chore: initial commit for asepharyana-hub-scraper
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
name: Notify Parent Repo
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
dispatch:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Trigger root monorepo build
|
||||
uses: peter-evans/repository-dispatch@v3
|
||||
with:
|
||||
token: ${{ secrets.DISPATCH_TOKEN }}
|
||||
repository: MythEclipse/ultimate-asepharyana.tech
|
||||
event-type: submodule-updated
|
||||
client-payload: |
|
||||
{
|
||||
"service": "scraper-api",
|
||||
"ref": "${{ github.ref }}",
|
||||
"sha": "${{ github.sha }}",
|
||||
"actor": "${{ github.actor }}"
|
||||
}
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
apps/gmw/
|
||||
# See https://docs.github.com/en/get-started/getting-started-with-git/ignoring-files for more about ignoring files.
|
||||
|
||||
# compiled output
|
||||
dist
|
||||
tmp
|
||||
out-tsc
|
||||
error.log
|
||||
# dependencies
|
||||
node_modules
|
||||
.bun/**
|
||||
.turbo/
|
||||
# IDEs and editors
|
||||
/.idea
|
||||
.project
|
||||
.classpath
|
||||
.c9/
|
||||
*.launch
|
||||
.settings/
|
||||
*.sublime-workspace
|
||||
**/target/**
|
||||
# IDE - VSCode
|
||||
.vscode/*
|
||||
!.vscode/settings.json
|
||||
!.vscode/tasks.json
|
||||
!.vscode/launch.json
|
||||
!.vscode/extensions.json
|
||||
|
||||
# misc
|
||||
/.sass-cache
|
||||
/connect.lock
|
||||
/coverage
|
||||
/libpeerconnection.log
|
||||
npm-debug.log
|
||||
yarn-error.log
|
||||
testem.log
|
||||
/typings
|
||||
|
||||
# System Files
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
.claude
|
||||
# Next.js
|
||||
.next
|
||||
out
|
||||
**/.codegraph/**
|
||||
**/.claude/**
|
||||
test-output
|
||||
**/**.env
|
||||
**/**.env.**
|
||||
vite.config.*.timestamp*
|
||||
vitest.config.*.timestamp*
|
||||
|
||||
|
||||
storybook-static
|
||||
~/.bun/**
|
||||
docs/dependency-map.md
|
||||
docs/handoff-log.jsonl
|
||||
docs/observability.md
|
||||
docs/quality-gates.json
|
||||
docs/workflow-state.json
|
||||
docs/todo.md
|
||||
**/vendor/
|
||||
|
||||
# moonrepo
|
||||
.moon/cache
|
||||
.~moon**
|
||||
@@ -0,0 +1,27 @@
|
||||
# AGENT.md - Universal AI Entry Point
|
||||
|
||||
This document serves as the unified entry point for all AI agents (Gemini, Claude, GPT, etc.) interacting with the **Scraping & CDN Service**.
|
||||
|
||||
## 🚀 Mission Statement
|
||||
|
||||
To provide a specialized, zero-bloat backend engine for high-concurrency web scraping and image persistent caching.
|
||||
|
||||
## 🛑 Global AI Protocols
|
||||
|
||||
As an AI agent, you **MUST** adhere to the following when working on this codebase:
|
||||
|
||||
1. **Professionalism**: Maintain a factual, technical, and objective tone.
|
||||
2. **No Hyperbole**: Prohibited from using marketing-speak or exaggerated praise (e.g., "amazing", "powerful").
|
||||
3. **Minimalism**: Prioritize the **Zero-Bloat Policy**. If a request adds unnecessary complexity or dependencies, challenge the user and suggest a leaner alternative.
|
||||
4. **Zero Suppression**: Never use suppression flags (`#[allow]`, `@ts-ignore`) to bypass warnings. Fix the underlying logic or types.
|
||||
5. **Observability**: Ensure critical business flows emit structured logs and request context where needed.
|
||||
|
||||
## 🔗 Technical Context
|
||||
|
||||
- **Architecture**: [GEMINI.md](file:///mnt/code/bp3/ultimate-asepharyana.tech/apps/scraper/GEMINI.md) (Logic flows, tech stack).
|
||||
- **Maintenance**: [Development Guide](file:///mnt/code/bp3/ultimate-asepharyana.tech/apps/scraper/docs/development.md) (Coding standards).
|
||||
- **Observability**: [Metrics Guide](file:///mnt/code/bp3/ultimate-asepharyana.tech/apps/scraper/docs/observability.md) (Standard telemetry).
|
||||
|
||||
---
|
||||
|
||||
*If you are an AI assistant, start by reading [GEMINI.md](file:///mnt/code/bp3/ultimate-asepharyana.tech/apps/scraper/GEMINI.md) for the full architectural context.*
|
||||
@@ -0,0 +1,177 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Project Overview
|
||||
|
||||
Scraper service — a Rust/Axum backend for web scraping (anime/komik data extraction) and image proxy/CDN caching. Serves as the backend engine consumed by the `apps/solidjs` frontend.
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
# Development
|
||||
cargo run # Start server (binds 0.0.0.0:4091)
|
||||
cargo test # Run all tests
|
||||
cargo clippy -- -D warnings # Lint (warnings are errors)
|
||||
cargo fmt # Auto-format all source files
|
||||
|
||||
# Release build (full LTO, single CGU, stripped)
|
||||
cargo build --release
|
||||
|
||||
# PM2 production
|
||||
pm2 start ecosystem.config.cjs --env production # Uses target/release/scraper
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
### Modular MVC + Service + Repository
|
||||
|
||||
Setiap module mengikuti arsitektur layered yang identik:
|
||||
|
||||
```
|
||||
Request → Router (route.rs) → Controller → Service → Repository → Parser
|
||||
│
|
||||
├── Redis (L1 cache)
|
||||
├── SeaORM/MySQL (L2, image_cache)
|
||||
└── External HTTP (alqanime.si, picser CDN)
|
||||
```
|
||||
|
||||
### Directory Layout
|
||||
|
||||
```
|
||||
src/
|
||||
├── main.rs # Entry point: builds Application, calls run()
|
||||
├── lib.rs # Public module declarations
|
||||
├── app.rs # Router assembly: modules + metrics + swagger + middleware layers
|
||||
├── bootstrap/mod.rs # Application::build(): tracing, Redis, browser pool, DB, AppState
|
||||
├── modules/ # Feature modules (vertical slices)
|
||||
│ ├── anime/ # Otakudesu anime scraper
|
||||
│ ├── anime2/ # Alqanime.si anime scraper
|
||||
│ ├── komik/ # Komik scraper
|
||||
│ └── proxy/ # Image proxy/cache/audit endpoints
|
||||
└── shared/ # Cross-cutting infrastructure
|
||||
├── config/ # Lazy-static AppConfig from env vars (fail-fast at startup)
|
||||
├── state/ # AppState (redis_pool, db, semaphore, event_bus)
|
||||
├── database/
|
||||
│ ├── traits/ # ScrapingRepository, ImageCacheRepository (async_trait)
|
||||
│ ├── repositories/ # SeaOrmImageCacheRepository (impl ImageCacheRepository)
|
||||
│ └── persistence/ # SeaORM entities (image_cache)
|
||||
├── services/images/ # ImageCache service + apply_cached_posters helper
|
||||
├── errors/ # AppError enum → axum IntoResponse (500/404 by variant)
|
||||
├── observability/ # Utoipa/Swagger OpenAPI doc
|
||||
├── scheduler/ # Cron jobs (daily cache cleanup at 2 AM)
|
||||
├── browser/ # Headless Chrome pool for JS-rendered scraping
|
||||
├── scrapers/ # Site-specific scrapers (otakudesu)
|
||||
├── utils/ # Cache helper, HTTP client, scraping helpers, retry, conversions
|
||||
├── middlewares/ # Logging, rate limiting
|
||||
├── events/ # EventBus for repair state updates
|
||||
└── types/ # ApiResponse<T>, shared entity types (HasPoster trait, Pagination)
|
||||
```
|
||||
|
||||
### Module Structure (identik untuk setiap module)
|
||||
|
||||
Setiap `src/modules/<name>/`:
|
||||
|
||||
| File | Peran | Pola |
|
||||
|---|---|---|
|
||||
| `route.rs` | Daftar endpoint, mapping URL → controller | `Router<Arc<AppState>>`, tidak ada logic |
|
||||
| `controller.rs` | Extract State/Path/Query/Body, panggil service | `Result<Json<T>, AppError>` |
|
||||
| `service.rs` | Business logic, caching, delegasi ke repository + parser | Struct dengan repo di-inject via constructor `new(repo: XRepository)` |
|
||||
| `repository.rs` | HTTP fetching, URL builders, DB queries | Struct + `impl ScrapingRepository` trait |
|
||||
| `parser.rs` | HTML parsing dengan `scraper` crate | Free functions → `Result<T, AppError>`, via `spawn_blocking` |
|
||||
| `schema.rs` | Validasi query/path/body params | Struct `Deserialize` + `ToSchema` |
|
||||
| `types.rs` | Response structs | `Serialize` + `ToSchema`, `impl HasPoster` jika punya poster |
|
||||
|
||||
### Dependency Injection
|
||||
|
||||
Semua service menerima dependency via constructor:
|
||||
|
||||
```rust
|
||||
// Controller creates and injects dependencies
|
||||
let repo = AnimeRepository::new();
|
||||
let service = AnimeService::new(repo);
|
||||
service.get_anime_index(app_state).await.map(Json)
|
||||
|
||||
// Service stores injected repo
|
||||
pub struct AnimeService {
|
||||
repository: AnimeRepository,
|
||||
}
|
||||
impl AnimeService {
|
||||
pub fn new(repository: AnimeRepository) -> Self { Self { repository } }
|
||||
}
|
||||
```
|
||||
|
||||
### Image Caching Architecture
|
||||
|
||||
Single unified image cache system:
|
||||
|
||||
1. **Trait**: `ImageCacheRepository` (`shared/database/traits/image_cache.rs`) — Redis ops, DB ops, locks, cache invalidation
|
||||
2. **Impl**: `SeaOrmImageCacheRepository` (`shared/database/repositories/image_cache.rs`)
|
||||
3. **Service**: `ImageCache` struct (`shared/services/images/cache.rs`) — download, MIME-verify dengan `infer`, upload ke Picser CDN, verifikasi CDN URL (10 retry dengan backoff)
|
||||
4. **Concurrency**: `Semaphore` (default 5 concurrent uploads) + request coalescing via `DashMap<broadcast::Sender>`
|
||||
5. **Lazy batch helper**: `cache_image_urls_batch_lazy()` — Redis batch check → DB batch check → background spawn untuk misses
|
||||
|
||||
### HasPoster Trait & apply_cached_posters
|
||||
|
||||
`HasPoster` trait di `shared/types/entities/anime.rs` memungkinkan generic poster caching:
|
||||
|
||||
```rust
|
||||
pub trait HasPoster {
|
||||
fn poster(&self) -> &str;
|
||||
fn set_poster(&mut self, url: String);
|
||||
}
|
||||
```
|
||||
|
||||
Semua item type dengan field `poster` mengimplementasikan trait ini (`OngoingAnimeItem`, `KomikItem`, `FilterAnimeItem`, `Recommendation`, dll).
|
||||
|
||||
`apply_cached_posters()` di `shared/services/images/cache.rs` menerima `&mut [T]` where `T: HasPoster`, menggantikan pola manual ~15 baris yang sebelumnya berulang di setiap service method.
|
||||
|
||||
### ScrapingRepository Trait
|
||||
|
||||
```rust
|
||||
#[async_trait]
|
||||
pub trait ScrapingRepository: Send + Sync {
|
||||
async fn fetch_html(&self, url: &str) -> Result<String, AppError>;
|
||||
}
|
||||
```
|
||||
|
||||
Semua module repository (`AnimeRepository`, `Anime2Repository`, `KomikRepository`, `ProxyRepository`) mengimplementasikan trait ini.
|
||||
|
||||
### Error Handling
|
||||
|
||||
`AppError` enum di `src/shared/errors/app_error.rs` — derives `thiserror::Error` dan implements `IntoResponse` (404 untuk `NotFound`, 500 untuk lainnya).
|
||||
|
||||
**Kontrak error per layer:**
|
||||
- **Parser** → `Result<T, AppError>`
|
||||
- **Repository** → `Result<T, AppError>` (via `ScrapingRepository` trait)
|
||||
- **Service** → `Result<T, AppError>` (tidak ada `Result<T, String>` atau `Box<dyn Error>`)
|
||||
- **Controller** → `Result<Json<T>, AppError>` (kecuali proxy yang return raw `Response`)
|
||||
|
||||
### Configuration
|
||||
|
||||
`src/shared/config/mod.rs` — global `CONFIG` lazy-static loaded from:
|
||||
1. `.env` file (dotenvy)
|
||||
2. `config/default.toml` / `config/{RUN_MODE}.toml`
|
||||
3. Environment variables (`APP__` prefix or legacy `DATABASE_URL`/`JWT_SECRET`/`REDIS_URL`)
|
||||
|
||||
Panics at startup if required config is missing — intentional fail-fast design.
|
||||
|
||||
## Constraints
|
||||
|
||||
- **No suppression flags**: `#[allow(...)]`, `#[ignore]`, `@ts-ignore` are prohibited. Fix the underlying issue.
|
||||
- **Lint strictness**: `unsafe_code = "forbid"`, `panic = "deny"`, `todo = "deny"`, `unimplemented = "deny"`, `unwrap_used = "warn"`, `expect_used = "warn"`
|
||||
- **Minimal dependencies**: Before adding a crate, evaluate if existing deps or std can handle it.
|
||||
- **Dead code**: Remove unused functions, types, modules rather than leaving them.
|
||||
- **Performance**: Use `spawn_blocking` for CPU-heavy work (HTML parsing).
|
||||
- **No duplicate infrastructure**: Satu trait, satu impl. Jangan membuat trait/repository duplikat seperti `ImageRepository` dan `ImageCacheRepository` yang berbeda.
|
||||
- **No thin wrappers**: Hindari wrapper tipis seperti `CacheImageUseCase` yang hanya meneruskan panggilan ke service lain.
|
||||
|
||||
## Useful Endpoints
|
||||
|
||||
- `GET /docs` — Swagger UI
|
||||
- `GET /api-docs/openapi.json` — OpenAPI spec
|
||||
- `POST /api/proxy/image-cache` — Cache an image URL
|
||||
- `POST /api/proxy/image-cache/audit` — Audit/repair cached images
|
||||
- `GET /api/anime/*` — Otakudesu scraping endpoints
|
||||
- `GET /api/anime2/*` — Alqanime scraping endpoints
|
||||
- `GET /api/komik/*` — Komik scraping endpoints
|
||||
Generated
+5564
File diff suppressed because it is too large
Load Diff
+132
@@ -0,0 +1,132 @@
|
||||
|
||||
[package]
|
||||
name = "scraper-service"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "A batteries-included, production-ready Rust web framework built on Axum"
|
||||
authors = ["Asep Haryana"]
|
||||
license = "MIT"
|
||||
repository = "https://github.com/MythEclipse/ultimate-asepharyana.tech"
|
||||
keywords = ["web", "framework", "axum", "api", "rest"]
|
||||
categories = ["web-programming::http-server", "web-programming::websocket"]
|
||||
default-run = "scraper"
|
||||
|
||||
# Dependensi yang dibutuhkan saat aplikasi berjalan
|
||||
[dependencies]
|
||||
axum = { version = "0.8.8", features = ["ws", "multipart", "macros"] }
|
||||
tokio = { version = "1.49.0", features = ["full"] }
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
dotenvy = "0.15"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
anyhow = "1.0"
|
||||
# sqlx removed - SeaORM uses it internally via sqlx-postgres feature
|
||||
sea-orm = { version = "1.1.19", features = ["sqlx-postgres", "runtime-tokio-rustls", "macros", "with-chrono", "with-uuid"] }
|
||||
uuid = { version = "1.10.0", features = ["v4", "serde"] }
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
|
||||
bytes = "1.11.0"
|
||||
futures = "0.3"
|
||||
reqwest = { version = "0.12.28", features = ["json", "stream", "multipart"] }
|
||||
http = "1.4.0"
|
||||
sha1 = "0.10.6"
|
||||
data-url = "0.3.2"
|
||||
base64 = "0.22.1"
|
||||
tokio-util = { version = "0.7.18", features = ["codec"] }
|
||||
async-trait = "0.1.89"
|
||||
regex = "1.12.2"
|
||||
infer = "0.19.0"
|
||||
|
||||
once_cell = "1.21.3"
|
||||
urlencoding = "2.1"
|
||||
|
||||
url = "2.5.8"
|
||||
rand = "0.8"
|
||||
tempfile = "3.24.0"
|
||||
mime_guess = "2.0.5"
|
||||
|
||||
tower-http = { version = "0.6.8", features = ["fs", "cors", "compression-gzip", "compression-br", "compression-zstd"] }
|
||||
backoff = { version = "0.4", features = ["futures", "tokio"] }
|
||||
dashmap = "6.1"
|
||||
deadpool-redis = { version = "0.22.1", features = ["serde"] }
|
||||
rayon = "1.11"
|
||||
tl = "0.7.8"
|
||||
tower = { version = "0.5", features = ["make"] }
|
||||
scraper = "0.25.0"
|
||||
flate2 = "1.1"
|
||||
redis = { version = "0.32.7", features = ["tokio-rustls-comp", "safe_iterators"] }
|
||||
thiserror = "2.0.18"
|
||||
itertools = "0.14"
|
||||
clap = { version = "4.5", features = ["derive"] }
|
||||
config = { version = "0.15.19", features = ["toml"] }
|
||||
governor = "0.10.4"
|
||||
tokio-cron-scheduler = "0.15.1"
|
||||
hex = "0.4.3"
|
||||
hmac = "0.12.1"
|
||||
sha2 = "0.10.9"
|
||||
log = "0.4"
|
||||
walkdir = "2.5"
|
||||
utoipa = { version = "5.0", features = ["axum_extras"] }
|
||||
utoipa-swagger-ui = { version = "9.0", features = ["axum"] }
|
||||
utoipa-axum = "0.2.0"
|
||||
|
||||
# OpenTelemetry metrics
|
||||
opentelemetry = { version = "0.27", features = ["metrics"] }
|
||||
opentelemetry_sdk = { version = "0.27", features = ["metrics", "rt-tokio"] }
|
||||
opentelemetry-otlp = { version = "0.27", features = ["metrics"] }
|
||||
opentelemetry-semantic-conventions = "0.27"
|
||||
|
||||
|
||||
# Dependensi yang hanya dibutuhkan untuk build script (build.rs)
|
||||
|
||||
# Dependensi yang hanya dibutuhkan untuk tes
|
||||
[dev-dependencies]
|
||||
sea-orm = { version = "1.1.19", features = ["sqlx-postgres", "runtime-tokio-rustls", "macros", "with-chrono", "with-uuid", "mock"] }
|
||||
|
||||
# Definisi targets secara eksplisit untuk menghindari peringatan cargo-chef (edition/plugin)
|
||||
[lib]
|
||||
name = "scraper_service"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "scraper"
|
||||
path = "src/main.rs"
|
||||
|
||||
|
||||
|
||||
[lints.rust]
|
||||
unsafe_code = "forbid"
|
||||
unused_variables = "deny"
|
||||
unused_imports = "deny"
|
||||
unused_must_use = "deny"
|
||||
|
||||
[lints.clippy]
|
||||
unwrap_used = "warn"
|
||||
expect_used = "warn"
|
||||
panic = "deny"
|
||||
todo = "deny"
|
||||
unimplemented = "deny"
|
||||
|
||||
[features]
|
||||
# default = ["ffmpeg"]
|
||||
ffmpeg = []
|
||||
|
||||
# Profile optimasi untuk production - fokus pada performa runtime maksimal
|
||||
[profile.release]
|
||||
opt-level = 3 # Optimasi level maksimal
|
||||
lto = "fat" # Full LTO untuk inlining maksimal dan binary optimal
|
||||
codegen-units = 1 # Single codegen unit untuk optimasi maksimal (lebih lambat build, binary lebih cepat)
|
||||
incremental = false # Disable incremental untuk optimasi penuh
|
||||
debug = false # No debug info untuk binary lebih kecil
|
||||
strip = true # Strip symbols untuk binary lebih kecil
|
||||
panic = "abort" # Abort on panic (binary lebih kecil dan lebih cepat)
|
||||
overflow-checks = false # Non-overflowing arithmetic for speed (disable in debug)
|
||||
|
||||
# Optimize dependencies too
|
||||
[profile.dev]
|
||||
opt-level = 0
|
||||
|
||||
[profile.bench]
|
||||
opt-level = 3
|
||||
lto = "fat"
|
||||
@@ -0,0 +1,69 @@
|
||||
# GEMINI.md - Codebase Architecture & Structure
|
||||
|
||||
Internal technical overview of the **Scraping & CDN Service** (`apps/scraper`) for automated data extraction and image persistence.
|
||||
|
||||
## 🌍 Context
|
||||
|
||||
- **`apps/scraper`**: Specialized backend engine (Axum).
|
||||
- **`apps/solidjs`**: Frontend consumer.
|
||||
- **`packages/services`**: Shared logic.
|
||||
|
||||
## 🤖 AI Assistant Guidelines
|
||||
|
||||
AI assistants (like Claude, Gemini, GPT) interacting with this codebase **MUST** adhere to the following protocols defined in **[AGENT.md](file:///mnt/code/bp3/ultimate-asepharyana.tech/apps/scraper/AGENT.md)**:
|
||||
|
||||
1. **Professional Tone**: Maintain a cold, technical, and objective tone.
|
||||
2. **No Hyperbole**: **PROHIBITED** from using marketing-speak or exaggerated praise (e.g., "amazing", "unparalleled", "powerful", "revolutionary").
|
||||
3. **Technical Accuracy**: Focus purely on implementation facts, data structures, and performance metrics.
|
||||
4. **Minimalist Adherence**: Always prioritize the **Zero-Bloat Policy**. If a request introduces unnecessary dependencies or logic, challenge the user and suggest a leaner alternative.
|
||||
5. **Documentation Consistency**: Ensure any generated documentation follows the established professional and objective style of the `docs/` folder.
|
||||
|
||||
## 🦀 `apps/scraper` - Backend Service
|
||||
|
||||
An asynchronous service for scraping and image proxying. All secondary web framework features (Authentication, Social, GraphQL) have been removed to reduce complexity.
|
||||
|
||||
### 📊 Tech Stack
|
||||
|
||||
- **Framework**: [Axum](https://github.com/tokio-rs/axum) (0.8.8) - Asynchronous Rust HTTP.
|
||||
- **ORM**: [SeaORM](https://www.sea-ql.org/SeaORM/) (MySQL) - Database abstraction.
|
||||
- **Caching**: `deadpool-redis` & `redis` - In-memory cache mapping.
|
||||
- **Observability**: Request ID tracing and structured logging.
|
||||
- **Scraping**: `scraper` (CSS Selectors) & remote Chrome via HTTP.
|
||||
|
||||
### 📂 Directory Structure (`apps/scraper/src`)
|
||||
|
||||
Organized as a hybrid of Vertical Slice and Clean Architecture.
|
||||
|
||||
| Directory | Description |
|
||||
| :--- | :--- |
|
||||
| **`bin/`** | Binary entry points and CLI tools. |
|
||||
| **`config/`** | Strongly-typed environment configuration. |
|
||||
| **`entities/`** | **SeaORM Entities**. Database schema mapping. |
|
||||
| **`routes/`** | **API Handlers**. Automatic routing system. |
|
||||
| **`services/`** | Business logic (e.g., `ImageCache` service). |
|
||||
| **`scraping/`** | Data extraction engines and parsers. |
|
||||
| **`helpers/`** | Shared utilities and cache helpers. |
|
||||
| **`middleware/`** | Axum layers (CORS, Compression). |
|
||||
| **`events/`** | Internal event bus for repair state updates. |
|
||||
| **`jobs/`** | Background task processing. |
|
||||
| **`scheduler/`** | Periodic tasks (Daily CDN audit). |
|
||||
| **`observability/`** | OpenAPI documentation, request ID tracing, and structured logging. |
|
||||
|
||||
### 🔑 Logic Flows
|
||||
|
||||
1. **Scraping**:
|
||||
`Request` -> `Router` -> `Handler` -> `Scraper Engine` -> `Redis` -> `Response`.
|
||||
2. **Image Proxy/CDN**:
|
||||
`Request` -> `ImageCache` -> `Cache Lookup` -> `Picser Upload (on miss)` -> `CDN URL`.
|
||||
|
||||
### 📜 Commands
|
||||
|
||||
- **Standard Run**: `cargo run`
|
||||
- **Optimized Build**: `cargo build --release`
|
||||
- **External Audit**: `POST /api/proxy/image-cache/audit`
|
||||
|
||||
## 🏗 Maintenance Constraints
|
||||
|
||||
- **Minimalist Approach**: New dependencies require impact evaluation.
|
||||
- **Lint Compliance**: Suppression flags (`#[allow]`) are prohibited.
|
||||
- **Performance-First**: Use `spawn_blocking` for CPU-heavy work (HTML parsing).
|
||||
@@ -0,0 +1,55 @@
|
||||
# Scraper API
|
||||
|
||||
Backend service berbasis Axum untuk scraping, image proxy/cache, dan endpoint API inti.
|
||||
|
||||
## Stack
|
||||
|
||||
- Rust + Axum
|
||||
- SeaORM (MySQL)
|
||||
- Redis (deadpool-redis)
|
||||
- Utoipa + Swagger UI
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
cargo run
|
||||
```
|
||||
|
||||
Server bind ke `0.0.0.0:${PORT}` dengan default port `4091`.
|
||||
|
||||
## Required Environment Variables
|
||||
|
||||
```env
|
||||
DATABASE_URL=mysql://asephs:hunterz@localhost:3306/sosmed
|
||||
JWT_SECRET=change-me
|
||||
REDIS_URL=redis://localhost:6379
|
||||
```
|
||||
|
||||
Optional yang sering dipakai:
|
||||
|
||||
```env
|
||||
RUST_LOG=info
|
||||
EXTERNAL_BROWSERLESS_WS=
|
||||
MINIO_ENDPOINT=
|
||||
MINIO_BUCKET_NAME=
|
||||
MINIO_ACCESS_KEY=
|
||||
MINIO_SECRET_KEY=
|
||||
```
|
||||
|
||||
## Useful Endpoints
|
||||
|
||||
- `GET /docs` - Swagger UI
|
||||
- `GET /api-docs/openapi.json` - OpenAPI JSON
|
||||
- `GET /api/anime2/*`
|
||||
- `GET /api/komik/*`
|
||||
- `POST /api/proxy/image-cache`
|
||||
- `POST /api/proxy/image-cache/audit`
|
||||
|
||||
## Notes
|
||||
|
||||
- Database akan dicek/dibuat saat startup jika `DATABASE_URL` bertipe MySQL.
|
||||
- Service ini juga menginisialisasi browser pool dan scheduler saat boot.
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
Executable
+73
@@ -0,0 +1,73 @@
|
||||
#!/bin/bash
|
||||
# Simple script to check image_cache database content
|
||||
|
||||
DATABASE_URL="mysql://asephs:hunterz@127.0.0.1:3306/sosmed"
|
||||
|
||||
echo "=== Checking Image Cache Database ==="
|
||||
echo ""
|
||||
echo "Attempting to connect via Rust binary..."
|
||||
echo ""
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
# Create temporary Rust script
|
||||
cat > /tmp/check_img_cache.rs << 'EOF'
|
||||
use sea_orm::{Database, ConnectionTrait, Statement};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let db = Database::connect("mysql://asephs:hunterz@127.0.0.1:3306/sosmed").await?;
|
||||
|
||||
println!("✓ Connected to database\n");
|
||||
|
||||
// Show all tables
|
||||
println!("=== ALL TABLES ===");
|
||||
let tables = db.query_all(Statement::from_string(
|
||||
db.get_database_backend(),
|
||||
"SHOW TABLES".to_owned()
|
||||
)).await?;
|
||||
for table in &tables {
|
||||
println!(" - {:?}", table);
|
||||
}
|
||||
|
||||
// Count image_cache records
|
||||
println!("\n=== IMAGE_CACHE TABLE ===");
|
||||
let count = db.query_one(Statement::from_string(
|
||||
db.get_database_backend(),
|
||||
"SELECT COUNT(*) as total FROM ImageCache".to_owned()
|
||||
)).await?;
|
||||
println!("Total records: {:?}", count);
|
||||
|
||||
// Show recent 10 records
|
||||
println!("\n=== RECENT RECORDS (Last 10) ===");
|
||||
let records = db.query_all(Statement::from_string(
|
||||
db.get_database_backend(),
|
||||
"SELECT id, originalUrl, cdnUrl, createdAt FROM ImageCache ORDER BY createdAt DESC LIMIT 10".to_owned()
|
||||
)).await?;
|
||||
|
||||
for (i, record) in records.iter().enumerate() {
|
||||
println!("\n[{}]", i + 1);
|
||||
println!(" ID: {:?}", record.try_get::<String>("", "id"));
|
||||
println!(" Original: {:?}", record.try_get::<String>("", "originalUrl"));
|
||||
println!(" CDN URL: {:?}", record.try_get::<String>("", "cdnUrl"));
|
||||
println!(" Created: {:?}", record.try_get::<chrono::DateTime<chrono::Utc>>("", "createdAt"));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
EOF
|
||||
|
||||
echo "Running database check..."
|
||||
rustc --edition 2021 /tmp/check_img_cache.rs -o /tmp/check_img_cache \
|
||||
-L dependency=target/debug/deps \
|
||||
--extern sea_orm=target/debug/deps/libsea_orm.rlib \
|
||||
--extern tokio=target/debug/deps/libtokio.rlib \
|
||||
--extern chrono=target/debug/deps/libchrono.rlib \
|
||||
2>/dev/null
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
/tmp/check_img_cache
|
||||
else
|
||||
echo "Rust compilation failed, using cargo script instead..."
|
||||
cargo script /tmp/check_img_cache.rs
|
||||
fi
|
||||
@@ -0,0 +1,167 @@
|
||||
# Clean-Modular Architecture Refactor Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Refactor the codebase into a rigid Clean-Modular architecture to improve maintainability and strictly enforce separation of concerns.
|
||||
|
||||
**Architecture:** A three-tier modular approach consisting of Presentation (API Handlers/DTOs), Core (Domain Models/Traits/Use Cases), and Infrastructure (Adapters/Repositories/Scrapers).
|
||||
|
||||
**Tech Stack:** Rust, Axum, SeaORM, Redis, reqwest.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Initialize Core Domain Models & Shared Errors
|
||||
|
||||
**Files:**
|
||||
- Create: `src/shared/errors/mod.rs`
|
||||
- Create: `src/core/models/image.rs`
|
||||
- Create: `src/core/models/mod.rs`
|
||||
- Create: `src/shared/mod.rs`
|
||||
|
||||
- [ ] **Step 1: Define shared application errors**
|
||||
- [ ] **Step 2: Define pure domain models for ImageCache**
|
||||
- [ ] **Step 3: Setup core and shared modules in `lib.rs`**
|
||||
|
||||
```rust
|
||||
// src/shared/errors/mod.rs
|
||||
use axum::{response::{IntoResponse, Response}, Json, http::StatusCode};
|
||||
use serde_json::json;
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum AppError {
|
||||
#[error("Not found: {0}")]
|
||||
NotFound(String),
|
||||
#[error("Internal error: {0}")]
|
||||
Internal(String),
|
||||
#[error("Validation error: {0}")]
|
||||
Validation(String),
|
||||
}
|
||||
|
||||
impl IntoResponse for AppError {
|
||||
fn into_response(self) -> Response {
|
||||
let (status, message) = match self {
|
||||
AppError::NotFound(m) => (StatusCode::NOT_FOUND, m),
|
||||
AppError::Internal(m) => (StatusCode::INTERNAL_SERVER_ERROR, m),
|
||||
AppError::Validation(m) => (StatusCode::BAD_REQUEST, m),
|
||||
};
|
||||
(status, Json(json!({ "error": message }))).into_response()
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Commit changes**
|
||||
```bash
|
||||
git add src/shared/errors/mod.rs src/core/models/image.rs
|
||||
git commit -m "feat: init core models and shared errors"
|
||||
```
|
||||
|
||||
### Task 2: Define Core Repository Traits
|
||||
|
||||
**Files:**
|
||||
- Create: `src/core/repositories/image_repository.rs`
|
||||
- Create: `src/core/repositories/mod.rs`
|
||||
|
||||
- [ ] **Step 1: Define ImageRepository trait in `core`**
|
||||
|
||||
```rust
|
||||
// src/core/repositories/image_repository.rs
|
||||
use async_trait::async_trait;
|
||||
use crate::core::models::image::ImageCache;
|
||||
use crate::shared::errors::AppError;
|
||||
|
||||
#[async_trait]
|
||||
pub trait ImageRepository: Send + Sync {
|
||||
async fn find_by_original_url(&self, url: &str) -> Result<Option<ImageCache>, AppError>;
|
||||
async fn save(&self, image: ImageCache) -> Result<(), AppError>;
|
||||
async fn delete_by_original_url(&self, url: &str) -> Result<(), AppError>;
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Commit changes**
|
||||
```bash
|
||||
git add src/core/repositories/
|
||||
git commit -m "feat: define core repository traits"
|
||||
```
|
||||
|
||||
### Task 3: Migrate SeaORM Entities to Infrastructure
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/infra/mod.rs`
|
||||
- Create: `src/infra/repositories/mysql_image_repository.rs`
|
||||
|
||||
- [ ] **Step 1: Implement ImageRepository for MySQL using SeaORM**
|
||||
- [ ] **Step 2: Move `src/entities/image_cache.rs` logic into the new repository implementation**
|
||||
- [ ] **Step 3: Update `src/infra/mod.rs` to expose repositories**
|
||||
|
||||
- [ ] **Step 4: Commit changes**
|
||||
```bash
|
||||
git add src/infra/repositories/
|
||||
git commit -m "feat: implement mysql image repository in infra"
|
||||
```
|
||||
|
||||
### Task 4: Implement Core Use Cases (Image Caching)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/core/use_cases/cache_image.rs`
|
||||
- Create: `src/core/use_cases/mod.rs`
|
||||
|
||||
- [ ] **Step 1: Implement `CacheImageUseCase`**
|
||||
- [ ] **Step 2: Orchestrate logic between repository, redis, and scraper/uploader**
|
||||
|
||||
- [ ] **Step 3: Commit changes**
|
||||
```bash
|
||||
git add src/core/use_cases/
|
||||
git commit -m "feat: implement image caching use cases"
|
||||
```
|
||||
|
||||
### Task 5: Refactor Scrapers into Infrastructure
|
||||
|
||||
**Files:**
|
||||
- Create: `src/core/repositories/scraping_repository.rs`
|
||||
- Create: `src/infra/scrapers/otakudesu.rs`
|
||||
|
||||
- [ ] **Step 1: Define Scraping traits in `core`**
|
||||
- [ ] **Step 2: Implement site-specific scrapers in `infra`**
|
||||
- [ ] **Step 3: Migrate existing logic from `src/scraping/`**
|
||||
|
||||
- [ ] **Step 4: Commit changes**
|
||||
```bash
|
||||
git add src/infra/scrapers/
|
||||
git commit -m "feat: migrate scrapers to infra adapters"
|
||||
```
|
||||
|
||||
### Task 6: Refactor Presentation Layer (API Handlers)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/presentation/api/anime_handler.rs`
|
||||
- Create: `src/presentation/api/mod.rs`
|
||||
- Create: `src/presentation/mod.rs`
|
||||
|
||||
- [ ] **Step 1: Migrate handlers from `src/routes/` to `presentation/api/`**
|
||||
- [ ] **Step 2: Update handlers to use Use Cases instead of direct service/helper calls**
|
||||
- [ ] **Step 3: Update global router in `src/main.rs` or `src/lib.rs`**
|
||||
|
||||
- [ ] **Step 4: Commit changes**
|
||||
```bash
|
||||
git add src/presentation/api/
|
||||
git commit -m "feat: refactor presentation layer api handlers"
|
||||
```
|
||||
|
||||
### Task 7: Global Cleanup & Verification
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/lib.rs`
|
||||
- Delete: `src/helpers/` (partially merged into shared/infra)
|
||||
- Delete: `src/services/` (merged into core/use_cases)
|
||||
- Delete: `src/routes/` (merged into presentation)
|
||||
|
||||
- [ ] **Step 1: Update `lib.rs` to reflect new module structure**
|
||||
- [ ] **Step 2: Remove old redundant directories**
|
||||
- [ ] **Step 3: Run full test suite**
|
||||
- [ ] **Step 4: Verify metrics endpoint**
|
||||
|
||||
- [ ] **Step 5: Final Commit**
|
||||
```bash
|
||||
git commit -m "refactor: complete clean-modular architecture overhaul"
|
||||
```
|
||||
@@ -0,0 +1,49 @@
|
||||
# Design Spec: Clean-Modular Architecture Refactor
|
||||
|
||||
**Date**: 2026-05-08
|
||||
**Topic**: Refactor `apps/rust` from Hybrid to Clean-Modular Architecture.
|
||||
|
||||
## 1. Purpose
|
||||
Standardize codebase structure for rigidity, maintainability, and clear separation of concerns (SOC) without violating the **Zero-Bloat Policy**.
|
||||
|
||||
## 2. Target Architecture
|
||||
Moving from current structure to a three-tier modular design:
|
||||
|
||||
### A. Presentation Layer (`src/presentation/`)
|
||||
- **API Handlers**: Pure Axum handlers.
|
||||
- **DTOs**: Request/Response models for external communication.
|
||||
- **Middleware**: Cross-cutting concerns (CORS, Metrics, Logging).
|
||||
|
||||
### B. Core Layer (`src/core/`) - The Domain
|
||||
- **Models**: Pure data structures (Plain Rust Objects).
|
||||
- **Repository Traits**: Abstract interfaces for data persistence.
|
||||
- **Use Cases**: Orchestration of business logic (e.g., `ScrapeAnime`, `ProcessImage`).
|
||||
- **Dependencies**: None (or minimal shared utils).
|
||||
|
||||
### C. Infrastructure Layer (`src/infra/`) - The Adapters
|
||||
- **Repositories**: SeaORM & Redis implementations of Core traits.
|
||||
- **Scrapers**: Site-specific parsing logic implementing Core scraping traits.
|
||||
- **External Clients**: HTTP Client (reqwest), Browser Pool.
|
||||
|
||||
### D. Shared Layer (`src/shared/`)
|
||||
- **Utils**: Low-level helpers (Date, JSON, String).
|
||||
- **Config**: Application configuration.
|
||||
- **Errors**: Centralized error handling.
|
||||
|
||||
## 3. Implementation Strategy
|
||||
1. **Phase 1**: Scaffold new directory structure.
|
||||
2. **Phase 2**: Migrate `models` and `entities` to `core/models` and `infra/repositories`.
|
||||
3. **Phase 3**: Refactor `scraping` logic into `infra/scrapers` and define traits in `core`.
|
||||
4. **Phase 4**: Move Axum handlers to `presentation/api` and update routing.
|
||||
5. **Phase 5**: Cleanup `helpers` into `shared/utils`.
|
||||
|
||||
## 4. Constraints
|
||||
- **Zero-Bloat**: No new heavy dependencies for the sake of abstraction.
|
||||
- **Performance**: Maintain latency metrics as defined in `docs/development.md`.
|
||||
- **SeaORM**: Entities stay in `infra/` to keep `core/` pure.
|
||||
|
||||
## 5. Success Criteria
|
||||
- All tests pass.
|
||||
- `/metrics` show no latency regression.
|
||||
- Circular dependencies are eliminated.
|
||||
- Folder structure matches this spec.
|
||||
@@ -0,0 +1,26 @@
|
||||
// PM2 Ecosystem Configuration for Rust
|
||||
module.exports = {
|
||||
apps: [
|
||||
{
|
||||
name: 'ultimate-rust',
|
||||
script: 'target/release/scraper',
|
||||
cwd: process.env.VPS_TARGET_DIR
|
||||
? `${process.env.VPS_TARGET_DIR}/apps/scraper`
|
||||
: '/home/asephs/ultimate-asepharyana.cloud/apps/scraper',
|
||||
interpreter: 'none',
|
||||
exec_mode: 'fork',
|
||||
autorestart: true,
|
||||
watch: false,
|
||||
max_memory_restart: '1G',
|
||||
env_production: {
|
||||
NODE_ENV: 'production',
|
||||
PORT: 4091,
|
||||
},
|
||||
// Logging configuration
|
||||
error_file: './logs/error.log',
|
||||
out_file: './logs/out.log',
|
||||
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
|
||||
merge_logs: true,
|
||||
},
|
||||
],
|
||||
};
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"name": "scraper",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "./target/release/scraper"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
# rustfmt configuration (stable features only)
|
||||
|
||||
max_width = 100
|
||||
hard_tabs = false
|
||||
tab_spaces = 4
|
||||
newline_style = "Unix"
|
||||
use_small_heuristics = "Default"
|
||||
reorder_imports = true
|
||||
reorder_modules = true
|
||||
remove_nested_parens = true
|
||||
edition = "2021"
|
||||
merge_derives = true
|
||||
use_try_shorthand = true
|
||||
use_field_init_shorthand = true
|
||||
force_explicit_abi = true
|
||||
Binary file not shown.
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
# Auto-fix warnings and lint Rust code
|
||||
|
||||
set -e
|
||||
|
||||
echo "🔧 Auto-fixing Rust warnings and linting..."
|
||||
|
||||
# Format code
|
||||
echo "📝 Running rustfmt..."
|
||||
cargo fmt
|
||||
|
||||
# Fix clippy warnings automatically
|
||||
echo "🔍 Running clippy auto-fix..."
|
||||
cargo clippy --fix --allow-dirty --allow-staged --all-targets
|
||||
|
||||
# Check for remaining issues
|
||||
echo "✅ Running final check..."
|
||||
cargo clippy --all-targets -- -D warnings
|
||||
|
||||
echo "✨ Done! Code is formatted and linted."
|
||||
Executable
+84
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare OpenAPI specifications for endpoint compatibility."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 3:
|
||||
print("Usage: compare-openapi.py <reference-openapi.json> <local-openapi.json>")
|
||||
sys.exit(1)
|
||||
|
||||
ref_path = Path(sys.argv[1])
|
||||
local_path = Path(sys.argv[2])
|
||||
|
||||
# Load reference OpenAPI
|
||||
try:
|
||||
ref_data = json.loads(ref_path.read_text())
|
||||
except FileNotFoundError:
|
||||
print(f"Error: Reference file not found: {ref_path}")
|
||||
sys.exit(1)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"Error: Invalid JSON in reference file: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
# Load local OpenAPI
|
||||
try:
|
||||
local_data = json.loads(local_path.read_text())
|
||||
except FileNotFoundError:
|
||||
print(f"Error: Local file not found: {local_path}")
|
||||
sys.exit(1)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"Error: Invalid JSON in local file: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
ref_paths = ref_data.get("paths", {})
|
||||
local_paths = local_data.get("paths", {})
|
||||
|
||||
http_methods = {"get", "put", "post", "delete", "options", "head", "patch", "trace"}
|
||||
|
||||
has_differences = False
|
||||
|
||||
# Check for missing paths
|
||||
missing_paths = set(ref_paths.keys()) - set(local_paths.keys())
|
||||
if missing_paths:
|
||||
has_differences = True
|
||||
for path in sorted(missing_paths):
|
||||
print(f"Missing path: {path}")
|
||||
|
||||
# Check for extra paths
|
||||
extra_paths = set(local_paths.keys()) - set(ref_paths.keys())
|
||||
if extra_paths:
|
||||
has_differences = True
|
||||
for path in sorted(extra_paths):
|
||||
print(f"Extra path: {path}")
|
||||
|
||||
# Check for method mismatches in common paths
|
||||
common_paths = set(ref_paths.keys()) & set(local_paths.keys())
|
||||
for path in sorted(common_paths):
|
||||
ref_methods = set(method.lower() for method in ref_paths[path].keys() if method.lower() in http_methods)
|
||||
local_methods = set(method.lower() for method in local_paths[path].keys() if method.lower() in http_methods)
|
||||
|
||||
missing_methods = ref_methods - local_methods
|
||||
if missing_methods:
|
||||
has_differences = True
|
||||
for method in sorted(missing_methods):
|
||||
print(f"Missing method {method.upper()} for path: {path}")
|
||||
|
||||
extra_methods = local_methods - ref_methods
|
||||
if extra_methods:
|
||||
has_differences = True
|
||||
for method in sorted(extra_methods):
|
||||
print(f"Extra method {method.upper()} for path: {path}")
|
||||
|
||||
if has_differences:
|
||||
sys.exit(1)
|
||||
|
||||
print(f"OpenAPI paths/methods match: {len(ref_paths)} paths")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+15
@@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
# Script to run database migrations manually
|
||||
|
||||
echo "Running database migrations..."
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# Build and run the migration binary
|
||||
cargo run --bin migrate
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "✅ Migration completed successfully."
|
||||
else
|
||||
echo "❌ Migration failed."
|
||||
exit 1
|
||||
fi
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
# Pre-build hook to enforce best practices
|
||||
|
||||
set -e
|
||||
|
||||
echo "🔍 Running pre-build lint checks..."
|
||||
|
||||
# Format check
|
||||
echo "📝 Checking code formatting..."
|
||||
if ! cargo fmt -- --check; then
|
||||
echo "❌ Code not formatted. Run: cargo fmt"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Clippy check with denials
|
||||
echo "🔧 Running clippy..."
|
||||
cargo clippy --all-targets -- -D warnings
|
||||
|
||||
echo "✅ All lint checks passed!"
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Router;
|
||||
use sea_orm::DatabaseConnection;
|
||||
use tower_http::compression::{CompressionLayer, CompressionLevel};
|
||||
use tower_http::cors::CorsLayer;
|
||||
use utoipa::OpenApi;
|
||||
use utoipa_swagger_ui::SwaggerUi;
|
||||
|
||||
use crate::shared::observability::openapi::ApiDoc;
|
||||
use crate::shared::state::AppState;
|
||||
|
||||
pub async fn build_router(
|
||||
app_state: Arc<AppState>,
|
||||
db: Arc<DatabaseConnection>,
|
||||
) -> anyhow::Result<Router> {
|
||||
init_scheduler(db).await?;
|
||||
|
||||
let mut openapi = ApiDoc::openapi();
|
||||
openapi.merge(crate::shared::observability::openapi_modules::ModuleApiDoc::openapi());
|
||||
|
||||
let app = crate::modules::routes(Router::new())
|
||||
.merge(SwaggerUi::new("/docs").url("/api-docs/openapi.json", openapi))
|
||||
.with_state(app_state)
|
||||
.layer(axum::middleware::from_fn(
|
||||
crate::shared::observability::metrics::otel_metrics_middleware,
|
||||
))
|
||||
.layer(CompressionLayer::new().quality(CompressionLevel::Fastest))
|
||||
.layer(CorsLayer::permissive());
|
||||
|
||||
Ok(app)
|
||||
}
|
||||
|
||||
async fn init_scheduler(db: Arc<DatabaseConnection>) -> anyhow::Result<()> {
|
||||
let scheduler = crate::shared::scheduler::Scheduler::new()
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to create scheduler: {}", e))?;
|
||||
|
||||
let cache_cleanup = crate::shared::scheduler::CleanupOldCache::new(db);
|
||||
scheduler
|
||||
.add(cache_cleanup)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to add cache cleanup: {}", e))?;
|
||||
|
||||
scheduler
|
||||
.start()
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to start scheduler: {}", e))?;
|
||||
tracing::info!("✓ Scheduler started");
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
use scraper::Selector;
|
||||
use scraper_service::shared::utils::parse_html;
|
||||
/// Capture html5ever tree_builder warning evidence by parsing problematic HTML.
|
||||
///
|
||||
/// Build with: cargo build --bin capture_warning
|
||||
/// Run with: RUST_LOG=warn cargo run --bin capture_warning 2>&1
|
||||
///
|
||||
/// Evidence of the warning is captured through:
|
||||
/// 1. HTML that triggers foster_parenting in html5ever::tree_builder
|
||||
/// 2. Observable parsing behavior showing tree reconstruction
|
||||
/// 3. The call path from src/helpers::parse_html () to html5ever
|
||||
/// 4. Real endpoint context from /api/anime2/latest/{slug} route
|
||||
use std::fs;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
fn main() {
|
||||
// Initialize logging to capture WARN output from html5ever
|
||||
let env_filter = EnvFilter::from_default_env()
|
||||
.add_directive("warn".parse().expect("valid directive"))
|
||||
.add_directive("html5ever=warn".parse().expect("valid directive"));
|
||||
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(env_filter)
|
||||
.with_writer(std::io::stderr)
|
||||
.init();
|
||||
|
||||
println!("=== HTML5ever Tree Builder Foster Parenting Evidence ===\n");
|
||||
println!("Real Endpoint: GET /api/anime2/latest/{{slug}}");
|
||||
println!("Handler: src/routes/api/anime2/latest/[slug].rs:124");
|
||||
println!("Helper Path: src/helpers/web/scraping.rs::parse_html() -> Html::parse_document()\n");
|
||||
|
||||
// Load HTML fixture from shared test file
|
||||
let fixture_path = "src/bin/test_fixtures/foster_parenting_minimal.html";
|
||||
let test_html = fs::read_to_string(fixture_path)
|
||||
.expect(&format!("Failed to read fixture from {}", fixture_path));
|
||||
|
||||
println!("Input Request:");
|
||||
println!(" GET /api/anime2/latest/some-anime");
|
||||
println!(" Body: HTML containing misplaced text in <table>");
|
||||
println!(" Fixture: {}", fixture_path);
|
||||
println!(" Test HTML: {}\n", test_html);
|
||||
|
||||
println!("Parsing through src/helpers::parse_html()...\n");
|
||||
println!("--- BEGIN STDERR (logging output) ---");
|
||||
|
||||
// This parse_html() call routes through:
|
||||
// src/helpers::parse_html()
|
||||
// -> scraper crate Html::parse_document()
|
||||
// -> html5ever::parse() [version 0.36.1]
|
||||
// -> TreeBuilder::process_token()
|
||||
// -> TreeBuilder::foster_parent_in_body() which emits:
|
||||
// warn!("foster parenting not implemented")
|
||||
let document = parse_html(&test_html);
|
||||
|
||||
println!("--- END STDERR (logging output) ---\n");
|
||||
|
||||
// Analyze the result
|
||||
println!("Parse Output Evidence:\n");
|
||||
|
||||
// Check table structure
|
||||
let table_sel = Selector::parse("table").expect("Valid CSS selector");
|
||||
let tr_sel = Selector::parse("tr").expect("Valid CSS selector");
|
||||
let td_sel = Selector::parse("td").expect("Valid CSS selector");
|
||||
|
||||
let tables: Vec<_> = document.select(&table_sel).collect();
|
||||
println!(" ✓ Tables parsed: {}", tables.len());
|
||||
|
||||
let trs: Vec<_> = document.select(&tr_sel).collect();
|
||||
println!(" ✓ Table rows found: {}", trs.len());
|
||||
|
||||
let tds: Vec<_> = document.select(&td_sel).collect();
|
||||
println!(" ✓ Table cells found: {}", tds.len());
|
||||
|
||||
let body_sel = Selector::parse("body").expect("Valid CSS selector");
|
||||
if let Some(body) = document.select(&body_sel).next() {
|
||||
let body_text: String = body.text().collect();
|
||||
let trimmed = body_text.trim();
|
||||
println!("\n Body element text content:");
|
||||
println!(" '{}'", trimmed);
|
||||
|
||||
if trimmed.contains("orphaned text") {
|
||||
println!("\n ✓ EVIDENCE: 'orphaned text' moved OUT of <table>");
|
||||
println!(" This proves foster_parenting occurred!");
|
||||
}
|
||||
if trimmed.contains("more text") {
|
||||
println!(" ✓ EVIDENCE: 'more text' moved OUT of <table>");
|
||||
println!(" This confirms the adoption agency algorithm ran!");
|
||||
}
|
||||
}
|
||||
|
||||
println!("\n=== Proven Call Path ===");
|
||||
println!("Route: GET /api/anime2/latest/{{slug}}");
|
||||
println!("Request Handler: src/routes/api/anime2/latest/[slug].rs");
|
||||
println!(" -> latest() handler");
|
||||
println!(" -> fetch_latest_anime()");
|
||||
println!(" -> parse_latest_page(html, page)");
|
||||
println!(" -> crate::shared::utils::parse_html(html) [line 124]\n");
|
||||
|
||||
println!("Helper Function: src/helpers/web/scraping.rs");
|
||||
println!(" pub fn parse_html(html: &str) -> Html {{");
|
||||
println!(" Html::parse_document(html) // Line 34");
|
||||
println!(" }}\n");
|
||||
|
||||
println!("Call Stack to Warning:");
|
||||
println!(" 1. crate::shared::utils::parse_html() [src/helpers/web/scraping.rs:34]");
|
||||
println!(" 2. Html::parse_document() [scraper crate wrapper]");
|
||||
println!(" 3. html5ever::parse() [Cargo.toml: version 0.36.1]");
|
||||
println!(" 4. TreeBuilder::process_token()");
|
||||
println!(" 5. TreeBuilder::process_chars_in_table()");
|
||||
println!(" 6. TreeBuilder::foster_parent_in_body() [src/tree_builder/mod.rs:1227]");
|
||||
println!(" 7. warn!(\"foster parenting not implemented\") ← EMITTED ABOVE\n");
|
||||
|
||||
println!("=== Fixture Source ===");
|
||||
println!("Shared File: {}", fixture_path);
|
||||
println!("HTML Content: {}", test_html);
|
||||
println!("Expected Parsing Behavior: Text nodes are fostered out of table");
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
/// Test: Verify parsed output for foster_parenting_minimal.html fixture
|
||||
///
|
||||
/// This binary contains tests and assertions that validate the expected behavior
|
||||
/// when parsing HTML that triggers the html5ever::tree_builder::foster_parent_in_body() warning.
|
||||
///
|
||||
/// Uses shared fixture: src/bin/test_fixtures/foster_parenting_minimal.html
|
||||
/// Uses shared parser: src/helpers::parse_html()
|
||||
use scraper::Selector;
|
||||
use scraper_service::shared::utils::parse_html;
|
||||
use std::fs;
|
||||
|
||||
fn main() {
|
||||
println!("Running foster parenting regression tests...\n");
|
||||
|
||||
// Load HTML fixture from shared test file
|
||||
let fixture_path = "src/bin/test_fixtures/foster_parenting_minimal.html";
|
||||
let foster_parenting_html = fs::read_to_string(fixture_path)
|
||||
.expect(&format!("Failed to read fixture from {}", fixture_path));
|
||||
|
||||
test_foster_parenting_text_extraction(&foster_parenting_html);
|
||||
println!("✓ test_foster_parenting_text_extraction passed");
|
||||
|
||||
test_foster_parenting_table_structure(&foster_parenting_html);
|
||||
println!("✓ test_foster_parenting_table_structure passed");
|
||||
|
||||
test_expected_parsed_output_assertion(&foster_parenting_html);
|
||||
println!("✓ test_expected_parsed_output_assertion passed");
|
||||
|
||||
println!("\n✓ All assertions passed (3/3)");
|
||||
println!("\nFixture source: {}", fixture_path);
|
||||
println!("Parser source: src/helpers/web/scraping.rs::parse_html()");
|
||||
}
|
||||
|
||||
/// Test: Text nodes in <table> are foster-parented to body
|
||||
fn test_foster_parenting_text_extraction(html: &str) {
|
||||
let document = parse_html(html);
|
||||
let body_sel = Selector::parse("body").expect("Valid CSS selector");
|
||||
|
||||
let body_text: String = document
|
||||
.select(&body_sel)
|
||||
.next()
|
||||
.map(|el| el.text().collect())
|
||||
.unwrap_or_default();
|
||||
|
||||
assert!(
|
||||
body_text.contains("orphaned text"),
|
||||
"Text 'orphaned text' should be present in body (fostered from table)"
|
||||
);
|
||||
assert!(
|
||||
body_text.contains("more text"),
|
||||
"Text 'more text' should be present in body (fostered from table)"
|
||||
);
|
||||
assert!(
|
||||
body_text.contains("cell content"),
|
||||
"Cell content should still be present"
|
||||
);
|
||||
}
|
||||
|
||||
fn test_foster_parenting_table_structure(html: &str) {
|
||||
let document = parse_html(html);
|
||||
|
||||
let table_sel = Selector::parse("table").expect("Valid CSS selector");
|
||||
let tr_sel = Selector::parse("tr").expect("Valid CSS selector");
|
||||
let td_sel = Selector::parse("td").expect("Valid CSS selector");
|
||||
|
||||
let tables: Vec<_> = document.select(&table_sel).collect();
|
||||
assert_eq!(tables.len(), 1, "Should have exactly 1 table");
|
||||
|
||||
let rows: Vec<_> = document.select(&tr_sel).collect();
|
||||
assert_eq!(rows.len(), 1, "Should have exactly 1 row");
|
||||
|
||||
let cells: Vec<_> = document.select(&td_sel).collect();
|
||||
assert_eq!(cells.len(), 1, "Should have exactly 1 cell");
|
||||
|
||||
if let Some(cell) = cells.first() {
|
||||
let cell_text: String = cell.text().collect();
|
||||
assert_eq!(
|
||||
cell_text.trim(),
|
||||
"cell content",
|
||||
"Cell content should be preserved"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn test_expected_parsed_output_assertion(html: &str) {
|
||||
let document = parse_html(html);
|
||||
|
||||
let body_sel = Selector::parse("body").expect("Valid CSS selector");
|
||||
let body = document
|
||||
.select(&body_sel)
|
||||
.next()
|
||||
.expect("body should exist");
|
||||
let full_text: String = body.text().collect();
|
||||
|
||||
let expected_pattern = "orphaned textmore textcell content";
|
||||
assert!(
|
||||
full_text.contains(&expected_pattern)
|
||||
|| (full_text.contains("orphaned text")
|
||||
&& full_text.contains("more text")
|
||||
&& full_text.contains("cell content")),
|
||||
"Parsed output should contain all text content in fostered form. Got: '{}'",
|
||||
full_text
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn load_fixture() -> String {
|
||||
fs::read_to_string("src/bin/test_fixtures/foster_parenting_minimal.html")
|
||||
.expect("Failed to load fixture")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_foster_parenting_text_extraction_test() {
|
||||
let html = load_fixture();
|
||||
test_foster_parenting_text_extraction(&html);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_foster_parenting_table_structure_test() {
|
||||
let html = load_fixture();
|
||||
test_foster_parenting_table_structure(&html);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_expected_parsed_output_assertion_test() {
|
||||
let html = load_fixture();
|
||||
test_expected_parsed_output_assertion(&html);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
//! Complete API generator - combines model, migration, controller, service, repository
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
pub fn generate_full_api(name: &str, full: bool) -> Result<()> {
|
||||
println!("📦 Generating model...");
|
||||
let model_name = singularize(name);
|
||||
super::model::generate_model(&model_name, true, true, false)?;
|
||||
|
||||
if full {
|
||||
println!("🔧 Generating service...");
|
||||
super::service::generate_service(name, Some(&model_name))?;
|
||||
|
||||
println!("💾 Generating repository...");
|
||||
super::repository::generate_repository(name, &model_name)?;
|
||||
}
|
||||
|
||||
println!("🎮 Generating CRUD controller...");
|
||||
super::controller::generate_controller(name, true, Some(&model_name))?;
|
||||
|
||||
println!("\n✅ Complete API generated!");
|
||||
println!("\n📋 Generated files:");
|
||||
println!(
|
||||
" - src/entities/{}.rs (SeaORM model)",
|
||||
model_name.to_lowercase()
|
||||
);
|
||||
println!(
|
||||
" - migrations/m*_create_{}.rs (migration)",
|
||||
super::model::pluralize(&model_name)
|
||||
);
|
||||
|
||||
if full {
|
||||
println!(" - src/services/{}_service.rs (service layer)", name);
|
||||
println!(" - src/repositories/{}_repository.rs (repository)", name);
|
||||
}
|
||||
|
||||
println!(" - src/routes/api/{}/index.rs (list)", name);
|
||||
println!(" - src/routes/api/{}/[id].rs (get)", name);
|
||||
println!(" - src/routes/api/{}/create.rs (create)", name);
|
||||
println!(" - src/routes/api/{}/[id]/update.rs (update)", name);
|
||||
println!(" - src/routes/api/{}/[id]/delete.rs (delete)", name);
|
||||
|
||||
println!("\n🚀 Next steps:");
|
||||
println!(" 1. Run 'cargo build' to compile");
|
||||
println!(" 2. Run migrations: cargo run -- migration up");
|
||||
println!(" 3. Start server: cargo run");
|
||||
|
||||
println!("\n📡 Available endpoints:");
|
||||
println!(" GET /api/{} - List all", name);
|
||||
println!(" GET /api/{}/{{id}} - Get one", name);
|
||||
println!(" POST /api/{} - Create", name);
|
||||
println!(" PUT /api/{}/{{id}} - Update", name);
|
||||
println!(" DELETE /api/{}/{{id}} - Delete", name);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn singularize(word: &str) -> String {
|
||||
let lower = word.to_lowercase();
|
||||
|
||||
if lower.ends_with("ies") {
|
||||
format!("{}y", &lower[..lower.len() - 3])
|
||||
} else if lower.ends_with("es") {
|
||||
lower[..lower.len() - 2].to_string()
|
||||
} else if lower.ends_with('s') {
|
||||
lower[..lower.len() - 1].to_string()
|
||||
} else {
|
||||
lower
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_singularize() {
|
||||
assert_eq!(singularize("users"), "user");
|
||||
assert_eq!(singularize("categories"), "category");
|
||||
assert_eq!(singularize("posts"), "post");
|
||||
assert_eq!(singularize("boxes"), "box");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
//! API controller generator with CRUD operations
|
||||
|
||||
use anyhow::Result;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
pub fn generate_controller(name: &str, crud: bool, model: Option<&str>) -> Result<()> {
|
||||
let api_dir = Path::new("src/routes/api").join(name);
|
||||
fs::create_dir_all(&api_dir)?;
|
||||
|
||||
let model_name = model.map(|s| s.to_string()).unwrap_or_else(|| {
|
||||
// Singularize the resource name
|
||||
let singular = name.trim_end_matches('s');
|
||||
format!("{}{}", &singular[..1].to_uppercase(), &singular[1..])
|
||||
});
|
||||
|
||||
if crud {
|
||||
generate_crud_routes(&api_dir, name, &model_name)?;
|
||||
} else {
|
||||
generate_basic_controller(&api_dir, name);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn generate_crud_routes(api_dir: &Path, resource: &str, model: &str) -> Result<()> {
|
||||
// List all
|
||||
let index_content = generate_list_handler(resource, model);
|
||||
fs::write(api_dir.join("index.rs"), index_content)?;
|
||||
|
||||
// Get by ID
|
||||
let show_content = generate_show_handler(resource, model);
|
||||
fs::write(api_dir.join("[id].rs"), show_content)?;
|
||||
|
||||
// Create
|
||||
let create_content = generate_create_handler(resource, model);
|
||||
fs::write(api_dir.join("create.rs"), create_content)?;
|
||||
|
||||
// Update & Delete in [id] subdirectory
|
||||
fs::create_dir_all(api_dir.join("[id]"))?;
|
||||
|
||||
let update_content = generate_update_handler(resource, model);
|
||||
fs::write(api_dir.join("[id]/update.rs"), update_content)?;
|
||||
|
||||
let delete_content = generate_delete_handler(resource, model);
|
||||
fs::write(api_dir.join("[id]/delete.rs"), delete_content)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn generate_list_handler(resource: &str, model: &str) -> String {
|
||||
format!(
|
||||
r#"//! List all {resource}
|
||||
|
||||
use axum::{{Extension, Json, response::IntoResponse, Router}};
|
||||
use sea_orm::{{DatabaseConnection, EntityTrait}};
|
||||
use std::sync::Arc;
|
||||
use crate::shared::state::AppState;
|
||||
use crate::entities::{model_low}::{{Entity as {model}, Model}};
|
||||
|
||||
pub async fn list(
|
||||
Extension(db): Extension<DatabaseConnection>,
|
||||
) -> impl IntoResponse {{
|
||||
match {model}.find().all(&db).await {{
|
||||
Ok(items) => Json(items).into_response(),
|
||||
Err(e) => {{
|
||||
eprintln!("Error listing {resource}: {{}}", e);
|
||||
(axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to list {resource}").into_response()
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
|
||||
pub fn register_routes(router: Router<Arc<AppState>>) -> Router<Arc<AppState>> {{
|
||||
router
|
||||
}}
|
||||
"#,
|
||||
resource = resource,
|
||||
model_low = model.to_lowercase(),
|
||||
model = model
|
||||
)
|
||||
}
|
||||
|
||||
fn generate_show_handler(resource: &str, model: &str) -> String {
|
||||
let singular = resource.trim_end_matches('s');
|
||||
format!(
|
||||
r#"//! Get {singular} by ID
|
||||
|
||||
use axum::{{Extension, Json, extract::Path, response::IntoResponse, Router}};
|
||||
use sea_orm::{{DatabaseConnection, EntityTrait}};
|
||||
use std::sync::Arc;
|
||||
use crate::shared::state::AppState;
|
||||
use crate::entities::{model_low}::{{Entity as {model}, Model}};
|
||||
|
||||
pub async fn show(
|
||||
Path(id): Path<i32>,
|
||||
Extension(db): Extension<DatabaseConnection>,
|
||||
) -> impl IntoResponse {{
|
||||
match {model}.find_by_id(id).one(&db).await {{
|
||||
Ok(Some(item)) => Json(item).into_response(),
|
||||
Ok(None) => (axum::http::StatusCode::NOT_FOUND, "{singular} not found").into_response(),
|
||||
Err(e) => {{
|
||||
eprintln!("Error getting {singular}: {{}}", e);
|
||||
(axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to get {singular}").into_response()
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
|
||||
pub fn register_routes(router: Router<Arc<AppState>>) -> Router<Arc<AppState>> {{
|
||||
router
|
||||
}}
|
||||
"#,
|
||||
singular = singular,
|
||||
model_low = model.to_lowercase(),
|
||||
model = model,
|
||||
resource = resource
|
||||
)
|
||||
}
|
||||
|
||||
fn generate_create_handler(resource: &str, model: &str) -> String {
|
||||
let singular = resource.trim_end_matches('s');
|
||||
format!(
|
||||
r#"//! Create new {singular}
|
||||
|
||||
use axum::{{Extension, Json, response::IntoResponse, Router}};
|
||||
use sea_orm::{{ActiveModelTrait, DatabaseConnection, Set}};
|
||||
use serde::{{Deserialize, Serialize}};
|
||||
use std::sync::Arc;
|
||||
use crate::shared::state::AppState;
|
||||
use crate::entities::{model_low}::{{ActiveModel, Model}};
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct Create{model}Dto {{
|
||||
pub name: String,
|
||||
// Add your fields
|
||||
}}
|
||||
|
||||
pub async fn create(
|
||||
Extension(db): Extension<DatabaseConnection>,
|
||||
Json(data): Json<Create{model}Dto>,
|
||||
) -> impl IntoResponse {{
|
||||
let new_item = ActiveModel {{
|
||||
name: Set(data.name),
|
||||
..Default::default()
|
||||
}};
|
||||
|
||||
match new_item.insert(&db).await {{
|
||||
Ok(item) => (axum::http::StatusCode::CREATED, Json(item)).into_response(),
|
||||
Err(e) => {{
|
||||
eprintln!("Error creating {singular}: {{}}", e);
|
||||
(axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to create {singular}").into_response()
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
|
||||
pub fn register_routes(router: Router<Arc<AppState>>) -> Router<Arc<AppState>> {{
|
||||
router
|
||||
}}
|
||||
"#,
|
||||
singular = singular,
|
||||
model_low = model.to_lowercase(),
|
||||
model = model,
|
||||
resource = resource
|
||||
)
|
||||
}
|
||||
|
||||
fn generate_update_handler(resource: &str, model: &str) -> String {
|
||||
let singular = resource.trim_end_matches('s');
|
||||
format!(
|
||||
r#"//! Update {singular}
|
||||
|
||||
use axum::{{Extension, Json, extract::Path, response::IntoResponse, Router}};
|
||||
use sea_orm::{{ActiveModelTrait, DatabaseConnection, EntityTrait, Set}};
|
||||
use serde::{{Deserialize, Serialize}};
|
||||
use std::sync::Arc;
|
||||
use crate::shared::state::AppState;
|
||||
use crate::entities::{model_low}::{{ActiveModel, Entity as {model}, Model}};
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct Update{model}Dto {{
|
||||
pub name: Option<String>,
|
||||
// Add your fields
|
||||
}}
|
||||
|
||||
pub async fn update(
|
||||
Path(id): Path<i32>,
|
||||
Extension(db): Extension<DatabaseConnection>,
|
||||
Json(data): Json<Update{model}Dto>,
|
||||
) -> impl IntoResponse {{
|
||||
let item = match {model}.find_by_id(id).one(&db).await {{
|
||||
Ok(Some(item)) => item,
|
||||
Ok(None) => return (axum::http::StatusCode::NOT_FOUND, "{singular} not found").into_response(),
|
||||
Err(e) => {{
|
||||
eprintln!("Error finding {singular}: {{}}", e);
|
||||
return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to find {singular}").into_response();
|
||||
}}
|
||||
}};
|
||||
|
||||
let mut active_model: ActiveModel = item.into();
|
||||
if let Some(name) = data.name {{
|
||||
active_model.name = Set(name);
|
||||
}}
|
||||
|
||||
match active_model.update(&db).await {{
|
||||
Ok(updated) => Json(updated).into_response(),
|
||||
Err(e) => {{
|
||||
eprintln!("Error updating {singular}: {{}}", e);
|
||||
(axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to update {singular}").into_response()
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
|
||||
pub fn register_routes(router: Router<Arc<AppState>>) -> Router<Arc<AppState>> {{
|
||||
router
|
||||
}}
|
||||
"#,
|
||||
singular = singular,
|
||||
model_low = model.to_lowercase(),
|
||||
model = model,
|
||||
resource = resource
|
||||
)
|
||||
}
|
||||
|
||||
fn generate_delete_handler(resource: &str, model: &str) -> String {
|
||||
let singular = resource.trim_end_matches('s');
|
||||
format!(
|
||||
r#"//! Delete {singular}
|
||||
|
||||
use axum::{{Extension, extract::Path, response::IntoResponse, Router}};
|
||||
use sea_orm::{{ActiveModelTrait, DatabaseConnection, EntityTrait, IntoActiveModel}};
|
||||
use std::sync::Arc;
|
||||
use crate::shared::state::AppState;
|
||||
use crate::entities::{model_low}::{{Entity as {model}}};
|
||||
|
||||
pub async fn destroy(
|
||||
Path(id): Path<i32>,
|
||||
Extension(db): Extension<DatabaseConnection>,
|
||||
) -> impl IntoResponse {{
|
||||
let item = match {model}.find_by_id(id).one(&db).await {{
|
||||
Ok(Some(item)) => item,
|
||||
Ok(None) => return (axum::http::StatusCode::NOT_FOUND, "{singular} not found").into_response(),
|
||||
Err(e) => {{
|
||||
eprintln!("Error finding {singular}: {{}}", e);
|
||||
return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to find {singular}").into_response();
|
||||
}}
|
||||
}};
|
||||
|
||||
match item.into_active_model().delete(&db).await {{
|
||||
Ok(_) => axum::http::StatusCode::NO_CONTENT.into_response(),
|
||||
Err(e) => {{
|
||||
eprintln!("Error deleting {singular}: {{}}", e);
|
||||
(axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to delete {singular}").into_response()
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
|
||||
pub fn register_routes(router: Router<Arc<AppState>>) -> Router<Arc<AppState>> {{
|
||||
router
|
||||
}}
|
||||
"#,
|
||||
singular = singular,
|
||||
model_low = model.to_lowercase(),
|
||||
model = model,
|
||||
resource = resource
|
||||
)
|
||||
}
|
||||
|
||||
fn generate_basic_controller(api_dir: &Path, resource: &str) {
|
||||
let content = format!(
|
||||
r#"//! {resource} controller
|
||||
|
||||
use axum::Router;
|
||||
use std::sync::Arc;
|
||||
use crate::shared::state::AppState;
|
||||
|
||||
pub async fn index() -> &'static str {{
|
||||
"{resource} endpoint"
|
||||
}}
|
||||
|
||||
pub fn register_routes(router: Router<Arc<AppState>>) -> Router<Arc<AppState>> {{
|
||||
router
|
||||
}}
|
||||
"#,
|
||||
resource = resource
|
||||
);
|
||||
|
||||
let _ = fs::write(api_dir.join("index.rs"), content);
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
//! Database migration generator
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use chrono::Local;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
pub fn generate_migration(name: &str, table: Option<&str>) -> Result<()> {
|
||||
let timestamp = Local::now().format("%Y%m%d%H%M%S");
|
||||
let file_name = format!("m{}_{}.rs", timestamp, name);
|
||||
|
||||
let migrations_dir = Path::new("migrations");
|
||||
fs::create_dir_all(migrations_dir)?;
|
||||
|
||||
let content = if let Some(table_name) = table {
|
||||
generate_create_table_migration(table_name)
|
||||
} else {
|
||||
generate_empty_migration()
|
||||
};
|
||||
|
||||
let migration_path = migrations_dir.join(&file_name);
|
||||
fs::write(&migration_path, content)
|
||||
.with_context(|| format!("Failed to write migration: {:?}", migration_path))?;
|
||||
|
||||
update_migrations_mod(&file_name)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn generate_model_migration(table: &str, timestamps: bool, soft_delete: bool) -> Result<()> {
|
||||
let timestamp = Local::now().format("%Y%m%d%H%M%S");
|
||||
let name = format!("create_{}_table", table);
|
||||
let file_name = format!("m{}_{}.rs", timestamp, name);
|
||||
|
||||
let migrations_dir = Path::new("migrations");
|
||||
fs::create_dir_all(migrations_dir)?;
|
||||
|
||||
let content = generate_model_table_migration(table, timestamps, soft_delete);
|
||||
|
||||
let migration_path = migrations_dir.join(&file_name);
|
||||
fs::write(&migration_path, content)?;
|
||||
|
||||
update_migrations_mod(&file_name)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn generate_create_table_migration(table: &str) -> String {
|
||||
let struct_name = table
|
||||
.split('_')
|
||||
.map(|s| {
|
||||
let mut c = s.chars();
|
||||
match c.next() {
|
||||
None => String::new(),
|
||||
Some(f) => f.to_uppercase().chain(c).collect(),
|
||||
}
|
||||
})
|
||||
.collect::<String>();
|
||||
|
||||
format!(
|
||||
r#"use sea_orm_migration::prelude::*;
|
||||
|
||||
#[derive(DeriveMigrationName)]
|
||||
pub struct Migration;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl MigrationTrait for Migration {{
|
||||
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {{
|
||||
manager
|
||||
.create_table(
|
||||
Table::create()
|
||||
.table({table}::Table)
|
||||
.if_not_exists()
|
||||
.col(
|
||||
ColumnDef::new({table}::Id)
|
||||
.integer()
|
||||
.not_null()
|
||||
.auto_increment()
|
||||
.primary_key(),
|
||||
)
|
||||
.col(ColumnDef::new({table}::Name).string().not_null())
|
||||
.to_owned(),
|
||||
)
|
||||
.await
|
||||
}}
|
||||
|
||||
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {{
|
||||
manager
|
||||
.drop_table(Table::drop().table({table}::Table).to_owned())
|
||||
.await
|
||||
}}
|
||||
}}
|
||||
|
||||
#[derive(DeriveIden)]
|
||||
enum {table} {{
|
||||
Table,
|
||||
Id,
|
||||
Name,
|
||||
}}
|
||||
"#,
|
||||
table = struct_name
|
||||
)
|
||||
}
|
||||
|
||||
fn generate_model_table_migration(table: &str, timestamps: bool, soft_delete: bool) -> String {
|
||||
let table_pascal = table
|
||||
.split('_')
|
||||
.map(|s| {
|
||||
let mut c = s.chars();
|
||||
match c.next() {
|
||||
None => String::new(),
|
||||
Some(f) => f.to_uppercase().chain(c).collect(),
|
||||
}
|
||||
})
|
||||
.collect::<String>();
|
||||
|
||||
let timestamp_cols = if timestamps {
|
||||
format!(
|
||||
r#"
|
||||
.col(ColumnDef::new({}::CreatedAt).timestamp().null())
|
||||
.col(ColumnDef::new({}::UpdatedAt).timestamp().null())"#,
|
||||
table_pascal, table_pascal
|
||||
)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
|
||||
let soft_delete_col = if soft_delete {
|
||||
format!(
|
||||
r#"
|
||||
.col(ColumnDef::new({}::DeletedAt).timestamp().null())"#,
|
||||
table_pascal
|
||||
)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
|
||||
let enum_fields = if timestamps && soft_delete {
|
||||
format!(" CreatedAt,\n UpdatedAt,\n DeletedAt,")
|
||||
} else if timestamps {
|
||||
format!(" CreatedAt,\n UpdatedAt,")
|
||||
} else if soft_delete {
|
||||
format!(" DeletedAt,")
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
|
||||
format!(
|
||||
r#"use sea_orm_migration::prelude::*;
|
||||
|
||||
#[derive(DeriveMigrationName)]
|
||||
pub struct Migration;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl MigrationTrait for Migration {{
|
||||
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {{
|
||||
manager
|
||||
.create_table(
|
||||
Table::create()
|
||||
.table({table}::Table)
|
||||
.if_not_exists()
|
||||
.col(
|
||||
ColumnDef::new({table}::Id)
|
||||
.integer()
|
||||
.not_null()
|
||||
.auto_increment()
|
||||
.primary_key(),
|
||||
)
|
||||
.col(ColumnDef::new({table}::Name).string().not_null()){timestamps}{soft_delete}
|
||||
.to_owned(),
|
||||
)
|
||||
.await
|
||||
}}
|
||||
|
||||
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {{
|
||||
manager
|
||||
.drop_table(Table::drop().table({table}::Table).to_owned())
|
||||
.await
|
||||
}}
|
||||
}}
|
||||
|
||||
#[derive(DeriveIden)]
|
||||
enum {table} {{
|
||||
Table,
|
||||
Id,
|
||||
Name,
|
||||
{enum_fields}
|
||||
}}
|
||||
"#,
|
||||
table = table_pascal,
|
||||
timestamps = timestamp_cols,
|
||||
soft_delete = soft_delete_col,
|
||||
enum_fields = enum_fields
|
||||
)
|
||||
}
|
||||
|
||||
fn generate_empty_migration() -> String {
|
||||
format!(
|
||||
r#"use sea_orm_migration::prelude::*;
|
||||
|
||||
#[derive(DeriveMigrationName)]
|
||||
pub struct Migration;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl MigrationTrait for Migration {{
|
||||
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {{
|
||||
// Add your migration logic here
|
||||
Ok(())
|
||||
}}
|
||||
|
||||
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {{
|
||||
// Add your rollback logic here
|
||||
Ok(())
|
||||
}}
|
||||
}}
|
||||
"#
|
||||
)
|
||||
}
|
||||
|
||||
fn update_migrations_mod(file_name: &str) -> Result<()> {
|
||||
let mod_path = Path::new("migrations/mod.rs");
|
||||
let module_name = file_name.trim_end_matches(".rs");
|
||||
let module_line = format!("mod {};", module_name);
|
||||
|
||||
if mod_path.exists() {
|
||||
let content = fs::read_to_string(mod_path)?;
|
||||
if !content.contains(&module_line) {
|
||||
// Find the vec![] and add migration
|
||||
let new_content = if content.contains("vec![") {
|
||||
content.replace(
|
||||
"vec![",
|
||||
&format!("vec![\n Box::new({}::Migration),", module_name),
|
||||
)
|
||||
} else {
|
||||
format!("{}\n{}", content.trim(), module_line)
|
||||
};
|
||||
fs::write(mod_path, new_content)?;
|
||||
}
|
||||
} else {
|
||||
let initial_content = format!(
|
||||
r#"pub use sea_orm_migration::prelude::*;
|
||||
|
||||
{}
|
||||
|
||||
pub struct Migrator;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl MigratorTrait for Migrator {{
|
||||
fn migrations() -> Vec<Box<dyn MigrationTrait>> {{
|
||||
vec![
|
||||
Box::new({}::Migration),
|
||||
]
|
||||
}}
|
||||
}}
|
||||
"#,
|
||||
module_line, module_name
|
||||
);
|
||||
fs::write(mod_path, initial_content)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
// Module declarations for generators
|
||||
pub mod api;
|
||||
pub mod controller;
|
||||
pub mod migration;
|
||||
pub mod model;
|
||||
pub mod repository;
|
||||
pub mod service;
|
||||
@@ -0,0 +1,125 @@
|
||||
//! SeaORM model generator
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
pub fn generate_model(
|
||||
name: &str,
|
||||
with_migration: bool,
|
||||
timestamps: bool,
|
||||
soft_delete: bool,
|
||||
) -> Result<()> {
|
||||
let table_name = pluralize(name);
|
||||
|
||||
// Create entities directory
|
||||
let entities_dir = Path::new("src/entities");
|
||||
fs::create_dir_all(entities_dir)?;
|
||||
|
||||
// Generate model file
|
||||
let model_content = generate_model_content(name, &table_name, timestamps, soft_delete);
|
||||
let model_path = entities_dir.join(format!("{}.rs", name.to_lowercase()));
|
||||
fs::write(&model_path, model_content)
|
||||
.with_context(|| format!("Failed to write model file: {:?}", model_path))?;
|
||||
|
||||
// Update entities/mod.rs
|
||||
update_entities_mod(name)?;
|
||||
|
||||
// Generate migration if requested
|
||||
if with_migration {
|
||||
super::migration::generate_model_migration(&table_name, timestamps, soft_delete)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn generate_model_content(
|
||||
name: &str,
|
||||
table_name: &str,
|
||||
timestamps: bool,
|
||||
soft_delete: bool,
|
||||
) -> String {
|
||||
let timestamp_fields = if timestamps {
|
||||
r#"
|
||||
#[sea_orm(nullable)]
|
||||
pub created_at: Option<DateTimeUtc>,
|
||||
#[sea_orm(nullable)]
|
||||
pub updated_at: Option<DateTimeUtc>,"#
|
||||
} else {
|
||||
""
|
||||
};
|
||||
|
||||
let soft_delete_field = if soft_delete {
|
||||
r#"
|
||||
#[sea_orm(nullable)]
|
||||
pub deleted_at: Option<DateTimeUtc>,"#
|
||||
} else {
|
||||
""
|
||||
};
|
||||
|
||||
format!(
|
||||
r#"//! {} entity
|
||||
|
||||
use sea_orm::entity::prelude::*;
|
||||
use serde::{{Deserialize, Serialize}};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)]
|
||||
#[sea_orm(table_name = "{}")]
|
||||
pub struct Model {{
|
||||
#[sea_orm(primary_key)]
|
||||
pub id: i32,
|
||||
|
||||
// Add your fields here
|
||||
pub name: String,{}{}
|
||||
}}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
pub enum Relation {{}}
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {{}}
|
||||
"#,
|
||||
name, table_name, timestamp_fields, soft_delete_field
|
||||
)
|
||||
}
|
||||
|
||||
fn update_entities_mod(name: &str) -> Result<()> {
|
||||
let mod_path = Path::new("src/entities/mod.rs");
|
||||
let module_line = format!("pub mod {};", name.to_lowercase());
|
||||
|
||||
if mod_path.exists() {
|
||||
let content = fs::read_to_string(mod_path)?;
|
||||
if !content.contains(&module_line) {
|
||||
let new_content = format!("{}\n{}", content.trim(), module_line);
|
||||
fs::write(mod_path, new_content)?;
|
||||
}
|
||||
} else {
|
||||
fs::write(mod_path, format!("{}\n", module_line))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn pluralize(word: &str) -> String {
|
||||
let lower = word.to_lowercase();
|
||||
|
||||
if lower.ends_with('y') {
|
||||
format!("{}ies", &lower[..lower.len() - 1])
|
||||
} else if lower.ends_with('s') || lower.ends_with("ch") || lower.ends_with("sh") || lower.ends_with('x') {
|
||||
format!("{}es", lower)
|
||||
} else {
|
||||
format!("{}s", lower)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_pluralize() {
|
||||
assert_eq!(pluralize("User"), "users");
|
||||
assert_eq!(pluralize("Category"), "categories");
|
||||
assert_eq!(pluralize("Post"), "posts");
|
||||
assert_eq!(pluralize("Box"), "boxes");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
//! Repository pattern generator
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
pub fn generate_repository(name: &str, model: &str) -> Result<()> {
|
||||
let repos_dir = Path::new("src/repositories");
|
||||
fs::create_dir_all(repos_dir)?;
|
||||
|
||||
let repo_content = generate_repository_content(name, model);
|
||||
|
||||
let repo_path = repos_dir.join(format!("{}_repository.rs", name.to_lowercase()));
|
||||
fs::write(&repo_path, repo_content)
|
||||
.with_context(|| format!("Failed to write repository: {:?}", repo_path))?;
|
||||
|
||||
update_repositories_mod(name)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn generate_repository_content(name: &str, model: &str) -> String {
|
||||
format!(
|
||||
r#"//! {} repository
|
||||
|
||||
use sea_orm::*;
|
||||
use crate::entities::{}::{{Entity as {}, Model, ActiveModel, Column}};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct {}Repository {{
|
||||
db: DatabaseConnection,
|
||||
}}
|
||||
|
||||
impl {}Repository {{
|
||||
pub fn new(db: DatabaseConnection) -> Self {{
|
||||
Self {{ db }}
|
||||
}}
|
||||
|
||||
/// Find all records
|
||||
pub async fn find_all(&self) -> Result<Vec<Model>, DbErr> {{
|
||||
{}.find().all(&self.db).await
|
||||
}}
|
||||
|
||||
/// Find by ID
|
||||
pub async fn find_by_id(&self, id: i32) -> Result<Option<Model>, DbErr> {{
|
||||
{}.find_by_id(id).one(&self.db).await
|
||||
}}
|
||||
|
||||
/// Find with pagination
|
||||
pub async fn paginate(&self, page: u64, per_page: u64) -> Result<(Vec<Model>, u64), DbErr> {{
|
||||
let paginator = {}.find()
|
||||
.paginate(&self.db, per_page);
|
||||
|
||||
let total = paginator.num_items().await?;
|
||||
let items = paginator.fetch_page(page).await?;
|
||||
|
||||
Ok((items, total))
|
||||
}}
|
||||
|
||||
/// Create new record
|
||||
pub async fn create(&self, data: ActiveModel) -> Result<Model, DbErr> {{
|
||||
data.insert(&self.db).await
|
||||
}}
|
||||
|
||||
/// Update existing record
|
||||
pub async fn update(&self, data: ActiveModel) -> Result<Model, DbErr> {{
|
||||
data.update(&self.db).await
|
||||
}}
|
||||
|
||||
/// Delete by ID
|
||||
pub async fn delete(&self, id: i32) -> Result<DeleteResult, DbErr> {{
|
||||
{}.delete_by_id(id).exec(&self.db).await
|
||||
}}
|
||||
|
||||
/// Find by custom condition
|
||||
pub async fn find_by_name(&self, name: &str) -> Result<Vec<Model>, DbErr> {{
|
||||
{}.find()
|
||||
.filter(Column::Name.contains(name))
|
||||
.all(&self.db)
|
||||
.await
|
||||
}}
|
||||
}}
|
||||
"#,
|
||||
name,
|
||||
model.to_lowercase(),
|
||||
model,
|
||||
model,
|
||||
model,
|
||||
model,
|
||||
model,
|
||||
model,
|
||||
model,
|
||||
model
|
||||
)
|
||||
}
|
||||
|
||||
fn update_repositories_mod(name: &str) -> Result<()> {
|
||||
let mod_path = Path::new("src/repositories/mod.rs");
|
||||
let module_line = format!("pub mod {}_repository;", name.to_lowercase());
|
||||
|
||||
if mod_path.exists() {
|
||||
let content = fs::read_to_string(mod_path)?;
|
||||
if !content.contains(&module_line) {
|
||||
let new_content = format!("{}\n{}", content.trim(), module_line);
|
||||
fs::write(mod_path, new_content)?;
|
||||
}
|
||||
} else {
|
||||
fs::write(mod_path, format!("{}\n", module_line))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
//! Service layer generator
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
pub fn generate_service(name: &str, model: Option<&str>) -> Result<()> {
|
||||
let services_dir = Path::new("src/services");
|
||||
fs::create_dir_all(services_dir)?;
|
||||
|
||||
let model_name = model.unwrap_or(name);
|
||||
let service_content = generate_service_content(name, model_name);
|
||||
|
||||
let service_path = services_dir.join(format!("{}_service.rs", name.to_lowercase()));
|
||||
fs::write(&service_path, service_content)
|
||||
.with_context(|| format!("Failed to write service: {:?}", service_path))?;
|
||||
|
||||
update_services_mod(name)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn generate_service_content(name: &str, model: &str) -> String {
|
||||
format!(
|
||||
r#"//! {} service layer
|
||||
|
||||
use sea_orm::*;
|
||||
use crate::entities::{}::{{Entity as {}, Model, ActiveModel}};
|
||||
|
||||
pub struct {}Service {{
|
||||
db: DatabaseConnection,
|
||||
}}
|
||||
|
||||
impl {}Service {{
|
||||
pub fn new(db: DatabaseConnection) -> Self {{
|
||||
Self {{ db }}
|
||||
}}
|
||||
|
||||
pub async fn find_all(&self) -> Result<Vec<Model>, DbErr> {{
|
||||
{}.find().all(&self.db).await
|
||||
}}
|
||||
|
||||
pub async fn find_by_id(&self, id: i32) -> Result<Option<Model>, DbErr> {{
|
||||
{}.find_by_id(id).one(&self.db).await
|
||||
}}
|
||||
|
||||
pub async fn create(&self, data: ActiveModel) -> Result<Model, DbErr> {{
|
||||
data.insert(&self.db).await
|
||||
}}
|
||||
|
||||
pub async fn update(&self, id: i32, data: ActiveModel) -> Result<Model, DbErr> {{
|
||||
data.update(&self.db).await
|
||||
}}
|
||||
|
||||
pub async fn delete(&self, id: i32) -> Result<DeleteResult, DbErr> {{
|
||||
{}.delete_by_id(id).exec(&self.db).await
|
||||
}}
|
||||
}}
|
||||
"#,
|
||||
name,
|
||||
model.to_lowercase(),
|
||||
model,
|
||||
model,
|
||||
model,
|
||||
model,
|
||||
model,
|
||||
model
|
||||
)
|
||||
}
|
||||
|
||||
fn update_services_mod(name: &str) -> Result<()> {
|
||||
let mod_path = Path::new("src/services/mod.rs");
|
||||
let module_line = format!("pub mod {}_service;", name.to_lowercase());
|
||||
|
||||
if mod_path.exists() {
|
||||
let content = fs::read_to_string(mod_path)?;
|
||||
if !content.contains(&module_line) {
|
||||
let new_content = format!("{}\n{}", content.trim(), module_line);
|
||||
fs::write(mod_path, new_content)?;
|
||||
}
|
||||
} else {
|
||||
fs::write(mod_path, format!("{}\n", module_line))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
<table>orphaned text<tr><td>cell content</td></tr>more text</table>
|
||||
@@ -0,0 +1,118 @@
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
use axum::Router;
|
||||
use sea_orm::Database;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
use crate::shared::config::CONFIG;
|
||||
use crate::shared::database::get_redis_conn;
|
||||
use crate::shared::state::AppState;
|
||||
|
||||
pub struct Application {
|
||||
pub port: u16,
|
||||
router: Router,
|
||||
listener: TcpListener,
|
||||
}
|
||||
|
||||
impl Application {
|
||||
pub async fn build() -> anyhow::Result<Self> {
|
||||
// Initialize tracing. Default to warn/error globally unless RUST_LOG is explicitly set.
|
||||
let env_filter = match std::env::var("RUST_LOG") {
|
||||
Ok(filter) => EnvFilter::new(filter).add_directive("html5ever=error".parse()?),
|
||||
Err(_) => EnvFilter::new("warn,html5ever=error"),
|
||||
};
|
||||
|
||||
tracing_subscriber::fmt().with_env_filter(env_filter).init();
|
||||
|
||||
// Initialize OpenTelemetry metrics
|
||||
crate::shared::observability::metrics::init_otel_metrics();
|
||||
|
||||
tracing::info!("🚀 Scraper starting up...");
|
||||
tracing::info!(" Environment: {}", CONFIG.environment);
|
||||
|
||||
// Log thread configuration
|
||||
let worker_threads = std::thread::available_parallelism()
|
||||
.map(|n| n.get())
|
||||
.unwrap_or(1);
|
||||
tracing::info!(
|
||||
" Tokio Worker Threads: (Defaulting to CPU cores: {})",
|
||||
worker_threads
|
||||
);
|
||||
|
||||
// Redis
|
||||
let _ = get_redis_conn().await;
|
||||
|
||||
// Browser Pool
|
||||
tracing::info!("Initializing browser pool...");
|
||||
let browser_config = crate::shared::browser::BrowserPoolConfig::default();
|
||||
match crate::shared::browser::pool::init_browser_pool(browser_config).await {
|
||||
Ok(_) => tracing::info!("✓ Browser pool initialized"),
|
||||
Err(e) => tracing::error!("⚠️ Failed to initialize browser pool: {}", e),
|
||||
}
|
||||
|
||||
// Database
|
||||
let mut opt = sea_orm::ConnectOptions::new(CONFIG.database_url.clone());
|
||||
opt.max_connections(20)
|
||||
.min_connections(1)
|
||||
.connect_timeout(std::time::Duration::from_secs(
|
||||
CONFIG.db.connect_timeout_seconds,
|
||||
))
|
||||
.idle_timeout(std::time::Duration::from_secs(
|
||||
CONFIG.db.idle_timeout_seconds,
|
||||
))
|
||||
.acquire_timeout(std::time::Duration::from_secs(
|
||||
CONFIG.db.acquire_timeout_seconds,
|
||||
))
|
||||
.max_lifetime(std::time::Duration::from_secs(
|
||||
CONFIG.db.max_lifetime_seconds,
|
||||
))
|
||||
.sqlx_logging(CONFIG.log_level == "debug");
|
||||
|
||||
let db = Database::connect(opt)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to connect to database: {}", e))?;
|
||||
tracing::info!("✓ SeaORM database connection established");
|
||||
|
||||
// Schema & Seeding
|
||||
if let Err(e) = crate::shared::database::setup::init(&db).await {
|
||||
tracing::error!("Failed to init DB schema: {}", e);
|
||||
}
|
||||
|
||||
// App State components
|
||||
let db_arc = Arc::new(db);
|
||||
let image_processing_semaphore = Arc::new(tokio::sync::Semaphore::new(
|
||||
CONFIG.image_processing_concurrency,
|
||||
));
|
||||
let event_bus = Arc::new(crate::shared::events::bus::EventBus::new());
|
||||
|
||||
let redis_pool = crate::shared::database::redis_pool()
|
||||
.map_err(|e| anyhow::anyhow!("Failed to init Redis pool: {}", e))?;
|
||||
|
||||
let app_state = Arc::new(AppState {
|
||||
redis_pool,
|
||||
db: db_arc.clone(),
|
||||
image_processing_semaphore,
|
||||
event_bus: event_bus.clone(),
|
||||
});
|
||||
|
||||
let app = crate::app::build_router(app_state, db_arc.clone()).await?;
|
||||
|
||||
// Listener
|
||||
let port = CONFIG.server_port;
|
||||
let addr = SocketAddr::from(([0, 0, 0, 0], port));
|
||||
let listener = TcpListener::bind(&addr).await?;
|
||||
tracing::info!("Server listening on {}", listener.local_addr()?);
|
||||
|
||||
Ok(Self {
|
||||
port,
|
||||
router: app,
|
||||
listener,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn run(self) -> std::io::Result<()> {
|
||||
axum::serve(self.listener, self.router.into_make_service()).await
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
// Library root - clean organized module structure
|
||||
// All modules organized into logical folders
|
||||
|
||||
// ============================================================================
|
||||
// Core Framework
|
||||
// ============================================================================
|
||||
pub mod app;
|
||||
pub mod bootstrap;
|
||||
pub mod modules;
|
||||
pub mod shared;
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
#![doc = "Logging Setup"]
|
||||
use scraper_service::bootstrap::Application;
|
||||
|
||||
#[tokio::main(flavor = "multi_thread")]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
// Application setup and server logic is now encapsulated in `startup.rs`
|
||||
// This makes the main function clean and the app easier to integration test.
|
||||
let app = Application::build().await?;
|
||||
app.run().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
use crate::modules::anime::repository::AnimeRepository;
|
||||
use crate::modules::anime::service::AnimeService;
|
||||
use crate::shared::errors::AppError;
|
||||
use crate::shared::state::AppState;
|
||||
use axum::extract::{Path, State};
|
||||
use axum::Json;
|
||||
use std::sync::Arc;
|
||||
use tracing::info;
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime",
|
||||
tag = "anime",
|
||||
operation_id = "anime_index",
|
||||
responses(
|
||||
(status = 200, description = "Handles GET requests for the /api/anime endpoint.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn anime_index(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
) -> Result<Json<crate::modules::anime::types::AnimeData>, AppError> {
|
||||
info!("Handling request for anime index");
|
||||
let service = AnimeService::new(AnimeRepository::new());
|
||||
service.get_anime_index(app_state).await.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime/genre_list",
|
||||
tag = "anime",
|
||||
operation_id = "anime_genre_list",
|
||||
responses(
|
||||
(status = 200, description = "Handles GET requests for the /api/anime/genre_list endpoint.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn genres(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
) -> Result<Json<crate::modules::anime::types::GenresResponse>, AppError> {
|
||||
info!("Handling request for anime genres");
|
||||
let service = AnimeService::new(AnimeRepository::new());
|
||||
service.get_genres(app_state).await.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime/detail/{slug}",
|
||||
tag = "anime",
|
||||
operation_id = "anime_detail_slug",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves details for a specific detail by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn detail_slug(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
Path(slug): Path<String>,
|
||||
) -> Result<Json<crate::modules::anime::types::DetailResponse>, AppError> {
|
||||
info!("Starting request for detail slug: {}", slug);
|
||||
let service = AnimeService::new(AnimeRepository::new());
|
||||
service.get_anime_detail(app_state, slug).await.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime/complete_anime/{slug}",
|
||||
tag = "anime",
|
||||
operation_id = "anime_complete_anime_slug",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves details for a specific complete_anime by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn complete_anime_slug(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
Path(slug): Path<String>,
|
||||
) -> Result<Json<crate::modules::anime::types::ListResponse>, AppError> {
|
||||
info!("Starting request for complete_anime slug: {}", slug);
|
||||
let service = AnimeService::new(AnimeRepository::new());
|
||||
service
|
||||
.get_complete_anime_page(app_state, slug)
|
||||
.await
|
||||
.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime/full/{slug}",
|
||||
tag = "anime",
|
||||
operation_id = "anime_full_slug",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves full episode details for a specific episode by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn full_slug(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
Path(slug): Path<String>,
|
||||
) -> Result<Json<crate::modules::anime::types::FullResponse>, AppError> {
|
||||
info!("Starting request for full slug: {}", slug);
|
||||
let service = AnimeService::new(AnimeRepository::new());
|
||||
service.get_anime_full(app_state, slug).await.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime/ongoing_anime/{slug}",
|
||||
tag = "anime",
|
||||
operation_id = "anime_ongoing_anime_slug",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves details for a specific ongoing_anime by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn ongoing_anime_slug(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
Path(slug): Path<String>,
|
||||
) -> Result<Json<crate::modules::anime::types::OngoingAnimeResponse>, AppError> {
|
||||
info!("Starting request for ongoing_anime slug: {}", slug);
|
||||
let service = AnimeService::new(AnimeRepository::new());
|
||||
service
|
||||
.get_ongoing_anime_page(app_state, slug)
|
||||
.await
|
||||
.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime/latest/{slug}",
|
||||
tag = "anime",
|
||||
operation_id = "anime_latest_slug",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves details for a specific latest by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn latest_slug(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
Path(slug): Path<String>,
|
||||
) -> Result<Json<crate::modules::anime::types::LatestAnimeResponse>, AppError> {
|
||||
info!("Starting request for latest slug: {}", slug);
|
||||
let service = AnimeService::new(AnimeRepository::new());
|
||||
service
|
||||
.get_latest_anime_page(app_state, slug)
|
||||
.await
|
||||
.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime/search/{slug}",
|
||||
tag = "anime",
|
||||
operation_id = "anime_search_slug_index",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves details for a specific search by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn search_slug_index(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
Path(slug): Path<String>,
|
||||
) -> Result<Json<crate::modules::anime::types::SearchResponse>, AppError> {
|
||||
info!("Starting request for search slug: {}", slug);
|
||||
let service = AnimeService::new(AnimeRepository::new());
|
||||
service
|
||||
.get_search_anime_page(app_state, slug, "1".to_string())
|
||||
.await
|
||||
.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime/search/{slug}/{page}",
|
||||
tag = "anime",
|
||||
operation_id = "anime_search_slug_page",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves details for a specific search by slug and page.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn search_slug_page(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
Path((slug, page)): Path<(String, String)>,
|
||||
) -> Result<Json<crate::modules::anime::types::SearchResponse>, AppError> {
|
||||
info!("Starting request for search slug: {} page: {}", slug, page);
|
||||
let service = AnimeService::new(AnimeRepository::new());
|
||||
service
|
||||
.get_search_anime_page(app_state, slug, page)
|
||||
.await
|
||||
.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime/genre/{slug}",
|
||||
tag = "anime",
|
||||
operation_id = "anime_genre_slug_index",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves details for a specific genre by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn genre_slug_index(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
Path(slug): Path<String>,
|
||||
) -> Result<Json<crate::modules::anime::types::GenreListResponse>, AppError> {
|
||||
info!("Starting request for genre slug: {}", slug);
|
||||
let service = AnimeService::new(AnimeRepository::new());
|
||||
service
|
||||
.get_genre_anime_page(app_state, slug, "1".to_string())
|
||||
.await
|
||||
.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime/genre/{slug}/{page}",
|
||||
tag = "anime",
|
||||
operation_id = "anime_genre_slug_page",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves details for a specific genre by slug and page.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn genre_slug_page(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
Path((slug, page)): Path<(String, String)>,
|
||||
) -> Result<Json<crate::modules::anime::types::GenreListResponse>, AppError> {
|
||||
info!("Starting request for genre slug: {} page: {}", slug, page);
|
||||
let service = AnimeService::new(AnimeRepository::new());
|
||||
service
|
||||
.get_genre_anime_page(app_state, slug, page)
|
||||
.await
|
||||
.map(Json)
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
pub mod controller;
|
||||
pub mod parser;
|
||||
pub mod repository;
|
||||
pub mod route;
|
||||
pub mod schema;
|
||||
pub mod service;
|
||||
pub mod types;
|
||||
@@ -0,0 +1,632 @@
|
||||
use crate::modules::anime::types::*;
|
||||
use crate::shared::errors::AppError;
|
||||
use crate::shared::utils::parse_html;
|
||||
use crate::shared::utils::scraping::{
|
||||
attr, attr_from, attr_from_or, extract_slug, selector, text, text_from_or,
|
||||
};
|
||||
|
||||
pub fn parse_ongoing_anime(html: &str) -> Result<Vec<OngoingAnimeItem>, AppError> {
|
||||
let document = parse_html(html);
|
||||
let mut ongoing_anime = Vec::new();
|
||||
|
||||
let venz_selector = selector(".venz ul li").unwrap();
|
||||
let title_selector = selector(".thumbz h2.jdlflm").unwrap();
|
||||
let link_selector = selector("a").unwrap();
|
||||
let img_selector = selector("img").unwrap();
|
||||
let episode_selector = selector(".epz").unwrap();
|
||||
|
||||
for element in document.select(&venz_selector) {
|
||||
let title = text_from_or(&element, &title_selector, "");
|
||||
let href = attr_from(&element, &link_selector, "href").unwrap_or_default();
|
||||
let slug = extract_slug(&href);
|
||||
let poster = attr_from_or(&element, &img_selector, "src", "");
|
||||
let current_episode = text_from_or(&element, &episode_selector, "N/A");
|
||||
let anime_url = attr_from_or(&element, &link_selector, "href", "");
|
||||
|
||||
if !title.is_empty() {
|
||||
ongoing_anime.push(OngoingAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
current_episode,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(ongoing_anime)
|
||||
}
|
||||
|
||||
pub fn parse_complete_anime(html: &str) -> Result<Vec<CompleteAnimeItem>, AppError> {
|
||||
let document = parse_html(html);
|
||||
let mut complete_anime = Vec::new();
|
||||
|
||||
let venz_selector = selector(".venz ul li").unwrap();
|
||||
let title_selector = selector(".thumbz h2.jdlflm").unwrap();
|
||||
let link_selector = selector("a").unwrap();
|
||||
let img_selector = selector("img").unwrap();
|
||||
let episode_selector = selector(".epz").unwrap();
|
||||
|
||||
for element in document.select(&venz_selector) {
|
||||
let title = text_from_or(&element, &title_selector, "");
|
||||
let href = attr_from(&element, &link_selector, "href").unwrap_or_default();
|
||||
let slug = extract_slug(&href);
|
||||
let poster = attr_from_or(&element, &img_selector, "src", "");
|
||||
let episode_count = text_from_or(&element, &episode_selector, "N/A");
|
||||
let anime_url = attr_from_or(&element, &link_selector, "href", "");
|
||||
|
||||
if !title.is_empty() {
|
||||
complete_anime.push(CompleteAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
episode_count,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(complete_anime)
|
||||
}
|
||||
|
||||
pub fn parse_genres(html: &str) -> Result<Vec<Genre>, AppError> {
|
||||
let document = parse_html(html);
|
||||
let mut genres = Vec::new();
|
||||
let genre_selector = selector(".genres li a, .genre-list a").unwrap();
|
||||
|
||||
for element in document.select(&genre_selector) {
|
||||
let name = text(&element);
|
||||
let url = attr(&element, "href").unwrap_or_default();
|
||||
let slug = extract_slug(&url);
|
||||
|
||||
if !name.is_empty() && !slug.is_empty() {
|
||||
genres.push(Genre { name, slug, url });
|
||||
}
|
||||
}
|
||||
|
||||
Ok(genres)
|
||||
}
|
||||
|
||||
pub fn parse_anime_detail_document(html: &str) -> Result<AnimeDetailData, AppError> {
|
||||
let document = parse_html(html);
|
||||
|
||||
let info_selector = selector(".infozingle p").unwrap();
|
||||
let poster_selector = selector(".fotoanime img").unwrap();
|
||||
let synopsis_selector = selector(".sinopc").unwrap();
|
||||
let genre_link_selector = selector("a").unwrap();
|
||||
let episode_list_selector = selector(".episodelist ul li a").unwrap();
|
||||
let recommendation_selector = selector("#recommend-anime-series .isi-anime").unwrap();
|
||||
let recommendation_title_selector = selector(".judul-anime a").unwrap();
|
||||
let recommendation_img_selector = selector("img").unwrap();
|
||||
|
||||
let mut title = String::new();
|
||||
let mut alternative_title = String::new();
|
||||
let mut r#type: Option<String> = None;
|
||||
let mut status: Option<String> = None;
|
||||
let mut release_date = String::new();
|
||||
let mut studio = String::new();
|
||||
let producers = Vec::new();
|
||||
|
||||
for element in document.select(&info_selector) {
|
||||
let text = text(&element);
|
||||
if text.contains("Judul:") {
|
||||
title = text.replace("Judul:", "").trim().to_string();
|
||||
} else if text.contains("Japanese:") {
|
||||
alternative_title = text.replace("Japanese:", "").trim().to_string();
|
||||
} else if text.contains("Type:") {
|
||||
let type_str = text.replace("Type:", "").trim().to_string();
|
||||
if !type_str.is_empty() {
|
||||
r#type = Some(type_str);
|
||||
}
|
||||
} else if text.contains("Status:") {
|
||||
let status_str = text.replace("Status:", "").trim().to_string();
|
||||
if !status_str.is_empty() {
|
||||
status = Some(status_str);
|
||||
}
|
||||
} else if text.contains("Tanggal Rilis:") {
|
||||
release_date = text.replace("Tanggal Rilis:", "").trim().to_string();
|
||||
} else if text.contains("Studio:") {
|
||||
studio = text.replace("Studio:", "").trim().to_string();
|
||||
}
|
||||
}
|
||||
|
||||
let poster = document
|
||||
.select(&poster_selector)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("src"))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let synopsis = text_from_or(&document.root_element(), &synopsis_selector, "");
|
||||
|
||||
let mut genres = Vec::new();
|
||||
if let Some(genres_element) = document
|
||||
.select(&info_selector)
|
||||
.find(|e| text(&e).contains("Genres:"))
|
||||
{
|
||||
for genre_link in genres_element.select(&genre_link_selector) {
|
||||
let name = text(&genre_link);
|
||||
let anime_url = attr(&genre_link, "href").unwrap_or_default();
|
||||
let genre_slug = extract_slug(&anime_url);
|
||||
genres.push(DetailGenre {
|
||||
name,
|
||||
slug: genre_slug,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let mut episode_lists = Vec::new();
|
||||
for element in document.select(&episode_list_selector) {
|
||||
let episode = text(&element);
|
||||
let href = attr(&element, "href").unwrap_or_default();
|
||||
let slug = extract_slug(&href);
|
||||
episode_lists.push(EpisodeList { episode, slug });
|
||||
}
|
||||
|
||||
let mut recommendations = Vec::new();
|
||||
for element in document.select(&recommendation_selector) {
|
||||
let title = text_from_or(&element, &recommendation_title_selector, "");
|
||||
let poster = attr_from_or(&element, &recommendation_img_selector, "src", "");
|
||||
let href = element
|
||||
.select(&genre_link_selector)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("href"))
|
||||
.unwrap_or("");
|
||||
|
||||
let slug = extract_slug(href);
|
||||
|
||||
recommendations.push(Recommendation {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
status: None,
|
||||
r#type: None,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(AnimeDetailData {
|
||||
title,
|
||||
alternative_title,
|
||||
poster,
|
||||
r#type,
|
||||
status,
|
||||
release_date,
|
||||
studio,
|
||||
genres,
|
||||
synopsis,
|
||||
episode_lists,
|
||||
batch: vec![],
|
||||
producers,
|
||||
recommendations,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn parse_anime_page(
|
||||
html: &str,
|
||||
slug: &str,
|
||||
) -> Result<(Vec<CompleteAnimeListItem>, Pagination), AppError> {
|
||||
let document = parse_html(html);
|
||||
let mut anime_list = Vec::new();
|
||||
|
||||
let item_selector = selector(".venz ul li").unwrap();
|
||||
let title_selector = selector(".thumbz h2.jdlflm").unwrap();
|
||||
let link_selector = selector("a").unwrap();
|
||||
let img_selector = selector("img").unwrap();
|
||||
let episode_selector = selector(".epz").unwrap();
|
||||
let pagination_selector = selector(".pagenavix .page-numbers:not(.next)").unwrap();
|
||||
let next_selector = selector(".pagenavix .next.page-numbers").unwrap();
|
||||
|
||||
for element in document.select(&item_selector) {
|
||||
let title = text_from_or(&element, &title_selector, "");
|
||||
let anime_url = attr_from_or(&element, &link_selector, "href", "");
|
||||
let slug = extract_slug(&anime_url);
|
||||
let poster = attr_from_or(&element, &img_selector, "src", "");
|
||||
let episode_count = text_from_or(&element, &episode_selector, "N/A");
|
||||
|
||||
if !title.is_empty() {
|
||||
anime_list.push(CompleteAnimeListItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
episode_count,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let current_page = slug.parse::<u32>().unwrap_or(1);
|
||||
let last_visible_page = document
|
||||
.select(&pagination_selector)
|
||||
.next_back()
|
||||
.and_then(|e| e.text().collect::<String>().trim().parse::<u32>().ok())
|
||||
.unwrap_or(1);
|
||||
|
||||
let has_next_page = document.select(&next_selector).next().is_some();
|
||||
let next_page = if has_next_page {
|
||||
Some(current_page + 1)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let has_previous_page = current_page > 1;
|
||||
let previous_page = if has_previous_page {
|
||||
Some(current_page - 1)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let pagination = Pagination {
|
||||
current_page,
|
||||
last_visible_page,
|
||||
has_next_page,
|
||||
next_page,
|
||||
has_previous_page,
|
||||
previous_page,
|
||||
};
|
||||
|
||||
Ok((anime_list, pagination))
|
||||
}
|
||||
|
||||
pub fn parse_ongoing_anime_document(
|
||||
html: &str,
|
||||
slug: &str,
|
||||
) -> Result<(Vec<OngoingAnimeListItem>, Pagination), AppError> {
|
||||
let document = parse_html(html);
|
||||
let mut anime_list = Vec::new();
|
||||
|
||||
let item_selector = selector(".venz ul li").unwrap();
|
||||
let title_selector = selector(".thumbz h2.jdlflm").unwrap();
|
||||
let link_selector = selector("a").unwrap();
|
||||
let img_selector = selector("img").unwrap();
|
||||
let score_selector = selector(".epz").unwrap();
|
||||
let pagination_selector = selector(".pagenavix .page-numbers:not(.next)").unwrap();
|
||||
let next_selector = selector(".pagenavix .next.page-numbers").unwrap();
|
||||
|
||||
for element in document.select(&item_selector) {
|
||||
let title = text_from_or(&element, &title_selector, "");
|
||||
let anime_url = attr_from_or(&element, &link_selector, "href", "");
|
||||
let slug = extract_slug(&anime_url);
|
||||
let poster = attr_from_or(&element, &img_selector, "src", "");
|
||||
let score = text_from_or(&element, &score_selector, "N/A");
|
||||
|
||||
if !title.is_empty() {
|
||||
anime_list.push(OngoingAnimeListItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
score,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let current_page = slug.parse::<u32>().unwrap_or(1);
|
||||
let last_visible_page = document
|
||||
.select(&pagination_selector)
|
||||
.next_back()
|
||||
.and_then(|e| e.text().collect::<String>().trim().parse::<u32>().ok())
|
||||
.unwrap_or(1);
|
||||
|
||||
let has_next_page = document.select(&next_selector).next().is_some();
|
||||
let next_page = if has_next_page {
|
||||
Some(current_page + 1)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let has_previous_page = current_page > 1;
|
||||
let previous_page = if has_previous_page {
|
||||
Some(current_page - 1)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let pagination = Pagination {
|
||||
current_page,
|
||||
last_visible_page,
|
||||
has_next_page,
|
||||
next_page,
|
||||
has_previous_page,
|
||||
previous_page,
|
||||
};
|
||||
|
||||
Ok((anime_list, pagination))
|
||||
}
|
||||
|
||||
pub fn parse_latest_anime_document(
|
||||
html: &str,
|
||||
slug: &str,
|
||||
) -> Result<(Vec<LatestAnimeItem>, Pagination), AppError> {
|
||||
let document = parse_html(html);
|
||||
let mut anime_list = Vec::new();
|
||||
|
||||
let item_selector = selector(".venz ul li").unwrap();
|
||||
let title_selector = selector(".thumbz h2.jdlflm").unwrap();
|
||||
let link_selector = selector("a").unwrap();
|
||||
let img_selector = selector("img").unwrap();
|
||||
let episode_selector = selector(".epz").unwrap();
|
||||
let pagination_selector = selector(".pagenavix .page-numbers:not(.next)").unwrap();
|
||||
let next_selector = selector(".pagenavix .next.page-numbers").unwrap();
|
||||
|
||||
for element in document.select(&item_selector) {
|
||||
let title = text_from_or(&element, &title_selector, "");
|
||||
let anime_url = attr_from_or(&element, &link_selector, "href", "");
|
||||
let slug = extract_slug(&anime_url);
|
||||
let poster = attr_from_or(&element, &img_selector, "src", "");
|
||||
let episode = text_from_or(&element, &episode_selector, "N/A");
|
||||
|
||||
if !title.is_empty() {
|
||||
anime_list.push(LatestAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
episode,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let current_page = slug.parse::<u32>().unwrap_or(1);
|
||||
let last_visible_page = document
|
||||
.select(&pagination_selector)
|
||||
.next_back()
|
||||
.and_then(|e| e.text().collect::<String>().trim().parse::<u32>().ok())
|
||||
.unwrap_or(1);
|
||||
|
||||
let has_next_page = document.select(&next_selector).next().is_some();
|
||||
let next_page = if has_next_page {
|
||||
Some(current_page + 1)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let has_previous_page = current_page > 1;
|
||||
let previous_page = if has_previous_page {
|
||||
Some(current_page - 1)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let pagination = Pagination {
|
||||
current_page,
|
||||
last_visible_page,
|
||||
has_next_page,
|
||||
next_page,
|
||||
has_previous_page,
|
||||
previous_page,
|
||||
};
|
||||
|
||||
Ok((anime_list, pagination))
|
||||
}
|
||||
|
||||
pub fn parse_search_anime_document(
|
||||
html: &str,
|
||||
slug: &str,
|
||||
) -> Result<(Vec<SearchAnimeItem>, Pagination), AppError> {
|
||||
let document = parse_html(html);
|
||||
let mut anime_list = Vec::new();
|
||||
|
||||
let item_selector = selector(".venz ul li").unwrap();
|
||||
let title_selector = selector(".thumbz h2.jdlflm").unwrap();
|
||||
let link_selector = selector("a").unwrap();
|
||||
let img_selector = selector("img").unwrap();
|
||||
let episode_selector = selector(".epz").unwrap();
|
||||
let genre_selector = selector(".genre-tag").unwrap();
|
||||
let status_selector = selector(".status").unwrap();
|
||||
let rating_selector = selector(".rating").unwrap();
|
||||
let pagination_selector = selector(".pagenavix .page-numbers:not(.next)").unwrap();
|
||||
let next_selector = selector(".pagenavix .next.page-numbers").unwrap();
|
||||
|
||||
for element in document.select(&item_selector) {
|
||||
let title = text_from_or(&element, &title_selector, "");
|
||||
let anime_url = attr_from_or(&element, &link_selector, "href", "");
|
||||
let slug = extract_slug(&anime_url);
|
||||
let poster = attr_from_or(&element, &img_selector, "src", "");
|
||||
let episode = text_from_or(&element, &episode_selector, "N/A");
|
||||
|
||||
let mut genres = Vec::new();
|
||||
for genre_elem in element.select(&genre_selector) {
|
||||
genres.push(text(&genre_elem));
|
||||
}
|
||||
|
||||
let status = text_from_or(&element, &status_selector, "");
|
||||
let rating = text_from_or(&element, &rating_selector, "");
|
||||
|
||||
if !title.is_empty() {
|
||||
anime_list.push(SearchAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
episode,
|
||||
anime_url,
|
||||
genres,
|
||||
status,
|
||||
rating,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let current_page = slug.parse::<u32>().unwrap_or(1);
|
||||
let last_visible_page = document
|
||||
.select(&pagination_selector)
|
||||
.next_back()
|
||||
.and_then(|e| e.text().collect::<String>().trim().parse::<u32>().ok())
|
||||
.unwrap_or(1);
|
||||
|
||||
let has_next_page = document.select(&next_selector).next().is_some();
|
||||
let next_page = if has_next_page {
|
||||
Some(current_page + 1)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let has_previous_page = current_page > 1;
|
||||
let previous_page = if has_previous_page {
|
||||
Some(current_page - 1)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let pagination = Pagination {
|
||||
current_page,
|
||||
last_visible_page,
|
||||
has_next_page,
|
||||
next_page,
|
||||
has_previous_page,
|
||||
previous_page,
|
||||
};
|
||||
|
||||
Ok((anime_list, pagination))
|
||||
}
|
||||
|
||||
pub fn parse_genre_anime_document(
|
||||
html: &str,
|
||||
slug: &str,
|
||||
) -> Result<(Vec<GenreAnimeItem>, Pagination), AppError> {
|
||||
let document = parse_html(html);
|
||||
let mut anime_list = Vec::new();
|
||||
|
||||
let item_selector = selector(".venz ul li").unwrap();
|
||||
let title_selector = selector(".thumbz h2.jdlflm").unwrap();
|
||||
let link_selector = selector("a").unwrap();
|
||||
let img_selector = selector("img").unwrap();
|
||||
let episode_selector = selector(".epz").unwrap();
|
||||
let pagination_selector = selector(".pagenavix .page-numbers:not(.next)").unwrap();
|
||||
let next_selector = selector(".pagenavix .next.page-numbers").unwrap();
|
||||
|
||||
for element in document.select(&item_selector) {
|
||||
let title = text_from_or(&element, &title_selector, "");
|
||||
let anime_url = attr_from_or(&element, &link_selector, "href", "");
|
||||
let slug = extract_slug(&anime_url);
|
||||
let poster = attr_from_or(&element, &img_selector, "src", "");
|
||||
let episode = text_from_or(&element, &episode_selector, "N/A");
|
||||
|
||||
if !title.is_empty() {
|
||||
anime_list.push(GenreAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
episode,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let current_page = slug.parse::<u32>().unwrap_or(1);
|
||||
let last_visible_page = document
|
||||
.select(&pagination_selector)
|
||||
.next_back()
|
||||
.and_then(|e| e.text().collect::<String>().trim().parse::<u32>().ok())
|
||||
.unwrap_or(1);
|
||||
|
||||
let has_next_page = document.select(&next_selector).next().is_some();
|
||||
let next_page = if has_next_page {
|
||||
Some(current_page + 1)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let has_previous_page = current_page > 1;
|
||||
let previous_page = if has_previous_page {
|
||||
Some(current_page - 1)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let pagination = Pagination {
|
||||
current_page,
|
||||
last_visible_page,
|
||||
has_next_page,
|
||||
next_page,
|
||||
has_previous_page,
|
||||
previous_page,
|
||||
};
|
||||
|
||||
Ok((anime_list, pagination))
|
||||
}
|
||||
|
||||
pub fn parse_anime_full_document(html: &str, slug: &str) -> Result<AnimeFullData, AppError> {
|
||||
let document = parse_html(html);
|
||||
|
||||
let episode_title_selector = selector("h1.posttl").unwrap();
|
||||
let image_selector = selector(".cukder img").unwrap();
|
||||
let stream_selector = selector("#embed_holder iframe").unwrap();
|
||||
let download_item_selector = selector(".download ul li").unwrap();
|
||||
let resolution_selector = selector("strong").unwrap();
|
||||
let link_selector = selector("a").unwrap();
|
||||
let next_episode_selector = selector(".flir a[title*='Episode Selanjutnya']").unwrap();
|
||||
let previous_episode_selector = selector(".flir a[title*='Episode Sebelumnya']").unwrap();
|
||||
|
||||
let episode = document
|
||||
.select(&episode_title_selector)
|
||||
.next()
|
||||
.map(|e| text(&e))
|
||||
.unwrap_or_default();
|
||||
|
||||
let episode_number = episode
|
||||
.split("Episode")
|
||||
.nth(1)
|
||||
.map(|s| s.trim().to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let image_url = document
|
||||
.select(&image_selector)
|
||||
.next()
|
||||
.and_then(|e| attr(&e, "src"))
|
||||
.unwrap_or_default();
|
||||
|
||||
let stream_url = document
|
||||
.select(&stream_selector)
|
||||
.next()
|
||||
.and_then(|e| attr(&e, "src"))
|
||||
.unwrap_or_default();
|
||||
|
||||
let mut download_urls = std::collections::HashMap::new();
|
||||
|
||||
for element in document.select(&download_item_selector) {
|
||||
let resolution = element
|
||||
.select(&resolution_selector)
|
||||
.next()
|
||||
.map(|e| text(&e))
|
||||
.unwrap_or_default();
|
||||
|
||||
let mut links = Vec::new();
|
||||
for link_element in element.select(&link_selector) {
|
||||
let server = text(&link_element);
|
||||
let url = attr(&link_element, "href").unwrap_or_default();
|
||||
links.push(DownloadLink { server, url });
|
||||
}
|
||||
|
||||
if !resolution.is_empty() && !links.is_empty() {
|
||||
download_urls.insert(resolution, links);
|
||||
}
|
||||
}
|
||||
|
||||
let next_episode_element = document.select(&next_episode_selector).next();
|
||||
let previous_episode_element = document.select(&previous_episode_selector).next();
|
||||
|
||||
let next_episode_slug = next_episode_element
|
||||
.and_then(|e| attr(&e, "href"))
|
||||
.and_then(|href| {
|
||||
href.split('/')
|
||||
.nth(href.split('/').count().saturating_sub(2))
|
||||
.map(|s| s.to_string() + "/")
|
||||
});
|
||||
|
||||
let previous_episode_slug = previous_episode_element
|
||||
.and_then(|e| attr(&e, "href"))
|
||||
.and_then(|href| {
|
||||
href.split('/')
|
||||
.nth(href.split('/').count().saturating_sub(2))
|
||||
.map(|s| s.to_string() + "/")
|
||||
});
|
||||
|
||||
Ok(AnimeFullData {
|
||||
episode,
|
||||
episode_number,
|
||||
anime: AnimeInfo {
|
||||
slug: slug.to_string(),
|
||||
},
|
||||
has_next_episode: next_episode_slug.is_some(),
|
||||
next_episode: next_episode_slug.map(|s| EpisodeInfo { slug: s }),
|
||||
has_previous_episode: previous_episode_slug.is_some(),
|
||||
previous_episode: previous_episode_slug.map(|s| EpisodeInfo { slug: s }),
|
||||
stream_url,
|
||||
download_urls,
|
||||
image_url,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
use crate::modules::anime::parser;
|
||||
use crate::modules::anime::types::*;
|
||||
use crate::shared::database::traits::scraping_repository::ScrapingRepository;
|
||||
use crate::shared::errors::AppError;
|
||||
use crate::shared::utils::web::proxy_fetch::fetch_with_proxy;
|
||||
use crate::shared::utils::web::scraping_urls::{get_otakudesu_url, OTAKUDESU_BASE_URL};
|
||||
use crate::shared::utils::{default_backoff, fetch_html_with_retry, transient};
|
||||
use async_trait::async_trait;
|
||||
use backoff::future::retry;
|
||||
use tracing::{info, warn};
|
||||
|
||||
pub struct AnimeRepository;
|
||||
|
||||
impl Default for AnimeRepository {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl AnimeRepository {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ScrapingRepository for AnimeRepository {
|
||||
async fn fetch_html(&self, url: &str) -> Result<String, AppError> {
|
||||
fetch_html_with_retry(url).await
|
||||
}
|
||||
}
|
||||
|
||||
impl AnimeRepository {
|
||||
pub fn base_url(&self) -> String {
|
||||
get_otakudesu_url()
|
||||
}
|
||||
|
||||
pub fn index_urls(&self) -> (String, String) {
|
||||
let base = self.base_url();
|
||||
(
|
||||
format!("{}/ongoing-anime/", base),
|
||||
format!("{}/complete-anime/", base),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn genres_url(&self) -> String {
|
||||
format!("{}/genre-list/", self.base_url())
|
||||
}
|
||||
|
||||
pub fn detail_url(&self, slug: &str) -> String {
|
||||
format!("{}/anime/{}", OTAKUDESU_BASE_URL, slug)
|
||||
}
|
||||
|
||||
pub fn page_url(&self, category: &str, page: &str) -> String {
|
||||
format!("{}/{}/page/{}/", OTAKUDESU_BASE_URL, category, page)
|
||||
}
|
||||
|
||||
pub fn search_url(&self, query: &str, page: &str) -> String {
|
||||
if page == "1" {
|
||||
format!("{}/search/{}/", self.base_url(), query)
|
||||
} else {
|
||||
format!("{}/search/{}/page/{}/", self.base_url(), query, page)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn genre_page_url(&self, genre_slug: &str, page: &str) -> String {
|
||||
format!("{}/genre/{}/page/{}/", self.base_url(), genre_slug, page)
|
||||
}
|
||||
|
||||
pub fn full_episode_url(&self, slug: &str) -> String {
|
||||
format!("{}/episode/{}", OTAKUDESU_BASE_URL, slug)
|
||||
}
|
||||
|
||||
pub async fn fetch_anime_index(&self) -> Result<AnimeData, AppError> {
|
||||
let (ongoing_url, complete_url) = self.index_urls();
|
||||
|
||||
let (ongoing_html, complete_html) = tokio::join!(
|
||||
self.fetch_html(&ongoing_url),
|
||||
self.fetch_html(&complete_url)
|
||||
);
|
||||
|
||||
let ongoing_html = ongoing_html?;
|
||||
let complete_html = complete_html?;
|
||||
|
||||
let ongoing_anime =
|
||||
tokio::task::spawn_blocking(move || parser::parse_ongoing_anime(&ongoing_html))
|
||||
.await??;
|
||||
let complete_anime =
|
||||
tokio::task::spawn_blocking(move || parser::parse_complete_anime(&complete_html))
|
||||
.await??;
|
||||
|
||||
Ok(AnimeData {
|
||||
ongoing_anime,
|
||||
complete_anime,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn fetch_genres(&self) -> Result<Vec<Genre>, AppError> {
|
||||
let html = self.fetch_html(&self.genres_url()).await?;
|
||||
tokio::task::spawn_blocking(move || parser::parse_genres(&html)).await?
|
||||
}
|
||||
|
||||
pub async fn fetch_anime_detail(&self, slug: &str) -> Result<AnimeDetailData, AppError> {
|
||||
let url = self.detail_url(slug);
|
||||
let html = self
|
||||
.fetch_with_proxy_retry(&url)
|
||||
.await
|
||||
.map_err(|e| AppError::ScraperError(e.to_string()))?;
|
||||
|
||||
tokio::task::spawn_blocking(move || parser::parse_anime_detail_document(&html)).await?
|
||||
}
|
||||
|
||||
pub async fn fetch_complete_anime_page(
|
||||
&self,
|
||||
slug: &str,
|
||||
) -> Result<(Vec<CompleteAnimeListItem>, Pagination), AppError> {
|
||||
let url = self.page_url("complete-anime", slug);
|
||||
let html = self.fetch_html(&url).await?;
|
||||
let slug_owned = slug.to_string();
|
||||
tokio::task::spawn_blocking(move || parser::parse_anime_page(&html, &slug_owned)).await?
|
||||
}
|
||||
|
||||
pub async fn fetch_ongoing_anime_page(
|
||||
&self,
|
||||
slug: &str,
|
||||
) -> Result<(Vec<OngoingAnimeListItem>, Pagination), AppError> {
|
||||
let url = self.page_url("ongoing-anime", slug);
|
||||
let html = self.fetch_html(&url).await?;
|
||||
let slug_owned = slug.to_string();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
parser::parse_ongoing_anime_document(&html, &slug_owned)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
pub async fn fetch_latest_anime_page(
|
||||
&self,
|
||||
slug: &str,
|
||||
) -> Result<(Vec<LatestAnimeItem>, Pagination), AppError> {
|
||||
let url = self.page_url("latest-anime", slug);
|
||||
let html = self.fetch_html(&url).await?;
|
||||
let slug_owned = slug.to_string();
|
||||
tokio::task::spawn_blocking(move || parser::parse_latest_anime_document(&html, &slug_owned))
|
||||
.await?
|
||||
}
|
||||
|
||||
pub async fn fetch_search_anime_page(
|
||||
&self,
|
||||
slug: &str,
|
||||
page: &str,
|
||||
) -> Result<(Vec<SearchAnimeItem>, Pagination), AppError> {
|
||||
let url = self.search_url(slug, page);
|
||||
let html = self.fetch_html(&url).await?;
|
||||
let page_owned = page.to_string();
|
||||
tokio::task::spawn_blocking(move || parser::parse_search_anime_document(&html, &page_owned))
|
||||
.await?
|
||||
}
|
||||
|
||||
pub async fn fetch_genre_anime_page(
|
||||
&self,
|
||||
genre_slug: &str,
|
||||
page: &str,
|
||||
) -> Result<(Vec<GenreAnimeItem>, Pagination), AppError> {
|
||||
let url = self.genre_page_url(genre_slug, page);
|
||||
let html = self.fetch_html(&url).await?;
|
||||
let page_owned = page.to_string();
|
||||
tokio::task::spawn_blocking(move || parser::parse_genre_anime_document(&html, &page_owned))
|
||||
.await?
|
||||
}
|
||||
|
||||
pub async fn fetch_anime_full(&self, slug: &str) -> Result<AnimeFullData, AppError> {
|
||||
let url = self.full_episode_url(slug);
|
||||
let html = self.fetch_html(&url).await?;
|
||||
let slug_owned = slug.to_string();
|
||||
tokio::task::spawn_blocking(move || parser::parse_anime_full_document(&html, &slug_owned))
|
||||
.await?
|
||||
}
|
||||
|
||||
async fn fetch_with_proxy_retry(&self, url: &str) -> Result<String, AppError> {
|
||||
let backoff = default_backoff();
|
||||
let url_owned = url.to_string();
|
||||
let fetch_op = || async {
|
||||
info!("Fetching URL: {}", url_owned);
|
||||
match fetch_with_proxy(&url_owned).await {
|
||||
Ok(response) => {
|
||||
info!("Successfully fetched URL: {}", url_owned);
|
||||
Ok(response.data)
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Failed to fetch URL: {}, error: {:?}", url_owned, e);
|
||||
Err(transient(e))
|
||||
}
|
||||
}
|
||||
};
|
||||
retry(backoff, fetch_op)
|
||||
.await
|
||||
.map_err(|e| AppError::ScraperError(e.to_string()))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
use crate::modules::anime::controller;
|
||||
use crate::shared::state::AppState;
|
||||
use axum::Router;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub fn routes(router: Router<Arc<AppState>>) -> Router<Arc<AppState>> {
|
||||
router
|
||||
.route("/api/anime", axum::routing::get(controller::anime_index))
|
||||
.route(
|
||||
"/api/anime/genre_list",
|
||||
axum::routing::get(controller::genres),
|
||||
)
|
||||
.route(
|
||||
"/api/anime/detail/{slug}",
|
||||
axum::routing::get(controller::detail_slug),
|
||||
)
|
||||
.route(
|
||||
"/api/anime/complete_anime/{slug}",
|
||||
axum::routing::get(controller::complete_anime_slug),
|
||||
)
|
||||
.route(
|
||||
"/api/anime/full/{slug}",
|
||||
axum::routing::get(controller::full_slug),
|
||||
)
|
||||
.route(
|
||||
"/api/anime/ongoing_anime/{slug}",
|
||||
axum::routing::get(controller::ongoing_anime_slug),
|
||||
)
|
||||
.route(
|
||||
"/api/anime/latest/{slug}",
|
||||
axum::routing::get(controller::latest_slug),
|
||||
)
|
||||
.route(
|
||||
"/api/anime/search/{slug}",
|
||||
axum::routing::get(controller::search_slug_index),
|
||||
)
|
||||
.route(
|
||||
"/api/anime/search/{slug}/{page}",
|
||||
axum::routing::get(controller::search_slug_page),
|
||||
)
|
||||
.route(
|
||||
"/api/anime/genre/{slug}",
|
||||
axum::routing::get(controller::genre_slug_index),
|
||||
)
|
||||
.route(
|
||||
"/api/anime/genre/{slug}/{page}",
|
||||
axum::routing::get(controller::genre_slug_page),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
use serde::Deserialize;
|
||||
use utoipa::ToSchema;
|
||||
|
||||
#[derive(Deserialize, ToSchema)]
|
||||
pub struct SearchQuery {
|
||||
pub q: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, ToSchema)]
|
||||
pub struct SlugPath {
|
||||
pub slug: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, ToSchema)]
|
||||
pub struct SlugPagePath {
|
||||
pub slug: String,
|
||||
pub page: String,
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
use crate::shared::types::entities::anime::HasPoster;
|
||||
use crate::shared::state::AppState;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Cache poster URLs for a collection of anime items
|
||||
/// This is a fire-and-forget operation that triggers lazy background caching
|
||||
pub async fn cache_posters<T: HasPoster>(app_state: &Arc<AppState>, items: &[T]) {
|
||||
let posters: Vec<String> = items.iter().map(|item| item.poster().to_string()).collect();
|
||||
|
||||
if posters.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let db = app_state.db.clone();
|
||||
let redis = app_state.redis_pool.clone();
|
||||
|
||||
crate::shared::services::images::cache::cache_image_urls_batch_lazy(
|
||||
db,
|
||||
&redis,
|
||||
posters,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Cache poster URLs and update items with cached URLs
|
||||
/// Returns the updated items with CDN URLs
|
||||
pub async fn cache_and_update_posters<T: HasPoster + Clone>(
|
||||
app_state: &Arc<AppState>,
|
||||
mut items: Vec<T>,
|
||||
) -> Vec<T> {
|
||||
let posters: Vec<String> = items.iter().map(|item| item.poster().to_string()).collect();
|
||||
|
||||
if posters.is_empty() {
|
||||
return items;
|
||||
}
|
||||
|
||||
let db = app_state.db.clone();
|
||||
let redis = app_state.redis_pool.clone();
|
||||
|
||||
let cached_posters = crate::shared::services::images::cache::cache_image_urls_batch_lazy(
|
||||
db,
|
||||
&redis,
|
||||
posters,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
// Update items with cached URLs
|
||||
for (i, item) in items.iter_mut().enumerate() {
|
||||
if let Some(url) = cached_posters.get(i) {
|
||||
item.set_poster(url.clone());
|
||||
}
|
||||
}
|
||||
|
||||
items
|
||||
}
|
||||
|
||||
/// Cache multiple collections of posters and update them
|
||||
/// Useful when you have different types of items (e.g., ongoing and complete anime)
|
||||
pub async fn cache_multiple_collections(
|
||||
app_state: &Arc<AppState>,
|
||||
collections: Vec<Vec<String>>,
|
||||
) -> Vec<String> {
|
||||
let all_posters: Vec<String> = collections.into_iter().flatten().collect();
|
||||
|
||||
if all_posters.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let db = app_state.db.clone();
|
||||
let redis = app_state.redis_pool.clone();
|
||||
|
||||
crate::shared::services::images::cache::cache_image_urls_batch_lazy(
|
||||
db,
|
||||
&redis,
|
||||
all_posters,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::modules::anime::repository::AnimeRepository;
|
||||
use crate::modules::anime::types::*;
|
||||
use crate::shared::errors::AppError;
|
||||
use crate::shared::state::AppState;
|
||||
use crate::shared::utils::Cache;
|
||||
|
||||
const INDEX_CACHE_TTL: u64 = 10;
|
||||
const GENRE_LIST_CACHE_TTL: u64 = 3600;
|
||||
const DEFAULT_CACHE_TTL: u64 = 300;
|
||||
|
||||
pub struct AnimeService {
|
||||
repository: AnimeRepository,
|
||||
}
|
||||
|
||||
impl AnimeService {
|
||||
pub fn new(repository: AnimeRepository) -> Self {
|
||||
Self { repository }
|
||||
}
|
||||
|
||||
pub async fn get_anime_index(&self, app_state: Arc<AppState>) -> Result<AnimeData, AppError> {
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
|
||||
cache
|
||||
.get_or_set("anime:index:v2", INDEX_CACHE_TTL, || async {
|
||||
let mut data = self
|
||||
.repository
|
||||
.fetch_anime_index()
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
if data.ongoing_anime.is_empty() && data.complete_anime.is_empty() {
|
||||
return Err("Empty anime index — refusing to cache".to_string());
|
||||
}
|
||||
|
||||
let mut posters: Vec<String> = data
|
||||
.ongoing_anime
|
||||
.iter()
|
||||
.map(|item| item.poster.clone())
|
||||
.collect();
|
||||
posters.extend(data.complete_anime.iter().map(|item| item.poster.clone()));
|
||||
|
||||
let cached_posters =
|
||||
crate::shared::services::images::cache::cache_image_urls_batch_lazy(
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
posters,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
let ongoing_len = data.ongoing_anime.len();
|
||||
for (i, item) in data.ongoing_anime.iter_mut().enumerate() {
|
||||
if let Some(url) = cached_posters.get(i) {
|
||||
item.poster = url.clone();
|
||||
}
|
||||
}
|
||||
for (i, item) in data.complete_anime.iter_mut().enumerate() {
|
||||
if let Some(url) = cached_posters.get(ongoing_len + i) {
|
||||
item.poster = url.clone();
|
||||
}
|
||||
}
|
||||
|
||||
Ok(data)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| AppError::ScraperError(e))
|
||||
}
|
||||
|
||||
pub async fn get_genres(&self, app_state: Arc<AppState>) -> Result<GenresResponse, AppError> {
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
|
||||
cache
|
||||
.get_or_set("anime:genres:list", GENRE_LIST_CACHE_TTL, || async {
|
||||
let genres = self
|
||||
.repository
|
||||
.fetch_genres()
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(GenresResponse {
|
||||
status: "Ok".to_string(),
|
||||
data: genres,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|e| AppError::ScraperError(e))
|
||||
}
|
||||
|
||||
pub async fn get_anime_detail(
|
||||
&self,
|
||||
app_state: Arc<AppState>,
|
||||
slug: String,
|
||||
) -> Result<DetailResponse, AppError> {
|
||||
let cache_key = format!("anime:detail:{}", slug);
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async {
|
||||
let mut data = self
|
||||
.repository
|
||||
.fetch_anime_detail(&slug)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
data.poster = crate::shared::services::images::cache::get_cached_or_original(
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
&data.poster,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
let rec_posters: Vec<String> = data
|
||||
.recommendations
|
||||
.iter()
|
||||
.map(|r| r.poster.clone())
|
||||
.collect();
|
||||
let cached_rec_posters =
|
||||
crate::shared::services::images::cache::cache_image_urls_batch_lazy(
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
rec_posters,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
for (i, rec) in data.recommendations.iter_mut().enumerate() {
|
||||
if let Some(url) = cached_rec_posters.get(i) {
|
||||
rec.poster = url.clone();
|
||||
}
|
||||
}
|
||||
|
||||
Ok(DetailResponse {
|
||||
status: Some("Ok".to_string()),
|
||||
data,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|e| AppError::ScraperError(e))
|
||||
}
|
||||
|
||||
pub async fn get_complete_anime_page(
|
||||
&self,
|
||||
app_state: Arc<AppState>,
|
||||
slug: String,
|
||||
) -> Result<ListResponse, AppError> {
|
||||
let cache_key = format!("anime:complete:{}", slug);
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async {
|
||||
let (anime_list, pagination) = self
|
||||
.repository
|
||||
.fetch_complete_anime_page(&slug)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let total = anime_list.len() as i64;
|
||||
Ok(ListResponse {
|
||||
message: "Success".to_string(),
|
||||
data: anime_list,
|
||||
total: Some(total),
|
||||
pagination: Some(pagination),
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|e| AppError::ScraperError(e))
|
||||
}
|
||||
|
||||
pub async fn get_ongoing_anime_page(
|
||||
&self,
|
||||
app_state: Arc<AppState>,
|
||||
slug: String,
|
||||
) -> Result<OngoingAnimeResponse, AppError> {
|
||||
let cache_key = format!("anime:ongoing:{}", slug);
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async {
|
||||
let (anime_list, pagination) = self
|
||||
.repository
|
||||
.fetch_ongoing_anime_page(&slug)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(OngoingAnimeResponse {
|
||||
status: "Ok".to_string(),
|
||||
data: anime_list,
|
||||
pagination,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|e| AppError::ScraperError(e))
|
||||
}
|
||||
|
||||
pub async fn get_latest_anime_page(
|
||||
&self,
|
||||
app_state: Arc<AppState>,
|
||||
slug: String,
|
||||
) -> Result<LatestAnimeResponse, AppError> {
|
||||
let cache_key = format!("anime:latest:{}", slug);
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async {
|
||||
let (anime_list, pagination) = self
|
||||
.repository
|
||||
.fetch_latest_anime_page(&slug)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(LatestAnimeResponse {
|
||||
status: "Ok".to_string(),
|
||||
data: anime_list,
|
||||
pagination,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|e| AppError::ScraperError(e))
|
||||
}
|
||||
|
||||
pub async fn get_search_anime_page(
|
||||
&self,
|
||||
app_state: Arc<AppState>,
|
||||
slug: String,
|
||||
page: String,
|
||||
) -> Result<SearchResponse, AppError> {
|
||||
let cache_key = format!("anime:search:{}:{}", slug, page);
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async {
|
||||
let (anime_list, pagination) = self
|
||||
.repository
|
||||
.fetch_search_anime_page(&slug, &page)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(SearchResponse {
|
||||
status: "Ok".to_string(),
|
||||
data: anime_list,
|
||||
pagination,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|e| AppError::ScraperError(e))
|
||||
}
|
||||
|
||||
pub async fn get_genre_anime_page(
|
||||
&self,
|
||||
app_state: Arc<AppState>,
|
||||
genre_slug: String,
|
||||
page: String,
|
||||
) -> Result<GenreListResponse, AppError> {
|
||||
let cache_key = format!("anime:genre:{}:{}", genre_slug, page);
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async {
|
||||
let (anime_list, pagination) = self
|
||||
.repository
|
||||
.fetch_genre_anime_page(&genre_slug, &page)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(GenreListResponse {
|
||||
status: "Ok".to_string(),
|
||||
data: anime_list,
|
||||
pagination,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|e| AppError::ScraperError(e))
|
||||
}
|
||||
|
||||
pub async fn get_anime_full(
|
||||
&self,
|
||||
app_state: Arc<AppState>,
|
||||
slug: String,
|
||||
) -> Result<FullResponse, AppError> {
|
||||
let cache_key = format!("anime:full:{}", slug);
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async {
|
||||
let data = self
|
||||
.repository
|
||||
.fetch_anime_full(&slug)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(FullResponse {
|
||||
status: "Ok".to_string(),
|
||||
data,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|e| AppError::ScraperError(e))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
// Index endpoint types
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct OngoingAnimeItem {
|
||||
pub title: String,
|
||||
pub slug: String,
|
||||
pub poster: String,
|
||||
pub current_episode: String,
|
||||
pub anime_url: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct CompleteAnimeItem {
|
||||
pub title: String,
|
||||
pub slug: String,
|
||||
pub poster: String,
|
||||
pub episode_count: String,
|
||||
pub anime_url: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct AnimeData {
|
||||
pub ongoing_anime: Vec<OngoingAnimeItem>,
|
||||
pub complete_anime: Vec<CompleteAnimeItem>,
|
||||
}
|
||||
|
||||
// Genre list types
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct Genre {
|
||||
pub name: String,
|
||||
pub slug: String,
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct GenresResponse {
|
||||
pub status: String,
|
||||
pub data: Vec<Genre>,
|
||||
}
|
||||
|
||||
// Detail endpoint types
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct DetailGenre {
|
||||
pub name: String,
|
||||
pub slug: String,
|
||||
pub anime_url: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct EpisodeList {
|
||||
pub episode: String,
|
||||
pub slug: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct Recommendation {
|
||||
pub title: String,
|
||||
pub slug: String,
|
||||
pub poster: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub status: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub r#type: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct AnimeDetailData {
|
||||
pub title: String,
|
||||
pub alternative_title: String,
|
||||
pub poster: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub r#type: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub status: Option<String>,
|
||||
pub release_date: String,
|
||||
pub studio: String,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
pub genres: Vec<DetailGenre>,
|
||||
pub synopsis: String,
|
||||
pub episode_lists: Vec<EpisodeList>,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
pub batch: Vec<EpisodeList>,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
pub producers: Vec<String>,
|
||||
pub recommendations: Vec<Recommendation>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct DetailResponse {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub status: Option<String>,
|
||||
pub data: AnimeDetailData,
|
||||
}
|
||||
|
||||
// Complete anime list types
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct CompleteAnimeListItem {
|
||||
pub title: String,
|
||||
pub slug: String,
|
||||
pub poster: String,
|
||||
pub episode_count: String,
|
||||
pub anime_url: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct Pagination {
|
||||
pub current_page: u32,
|
||||
pub last_visible_page: u32,
|
||||
pub has_next_page: bool,
|
||||
pub next_page: Option<u32>,
|
||||
pub has_previous_page: bool,
|
||||
pub previous_page: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct ListResponse {
|
||||
pub message: String,
|
||||
pub data: Vec<CompleteAnimeListItem>,
|
||||
pub total: Option<i64>,
|
||||
pub pagination: Option<Pagination>,
|
||||
}
|
||||
|
||||
// Full episode types
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct AnimeInfo {
|
||||
pub slug: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct EpisodeInfo {
|
||||
pub slug: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct DownloadLink {
|
||||
pub server: String,
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct AnimeFullData {
|
||||
pub episode: String,
|
||||
pub episode_number: String,
|
||||
pub anime: AnimeInfo,
|
||||
pub has_next_episode: bool,
|
||||
pub next_episode: Option<EpisodeInfo>,
|
||||
pub has_previous_episode: bool,
|
||||
pub previous_episode: Option<EpisodeInfo>,
|
||||
pub stream_url: String,
|
||||
pub download_urls: std::collections::HashMap<String, Vec<DownloadLink>>,
|
||||
pub image_url: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct FullResponse {
|
||||
pub status: String,
|
||||
pub data: AnimeFullData,
|
||||
}
|
||||
|
||||
// Ongoing anime list types
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct OngoingAnimeListItem {
|
||||
pub title: String,
|
||||
pub slug: String,
|
||||
pub poster: String,
|
||||
pub score: String,
|
||||
pub anime_url: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct OngoingAnimeResponse {
|
||||
pub status: String,
|
||||
pub data: Vec<OngoingAnimeListItem>,
|
||||
pub pagination: Pagination,
|
||||
}
|
||||
|
||||
// Latest anime types
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct LatestAnimeItem {
|
||||
pub title: String,
|
||||
pub slug: String,
|
||||
pub poster: String,
|
||||
pub episode: String,
|
||||
pub anime_url: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct LatestAnimeResponse {
|
||||
pub status: String,
|
||||
pub data: Vec<LatestAnimeItem>,
|
||||
pub pagination: Pagination,
|
||||
}
|
||||
|
||||
// Search types
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct SearchAnimeItem {
|
||||
pub title: String,
|
||||
pub slug: String,
|
||||
pub poster: String,
|
||||
pub episode: String,
|
||||
pub anime_url: String,
|
||||
pub genres: Vec<String>,
|
||||
pub status: String,
|
||||
pub rating: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct SearchResponse {
|
||||
pub status: String,
|
||||
pub data: Vec<SearchAnimeItem>,
|
||||
pub pagination: Pagination,
|
||||
}
|
||||
|
||||
// Genre list by slug types
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct GenreAnimeItem {
|
||||
pub title: String,
|
||||
pub slug: String,
|
||||
pub poster: String,
|
||||
pub episode: String,
|
||||
pub anime_url: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct GenreListResponse {
|
||||
pub status: String,
|
||||
pub data: Vec<GenreAnimeItem>,
|
||||
pub pagination: Pagination,
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::{
|
||||
extract::{Path, Query, State},
|
||||
Json,
|
||||
};
|
||||
|
||||
use crate::modules::anime2::repository::Anime2Repository;
|
||||
use crate::modules::anime2::schema::FilterQuery;
|
||||
use crate::modules::anime2::service::Anime2Service;
|
||||
use crate::shared::errors::AppError;
|
||||
use crate::shared::state::AppState;
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime2",
|
||||
tag = "anime2",
|
||||
operation_id = "anime2_index",
|
||||
responses(
|
||||
(status = 200, description = "Handles GET requests for the /api/anime2 endpoint.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn index(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
) -> Result<Json<crate::modules::anime2::types::Anime2Response>, AppError> {
|
||||
let service = Anime2Service::new(Anime2Repository::new());
|
||||
Ok(Json(service.index(app_state).await?))
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime2/genre_list",
|
||||
tag = "anime2",
|
||||
operation_id = "anime2_genre_list",
|
||||
responses(
|
||||
(status = 200, description = "Handles GET requests for the /api/anime2/genre_list endpoint.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn genre_list(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
) -> Result<Json<crate::modules::anime2::types::GenresResponse>, AppError> {
|
||||
let service = Anime2Service::new(Anime2Repository::new());
|
||||
Ok(Json(service.genre_list(app_state).await?))
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime2/filter",
|
||||
tag = "anime2",
|
||||
operation_id = "anime2_filter",
|
||||
responses(
|
||||
(status = 200, description = "Handles GET requests for the /api/anime2/filter endpoint.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn filter(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
Query(params): Query<FilterQuery>,
|
||||
) -> Result<Json<crate::modules::anime2::types::FilterResponse>, AppError> {
|
||||
let page = params.page.unwrap_or(1);
|
||||
let genre = params.genre.clone();
|
||||
let status = params.status.clone();
|
||||
let anime_type = params.r#type.clone();
|
||||
let order = params.order.clone().unwrap_or("update".to_string());
|
||||
|
||||
let service = Anime2Service::new(Anime2Repository::new());
|
||||
Ok(Json(
|
||||
service
|
||||
.filter(app_state, page, genre, status, anime_type, order)
|
||||
.await?,
|
||||
))
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime2/detail/{slug}",
|
||||
tag = "anime2",
|
||||
operation_id = "anime2_detail_slug",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves details for a specific detail by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn detail_slug(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
Path(slug): Path<String>,
|
||||
) -> Result<Json<crate::modules::anime2::types::DetailResponse>, AppError> {
|
||||
let service = Anime2Service::new(Anime2Repository::new());
|
||||
Ok(Json(service.detail(app_state, slug).await?))
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime2/genre/{slug}",
|
||||
tag = "anime2",
|
||||
operation_id = "anime2_genre_slug_index",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves details for a specific genre by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn genre_slug_index(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
Path(slug): Path<String>,
|
||||
) -> Result<
|
||||
Json<
|
||||
crate::shared::types::ApiResponse<
|
||||
Vec<crate::shared::types::entities::anime::GenreAnimeItem>,
|
||||
>,
|
||||
>,
|
||||
AppError,
|
||||
> {
|
||||
let service = Anime2Service::new(Anime2Repository::new());
|
||||
Ok(Json(service.genre_slug(app_state, slug, 1).await?))
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime2/genre/{slug}/{page}",
|
||||
tag = "anime2",
|
||||
operation_id = "anime2_genre_slug_page",
|
||||
responses(
|
||||
(status = 200, description = "Handles GET requests for the /api/anime2/genre/{slug}/{page} endpoint.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn genre_slug_page(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
Path((slug, page)): Path<(String, u32)>,
|
||||
) -> Result<
|
||||
Json<
|
||||
crate::shared::types::ApiResponse<
|
||||
Vec<crate::shared::types::entities::anime::GenreAnimeItem>,
|
||||
>,
|
||||
>,
|
||||
AppError,
|
||||
> {
|
||||
let service = Anime2Service::new(Anime2Repository::new());
|
||||
Ok(Json(service.genre_slug(app_state, slug, page).await?))
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime2/search/{slug}",
|
||||
tag = "anime2",
|
||||
operation_id = "anime2_search_slug_index",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves details for a specific search by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn search_slug_index(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
Path(slug): Path<String>,
|
||||
) -> Result<
|
||||
Json<
|
||||
crate::shared::types::ApiResponse<
|
||||
Vec<crate::shared::types::entities::anime::SearchAnimeItem>,
|
||||
>,
|
||||
>,
|
||||
AppError,
|
||||
> {
|
||||
let service = Anime2Service::new(Anime2Repository::new());
|
||||
Ok(Json(service.search(app_state, slug, 1).await?))
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime2/search/{slug}/{page}",
|
||||
tag = "anime2",
|
||||
operation_id = "anime2_search_slug_page",
|
||||
responses(
|
||||
(status = 200, description = "Handles GET requests for the /api/anime2/search/{slug}/{page} endpoint.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn search_slug_page(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
Path((slug, page)): Path<(String, u32)>,
|
||||
) -> Result<
|
||||
Json<
|
||||
crate::shared::types::ApiResponse<
|
||||
Vec<crate::shared::types::entities::anime::SearchAnimeItem>,
|
||||
>,
|
||||
>,
|
||||
AppError,
|
||||
> {
|
||||
let service = Anime2Service::new(Anime2Repository::new());
|
||||
Ok(Json(service.search(app_state, slug, page).await?))
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime2/latest/{slug}",
|
||||
tag = "anime2",
|
||||
operation_id = "anime2_latest_slug",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves details for a specific latest by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn latest_slug(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
Path(slug): Path<String>,
|
||||
) -> Result<
|
||||
Json<
|
||||
crate::shared::types::ApiResponse<
|
||||
Vec<crate::shared::types::entities::anime::LatestAnimeItem>,
|
||||
>,
|
||||
>,
|
||||
AppError,
|
||||
> {
|
||||
let page = slug
|
||||
.parse::<u32>()
|
||||
.map_err(|_| AppError::ScraperError(format!("Invalid page number: {}", slug)))?;
|
||||
let service = Anime2Service::new(Anime2Repository::new());
|
||||
Ok(Json(service.latest(app_state, page).await?))
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime2/ongoing_anime/{slug}",
|
||||
tag = "anime2",
|
||||
operation_id = "anime2_ongoing_anime_slug",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves details for a specific ongoing_anime by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn ongoing_anime_slug(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
Path(slug): Path<String>,
|
||||
) -> Result<
|
||||
Json<
|
||||
crate::shared::types::ApiResponse<
|
||||
Vec<crate::shared::types::entities::anime::OngoingAnimeItemWithScore>,
|
||||
>,
|
||||
>,
|
||||
AppError,
|
||||
> {
|
||||
let page = slug
|
||||
.parse::<u32>()
|
||||
.map_err(|_| AppError::ScraperError(format!("Invalid page number: {}", slug)))?;
|
||||
let service = Anime2Service::new(Anime2Repository::new());
|
||||
Ok(Json(service.ongoing_anime(app_state, page).await?))
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/anime2/complete_anime/{slug}",
|
||||
tag = "anime2",
|
||||
operation_id = "anime2_complete_anime_slug",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves details for a specific complete_anime by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn complete_anime_slug(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
Path(slug): Path<String>,
|
||||
) -> Result<
|
||||
Json<
|
||||
crate::shared::types::ApiResponse<
|
||||
Vec<crate::shared::types::entities::anime::CompleteAnimeItem>,
|
||||
>,
|
||||
>,
|
||||
AppError,
|
||||
> {
|
||||
let page = slug
|
||||
.parse::<u32>()
|
||||
.map_err(|_| AppError::ScraperError(format!("Invalid page number: {}", slug)))?;
|
||||
let service = Anime2Service::new(Anime2Repository::new());
|
||||
Ok(Json(service.complete_anime(app_state, page).await?))
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
pub mod controller;
|
||||
pub mod parser;
|
||||
pub mod repository;
|
||||
pub mod route;
|
||||
pub mod schema;
|
||||
pub mod service;
|
||||
pub mod types;
|
||||
@@ -0,0 +1,804 @@
|
||||
use crate::shared::errors::AppError;
|
||||
use crate::shared::utils::parse_html;
|
||||
use crate::shared::utils::scraping::{attr, extract_slug, selector, text, text_from_or};
|
||||
use once_cell::sync::Lazy;
|
||||
use regex::Regex;
|
||||
use scraper::Selector;
|
||||
|
||||
static ITEM_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse("article.bs").unwrap());
|
||||
static TITLE_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse(".tt h2").unwrap());
|
||||
static IMG_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse("img").unwrap());
|
||||
static SCORE_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse(".numscore").unwrap());
|
||||
static STATUS_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse(".status").unwrap());
|
||||
static TYPE_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse(".type").unwrap());
|
||||
static LINK_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse("a").unwrap());
|
||||
static PAGINATION_SELECTOR: Lazy<Selector> =
|
||||
Lazy::new(|| Selector::parse(".pagination .page-numbers:not(.next)").unwrap());
|
||||
static NEXT_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse(".pagination .next").unwrap());
|
||||
static SLUG_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"/([^/]+)/?$").unwrap());
|
||||
static GENRE_SLUG_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"genre-(.+)$").unwrap());
|
||||
pub fn parse_ongoing_anime(
|
||||
html: &str,
|
||||
) -> Result<Vec<crate::shared::types::entities::anime::OngoingAnimeItem>, AppError> {
|
||||
let items = parse_ongoing_anime_with_score(html)?;
|
||||
Ok(items
|
||||
.into_iter()
|
||||
.map(
|
||||
|item| crate::shared::types::entities::anime::OngoingAnimeItem {
|
||||
title: item.title,
|
||||
slug: item.slug,
|
||||
poster: item.poster,
|
||||
current_episode: item.score,
|
||||
anime_url: item.anime_url,
|
||||
},
|
||||
)
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub fn parse_complete_anime(
|
||||
html: &str,
|
||||
) -> Result<Vec<crate::shared::types::entities::anime::CompleteAnimeItem>, AppError> {
|
||||
let document = parse_html(html);
|
||||
let mut complete_anime = Vec::new();
|
||||
|
||||
for element in document.select(&ITEM_SELECTOR) {
|
||||
let title = element
|
||||
.select(&TITLE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or_default();
|
||||
let poster = element
|
||||
.select(&IMG_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("src").or(e.value().attr("data-src")))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let anime_url = element
|
||||
.select(&LINK_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("href"))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let slug = SLUG_REGEX
|
||||
.captures(&anime_url)
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let episode_count = text_from_or(&element, &STATUS_SELECTOR, "N/A");
|
||||
|
||||
if !title.is_empty() {
|
||||
complete_anime.push(crate::shared::types::entities::anime::CompleteAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
episode_count,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(complete_anime)
|
||||
}
|
||||
|
||||
pub fn parse_genres(html: &str) -> Result<Vec<crate::modules::anime2::types::Genre>, AppError> {
|
||||
let document = parse_html(html);
|
||||
let mut genres = Vec::new();
|
||||
let genre_label_selector = selector("label[for^=\"genre-\"]").ok_or_else(|| {
|
||||
AppError::ScraperError("Invalid selector: label[for^=\"genre-\"]".to_string())
|
||||
})?;
|
||||
|
||||
for element in document.select(&genre_label_selector) {
|
||||
let name = text(&element).trim().to_string();
|
||||
let for_attr = attr(&element, "for").unwrap_or_default();
|
||||
|
||||
let slug = GENRE_SLUG_REGEX
|
||||
.captures(&for_attr)
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if !name.is_empty() && !slug.is_empty() {
|
||||
genres.push(crate::modules::anime2::types::Genre { name, slug });
|
||||
}
|
||||
}
|
||||
|
||||
Ok(genres)
|
||||
}
|
||||
|
||||
pub fn parse_filter_page(
|
||||
html: &str,
|
||||
current_page: u32,
|
||||
) -> Result<
|
||||
(
|
||||
Vec<crate::shared::types::entities::anime::FilterAnimeItem>,
|
||||
crate::shared::types::entities::anime::Pagination,
|
||||
),
|
||||
AppError,
|
||||
> {
|
||||
let document = parse_html(html);
|
||||
let mut anime_list = Vec::new();
|
||||
|
||||
for element in document.select(&ITEM_SELECTOR) {
|
||||
let title = element
|
||||
.select(&TITLE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let poster = element
|
||||
.select(&IMG_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("src").or(e.value().attr("data-src")))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let score = element
|
||||
.select(&SCORE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or("N/A".to_string());
|
||||
|
||||
let status = element
|
||||
.select(&STATUS_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or("Unknown".to_string());
|
||||
|
||||
let anime_type = element
|
||||
.select(&TYPE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or("Unknown".to_string());
|
||||
|
||||
let anime_url = element
|
||||
.select(&LINK_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("href"))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let slug = SLUG_REGEX
|
||||
.captures(&anime_url)
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if !title.is_empty() {
|
||||
anime_list.push(crate::shared::types::entities::anime::FilterAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
score,
|
||||
status,
|
||||
r#type: anime_type,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let last_visible_page = document
|
||||
.select(&PAGINATION_SELECTOR)
|
||||
.next_back()
|
||||
.map(|e| {
|
||||
e.text()
|
||||
.collect::<String>()
|
||||
.trim()
|
||||
.parse::<u32>()
|
||||
.unwrap_or(1)
|
||||
})
|
||||
.unwrap_or(1);
|
||||
|
||||
let has_next_page = document.select(&NEXT_SELECTOR).next().is_some();
|
||||
let pagination = crate::shared::types::entities::anime::Pagination {
|
||||
current_page,
|
||||
last_visible_page,
|
||||
has_next_page,
|
||||
next_page: if has_next_page {
|
||||
Some(current_page + 1)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
has_previous_page: current_page > 1,
|
||||
previous_page: if current_page > 1 {
|
||||
Some(current_page - 1)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
};
|
||||
|
||||
Ok((anime_list, pagination))
|
||||
}
|
||||
|
||||
pub fn parse_genre_anime(
|
||||
html: &str,
|
||||
) -> Result<Vec<crate::shared::types::entities::anime::GenreAnimeItem>, AppError> {
|
||||
let document = parse_html(html);
|
||||
let mut anime_list = Vec::new();
|
||||
|
||||
for element in document.select(&ITEM_SELECTOR) {
|
||||
let title = element
|
||||
.select(&TITLE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let poster = element
|
||||
.select(&IMG_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("src").or(e.value().attr("data-src")))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let score = element
|
||||
.select(&SCORE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or("N/A".to_string());
|
||||
|
||||
let status = element
|
||||
.select(&STATUS_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or("Unknown".to_string());
|
||||
|
||||
let anime_url = element
|
||||
.select(&LINK_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("href"))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let slug = SLUG_REGEX
|
||||
.captures(&anime_url)
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if !title.is_empty() {
|
||||
anime_list.push(crate::shared::types::entities::anime::GenreAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
score,
|
||||
status,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(anime_list)
|
||||
}
|
||||
|
||||
pub fn parse_search_anime(
|
||||
html: &str,
|
||||
) -> Result<Vec<crate::shared::types::entities::anime::SearchAnimeItem>, AppError> {
|
||||
let document = parse_html(html);
|
||||
let mut anime_list = Vec::new();
|
||||
|
||||
for element in document.select(&ITEM_SELECTOR) {
|
||||
let title = element
|
||||
.select(&TITLE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let poster = element
|
||||
.select(&IMG_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("src").or(e.value().attr("data-src")))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let anime_url = element
|
||||
.select(&LINK_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("href"))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let slug = SLUG_REGEX
|
||||
.captures(&anime_url)
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if !title.is_empty() {
|
||||
anime_list.push(crate::shared::types::entities::anime::SearchAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
description: String::new(),
|
||||
anime_url,
|
||||
genres: Vec::new(),
|
||||
rating: "N/A".to_string(),
|
||||
r#type: "Unknown".to_string(),
|
||||
season: "Unknown".to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(anime_list)
|
||||
}
|
||||
|
||||
pub fn parse_latest_anime(
|
||||
html: &str,
|
||||
) -> Result<Vec<crate::shared::types::entities::anime::LatestAnimeItem>, AppError> {
|
||||
let document = parse_html(html);
|
||||
let mut anime_list = Vec::new();
|
||||
|
||||
for element in document.select(&ITEM_SELECTOR) {
|
||||
let title = element
|
||||
.select(&TITLE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let poster = element
|
||||
.select(&IMG_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("src").or(e.value().attr("data-src")))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let score = element
|
||||
.select(&SCORE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or("N/A".to_string());
|
||||
|
||||
let anime_url = element
|
||||
.select(&LINK_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("href"))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let slug = SLUG_REGEX
|
||||
.captures(&anime_url)
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if !title.is_empty() {
|
||||
anime_list.push(crate::shared::types::entities::anime::LatestAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
current_episode: "N/A".to_string(),
|
||||
score,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(anime_list)
|
||||
}
|
||||
|
||||
pub fn parse_ongoing_anime_with_score(
|
||||
html: &str,
|
||||
) -> Result<Vec<crate::shared::types::entities::anime::OngoingAnimeItemWithScore>, AppError> {
|
||||
let document = parse_html(html);
|
||||
let mut anime_list = Vec::new();
|
||||
|
||||
for element in document.select(&ITEM_SELECTOR) {
|
||||
let title = element
|
||||
.select(&TITLE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let poster = element
|
||||
.select(&IMG_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("src").or(e.value().attr("data-src")))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let score = element
|
||||
.select(&SCORE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or("N/A".to_string());
|
||||
|
||||
let anime_url = element
|
||||
.select(&LINK_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("href"))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let slug = SLUG_REGEX
|
||||
.captures(&anime_url)
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if !title.is_empty() {
|
||||
anime_list.push(
|
||||
crate::shared::types::entities::anime::OngoingAnimeItemWithScore {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
score,
|
||||
anime_url,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(anime_list)
|
||||
}
|
||||
|
||||
pub fn parse_pagination(
|
||||
document: &scraper::Html,
|
||||
current_page: u32,
|
||||
) -> Result<crate::shared::types::entities::anime::Pagination, String> {
|
||||
let last_visible_page = document
|
||||
.select(&PAGINATION_SELECTOR)
|
||||
.next_back()
|
||||
.map(|e| {
|
||||
e.text()
|
||||
.collect::<String>()
|
||||
.trim()
|
||||
.parse::<u32>()
|
||||
.unwrap_or(1)
|
||||
})
|
||||
.unwrap_or(1);
|
||||
|
||||
let has_next_page = document.select(&NEXT_SELECTOR).next().is_some();
|
||||
let pagination = crate::shared::types::entities::anime::Pagination {
|
||||
current_page,
|
||||
last_visible_page,
|
||||
has_next_page,
|
||||
next_page: if has_next_page {
|
||||
Some(current_page + 1)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
has_previous_page: current_page > 1,
|
||||
previous_page: if current_page > 1 {
|
||||
Some(current_page - 1)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
};
|
||||
|
||||
Ok(pagination)
|
||||
}
|
||||
|
||||
pub fn parse_pagination_with_string(
|
||||
document: &scraper::Html,
|
||||
current_page: u32,
|
||||
) -> Result<crate::shared::types::entities::anime::PaginationWithStringPages, String> {
|
||||
let last_visible_page = document
|
||||
.select(&PAGINATION_SELECTOR)
|
||||
.next_back()
|
||||
.map(|e| {
|
||||
e.text()
|
||||
.collect::<String>()
|
||||
.trim()
|
||||
.parse::<u32>()
|
||||
.unwrap_or(1)
|
||||
})
|
||||
.unwrap_or(1);
|
||||
|
||||
let has_next_page = document.select(&NEXT_SELECTOR).next().is_some();
|
||||
let pagination = crate::shared::types::entities::anime::PaginationWithStringPages {
|
||||
current_page,
|
||||
last_visible_page,
|
||||
has_next_page,
|
||||
next_page: if has_next_page {
|
||||
Some((current_page + 1).to_string())
|
||||
} else {
|
||||
None
|
||||
},
|
||||
has_previous_page: current_page > 1,
|
||||
previous_page: if current_page > 1 {
|
||||
Some((current_page - 1).to_string())
|
||||
} else {
|
||||
None
|
||||
},
|
||||
};
|
||||
|
||||
Ok(pagination)
|
||||
}
|
||||
|
||||
pub fn parse_anime_detail(
|
||||
html: &str,
|
||||
) -> Result<crate::modules::anime2::types::AnimeDetailData, AppError> {
|
||||
let document = parse_html(html);
|
||||
|
||||
let title_selector = selector(".entry-title")
|
||||
.ok_or_else(|| AppError::ScraperError("Invalid selector: .entry-title".to_string()))?;
|
||||
let alt_title_selector = selector(".alter")
|
||||
.ok_or_else(|| AppError::ScraperError("Invalid selector: .alter".to_string()))?;
|
||||
let poster_selector = selector(".thumb img, .thumbook img, .wp-post-image, .ts-post-image")
|
||||
.ok_or_else(|| {
|
||||
AppError::ScraperError(
|
||||
"Invalid selector: .thumb img, .thumbook img, .wp-post-image, .ts-post-image"
|
||||
.to_string(),
|
||||
)
|
||||
})?;
|
||||
let poster2_selector = selector(".bigcover img, .bixbox.animefull .bigcover .ime img")
|
||||
.ok_or_else(|| {
|
||||
AppError::ScraperError(
|
||||
"Invalid selector: .bigcover img, .bixbox.animefull .bigcover .ime img".to_string(),
|
||||
)
|
||||
})?;
|
||||
let spe_span_selector = selector(".info-content .spe span").ok_or_else(|| {
|
||||
AppError::ScraperError("Invalid selector: .info-content .spe span".to_string())
|
||||
})?;
|
||||
let a_selector =
|
||||
selector("a").ok_or_else(|| AppError::ScraperError("Invalid selector: a".to_string()))?;
|
||||
let synopsis_selector = selector(".entry-content p")
|
||||
.ok_or_else(|| AppError::ScraperError("Invalid selector: .entry-content p".to_string()))?;
|
||||
let genre_selector = selector(".genxed a")
|
||||
.ok_or_else(|| AppError::ScraperError("Invalid selector: .genxed a".to_string()))?;
|
||||
let download_container_selector = selector(".soraddl.dlone")
|
||||
.ok_or_else(|| AppError::ScraperError("Invalid selector: .soraddl.dlone".to_string()))?;
|
||||
let resolution_selector = selector(".res")
|
||||
.ok_or_else(|| AppError::ScraperError("Invalid selector: .res".to_string()))?;
|
||||
let link_selector = selector(".slink a")
|
||||
.ok_or_else(|| AppError::ScraperError("Invalid selector: .slink a".to_string()))?;
|
||||
let h3_selector =
|
||||
selector("h3").ok_or_else(|| AppError::ScraperError("Invalid selector: h3".to_string()))?;
|
||||
let recommendation_selector = selector(".listupd .bs")
|
||||
.ok_or_else(|| AppError::ScraperError("Invalid selector: .listupd .bs".to_string()))?;
|
||||
let rec_title_selector = selector(".ntitle")
|
||||
.ok_or_else(|| AppError::ScraperError("Invalid selector: .ntitle".to_string()))?;
|
||||
let rec_img_selector = selector("img")
|
||||
.ok_or_else(|| AppError::ScraperError("Invalid selector: img".to_string()))?;
|
||||
let status_selector = selector(".status")
|
||||
.ok_or_else(|| AppError::ScraperError("Invalid selector: .status".to_string()))?;
|
||||
let type_selector = selector(".typez")
|
||||
.ok_or_else(|| AppError::ScraperError("Invalid selector: .typez".to_string()))?;
|
||||
|
||||
let title = text_from_or(&document.root_element(), &title_selector, "");
|
||||
let alternative_title = text_from_or(&document.root_element(), &alt_title_selector, "");
|
||||
|
||||
let poster = document
|
||||
.select(&poster_selector)
|
||||
.next()
|
||||
.and_then(|e| {
|
||||
attr(&e, "src")
|
||||
.or_else(|| attr(&e, "data-src"))
|
||||
.or_else(|| attr(&e, "data-lazy-src"))
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let poster2 = document
|
||||
.select(&poster2_selector)
|
||||
.next()
|
||||
.and_then(|e| {
|
||||
attr(&e, "src")
|
||||
.or_else(|| attr(&e, "data-src"))
|
||||
.or_else(|| attr(&e, "data-lazy-src"))
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let r#type = document
|
||||
.select(&spe_span_selector)
|
||||
.find(|e| text(&e).contains("Tipe:"))
|
||||
.and_then(|span| span.select(&a_selector).next())
|
||||
.map(|e| text(&e))
|
||||
.unwrap_or_default();
|
||||
|
||||
let release_date = document
|
||||
.select(&spe_span_selector)
|
||||
.find(|e| text(&e).contains("Dirilis:"))
|
||||
.map(|e| text(&e))
|
||||
.unwrap_or_default();
|
||||
|
||||
let status = document
|
||||
.select(&spe_span_selector)
|
||||
.find(|e| text(&e).contains("Status:"))
|
||||
.map(|e| text(&e))
|
||||
.unwrap_or_default();
|
||||
|
||||
let synopsis = text_from_or(&document.root_element(), &synopsis_selector, "");
|
||||
|
||||
let studio = document
|
||||
.select(&spe_span_selector)
|
||||
.find(|e| text(&e).contains("Studio:"))
|
||||
.and_then(|span| span.select(&a_selector).next())
|
||||
.map(|e| text(&e))
|
||||
.unwrap_or_default();
|
||||
|
||||
let mut genres = Vec::new();
|
||||
for element in document.select(&genre_selector) {
|
||||
let name = text(&element);
|
||||
let anime_url = attr(&element, "href").unwrap_or_default();
|
||||
let genre_slug = extract_slug(&anime_url);
|
||||
genres.push(crate::modules::anime2::types::DetailGenre {
|
||||
name,
|
||||
slug: genre_slug,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
|
||||
let mut batch = Vec::new();
|
||||
let mut ova = Vec::new();
|
||||
let mut downloads = Vec::new();
|
||||
|
||||
for element in document.select(&download_container_selector) {
|
||||
let title = element
|
||||
.select(&h3_selector)
|
||||
.next()
|
||||
.map(|e| text(&e))
|
||||
.unwrap_or_else(|| "Unknown".to_string());
|
||||
|
||||
let category = title.to_lowercase();
|
||||
let is_batch = category.contains("batch");
|
||||
let is_ova = category.contains("ova");
|
||||
|
||||
let mut all_links = Vec::new();
|
||||
|
||||
let row_selector = selector("table tr")
|
||||
.ok_or_else(|| AppError::ScraperError("Invalid selector: table tr".to_string()))?;
|
||||
for row in element.select(&row_selector) {
|
||||
let resolution = text_from_or(&row, &resolution_selector, "");
|
||||
|
||||
for link_element in row.select(&link_selector) {
|
||||
let provider = text(&link_element);
|
||||
let url = attr(&link_element, "href").unwrap_or_default();
|
||||
|
||||
let name = if !resolution.is_empty() {
|
||||
format!("{} - {}", resolution, provider)
|
||||
} else {
|
||||
provider
|
||||
};
|
||||
|
||||
all_links.push(crate::modules::anime2::types::Link { name, url });
|
||||
}
|
||||
}
|
||||
|
||||
let download_item = crate::modules::anime2::types::DownloadItem {
|
||||
resolution: title,
|
||||
links: all_links,
|
||||
};
|
||||
|
||||
if is_batch {
|
||||
batch.push(download_item);
|
||||
} else if is_ova {
|
||||
ova.push(download_item);
|
||||
} else {
|
||||
downloads.push(download_item);
|
||||
}
|
||||
}
|
||||
|
||||
let mut recommendations = Vec::new();
|
||||
for element in document.select(&recommendation_selector) {
|
||||
let title = text_from_or(&element, &rec_title_selector, "");
|
||||
|
||||
let anime_url = element
|
||||
.select(&a_selector)
|
||||
.next()
|
||||
.and_then(|e| attr(&e, "href"))
|
||||
.unwrap_or_default();
|
||||
|
||||
let rec_slug = extract_slug(&anime_url);
|
||||
|
||||
let poster = element
|
||||
.select(&rec_img_selector)
|
||||
.next()
|
||||
.and_then(|e| attr(&e, "data-src").or_else(|| attr(&e, "src")))
|
||||
.unwrap_or_default();
|
||||
|
||||
let status = text_from_or(&element, &status_selector, "");
|
||||
|
||||
let r#type = text_from_or(&element, &type_selector, "");
|
||||
|
||||
recommendations.push(crate::modules::anime2::types::Recommendation {
|
||||
title,
|
||||
slug: rec_slug,
|
||||
poster,
|
||||
status,
|
||||
r#type,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(crate::modules::anime2::types::AnimeDetailData {
|
||||
title,
|
||||
alternative_title,
|
||||
poster,
|
||||
poster2,
|
||||
r#type,
|
||||
release_date,
|
||||
status,
|
||||
synopsis,
|
||||
studio,
|
||||
genres,
|
||||
producers: vec![],
|
||||
recommendations,
|
||||
batch,
|
||||
ova,
|
||||
downloads,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn parse_genre_page(
|
||||
html: &str,
|
||||
current_page: u32,
|
||||
) -> Result<
|
||||
(
|
||||
Vec<crate::shared::types::entities::anime::GenreAnimeItem>,
|
||||
crate::shared::types::entities::anime::Pagination,
|
||||
),
|
||||
AppError,
|
||||
> {
|
||||
let document = parse_html(html);
|
||||
let anime_list = parse_genre_anime(html)?;
|
||||
let pagination = parse_pagination(&document, current_page)?;
|
||||
Ok((anime_list, pagination))
|
||||
}
|
||||
|
||||
pub fn parse_search_page(
|
||||
html: &str,
|
||||
current_page: u32,
|
||||
) -> Result<
|
||||
(
|
||||
Vec<crate::shared::types::entities::anime::SearchAnimeItem>,
|
||||
crate::shared::types::entities::anime::PaginationWithStringPages,
|
||||
),
|
||||
AppError,
|
||||
> {
|
||||
let document = parse_html(html);
|
||||
let data = parse_search_anime(html)?;
|
||||
let pagination = parse_pagination_with_string(&document, current_page)?;
|
||||
Ok((data, pagination))
|
||||
}
|
||||
|
||||
pub fn parse_latest_page(
|
||||
html: &str,
|
||||
current_page: u32,
|
||||
) -> Result<
|
||||
(
|
||||
Vec<crate::shared::types::entities::anime::LatestAnimeItem>,
|
||||
crate::shared::types::entities::anime::Pagination,
|
||||
),
|
||||
AppError,
|
||||
> {
|
||||
let document = parse_html(html);
|
||||
let anime_list = parse_latest_anime(html)?;
|
||||
let pagination = parse_pagination(&document, current_page)?;
|
||||
Ok((anime_list, pagination))
|
||||
}
|
||||
|
||||
pub fn parse_ongoing_page(
|
||||
html: &str,
|
||||
current_page: u32,
|
||||
) -> Result<
|
||||
(
|
||||
Vec<crate::shared::types::entities::anime::OngoingAnimeItemWithScore>,
|
||||
crate::shared::types::entities::anime::Pagination,
|
||||
),
|
||||
AppError,
|
||||
> {
|
||||
let document = parse_html(html);
|
||||
let anime_list = parse_ongoing_anime_with_score(html)?;
|
||||
let pagination = parse_pagination(&document, current_page)?;
|
||||
Ok((anime_list, pagination))
|
||||
}
|
||||
|
||||
pub fn parse_complete_page(
|
||||
html: &str,
|
||||
current_page: u32,
|
||||
) -> Result<
|
||||
(
|
||||
Vec<crate::shared::types::entities::anime::CompleteAnimeItem>,
|
||||
crate::shared::types::entities::anime::Pagination,
|
||||
),
|
||||
AppError,
|
||||
> {
|
||||
let document = parse_html(html);
|
||||
let anime_list = parse_complete_anime(html)?;
|
||||
let pagination = parse_pagination(&document, current_page)?;
|
||||
Ok((anime_list, pagination))
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
use crate::shared::database::traits::scraping_repository::ScrapingRepository;
|
||||
use crate::shared::errors::AppError;
|
||||
use crate::shared::utils::web::proxy_fetch::fetch_with_proxy_only;
|
||||
use async_trait::async_trait;
|
||||
use tracing::warn;
|
||||
|
||||
const BASE_URL: &str = "https://alqanime.si";
|
||||
const BASE_DETAIL_URL: &str = "https://alqanime.net";
|
||||
|
||||
pub struct Anime2Repository;
|
||||
|
||||
impl Default for Anime2Repository {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Anime2Repository {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
pub fn index_ongoing_url(&self) -> String {
|
||||
format!("{}/anime/?status=ongoing&type=&order=update", BASE_URL)
|
||||
}
|
||||
|
||||
pub fn index_complete_url(&self) -> String {
|
||||
format!("{}/anime/?status=completed&type=&order=update", BASE_URL)
|
||||
}
|
||||
|
||||
pub fn genre_list_url(&self) -> String {
|
||||
format!("{}/anime/", BASE_URL)
|
||||
}
|
||||
|
||||
pub fn filter_url(&self, page: u32, order: &str) -> String {
|
||||
if page > 1 {
|
||||
format!("{}/anime/page/{}/?order={}", BASE_URL, page, order)
|
||||
} else {
|
||||
format!("{}/anime/?order={}", BASE_URL, order)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn detail_url(&self, slug: &str) -> String {
|
||||
format!("{}/{}/", BASE_DETAIL_URL, slug)
|
||||
}
|
||||
|
||||
pub fn detail_image_url(&self, slug: &str) -> String {
|
||||
format!("{}/anime/{}/", BASE_URL, slug)
|
||||
}
|
||||
|
||||
pub fn genre_page_url(&self, genre_slug: &str, page: u32) -> String {
|
||||
if page > 1 {
|
||||
format!(
|
||||
"{}/anime/page/{}/?genre[]={}&order=update",
|
||||
BASE_URL, page, genre_slug
|
||||
)
|
||||
} else {
|
||||
format!("{}/anime/?genre[]={}&order=update", BASE_URL, genre_slug)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn search_url(&self, query: &str, page: u32) -> String {
|
||||
let encoded = urlencoding::encode(query);
|
||||
if page == 1 {
|
||||
format!("{}/?s={}", BASE_URL, encoded)
|
||||
} else {
|
||||
format!("{}/page/{}/?s={}", BASE_URL, page, encoded)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn latest_url(&self, page: u32) -> String {
|
||||
format!(
|
||||
"{}/anime/page/{}/?status=&type=&order=latest",
|
||||
BASE_URL, page
|
||||
)
|
||||
}
|
||||
|
||||
pub fn ongoing_url(&self, page: u32) -> String {
|
||||
format!(
|
||||
"{}/anime/page/{}/?status=ongoing&type=&order=update",
|
||||
BASE_URL, page
|
||||
)
|
||||
}
|
||||
|
||||
pub fn complete_url(&self, page: u32) -> String {
|
||||
format!(
|
||||
"{}/anime/page/{}/?status=completed&order=update",
|
||||
BASE_URL, page
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ScrapingRepository for Anime2Repository {
|
||||
async fn fetch_html(&self, url: &str) -> Result<String, AppError> {
|
||||
let response = fetch_with_proxy_only(url).await?;
|
||||
if response.data.trim().is_empty() {
|
||||
warn!("Anime2 browserless fetch returned empty body for {}", url);
|
||||
}
|
||||
Ok(response.data)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::{routing::get, Router};
|
||||
|
||||
use crate::modules::anime2::controller;
|
||||
use crate::shared::state::AppState;
|
||||
|
||||
pub fn routes(router: Router<Arc<AppState>>) -> Router<Arc<AppState>> {
|
||||
router
|
||||
.route("/api/anime2", get(controller::index))
|
||||
.route(
|
||||
"/api/anime2/complete_anime/{slug}",
|
||||
get(controller::complete_anime_slug),
|
||||
)
|
||||
.route("/api/anime2/detail/{slug}", get(controller::detail_slug))
|
||||
.route("/api/anime2/filter", get(controller::filter))
|
||||
.route("/api/anime2/genre_list", get(controller::genre_list))
|
||||
.route(
|
||||
"/api/anime2/genre/{slug}",
|
||||
get(controller::genre_slug_index),
|
||||
)
|
||||
.route(
|
||||
"/api/anime2/genre/{slug}/{page}",
|
||||
get(controller::genre_slug_page),
|
||||
)
|
||||
.route("/api/anime2/latest/{slug}", get(controller::latest_slug))
|
||||
.route(
|
||||
"/api/anime2/ongoing_anime/{slug}",
|
||||
get(controller::ongoing_anime_slug),
|
||||
)
|
||||
.route(
|
||||
"/api/anime2/search/{slug}",
|
||||
get(controller::search_slug_index),
|
||||
)
|
||||
.route(
|
||||
"/api/anime2/search/{slug}/{page}",
|
||||
get(controller::search_slug_page),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
use serde::Deserialize;
|
||||
use utoipa::ToSchema;
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, ToSchema)]
|
||||
pub struct SlugPath {
|
||||
pub slug: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, ToSchema)]
|
||||
pub struct SlugPagePath {
|
||||
pub slug: String,
|
||||
pub page: u32,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, ToSchema)]
|
||||
pub struct FilterQuery {
|
||||
pub page: Option<u32>,
|
||||
pub genre: Option<String>,
|
||||
pub status: Option<String>,
|
||||
pub r#type: Option<String>,
|
||||
pub order: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, ToSchema)]
|
||||
pub struct GenreQuery {
|
||||
pub page: Option<u32>,
|
||||
pub status: Option<String>,
|
||||
pub order: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, ToSchema)]
|
||||
pub struct SearchQuery {
|
||||
pub q: Option<String>,
|
||||
}
|
||||
@@ -0,0 +1,359 @@
|
||||
use crate::shared::types::entities::anime::*;
|
||||
use crate::shared::utils::parse_html;
|
||||
use crate::shared::utils::scraping::{
|
||||
attr, attr_from, attr_from_or, extract_slug, selector, text, text_from_or,
|
||||
};
|
||||
use scraper::{Html, Selector};
|
||||
|
||||
// ============================================================================
|
||||
// SELECTORS
|
||||
// ============================================================================
|
||||
|
||||
/// Common selectors used across anime parsing
|
||||
pub struct AnimeSelectors {
|
||||
pub item: Selector,
|
||||
pub title: Selector,
|
||||
pub link: Selector,
|
||||
pub img: Selector,
|
||||
pub episode: Selector,
|
||||
pub score: Selector,
|
||||
pub status: Selector,
|
||||
pub genre: Selector,
|
||||
pub rating: Selector,
|
||||
pub type_sel: Selector,
|
||||
pub season: Selector,
|
||||
pub desc: Selector,
|
||||
}
|
||||
|
||||
impl AnimeSelectors {
|
||||
pub fn new() -> Result<Self, String> {
|
||||
Ok(Self {
|
||||
item: selector("article.bs").ok_or("Invalid selector: article.bs")?,
|
||||
title: selector(".tt h2").ok_or("Invalid selector: .tt h2")?,
|
||||
link: selector("a").ok_or("Invalid selector: a")?,
|
||||
img: selector("img").ok_or("Invalid selector: img")?,
|
||||
episode: selector(".epx").ok_or("Invalid selector: .epx")?,
|
||||
score: selector(".numscore").ok_or("Invalid selector: .numscore")?,
|
||||
status: selector(".status").ok_or("Invalid selector: .status")?,
|
||||
genre: selector(".genres a").ok_or("Invalid selector: .genres a")?,
|
||||
rating: selector(".score").ok_or("Invalid selector: .score")?,
|
||||
type_sel: selector(".typez").ok_or("Invalid selector: .typez")?,
|
||||
season: selector(".season").ok_or("Invalid selector: .season")?,
|
||||
desc: selector(".data .typez").ok_or("Invalid selector: .data .typez")?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for AnimeSelectors {
|
||||
fn default() -> Self {
|
||||
Self::new().expect("Valid CSS selectors")
|
||||
}
|
||||
}
|
||||
|
||||
// Global lazy static instance for selectors to avoid reallocation per parse
|
||||
use once_cell::sync::Lazy;
|
||||
static ANIME_SELECTORS: Lazy<Result<AnimeSelectors, String>> =
|
||||
Lazy::new(|| AnimeSelectors::new().map_err(|e| format!("Failed to create selectors: {}", e)));
|
||||
|
||||
// ============================================================================
|
||||
// HELPER FUNCTIONS
|
||||
// ============================================================================
|
||||
|
||||
/// Extract poster URL from an element, checking both src and data-src attributes
|
||||
pub fn extract_poster(element: &scraper::ElementRef, img_selector: &Selector) -> String {
|
||||
element
|
||||
.select(img_selector)
|
||||
.next()
|
||||
.and_then(|e| attr(&e, "src").or(attr(&e, "data-src")))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// ANIME PARSERS
|
||||
// ============================================================================
|
||||
|
||||
/// Parse ongoing anime items from HTML
|
||||
pub fn parse_ongoing_anime(
|
||||
html: &str,
|
||||
) -> Result<Vec<OngoingAnimeItem>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let document = parse_html(html);
|
||||
let selectors = ANIME_SELECTORS.as_ref().map_err(|e| e.clone())?;
|
||||
let mut items = Vec::new();
|
||||
|
||||
for element in document.select(&selectors.item) {
|
||||
let title = text_from_or(&element, &selectors.title, "");
|
||||
if title.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let href = attr_from_or(&element, &selectors.link, "href", "");
|
||||
let slug = extract_slug(&href);
|
||||
let poster = extract_poster(&element, &selectors.img);
|
||||
let current_episode = text_from_or(&element, &selectors.episode, "N/A");
|
||||
let anime_url = attr_from_or(&element, &selectors.link, "href", "");
|
||||
|
||||
items.push(OngoingAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
current_episode,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
/// Parse ongoing anime items with score from HTML
|
||||
pub fn parse_ongoing_anime_with_score(
|
||||
html: &str,
|
||||
) -> Result<Vec<OngoingAnimeItemWithScore>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let document = parse_html(html);
|
||||
let selectors = ANIME_SELECTORS.as_ref().map_err(|e| e.clone())?;
|
||||
let mut items = Vec::new();
|
||||
|
||||
for element in document.select(&selectors.item) {
|
||||
let title = text_from_or(&element, &selectors.title, "");
|
||||
if title.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let poster = extract_poster(&element, &selectors.img);
|
||||
let score = text_from_or(&element, &selectors.score, "N/A");
|
||||
let anime_url = attr_from_or(&element, &selectors.link, "href", "");
|
||||
let slug = extract_slug(&anime_url);
|
||||
|
||||
items.push(OngoingAnimeItemWithScore {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
score,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
/// Parse complete anime items from HTML
|
||||
pub fn parse_complete_anime(
|
||||
html: &str,
|
||||
) -> Result<Vec<CompleteAnimeItem>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let document = parse_html(html);
|
||||
let selectors = ANIME_SELECTORS.as_ref().map_err(|e| e.clone())?;
|
||||
let mut items = Vec::new();
|
||||
|
||||
for element in document.select(&selectors.item) {
|
||||
let title = text_from_or(&element, &selectors.title, "");
|
||||
if title.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let href = attr_from_or(&element, &selectors.link, "href", "");
|
||||
let slug = extract_slug(&href);
|
||||
let poster = extract_poster(&element, &selectors.img);
|
||||
let episode_count = text_from_or(&element, &selectors.episode, "N/A");
|
||||
let anime_url = attr_from_or(&element, &selectors.link, "href", "");
|
||||
|
||||
items.push(CompleteAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
episode_count,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
/// Parse latest anime items from HTML
|
||||
pub fn parse_latest_anime(
|
||||
html: &str,
|
||||
) -> Result<Vec<LatestAnimeItem>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let document = parse_html(html);
|
||||
let selectors = ANIME_SELECTORS.as_ref().map_err(|e| e.clone())?;
|
||||
let mut items = Vec::new();
|
||||
|
||||
for element in document.select(&selectors.item) {
|
||||
let title = text_from_or(&element, &selectors.title, "");
|
||||
if title.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let poster = extract_poster(&element, &selectors.img);
|
||||
let current_episode = text_from_or(&element, &selectors.episode, "N/A");
|
||||
let score = text_from_or(&element, &selectors.score, "N/A");
|
||||
let anime_url = attr_from_or(&element, &selectors.link, "href", "");
|
||||
let slug = extract_slug(&anime_url);
|
||||
|
||||
items.push(LatestAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
current_episode,
|
||||
score,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
/// Parse search results from HTML
|
||||
pub fn parse_search_anime(
|
||||
html: &str,
|
||||
) -> Result<Vec<SearchAnimeItem>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let document = parse_html(html);
|
||||
let selectors = ANIME_SELECTORS.as_ref().map_err(|e| e.clone())?;
|
||||
let mut items = Vec::new();
|
||||
|
||||
for element in document.select(&selectors.item) {
|
||||
let title = text_from_or(&element, &selectors.title, "");
|
||||
if title.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let href = attr_from(&element, &selectors.link, "href").unwrap_or_default();
|
||||
let slug = extract_slug(&href);
|
||||
let poster = extract_poster(&element, &selectors.img);
|
||||
let description = text_from_or(&element, &selectors.desc, "");
|
||||
let anime_url = attr_from_or(&element, &selectors.link, "href", "");
|
||||
let genres = element.select(&selectors.genre).map(|e| text(&e)).collect();
|
||||
let rating = text_from_or(&element, &selectors.rating, "");
|
||||
let r#type = text_from_or(&element, &selectors.type_sel, "");
|
||||
let season = text_from_or(&element, &selectors.season, "");
|
||||
|
||||
items.push(SearchAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
description,
|
||||
anime_url,
|
||||
genres,
|
||||
rating,
|
||||
r#type,
|
||||
season,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
/// Parse genre-filtered anime items from HTML
|
||||
pub fn parse_genre_anime(
|
||||
html: &str,
|
||||
) -> Result<Vec<GenreAnimeItem>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let document = parse_html(html);
|
||||
let selectors = ANIME_SELECTORS.as_ref().map_err(|e| e.clone())?;
|
||||
let mut items = Vec::new();
|
||||
|
||||
for element in document.select(&selectors.item) {
|
||||
let title = text_from_or(&element, &selectors.title, "");
|
||||
if title.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let poster = extract_poster(&element, &selectors.img);
|
||||
let score = text_from_or(&element, &selectors.score, "N/A");
|
||||
let status = text_from_or(&element, &selectors.status, "Unknown");
|
||||
let anime_url = attr_from_or(&element, &selectors.link, "href", "");
|
||||
let slug = extract_slug(&anime_url);
|
||||
|
||||
items.push(GenreAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
score,
|
||||
status,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// PAGINATION PARSERS
|
||||
// ============================================================================
|
||||
|
||||
/// Parse pagination from HTML document
|
||||
pub fn parse_pagination(document: &Html, current_page: u32) -> Result<Pagination, String> {
|
||||
let pagination_selector =
|
||||
selector(".pagination .page-numbers:not(.next)").ok_or("Invalid selector")?;
|
||||
let next_selector = selector(".pagination .next").ok_or("Invalid selector")?;
|
||||
|
||||
let last_visible_page = document
|
||||
.select(&pagination_selector)
|
||||
.next_back()
|
||||
.and_then(|e| text(&e).trim().parse::<u32>().ok())
|
||||
.unwrap_or(current_page);
|
||||
|
||||
let has_next_page = document.select(&next_selector).next().is_some();
|
||||
let next_page = if has_next_page {
|
||||
Some(current_page + 1)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let has_previous_page = current_page > 1;
|
||||
let previous_page = if has_previous_page {
|
||||
Some(current_page - 1)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok(Pagination {
|
||||
current_page,
|
||||
last_visible_page,
|
||||
has_next_page,
|
||||
next_page,
|
||||
has_previous_page,
|
||||
previous_page,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse pagination with string-based page numbers (for search results)
|
||||
pub fn parse_pagination_with_string(
|
||||
document: &Html,
|
||||
current_page: u32,
|
||||
) -> Result<PaginationWithStringPages, String> {
|
||||
let pagination_selector =
|
||||
selector(".pagination .page-numbers:not(.next)").ok_or("Invalid selector")?;
|
||||
let next_selector = selector(".pagination .next").ok_or("Invalid selector")?;
|
||||
|
||||
let last_visible_page = document
|
||||
.select(&pagination_selector)
|
||||
.last()
|
||||
.and_then(|e| text(&e).trim().parse::<u32>().ok())
|
||||
.unwrap_or(current_page);
|
||||
|
||||
let has_next_page = document.select(&next_selector).next().is_some();
|
||||
|
||||
let next_page = if has_next_page {
|
||||
document
|
||||
.select(&next_selector)
|
||||
.next()
|
||||
.and_then(|e| attr(&e, "href"))
|
||||
.and_then(|href| href.split("/page/").nth(1).map(|s| s.to_string()))
|
||||
.and_then(|s| s.split('/').next().map(|s| s.to_string()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let has_previous_page = current_page > 1;
|
||||
let previous_page = if has_previous_page {
|
||||
Some((current_page - 1).to_string())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok(PaginationWithStringPages {
|
||||
current_page,
|
||||
last_visible_page,
|
||||
has_next_page,
|
||||
next_page,
|
||||
has_previous_page,
|
||||
previous_page,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,453 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::modules::anime2::parser;
|
||||
use crate::modules::anime2::repository::Anime2Repository;
|
||||
use crate::modules::anime2::types::{DetailResponse, GenresResponse};
|
||||
use crate::shared::database::traits::scraping_repository::ScrapingRepository;
|
||||
use crate::shared::errors::AppError;
|
||||
use crate::shared::services::images::cache::{
|
||||
apply_cached_posters, cache_image_urls_batch_lazy, get_cached_or_original,
|
||||
};
|
||||
use crate::shared::state::AppState;
|
||||
use crate::shared::types::ApiResponse;
|
||||
use crate::shared::utils::Cache;
|
||||
|
||||
const INDEX_CACHE_TTL: u64 = 300;
|
||||
const GENRE_LIST_CACHE_TTL: u64 = 3600;
|
||||
const FILTER_CACHE_TTL: u64 = 300;
|
||||
const DETAIL_CACHE_TTL: u64 = 300;
|
||||
const GENRE_CACHE_TTL: u64 = 300;
|
||||
const SEARCH_CACHE_TTL: u64 = 300;
|
||||
const LATEST_CACHE_TTL: u64 = 120;
|
||||
const ONGOING_CACHE_TTL: u64 = 300;
|
||||
const COMPLETE_CACHE_TTL: u64 = 300;
|
||||
|
||||
pub struct Anime2Service {
|
||||
repository: Anime2Repository,
|
||||
}
|
||||
|
||||
impl Anime2Service {
|
||||
pub fn new(repository: Anime2Repository) -> Self {
|
||||
Self { repository }
|
||||
}
|
||||
|
||||
pub async fn index(
|
||||
&self,
|
||||
app_state: Arc<AppState>,
|
||||
) -> Result<crate::modules::anime2::types::Anime2Response, AppError> {
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
|
||||
cache
|
||||
.get_or_set("anime2:index", INDEX_CACHE_TTL, || async {
|
||||
let ongoing_html = self
|
||||
.repository
|
||||
.fetch_html(&self.repository.index_ongoing_url())
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let complete_html = self
|
||||
.repository
|
||||
.fetch_html(&self.repository.index_complete_url())
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let mut data = tokio::task::spawn_blocking(move || {
|
||||
Ok::<_, String>((
|
||||
parser::parse_ongoing_anime(&ongoing_html).map_err(|e| e.to_string())?,
|
||||
parser::parse_complete_anime(&complete_html).map_err(|e| e.to_string())?,
|
||||
))
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())??;
|
||||
|
||||
let mut posters: Vec<String> =
|
||||
data.0.iter().map(|item| item.poster.clone()).collect();
|
||||
posters.extend(data.1.iter().map(|item| item.poster.clone()));
|
||||
|
||||
let cached_posters = cache_image_urls_batch_lazy(
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
posters,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
let ongoing_len = data.0.len();
|
||||
for (i, item) in data.0.iter_mut().enumerate() {
|
||||
if let Some(url) = cached_posters.get(i) {
|
||||
item.poster = url.clone();
|
||||
}
|
||||
}
|
||||
for (i, item) in data.1.iter_mut().enumerate() {
|
||||
if let Some(url) = cached_posters.get(ongoing_len + i) {
|
||||
item.poster = url.clone();
|
||||
}
|
||||
}
|
||||
|
||||
Ok(crate::modules::anime2::types::Anime2Response {
|
||||
status: "Ok".to_string(),
|
||||
data: crate::modules::anime2::types::Anime2Data {
|
||||
ongoing_anime: data.0,
|
||||
complete_anime: data.1,
|
||||
},
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(AppError::ScraperError)
|
||||
}
|
||||
|
||||
pub async fn genre_list(&self, app_state: Arc<AppState>) -> Result<GenresResponse, AppError> {
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
|
||||
cache
|
||||
.get_or_set("anime2:genres:list:v3", GENRE_LIST_CACHE_TTL, || async {
|
||||
let html = self
|
||||
.repository
|
||||
.fetch_html(&self.repository.genre_list_url())
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let genres = tokio::task::spawn_blocking(move || {
|
||||
parser::parse_genres(&html).map_err(|e| e.to_string())
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())??;
|
||||
|
||||
Ok(GenresResponse {
|
||||
status: "Ok".to_string(),
|
||||
data: genres,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(AppError::ScraperError)
|
||||
}
|
||||
|
||||
pub async fn filter(
|
||||
&self,
|
||||
app_state: Arc<AppState>,
|
||||
page: u32,
|
||||
genre: Option<String>,
|
||||
status: Option<String>,
|
||||
anime_type: Option<String>,
|
||||
order: String,
|
||||
) -> Result<crate::modules::anime2::types::FilterResponse, AppError> {
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
let cache_key = format!(
|
||||
"anime2:filter:{}:{:?}:{:?}:{:?}:{}",
|
||||
page, genre, status, anime_type, order
|
||||
);
|
||||
let genre_clone = genre.clone();
|
||||
let status_clone = status.clone();
|
||||
let anime_type_clone = anime_type.clone();
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, FILTER_CACHE_TTL, || async {
|
||||
let mut url = self.repository.filter_url(page, &order);
|
||||
|
||||
if let Some(g) = &genre {
|
||||
for genre_item in g.split(',') {
|
||||
url.push_str(&format!("&genre[]={}", genre_item.trim()));
|
||||
}
|
||||
}
|
||||
if let Some(s) = &status {
|
||||
url.push_str(&format!("&status={}", s));
|
||||
}
|
||||
if let Some(t) = &anime_type {
|
||||
url.push_str(&format!("&type={}", t));
|
||||
}
|
||||
|
||||
let html = self
|
||||
.repository
|
||||
.fetch_html(&url)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let (data, pagination) = tokio::task::spawn_blocking(move || {
|
||||
parser::parse_filter_page(&html, page).map_err(|e| e.to_string())
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())??;
|
||||
|
||||
let mut final_data = data;
|
||||
apply_cached_posters(
|
||||
&mut final_data,
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(crate::modules::anime2::types::FilterResponse {
|
||||
success: true,
|
||||
data: final_data,
|
||||
pagination,
|
||||
filters_applied: crate::modules::anime2::types::FiltersApplied {
|
||||
genre: genre_clone,
|
||||
status: status_clone,
|
||||
r#type: anime_type_clone,
|
||||
order: order.clone(),
|
||||
},
|
||||
status: "Ok".to_string(),
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(AppError::ScraperError)
|
||||
}
|
||||
|
||||
pub async fn detail(
|
||||
&self,
|
||||
app_state: Arc<AppState>,
|
||||
slug: String,
|
||||
) -> Result<DetailResponse, AppError> {
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
let cache_key = format!("anime2:detail:{}", slug);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, DETAIL_CACHE_TTL, || async {
|
||||
let detail_url = self.repository.detail_url(&slug);
|
||||
let image_url = self.repository.detail_image_url(&slug);
|
||||
let (detail_html, image_html) = tokio::join!(
|
||||
self.repository.fetch_html(&detail_url),
|
||||
self.repository.fetch_html(&image_url)
|
||||
);
|
||||
let detail_html = detail_html.map_err(|e| e.to_string())?;
|
||||
let image_html = image_html.ok();
|
||||
|
||||
let mut data = tokio::task::spawn_blocking(move || {
|
||||
let mut data =
|
||||
parser::parse_anime_detail(&detail_html).map_err(|e| e.to_string())?;
|
||||
if let Some(image_html) = image_html {
|
||||
if let Ok(image_data) = parser::parse_anime_detail(&image_html) {
|
||||
if !image_data.poster.is_empty() {
|
||||
data.poster = image_data.poster;
|
||||
}
|
||||
if !image_data.poster2.is_empty() {
|
||||
data.poster2 = image_data.poster2;
|
||||
}
|
||||
data.recommendations = image_data.recommendations;
|
||||
}
|
||||
}
|
||||
Ok::<_, String>(data)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())??;
|
||||
|
||||
data.poster = get_cached_or_original(
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
&data.poster,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
data.poster2 = get_cached_or_original(
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
&data.poster2,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
apply_cached_posters(
|
||||
&mut data.recommendations,
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(DetailResponse {
|
||||
status: "Ok".to_string(),
|
||||
data,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(AppError::ScraperError)
|
||||
}
|
||||
|
||||
pub async fn genre_slug(
|
||||
&self,
|
||||
app_state: Arc<AppState>,
|
||||
genre_slug: String,
|
||||
page: u32,
|
||||
) -> Result<ApiResponse<Vec<crate::shared::types::entities::anime::GenreAnimeItem>>, AppError>
|
||||
{
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
let cache_key = format!("anime2:genre:{}:{}", genre_slug, page);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, GENRE_CACHE_TTL, || async {
|
||||
let html = self
|
||||
.repository
|
||||
.fetch_html(&self.repository.genre_page_url(&genre_slug, page))
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let (data, _pagination) = tokio::task::spawn_blocking(move || {
|
||||
parser::parse_genre_page(&html, page).map_err(|e| e.to_string())
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())??;
|
||||
|
||||
let mut final_data = data;
|
||||
apply_cached_posters(
|
||||
&mut final_data,
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(ApiResponse::success(final_data))
|
||||
})
|
||||
.await
|
||||
.map_err(AppError::ScraperError)
|
||||
}
|
||||
|
||||
pub async fn search(
|
||||
&self,
|
||||
app_state: Arc<AppState>,
|
||||
query: String,
|
||||
page: u32,
|
||||
) -> Result<ApiResponse<Vec<crate::shared::types::entities::anime::SearchAnimeItem>>, AppError>
|
||||
{
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
let cache_key = format!("anime2:search:{}:{}", query, page);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, SEARCH_CACHE_TTL, || async {
|
||||
let html = self
|
||||
.repository
|
||||
.fetch_html(&self.repository.search_url(&query, page))
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let (data, _pagination) = tokio::task::spawn_blocking(move || {
|
||||
parser::parse_search_page(&html, page).map_err(|e| e.to_string())
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())??;
|
||||
|
||||
let mut final_data = data;
|
||||
apply_cached_posters(
|
||||
&mut final_data,
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(ApiResponse::success(final_data))
|
||||
})
|
||||
.await
|
||||
.map_err(AppError::ScraperError)
|
||||
}
|
||||
|
||||
pub async fn latest(
|
||||
&self,
|
||||
app_state: Arc<AppState>,
|
||||
page: u32,
|
||||
) -> Result<ApiResponse<Vec<crate::shared::types::entities::anime::LatestAnimeItem>>, AppError>
|
||||
{
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
let cache_key = format!("anime2:latest:{}", page);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, LATEST_CACHE_TTL, || async {
|
||||
let html = self
|
||||
.repository
|
||||
.fetch_html(&self.repository.latest_url(page))
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let (data, _pagination) = tokio::task::spawn_blocking(move || {
|
||||
parser::parse_latest_page(&html, page).map_err(|e| e.to_string())
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())??;
|
||||
|
||||
let mut final_data = data;
|
||||
apply_cached_posters(
|
||||
&mut final_data,
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(ApiResponse::success(final_data))
|
||||
})
|
||||
.await
|
||||
.map_err(AppError::ScraperError)
|
||||
}
|
||||
|
||||
pub async fn ongoing_anime(
|
||||
&self,
|
||||
app_state: Arc<AppState>,
|
||||
page: u32,
|
||||
) -> Result<
|
||||
ApiResponse<Vec<crate::shared::types::entities::anime::OngoingAnimeItemWithScore>>,
|
||||
AppError,
|
||||
> {
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
let cache_key = format!("anime2:ongoing:{}", page);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, ONGOING_CACHE_TTL, || async {
|
||||
let html = self
|
||||
.repository
|
||||
.fetch_html(&self.repository.ongoing_url(page))
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let (data, _pagination) = tokio::task::spawn_blocking(move || {
|
||||
parser::parse_ongoing_page(&html, page).map_err(|e| e.to_string())
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())??;
|
||||
|
||||
let mut final_data = data;
|
||||
apply_cached_posters(
|
||||
&mut final_data,
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(ApiResponse::success(final_data))
|
||||
})
|
||||
.await
|
||||
.map_err(AppError::ScraperError)
|
||||
}
|
||||
|
||||
pub async fn complete_anime(
|
||||
&self,
|
||||
app_state: Arc<AppState>,
|
||||
page: u32,
|
||||
) -> Result<ApiResponse<Vec<crate::shared::types::entities::anime::CompleteAnimeItem>>, AppError>
|
||||
{
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
let cache_key = format!("anime2:complete:{}", page);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, COMPLETE_CACHE_TTL, || async {
|
||||
let html = self
|
||||
.repository
|
||||
.fetch_html(&self.repository.complete_url(page))
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let (data, _pagination) = tokio::task::spawn_blocking(move || {
|
||||
parser::parse_complete_page(&html, page).map_err(|e| e.to_string())
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())??;
|
||||
|
||||
let mut final_data = data;
|
||||
apply_cached_posters(
|
||||
&mut final_data,
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(ApiResponse::success(final_data))
|
||||
})
|
||||
.await
|
||||
.map_err(AppError::ScraperError)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
use crate::shared::types::entities::anime::HasPoster;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct Anime2Data {
|
||||
pub ongoing_anime: Vec<crate::shared::types::entities::anime::OngoingAnimeItem>,
|
||||
pub complete_anime: Vec<crate::shared::types::entities::anime::CompleteAnimeItem>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct Anime2Response {
|
||||
pub status: String,
|
||||
pub data: Anime2Data,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct Genre {
|
||||
pub name: String,
|
||||
pub slug: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct GenresResponse {
|
||||
pub status: String,
|
||||
pub data: Vec<Genre>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct FiltersApplied {
|
||||
pub genre: Option<String>,
|
||||
pub status: Option<String>,
|
||||
pub r#type: Option<String>,
|
||||
pub order: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct FilterResponse {
|
||||
pub success: bool,
|
||||
pub data: Vec<crate::shared::types::entities::anime::FilterAnimeItem>,
|
||||
pub pagination: crate::shared::types::entities::anime::Pagination,
|
||||
pub filters_applied: FiltersApplied,
|
||||
pub status: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct AnimeDetailData {
|
||||
pub title: String,
|
||||
pub alternative_title: String,
|
||||
pub poster: String,
|
||||
pub poster2: String,
|
||||
pub r#type: String,
|
||||
pub release_date: String,
|
||||
pub status: String,
|
||||
pub synopsis: String,
|
||||
pub studio: String,
|
||||
pub genres: Vec<DetailGenre>,
|
||||
pub producers: Vec<String>,
|
||||
pub recommendations: Vec<Recommendation>,
|
||||
pub batch: Vec<DownloadItem>,
|
||||
pub ova: Vec<DownloadItem>,
|
||||
pub downloads: Vec<DownloadItem>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct DetailGenre {
|
||||
pub name: String,
|
||||
pub slug: String,
|
||||
pub anime_url: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct Link {
|
||||
pub name: String,
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct DownloadItem {
|
||||
pub resolution: String,
|
||||
pub links: Vec<Link>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct Recommendation {
|
||||
pub title: String,
|
||||
pub slug: String,
|
||||
pub poster: String,
|
||||
pub status: String,
|
||||
pub r#type: String,
|
||||
}
|
||||
|
||||
impl HasPoster for Recommendation {
|
||||
fn poster(&self) -> &str {
|
||||
&self.poster
|
||||
}
|
||||
fn set_poster(&mut self, url: String) {
|
||||
self.poster = url;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct DetailResponse {
|
||||
pub status: String,
|
||||
pub data: AnimeDetailData,
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::{extract::State, Json};
|
||||
|
||||
use crate::modules::komik::repository::KomikRepository;
|
||||
use crate::modules::komik::service::KomikService;
|
||||
use crate::shared::errors::AppError;
|
||||
use crate::shared::state::AppState;
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/komik/genre_list",
|
||||
tag = "komik",
|
||||
operation_id = "komik_genre_list",
|
||||
responses(
|
||||
(status = 200, description = "Handles GET requests for the /api/komik/genre_list endpoint.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn genre_list(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
) -> Result<Json<crate::modules::komik::types::GenresResponse>, AppError> {
|
||||
let service = KomikService::new(KomikRepository::new());
|
||||
service.genre_list(app_state).await.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/komik/chapter/{slug}",
|
||||
tag = "komik",
|
||||
operation_id = "komik_chapter_slug",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves details for a specific chapter by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn chapter_slug(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
axum::extract::Path(slug): axum::extract::Path<String>,
|
||||
) -> Result<Json<crate::modules::komik::types::ChapterResponse>, AppError> {
|
||||
let service = KomikService::new(KomikRepository::new());
|
||||
service.chapter_slug(slug, app_state).await.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/komik/detail/{slug}",
|
||||
tag = "komik",
|
||||
operation_id = "komik_detail_slug",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves details for a specific detail by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn detail_slug(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
axum::extract::Path(slug): axum::extract::Path<String>,
|
||||
) -> Result<Json<crate::modules::komik::types::DetailResponse>, AppError> {
|
||||
let service = KomikService::new(KomikRepository::new());
|
||||
service.detail_slug(slug, app_state).await.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/komik/genre/{slug}",
|
||||
tag = "komik",
|
||||
operation_id = "komik_genre_slug_index",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves details for a specific genre by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn genre_slug(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
axum::extract::Path(slug): axum::extract::Path<String>,
|
||||
) -> Result<Json<crate::modules::komik::types::GenreKomikResponse>, AppError> {
|
||||
let service = KomikService::new(KomikRepository::new());
|
||||
service.genre_slug(slug, app_state).await.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/komik/genre/{slug}/{page}",
|
||||
tag = "komik",
|
||||
operation_id = "komik_genre_slug_page",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves paginated genre results by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn genre_slug_page(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
axum::extract::Path((slug, page)): axum::extract::Path<(String, String)>,
|
||||
) -> Result<Json<crate::modules::komik::types::GenreKomikResponse>, AppError> {
|
||||
let page_num = page
|
||||
.parse::<u32>()
|
||||
.map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?;
|
||||
let service = KomikService::new(KomikRepository::new());
|
||||
service
|
||||
.genre_slug_page(slug, page_num, app_state)
|
||||
.await
|
||||
.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/komik/manga/{slug}",
|
||||
tag = "komik",
|
||||
operation_id = "komik_manga_slug",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves manga details by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn manga_slug(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
axum::extract::Path(slug): axum::extract::Path<String>,
|
||||
) -> Result<Json<crate::modules::komik::types::GenreKomikResponse>, AppError> {
|
||||
let service = KomikService::new(KomikRepository::new());
|
||||
service.manga_slug(slug, app_state).await.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/komik/manhua/{slug}",
|
||||
tag = "komik",
|
||||
operation_id = "komik_manhua_slug",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves manhua details by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn manhua_slug(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
axum::extract::Path(slug): axum::extract::Path<String>,
|
||||
) -> Result<Json<crate::modules::komik::types::GenreKomikResponse>, AppError> {
|
||||
let service = KomikService::new(KomikRepository::new());
|
||||
service.manhua_slug(slug, app_state).await.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/komik/manhwa/{slug}",
|
||||
tag = "komik",
|
||||
operation_id = "komik_manhwa_slug",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves manhwa details by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn manhwa_slug(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
axum::extract::Path(slug): axum::extract::Path<String>,
|
||||
) -> Result<Json<crate::modules::komik::types::GenreKomikResponse>, AppError> {
|
||||
let service = KomikService::new(KomikRepository::new());
|
||||
service.manhwa_slug(slug, app_state).await.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/komik/popular/{slug}",
|
||||
tag = "komik",
|
||||
operation_id = "komik_popular_slug",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves popular komik details by slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn popular_slug(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
axum::extract::Path(slug): axum::extract::Path<String>,
|
||||
) -> Result<Json<crate::modules::komik::types::GenreKomikResponse>, AppError> {
|
||||
let service = KomikService::new(KomikRepository::new());
|
||||
service.popular_slug(slug, app_state).await.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/komik/search/{slug}",
|
||||
tag = "komik",
|
||||
operation_id = "komik_search_slug_index",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves search results by query slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn search_slug(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
axum::extract::Path(slug): axum::extract::Path<String>,
|
||||
) -> Result<Json<crate::modules::komik::types::SearchKomikResponse>, AppError> {
|
||||
let service = KomikService::new(KomikRepository::new());
|
||||
service.search_slug(slug, app_state).await.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/komik/search/{slug}/{page}",
|
||||
tag = "komik",
|
||||
operation_id = "komik_search_slug_page",
|
||||
responses(
|
||||
(status = 200, description = "Retrieves paginated search results by query slug.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn search_slug_page(
|
||||
State(app_state): State<Arc<AppState>>,
|
||||
axum::extract::Path((slug, page)): axum::extract::Path<(String, String)>,
|
||||
) -> Result<Json<crate::modules::komik::types::SearchKomikResponse>, AppError> {
|
||||
let page_num = page
|
||||
.parse::<u32>()
|
||||
.map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?;
|
||||
let service = KomikService::new(KomikRepository::new());
|
||||
service
|
||||
.search_slug_page(slug, page_num, app_state)
|
||||
.await
|
||||
.map(Json)
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
pub mod controller;
|
||||
pub mod parser;
|
||||
pub mod repository;
|
||||
pub mod route;
|
||||
pub mod schema;
|
||||
pub mod service;
|
||||
pub mod types;
|
||||
@@ -0,0 +1,592 @@
|
||||
use crate::modules::komik::types::{ChapterData, DetailData, Genre, KomikItem, Pagination};
|
||||
use crate::shared::utils::parse_html;
|
||||
use crate::shared::utils::scraping::{attr, attr_from, attr_from_or, selector, text, text_from_or};
|
||||
use once_cell::sync::Lazy;
|
||||
use rayon::prelude::*;
|
||||
use regex::Regex;
|
||||
use tracing::info;
|
||||
|
||||
static TD_LAST_SELECTOR: Lazy<scraper::Selector> = Lazy::new(|| selector("td:last-child").unwrap());
|
||||
static TITLE_SELECTOR: Lazy<scraper::Selector> =
|
||||
Lazy::new(|| selector("div#Judul h1 span[itemprop=\"name\"]").unwrap());
|
||||
static H1_SELECTOR: Lazy<scraper::Selector> = Lazy::new(|| selector("h1").unwrap());
|
||||
static TITLE_TAG_SELECTOR: Lazy<scraper::Selector> = Lazy::new(|| selector("title").unwrap());
|
||||
static INFO_ROW_SELECTOR: Lazy<scraper::Selector> =
|
||||
Lazy::new(|| selector("table.inftable tr").unwrap());
|
||||
static POSTER_SELECTOR: Lazy<scraper::Selector> =
|
||||
Lazy::new(|| selector("section#Informasi .ims img").unwrap());
|
||||
static DESC_SELECTOR: Lazy<scraper::Selector> = Lazy::new(|| selector("p.desc").unwrap());
|
||||
static CHAPTER_LIST_SELECTOR: Lazy<scraper::Selector> =
|
||||
Lazy::new(|| selector("#Daftar_Chapter tr, tbody#daftarChapter tr").unwrap());
|
||||
static DATE_LINK_SELECTOR: Lazy<scraper::Selector> =
|
||||
Lazy::new(|| selector("td.tanggalseries, .tanggalseries").unwrap());
|
||||
static JUDUL2_SELECTOR: Lazy<scraper::Selector> = Lazy::new(|| selector("div.judul2").unwrap());
|
||||
static GENRE_SELECTOR: Lazy<scraper::Selector> = Lazy::new(|| selector("ul.genre li a").unwrap());
|
||||
static CHAPTER_LINK_SELECTOR: Lazy<scraper::Selector> =
|
||||
Lazy::new(|| selector("td.judulseries a").unwrap());
|
||||
static CHAPTER_TITLE_REGEX: Lazy<Regex> =
|
||||
Lazy::new(|| Regex::new(r"(?i)(?:chapter|ch\.?)\s*([\d\.]+)").unwrap());
|
||||
static CHAPTER_NUMBER_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"([\d\.]+)").unwrap());
|
||||
|
||||
pub fn parse_genres(html: &str) -> Result<Vec<Genre>, String> {
|
||||
let document = parse_html(html);
|
||||
let mut genres = Vec::new();
|
||||
|
||||
let genre_selector =
|
||||
selector("#Genre .ls3, section#Genre .ls3, .ls3").ok_or("Selector error".to_string())?;
|
||||
let genre_name_selector = selector(".ls3p h4, h4").ok_or("Selector error".to_string())?;
|
||||
let genre_link_selector = selector("a[href*='/genre/']").ok_or("Selector error".to_string())?;
|
||||
let slug_regex = Regex::new(r"/genre/([^/]+)").unwrap();
|
||||
|
||||
for element in document.select(&genre_selector) {
|
||||
let name = text_from_or(&element, &genre_name_selector, "");
|
||||
let href = attr_from(&element, &genre_link_selector, "href").unwrap_or_default();
|
||||
|
||||
let slug = slug_regex
|
||||
.captures(&href)
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if !name.is_empty() && !slug.is_empty() {
|
||||
genres.push(Genre {
|
||||
name,
|
||||
slug,
|
||||
count: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
info!("Parsed {} genres", genres.len());
|
||||
Ok(genres)
|
||||
}
|
||||
|
||||
pub fn parse_komik_chapter_document(html: &str, chapter_url: &str) -> Result<ChapterData, String> {
|
||||
let document = parse_html(html);
|
||||
let _start_time = std::time::Instant::now();
|
||||
info!("Starting to parse komik chapter document");
|
||||
|
||||
let title_selector = selector("title").ok_or("Selector error".to_string())?;
|
||||
let prev_chapter_selector = selector(
|
||||
"a[aria-label='Prev'][href*='chapter'], .nxpr a:not(.rl):not([href*='#Chapter']), .chprev a, a.prev",
|
||||
)
|
||||
.ok_or("Selector error".to_string())?;
|
||||
let next_chapter_selector = selector(
|
||||
"a[aria-label='Next'][href*='chapter'], .nxpr a.rl, .nxpr a.next, .chnext a, a.next",
|
||||
)
|
||||
.ok_or("Selector error".to_string())?;
|
||||
let image_selector =
|
||||
selector("#Baca_Komik img, img.klazy.ww").ok_or("Selector error".to_string())?;
|
||||
|
||||
let title = document
|
||||
.select(&title_selector)
|
||||
.next()
|
||||
.map(|e| {
|
||||
let full_title = text(&e);
|
||||
if let Some(start) = full_title.find("Komik ") {
|
||||
if let Some(end) = full_title.find(" - Komiku") {
|
||||
full_title[start + 6..end].trim().to_string()
|
||||
} else {
|
||||
full_title
|
||||
}
|
||||
} else {
|
||||
full_title
|
||||
}
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let next_chapter_id = document
|
||||
.select(&next_chapter_selector)
|
||||
.next()
|
||||
.and_then(|e| attr(&e, "href"))
|
||||
.map(|href| {
|
||||
href.trim_end_matches('/')
|
||||
.split('/')
|
||||
.filter(|s| !s.is_empty())
|
||||
.next_back()
|
||||
.unwrap_or("")
|
||||
.to_string()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
fn get_previous_chapter_id(chapter_url: &str) -> String {
|
||||
const CHAPTER_PATTERN: &str = "chapter-";
|
||||
|
||||
if let Some(pattern_pos) = chapter_url.rfind(CHAPTER_PATTERN) {
|
||||
let prefix = &chapter_url[0..pattern_pos];
|
||||
let suffix = &chapter_url[pattern_pos + CHAPTER_PATTERN.len()..];
|
||||
|
||||
let chapter_num = suffix
|
||||
.chars()
|
||||
.take_while(|c| c.is_ascii_digit())
|
||||
.collect::<String>();
|
||||
|
||||
if let Ok(num) = chapter_num.parse::<u32>() {
|
||||
let prev_num = num.saturating_sub(1);
|
||||
|
||||
let formatted_num = if chapter_num.starts_with('0') {
|
||||
format!("{:0width$}", prev_num, width = chapter_num.len())
|
||||
} else {
|
||||
prev_num.to_string()
|
||||
};
|
||||
|
||||
return format!("{}{}{}", prefix, CHAPTER_PATTERN, formatted_num);
|
||||
}
|
||||
}
|
||||
|
||||
String::new()
|
||||
}
|
||||
|
||||
let prev_chapter_id_from_url = get_previous_chapter_id(chapter_url);
|
||||
let prev_chapter_id = if !prev_chapter_id_from_url.is_empty() {
|
||||
prev_chapter_id_from_url
|
||||
} else {
|
||||
document
|
||||
.select(&prev_chapter_selector)
|
||||
.next()
|
||||
.and_then(|e| attr(&e, "href"))
|
||||
.map(|href| {
|
||||
href.trim_end_matches('/')
|
||||
.split('/')
|
||||
.filter(|s| !s.is_empty())
|
||||
.next_back()
|
||||
.unwrap_or("")
|
||||
.to_string()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
};
|
||||
|
||||
fn get_list_chapter_from_url(chapter_url: &str) -> String {
|
||||
if let Some(pos) = chapter_url.rfind("-chapter-") {
|
||||
chapter_url[..pos].to_string()
|
||||
} else {
|
||||
chapter_url.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
let list_chapter = get_list_chapter_from_url(chapter_url);
|
||||
|
||||
let mut images = Vec::new();
|
||||
let forbidden_images = [
|
||||
"https://flagcdn.com/32x24/jp.png",
|
||||
"https://flagcdn.com/32x24/kr.png",
|
||||
"https://flagcdn.com/32x24/cn.png",
|
||||
"https://www.gstatic.com/firebasejs/ui/2.0.0/images/auth/google.svg",
|
||||
"https://www.gravatar.com/avatar/?d=mp&s=80",
|
||||
"/asset/img/komikuplus2.jpg",
|
||||
"https://komiku.org/asset/img/Loading.gif",
|
||||
];
|
||||
for el in document.select(&image_selector) {
|
||||
if let Some(src) = attr(&el, "src")
|
||||
.or_else(|| attr(&el, "data-src"))
|
||||
.or_else(|| attr(&el, "data-lazy-src"))
|
||||
.or_else(|| {
|
||||
attr(&el, "srcset").and_then(|s| s.split_whitespace().next().map(|s| s.to_string()))
|
||||
})
|
||||
{
|
||||
if !forbidden_images.contains(&src.as_str()) {
|
||||
images.push(src);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(ChapterData {
|
||||
title,
|
||||
next_chapter_id,
|
||||
prev_chapter_id,
|
||||
list_chapter,
|
||||
images,
|
||||
})
|
||||
}
|
||||
|
||||
fn clean_text(text: String) -> String {
|
||||
text.replace(['\n', '\t'], " ").trim().to_string()
|
||||
}
|
||||
|
||||
fn extract_value_after_keyword(full_text: &str, keywords: &[&str], default_index: usize) -> String {
|
||||
let lower_text = full_text.to_lowercase();
|
||||
for keyword in keywords {
|
||||
if let Some(pos) = lower_text.find(&format!("{}:", keyword)) {
|
||||
return clean_text(full_text[pos + keyword.len() + 1..].to_string());
|
||||
} else if let Some(pos) = lower_text.find(&format!("{} ", keyword)) {
|
||||
return clean_text(full_text[pos + keyword.len() + 1..].to_string());
|
||||
}
|
||||
}
|
||||
if let Some(colon_pos) = full_text.find(':') {
|
||||
return clean_text(full_text[colon_pos + 1..].to_string());
|
||||
}
|
||||
clean_text(full_text[default_index..].to_string())
|
||||
}
|
||||
|
||||
fn find_table_row_with_text<'a>(
|
||||
info_rows: &[scraper::ElementRef<'a>],
|
||||
text_fragments: &[&str],
|
||||
) -> Option<String> {
|
||||
let lower_text_fragments: Vec<String> =
|
||||
text_fragments.iter().map(|&s| s.to_lowercase()).collect();
|
||||
let td_last_selector = &*TD_LAST_SELECTOR;
|
||||
|
||||
info_rows
|
||||
.iter()
|
||||
.find(|row| {
|
||||
let row_text = text(row).to_lowercase();
|
||||
lower_text_fragments
|
||||
.iter()
|
||||
.any(|fragment| row_text.contains(fragment))
|
||||
})
|
||||
.and_then(|row| {
|
||||
text_from_or(row, td_last_selector, "")
|
||||
.trim()
|
||||
.to_string()
|
||||
.into()
|
||||
})
|
||||
}
|
||||
|
||||
pub fn parse_komik_detail_document(html: &str) -> Result<DetailData, String> {
|
||||
let start_time = std::time::Instant::now();
|
||||
info!("Starting to parse komik detail document");
|
||||
|
||||
let document = parse_html(html);
|
||||
|
||||
let title_selector = &*TITLE_SELECTOR;
|
||||
let h1_selector = &*H1_SELECTOR;
|
||||
let title_tag_selector = &*TITLE_TAG_SELECTOR;
|
||||
let info_row_selector = &*INFO_ROW_SELECTOR;
|
||||
let poster_selector = &*POSTER_SELECTOR;
|
||||
let desc_selector = &*DESC_SELECTOR;
|
||||
let chapter_list_selector = &*CHAPTER_LIST_SELECTOR;
|
||||
let date_link_selector = &*DATE_LINK_SELECTOR;
|
||||
let judul2_selector = &*JUDUL2_SELECTOR;
|
||||
let genre_selector = &*GENRE_SELECTOR;
|
||||
let chapter_link_selector = &*CHAPTER_LINK_SELECTOR;
|
||||
|
||||
let title = document
|
||||
.select(&title_selector)
|
||||
.next()
|
||||
.map(|e| {
|
||||
let text = clean_text(text(&e));
|
||||
text.replace("Komik ", "")
|
||||
.replace("Manga ", "")
|
||||
.replace("Manhua ", "")
|
||||
.replace("Manhwa ", "")
|
||||
.trim()
|
||||
.to_string()
|
||||
})
|
||||
.or_else(|| {
|
||||
document
|
||||
.select(&h1_selector)
|
||||
.next()
|
||||
.map(|e| clean_text(text(&e)))
|
||||
})
|
||||
.or_else(|| {
|
||||
document.select(&title_tag_selector).next().map(|e| {
|
||||
let text = clean_text(text(&e));
|
||||
if text.contains("Komik ") {
|
||||
text.replace("Komik ", "").trim().to_string()
|
||||
} else {
|
||||
text.trim().to_string()
|
||||
}
|
||||
})
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let info_rows_vec: Vec<scraper::ElementRef> = document.select(&info_row_selector).collect();
|
||||
let info_rows = &info_rows_vec[..];
|
||||
|
||||
let status = info_rows
|
||||
.iter()
|
||||
.find_map(|&row| {
|
||||
let full_text = text(&row);
|
||||
if full_text.to_lowercase().contains("status") {
|
||||
Some(
|
||||
extract_value_after_keyword(&full_text, &["status"], 0)
|
||||
.replace("Status", "")
|
||||
.replace("Jenis Komik", "")
|
||||
.replace("Type", "")
|
||||
.trim()
|
||||
.to_string(),
|
||||
)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let r#type = info_rows
|
||||
.iter()
|
||||
.find_map(|&row| {
|
||||
let full_text = text(&row);
|
||||
if full_text.to_lowercase().contains("jenis komik")
|
||||
|| full_text.to_lowercase().contains("type")
|
||||
{
|
||||
Some(
|
||||
extract_value_after_keyword(&full_text, &["jenis komik", "type"], 0)
|
||||
.replace("Jenis Komik", "")
|
||||
.replace("Type", "")
|
||||
.trim()
|
||||
.to_string(),
|
||||
)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let author = info_rows
|
||||
.iter()
|
||||
.find_map(|&row| {
|
||||
let full_text = text(&row);
|
||||
if full_text.to_lowercase().contains("pengarang")
|
||||
|| full_text.to_lowercase().contains("author")
|
||||
|| full_text.to_lowercase().contains("artist")
|
||||
{
|
||||
Some(
|
||||
extract_value_after_keyword(&full_text, &["pengarang", "author", "artist"], 0)
|
||||
.replace("Pengarang", "")
|
||||
.replace("Author", "")
|
||||
.replace("pengarang", "")
|
||||
.replace("author", "")
|
||||
.replace("Artist", "")
|
||||
.replace("artist", "")
|
||||
.trim()
|
||||
.to_string(),
|
||||
)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let poster = document
|
||||
.select(&poster_selector)
|
||||
.next()
|
||||
.and_then(|e| attr(&e, "src"))
|
||||
.map(|s| s.split('?').next().unwrap_or(&s).to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let description = document
|
||||
.select(&desc_selector)
|
||||
.map(|e| clean_text(text(&e)))
|
||||
.filter(|t| t.len() > 50)
|
||||
.collect::<Vec<String>>()
|
||||
.join("\n")
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
let release_date = find_table_row_with_text(info_rows, &["tanggal rilis", "release date"])
|
||||
.map(clean_text)
|
||||
.unwrap_or_else(|| {
|
||||
document
|
||||
.select(&chapter_list_selector)
|
||||
.next_back()
|
||||
.and_then(|last| last.select(&date_link_selector).next())
|
||||
.map(|e| clean_text(text(&e)))
|
||||
.unwrap_or_default()
|
||||
});
|
||||
|
||||
let total_chapter = find_table_row_with_text(info_rows, &["total chapter", "total chapters"])
|
||||
.unwrap_or_else(|| {
|
||||
let count = document.select(&chapter_list_selector).count();
|
||||
if count > 0 {
|
||||
count.to_string()
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
});
|
||||
|
||||
let updated_on = find_table_row_with_text(info_rows, &["diperbarui", "updated"])
|
||||
.or_else(|| {
|
||||
document.select(&judul2_selector).next().map(|e| {
|
||||
let text_str = clean_text(text(&e));
|
||||
text_str.split("• ").nth(1).unwrap_or("").trim().to_string()
|
||||
})
|
||||
})
|
||||
.unwrap_or_else(|| {
|
||||
document
|
||||
.select(&chapter_list_selector)
|
||||
.next()
|
||||
.and_then(|first| first.select(&date_link_selector).next())
|
||||
.map(|e| clean_text(text(&e)))
|
||||
.unwrap_or_default()
|
||||
});
|
||||
|
||||
let mut genres = Vec::new();
|
||||
for element in document.select(&genre_selector) {
|
||||
let genre = clean_text(text(&element));
|
||||
if !genre.is_empty() {
|
||||
genres.push(genre);
|
||||
}
|
||||
}
|
||||
|
||||
let raw_chapter_data: Vec<(String, String, String)> = document
|
||||
.select(&chapter_list_selector)
|
||||
.filter_map(|el| {
|
||||
let chapter_link_element = el.select(&chapter_link_selector).next();
|
||||
let date_element = el.select(&date_link_selector).next();
|
||||
|
||||
let chapter_text = chapter_link_element
|
||||
.as_ref()
|
||||
.map(|e| clean_text(text(e)))
|
||||
.unwrap_or_default();
|
||||
|
||||
let date_text = date_element
|
||||
.map(|e| clean_text(text(&e)))
|
||||
.unwrap_or_default();
|
||||
|
||||
let href_text = chapter_link_element
|
||||
.and_then(|e| attr(&e, "href"))
|
||||
.unwrap_or_default();
|
||||
|
||||
if !chapter_text.is_empty() || !date_text.is_empty() || !href_text.is_empty() {
|
||||
Some((chapter_text, date_text, href_text))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let chapters: Vec<crate::modules::komik::types::Chapter> = raw_chapter_data
|
||||
.par_iter()
|
||||
.filter_map(|(chapter_text, date_text, href_text)| {
|
||||
let chapter = {
|
||||
let trimmed_chapter_text = chapter_text.trim();
|
||||
if let Some(captures) = CHAPTER_TITLE_REGEX.captures(trimmed_chapter_text) {
|
||||
captures
|
||||
.get(1)
|
||||
.map_or(trimmed_chapter_text.to_string(), |m| m.as_str().to_string())
|
||||
} else if let Some(captures) = CHAPTER_NUMBER_REGEX.captures(trimmed_chapter_text) {
|
||||
captures
|
||||
.get(1)
|
||||
.map_or(trimmed_chapter_text.to_string(), |m| m.as_str().to_string())
|
||||
} else {
|
||||
trimmed_chapter_text.to_string()
|
||||
}
|
||||
};
|
||||
|
||||
let date = date_text.trim().to_string();
|
||||
|
||||
let chapter_id = href_text
|
||||
.split('/')
|
||||
.filter(|s| !s.is_empty())
|
||||
.next_back()
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if !chapter_id.is_empty() {
|
||||
Some(crate::modules::komik::types::Chapter {
|
||||
chapter,
|
||||
date,
|
||||
chapter_id,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let duration = start_time.elapsed();
|
||||
info!("Parsed komik detail document in {:?}", duration);
|
||||
|
||||
Ok(DetailData {
|
||||
title,
|
||||
poster,
|
||||
description,
|
||||
status,
|
||||
r#type,
|
||||
release_date,
|
||||
author,
|
||||
total_chapter,
|
||||
updated_on,
|
||||
genres,
|
||||
chapters,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn parse_genre_page(
|
||||
html: &str,
|
||||
current_page: u32,
|
||||
) -> Result<(Vec<KomikItem>, Pagination), String> {
|
||||
let document = parse_html(html);
|
||||
let mut komik_list = Vec::new();
|
||||
|
||||
let item_selector =
|
||||
selector(".bge, article, .ls4, .ls2").ok_or("Selector error".to_string())?;
|
||||
let title_selector = selector(".kan h3, h3 a, h4 a").ok_or("Selector error".to_string())?;
|
||||
let img_selector = selector(".bgei img, img.lazy, img").ok_or("Selector error".to_string())?;
|
||||
let chapter_selector = selector(".new1:last-of-type a span:last-child, .new1 a:last-child span:last-child, .ls4s a, .ls24, .ls2l a")
|
||||
.ok_or("Selector error".to_string())?;
|
||||
let score_selector = selector(".up, .numscore, .epx").ok_or("Selector error".to_string())?;
|
||||
let type_selector = selector(".tpe1_inf, .ls3p, .type").ok_or("Selector error".to_string())?;
|
||||
let link_selector =
|
||||
selector(".kan h3 a, .bgei a, h3 a, h4 a, a").ok_or("Selector error".to_string())?;
|
||||
let next_selector = selector("span[hx-get]").ok_or("Selector error".to_string())?;
|
||||
let slug_regex = Regex::new(r"/([^/]+)/?$").unwrap();
|
||||
|
||||
for element in document.select(&item_selector) {
|
||||
let title = text_from_or(&element, &title_selector, "");
|
||||
|
||||
let poster = element
|
||||
.select(&img_selector)
|
||||
.next()
|
||||
.and_then(|e| attr(&e, "data-src").or(attr(&e, "src")))
|
||||
.unwrap_or_else(|| "".to_string())
|
||||
.to_string();
|
||||
|
||||
let chapter = text_from_or(&element, &chapter_selector, "N/A");
|
||||
|
||||
let score = text_from_or(&element, &score_selector, "N/A");
|
||||
|
||||
let komik_type = text_from_or(&element, &type_selector, "Unknown")
|
||||
.split_whitespace()
|
||||
.next()
|
||||
.unwrap_or("Unknown")
|
||||
.to_string();
|
||||
|
||||
let komik_url = attr_from_or(&element, &link_selector, "href", "");
|
||||
|
||||
let slug = slug_regex
|
||||
.captures(&komik_url)
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if !title.is_empty() {
|
||||
komik_list.push(KomikItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
chapter,
|
||||
score,
|
||||
r#type: komik_type,
|
||||
komik_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let has_next_page = document.select(&next_selector).next().is_some();
|
||||
let last_visible_page = if has_next_page {
|
||||
current_page + 1
|
||||
} else {
|
||||
current_page
|
||||
};
|
||||
let pagination = Pagination {
|
||||
current_page,
|
||||
last_visible_page,
|
||||
has_next_page,
|
||||
next_page: if has_next_page {
|
||||
Some(current_page + 1)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
has_previous_page: current_page > 1,
|
||||
previous_page: if current_page > 1 {
|
||||
Some(current_page - 1)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
};
|
||||
|
||||
Ok((komik_list, pagination))
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
use crate::shared::database::traits::scraping_repository::ScrapingRepository;
|
||||
use crate::shared::errors::AppError;
|
||||
use crate::shared::utils::fetch_html_with_retry;
|
||||
use crate::shared::utils::web::scraping_urls::{get_komik_api_url, get_komik_url};
|
||||
use async_trait::async_trait;
|
||||
|
||||
pub struct KomikRepository;
|
||||
|
||||
impl Default for KomikRepository {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl KomikRepository {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
pub fn api_url(&self) -> String {
|
||||
get_komik_api_url()
|
||||
}
|
||||
|
||||
pub fn base_url(&self) -> String {
|
||||
get_komik_url()
|
||||
}
|
||||
|
||||
pub fn genre_url(&self, genre_slug: &str, page: u32) -> String {
|
||||
if page == 1 {
|
||||
format!("{}/genre/{}/", self.api_url(), genre_slug)
|
||||
} else {
|
||||
format!("{}/genre/{}/page/{}/", self.api_url(), genre_slug, page)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn search_url(&self, query: &str, page: u32) -> String {
|
||||
if page == 1 {
|
||||
format!("{}/search/{}/", self.api_url(), query)
|
||||
} else {
|
||||
format!("{}/search/{}/page/{}/", self.api_url(), query, page)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn manga_list_url(&self, page: u32) -> String {
|
||||
format!("{}/manga/page/{}/?tipe=manga", self.api_url(), page)
|
||||
}
|
||||
|
||||
pub fn manhua_list_url(&self, page: u32) -> String {
|
||||
format!("{}/manga/page/{}/?tipe=manhua", self.api_url(), page)
|
||||
}
|
||||
|
||||
pub fn manhwa_list_url(&self, page: u32) -> String {
|
||||
format!("{}/manga/page/{}/?tipe=manhwa", self.api_url(), page)
|
||||
}
|
||||
|
||||
pub fn popular_list_url(&self, page: u32) -> String {
|
||||
format!(
|
||||
"{}/manga/page/{}/?orderby=meta_value_num",
|
||||
self.api_url(),
|
||||
page
|
||||
)
|
||||
}
|
||||
|
||||
pub fn detail_url(&self, slug: &str) -> String {
|
||||
format!("{}/manga/{}/", self.base_url(), slug)
|
||||
}
|
||||
|
||||
pub fn chapter_url(&self, chapter_url: &str) -> String {
|
||||
format!("{}/{}", self.base_url(), chapter_url)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ScrapingRepository for KomikRepository {
|
||||
async fn fetch_html(&self, url: &str) -> Result<String, AppError> {
|
||||
fetch_html_with_retry(url).await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::{routing::get, Router};
|
||||
|
||||
use crate::modules::komik::controller;
|
||||
use crate::shared::state::AppState;
|
||||
|
||||
pub fn routes(router: Router<Arc<AppState>>) -> Router<Arc<AppState>> {
|
||||
router
|
||||
.route("/api/komik/genre_list", get(controller::genre_list))
|
||||
.route("/api/komik/chapter/{slug}", get(controller::chapter_slug))
|
||||
.route("/api/komik/detail/{slug}", get(controller::detail_slug))
|
||||
.route("/api/komik/genre/{slug}", get(controller::genre_slug))
|
||||
.route(
|
||||
"/api/komik/genre/{slug}/{page}",
|
||||
get(controller::genre_slug_page),
|
||||
)
|
||||
.route("/api/komik/manga/{slug}", get(controller::manga_slug))
|
||||
.route("/api/komik/manhua/{slug}", get(controller::manhua_slug))
|
||||
.route("/api/komik/manhwa/{slug}", get(controller::manhwa_slug))
|
||||
.route("/api/komik/popular/{slug}", get(controller::popular_slug))
|
||||
.route("/api/komik/search/{slug}", get(controller::search_slug))
|
||||
.route(
|
||||
"/api/komik/search/{slug}/{page}",
|
||||
get(controller::search_slug_page),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
use serde::Deserialize;
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct SlugPath {
|
||||
pub slug: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct SlugPagePath {
|
||||
pub slug: String,
|
||||
pub page: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct ChapterQuery {
|
||||
/// URL-friendly identifier for the chapter (typically the chapter slug or URL path)
|
||||
pub chapter_url: Option<String>,
|
||||
}
|
||||
@@ -0,0 +1,421 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::modules::komik::parser;
|
||||
use crate::modules::komik::repository::KomikRepository;
|
||||
use crate::modules::komik::types::{
|
||||
ChapterResponse, DetailResponse, GenreKomikResponse, GenresResponse, SearchKomikResponse,
|
||||
};
|
||||
use crate::shared::database::traits::scraping_repository::ScrapingRepository;
|
||||
use crate::shared::errors::AppError;
|
||||
use crate::shared::services::images::cache::{
|
||||
apply_cached_posters, cache_image_urls_batch_lazy, get_cached_or_original,
|
||||
};
|
||||
use crate::shared::state::AppState;
|
||||
use crate::shared::utils::Cache;
|
||||
|
||||
const GENRE_LIST_CACHE_TTL: u64 = 3600;
|
||||
const GENRE_CACHE_TTL: u64 = 300;
|
||||
const DETAIL_CACHE_TTL: u64 = 300;
|
||||
const CHAPTER_CACHE_TTL: u64 = 300;
|
||||
const SEARCH_CACHE_TTL: u64 = 300;
|
||||
|
||||
pub struct KomikService {
|
||||
repository: KomikRepository,
|
||||
}
|
||||
|
||||
impl KomikService {
|
||||
pub fn new(repository: KomikRepository) -> Self {
|
||||
Self { repository }
|
||||
}
|
||||
|
||||
pub async fn genre_list(&self, app_state: Arc<AppState>) -> Result<GenresResponse, AppError> {
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
let cache_key = "komik:genres:list:v3";
|
||||
|
||||
cache
|
||||
.get_or_set(cache_key, GENRE_LIST_CACHE_TTL, || async {
|
||||
let html = self
|
||||
.repository
|
||||
.fetch_html(&self.repository.api_url())
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let genres = tokio::task::spawn_blocking(move || parser::parse_genres(&html))
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(GenresResponse {
|
||||
status: "Ok".to_string(),
|
||||
data: genres,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(AppError::ScraperError)
|
||||
}
|
||||
|
||||
pub async fn genre_slug(
|
||||
&self,
|
||||
genre_slug: String,
|
||||
app_state: Arc<AppState>,
|
||||
) -> Result<GenreKomikResponse, AppError> {
|
||||
let page = 1;
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
let cache_key = format!("komik:genre:{}:{}:v2", genre_slug, page);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, GENRE_CACHE_TTL, || async {
|
||||
let url = self.repository.genre_url(&genre_slug, page);
|
||||
let html = self
|
||||
.repository
|
||||
.fetch_html(&url)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let (mut komik_list, pagination) =
|
||||
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
apply_cached_posters(
|
||||
&mut komik_list,
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(GenreKomikResponse {
|
||||
status: "Ok".to_string(),
|
||||
genre: genre_slug.clone(),
|
||||
data: komik_list,
|
||||
pagination,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(AppError::ScraperError)
|
||||
}
|
||||
|
||||
pub async fn genre_slug_page(
|
||||
&self,
|
||||
genre_slug: String,
|
||||
page: u32,
|
||||
app_state: Arc<AppState>,
|
||||
) -> Result<GenreKomikResponse, AppError> {
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
let cache_key = format!("komik:genre:{}:{}:v2", genre_slug, page);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, GENRE_CACHE_TTL, || async {
|
||||
let url = self.repository.genre_url(&genre_slug, page);
|
||||
let html = self
|
||||
.repository
|
||||
.fetch_html(&url)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let (mut komik_list, pagination) =
|
||||
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
apply_cached_posters(
|
||||
&mut komik_list,
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(GenreKomikResponse {
|
||||
status: "Ok".to_string(),
|
||||
genre: genre_slug.clone(),
|
||||
data: komik_list,
|
||||
pagination,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(AppError::ScraperError)
|
||||
}
|
||||
|
||||
pub async fn detail_slug(
|
||||
&self,
|
||||
komik_id: String,
|
||||
app_state: Arc<AppState>,
|
||||
) -> Result<DetailResponse, AppError> {
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
let cache_key = format!("komik:detail:{}", komik_id);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, DETAIL_CACHE_TTL, || async {
|
||||
let url = self.repository.detail_url(&komik_id);
|
||||
let html = self
|
||||
.repository
|
||||
.fetch_html(&url)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let mut data =
|
||||
tokio::task::spawn_blocking(move || parser::parse_komik_detail_document(&html))
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
if !data.poster.is_empty() {
|
||||
data.poster = get_cached_or_original(
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
&data.poster,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
Ok(DetailResponse { status: true, data })
|
||||
})
|
||||
.await
|
||||
.map_err(AppError::ScraperError)
|
||||
}
|
||||
|
||||
pub async fn chapter_slug(
|
||||
&self,
|
||||
chapter_url: String,
|
||||
app_state: Arc<AppState>,
|
||||
) -> Result<ChapterResponse, AppError> {
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
let cache_key = format!("komik:chapter:{}", chapter_url);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, CHAPTER_CACHE_TTL, || async {
|
||||
let url = self.repository.chapter_url(&chapter_url);
|
||||
let html = self
|
||||
.repository
|
||||
.fetch_html(&url)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let mut data = tokio::task::spawn_blocking({
|
||||
let chapter_url = chapter_url.clone();
|
||||
move || parser::parse_komik_chapter_document(&html, &chapter_url)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
data.images = cache_image_urls_batch_lazy(
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
data.images,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(ChapterResponse {
|
||||
message: "Ok".to_string(),
|
||||
data,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(AppError::ScraperError)
|
||||
}
|
||||
|
||||
pub async fn manga_slug(
|
||||
&self,
|
||||
page_slug: String,
|
||||
app_state: Arc<AppState>,
|
||||
) -> Result<GenreKomikResponse, AppError> {
|
||||
let page = page_slug
|
||||
.parse::<u32>()
|
||||
.map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?;
|
||||
self.list_by_url(
|
||||
"manga",
|
||||
page,
|
||||
self.repository.manga_list_url(page),
|
||||
app_state,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn manhua_slug(
|
||||
&self,
|
||||
page_slug: String,
|
||||
app_state: Arc<AppState>,
|
||||
) -> Result<GenreKomikResponse, AppError> {
|
||||
let page = page_slug
|
||||
.parse::<u32>()
|
||||
.map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?;
|
||||
self.list_by_url(
|
||||
"manhua",
|
||||
page,
|
||||
self.repository.manhua_list_url(page),
|
||||
app_state,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn manhwa_slug(
|
||||
&self,
|
||||
page_slug: String,
|
||||
app_state: Arc<AppState>,
|
||||
) -> Result<GenreKomikResponse, AppError> {
|
||||
let page = page_slug
|
||||
.parse::<u32>()
|
||||
.map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?;
|
||||
self.list_by_url(
|
||||
"manhwa",
|
||||
page,
|
||||
self.repository.manhwa_list_url(page),
|
||||
app_state,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn popular_slug(
|
||||
&self,
|
||||
page_slug: String,
|
||||
app_state: Arc<AppState>,
|
||||
) -> Result<GenreKomikResponse, AppError> {
|
||||
let page = page_slug
|
||||
.parse::<u32>()
|
||||
.map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?;
|
||||
self.list_by_url(
|
||||
"popular",
|
||||
page,
|
||||
self.repository.popular_list_url(page),
|
||||
app_state,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_by_url(
|
||||
&self,
|
||||
list_name: &str,
|
||||
page: u32,
|
||||
url: String,
|
||||
app_state: Arc<AppState>,
|
||||
) -> Result<GenreKomikResponse, AppError> {
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
let cache_key = format!("komik:list:{}:{}:v2", list_name, page);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, GENRE_CACHE_TTL, || async {
|
||||
let html = self
|
||||
.repository
|
||||
.fetch_html(&url)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let (mut komik_list, pagination) =
|
||||
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
if komik_list.is_empty() {
|
||||
return Err(format!("Empty komik {} page {}", list_name, page));
|
||||
}
|
||||
|
||||
apply_cached_posters(
|
||||
&mut komik_list,
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(GenreKomikResponse {
|
||||
status: "Ok".to_string(),
|
||||
genre: list_name.to_string(),
|
||||
data: komik_list,
|
||||
pagination,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(AppError::ScraperError)
|
||||
}
|
||||
|
||||
pub async fn search_slug(
|
||||
&self,
|
||||
query: String,
|
||||
app_state: Arc<AppState>,
|
||||
) -> Result<SearchKomikResponse, AppError> {
|
||||
let page = 1;
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
let cache_key = format!("komik:search:{}:{}", query, page);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, SEARCH_CACHE_TTL, || async {
|
||||
let url = self.repository.search_url(&query, page);
|
||||
let html = self
|
||||
.repository
|
||||
.fetch_html(&url)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let (mut komik_list, pagination) =
|
||||
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
apply_cached_posters(
|
||||
&mut komik_list,
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(SearchKomikResponse {
|
||||
status: "Ok".to_string(),
|
||||
data: komik_list,
|
||||
pagination,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(AppError::ScraperError)
|
||||
}
|
||||
|
||||
pub async fn search_slug_page(
|
||||
&self,
|
||||
query: String,
|
||||
page: u32,
|
||||
app_state: Arc<AppState>,
|
||||
) -> Result<SearchKomikResponse, AppError> {
|
||||
let cache = Cache::new(&app_state.redis_pool);
|
||||
let cache_key = format!("komik:search:{}:{}", query, page);
|
||||
|
||||
cache
|
||||
.get_or_set(&cache_key, SEARCH_CACHE_TTL, || async {
|
||||
let url = self.repository.search_url(&query, page);
|
||||
let html = self
|
||||
.repository
|
||||
.fetch_html(&url)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let (mut komik_list, pagination) =
|
||||
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
apply_cached_posters(
|
||||
&mut komik_list,
|
||||
app_state.db.clone(),
|
||||
&app_state.redis_pool,
|
||||
Some(app_state.image_processing_semaphore.clone()),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(SearchKomikResponse {
|
||||
status: "Ok".to_string(),
|
||||
data: komik_list,
|
||||
pagination,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(AppError::ScraperError)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
use crate::shared::types::entities::anime::HasPoster;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct Genre {
|
||||
pub name: String,
|
||||
pub slug: String,
|
||||
pub count: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct GenresResponse {
|
||||
pub status: String,
|
||||
pub data: Vec<Genre>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct ChapterData {
|
||||
pub title: String,
|
||||
pub next_chapter_id: String,
|
||||
pub prev_chapter_id: String,
|
||||
pub list_chapter: String,
|
||||
pub images: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct ChapterResponse {
|
||||
pub message: String,
|
||||
pub data: ChapterData,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct Chapter {
|
||||
pub chapter: String,
|
||||
pub date: String,
|
||||
pub chapter_id: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct DetailData {
|
||||
pub title: String,
|
||||
pub poster: String,
|
||||
pub description: String,
|
||||
pub status: String,
|
||||
pub r#type: String,
|
||||
pub release_date: String,
|
||||
pub author: String,
|
||||
pub total_chapter: String,
|
||||
pub updated_on: String,
|
||||
pub genres: Vec<String>,
|
||||
pub chapters: Vec<Chapter>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct DetailResponse {
|
||||
pub status: bool,
|
||||
pub data: DetailData,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, ToSchema)]
|
||||
pub struct KomikDetailRequest {
|
||||
pub komik_id: String,
|
||||
pub chapter_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Clone, ToSchema)]
|
||||
pub enum KomikDetailEvent {
|
||||
Chapter(Chapter),
|
||||
Detail(DetailData),
|
||||
Error(String),
|
||||
EndOfStream,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct KomikItem {
|
||||
pub title: String,
|
||||
pub slug: String,
|
||||
pub poster: String,
|
||||
pub chapter: String,
|
||||
pub score: String,
|
||||
pub r#type: String,
|
||||
pub komik_url: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct Pagination {
|
||||
pub current_page: u32,
|
||||
pub last_visible_page: u32,
|
||||
pub has_next_page: bool,
|
||||
pub next_page: Option<u32>,
|
||||
pub has_previous_page: bool,
|
||||
pub previous_page: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct GenreKomikResponse {
|
||||
pub status: String,
|
||||
pub genre: String,
|
||||
pub data: Vec<KomikItem>,
|
||||
pub pagination: Pagination,
|
||||
}
|
||||
|
||||
impl HasPoster for KomikItem {
|
||||
fn poster(&self) -> &str {
|
||||
&self.poster
|
||||
}
|
||||
fn set_poster(&mut self, url: String) {
|
||||
self.poster = url;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
||||
pub struct SearchKomikResponse {
|
||||
pub status: String,
|
||||
pub data: Vec<KomikItem>,
|
||||
pub pagination: Pagination,
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
use axum::Router;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::shared::state::AppState;
|
||||
|
||||
pub mod anime;
|
||||
pub mod anime2;
|
||||
pub mod komik;
|
||||
pub mod proxy;
|
||||
|
||||
pub fn routes(router: Router<Arc<AppState>>) -> Router<Arc<AppState>> {
|
||||
let router = anime::route::routes(router);
|
||||
let router = anime2::route::routes(router);
|
||||
let router = komik::route::routes(router);
|
||||
let router = proxy::route::routes(router);
|
||||
router
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
use crate::modules::proxy::repository::ProxyRepository;
|
||||
use crate::modules::proxy::schema::{AuditImageCacheRequest, ImageCacheRequest, ProxyParams};
|
||||
use crate::modules::proxy::service::ProxyService;
|
||||
use crate::modules::proxy::types::{AuditImageCacheResponse, ImageCacheResponse};
|
||||
use crate::shared::database::repositories::image_cache::SeaOrmImageCacheRepository;
|
||||
use crate::shared::errors::AppError;
|
||||
use crate::shared::state::AppState;
|
||||
use axum::extract::{Json, Query, State};
|
||||
use axum::response::Response;
|
||||
use std::sync::Arc;
|
||||
|
||||
fn make_service(state: &Arc<AppState>) -> ProxyService {
|
||||
let repo = Arc::new(SeaOrmImageCacheRepository::new(
|
||||
state.db.clone(),
|
||||
state.redis_pool.clone(),
|
||||
));
|
||||
ProxyService::new(ProxyRepository::new(), repo)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/proxy/croxy",
|
||||
tag = "proxy",
|
||||
operation_id = "proxy_croxy",
|
||||
params(ProxyParams),
|
||||
responses(
|
||||
(status = 200, description = "Handles GET requests for the /api/proxy/croxy endpoint.", body = serde_json::Value),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn fetch_with_proxy_only(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Query(params): Query<ProxyParams>,
|
||||
) -> Result<Response, AppError> {
|
||||
make_service(&state).fetch_with_proxy_only(params).await
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
post,
|
||||
path = "/api/proxy/image-cache",
|
||||
tag = "proxy",
|
||||
operation_id = "proxy_image_cache",
|
||||
request_body = ImageCacheRequest,
|
||||
responses(
|
||||
(status = 200, description = "Cache an image to CDN and return the cached URL", body = ImageCacheResponse),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn image_cache(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Json(req): Json<ImageCacheRequest>,
|
||||
) -> Result<Json<ImageCacheResponse>, AppError> {
|
||||
make_service(&state).image_cache(state, req).await.map(Json)
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
post,
|
||||
path = "/api/proxy/image-cache/audit",
|
||||
tag = "proxy",
|
||||
operation_id = "proxy_image_cache_audit",
|
||||
request_body = AuditImageCacheRequest,
|
||||
responses(
|
||||
(status = 200, description = "Audit an image cache entry", body = AuditImageCacheResponse),
|
||||
(status = 500, description = "Internal Server Error", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn audit_image_cache(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Json(req): Json<AuditImageCacheRequest>,
|
||||
) -> Result<Json<AuditImageCacheResponse>, AppError> {
|
||||
make_service(&state)
|
||||
.audit_image_cache(state, req)
|
||||
.await
|
||||
.map(Json)
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
pub mod controller;
|
||||
pub mod parser;
|
||||
pub mod repository;
|
||||
pub mod route;
|
||||
pub mod schema;
|
||||
pub mod service;
|
||||
pub mod types;
|
||||
@@ -0,0 +1 @@
|
||||
// Proxy endpoints do not parse HTML or structured upstream payloads.
|
||||
@@ -0,0 +1,29 @@
|
||||
use crate::shared::database::traits::scraping_repository::ScrapingRepository;
|
||||
use crate::shared::errors::AppError;
|
||||
use crate::shared::utils::web::proxy_fetch::{self, FetchResult};
|
||||
use async_trait::async_trait;
|
||||
|
||||
pub struct ProxyRepository;
|
||||
|
||||
impl Default for ProxyRepository {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl ProxyRepository {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
pub async fn fetch_with_proxy_url(&self, url: &str) -> Result<FetchResult, AppError> {
|
||||
proxy_fetch::fetch_with_proxy(url).await
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ScrapingRepository for ProxyRepository {
|
||||
async fn fetch_html(&self, url: &str) -> Result<String, AppError> {
|
||||
self.fetch_with_proxy_url(url).await.map(|r| r.data)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
use crate::modules::proxy::controller;
|
||||
use crate::shared::state::AppState;
|
||||
use axum::Router;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub fn routes(router: Router<Arc<AppState>>) -> Router<Arc<AppState>> {
|
||||
router
|
||||
.route(
|
||||
"/api/proxy/croxy",
|
||||
axum::routing::get(controller::fetch_with_proxy_only),
|
||||
)
|
||||
.route(
|
||||
"/api/proxy/image-cache",
|
||||
axum::routing::post(controller::image_cache),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
use serde::Deserialize;
|
||||
use utoipa::IntoParams;
|
||||
use utoipa::ToSchema;
|
||||
|
||||
/// Query parameters for proxy fetch (GET)
|
||||
#[derive(Debug, Deserialize, ToSchema, IntoParams)]
|
||||
pub struct ProxyParams {
|
||||
/// URL to fetch via proxy
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
/// Request body for image cache (POST)
|
||||
#[derive(Debug, Deserialize, ToSchema)]
|
||||
pub struct ImageCacheRequest {
|
||||
/// Original image URL to cache
|
||||
pub url: String,
|
||||
/// If true, returns original URL immediately and caches in background
|
||||
#[serde(default)]
|
||||
pub lazy: bool,
|
||||
}
|
||||
|
||||
/// Request body for auditing image cache (POST)
|
||||
#[derive(Debug, Deserialize, ToSchema)]
|
||||
pub struct AuditImageCacheRequest {
|
||||
/// Original image URL to audit
|
||||
pub url: String,
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
use crate::modules::proxy::repository::ProxyRepository;
|
||||
use crate::modules::proxy::schema::{AuditImageCacheRequest, ImageCacheRequest, ProxyParams};
|
||||
use crate::modules::proxy::types::{AuditImageCacheResponse, ImageCacheResponse};
|
||||
use crate::shared::config::CONFIG;
|
||||
use crate::shared::database::traits::image_cache::ImageCacheRepository;
|
||||
use crate::shared::errors::AppError;
|
||||
use crate::shared::events::bus::ImageRepaired;
|
||||
use crate::shared::services::images::cache::ImageCache;
|
||||
use crate::shared::state::AppState;
|
||||
use axum::http::StatusCode;
|
||||
use axum::response::Response;
|
||||
use std::sync::Arc;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
pub struct ProxyService {
|
||||
repository: ProxyRepository,
|
||||
image_cache_repo: Arc<dyn ImageCacheRepository>,
|
||||
}
|
||||
|
||||
impl ProxyService {
|
||||
pub fn new(
|
||||
repository: ProxyRepository,
|
||||
image_cache_repo: Arc<dyn ImageCacheRepository>,
|
||||
) -> Self {
|
||||
Self {
|
||||
repository,
|
||||
image_cache_repo,
|
||||
}
|
||||
}
|
||||
|
||||
fn build_image_cache(&self) -> ImageCache {
|
||||
ImageCache::new(self.image_cache_repo.clone())
|
||||
}
|
||||
|
||||
pub async fn fetch_with_proxy_only(&self, params: ProxyParams) -> Result<Response, AppError> {
|
||||
let url = params.url;
|
||||
match self.repository.fetch_with_proxy_url(&url).await {
|
||||
Ok(fetch_result) => {
|
||||
let mut builder = Response::builder().status(StatusCode::OK);
|
||||
if let Some(content_type) = fetch_result.content_type {
|
||||
builder = builder.header("Content-Type", content_type);
|
||||
}
|
||||
Ok(builder.body(fetch_result.data.into())?)
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Proxy fetch error: {:?}", e);
|
||||
Err(AppError::Other(format!(
|
||||
"Failed to fetch URL via proxy: {}",
|
||||
e
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn image_cache(
|
||||
&self,
|
||||
state: Arc<AppState>,
|
||||
req: ImageCacheRequest,
|
||||
) -> Result<ImageCacheResponse, AppError> {
|
||||
let cache = self
|
||||
.build_image_cache()
|
||||
.with_semaphore(state.image_processing_semaphore.clone());
|
||||
|
||||
if let Some(cdn_url) = cache.get_cdn_url(&req.url).await {
|
||||
return Ok(ImageCacheResponse {
|
||||
success: true,
|
||||
original_url: req.url,
|
||||
cdn_url,
|
||||
from_cache: true,
|
||||
pending: None,
|
||||
});
|
||||
}
|
||||
|
||||
if req.lazy {
|
||||
let url = req.url.clone();
|
||||
let repo = self.image_cache_repo.clone();
|
||||
let semaphore = state.image_processing_semaphore.clone();
|
||||
tokio::spawn(async move {
|
||||
let cache = ImageCache::new(repo).with_semaphore(semaphore);
|
||||
match cache.get_or_cache(&url).await {
|
||||
Ok(cdn) => info!("[LazyCache] Cached {} -> {}", url, cdn),
|
||||
Err(e) => warn!("[LazyCache] Failed {}: {}", url, e),
|
||||
}
|
||||
});
|
||||
return Ok(ImageCacheResponse {
|
||||
success: true,
|
||||
original_url: req.url.clone(),
|
||||
cdn_url: req.url,
|
||||
from_cache: false,
|
||||
pending: Some(true),
|
||||
});
|
||||
}
|
||||
|
||||
match cache.get_or_cache(&req.url).await {
|
||||
Ok(cdn_url) => Ok(ImageCacheResponse {
|
||||
success: true,
|
||||
original_url: req.url,
|
||||
cdn_url,
|
||||
from_cache: false,
|
||||
pending: None,
|
||||
}),
|
||||
Err(e) => {
|
||||
error!("ImageCache error: {}", e);
|
||||
Ok(ImageCacheResponse {
|
||||
success: false,
|
||||
original_url: req.url.clone(),
|
||||
cdn_url: req.url,
|
||||
from_cache: false,
|
||||
pending: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn audit_image_cache(
|
||||
&self,
|
||||
state: Arc<AppState>,
|
||||
req: AuditImageCacheRequest,
|
||||
) -> Result<AuditImageCacheResponse, AppError> {
|
||||
let cache = self.build_image_cache();
|
||||
|
||||
let mut cdn_opt = cache.get_cdn_url(&req.url).await;
|
||||
let mut original = req.url.clone();
|
||||
if cdn_opt.is_none() {
|
||||
if let Some(orig) = cache.find_original_from_cdn(&req.url).await {
|
||||
info!(
|
||||
"SmartAudit: {} recognized as CDN, original {}",
|
||||
req.url, orig
|
||||
);
|
||||
original = orig;
|
||||
cdn_opt = Some(req.url.clone());
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(cdn_url) = cdn_opt {
|
||||
let client = crate::shared::utils::web::http_client::http_client().client();
|
||||
let mut accessible = false;
|
||||
match client.get(cdn_url.clone()).send().await {
|
||||
Ok(resp) if resp.status().is_success() => {
|
||||
if let Ok(bytes) = resp.bytes().await {
|
||||
if infer::get(&bytes)
|
||||
.map(|k| k.mime_type().starts_with("image/"))
|
||||
.unwrap_or(false)
|
||||
{
|
||||
accessible = true;
|
||||
} else {
|
||||
warn!("CDN {} returned non-image content", cdn_url);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(resp) => warn!("CDN {} status {}", cdn_url, resp.status()),
|
||||
Err(e) => warn!("CDN {} fetch error {}", cdn_url, e),
|
||||
}
|
||||
if accessible {
|
||||
return Ok(AuditImageCacheResponse {
|
||||
success: true,
|
||||
original_url: original,
|
||||
cdn_url: Some(cdn_url),
|
||||
was_accessible: true,
|
||||
re_uploaded: false,
|
||||
message: "CDN URL is accessible and the image is valid".to_string(),
|
||||
});
|
||||
}
|
||||
info!("CDN {} inaccessible, purging and reuploading", cdn_url);
|
||||
let picser_delete_url = &CONFIG.urls.picser_api_url;
|
||||
if let Some(filename) = cdn_url.split('/').last() {
|
||||
let payload = serde_json::json!({ "filename": filename });
|
||||
match client.delete(picser_delete_url).json(&payload).send().await {
|
||||
Ok(r) if r.status().is_success() => info!("Deleted {} via Picser", filename),
|
||||
Ok(r) => warn!("Picser delete {} status {}", filename, r.status()),
|
||||
Err(e) => warn!("Picser delete error {}: {}", filename, e),
|
||||
}
|
||||
}
|
||||
let _ = cache.invalidate(&original).await;
|
||||
match cache.get_or_cache(&original).await {
|
||||
Ok(new_cdn) => {
|
||||
state
|
||||
.event_bus
|
||||
.publish(ImageRepaired {
|
||||
original_url: original.clone(),
|
||||
cdn_url: new_cdn.clone(),
|
||||
})
|
||||
.await;
|
||||
Ok(AuditImageCacheResponse {
|
||||
success: true,
|
||||
original_url: original,
|
||||
cdn_url: Some(new_cdn),
|
||||
was_accessible: false,
|
||||
re_uploaded: true,
|
||||
message: "CDN URL was inaccessible, re-uploaded".to_string(),
|
||||
})
|
||||
}
|
||||
Err(e) => Ok(AuditImageCacheResponse {
|
||||
success: false,
|
||||
original_url: original,
|
||||
cdn_url: None,
|
||||
was_accessible: false,
|
||||
re_uploaded: false,
|
||||
message: format!("Re-upload failed: {}", e),
|
||||
}),
|
||||
}
|
||||
} else {
|
||||
match cache.get_or_cache(&original).await {
|
||||
Ok(new_cdn) => {
|
||||
state
|
||||
.event_bus
|
||||
.publish(ImageRepaired {
|
||||
original_url: original.clone(),
|
||||
cdn_url: new_cdn.clone(),
|
||||
})
|
||||
.await;
|
||||
Ok(AuditImageCacheResponse {
|
||||
success: true,
|
||||
original_url: original,
|
||||
cdn_url: Some(new_cdn),
|
||||
was_accessible: false,
|
||||
re_uploaded: true,
|
||||
message: "Cached newly".to_string(),
|
||||
})
|
||||
}
|
||||
Err(e) => Ok(AuditImageCacheResponse {
|
||||
success: false,
|
||||
original_url: original,
|
||||
cdn_url: None,
|
||||
was_accessible: false,
|
||||
re_uploaded: false,
|
||||
message: format!("Cache failed: {}", e),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
use serde::Serialize;
|
||||
use utoipa::ToSchema;
|
||||
|
||||
/// Response for image cache POST
|
||||
#[derive(Debug, Serialize, ToSchema)]
|
||||
pub struct ImageCacheResponse {
|
||||
pub success: bool,
|
||||
pub original_url: String,
|
||||
pub cdn_url: String,
|
||||
pub from_cache: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub pending: Option<bool>,
|
||||
}
|
||||
|
||||
/// Response for image cache audit POST
|
||||
#[derive(Debug, Serialize, ToSchema)]
|
||||
pub struct AuditImageCacheResponse {
|
||||
pub success: bool,
|
||||
pub original_url: String,
|
||||
pub cdn_url: Option<String>,
|
||||
pub was_accessible: bool,
|
||||
pub re_uploaded: bool,
|
||||
pub message: String,
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
pub mod pool;
|
||||
|
||||
pub use pool::{init_browser_pool, BrowserPoolConfig};
|
||||
@@ -0,0 +1,483 @@
|
||||
//! Browser pool for managing a single remote browser with multiple tabs.
|
||||
//!
|
||||
//! This pool maintains a connection to a remote Chrome DevTools Protocol (CDP)
|
||||
//! endpoint and provides tabs on-demand for scraping. Tabs are returned to the
|
||||
//! pool after use. Requires EXTERNAL_BROWSERLESS_WS or CHROME_REMOTE_WS to be set.
|
||||
|
||||
use reqwest::StatusCode;
|
||||
use serde_json::json;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::{Mutex, Semaphore};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
fn build_http_endpoint(remote_url: &str, path: &str) -> anyhow::Result<String> {
|
||||
let mut url = reqwest::Url::parse(remote_url)
|
||||
.map_err(|e| anyhow::anyhow!("Invalid remote browser URL '{}': {}", remote_url, e))?;
|
||||
|
||||
match url.scheme() {
|
||||
"ws" => {
|
||||
url.set_scheme("http")
|
||||
.map_err(|_| anyhow::anyhow!("Failed to convert scheme from ws to http"))?;
|
||||
}
|
||||
"wss" => {
|
||||
url.set_scheme("https")
|
||||
.map_err(|_| anyhow::anyhow!("Failed to convert scheme from wss to https"))?;
|
||||
}
|
||||
"http" | "https" => {}
|
||||
scheme => {
|
||||
return Err(anyhow::anyhow!(
|
||||
"Unsupported browser URL scheme '{}'. Expected ws:// or wss://",
|
||||
scheme
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let normalized = path.trim_start_matches('/');
|
||||
url.set_path(normalized);
|
||||
|
||||
Ok(url.to_string())
|
||||
}
|
||||
|
||||
/// Configuration for the browser pool.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BrowserPoolConfig {
|
||||
/// Remote Chrome DevTools Protocol WebSocket URL (required).
|
||||
/// Must be set via `EXTERNAL_BROWSERLESS_WS` or `CHROME_REMOTE_WS` environment variable.
|
||||
/// Takes precedence from: EXTERNAL_BROWSERLESS_WS → CHROME_REMOTE_WS → None
|
||||
pub remote_websocket_url: String,
|
||||
/// Maximum number of concurrent tabs
|
||||
pub max_tabs: usize,
|
||||
}
|
||||
|
||||
impl Default for BrowserPoolConfig {
|
||||
fn default() -> Self {
|
||||
// Priority (highest → lowest):
|
||||
// 1. EXTERNAL_BROWSERLESS_WS — explicit operator override, always wins
|
||||
// 2. CHROME_REMOTE_WS — may be injected by Coolify/Docker networking;
|
||||
// rejected when it resolves to the unroutable Docker alias
|
||||
let external = std::env::var("EXTERNAL_BROWSERLESS_WS").ok();
|
||||
let chrome_remote = std::env::var("CHROME_REMOTE_WS").ok();
|
||||
|
||||
let remote_websocket_url = if let Some(ext) = external {
|
||||
tracing::info!("🌐 Browser: using EXTERNAL_BROWSERLESS_WS");
|
||||
ext
|
||||
} else if let Some(ref cr) = chrome_remote {
|
||||
if cr == "ws://browserless:3000" {
|
||||
eprintln!(
|
||||
"CHROME_REMOTE_WS is set to the unroutable Docker alias \"ws://browserless:3000\" \
|
||||
and EXTERNAL_BROWSERLESS_WS is unset. Remote browser connectivity is required."
|
||||
);
|
||||
std::process::exit(1);
|
||||
} else {
|
||||
tracing::info!("🌐 Browser: using CHROME_REMOTE_WS");
|
||||
cr.clone()
|
||||
}
|
||||
} else {
|
||||
eprintln!(
|
||||
"Remote browser URL not configured. Set EXTERNAL_BROWSERLESS_WS or CHROME_REMOTE_WS environment variable."
|
||||
);
|
||||
std::process::exit(1);
|
||||
};
|
||||
|
||||
Self {
|
||||
remote_websocket_url,
|
||||
max_tabs: 10,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A pool of browser tabs backed by a single remote browser instance.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```ignore
|
||||
/// use scraper_service::browser::{BrowserPool, BrowserPoolConfig};
|
||||
///
|
||||
/// let pool = BrowserPool::new(BrowserPoolConfig::default()).await?;
|
||||
/// let tab = pool.get_tab().await?;
|
||||
/// tab.goto("https://example.com").await?;
|
||||
/// let html = tab.content().await?;
|
||||
/// ```
|
||||
pub struct BrowserPool {
|
||||
/// Available (idle) tab IDs
|
||||
available_tabs: Mutex<Vec<String>>,
|
||||
/// Semaphore to limit concurrent tabs
|
||||
semaphore: Arc<Semaphore>,
|
||||
/// Configuration
|
||||
config: BrowserPoolConfig,
|
||||
/// Counter for generating unique tab IDs
|
||||
tab_counter: AtomicU64,
|
||||
}
|
||||
|
||||
impl BrowserPool {
|
||||
/// Create a new browser pool connecting to a remote CDP endpoint.
|
||||
///
|
||||
/// The pool will attempt to connect to the remote browser URL specified in
|
||||
/// BrowserPoolConfig. If the connection fails, an error is returned.
|
||||
pub async fn new(config: BrowserPoolConfig) -> anyhow::Result<Arc<Self>> {
|
||||
info!(
|
||||
"Initializing browser pool (remote: {}) with max {} tabs",
|
||||
config.remote_websocket_url, config.max_tabs
|
||||
);
|
||||
|
||||
// Verify connection to remote browser with a simple health check
|
||||
Self::verify_remote_connection(&config).await?;
|
||||
|
||||
let pool = Arc::new(Self {
|
||||
available_tabs: Mutex::new(Vec::new()),
|
||||
semaphore: Arc::new(Semaphore::new(config.max_tabs)),
|
||||
config: config.clone(),
|
||||
tab_counter: AtomicU64::new(0),
|
||||
});
|
||||
|
||||
info!("Browser pool initialized (remote-only)");
|
||||
Ok(pool)
|
||||
}
|
||||
|
||||
/// Verify connection to the remote browser.
|
||||
async fn verify_remote_connection(config: &BrowserPoolConfig) -> anyhow::Result<()> {
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
// Browserless/CDP deployments vary by exposed endpoint.
|
||||
// Try multiple known paths and succeed on any valid HTTP response from the host.
|
||||
let check_paths = ["json/version", "health", ""];
|
||||
let mut last_non_success: Option<(String, StatusCode)> = None;
|
||||
|
||||
for path in check_paths {
|
||||
let endpoint = build_http_endpoint(&config.remote_websocket_url, path)?;
|
||||
let response = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
client.get(&endpoint).send(),
|
||||
)
|
||||
.await;
|
||||
|
||||
match response {
|
||||
Ok(Ok(r)) if r.status().is_success() => {
|
||||
info!("Remote browser health check passed via {}", endpoint);
|
||||
return Ok(());
|
||||
}
|
||||
Ok(Ok(r))
|
||||
if r.status() == StatusCode::UNAUTHORIZED
|
||||
|| r.status() == StatusCode::FORBIDDEN =>
|
||||
{
|
||||
return Err(anyhow::anyhow!(
|
||||
"Remote browser reachable but authentication failed at {}: {}",
|
||||
endpoint,
|
||||
r.status()
|
||||
));
|
||||
}
|
||||
Ok(Ok(r)) => {
|
||||
last_non_success = Some((endpoint, r.status()));
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
return Err(anyhow::anyhow!(
|
||||
"Failed to connect to remote browser: {}",
|
||||
e
|
||||
));
|
||||
}
|
||||
Err(_) => {
|
||||
return Err(anyhow::anyhow!("Remote browser connection timeout"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some((endpoint, status)) = last_non_success {
|
||||
warn!(
|
||||
"Remote browser reachable but no known health endpoint succeeded (last: {} -> {})",
|
||||
endpoint, status
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get a tab from the pool.
|
||||
///
|
||||
/// This will reuse an existing idle tab or create a new one.
|
||||
/// The returned `PooledTab` automatically returns to the pool when dropped.
|
||||
pub async fn get_tab(self: &Arc<Self>) -> anyhow::Result<PooledTab> {
|
||||
// Acquire semaphore permit (limits concurrent tabs)
|
||||
let permit = self.semaphore.clone().acquire_owned().await?;
|
||||
|
||||
// Try to reuse an existing tab ID, or generate a new one
|
||||
let tab_id = {
|
||||
let mut tabs = self.available_tabs.lock().await;
|
||||
tabs.pop().unwrap_or_else(|| {
|
||||
let id = self.tab_counter.fetch_add(1, Ordering::SeqCst);
|
||||
format!("tab-{}", id)
|
||||
})
|
||||
};
|
||||
|
||||
debug!("Allocated tab: {}", tab_id);
|
||||
|
||||
Ok(PooledTab {
|
||||
tab_id,
|
||||
cdp_url: self.config.remote_websocket_url.clone(),
|
||||
pool: Arc::clone(self),
|
||||
_permit: permit,
|
||||
})
|
||||
}
|
||||
|
||||
/// Get the number of available (idle) tabs in the pool.
|
||||
pub async fn available_count(&self) -> usize {
|
||||
self.available_tabs.lock().await.len()
|
||||
}
|
||||
|
||||
/// Close the browser pool.
|
||||
pub async fn close(&self) -> anyhow::Result<()> {
|
||||
info!("Closing browser pool");
|
||||
self.available_tabs.lock().await.clear();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// A tab borrowed from the pool (remote CDP-based).
|
||||
///
|
||||
/// This is a lightweight wrapper that communicates with a remote Chrome instance
|
||||
/// via HTTP calls to the browser service. The tab is returned to the pool when dropped.
|
||||
pub struct PooledTab {
|
||||
/// Unique identifier for this tab
|
||||
pub tab_id: String,
|
||||
/// Remote CDP/browser service URL
|
||||
cdp_url: String,
|
||||
/// Pool reference for returning on drop
|
||||
pool: Arc<BrowserPool>,
|
||||
/// Semaphore permit (released on drop)
|
||||
_permit: tokio::sync::OwnedSemaphorePermit,
|
||||
}
|
||||
|
||||
impl PooledTab {
|
||||
/// Navigate to a URL.
|
||||
pub async fn goto(&self, url: &str) -> anyhow::Result<()> {
|
||||
let client = reqwest::Client::new();
|
||||
let timeout = std::time::Duration::from_secs(15);
|
||||
let endpoint = build_http_endpoint(&self.cdp_url, "goto")?;
|
||||
|
||||
let response = tokio::time::timeout(
|
||||
timeout,
|
||||
client.post(&endpoint).json(&json!({ "url": url })).send(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| anyhow::anyhow!("Timeout navigating to {}", url))?
|
||||
.map_err(|e| anyhow::anyhow!("Failed to navigate to {}: {}", url, e))?;
|
||||
|
||||
if response.status().is_success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(anyhow::anyhow!(
|
||||
"Failed to navigate to {}: {}",
|
||||
url,
|
||||
response.status()
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the page content (HTML).
|
||||
pub async fn content(&self) -> anyhow::Result<String> {
|
||||
let client = reqwest::Client::new();
|
||||
let timeout = std::time::Duration::from_secs(15);
|
||||
let endpoint = build_http_endpoint(&self.cdp_url, "content")?;
|
||||
|
||||
let response = tokio::time::timeout(timeout, client.post(&endpoint).send())
|
||||
.await
|
||||
.map_err(|_| anyhow::anyhow!("Timeout getting page content"))?
|
||||
.map_err(|e| anyhow::anyhow!("Failed to get page content: {}", e))?;
|
||||
|
||||
response
|
||||
.text()
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to read response body: {}", e))
|
||||
}
|
||||
|
||||
/// Execute JavaScript and return the result.
|
||||
pub async fn evaluate<T: serde::de::DeserializeOwned>(
|
||||
&self,
|
||||
expression: &str,
|
||||
) -> anyhow::Result<T> {
|
||||
let client = reqwest::Client::new();
|
||||
let timeout = std::time::Duration::from_secs(15);
|
||||
let endpoint = build_http_endpoint(&self.cdp_url, "evaluate")?;
|
||||
|
||||
let response = tokio::time::timeout(
|
||||
timeout,
|
||||
client
|
||||
.post(&endpoint)
|
||||
.json(&json!({ "expression": expression }))
|
||||
.send(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| anyhow::anyhow!("Timeout evaluating JS"))?
|
||||
.map_err(|e| anyhow::anyhow!("Failed to evaluate JS: {}", e))?;
|
||||
|
||||
let data: serde_json::Value = response
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to parse JS result: {}", e))?;
|
||||
|
||||
serde_json::from_value(data).map_err(|e| anyhow::anyhow!("Invalid JS result: {}", e))
|
||||
}
|
||||
|
||||
/// Wait for a selector to appear.
|
||||
pub async fn wait_for_selector(&self, selector: &str) -> anyhow::Result<()> {
|
||||
let client = reqwest::Client::new();
|
||||
let timeout = std::time::Duration::from_secs(15);
|
||||
let endpoint = build_http_endpoint(&self.cdp_url, "waitForSelector")?;
|
||||
|
||||
let response = tokio::time::timeout(
|
||||
timeout,
|
||||
client
|
||||
.post(&endpoint)
|
||||
.json(&json!({ "selector": selector }))
|
||||
.send(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| anyhow::anyhow!("Timeout waiting for selector '{}'", selector))?
|
||||
.map_err(|e| anyhow::anyhow!("Selector '{}' error: {}", selector, e))?;
|
||||
|
||||
if response.status().is_success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(anyhow::anyhow!(
|
||||
"Selector '{}' not found: {}",
|
||||
selector,
|
||||
response.status()
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Click an element by selector.
|
||||
pub async fn click(&self, selector: &str) -> anyhow::Result<()> {
|
||||
let client = reqwest::Client::new();
|
||||
let timeout = std::time::Duration::from_secs(15);
|
||||
let endpoint = build_http_endpoint(&self.cdp_url, "click")?;
|
||||
|
||||
let response = tokio::time::timeout(
|
||||
timeout,
|
||||
client
|
||||
.post(&endpoint)
|
||||
.json(&json!({ "selector": selector }))
|
||||
.send(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| anyhow::anyhow!("Timeout clicking element"))?
|
||||
.map_err(|e| anyhow::anyhow!("Click error: {}", e))?;
|
||||
|
||||
if response.status().is_success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(anyhow::anyhow!(
|
||||
"Failed to click '{}': {}",
|
||||
selector,
|
||||
response.status()
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Type text into an element.
|
||||
pub async fn type_text(&self, selector: &str, text: &str) -> anyhow::Result<()> {
|
||||
let client = reqwest::Client::new();
|
||||
let timeout = std::time::Duration::from_secs(15);
|
||||
let endpoint = build_http_endpoint(&self.cdp_url, "type")?;
|
||||
|
||||
let response = tokio::time::timeout(
|
||||
timeout,
|
||||
client
|
||||
.post(&endpoint)
|
||||
.json(&json!({ "selector": selector, "text": text }))
|
||||
.send(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| anyhow::anyhow!("Timeout typing text"))?
|
||||
.map_err(|e| anyhow::anyhow!("Type error: {}", e))?;
|
||||
|
||||
if response.status().is_success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(anyhow::anyhow!(
|
||||
"Failed to type into '{}': {}",
|
||||
selector,
|
||||
response.status()
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Take a screenshot as PNG bytes.
|
||||
pub async fn screenshot(&self) -> anyhow::Result<Vec<u8>> {
|
||||
let client = reqwest::Client::new();
|
||||
let timeout = std::time::Duration::from_secs(15);
|
||||
let endpoint = build_http_endpoint(&self.cdp_url, "screenshot")?;
|
||||
|
||||
let response = tokio::time::timeout(
|
||||
timeout,
|
||||
client
|
||||
.post(&endpoint)
|
||||
.json(&json!({ "fullPage": true }))
|
||||
.send(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| anyhow::anyhow!("Timeout taking screenshot"))?
|
||||
.map_err(|e| anyhow::anyhow!("Failed to take screenshot: {}", e))?;
|
||||
|
||||
response
|
||||
.bytes()
|
||||
.await
|
||||
.map(|b| b.to_vec())
|
||||
.map_err(|e| anyhow::anyhow!("Failed to read screenshot bytes: {}", e))
|
||||
}
|
||||
|
||||
/// Get the current URL.
|
||||
pub async fn url(&self) -> anyhow::Result<String> {
|
||||
let client = reqwest::Client::new();
|
||||
let timeout = std::time::Duration::from_secs(10);
|
||||
let endpoint = build_http_endpoint(&self.cdp_url, "url")?;
|
||||
|
||||
let response = tokio::time::timeout(timeout, client.post(&endpoint).send())
|
||||
.await
|
||||
.map_err(|_| anyhow::anyhow!("Timeout getting URL"))?
|
||||
.map_err(|e| anyhow::anyhow!("Failed to get URL: {}", e))?;
|
||||
|
||||
response
|
||||
.text()
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to read URL: {}", e))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PooledTab {
|
||||
fn drop(&mut self) {
|
||||
let pool = Arc::clone(&self.pool);
|
||||
let tab_id = self.tab_id.clone();
|
||||
|
||||
let rt = tokio::runtime::Handle::try_current();
|
||||
if let Ok(handle) = rt {
|
||||
handle.spawn(async move {
|
||||
// Return tab ID to available pool for reuse
|
||||
pool.available_tabs.lock().await.push(tab_id);
|
||||
});
|
||||
} else {
|
||||
warn!("Cannot return tab: no tokio runtime available");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Global browser pool instance
|
||||
use once_cell::sync::OnceCell;
|
||||
|
||||
static BROWSER_POOL: OnceCell<Arc<BrowserPool>> = OnceCell::new();
|
||||
|
||||
/// Initialize the global browser pool.
|
||||
/// Call this once at application startup.
|
||||
pub async fn init_browser_pool(config: BrowserPoolConfig) -> anyhow::Result<()> {
|
||||
let pool = BrowserPool::new(config).await?;
|
||||
BROWSER_POOL
|
||||
.set(pool)
|
||||
.map_err(|_| anyhow::anyhow!("Browser pool already initialized"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get the global browser pool.
|
||||
/// Returns None if not initialized.
|
||||
pub fn get_browser_pool() -> Option<Arc<BrowserPool>> {
|
||||
BROWSER_POOL.get().cloned()
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
//! Type-safe application configuration.
|
||||
//!
|
||||
//! This module provides a strongly-typed configuration system that:
|
||||
//! - Loads from environment variables and optional TOML files
|
||||
//! - Fails fast at startup if required variables are missing
|
||||
//! - Supports hierarchical configuration (default -> environment-specific)
|
||||
|
||||
use config::{Config, ConfigError, Environment, File};
|
||||
use once_cell::sync::Lazy;
|
||||
use serde::Deserialize;
|
||||
use std::env;
|
||||
|
||||
/// Application configuration loaded at startup.
|
||||
/// All fields are required unless marked as `Option<T>`.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct AppConfig {
|
||||
/// Database connection URL (PostgreSQL)
|
||||
pub database_url: String,
|
||||
|
||||
/// Secret key for JWT signing
|
||||
pub jwt_secret: String,
|
||||
|
||||
/// Redis connection URL
|
||||
#[serde(default)]
|
||||
pub redis_url: String,
|
||||
|
||||
/// Server port to bind to
|
||||
#[serde(default = "default_port")]
|
||||
pub server_port: u16,
|
||||
|
||||
/// Environment (development, staging, production)
|
||||
#[serde(default = "default_env")]
|
||||
pub environment: String,
|
||||
|
||||
/// Allowed CORS origins (comma-separated)
|
||||
#[serde(default)]
|
||||
pub cors_origins: Vec<String>,
|
||||
|
||||
/// Log level (trace, debug, info, warn, error)
|
||||
#[serde(default = "default_log_level")]
|
||||
pub log_level: String,
|
||||
|
||||
/// SMTP configuration for emails (optional)
|
||||
pub smtp: Option<SmtpConfig>,
|
||||
|
||||
/// Database pool configuration
|
||||
#[serde(default)]
|
||||
pub db: DbConfig,
|
||||
|
||||
/// Max concurrent image processing tasks
|
||||
#[serde(default = "default_image_processing_concurrency")]
|
||||
pub image_processing_concurrency: usize,
|
||||
|
||||
/// Domain/URL values that may change between deployments
|
||||
#[serde(default)]
|
||||
pub urls: UrlConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct UrlConfig {
|
||||
#[serde(default = "default_site_url")]
|
||||
pub site_url: String,
|
||||
#[serde(default = "default_picser_api_url")]
|
||||
pub picser_api_url: String,
|
||||
#[serde(default = "default_fallback_upload_api_url")]
|
||||
pub fallback_upload_api_url: String,
|
||||
}
|
||||
|
||||
impl Default for UrlConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
site_url: default_site_url(),
|
||||
picser_api_url: default_picser_api_url(),
|
||||
fallback_upload_api_url: default_fallback_upload_api_url(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct DbConfig {
|
||||
#[serde(default = "default_db_max_connections")]
|
||||
pub max_connections: u32,
|
||||
#[serde(default = "default_db_min_connections")]
|
||||
pub min_connections: u32,
|
||||
#[serde(default = "default_db_connect_timeout")]
|
||||
pub connect_timeout_seconds: u64,
|
||||
#[serde(default = "default_db_idle_timeout")]
|
||||
pub idle_timeout_seconds: u64,
|
||||
#[serde(default = "default_db_acquire_timeout")]
|
||||
pub acquire_timeout_seconds: u64,
|
||||
#[serde(default = "default_db_max_lifetime")]
|
||||
pub max_lifetime_seconds: u64,
|
||||
}
|
||||
|
||||
impl Default for DbConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_connections: default_db_max_connections(),
|
||||
min_connections: default_db_min_connections(),
|
||||
connect_timeout_seconds: default_db_connect_timeout(),
|
||||
idle_timeout_seconds: default_db_idle_timeout(),
|
||||
acquire_timeout_seconds: default_db_acquire_timeout(),
|
||||
max_lifetime_seconds: default_db_max_lifetime(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// SMTP configuration for sending emails
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct SmtpConfig {
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
pub username: String,
|
||||
pub password: String,
|
||||
pub from_email: String,
|
||||
pub from_name: String,
|
||||
}
|
||||
|
||||
/// MinIO/S3-compatible storage configuration
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MinioConfig {
|
||||
/// MinIO endpoint URL (e.g., "https://cdn.asepharyana.my.id")
|
||||
pub endpoint: String,
|
||||
/// Bucket name
|
||||
pub bucket_name: String,
|
||||
/// Access key / username
|
||||
pub access_key: String,
|
||||
/// Secret key / password
|
||||
pub secret_key: String,
|
||||
/// Use HTTPS (true) or HTTP (false)
|
||||
pub secure: bool,
|
||||
/// AWS region (default: us-east-1)
|
||||
pub region: String,
|
||||
/// Public URL for serving files (optional)
|
||||
pub public_url: Option<String>,
|
||||
/// Prefix for avatar files (e.g., "avatars")
|
||||
pub avatar_prefix: String,
|
||||
}
|
||||
|
||||
impl MinioConfig {
|
||||
/// Load MinIO configuration from environment variables
|
||||
pub fn from_env() -> Option<Self> {
|
||||
let endpoint = env::var("MINIO_ENDPOINT").ok()?;
|
||||
let bucket_name = env::var("MINIO_BUCKET_NAME").ok()?;
|
||||
let access_key = env::var("MINIO_ACCESS_KEY").ok()?;
|
||||
let secret_key = env::var("MINIO_SECRET_KEY").ok()?;
|
||||
|
||||
let secure = env::var("MINIO_SECURE")
|
||||
.map(|v| v.to_lowercase() == "true")
|
||||
.unwrap_or(true);
|
||||
|
||||
let region = env::var("MINIO_REGION").unwrap_or_else(|_| "us-east-1".to_string());
|
||||
|
||||
let public_url = env::var("MINIO_PUBLIC_URL").ok();
|
||||
|
||||
let avatar_prefix =
|
||||
env::var("MINIO_AVATAR_PREFIX").unwrap_or_else(|_| "avatars".to_string());
|
||||
|
||||
Some(Self {
|
||||
endpoint,
|
||||
bucket_name,
|
||||
access_key,
|
||||
secret_key,
|
||||
secure,
|
||||
region,
|
||||
public_url,
|
||||
avatar_prefix,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn default_port() -> u16 {
|
||||
4091
|
||||
}
|
||||
|
||||
fn default_env() -> String {
|
||||
"development".to_string()
|
||||
}
|
||||
|
||||
fn default_log_level() -> String {
|
||||
"info".to_string()
|
||||
}
|
||||
|
||||
fn default_image_processing_concurrency() -> usize {
|
||||
5
|
||||
}
|
||||
|
||||
fn default_site_url() -> String {
|
||||
"https://asepharyana.my.id".to_string()
|
||||
}
|
||||
|
||||
fn default_picser_api_url() -> String {
|
||||
"https://picser.asepharyana.my.id/api/upload".to_string()
|
||||
}
|
||||
|
||||
fn default_fallback_upload_api_url() -> String {
|
||||
"https://upload.asepharyana.my.id/api/upload".to_string()
|
||||
}
|
||||
|
||||
fn default_db_max_connections() -> u32 {
|
||||
100
|
||||
}
|
||||
|
||||
fn default_db_min_connections() -> u32 {
|
||||
10
|
||||
}
|
||||
|
||||
fn default_db_connect_timeout() -> u64 {
|
||||
5
|
||||
}
|
||||
|
||||
fn default_db_idle_timeout() -> u64 {
|
||||
300
|
||||
}
|
||||
|
||||
fn default_db_acquire_timeout() -> u64 {
|
||||
10
|
||||
}
|
||||
|
||||
fn default_db_max_lifetime() -> u64 {
|
||||
1800
|
||||
}
|
||||
|
||||
impl AppConfig {
|
||||
/// Load configuration from environment and optional config files.
|
||||
///
|
||||
/// Priority (highest to lowest):
|
||||
/// 1. Environment variables (prefixed with APP_)
|
||||
/// 2. `config/{environment}.toml`
|
||||
/// 3. `config/default.toml`
|
||||
pub fn load() -> Result<Self, ConfigError> {
|
||||
// Load .env file first
|
||||
if let Err(e) = dotenvy::dotenv() {
|
||||
tracing::debug!("Could not load .env file: {}", e);
|
||||
}
|
||||
|
||||
let run_mode = env::var("RUN_MODE").unwrap_or_else(|_| "development".into());
|
||||
|
||||
let config = Config::builder()
|
||||
// Start with default config file
|
||||
.add_source(File::with_name("config/default").required(false))
|
||||
// Layer on environment-specific values
|
||||
.add_source(File::with_name(&format!("config/{}", run_mode)).required(false))
|
||||
// Add environment variables (with APP_ prefix)
|
||||
.add_source(
|
||||
Environment::with_prefix("APP")
|
||||
.separator("__")
|
||||
.try_parsing(true)
|
||||
.list_separator(","),
|
||||
)
|
||||
// Map legacy env vars to new config structure
|
||||
.set_override_option("database_url", env::var("DATABASE_URL").ok())?
|
||||
.set_override_option("jwt_secret", env::var("JWT_SECRET").ok())?
|
||||
.set_override_option("redis_url", env::var("REDIS_URL").ok())?
|
||||
.build()?;
|
||||
|
||||
config.try_deserialize()
|
||||
}
|
||||
|
||||
/// Check if running in production mode
|
||||
pub fn is_production(&self) -> bool {
|
||||
self.environment == "production"
|
||||
}
|
||||
|
||||
/// Check if running in development mode
|
||||
pub fn is_development(&self) -> bool {
|
||||
self.environment == "development"
|
||||
}
|
||||
}
|
||||
|
||||
/// Global configuration instance, loaded once at startup.
|
||||
/// Panics if configuration is invalid - this is intentional for fail-fast behavior.
|
||||
pub static CONFIG: Lazy<AppConfig> = Lazy::new(|| {
|
||||
AppConfig::load().unwrap_or_else(|e| {
|
||||
eprintln!("❌ Failed to load configuration: {}", e);
|
||||
eprintln!(" Make sure all required environment variables are set:");
|
||||
eprintln!(" - DATABASE_URL");
|
||||
eprintln!(" - JWT_SECRET");
|
||||
eprintln!(" - REDIS_URL (or APP_REDIS_URL)");
|
||||
std::process::exit(1);
|
||||
})
|
||||
});
|
||||
|
||||
/// Global MinIO configuration, loaded from environment variables.
|
||||
/// Returns None if required MINIO_* variables are not set.
|
||||
pub static MINIO_CONFIG: Lazy<Option<MinioConfig>> = Lazy::new(|| {
|
||||
let _ = dotenvy::dotenv();
|
||||
MinioConfig::from_env()
|
||||
});
|
||||
@@ -0,0 +1,7 @@
|
||||
pub mod persistence;
|
||||
pub mod redis;
|
||||
pub mod repositories;
|
||||
pub mod setup;
|
||||
pub mod traits;
|
||||
|
||||
pub use redis::{get_redis_conn, get_redis_pool, redis_pool};
|
||||
@@ -0,0 +1,71 @@
|
||||
//! `SeaORM` Entity for ImageCache - stores URL mappings for CDN caching
|
||||
|
||||
use sea_orm::entity::prelude::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Copy, Clone, Default, Debug, DeriveEntity)]
|
||||
pub struct Entity;
|
||||
|
||||
impl EntityName for Entity {
|
||||
fn table_name(&self) -> &str {
|
||||
"ImageCache"
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel, Eq, Serialize, Deserialize)]
|
||||
pub struct Model {
|
||||
pub id: String,
|
||||
pub original_url: String,
|
||||
pub cdn_url: String,
|
||||
pub created_at: DateTimeUtc,
|
||||
pub expires_at: Option<DateTimeUtc>,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
|
||||
pub enum Column {
|
||||
Id,
|
||||
#[sea_orm(column_name = "original_url")]
|
||||
OriginalUrl,
|
||||
#[sea_orm(column_name = "cdn_url")]
|
||||
CdnUrl,
|
||||
#[sea_orm(column_name = "created_at")]
|
||||
CreatedAt,
|
||||
#[sea_orm(column_name = "expires_at")]
|
||||
ExpiresAt,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)]
|
||||
pub enum PrimaryKey {
|
||||
Id,
|
||||
}
|
||||
|
||||
impl PrimaryKeyTrait for PrimaryKey {
|
||||
type ValueType = String;
|
||||
fn auto_increment() -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter)]
|
||||
pub enum Relation {}
|
||||
|
||||
impl ColumnTrait for Column {
|
||||
type EntityName = Entity;
|
||||
fn def(&self) -> ColumnDef {
|
||||
match self {
|
||||
Self::Id => ColumnType::String(StringLen::N(36u32)).def(),
|
||||
Self::OriginalUrl => ColumnType::String(StringLen::N(512u32)).def().unique(),
|
||||
Self::CdnUrl => ColumnType::String(StringLen::N(512u32)).def(),
|
||||
Self::CreatedAt => ColumnType::TimestampWithTimeZone.def(),
|
||||
Self::ExpiresAt => ColumnType::TimestampWithTimeZone.def().null(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RelationTrait for Relation {
|
||||
fn def(&self) -> RelationDef {
|
||||
match *self {}
|
||||
}
|
||||
}
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {}
|
||||
@@ -0,0 +1 @@
|
||||
pub mod image_cache;
|
||||
@@ -0,0 +1 @@
|
||||
pub mod entities;
|
||||
@@ -0,0 +1,74 @@
|
||||
//! Redis connection utility with tracing for connection lifecycle and errors.
|
||||
//!
|
||||
//! Uses the type-safe CONFIG for Redis connection parameters.
|
||||
|
||||
use crate::shared::config::CONFIG;
|
||||
use crate::shared::errors::AppError;
|
||||
use deadpool_redis::{Manager, Pool};
|
||||
use once_cell::sync::Lazy;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
static REDIS_POOL_INIT: Lazy<Result<Pool, String>> = Lazy::new(|| {
|
||||
let redis_url = if !CONFIG.redis_url.is_empty() {
|
||||
CONFIG.redis_url.clone()
|
||||
} else {
|
||||
let host = std::env::var("REDIS_HOST").unwrap_or_else(|_| "127.0.0.1".to_string());
|
||||
let port = std::env::var("REDIS_PORT").unwrap_or_else(|_| "6379".to_string());
|
||||
let password = std::env::var("REDIS_PASSWORD").unwrap_or_default();
|
||||
|
||||
if password.is_empty() {
|
||||
format!("redis://{}:{}", host, port)
|
||||
} else {
|
||||
format!("redis://:{}@{}:{}", password, host, port)
|
||||
}
|
||||
};
|
||||
|
||||
info!("Initializing Redis connection pool for URL: {}", redis_url);
|
||||
|
||||
Manager::new(redis_url)
|
||||
.map_err(|e| format!("Failed to create Redis manager: {}", e))
|
||||
.and_then(|manager| {
|
||||
Pool::builder(manager)
|
||||
.max_size(100)
|
||||
.wait_timeout(Some(std::time::Duration::from_millis(200)))
|
||||
.runtime(deadpool_redis::Runtime::Tokio1)
|
||||
.build()
|
||||
.map_err(|e| format!("Failed to create Redis connection pool: {}", e))
|
||||
})
|
||||
});
|
||||
|
||||
/// Get the Redis connection pool (for internal use).
|
||||
pub fn get_redis_pool() -> Result<&'static Pool, String> {
|
||||
REDIS_POOL_INIT.as_ref().map_err(|e| e.clone())
|
||||
}
|
||||
|
||||
/// Get a cloned reference to the Redis pool.
|
||||
pub fn redis_pool() -> Result<Pool, String> {
|
||||
get_redis_pool().map(|p| (*p).clone())
|
||||
}
|
||||
|
||||
/// Get an async connection from the pool with retry backoff.
|
||||
pub async fn get_redis_conn() -> Result<deadpool_redis::Connection, AppError> {
|
||||
let pool = get_redis_pool().map_err(|e| AppError::Other(e))?;
|
||||
let mut retries = 5;
|
||||
let mut wait = std::time::Duration::from_millis(100);
|
||||
|
||||
loop {
|
||||
match pool.get().await {
|
||||
Ok(conn) => {
|
||||
debug!("Successfully retrieved Redis connection from pool.");
|
||||
return Ok(conn);
|
||||
}
|
||||
Err(e) => {
|
||||
if retries <= 0 {
|
||||
error!("Failed to get Redis connection after retries: {:?}", e);
|
||||
return Err(AppError::from(e));
|
||||
}
|
||||
debug!("Redis connection failed, retrying in {:?}: {:?}", wait, e);
|
||||
tokio::time::sleep(wait).await;
|
||||
wait = std::cmp::min(wait * 2, std::time::Duration::from_secs(5));
|
||||
retries -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
use crate::shared::database::persistence::entities::image_cache;
|
||||
use crate::shared::database::traits::image_cache::ImageCacheRepository;
|
||||
use crate::shared::utils::Cache;
|
||||
use async_trait::async_trait;
|
||||
use chrono::Utc;
|
||||
use deadpool_redis::Pool as RedisPool;
|
||||
use sea_orm::{ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, Set};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub struct SeaOrmImageCacheRepository {
|
||||
db: Arc<DatabaseConnection>,
|
||||
redis: RedisPool,
|
||||
}
|
||||
|
||||
impl SeaOrmImageCacheRepository {
|
||||
pub fn new(db: Arc<DatabaseConnection>, redis: RedisPool) -> Self {
|
||||
Self { db, redis }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ImageCacheRepository for SeaOrmImageCacheRepository {
|
||||
async fn get_from_redis(&self, key: &str) -> Option<String> {
|
||||
Cache::new(&self.redis).get::<String>(key).await
|
||||
}
|
||||
|
||||
async fn set_in_redis(&self, key: &str, value: &str, ttl: u64) -> Result<(), String> {
|
||||
Cache::new(&self.redis)
|
||||
.set_with_ttl(key, &value, ttl)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
async fn get_from_db(&self, original_url: &str) -> Result<Option<String>, String> {
|
||||
let entry = image_cache::Entity::find()
|
||||
.filter(image_cache::Column::OriginalUrl.eq(original_url))
|
||||
.one(self.db.as_ref())
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(entry.map(|m| m.cdn_url))
|
||||
}
|
||||
|
||||
async fn save_to_db(&self, original_url: &str, cdn_url: &str) -> Result<(), String> {
|
||||
let model = image_cache::ActiveModel {
|
||||
id: Set(uuid::Uuid::new_v4().to_string()),
|
||||
original_url: Set(original_url.to_string()),
|
||||
cdn_url: Set(cdn_url.to_string()),
|
||||
created_at: Set(Utc::now()),
|
||||
expires_at: Set(None),
|
||||
};
|
||||
|
||||
model
|
||||
.insert(self.db.as_ref())
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn find_original_from_cdn(&self, cdn_url: &str) -> Result<Option<String>, String> {
|
||||
let entry = image_cache::Entity::find()
|
||||
.filter(image_cache::Column::CdnUrl.eq(cdn_url))
|
||||
.one(self.db.as_ref())
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(entry.map(|m| m.original_url))
|
||||
}
|
||||
|
||||
async fn delete_from_db(&self, original_url: &str) -> Result<(), String> {
|
||||
image_cache::Entity::delete_many()
|
||||
.filter(image_cache::Column::OriginalUrl.eq(original_url))
|
||||
.exec(self.db.as_ref())
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete_from_redis(&self, key: &str) -> Result<(), String> {
|
||||
Cache::new(&self.redis)
|
||||
.delete(key)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
async fn get_lock(&self, key: &str) -> bool {
|
||||
Cache::new(&self.redis).get::<bool>(key).await.is_some()
|
||||
}
|
||||
|
||||
async fn set_lock(&self, key: &str, ttl: u64) -> Result<(), String> {
|
||||
Cache::new(&self.redis)
|
||||
.set_with_ttl(key, &true, ttl)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
async fn release_lock(&self, key: &str) -> Result<(), String> {
|
||||
self.delete_from_redis(key).await
|
||||
}
|
||||
|
||||
async fn invalidate_api_caches(&self, patterns: Vec<&str>) -> Result<(), String> {
|
||||
use deadpool_redis::redis::{cmd, AsyncCommands};
|
||||
|
||||
let mut conn = self.redis.get().await.map_err(|e| e.to_string())?;
|
||||
|
||||
for pattern in patterns {
|
||||
let mut cursor: u64 = 0;
|
||||
loop {
|
||||
let (new_cursor, keys): (u64, Vec<String>) = cmd("SCAN")
|
||||
.arg(cursor)
|
||||
.arg("MATCH")
|
||||
.arg(pattern)
|
||||
.arg("COUNT")
|
||||
.arg(100)
|
||||
.query_async(&mut *conn)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
if !keys.is_empty() {
|
||||
let _: usize = conn.del(&keys).await.map_err(|e| e.to_string())?;
|
||||
}
|
||||
|
||||
cursor = new_cursor;
|
||||
if cursor == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
pub mod image_cache;
|
||||
@@ -0,0 +1,44 @@
|
||||
use crate::shared::database::persistence::entities::image_cache;
|
||||
use sea_orm::{ConnectionTrait, DatabaseConnection, Schema, Statement};
|
||||
use tracing::info;
|
||||
|
||||
pub async fn init(db: &DatabaseConnection) -> Result<(), sea_orm::DbErr> {
|
||||
info!("🚀 Initializing database schema...");
|
||||
let backend = db.get_database_backend();
|
||||
let schema = Schema::new(backend);
|
||||
|
||||
let tables = vec![(
|
||||
"ImageCache",
|
||||
schema
|
||||
.create_table_from_entity(image_cache::Entity)
|
||||
.if_not_exists()
|
||||
.to_owned(),
|
||||
)];
|
||||
|
||||
for (name, stmt) in tables {
|
||||
match db.execute(backend.build(&stmt)).await {
|
||||
Ok(_) => info!(" ✓ Table '{}' checked/created", name),
|
||||
Err(e) => {
|
||||
tracing::error!(" [!] Failed to create table '{}': {}", name, e);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let index_sql =
|
||||
"CREATE INDEX IF NOT EXISTS idx_image_cache_cdn_url ON \"ImageCache\" (cdn_url)";
|
||||
match db.execute(Statement::from_string(backend, index_sql)).await {
|
||||
Ok(_) => info!(" ✓ Index 'idx_image_cache_cdn_url' ensured"),
|
||||
Err(e) => {
|
||||
let err_str = e.to_string();
|
||||
if err_str.contains("already exists") || err_str.contains("duplicate") {
|
||||
info!(" ✓ Index 'idx_image_cache_cdn_url' already exists");
|
||||
} else {
|
||||
tracing::error!(" [!] Failed to create index on ImageCache: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
info!("✅ Database schema initialization complete.");
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
use async_trait::async_trait;
|
||||
|
||||
#[async_trait]
|
||||
pub trait ImageCacheRepository: Send + Sync {
|
||||
async fn get_from_redis(&self, key: &str) -> Option<String>;
|
||||
async fn set_in_redis(&self, key: &str, value: &str, ttl: u64) -> Result<(), String>;
|
||||
async fn get_from_db(&self, original_url: &str) -> Result<Option<String>, String>;
|
||||
async fn save_to_db(&self, original_url: &str, cdn_url: &str) -> Result<(), String>;
|
||||
async fn find_original_from_cdn(&self, cdn_url: &str) -> Result<Option<String>, String>;
|
||||
async fn delete_from_db(&self, original_url: &str) -> Result<(), String>;
|
||||
async fn delete_from_redis(&self, key: &str) -> Result<(), String>;
|
||||
async fn get_lock(&self, key: &str) -> bool;
|
||||
async fn set_lock(&self, key: &str, ttl: u64) -> Result<(), String>;
|
||||
async fn release_lock(&self, key: &str) -> Result<(), String>;
|
||||
async fn invalidate_api_caches(&self, patterns: Vec<&str>) -> Result<(), String>;
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
pub mod image_cache;
|
||||
pub mod scraping_repository;
|
||||
|
||||
pub use scraping_repository::ScrapingRepository;
|
||||
@@ -0,0 +1,7 @@
|
||||
use crate::shared::errors::AppError;
|
||||
use async_trait::async_trait;
|
||||
|
||||
#[async_trait]
|
||||
pub trait ScrapingRepository: Send + Sync {
|
||||
async fn fetch_html(&self, url: &str) -> Result<String, AppError>;
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
use axum::response::IntoResponse;
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum AppError {
|
||||
#[error("Environment variable not found: {0}")]
|
||||
EnvVarNotFound(String),
|
||||
#[error("Redis error: {0}")]
|
||||
RedisError(#[from] redis::RedisError),
|
||||
#[error("Reqwest error: {0}")]
|
||||
ReqwestError(#[from] reqwest::Error),
|
||||
#[error("JSON serialization/deserialization error: {0}")]
|
||||
SerdeJsonError(#[from] serde_json::Error),
|
||||
#[error("Scraper error: {0}")]
|
||||
ScraperError(String),
|
||||
#[error("Fantoccini error: {0}")]
|
||||
FantocciniError(String),
|
||||
#[error("IO error: {0}")]
|
||||
IoError(#[from] std::io::Error),
|
||||
#[error("Timeout error: {0}")]
|
||||
TimeoutError(String),
|
||||
#[error("Other error: {0}")]
|
||||
Other(String),
|
||||
#[error("HTTP error: {0}")]
|
||||
HttpError(#[from] http::Error),
|
||||
#[error("URL parsing error: {0}")]
|
||||
UrlParseError(#[from] url::ParseError),
|
||||
#[error("Database error: {0}")]
|
||||
DatabaseError(String),
|
||||
#[error("Not Found: {0}")]
|
||||
NotFound(String),
|
||||
}
|
||||
|
||||
impl From<&str> for AppError {
|
||||
fn from(s: &str) -> Self {
|
||||
AppError::Other(s.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<String> for AppError {
|
||||
fn from(s: String) -> Self {
|
||||
AppError::Other(s)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Box<dyn std::error::Error + Send + Sync>> for AppError {
|
||||
fn from(err: Box<dyn std::error::Error + Send + Sync>) -> Self {
|
||||
AppError::Other(err.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<anyhow::Error> for AppError {
|
||||
fn from(err: anyhow::Error) -> Self {
|
||||
AppError::Other(err.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<deadpool_redis::PoolError> for AppError {
|
||||
fn from(err: deadpool_redis::PoolError) -> Self {
|
||||
AppError::Other(err.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<tokio::task::JoinError> for AppError {
|
||||
fn from(err: tokio::task::JoinError) -> Self {
|
||||
AppError::Other(err.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl IntoResponse for AppError {
|
||||
fn into_response(self) -> axum::response::Response {
|
||||
let (status, error_message) = match self {
|
||||
AppError::NotFound(_) => (http::StatusCode::NOT_FOUND, self.to_string()),
|
||||
AppError::DatabaseError(_) => {
|
||||
(http::StatusCode::INTERNAL_SERVER_ERROR, self.to_string())
|
||||
}
|
||||
_ => (http::StatusCode::INTERNAL_SERVER_ERROR, self.to_string()),
|
||||
};
|
||||
|
||||
// Note: crate::shared::types needs to be available.
|
||||
// If not, we might need to adjust this line or ensure types are there.
|
||||
// Since we are validating structure, let's assume types is in core/types.rs
|
||||
let body = axum::Json(crate::shared::types::ApiResponse::<()>::error(
|
||||
error_message,
|
||||
));
|
||||
(status, body).into_response()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
pub mod app_error;
|
||||
pub use app_error::AppError;
|
||||
@@ -0,0 +1,155 @@
|
||||
//! Event bus implementation.
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use std::{any::TypeId, collections::HashMap, sync::Arc};
|
||||
use tokio::sync::{broadcast, RwLock};
|
||||
use tracing::{debug, info};
|
||||
|
||||
/// Trait for events that can be published.
|
||||
pub trait Event: Clone + Send + Sync + 'static {
|
||||
/// Event name for logging/debugging.
|
||||
const NAME: &'static str;
|
||||
}
|
||||
|
||||
/// Trait for event handlers.
|
||||
#[async_trait]
|
||||
pub trait EventHandler<E: Event>: Send + Sync {
|
||||
async fn handle(&self, event: E);
|
||||
}
|
||||
|
||||
/// The event bus for publishing and subscribing to events.
|
||||
pub struct EventBus {
|
||||
channels: RwLock<HashMap<TypeId, Box<dyn std::any::Any + Send + Sync>>>,
|
||||
}
|
||||
|
||||
impl EventBus {
|
||||
/// Create a new event bus.
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
channels: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Publish an event to all subscribers.
|
||||
pub async fn publish<E: Event>(&self, event: E) {
|
||||
let type_id = TypeId::of::<E>();
|
||||
let channels = self.channels.read().await;
|
||||
|
||||
if let Some(sender) = channels.get(&type_id) {
|
||||
if let Some(tx) = sender.downcast_ref::<broadcast::Sender<E>>() {
|
||||
let _ = tx.send(event);
|
||||
debug!("Published event: {}", E::NAME);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Subscribe to events of a specific type.
|
||||
/// Returns a receiver that can be used to receive events.
|
||||
pub async fn subscribe<E: Event>(&self) -> broadcast::Receiver<E> {
|
||||
let type_id = TypeId::of::<E>();
|
||||
|
||||
// Check if channel exists
|
||||
{
|
||||
let channels = self.channels.read().await;
|
||||
if let Some(sender) = channels.get(&type_id) {
|
||||
if let Some(tx) = sender.downcast_ref::<broadcast::Sender<E>>() {
|
||||
return tx.subscribe();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create new channel
|
||||
let (tx, rx) = broadcast::channel::<E>(100);
|
||||
{
|
||||
let mut channels = self.channels.write().await;
|
||||
channels.insert(type_id, Box::new(tx));
|
||||
}
|
||||
|
||||
// Re-get the receiver from the stored sender
|
||||
let channels = self.channels.read().await;
|
||||
if let Some(sender) = channels.get(&type_id) {
|
||||
if let Some(tx) = sender.downcast_ref::<broadcast::Sender<E>>() {
|
||||
return tx.subscribe();
|
||||
}
|
||||
}
|
||||
|
||||
rx
|
||||
}
|
||||
|
||||
/// Register a handler for a specific event type.
|
||||
/// The handler will be called whenever an event of that type is published.
|
||||
pub async fn on<E: Event, H: EventHandler<E> + 'static>(&self, handler: H) {
|
||||
let mut rx = self.subscribe::<E>().await;
|
||||
let handler = Arc::new(handler);
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
match rx.recv().await {
|
||||
Ok(event) => {
|
||||
handler.handle(event).await;
|
||||
}
|
||||
Err(broadcast::error::RecvError::Closed) => break,
|
||||
Err(broadcast::error::RecvError::Lagged(n)) => {
|
||||
tracing::warn!("Event handler lagged by {} events", n);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
info!("Registered handler for event: {}", E::NAME);
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for EventBus {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
// Common events
|
||||
/// User registered event.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct UserRegistered {
|
||||
pub user_id: String,
|
||||
pub email: String,
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
impl Event for UserRegistered {
|
||||
const NAME: &'static str = "user.registered";
|
||||
}
|
||||
|
||||
/// User logged in event.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct UserLoggedIn {
|
||||
pub user_id: String,
|
||||
pub ip_address: Option<String>,
|
||||
}
|
||||
|
||||
impl Event for UserLoggedIn {
|
||||
const NAME: &'static str = "user.logged_in";
|
||||
}
|
||||
|
||||
/// Order created event.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct OrderCreated {
|
||||
pub order_id: String,
|
||||
pub user_id: String,
|
||||
pub total: f64,
|
||||
}
|
||||
|
||||
impl Event for OrderCreated {
|
||||
const NAME: &'static str = "order.created";
|
||||
}
|
||||
|
||||
/// Image repaired event.
|
||||
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
|
||||
pub struct ImageRepaired {
|
||||
pub original_url: String,
|
||||
pub cdn_url: String,
|
||||
}
|
||||
|
||||
impl Event for ImageRepaired {
|
||||
const NAME: &'static str = "image.repaired";
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
pub mod bus;
|
||||
@@ -0,0 +1,142 @@
|
||||
//! Graceful shutdown with proper resource cleanup.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use tokio::signal;
|
||||
use tokio::sync::Notify;
|
||||
use tokio::time::{sleep, Duration};
|
||||
use tracing::info;
|
||||
|
||||
/// Graceful shutdown coordinator.
|
||||
pub struct ShutdownCoordinator {
|
||||
/// Shutdown signal flag
|
||||
is_shutting_down: Arc<AtomicBool>,
|
||||
/// Notify for graceful shutdown
|
||||
shutdown_notify: Arc<Notify>,
|
||||
}
|
||||
|
||||
impl ShutdownCoordinator {
|
||||
/// Create a new shutdown coordinator.
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
is_shutting_down: Arc::new(AtomicBool::new(false)),
|
||||
shutdown_notify: Arc::new(Notify::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if shutdown is in progress.
|
||||
pub fn is_shutting_down(&self) -> bool {
|
||||
self.is_shutting_down.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Start shutdown process.
|
||||
pub fn initiate_shutdown(&self) {
|
||||
info!("🛑 Initiating graceful shutdown...");
|
||||
self.is_shutting_down.store(true, Ordering::Relaxed);
|
||||
self.shutdown_notify.notify_waiters();
|
||||
}
|
||||
|
||||
/// Wait for shutdown signal.
|
||||
pub async fn wait_for_shutdown_signal(&self) {
|
||||
let ctrl_c = async {
|
||||
if let Err(e) = signal::ctrl_c().await {
|
||||
tracing::error!("Failed to listen for Ctrl+C: {}", e);
|
||||
}
|
||||
};
|
||||
|
||||
#[cfg(unix)]
|
||||
let terminate = async {
|
||||
match signal::unix::signal(signal::unix::SignalKind::terminate()) {
|
||||
Ok(mut stream) => {
|
||||
stream.recv().await;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to listen for SIGTERM: {}", e);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
#[cfg(not(unix))]
|
||||
let terminate = std::future::pending::<()>();
|
||||
|
||||
tokio::select! {
|
||||
_ = ctrl_c => {
|
||||
info!("Received Ctrl+C signal");
|
||||
}
|
||||
_ = terminate => {
|
||||
info!("Received SIGTERM signal");
|
||||
}
|
||||
}
|
||||
|
||||
self.initiate_shutdown();
|
||||
}
|
||||
|
||||
/// Perform cleanup operations.
|
||||
pub async fn cleanup(&self) {
|
||||
info!("🧹 Starting cleanup operations...");
|
||||
|
||||
// Give active requests time to finish
|
||||
info!("Waiting for active requests to complete...");
|
||||
sleep(Duration::from_secs(5)).await;
|
||||
|
||||
info!("✅ Cleanup completed");
|
||||
}
|
||||
|
||||
/// Get a handle for checking shutdown status.
|
||||
pub fn handle(&self) -> ShutdownHandle {
|
||||
ShutdownHandle {
|
||||
is_shutting_down: Arc::clone(&self.is_shutting_down),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ShutdownCoordinator {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle for checking shutdown status.
|
||||
#[derive(Clone)]
|
||||
pub struct ShutdownHandle {
|
||||
is_shutting_down: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl ShutdownHandle {
|
||||
/// Check if shutdown is in progress.
|
||||
pub fn is_shutting_down(&self) -> bool {
|
||||
self.is_shutting_down.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn wait_for_shutdown_and_cleanup() {
|
||||
let coordinator = ShutdownCoordinator::new();
|
||||
|
||||
coordinator.wait_for_shutdown_signal().await;
|
||||
coordinator.cleanup().await;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_shutdown_coordinator() {
|
||||
let coordinator = ShutdownCoordinator::new();
|
||||
assert!(!coordinator.is_shutting_down());
|
||||
|
||||
coordinator.initiate_shutdown();
|
||||
assert!(coordinator.is_shutting_down());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_shutdown_handle() {
|
||||
let coordinator = ShutdownCoordinator::new();
|
||||
let handle = coordinator.handle();
|
||||
|
||||
assert!(!handle.is_shutting_down());
|
||||
|
||||
coordinator.initiate_shutdown();
|
||||
assert!(handle.is_shutting_down());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
pub mod cleanup;
|
||||
pub mod shutdown;
|
||||
@@ -0,0 +1,116 @@
|
||||
//! Graceful shutdown implementation.
|
||||
|
||||
use std::future::Future;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::signal;
|
||||
use tokio::sync::broadcast;
|
||||
use tracing::info;
|
||||
|
||||
/// Graceful shutdown controller.
|
||||
pub struct GracefulShutdown {
|
||||
shutdown_tx: broadcast::Sender<()>,
|
||||
is_shutting_down: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl GracefulShutdown {
|
||||
/// Create a new graceful shutdown controller.
|
||||
pub fn new() -> Self {
|
||||
let (shutdown_tx, _) = broadcast::channel(1);
|
||||
Self {
|
||||
shutdown_tx,
|
||||
is_shutting_down: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if shutdown has been initiated.
|
||||
pub fn is_shutting_down(&self) -> bool {
|
||||
self.is_shutting_down.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Get a receiver for shutdown notifications.
|
||||
pub fn subscribe(&self) -> broadcast::Receiver<()> {
|
||||
self.shutdown_tx.subscribe()
|
||||
}
|
||||
|
||||
/// Initiate shutdown.
|
||||
pub fn shutdown(&self) {
|
||||
if !self.is_shutting_down.swap(true, Ordering::SeqCst) {
|
||||
info!("🛑 Initiating graceful shutdown...");
|
||||
let _ = self.shutdown_tx.send(());
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait for shutdown signal and then gracefully drain.
|
||||
pub async fn wait_for_shutdown(&self, drain_timeout: Duration) {
|
||||
// Wait for shutdown signal
|
||||
shutdown_signal().await;
|
||||
|
||||
self.shutdown();
|
||||
|
||||
// Give time for in-flight requests to complete
|
||||
info!(
|
||||
"⏳ Waiting {}s for in-flight requests...",
|
||||
drain_timeout.as_secs()
|
||||
);
|
||||
tokio::time::sleep(drain_timeout).await;
|
||||
|
||||
info!("✅ Graceful shutdown complete");
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for GracefulShutdown {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Clone for GracefulShutdown {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
shutdown_tx: self.shutdown_tx.clone(),
|
||||
is_shutting_down: Arc::clone(&self.is_shutting_down),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait for a shutdown signal (SIGTERM, SIGINT, or Ctrl+C).
|
||||
pub async fn shutdown_signal() {
|
||||
let ctrl_c = async {
|
||||
if let Err(e) = signal::ctrl_c().await {
|
||||
tracing::error!("Failed to install Ctrl+C handler: {}", e);
|
||||
}
|
||||
};
|
||||
|
||||
#[cfg(unix)]
|
||||
let terminate = async {
|
||||
match signal::unix::signal(signal::unix::SignalKind::terminate()) {
|
||||
Ok(mut stream) => {
|
||||
stream.recv().await;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to install SIGTERM handler: {}", e);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
#[cfg(not(unix))]
|
||||
let terminate = std::future::pending::<()>();
|
||||
|
||||
tokio::select! {
|
||||
_ = ctrl_c => {
|
||||
info!("📥 Received Ctrl+C signal");
|
||||
}
|
||||
_ = terminate => {
|
||||
info!("📥 Received SIGTERM signal");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a shutdown future that can be used with axum's serve.
|
||||
pub fn create_shutdown_signal() -> impl Future<Output = ()> + Send + 'static {
|
||||
async {
|
||||
shutdown_signal().await;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
//! Health check endpoint implementations.
|
||||
|
||||
use axum::{http::StatusCode, response::IntoResponse, Json};
|
||||
use serde::Serialize;
|
||||
use std::time::Instant;
|
||||
|
||||
use once_cell::sync::Lazy;
|
||||
|
||||
static START_TIME: Lazy<Instant> = Lazy::new(Instant::now);
|
||||
|
||||
/// Health status response.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct HealthStatus {
|
||||
pub status: &'static str,
|
||||
pub version: &'static str,
|
||||
pub uptime_seconds: u64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub checks: Option<HealthChecks>,
|
||||
}
|
||||
|
||||
/// Individual health checks.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct HealthChecks {
|
||||
pub database: CheckResult,
|
||||
pub redis: CheckResult,
|
||||
}
|
||||
|
||||
/// Result of a health check.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct CheckResult {
|
||||
pub status: &'static str,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub latency_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Simple health check - just returns OK.
|
||||
/// Use for liveness probes (Kubernetes: /healthz).
|
||||
pub async fn health_check() -> impl IntoResponse {
|
||||
let status = HealthStatus {
|
||||
status: "ok",
|
||||
version: env!("CARGO_PKG_VERSION"),
|
||||
uptime_seconds: START_TIME.elapsed().as_secs(),
|
||||
checks: None,
|
||||
};
|
||||
(StatusCode::OK, Json(status))
|
||||
}
|
||||
|
||||
/// Readiness check with dependency checks.
|
||||
/// Use for readiness probes (Kubernetes: /readyz).
|
||||
pub async fn readiness_check() -> impl IntoResponse {
|
||||
let uptime = START_TIME.elapsed().as_secs();
|
||||
|
||||
// Check Redis
|
||||
let redis_check = check_redis().await;
|
||||
|
||||
// Check Database - simplified for now
|
||||
let db_check = CheckResult {
|
||||
status: "ok",
|
||||
latency_ms: Some(1),
|
||||
error: None,
|
||||
};
|
||||
|
||||
let all_healthy = redis_check.status == "ok" && db_check.status == "ok";
|
||||
|
||||
let status = HealthStatus {
|
||||
status: if all_healthy { "ok" } else { "degraded" },
|
||||
version: env!("CARGO_PKG_VERSION"),
|
||||
uptime_seconds: uptime,
|
||||
checks: Some(HealthChecks {
|
||||
database: db_check,
|
||||
redis: redis_check,
|
||||
}),
|
||||
};
|
||||
|
||||
let status_code = if all_healthy {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
};
|
||||
|
||||
(status_code, Json(status))
|
||||
}
|
||||
|
||||
/// Check Redis connectivity using PING command.
|
||||
async fn check_redis() -> CheckResult {
|
||||
use crate::shared::database::get_redis_pool;
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
let pool = match get_redis_pool() {
|
||||
Ok(p) => p,
|
||||
Err(e) => {
|
||||
return CheckResult {
|
||||
status: "error",
|
||||
latency_ms: Some(start.elapsed().as_millis() as u64),
|
||||
error: Some(format!("Pool error: {}", e)),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
match pool.get().await {
|
||||
Ok(mut conn) => {
|
||||
let result: Result<String, _> = redis::cmd("PING").query_async(&mut *conn).await;
|
||||
match result {
|
||||
Ok(response) if response == "PONG" => CheckResult {
|
||||
status: "ok",
|
||||
latency_ms: Some(start.elapsed().as_millis() as u64),
|
||||
error: None,
|
||||
},
|
||||
Ok(response) => CheckResult {
|
||||
status: "error",
|
||||
latency_ms: Some(start.elapsed().as_millis() as u64),
|
||||
error: Some(format!("Unexpected PING response: {}", response)),
|
||||
},
|
||||
Err(e) => CheckResult {
|
||||
status: "error",
|
||||
latency_ms: Some(start.elapsed().as_millis() as u64),
|
||||
error: Some(e.to_string()),
|
||||
},
|
||||
}
|
||||
}
|
||||
Err(e) => CheckResult {
|
||||
status: "error",
|
||||
latency_ms: Some(start.elapsed().as_millis() as u64),
|
||||
error: Some(e.to_string()),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Standalone health endpoint (no dependencies required).
|
||||
pub async fn simple_health() -> impl IntoResponse {
|
||||
Json(serde_json::json!({
|
||||
"status": "ok",
|
||||
"timestamp": chrono::Utc::now().to_rfc3339()
|
||||
}))
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
pub mod endpoints;
|
||||
@@ -0,0 +1,2 @@
|
||||
pub mod queue;
|
||||
pub mod worker;
|
||||
@@ -0,0 +1,204 @@
|
||||
//! Job queue implementation using Redis.
|
||||
//!
|
||||
//! Provides a simple but robust job queue system for background processing.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use deadpool_redis::Pool;
|
||||
use redis::AsyncCommands;
|
||||
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Status of a queued job.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum JobStatus {
|
||||
/// Job is waiting in the queue
|
||||
Pending,
|
||||
/// Job is currently being processed
|
||||
Processing,
|
||||
/// Job completed successfully
|
||||
Completed,
|
||||
/// Job failed with an error
|
||||
Failed,
|
||||
/// Job was manually cancelled
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
/// Metadata for a queued job.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct JobMeta {
|
||||
pub id: String,
|
||||
pub job_type: String,
|
||||
pub status: JobStatus,
|
||||
pub created_at: chrono::DateTime<chrono::Utc>,
|
||||
pub started_at: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub completed_at: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub attempts: u32,
|
||||
pub max_attempts: u32,
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Trait for background jobs.
|
||||
///
|
||||
/// Implement this trait to define a job that can be queued and processed.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```ignore
|
||||
/// use scraper_service::jobs::{Job, JobDispatcher};
|
||||
/// use serde::{Serialize, Deserialize};
|
||||
/// use async_trait::async_trait;
|
||||
///
|
||||
/// #[derive(Serialize, Deserialize)]
|
||||
/// struct SendWelcomeEmail {
|
||||
/// user_id: String,
|
||||
/// email: String,
|
||||
/// }
|
||||
///
|
||||
/// #[async_trait]
|
||||
/// impl Job for SendWelcomeEmail {
|
||||
/// const NAME: &'static str = "send_welcome_email";
|
||||
/// const MAX_ATTEMPTS: u32 = 3;
|
||||
///
|
||||
/// async fn handle(&self) -> anyhow::Result<()> {
|
||||
/// // Send the email...
|
||||
/// println!("Sending welcome email to {}", self.email);
|
||||
/// Ok(())
|
||||
/// }
|
||||
/// }
|
||||
///
|
||||
/// // Dispatch the job
|
||||
/// let job = SendWelcomeEmail {
|
||||
/// user_id: "123".to_string(),
|
||||
/// email: "user@example.com".to_string(),
|
||||
/// };
|
||||
/// dispatcher.dispatch(job).await?;
|
||||
/// ```
|
||||
#[async_trait]
|
||||
pub trait Job: Serialize + DeserializeOwned + Send + Sync {
|
||||
/// Unique name for this job type.
|
||||
const NAME: &'static str;
|
||||
|
||||
/// Maximum number of retry attempts.
|
||||
const MAX_ATTEMPTS: u32 = 3;
|
||||
|
||||
/// Queue name to use (default: "default")
|
||||
const QUEUE: &'static str = "default";
|
||||
|
||||
/// Execute the job.
|
||||
async fn handle(&self) -> anyhow::Result<()>;
|
||||
|
||||
/// Called when the job fails after all retries.
|
||||
async fn failed(&self, error: &str) {
|
||||
tracing::error!("Job {} failed: {}", Self::NAME, error);
|
||||
}
|
||||
}
|
||||
|
||||
/// Job dispatcher for queuing jobs.
|
||||
#[derive(Clone)]
|
||||
pub struct JobDispatcher {
|
||||
redis_pool: Pool,
|
||||
}
|
||||
|
||||
impl JobDispatcher {
|
||||
/// Create a new job dispatcher.
|
||||
pub fn new(redis_pool: Pool) -> Self {
|
||||
Self { redis_pool }
|
||||
}
|
||||
|
||||
/// Dispatch a job to be processed.
|
||||
pub async fn dispatch<J: Job>(&self, job: J) -> anyhow::Result<String> {
|
||||
let job_id = Uuid::new_v4().to_string();
|
||||
let queue_key = format!("jobs:queue:{}", J::QUEUE);
|
||||
let job_key = format!("jobs:data:{}", job_id);
|
||||
|
||||
let payload = serde_json::to_string(&job)?;
|
||||
|
||||
let meta = JobMeta {
|
||||
id: job_id.clone(),
|
||||
job_type: J::NAME.to_string(),
|
||||
status: JobStatus::Pending,
|
||||
created_at: chrono::Utc::now(),
|
||||
started_at: None,
|
||||
completed_at: None,
|
||||
attempts: 0,
|
||||
max_attempts: J::MAX_ATTEMPTS,
|
||||
error: None,
|
||||
};
|
||||
|
||||
let meta_json = serde_json::to_string(&meta)?;
|
||||
|
||||
let mut conn = self.redis_pool.get().await?;
|
||||
|
||||
// Store job data
|
||||
let _: () = conn.set(&job_key, payload).await?;
|
||||
let _: () = conn.set(format!("{}:meta", job_key), meta_json).await?;
|
||||
|
||||
// Push to queue
|
||||
let _: () = conn.rpush(&queue_key, &job_id).await?;
|
||||
|
||||
tracing::info!("Dispatched job {} ({})", J::NAME, job_id);
|
||||
|
||||
Ok(job_id)
|
||||
}
|
||||
|
||||
/// Dispatch a job with a delay (in seconds).
|
||||
pub async fn dispatch_delayed<J: Job>(
|
||||
&self,
|
||||
job: J,
|
||||
delay_seconds: u64,
|
||||
) -> anyhow::Result<String> {
|
||||
let job_id = Uuid::new_v4().to_string();
|
||||
let delayed_key = "jobs:delayed";
|
||||
let job_key = format!("jobs:data:{}", job_id);
|
||||
|
||||
let payload = serde_json::to_string(&job)?;
|
||||
let execute_at = chrono::Utc::now().timestamp() + delay_seconds as i64;
|
||||
|
||||
let meta = JobMeta {
|
||||
id: job_id.clone(),
|
||||
job_type: J::NAME.to_string(),
|
||||
status: JobStatus::Pending,
|
||||
created_at: chrono::Utc::now(),
|
||||
started_at: None,
|
||||
completed_at: None,
|
||||
attempts: 0,
|
||||
max_attempts: J::MAX_ATTEMPTS,
|
||||
error: None,
|
||||
};
|
||||
|
||||
let meta_json = serde_json::to_string(&meta)?;
|
||||
|
||||
let mut conn = self.redis_pool.get().await?;
|
||||
|
||||
// Store job data
|
||||
let _: () = conn.set(&job_key, payload).await?;
|
||||
let _: () = conn.set(format!("{}:meta", job_key), meta_json).await?;
|
||||
|
||||
// Add to delayed sorted set (score = execution timestamp)
|
||||
let delayed_entry = format!("{}:{}", J::QUEUE, job_id);
|
||||
let _: () = conn.zadd(delayed_key, delayed_entry, execute_at).await?;
|
||||
|
||||
tracing::info!(
|
||||
"Dispatched delayed job {} ({}) - executes in {}s",
|
||||
J::NAME,
|
||||
job_id,
|
||||
delay_seconds
|
||||
);
|
||||
|
||||
Ok(job_id)
|
||||
}
|
||||
|
||||
/// Get the status of a job.
|
||||
pub async fn status(&self, job_id: &str) -> anyhow::Result<Option<JobMeta>> {
|
||||
let meta_key = format!("jobs:data:{}:meta", job_id);
|
||||
let mut conn = self.redis_pool.get().await?;
|
||||
|
||||
let meta_json: Option<String> = conn.get(&meta_key).await?;
|
||||
|
||||
match meta_json {
|
||||
Some(json) => Ok(Some(serde_json::from_str(&json)?)),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
//! Job worker for processing background jobs.
|
||||
//!
|
||||
//! The worker runs as a separate process and continuously polls
|
||||
//! the job queue for work.
|
||||
|
||||
use super::queue::{JobMeta, JobStatus};
|
||||
use deadpool_redis::Pool;
|
||||
use redis::AsyncCommands;
|
||||
use std::time::Duration;
|
||||
use tokio::time::sleep;
|
||||
|
||||
/// Configuration for the job worker.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct WorkerConfig {
|
||||
/// Queues to process (in priority order)
|
||||
pub queues: Vec<String>,
|
||||
/// Number of concurrent jobs to process
|
||||
pub concurrency: usize,
|
||||
/// Sleep duration when no jobs are available
|
||||
pub sleep_duration: Duration,
|
||||
}
|
||||
|
||||
impl Default for WorkerConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
queues: vec!["default".to_string()],
|
||||
concurrency: 4,
|
||||
sleep_duration: Duration::from_secs(1),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Job worker that processes queued jobs.
|
||||
pub struct Worker {
|
||||
redis_pool: Pool,
|
||||
config: WorkerConfig,
|
||||
/// Registry of job handlers by name
|
||||
handlers: std::collections::HashMap<&'static str, Box<dyn JobHandler>>,
|
||||
}
|
||||
|
||||
/// Trait for job handlers (type-erased).
|
||||
#[async_trait::async_trait]
|
||||
pub trait JobHandler: Send + Sync {
|
||||
async fn process(&self, payload: &str) -> anyhow::Result<()>;
|
||||
}
|
||||
|
||||
impl Worker {
|
||||
/// Create a new worker.
|
||||
pub fn new(redis_pool: Pool, config: WorkerConfig) -> Self {
|
||||
Self {
|
||||
redis_pool,
|
||||
config,
|
||||
handlers: std::collections::HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Register a job handler.
|
||||
pub fn register<H: JobHandler + 'static>(&mut self, name: &'static str, handler: H) {
|
||||
self.handlers.insert(name, Box::new(handler));
|
||||
}
|
||||
|
||||
/// Run the worker loop.
|
||||
pub async fn run(&self) -> anyhow::Result<()> {
|
||||
tracing::info!(
|
||||
"🔧 Worker started - processing queues: {:?}",
|
||||
self.config.queues
|
||||
);
|
||||
|
||||
loop {
|
||||
let mut processed = false;
|
||||
|
||||
for queue in &self.config.queues {
|
||||
if let Some(job_id) = self.pop_job(queue).await? {
|
||||
self.process_job(queue, &job_id).await?;
|
||||
processed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if !processed {
|
||||
// No jobs available, sleep
|
||||
sleep(self.config.sleep_duration).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Pop a job from the queue.
|
||||
async fn pop_job(&self, queue: &str) -> anyhow::Result<Option<String>> {
|
||||
let queue_key = format!("jobs:queue:{}", queue);
|
||||
let mut conn = self.redis_pool.get().await?;
|
||||
|
||||
let job_id: Option<String> = conn.lpop(&queue_key, None).await?;
|
||||
Ok(job_id)
|
||||
}
|
||||
|
||||
/// Process a single job.
|
||||
async fn process_job(&self, queue: &str, job_id: &str) -> anyhow::Result<()> {
|
||||
let job_key = format!("jobs:data:{}", job_id);
|
||||
let meta_key = format!("{}:meta", job_key);
|
||||
|
||||
let mut conn = self.redis_pool.get().await?;
|
||||
|
||||
// Get job metadata
|
||||
let meta_json: Option<String> = conn.get(&meta_key).await?;
|
||||
let mut meta: JobMeta = match meta_json {
|
||||
Some(json) => serde_json::from_str(&json)?,
|
||||
None => {
|
||||
tracing::warn!("Job {} not found", job_id);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
// Get job payload
|
||||
let payload: Option<String> = conn.get(&job_key).await?;
|
||||
let payload = match payload {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
tracing::warn!("Job {} payload not found", job_id);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
// Update status to processing
|
||||
meta.status = JobStatus::Processing;
|
||||
meta.started_at = Some(chrono::Utc::now());
|
||||
meta.attempts += 1;
|
||||
let _: () = conn.set(&meta_key, serde_json::to_string(&meta)?).await?;
|
||||
|
||||
tracing::info!(
|
||||
"Processing job {} ({}) - attempt {}/{}",
|
||||
meta.job_type,
|
||||
job_id,
|
||||
meta.attempts,
|
||||
meta.max_attempts
|
||||
);
|
||||
|
||||
// Find handler
|
||||
let handler = match self.handlers.get(meta.job_type.as_str()) {
|
||||
Some(h) => h,
|
||||
None => {
|
||||
tracing::error!("No handler registered for job type: {}", meta.job_type);
|
||||
meta.status = JobStatus::Failed;
|
||||
meta.error = Some(format!("No handler for job type: {}", meta.job_type));
|
||||
meta.completed_at = Some(chrono::Utc::now());
|
||||
let _: () = conn.set(&meta_key, serde_json::to_string(&meta)?).await?;
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
// Execute job
|
||||
match handler.process(&payload).await {
|
||||
Ok(()) => {
|
||||
meta.status = JobStatus::Completed;
|
||||
meta.completed_at = Some(chrono::Utc::now());
|
||||
tracing::info!("Job {} completed successfully", job_id);
|
||||
}
|
||||
Err(e) => {
|
||||
let error_msg = e.to_string();
|
||||
tracing::error!("Job {} failed: {}", job_id, error_msg);
|
||||
|
||||
if meta.attempts >= meta.max_attempts {
|
||||
meta.status = JobStatus::Failed;
|
||||
meta.error = Some(error_msg);
|
||||
meta.completed_at = Some(chrono::Utc::now());
|
||||
} else {
|
||||
// Retry - push back to queue
|
||||
meta.status = JobStatus::Pending;
|
||||
meta.error = Some(format!("Attempt {} failed: {}", meta.attempts, error_msg));
|
||||
let queue_key = format!("jobs:queue:{}", queue);
|
||||
let _: () = conn.rpush(&queue_key, job_id).await?;
|
||||
tracing::info!("Job {} queued for retry", job_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save final status
|
||||
let _: () = conn.set(&meta_key, serde_json::to_string(&meta)?).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
//! Request/Response logging middleware.
|
||||
//!
|
||||
//! Provides structured logging for HTTP requests with timing and correlation.
|
||||
//!
|
||||
//! # Example
|
||||
//!
|
||||
//! ```ignore
|
||||
//! use scraper_service::shared::middlewares::logging::{LoggingConfig, with_logging};
|
||||
//! use axum::Router;
|
||||
//!
|
||||
//! let app = Router::new()
|
||||
//! .route("/api/test", get(handler))
|
||||
//! .layer(axum::middleware::from_fn(with_logging(LoggingConfig::default())));
|
||||
//! ```
|
||||
|
||||
use axum::{body::Body, extract::Request, http::StatusCode, middleware::Next, response::Response};
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Logging configuration.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct LoggingConfig {
|
||||
/// Log request headers
|
||||
pub log_headers: bool,
|
||||
/// Log request body (be careful with sensitive data)
|
||||
pub log_body: bool,
|
||||
/// Log response body
|
||||
pub log_response_body: bool,
|
||||
/// Maximum body size to log (bytes)
|
||||
pub max_body_size: usize,
|
||||
/// Paths to exclude from logging
|
||||
pub exclude_paths: HashSet<String>,
|
||||
/// Log level for successful requests
|
||||
pub success_level: LogLevel,
|
||||
/// Log level for client errors (4xx)
|
||||
pub client_error_level: LogLevel,
|
||||
/// Log level for server errors (5xx)
|
||||
pub server_error_level: LogLevel,
|
||||
}
|
||||
|
||||
/// Log level enum.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum LogLevel {
|
||||
Trace,
|
||||
Debug,
|
||||
Info,
|
||||
Warn,
|
||||
Error,
|
||||
}
|
||||
|
||||
impl Default for LoggingConfig {
|
||||
fn default() -> Self {
|
||||
let mut exclude = HashSet::new();
|
||||
exclude.insert("/health".to_string());
|
||||
exclude.insert("/favicon.ico".to_string());
|
||||
|
||||
Self {
|
||||
log_headers: false,
|
||||
log_body: false,
|
||||
log_response_body: false,
|
||||
max_body_size: 1024,
|
||||
exclude_paths: exclude,
|
||||
success_level: LogLevel::Info,
|
||||
client_error_level: LogLevel::Warn,
|
||||
server_error_level: LogLevel::Error,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl LoggingConfig {
|
||||
/// Create a new logging config.
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Enable header logging.
|
||||
pub fn with_headers(mut self) -> Self {
|
||||
self.log_headers = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a path to exclude from logging.
|
||||
pub fn exclude_path(mut self, path: &str) -> Self {
|
||||
self.exclude_paths.insert(path.to_string());
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Request ID extension for correlation.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct RequestId(pub String);
|
||||
|
||||
impl RequestId {
|
||||
/// Generate a new request ID.
|
||||
pub fn new() -> Self {
|
||||
Self(Uuid::new_v4().to_string())
|
||||
}
|
||||
|
||||
/// Get the request ID as a string.
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for RequestId {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Logging middleware.
|
||||
pub async fn logging_middleware(config: Arc<LoggingConfig>, req: Request, next: Next) -> Response {
|
||||
let path = req.uri().path();
|
||||
|
||||
if config.exclude_paths.contains(path) {
|
||||
return next.run(req).await;
|
||||
}
|
||||
|
||||
let request_id = req
|
||||
.headers()
|
||||
.get("X-Request-ID")
|
||||
.and_then(|h| h.to_str().ok())
|
||||
.map(|s| RequestId(s.to_string()))
|
||||
.unwrap_or_else(RequestId::new);
|
||||
|
||||
let method = req.method().clone();
|
||||
let uri = req.uri().clone();
|
||||
let path = uri.path().to_string();
|
||||
let query = uri.query().map(str::to_owned);
|
||||
|
||||
let headers_log = if config.log_headers {
|
||||
let headers: Vec<String> = req
|
||||
.headers()
|
||||
.iter()
|
||||
.filter(|(name, _)| !is_sensitive_header(name.as_str()))
|
||||
.map(|(name, value)| format!("{}: {}", name, value.to_str().unwrap_or("<binary>")))
|
||||
.collect();
|
||||
Some(headers)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut req = req;
|
||||
req.extensions_mut().insert(request_id.clone());
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
tracing::info!(
|
||||
request_id = %request_id.0,
|
||||
method = %method,
|
||||
path = %path,
|
||||
query = ?query,
|
||||
"→ Request started"
|
||||
);
|
||||
|
||||
if let Some(ref headers) = headers_log {
|
||||
tracing::debug!(request_id = %request_id.0, headers = ?headers, "Request headers");
|
||||
}
|
||||
|
||||
let response = next.run(req).await;
|
||||
|
||||
let duration = start.elapsed();
|
||||
let status = response.status();
|
||||
|
||||
match status.as_u16() {
|
||||
100..=399 => log_response(
|
||||
config.success_level,
|
||||
&request_id,
|
||||
status,
|
||||
duration,
|
||||
&method,
|
||||
&path,
|
||||
),
|
||||
400..=499 => log_response(
|
||||
config.client_error_level,
|
||||
&request_id,
|
||||
status,
|
||||
duration,
|
||||
&method,
|
||||
&path,
|
||||
),
|
||||
_ => log_response(
|
||||
config.server_error_level,
|
||||
&request_id,
|
||||
status,
|
||||
duration,
|
||||
&method,
|
||||
&path,
|
||||
),
|
||||
}
|
||||
|
||||
response
|
||||
}
|
||||
|
||||
fn is_sensitive_header(name: &str) -> bool {
|
||||
name.eq_ignore_ascii_case("authorization")
|
||||
|| name.eq_ignore_ascii_case("cookie")
|
||||
|| name.eq_ignore_ascii_case("x-api-key")
|
||||
}
|
||||
|
||||
fn log_response(
|
||||
level: LogLevel,
|
||||
request_id: &RequestId,
|
||||
status: StatusCode,
|
||||
duration: std::time::Duration,
|
||||
method: &axum::http::Method,
|
||||
path: &str,
|
||||
) {
|
||||
let duration_ms = duration.as_millis();
|
||||
|
||||
match level {
|
||||
LogLevel::Trace => tracing::trace!(
|
||||
request_id = %request_id.0,
|
||||
status = %status,
|
||||
duration_ms = %duration_ms,
|
||||
method = %method,
|
||||
path = %path,
|
||||
"← Response completed"
|
||||
),
|
||||
LogLevel::Debug => tracing::debug!(
|
||||
request_id = %request_id.0,
|
||||
status = %status,
|
||||
duration_ms = %duration_ms,
|
||||
method = %method,
|
||||
path = %path,
|
||||
"← Response completed"
|
||||
),
|
||||
LogLevel::Info => tracing::info!(
|
||||
request_id = %request_id.0,
|
||||
status = %status,
|
||||
duration_ms = %duration_ms,
|
||||
method = %method,
|
||||
path = %path,
|
||||
"← Response completed"
|
||||
),
|
||||
LogLevel::Warn => tracing::warn!(
|
||||
request_id = %request_id.0,
|
||||
status = %status,
|
||||
duration_ms = %duration_ms,
|
||||
method = %method,
|
||||
path = %path,
|
||||
"← Response completed"
|
||||
),
|
||||
LogLevel::Error => tracing::error!(
|
||||
request_id = %request_id.0,
|
||||
status = %status,
|
||||
duration_ms = %duration_ms,
|
||||
method = %method,
|
||||
path = %path,
|
||||
"← Response completed"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create logging middleware.
|
||||
pub fn with_logging(
|
||||
config: LoggingConfig,
|
||||
) -> impl Fn(
|
||||
Request<Body>,
|
||||
Next,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Response> + Send>>
|
||||
+ Clone
|
||||
+ Send {
|
||||
let config = Arc::new(config);
|
||||
move |req: Request<Body>, next: Next| {
|
||||
let config = config.clone();
|
||||
Box::pin(async move { logging_middleware(config, req, next).await })
|
||||
}
|
||||
}
|
||||
|
||||
/// Extractor for RequestId in route handlers.
|
||||
impl<S> axum::extract::FromRequestParts<S> for RequestId
|
||||
where
|
||||
S: Send + Sync,
|
||||
{
|
||||
type Rejection = (StatusCode, &'static str);
|
||||
|
||||
async fn from_request_parts(
|
||||
parts: &mut axum::http::request::Parts,
|
||||
_state: &S,
|
||||
) -> Result<Self, Self::Rejection> {
|
||||
parts.extensions.get::<RequestId>().cloned().ok_or((
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"RequestId not found. Did you add logging middleware?",
|
||||
))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
pub mod ratelimit;
|
||||
pub use ratelimit::rate_limit_middleware;
|
||||
@@ -0,0 +1,101 @@
|
||||
//! Rate limiter middleware using the governor crate.
|
||||
//!
|
||||
//! Provides token-bucket based rate limiting with configurable limits.
|
||||
//! Default: 20 requests per IP address, burst 50.
|
||||
|
||||
use axum::{
|
||||
extract::Request,
|
||||
http::StatusCode,
|
||||
middleware::Next,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use governor::{
|
||||
clock::DefaultClock, state::keyed::DefaultKeyedStateStore, Quota,
|
||||
RateLimiter as GovernorRateLimiter,
|
||||
};
|
||||
use once_cell::sync::Lazy;
|
||||
use serde_json::json;
|
||||
use std::{num::NonZeroU32, sync::Arc, time::Duration};
|
||||
use tracing::warn;
|
||||
|
||||
/// Global IP-keyed rate limiter instance.
|
||||
/// Configured for 20 requests per second with a burst of 50.
|
||||
static IP_LIMITER: Lazy<
|
||||
Arc<GovernorRateLimiter<String, DefaultKeyedStateStore<String>, DefaultClock>>,
|
||||
> = Lazy::new(|| {
|
||||
let quota = Quota::with_period(Duration::from_millis(50))
|
||||
.unwrap()
|
||||
.allow_burst(NonZeroU32::new(50).unwrap());
|
||||
|
||||
Arc::new(GovernorRateLimiter::keyed(quota))
|
||||
});
|
||||
|
||||
/// Rate limiter configuration.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RateLimiterConfig {
|
||||
/// Requests per second
|
||||
pub requests_per_second: u32,
|
||||
/// Burst size (max requests that can be made instantly)
|
||||
pub burst_size: u32,
|
||||
}
|
||||
|
||||
impl Default for RateLimiterConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
requests_per_second: 20,
|
||||
burst_size: 50,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a custom rate limiter with specific configuration.
|
||||
pub fn create_rate_limiter(
|
||||
config: RateLimiterConfig,
|
||||
) -> Arc<GovernorRateLimiter<String, DefaultKeyedStateStore<String>, DefaultClock>> {
|
||||
let period_ms = 1000 / config.requests_per_second.max(1);
|
||||
let quota = Quota::with_period(Duration::from_millis(period_ms as u64))
|
||||
.unwrap()
|
||||
.allow_burst(NonZeroU32::new(config.burst_size.max(1)).unwrap());
|
||||
|
||||
Arc::new(GovernorRateLimiter::keyed(quota))
|
||||
}
|
||||
|
||||
fn extract_ip(req: &Request) -> String {
|
||||
if let Some(ip) = req.headers().get("X-Forwarded-For") {
|
||||
if let Ok(ip_str) = ip.to_str() {
|
||||
if let Some(first_ip) = ip_str.split(',').next() {
|
||||
return first_ip.trim().to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(ip) = req.headers().get("X-Real-IP") {
|
||||
if let Ok(ip_str) = ip.to_str() {
|
||||
return ip_str.trim().to_string();
|
||||
}
|
||||
}
|
||||
"unknown".to_string()
|
||||
}
|
||||
|
||||
/// Rate limiting middleware using the IP-keyed limiter.
|
||||
///
|
||||
/// Returns 429 Too Many Requests if the limit is exceeded.
|
||||
pub async fn rate_limit_middleware(req: Request, next: Next) -> Response {
|
||||
let client_ip = extract_ip(&req);
|
||||
|
||||
match IP_LIMITER.check_key(&client_ip) {
|
||||
Ok(_) => next.run(req).await,
|
||||
Err(_) => {
|
||||
warn!("Rate limit exceeded for IP: {}", client_ip);
|
||||
(
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
Json(json!({
|
||||
"error": "Too many requests",
|
||||
"code": "RATE_LIMIT_EXCEEDED",
|
||||
"retry_after_ms": 1000
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
pub mod browser;
|
||||
pub mod config;
|
||||
pub mod database;
|
||||
pub mod errors;
|
||||
pub mod events;
|
||||
pub mod graceful;
|
||||
pub mod health;
|
||||
pub mod middlewares;
|
||||
pub mod observability;
|
||||
pub mod routing;
|
||||
pub mod scheduler;
|
||||
pub mod scrapers;
|
||||
pub mod services;
|
||||
pub mod state;
|
||||
pub mod testing;
|
||||
pub mod types;
|
||||
pub mod utils;
|
||||
@@ -0,0 +1,171 @@
|
||||
//! OpenTelemetry metrics initialization for the scraper service.
|
||||
//!
|
||||
//! Provides:
|
||||
//! - Global `MeterProvider` connected via OTLP gRPC to the metrics backend
|
||||
//! - Standard HTTP server metrics middleware
|
||||
//!
|
||||
//! Environment:
|
||||
//! OTEL_EXPORTER_OTLP_ENDPOINT — default: http://localhost:4317
|
||||
//! OTEL_SERVICE_NAME — default: scraper-api
|
||||
//! OTEL_METRICS_EXPORT_INTERVAL — default: 5000 (ms)
|
||||
|
||||
use axum::{extract::Request, middleware::Next, response::Response};
|
||||
use opentelemetry::{
|
||||
global,
|
||||
metrics::{Counter, Histogram, Meter, UpDownCounter},
|
||||
KeyValue,
|
||||
};
|
||||
use opentelemetry_otlp::WithExportConfig;
|
||||
use opentelemetry_sdk::{metrics::MeterProviderBuilder, metrics::PeriodicReader, Resource};
|
||||
use std::sync::OnceLock;
|
||||
use std::time::Instant;
|
||||
|
||||
static METER: OnceLock<Meter> = OnceLock::new();
|
||||
static PROVIDER: OnceLock<opentelemetry_sdk::metrics::SdkMeterProvider> = OnceLock::new();
|
||||
|
||||
fn meter() -> &'static Meter {
|
||||
METER.get().expect("OTel meter not initialized — call init_otel_metrics first")
|
||||
}
|
||||
|
||||
/// Initialize the global OTLP MeterProvider.
|
||||
/// Safe to call multiple times — subsequent calls are no-ops.
|
||||
pub fn init_otel_metrics() {
|
||||
if METER.get().is_some() {
|
||||
return;
|
||||
}
|
||||
|
||||
let otel_endpoint =
|
||||
std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").unwrap_or_else(|_| "http://localhost:4317".into());
|
||||
let service_name =
|
||||
std::env::var("OTEL_SERVICE_NAME").unwrap_or_else(|_| "scraper-api".into());
|
||||
let export_interval_ms: u64 = std::env::var("OTEL_METRICS_EXPORT_INTERVAL")
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(5000);
|
||||
|
||||
// Build the gRPC OTLP exporter
|
||||
let exporter = opentelemetry_otlp::MetricExporter::builder()
|
||||
.with_tonic()
|
||||
.with_endpoint(otel_endpoint.clone())
|
||||
.build()
|
||||
.expect("Failed to create OTLP metric exporter");
|
||||
|
||||
let reader = PeriodicReader::builder(exporter, opentelemetry_sdk::runtime::Tokio)
|
||||
.with_interval(std::time::Duration::from_millis(export_interval_ms))
|
||||
.build();
|
||||
|
||||
let resource = Resource::new(vec![
|
||||
KeyValue::new("service.name", service_name.clone()),
|
||||
]);
|
||||
|
||||
let provider = MeterProviderBuilder::default()
|
||||
.with_resource(resource)
|
||||
.with_reader(reader)
|
||||
.build();
|
||||
|
||||
// Keep a handle so we can shut it down later
|
||||
let _ = PROVIDER.set(provider.clone());
|
||||
|
||||
global::set_meter_provider(provider);
|
||||
|
||||
let m = global::meter("scraper-http-server");
|
||||
let _ = METER.set(m);
|
||||
|
||||
tracing::info!(otel_endpoint, service_name, "OTel metrics initialized");
|
||||
}
|
||||
|
||||
/// Shut down the global MeterProvider, flushing pending exports.
|
||||
pub async fn shutdown_otel_metrics() {
|
||||
if let Some(provider) = PROVIDER.get() {
|
||||
if let Err(e) = provider.shutdown() {
|
||||
tracing::warn!(error = %e, "OTel metrics shutdown error");
|
||||
} else {
|
||||
tracing::info!("OTel metrics shut down");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// HTTP metrics middleware
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Axum middleware that records standard HTTP server metrics for every request.
|
||||
///
|
||||
/// This middleware must be added as a layer **after** `init_otel_metrics()` has been called.
|
||||
///
|
||||
/// Metrics emitted:
|
||||
/// - `http.server.request_count` — Counter { method, path, status }
|
||||
/// - `http.server.request_duration_ms` — Histogram { method, path, status }
|
||||
/// - `http.server.request_in_flight` — UpDownCounter { method, path }
|
||||
pub async fn otel_metrics_middleware(req: Request, next: Next) -> Response {
|
||||
let method = req.method().to_string();
|
||||
let path = req.uri().path().to_string();
|
||||
|
||||
let in_flight = get_in_flight_counter();
|
||||
let counter = get_request_counter();
|
||||
let duration = get_duration_histogram();
|
||||
|
||||
// Record in-flight
|
||||
in_flight.add(
|
||||
1,
|
||||
&[
|
||||
KeyValue::new("method", method.clone()),
|
||||
KeyValue::new("path", path.clone()),
|
||||
],
|
||||
);
|
||||
|
||||
let start = Instant::now();
|
||||
let response = next.run(req).await;
|
||||
let elapsed_ms = start.elapsed().as_millis() as f64;
|
||||
|
||||
let status = response.status().as_u16().to_string();
|
||||
|
||||
// Record request count + duration
|
||||
let attrs = [
|
||||
KeyValue::new("method", method),
|
||||
KeyValue::new("path", path),
|
||||
KeyValue::new("status", status),
|
||||
];
|
||||
counter.add(1, &attrs);
|
||||
duration.record(elapsed_ms, &attrs);
|
||||
|
||||
// Decrement in-flight
|
||||
in_flight.add(-1, &[]);
|
||||
|
||||
response
|
||||
}
|
||||
|
||||
/// Lazily-initialised instruments guarded by OnceLock.
|
||||
fn get_in_flight_counter() -> UpDownCounter<i64> {
|
||||
static INST: OnceLock<UpDownCounter<i64>> = OnceLock::new();
|
||||
INST.get_or_init(|| {
|
||||
meter()
|
||||
.i64_up_down_counter("http.server.request_in_flight")
|
||||
.with_description("Number of HTTP requests currently in flight")
|
||||
.build()
|
||||
})
|
||||
.clone()
|
||||
}
|
||||
|
||||
fn get_request_counter() -> Counter<u64> {
|
||||
static INST: OnceLock<Counter<u64>> = OnceLock::new();
|
||||
INST.get_or_init(|| {
|
||||
meter()
|
||||
.u64_counter("http.server.request_count")
|
||||
.with_description("Total number of HTTP requests received")
|
||||
.build()
|
||||
})
|
||||
.clone()
|
||||
}
|
||||
|
||||
fn get_duration_histogram() -> Histogram<f64> {
|
||||
static INST: OnceLock<Histogram<f64>> = OnceLock::new();
|
||||
INST.get_or_init(|| {
|
||||
meter()
|
||||
.f64_histogram("http.server.request_duration_ms")
|
||||
.with_description("Duration of HTTP requests in milliseconds")
|
||||
.with_unit("ms")
|
||||
.build()
|
||||
})
|
||||
.clone()
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
pub mod metrics;
|
||||
pub mod openapi;
|
||||
pub mod openapi_modules;
|
||||
pub mod request_id;
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user