From 80c96eaa429291962b58cd3bc720c638251fb8bb Mon Sep 17 00:00:00 2001 From: asepharyana Date: Thu, 9 Jul 2026 22:07:03 +0700 Subject: [PATCH] chore: initial commit for asepharyana-hub-scraper --- .github/workflows/notify-parent.yml | 24 + .gitignore | 67 + AGENT.md | 27 + CLAUDE.md | 177 + Cargo.lock | 5564 +++++++++++++++++ Cargo.toml | 132 + GEMINI.md | 69 + README.md | 55 + check_image_cache_db.sh | 73 + .../2026-05-08-refactor-clean-architecture.md | 167 + .../2026-05-08-clean-architecture-design.md | 49 + ecosystem.config.cjs | 26 + package.json | 8 + rustfmt.toml | 15 + .../compare-openapi.cpython-312.pyc | Bin 0 -> 4492 bytes scripts/auto-lint.sh | 20 + scripts/compare-openapi.py | 84 + scripts/migrate.sh | 15 + scripts/pre-build-lint.sh | 19 + src/app.rs | 51 + src/bin/capture_warning.rs | 117 + src/bin/foster_parenting_assertion.rs | 132 + src/bin/scaffold_enhanced/generators/api.rs | 83 + .../generators/controller.rs | 287 + .../scaffold_enhanced/generators/migration.rs | 262 + src/bin/scaffold_enhanced/generators/mod.rs | 7 + src/bin/scaffold_enhanced/generators/model.rs | 125 + .../generators/repository.rs | 112 + .../scaffold_enhanced/generators/service.rs | 86 + .../foster_parenting_minimal.html | 1 + src/bootstrap/mod.rs | 118 + src/lib.rs | 10 + src/main.rs | 12 + src/modules/anime/controller.rs | 236 + src/modules/anime/mod.rs | 7 + src/modules/anime/parser.rs | 632 ++ src/modules/anime/repository.rs | 199 + src/modules/anime/route.rs | 49 + src/modules/anime/schema.rs | 18 + src/modules/anime/scraping/cache.rs | 81 + src/modules/anime/service.rs | 295 + src/modules/anime/types.rs | 231 + src/modules/anime2/controller.rs | 276 + src/modules/anime2/mod.rs | 7 + src/modules/anime2/parser.rs | 804 +++ src/modules/anime2/repository.rs | 102 + src/modules/anime2/route.rs | 39 + src/modules/anime2/schema.rs | 34 + src/modules/anime2/scraping.rs | 359 ++ src/modules/anime2/service.rs | 453 ++ src/modules/anime2/types.rs | 106 + src/modules/komik/controller.rs | 217 + src/modules/komik/mod.rs | 7 + src/modules/komik/parser.rs | 592 ++ src/modules/komik/repository.rs | 78 + src/modules/komik/route.rs | 27 + src/modules/komik/schema.rs | 18 + src/modules/komik/service.rs | 421 ++ src/modules/komik/types.rs | 118 + src/modules/mod.rs | 17 + src/modules/proxy/controller.rs | 75 + src/modules/proxy/mod.rs | 7 + src/modules/proxy/parser.rs | 1 + src/modules/proxy/repository.rs | 29 + src/modules/proxy/route.rs | 16 + src/modules/proxy/schema.rs | 27 + src/modules/proxy/service.rs | 232 + src/modules/proxy/types.rs | 24 + src/shared/browser/mod.rs | 3 + src/shared/browser/pool.rs | 483 ++ src/shared/config/mod.rs | 289 + src/shared/database/mod.rs | 7 + .../persistence/entities/image_cache.rs | 71 + .../database/persistence/entities/mod.rs | 1 + src/shared/database/persistence/mod.rs | 1 + src/shared/database/redis.rs | 74 + .../database/repositories/image_cache.rs | 129 + src/shared/database/repositories/mod.rs | 1 + src/shared/database/setup.rs | 44 + src/shared/database/traits/image_cache.rs | 16 + src/shared/database/traits/mod.rs | 4 + .../database/traits/scraping_repository.rs | 7 + src/shared/errors/app_error.rs | 88 + src/shared/errors/mod.rs | 2 + src/shared/events/bus.rs | 155 + src/shared/events/mod.rs | 1 + src/shared/graceful/cleanup.rs | 142 + src/shared/graceful/mod.rs | 2 + src/shared/graceful/shutdown.rs | 116 + src/shared/health/endpoints.rs | 138 + src/shared/health/mod.rs | 1 + src/shared/jobs/mod.rs | 2 + src/shared/jobs/queue.rs | 204 + src/shared/jobs/worker.rs | 180 + src/shared/middlewares/logging.rs | 289 + src/shared/middlewares/mod.rs | 2 + src/shared/middlewares/ratelimit.rs | 101 + src/shared/mod.rs | 17 + src/shared/observability/metrics.rs | 171 + src/shared/observability/mod.rs | 4 + src/shared/observability/openapi.rs | 14 + src/shared/observability/openapi_modules.rs | 85 + src/shared/observability/request_id.rs | 85 + src/shared/routing/mod.rs | 3 + src/shared/routing/versioning.rs | 135 + src/shared/scheduler/cleanup_cache.rs | 222 + src/shared/scheduler/mod.rs | 5 + src/shared/scheduler/runner.rs | 93 + src/shared/scrapers/mod.rs | 1 + src/shared/scrapers/otakudesu.rs | 115 + src/shared/services/images/cache.rs | 1140 ++++ src/shared/services/images/mod.rs | 1 + src/shared/services/mod.rs | 1 + src/shared/state/mod.rs | 17 + src/shared/testing/app.rs | 226 + src/shared/testing/mod.rs | 1 + src/shared/types/api_response.rs | 28 + src/shared/types/entities/anime.rs | 208 + src/shared/types/entities/image.rs | 11 + src/shared/types/entities/mod.rs | 3 + src/shared/types/entities/types.rs | 56 + src/shared/types/mod.rs | 4 + src/shared/utils/core/api_response.rs | 301 + src/shared/utils/core/errors.rs | 101 + src/shared/utils/core/handler.rs | 99 + src/shared/utils/core/mod.rs | 6 + src/shared/utils/core/pagination.rs | 105 + src/shared/utils/core/prelude.rs | 33 + src/shared/utils/core/response.rs | 148 + src/shared/utils/data/collections.rs | 169 + src/shared/utils/data/convert/bools.rs | 84 + src/shared/utils/data/convert/bytes.rs | 130 + src/shared/utils/data/convert/char.rs | 65 + src/shared/utils/data/convert/collections.rs | 103 + src/shared/utils/data/convert/color.rs | 70 + src/shared/utils/data/convert/mod.rs | 62 + src/shared/utils/data/convert/network.rs | 60 + src/shared/utils/data/convert/numeric.rs | 532 ++ src/shared/utils/data/convert/path.rs | 62 + src/shared/utils/data/convert/pointers.rs | 58 + src/shared/utils/data/convert/result.rs | 39 + src/shared/utils/data/convert/string.rs | 134 + src/shared/utils/data/convert/time.rs | 142 + src/shared/utils/data/datetime.rs | 107 + src/shared/utils/data/json.rs | 137 + src/shared/utils/data/mod.rs | 7 + src/shared/utils/data/numbers.rs | 151 + src/shared/utils/data/string.rs | 127 + src/shared/utils/data/text.rs | 163 + src/shared/utils/dev/async_utils.rs | 144 + src/shared/utils/dev/logging.rs | 146 + src/shared/utils/dev/mod.rs | 6 + src/shared/utils/dev/performance.rs | 88 + src/shared/utils/dev/result_ext.rs | 169 + src/shared/utils/dev/serde_helpers.rs | 183 + src/shared/utils/dev/testing.rs | 205 + src/shared/utils/infra/bulk.rs | 166 + src/shared/utils/infra/console.rs | 274 + src/shared/utils/infra/encryption.rs | 214 + src/shared/utils/infra/env.rs | 117 + src/shared/utils/infra/form_request.rs | 302 + src/shared/utils/infra/health_check.rs | 256 + src/shared/utils/infra/import_export.rs | 223 + src/shared/utils/infra/mod.rs | 14 + src/shared/utils/infra/query_profiler.rs | 277 + src/shared/utils/infra/resource.rs | 198 + src/shared/utils/infra/ryzen_cdn.rs | 67 + src/shared/utils/infra/searchable.rs | 190 + src/shared/utils/infra/transaction.rs | 179 + src/shared/utils/infra/uuid_utils.rs | 78 + src/shared/utils/infra/versioning.rs | 191 + src/shared/utils/io/cache.rs | 155 + src/shared/utils/io/cache_tags.rs | 306 + src/shared/utils/io/cache_ttl.rs | 84 + src/shared/utils/io/file.rs | 132 + src/shared/utils/io/mod.rs | 6 + src/shared/utils/io/retry.rs | 66 + src/shared/utils/io/soft_delete.rs | 86 + src/shared/utils/mod.rs | 190 + src/shared/utils/web/http.rs | 26 + src/shared/utils/web/http_client.rs | 135 + src/shared/utils/web/mod.rs | 8 + src/shared/utils/web/proxy_fetch.rs | 399 ++ src/shared/utils/web/query.rs | 278 + src/shared/utils/web/request.rs | 141 + src/shared/utils/web/scraping.rs | 189 + src/shared/utils/web/scraping_urls.rs | 30 + src/shared/utils/web/url.rs | 159 + src/shared/utils/web/validation.rs | 156 + tools/auto-lint/Cargo.toml | 10 + 190 files changed, 28465 insertions(+) create mode 100644 .github/workflows/notify-parent.yml create mode 100644 .gitignore create mode 100644 AGENT.md create mode 100644 CLAUDE.md create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 GEMINI.md create mode 100644 README.md create mode 100755 check_image_cache_db.sh create mode 100644 docs/superpowers/plans/2026-05-08-refactor-clean-architecture.md create mode 100644 docs/superpowers/specs/2026-05-08-clean-architecture-design.md create mode 100644 ecosystem.config.cjs create mode 100644 package.json create mode 100644 rustfmt.toml create mode 100644 scripts/__pycache__/compare-openapi.cpython-312.pyc create mode 100755 scripts/auto-lint.sh create mode 100755 scripts/compare-openapi.py create mode 100755 scripts/migrate.sh create mode 100755 scripts/pre-build-lint.sh create mode 100644 src/app.rs create mode 100644 src/bin/capture_warning.rs create mode 100644 src/bin/foster_parenting_assertion.rs create mode 100644 src/bin/scaffold_enhanced/generators/api.rs create mode 100644 src/bin/scaffold_enhanced/generators/controller.rs create mode 100644 src/bin/scaffold_enhanced/generators/migration.rs create mode 100644 src/bin/scaffold_enhanced/generators/mod.rs create mode 100644 src/bin/scaffold_enhanced/generators/model.rs create mode 100644 src/bin/scaffold_enhanced/generators/repository.rs create mode 100644 src/bin/scaffold_enhanced/generators/service.rs create mode 100644 src/bin/test_fixtures/foster_parenting_minimal.html create mode 100644 src/bootstrap/mod.rs create mode 100644 src/lib.rs create mode 100644 src/main.rs create mode 100644 src/modules/anime/controller.rs create mode 100644 src/modules/anime/mod.rs create mode 100644 src/modules/anime/parser.rs create mode 100644 src/modules/anime/repository.rs create mode 100644 src/modules/anime/route.rs create mode 100644 src/modules/anime/schema.rs create mode 100644 src/modules/anime/scraping/cache.rs create mode 100644 src/modules/anime/service.rs create mode 100644 src/modules/anime/types.rs create mode 100644 src/modules/anime2/controller.rs create mode 100644 src/modules/anime2/mod.rs create mode 100644 src/modules/anime2/parser.rs create mode 100644 src/modules/anime2/repository.rs create mode 100644 src/modules/anime2/route.rs create mode 100644 src/modules/anime2/schema.rs create mode 100644 src/modules/anime2/scraping.rs create mode 100644 src/modules/anime2/service.rs create mode 100644 src/modules/anime2/types.rs create mode 100644 src/modules/komik/controller.rs create mode 100644 src/modules/komik/mod.rs create mode 100644 src/modules/komik/parser.rs create mode 100644 src/modules/komik/repository.rs create mode 100644 src/modules/komik/route.rs create mode 100644 src/modules/komik/schema.rs create mode 100644 src/modules/komik/service.rs create mode 100644 src/modules/komik/types.rs create mode 100644 src/modules/mod.rs create mode 100644 src/modules/proxy/controller.rs create mode 100644 src/modules/proxy/mod.rs create mode 100644 src/modules/proxy/parser.rs create mode 100644 src/modules/proxy/repository.rs create mode 100644 src/modules/proxy/route.rs create mode 100644 src/modules/proxy/schema.rs create mode 100644 src/modules/proxy/service.rs create mode 100644 src/modules/proxy/types.rs create mode 100644 src/shared/browser/mod.rs create mode 100644 src/shared/browser/pool.rs create mode 100644 src/shared/config/mod.rs create mode 100644 src/shared/database/mod.rs create mode 100644 src/shared/database/persistence/entities/image_cache.rs create mode 100644 src/shared/database/persistence/entities/mod.rs create mode 100644 src/shared/database/persistence/mod.rs create mode 100644 src/shared/database/redis.rs create mode 100644 src/shared/database/repositories/image_cache.rs create mode 100644 src/shared/database/repositories/mod.rs create mode 100644 src/shared/database/setup.rs create mode 100644 src/shared/database/traits/image_cache.rs create mode 100644 src/shared/database/traits/mod.rs create mode 100644 src/shared/database/traits/scraping_repository.rs create mode 100644 src/shared/errors/app_error.rs create mode 100644 src/shared/errors/mod.rs create mode 100644 src/shared/events/bus.rs create mode 100644 src/shared/events/mod.rs create mode 100644 src/shared/graceful/cleanup.rs create mode 100644 src/shared/graceful/mod.rs create mode 100644 src/shared/graceful/shutdown.rs create mode 100644 src/shared/health/endpoints.rs create mode 100644 src/shared/health/mod.rs create mode 100644 src/shared/jobs/mod.rs create mode 100644 src/shared/jobs/queue.rs create mode 100644 src/shared/jobs/worker.rs create mode 100644 src/shared/middlewares/logging.rs create mode 100644 src/shared/middlewares/mod.rs create mode 100644 src/shared/middlewares/ratelimit.rs create mode 100644 src/shared/mod.rs create mode 100644 src/shared/observability/metrics.rs create mode 100644 src/shared/observability/mod.rs create mode 100644 src/shared/observability/openapi.rs create mode 100644 src/shared/observability/openapi_modules.rs create mode 100644 src/shared/observability/request_id.rs create mode 100644 src/shared/routing/mod.rs create mode 100644 src/shared/routing/versioning.rs create mode 100644 src/shared/scheduler/cleanup_cache.rs create mode 100644 src/shared/scheduler/mod.rs create mode 100644 src/shared/scheduler/runner.rs create mode 100644 src/shared/scrapers/mod.rs create mode 100644 src/shared/scrapers/otakudesu.rs create mode 100644 src/shared/services/images/cache.rs create mode 100644 src/shared/services/images/mod.rs create mode 100644 src/shared/services/mod.rs create mode 100644 src/shared/state/mod.rs create mode 100644 src/shared/testing/app.rs create mode 100644 src/shared/testing/mod.rs create mode 100644 src/shared/types/api_response.rs create mode 100644 src/shared/types/entities/anime.rs create mode 100644 src/shared/types/entities/image.rs create mode 100644 src/shared/types/entities/mod.rs create mode 100644 src/shared/types/entities/types.rs create mode 100644 src/shared/types/mod.rs create mode 100644 src/shared/utils/core/api_response.rs create mode 100644 src/shared/utils/core/errors.rs create mode 100644 src/shared/utils/core/handler.rs create mode 100644 src/shared/utils/core/mod.rs create mode 100644 src/shared/utils/core/pagination.rs create mode 100644 src/shared/utils/core/prelude.rs create mode 100644 src/shared/utils/core/response.rs create mode 100644 src/shared/utils/data/collections.rs create mode 100644 src/shared/utils/data/convert/bools.rs create mode 100644 src/shared/utils/data/convert/bytes.rs create mode 100644 src/shared/utils/data/convert/char.rs create mode 100644 src/shared/utils/data/convert/collections.rs create mode 100644 src/shared/utils/data/convert/color.rs create mode 100644 src/shared/utils/data/convert/mod.rs create mode 100644 src/shared/utils/data/convert/network.rs create mode 100644 src/shared/utils/data/convert/numeric.rs create mode 100644 src/shared/utils/data/convert/path.rs create mode 100644 src/shared/utils/data/convert/pointers.rs create mode 100644 src/shared/utils/data/convert/result.rs create mode 100644 src/shared/utils/data/convert/string.rs create mode 100644 src/shared/utils/data/convert/time.rs create mode 100644 src/shared/utils/data/datetime.rs create mode 100644 src/shared/utils/data/json.rs create mode 100644 src/shared/utils/data/mod.rs create mode 100644 src/shared/utils/data/numbers.rs create mode 100644 src/shared/utils/data/string.rs create mode 100644 src/shared/utils/data/text.rs create mode 100644 src/shared/utils/dev/async_utils.rs create mode 100644 src/shared/utils/dev/logging.rs create mode 100644 src/shared/utils/dev/mod.rs create mode 100644 src/shared/utils/dev/performance.rs create mode 100644 src/shared/utils/dev/result_ext.rs create mode 100644 src/shared/utils/dev/serde_helpers.rs create mode 100644 src/shared/utils/dev/testing.rs create mode 100644 src/shared/utils/infra/bulk.rs create mode 100644 src/shared/utils/infra/console.rs create mode 100644 src/shared/utils/infra/encryption.rs create mode 100644 src/shared/utils/infra/env.rs create mode 100644 src/shared/utils/infra/form_request.rs create mode 100644 src/shared/utils/infra/health_check.rs create mode 100644 src/shared/utils/infra/import_export.rs create mode 100644 src/shared/utils/infra/mod.rs create mode 100644 src/shared/utils/infra/query_profiler.rs create mode 100644 src/shared/utils/infra/resource.rs create mode 100644 src/shared/utils/infra/ryzen_cdn.rs create mode 100644 src/shared/utils/infra/searchable.rs create mode 100644 src/shared/utils/infra/transaction.rs create mode 100644 src/shared/utils/infra/uuid_utils.rs create mode 100644 src/shared/utils/infra/versioning.rs create mode 100644 src/shared/utils/io/cache.rs create mode 100644 src/shared/utils/io/cache_tags.rs create mode 100644 src/shared/utils/io/cache_ttl.rs create mode 100644 src/shared/utils/io/file.rs create mode 100644 src/shared/utils/io/mod.rs create mode 100644 src/shared/utils/io/retry.rs create mode 100644 src/shared/utils/io/soft_delete.rs create mode 100644 src/shared/utils/mod.rs create mode 100644 src/shared/utils/web/http.rs create mode 100644 src/shared/utils/web/http_client.rs create mode 100644 src/shared/utils/web/mod.rs create mode 100644 src/shared/utils/web/proxy_fetch.rs create mode 100644 src/shared/utils/web/query.rs create mode 100644 src/shared/utils/web/request.rs create mode 100644 src/shared/utils/web/scraping.rs create mode 100644 src/shared/utils/web/scraping_urls.rs create mode 100644 src/shared/utils/web/url.rs create mode 100644 src/shared/utils/web/validation.rs create mode 100644 tools/auto-lint/Cargo.toml diff --git a/.github/workflows/notify-parent.yml b/.github/workflows/notify-parent.yml new file mode 100644 index 0000000..cdbf361 --- /dev/null +++ b/.github/workflows/notify-parent.yml @@ -0,0 +1,24 @@ +name: Notify Parent Repo + +on: + push: + branches: + - main + +jobs: + dispatch: + runs-on: ubuntu-latest + steps: + - name: Trigger root monorepo build + uses: peter-evans/repository-dispatch@v3 + with: + token: ${{ secrets.DISPATCH_TOKEN }} + repository: MythEclipse/ultimate-asepharyana.tech + event-type: submodule-updated + client-payload: | + { + "service": "scraper-api", + "ref": "${{ github.ref }}", + "sha": "${{ github.sha }}", + "actor": "${{ github.actor }}" + } diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c9eb931 --- /dev/null +++ b/.gitignore @@ -0,0 +1,67 @@ +apps/gmw/ +# See https://docs.github.com/en/get-started/getting-started-with-git/ignoring-files for more about ignoring files. + +# compiled output +dist +tmp +out-tsc +error.log +# dependencies +node_modules +.bun/** +.turbo/ +# IDEs and editors +/.idea +.project +.classpath +.c9/ +*.launch +.settings/ +*.sublime-workspace +**/target/** +# IDE - VSCode +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json + +# misc +/.sass-cache +/connect.lock +/coverage +/libpeerconnection.log +npm-debug.log +yarn-error.log +testem.log +/typings + +# System Files +.DS_Store +Thumbs.db +.claude +# Next.js +.next +out +**/.codegraph/** +**/.claude/** +test-output +**/**.env +**/**.env.** +vite.config.*.timestamp* +vitest.config.*.timestamp* + + +storybook-static +~/.bun/** +docs/dependency-map.md +docs/handoff-log.jsonl +docs/observability.md +docs/quality-gates.json +docs/workflow-state.json +docs/todo.md +**/vendor/ + +# moonrepo +.moon/cache +.~moon** \ No newline at end of file diff --git a/AGENT.md b/AGENT.md new file mode 100644 index 0000000..2f12b61 --- /dev/null +++ b/AGENT.md @@ -0,0 +1,27 @@ +# AGENT.md - Universal AI Entry Point + +This document serves as the unified entry point for all AI agents (Gemini, Claude, GPT, etc.) interacting with the **Scraping & CDN Service**. + +## 🚀 Mission Statement + +To provide a specialized, zero-bloat backend engine for high-concurrency web scraping and image persistent caching. + +## 🛑 Global AI Protocols + +As an AI agent, you **MUST** adhere to the following when working on this codebase: + +1. **Professionalism**: Maintain a factual, technical, and objective tone. +2. **No Hyperbole**: Prohibited from using marketing-speak or exaggerated praise (e.g., "amazing", "powerful"). +3. **Minimalism**: Prioritize the **Zero-Bloat Policy**. If a request adds unnecessary complexity or dependencies, challenge the user and suggest a leaner alternative. +4. **Zero Suppression**: Never use suppression flags (`#[allow]`, `@ts-ignore`) to bypass warnings. Fix the underlying logic or types. +5. **Observability**: Ensure critical business flows emit structured logs and request context where needed. + +## 🔗 Technical Context + +- **Architecture**: [GEMINI.md](file:///mnt/code/bp3/ultimate-asepharyana.tech/apps/scraper/GEMINI.md) (Logic flows, tech stack). +- **Maintenance**: [Development Guide](file:///mnt/code/bp3/ultimate-asepharyana.tech/apps/scraper/docs/development.md) (Coding standards). +- **Observability**: [Metrics Guide](file:///mnt/code/bp3/ultimate-asepharyana.tech/apps/scraper/docs/observability.md) (Standard telemetry). + +--- + +*If you are an AI assistant, start by reading [GEMINI.md](file:///mnt/code/bp3/ultimate-asepharyana.tech/apps/scraper/GEMINI.md) for the full architectural context.* diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..0c31d36 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,177 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Scraper service — a Rust/Axum backend for web scraping (anime/komik data extraction) and image proxy/CDN caching. Serves as the backend engine consumed by the `apps/solidjs` frontend. + +## Commands + +```bash +# Development +cargo run # Start server (binds 0.0.0.0:4091) +cargo test # Run all tests +cargo clippy -- -D warnings # Lint (warnings are errors) +cargo fmt # Auto-format all source files + +# Release build (full LTO, single CGU, stripped) +cargo build --release + +# PM2 production +pm2 start ecosystem.config.cjs --env production # Uses target/release/scraper +``` + +## Architecture + +### Modular MVC + Service + Repository + +Setiap module mengikuti arsitektur layered yang identik: + +``` +Request → Router (route.rs) → Controller → Service → Repository → Parser + │ + ├── Redis (L1 cache) + ├── SeaORM/MySQL (L2, image_cache) + └── External HTTP (alqanime.si, picser CDN) +``` + +### Directory Layout + +``` +src/ +├── main.rs # Entry point: builds Application, calls run() +├── lib.rs # Public module declarations +├── app.rs # Router assembly: modules + metrics + swagger + middleware layers +├── bootstrap/mod.rs # Application::build(): tracing, Redis, browser pool, DB, AppState +├── modules/ # Feature modules (vertical slices) +│ ├── anime/ # Otakudesu anime scraper +│ ├── anime2/ # Alqanime.si anime scraper +│ ├── komik/ # Komik scraper +│ └── proxy/ # Image proxy/cache/audit endpoints +└── shared/ # Cross-cutting infrastructure + ├── config/ # Lazy-static AppConfig from env vars (fail-fast at startup) + ├── state/ # AppState (redis_pool, db, semaphore, event_bus) + ├── database/ + │ ├── traits/ # ScrapingRepository, ImageCacheRepository (async_trait) + │ ├── repositories/ # SeaOrmImageCacheRepository (impl ImageCacheRepository) + │ └── persistence/ # SeaORM entities (image_cache) + ├── services/images/ # ImageCache service + apply_cached_posters helper + ├── errors/ # AppError enum → axum IntoResponse (500/404 by variant) + ├── observability/ # Utoipa/Swagger OpenAPI doc + ├── scheduler/ # Cron jobs (daily cache cleanup at 2 AM) + ├── browser/ # Headless Chrome pool for JS-rendered scraping + ├── scrapers/ # Site-specific scrapers (otakudesu) + ├── utils/ # Cache helper, HTTP client, scraping helpers, retry, conversions + ├── middlewares/ # Logging, rate limiting + ├── events/ # EventBus for repair state updates + └── types/ # ApiResponse, shared entity types (HasPoster trait, Pagination) +``` + +### Module Structure (identik untuk setiap module) + +Setiap `src/modules//`: + +| File | Peran | Pola | +|---|---|---| +| `route.rs` | Daftar endpoint, mapping URL → controller | `Router>`, tidak ada logic | +| `controller.rs` | Extract State/Path/Query/Body, panggil service | `Result, AppError>` | +| `service.rs` | Business logic, caching, delegasi ke repository + parser | Struct dengan repo di-inject via constructor `new(repo: XRepository)` | +| `repository.rs` | HTTP fetching, URL builders, DB queries | Struct + `impl ScrapingRepository` trait | +| `parser.rs` | HTML parsing dengan `scraper` crate | Free functions → `Result`, via `spawn_blocking` | +| `schema.rs` | Validasi query/path/body params | Struct `Deserialize` + `ToSchema` | +| `types.rs` | Response structs | `Serialize` + `ToSchema`, `impl HasPoster` jika punya poster | + +### Dependency Injection + +Semua service menerima dependency via constructor: + +```rust +// Controller creates and injects dependencies +let repo = AnimeRepository::new(); +let service = AnimeService::new(repo); +service.get_anime_index(app_state).await.map(Json) + +// Service stores injected repo +pub struct AnimeService { + repository: AnimeRepository, +} +impl AnimeService { + pub fn new(repository: AnimeRepository) -> Self { Self { repository } } +} +``` + +### Image Caching Architecture + +Single unified image cache system: + +1. **Trait**: `ImageCacheRepository` (`shared/database/traits/image_cache.rs`) — Redis ops, DB ops, locks, cache invalidation +2. **Impl**: `SeaOrmImageCacheRepository` (`shared/database/repositories/image_cache.rs`) +3. **Service**: `ImageCache` struct (`shared/services/images/cache.rs`) — download, MIME-verify dengan `infer`, upload ke Picser CDN, verifikasi CDN URL (10 retry dengan backoff) +4. **Concurrency**: `Semaphore` (default 5 concurrent uploads) + request coalescing via `DashMap` +5. **Lazy batch helper**: `cache_image_urls_batch_lazy()` — Redis batch check → DB batch check → background spawn untuk misses + +### HasPoster Trait & apply_cached_posters + +`HasPoster` trait di `shared/types/entities/anime.rs` memungkinkan generic poster caching: + +```rust +pub trait HasPoster { + fn poster(&self) -> &str; + fn set_poster(&mut self, url: String); +} +``` + +Semua item type dengan field `poster` mengimplementasikan trait ini (`OngoingAnimeItem`, `KomikItem`, `FilterAnimeItem`, `Recommendation`, dll). + +`apply_cached_posters()` di `shared/services/images/cache.rs` menerima `&mut [T]` where `T: HasPoster`, menggantikan pola manual ~15 baris yang sebelumnya berulang di setiap service method. + +### ScrapingRepository Trait + +```rust +#[async_trait] +pub trait ScrapingRepository: Send + Sync { + async fn fetch_html(&self, url: &str) -> Result; +} +``` + +Semua module repository (`AnimeRepository`, `Anime2Repository`, `KomikRepository`, `ProxyRepository`) mengimplementasikan trait ini. + +### Error Handling + +`AppError` enum di `src/shared/errors/app_error.rs` — derives `thiserror::Error` dan implements `IntoResponse` (404 untuk `NotFound`, 500 untuk lainnya). + +**Kontrak error per layer:** +- **Parser** → `Result` +- **Repository** → `Result` (via `ScrapingRepository` trait) +- **Service** → `Result` (tidak ada `Result` atau `Box`) +- **Controller** → `Result, AppError>` (kecuali proxy yang return raw `Response`) + +### Configuration + +`src/shared/config/mod.rs` — global `CONFIG` lazy-static loaded from: +1. `.env` file (dotenvy) +2. `config/default.toml` / `config/{RUN_MODE}.toml` +3. Environment variables (`APP__` prefix or legacy `DATABASE_URL`/`JWT_SECRET`/`REDIS_URL`) + +Panics at startup if required config is missing — intentional fail-fast design. + +## Constraints + +- **No suppression flags**: `#[allow(...)]`, `#[ignore]`, `@ts-ignore` are prohibited. Fix the underlying issue. +- **Lint strictness**: `unsafe_code = "forbid"`, `panic = "deny"`, `todo = "deny"`, `unimplemented = "deny"`, `unwrap_used = "warn"`, `expect_used = "warn"` +- **Minimal dependencies**: Before adding a crate, evaluate if existing deps or std can handle it. +- **Dead code**: Remove unused functions, types, modules rather than leaving them. +- **Performance**: Use `spawn_blocking` for CPU-heavy work (HTML parsing). +- **No duplicate infrastructure**: Satu trait, satu impl. Jangan membuat trait/repository duplikat seperti `ImageRepository` dan `ImageCacheRepository` yang berbeda. +- **No thin wrappers**: Hindari wrapper tipis seperti `CacheImageUseCase` yang hanya meneruskan panggilan ke service lain. + +## Useful Endpoints + +- `GET /docs` — Swagger UI +- `GET /api-docs/openapi.json` — OpenAPI spec +- `POST /api/proxy/image-cache` — Cache an image URL +- `POST /api/proxy/image-cache/audit` — Audit/repair cached images +- `GET /api/anime/*` — Otakudesu scraping endpoints +- `GET /api/anime2/*` — Alqanime scraping endpoints +- `GET /api/komik/*` — Komik scraping endpoints diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..c0729af --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,5564 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "version_check", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "async-compression" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core 0.4.5", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit 0.7.3", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower 0.5.3", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +dependencies = [ + "axum-core 0.5.6", + "axum-macros", + "base64", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit 0.8.4", + "memchr", + "mime", + "multer", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sha1", + "sync_wrapper", + "tokio", + "tokio-tungstenite", + "tower 0.5.3", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "futures-core", + "getrandom 0.2.17", + "instant", + "pin-project-lite", + "rand 0.8.5", + "tokio", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bigdecimal" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +dependencies = [ + "serde_core", +] + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "borsh" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfcfdc083699101d5a7965e49925975f2f55060f94f9a05e7187be95d530ca59" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "brotli" +version = "8.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytecheck" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cc" +version = "1.2.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfb" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38f2da7a0a2c4ccf0065be06397cc26a81f4e528be095826eee9d4adbb8c60f" +dependencies = [ + "byteorder", + "fnv", + "uuid", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf 0.12.1", +] + +[[package]] +name = "clap" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "compression-codecs" +version = "0.4.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "config" +version = "0.15.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e68cfe19cd7d23ffde002c24ffa5cda73931913ef394d5eaaa32037dc940c0c" +dependencies = [ + "async-trait", + "convert_case", + "json5", + "pathdiff", + "ron", + "rust-ini", + "serde-untagged", + "serde_core", + "serde_json", + "toml", + "winnow", + "yaml-rust2", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "croner" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aa42bcd3d846ebf66e15bd528d1087f75d1c6c1c66ebff626178a106353c576" +dependencies = [ + "chrono", + "derive_builder", + "strum 0.27.2", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "cssparser" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae61cf9c0abb83bd659dab65b7e4e38d8236824c85f0f804f173567bda257d2" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf 0.13.1", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn 2.0.117", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" + +[[package]] +name = "data-url" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be1e0bca6c3637f992fc1cc7cbc52a78c1ef6db076dbf1059c4323d6a2048376" + +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "serde", + "tokio", +] + +[[package]] +name = "deadpool-redis" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4d00bce7a9cfd07ded19530621a7df17c4c3b1c4e8fc2262471de404ce539a" +dependencies = [ + "deadpool", + "redis", + "serde", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" +dependencies = [ + "tokio", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", + "serde", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.117", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.117", + "unicode-xid", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + +[[package]] +name = "ego-tree" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2972feb8dffe7bc8c5463b1dacda1b0dfbed3710e50f977d965429692d74cd8" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +dependencies = [ + "serde", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "governor" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9efcab3c1958580ff1f25a2a41be1668f7603d849bb63af523b208a3cc1223b8" +dependencies = [ + "cfg-if", + "dashmap", + "futures-sink", + "futures-timer", + "futures-util", + "getrandom 0.3.4", + "hashbrown 0.16.1", + "nonzero_ext", + "parking_lot", + "portable-atomic", + "quanta", + "rand 0.9.2", + "smallvec", + "spinning_top", + "web-time", +] + +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.13.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "html5ever" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6452c4751a24e1b99c3260d505eaeee76a050573e61f30ac2c924ddc7236f01e" +dependencies = [ + "log", + "markup5ever", +] + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2 0.6.3", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "infer" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a588916bfdfd92e71cacef98a63d9b1f0d74d6599980d11894290e7ddefffcf7" +dependencies = [ + "cfb", +] + +[[package]] +name = "inherent" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c727f80bfa4a6c6e2508d2f05b6f4bfce242030bd88ed15ae5331c5b5d30fba7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iri-string" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4c90f45aa2e6eacbe8645f77fdea542ac97a494bcd117a67df9ff4d611f995" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "json5" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" +dependencies = [ + "pest", + "pest_derive", + "serde", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" +dependencies = [ + "bitflags", + "libc", + "plain", + "redox_syscall 0.7.3", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c3294c4d74d0742910f8c7b466f44dda9eb2d5742c1e430138df290a1e8451c" +dependencies = [ + "log", + "tendril", + "web_atoms", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] + +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl" +version = "0.10.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "openssl-sys" +version = "0.9.112" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "opentelemetry" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab70038c28ed37b97d8ed414b6429d343a8bbf44c9f79ec854f3a643029ba6d7" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror 1.0.69", + "tracing", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" +dependencies = [ + "async-trait", + "futures-core", + "http", + "opentelemetry", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost", + "thiserror 1.0.69", + "tokio", + "tonic", + "tracing", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost", + "tonic", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52" + +[[package]] +name = "opentelemetry_sdk" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231e9d6ceef9b0b2546ddf52335785ce41252bc7474ee8ba05bfad277be13ab8" +dependencies = [ + "async-trait", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "opentelemetry", + "percent-encoding", + "rand 0.8.5", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tracing", +] + +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.5", +] + +[[package]] +name = "ouroboros" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pest" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0848c601009d37dfa3430c4666e147e49cdcf1b92ecd3e63657d8a5f19da662" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11f486f1ea21e6c10ed15d5a7c77165d0ee443402f0780849d1768e7d9d6fe77" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8040c4647b13b210a963c1ed407c1ff4fdfa01c31d6d2a098218702e6664f94f" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "pest_meta" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" +dependencies = [ + "pest", + "sha2", +] + +[[package]] +name = "pgvector" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc58e2d255979a31caa7cabfa7aac654af0354220719ab7a68520ae7a91e8c0b" +dependencies = [ + "serde", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared 0.12.1", +] + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_macros", + "phf_shared 0.13.1", + "serde", +] + +[[package]] +name = "phf_codegen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +dependencies = [ + "phf_generator", + "phf_shared 0.13.1", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared 0.13.1", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator", + "phf_shared 0.13.1", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.117", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "version_check", + "yansi", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redis" +version = "0.32.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "014cc767fefab6a3e798ca45112bccad9c6e0e218fbd49720042716c73cfef44" +dependencies = [ + "bytes", + "cfg-if", + "combine", + "futures-util", + "itoa", + "num-bigint", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "ryu", + "sha1_smol", + "socket2 0.6.3", + "tokio", + "tokio-rustls", + "tokio-util", + "url", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_syscall" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rend" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +dependencies = [ + "bytecheck", +] + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "mime_guess", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower 0.5.3", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rkyv" +version = "0.7.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1" +dependencies = [ + "bitvec", + "bytecheck", + "bytes", + "hashbrown 0.12.3", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ron" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd490c5b18261893f14449cbd28cb9c0b637aebf161cd77900bfdedaff21ec32" +dependencies = [ + "bitflags", + "once_cell", + "serde", + "serde_derive", + "typeid", + "unicode-ident", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rust-embed" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04113cb9355a377d83f06ef1f0a45b8ab8cd7d8b1288160717d66df5c7988d27" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0902e4c7c8e997159ab384e6d0fc91c221375f6894346ae107f47dd0f3ccaa" +dependencies = [ + "proc-macro2", + "quote", + "rust-embed-utils", + "syn 2.0.117", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1" +dependencies = [ + "sha2", + "walkdir", +] + +[[package]] +name = "rust-ini" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + +[[package]] +name = "rust_decimal" +version = "1.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ce901f9a19d251159075a4c37af514c3b8ef99c22e02dd8c19161cf397ee94a" +dependencies = [ + "arrayvec", + "borsh", + "bytes", + "num-traits", + "rand 0.8.5", + "rkyv", + "serde", + "serde_json", + "wasm-bindgen", +] + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scraper" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93cecd86d6259499c844440546d02f55f3e17bd286e529e48d1f9f67e92315cb" +dependencies = [ + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "precomputed-hash", + "selectors", + "tendril", +] + +[[package]] +name = "scraper-service" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.8.8", + "backoff", + "base64", + "bytes", + "chrono", + "clap", + "config", + "dashmap", + "data-url", + "deadpool-redis", + "dotenvy", + "flate2", + "futures", + "governor", + "hex", + "hmac", + "http", + "infer", + "itertools", + "log", + "mime_guess", + "once_cell", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", + "rand 0.8.5", + "rayon", + "redis", + "regex", + "reqwest", + "scraper", + "sea-orm", + "serde", + "serde_json", + "sha1", + "sha2", + "tempfile", + "thiserror 2.0.18", + "tl", + "tokio", + "tokio-cron-scheduler", + "tokio-util", + "tower 0.5.3", + "tower-http", + "tracing", + "tracing-subscriber", + "url", + "urlencoding", + "utoipa", + "utoipa-axum", + "utoipa-swagger-ui", + "uuid", + "walkdir", +] + +[[package]] +name = "sea-bae" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f694a6ab48f14bc063cfadff30ab551d3c7e46d8f81836c51989d548f44a2a25" +dependencies = [ + "heck 0.4.1", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "sea-orm" +version = "1.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d945f62558fac19e5988680d2fdf747b734c2dbc6ce2cb81ba33ed8dde5b103" +dependencies = [ + "async-stream", + "async-trait", + "bigdecimal", + "chrono", + "derive_more", + "futures-util", + "log", + "ouroboros", + "pgvector", + "rust_decimal", + "sea-orm-macros", + "sea-query", + "sea-query-binder", + "serde", + "serde_json", + "sqlx", + "strum 0.26.3", + "thiserror 2.0.18", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "sea-orm-macros" +version = "1.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c2e64a50a9cc8339f10a27577e10062c7f995488e469f2c95762c5ee847832" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "sea-bae", + "syn 2.0.117", + "unicode-ident", +] + +[[package]] +name = "sea-query" +version = "0.32.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a5d1c518eaf5eda38e5773f902b26ab6d5e9e9e2bb2349ca6c64cf96f80448c" +dependencies = [ + "bigdecimal", + "chrono", + "inherent", + "ordered-float", + "rust_decimal", + "serde_json", + "time", + "uuid", +] + +[[package]] +name = "sea-query-binder" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0019f47430f7995af63deda77e238c17323359af241233ec768aba1faea7608" +dependencies = [ + "bigdecimal", + "chrono", + "rust_decimal", + "sea-query", + "serde_json", + "sqlx", + "time", + "uuid", +] + +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "selectors" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feef350c36147532e1b79ea5c1f3791373e61cbd9a6a2615413b3807bb164fb7" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "log", + "new_debug_unreachable", + "phf 0.13.1", + "phf_codegen", + "precomputed-hash", + "rustc-hash", + "servo_arc", + "smallvec", +] + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde-untagged" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9faf48a4a2d2693be24c6289dbe26552776eb7737074e6722891fadbe6c5058" +dependencies = [ + "erased-serde", + "serde", + "serde_core", + "typeid", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_spanned" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876ac351060d4f882bb1032b6369eb0aef79ad9df1ea8bc404874d8cc3d0cd98" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "servo_arc" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spinning_top" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlx" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64", + "bigdecimal", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink", + "indexmap 2.13.0", + "log", + "memchr", + "once_cell", + "percent-encoding", + "rust_decimal", + "rustls", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror 2.0.18", + "time", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", + "webpki-roots 0.26.11", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.117", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +dependencies = [ + "dotenvy", + "either", + "heck 0.5.0", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.117", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +dependencies = [ + "atoi", + "base64", + "bigdecimal", + "bitflags", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand 0.8.5", + "rsa", + "rust_decimal", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.18", + "time", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64", + "bigdecimal", + "bitflags", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "num-bigint", + "once_cell", + "rand 0.8.5", + "rust_decimal", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.18", + "time", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "thiserror 2.0.18", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "string_cache" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared 0.13.1", + "precomputed-hash", +] + +[[package]] +name = "string_cache_codegen" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69" +dependencies = [ + "phf_generator", + "phf_shared 0.13.1", + "proc-macro2", + "quote", +] + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tl" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b130bd8a58c163224b44e217b4239ca7b927d82bf6cc2fea1fc561d15056e3f7" + +[[package]] +name = "tokio" +version = "1.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.6.3", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-cron-scheduler" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f50e41f200fd8ed426489bd356910ede4f053e30cebfbd59ef0f856f0d7432a" +dependencies = [ + "chrono", + "chrono-tz", + "croner", + "num-derive", + "num-traits", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "tokio-macros" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "1.1.0+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8195ca05e4eb728f4ba94f3e3291661320af739c4e43779cbdfae82ab239fcc" +dependencies = [ + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.0+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97251a7c317e03ad83774a8752a7e81fb6067740609f75ea2b585b569a59198f" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.8+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16bff38f1d86c47f9ff0647e6838d7bb362522bdf44006c7068c2b1e606f1f3c" +dependencies = [ + "indexmap 2.13.0", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.0+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2334f11ee363607eb04df9b8fc8a13ca1715a72ba8662a26ac285c98aabb4011" +dependencies = [ + "winnow", +] + +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum 0.7.9", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2 0.5.10", + "tokio", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.5", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "http-range-header", + "httpdate", + "iri-string", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower 0.5.3", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.2", + "sha1", + "thiserror 2.0.18", + "utf-8", +] + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "utoipa" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fcc29c80c21c31608227e0912b2d7fddba57ad76b606890627ba8ee7964e993" +dependencies = [ + "indexmap 2.13.0", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-axum" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c25bae5bccc842449ec0c5ddc5cbb6a3a1eaeac4503895dc105a1138f8234a0" +dependencies = [ + "axum 0.8.8", + "paste", + "tower-layer", + "tower-service", + "utoipa", +] + +[[package]] +name = "utoipa-gen" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d79d08d92ab8af4c5e8a6da20c47ae3f61a0f1dabc1997cdf2d082b757ca08b" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "syn 2.0.117", +] + +[[package]] +name = "utoipa-swagger-ui" +version = "9.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d047458f1b5b65237c2f6dc6db136945667f40a7668627b3490b9513a3d43a55" +dependencies = [ + "axum 0.8.8", + "base64", + "mime_guess", + "regex", + "rust-embed", + "serde", + "serde_json", + "url", + "utoipa", + "zip", +] + +[[package]] +name = "uuid" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.115" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6523d69017b7633e396a89c5efab138161ed5aafcbc8d3e5c5a42ae38f50495a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "serde", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d1faf851e778dfa54db7cd438b70758eba9755cb47403f3496edd7c8fc212f0" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.115" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e3a6c758eb2f701ed3d052ff5737f5bfe6614326ea7f3bbac7156192dc32e67" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.115" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921de2737904886b52bcbb237301552d05969a6f9c40d261eb0533c8b055fedf" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.117", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.115" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a93e946af942b58934c604527337bad9ae33ba1d5c6900bbb41c2c07c2364a93" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap 2.13.0", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap 2.13.0", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84cde8507f4d7cfcb1185b8cb5890c494ffea65edbe1ba82cfd63661c805ed94" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web_atoms" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a9779e9f04d2ac1ce317aee707aa2f6b773afba7b931222bff6983843b1576" +dependencies = [ + "phf 0.13.1", + "phf_codegen", + "string_cache", + "string_cache_codegen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.6", +] + +[[package]] +name = "webpki-roots" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck 0.5.0", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck 0.5.0", + "indexmap 2.13.0", + "prettyplease", + "syn 2.0.117", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.117", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap 2.13.0", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.13.0", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "yaml-rust2" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2462ea039c445496d8793d052e13787f2b90e750b833afee748e601c17621ed9" +dependencies = [ + "arraydeque", + "encoding_rs", + "hashlink", +] + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zip" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12598812502ed0105f607f941c386f43d441e00148fce9dec3ca5ffb0bde9308" +dependencies = [ + "arbitrary", + "crc32fast", + "flate2", + "indexmap 2.13.0", + "memchr", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..9bf8eb1 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,132 @@ + +[package] +name = "scraper-service" +version = "0.1.0" +edition = "2021" +description = "A batteries-included, production-ready Rust web framework built on Axum" +authors = ["Asep Haryana"] +license = "MIT" +repository = "https://github.com/MythEclipse/ultimate-asepharyana.tech" +keywords = ["web", "framework", "axum", "api", "rest"] +categories = ["web-programming::http-server", "web-programming::websocket"] +default-run = "scraper" + +# Dependensi yang dibutuhkan saat aplikasi berjalan +[dependencies] +axum = { version = "0.8.8", features = ["ws", "multipart", "macros"] } +tokio = { version = "1.49.0", features = ["full"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +dotenvy = "0.15" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +anyhow = "1.0" +# sqlx removed - SeaORM uses it internally via sqlx-postgres feature +sea-orm = { version = "1.1.19", features = ["sqlx-postgres", "runtime-tokio-rustls", "macros", "with-chrono", "with-uuid"] } +uuid = { version = "1.10.0", features = ["v4", "serde"] } +chrono = { version = "0.4", features = ["serde"] } + +bytes = "1.11.0" +futures = "0.3" +reqwest = { version = "0.12.28", features = ["json", "stream", "multipart"] } +http = "1.4.0" +sha1 = "0.10.6" +data-url = "0.3.2" +base64 = "0.22.1" +tokio-util = { version = "0.7.18", features = ["codec"] } +async-trait = "0.1.89" +regex = "1.12.2" +infer = "0.19.0" + +once_cell = "1.21.3" +urlencoding = "2.1" + +url = "2.5.8" +rand = "0.8" +tempfile = "3.24.0" +mime_guess = "2.0.5" + +tower-http = { version = "0.6.8", features = ["fs", "cors", "compression-gzip", "compression-br", "compression-zstd"] } +backoff = { version = "0.4", features = ["futures", "tokio"] } +dashmap = "6.1" +deadpool-redis = { version = "0.22.1", features = ["serde"] } +rayon = "1.11" +tl = "0.7.8" +tower = { version = "0.5", features = ["make"] } +scraper = "0.25.0" +flate2 = "1.1" +redis = { version = "0.32.7", features = ["tokio-rustls-comp", "safe_iterators"] } +thiserror = "2.0.18" +itertools = "0.14" +clap = { version = "4.5", features = ["derive"] } +config = { version = "0.15.19", features = ["toml"] } +governor = "0.10.4" +tokio-cron-scheduler = "0.15.1" +hex = "0.4.3" +hmac = "0.12.1" +sha2 = "0.10.9" +log = "0.4" +walkdir = "2.5" +utoipa = { version = "5.0", features = ["axum_extras"] } +utoipa-swagger-ui = { version = "9.0", features = ["axum"] } +utoipa-axum = "0.2.0" + +# OpenTelemetry metrics +opentelemetry = { version = "0.27", features = ["metrics"] } +opentelemetry_sdk = { version = "0.27", features = ["metrics", "rt-tokio"] } +opentelemetry-otlp = { version = "0.27", features = ["metrics"] } +opentelemetry-semantic-conventions = "0.27" + + +# Dependensi yang hanya dibutuhkan untuk build script (build.rs) + +# Dependensi yang hanya dibutuhkan untuk tes +[dev-dependencies] +sea-orm = { version = "1.1.19", features = ["sqlx-postgres", "runtime-tokio-rustls", "macros", "with-chrono", "with-uuid", "mock"] } + +# Definisi targets secara eksplisit untuk menghindari peringatan cargo-chef (edition/plugin) +[lib] +name = "scraper_service" +path = "src/lib.rs" + +[[bin]] +name = "scraper" +path = "src/main.rs" + + + +[lints.rust] +unsafe_code = "forbid" +unused_variables = "deny" +unused_imports = "deny" +unused_must_use = "deny" + +[lints.clippy] +unwrap_used = "warn" +expect_used = "warn" +panic = "deny" +todo = "deny" +unimplemented = "deny" + +[features] +# default = ["ffmpeg"] +ffmpeg = [] + +# Profile optimasi untuk production - fokus pada performa runtime maksimal +[profile.release] +opt-level = 3 # Optimasi level maksimal +lto = "fat" # Full LTO untuk inlining maksimal dan binary optimal +codegen-units = 1 # Single codegen unit untuk optimasi maksimal (lebih lambat build, binary lebih cepat) +incremental = false # Disable incremental untuk optimasi penuh +debug = false # No debug info untuk binary lebih kecil +strip = true # Strip symbols untuk binary lebih kecil +panic = "abort" # Abort on panic (binary lebih kecil dan lebih cepat) +overflow-checks = false # Non-overflowing arithmetic for speed (disable in debug) + +# Optimize dependencies too +[profile.dev] +opt-level = 0 + +[profile.bench] +opt-level = 3 +lto = "fat" diff --git a/GEMINI.md b/GEMINI.md new file mode 100644 index 0000000..74f2807 --- /dev/null +++ b/GEMINI.md @@ -0,0 +1,69 @@ +# GEMINI.md - Codebase Architecture & Structure + +Internal technical overview of the **Scraping & CDN Service** (`apps/scraper`) for automated data extraction and image persistence. + +## 🌍 Context + +- **`apps/scraper`**: Specialized backend engine (Axum). +- **`apps/solidjs`**: Frontend consumer. +- **`packages/services`**: Shared logic. + +## 🤖 AI Assistant Guidelines + +AI assistants (like Claude, Gemini, GPT) interacting with this codebase **MUST** adhere to the following protocols defined in **[AGENT.md](file:///mnt/code/bp3/ultimate-asepharyana.tech/apps/scraper/AGENT.md)**: + +1. **Professional Tone**: Maintain a cold, technical, and objective tone. +2. **No Hyperbole**: **PROHIBITED** from using marketing-speak or exaggerated praise (e.g., "amazing", "unparalleled", "powerful", "revolutionary"). +3. **Technical Accuracy**: Focus purely on implementation facts, data structures, and performance metrics. +4. **Minimalist Adherence**: Always prioritize the **Zero-Bloat Policy**. If a request introduces unnecessary dependencies or logic, challenge the user and suggest a leaner alternative. +5. **Documentation Consistency**: Ensure any generated documentation follows the established professional and objective style of the `docs/` folder. + +## 🦀 `apps/scraper` - Backend Service + +An asynchronous service for scraping and image proxying. All secondary web framework features (Authentication, Social, GraphQL) have been removed to reduce complexity. + +### 📊 Tech Stack + +- **Framework**: [Axum](https://github.com/tokio-rs/axum) (0.8.8) - Asynchronous Rust HTTP. +- **ORM**: [SeaORM](https://www.sea-ql.org/SeaORM/) (MySQL) - Database abstraction. +- **Caching**: `deadpool-redis` & `redis` - In-memory cache mapping. +- **Observability**: Request ID tracing and structured logging. +- **Scraping**: `scraper` (CSS Selectors) & remote Chrome via HTTP. + +### 📂 Directory Structure (`apps/scraper/src`) + +Organized as a hybrid of Vertical Slice and Clean Architecture. + +| Directory | Description | +| :--- | :--- | +| **`bin/`** | Binary entry points and CLI tools. | +| **`config/`** | Strongly-typed environment configuration. | +| **`entities/`** | **SeaORM Entities**. Database schema mapping. | +| **`routes/`** | **API Handlers**. Automatic routing system. | +| **`services/`** | Business logic (e.g., `ImageCache` service). | +| **`scraping/`** | Data extraction engines and parsers. | +| **`helpers/`** | Shared utilities and cache helpers. | +| **`middleware/`** | Axum layers (CORS, Compression). | +| **`events/`** | Internal event bus for repair state updates. | +| **`jobs/`** | Background task processing. | +| **`scheduler/`** | Periodic tasks (Daily CDN audit). | +| **`observability/`** | OpenAPI documentation, request ID tracing, and structured logging. | + +### 🔑 Logic Flows + +1. **Scraping**: + `Request` -> `Router` -> `Handler` -> `Scraper Engine` -> `Redis` -> `Response`. +2. **Image Proxy/CDN**: + `Request` -> `ImageCache` -> `Cache Lookup` -> `Picser Upload (on miss)` -> `CDN URL`. + +### 📜 Commands + +- **Standard Run**: `cargo run` +- **Optimized Build**: `cargo build --release` +- **External Audit**: `POST /api/proxy/image-cache/audit` + +## 🏗 Maintenance Constraints + +- **Minimalist Approach**: New dependencies require impact evaluation. +- **Lint Compliance**: Suppression flags (`#[allow]`) are prohibited. +- **Performance-First**: Use `spawn_blocking` for CPU-heavy work (HTML parsing). diff --git a/README.md b/README.md new file mode 100644 index 0000000..9151224 --- /dev/null +++ b/README.md @@ -0,0 +1,55 @@ +# Scraper API + +Backend service berbasis Axum untuk scraping, image proxy/cache, dan endpoint API inti. + +## Stack + +- Rust + Axum +- SeaORM (MySQL) +- Redis (deadpool-redis) +- Utoipa + Swagger UI + +## Quick Start + +```bash +cargo run +``` + +Server bind ke `0.0.0.0:${PORT}` dengan default port `4091`. + +## Required Environment Variables + +```env +DATABASE_URL=mysql://asephs:hunterz@localhost:3306/sosmed +JWT_SECRET=change-me +REDIS_URL=redis://localhost:6379 +``` + +Optional yang sering dipakai: + +```env +RUST_LOG=info +EXTERNAL_BROWSERLESS_WS= +MINIO_ENDPOINT= +MINIO_BUCKET_NAME= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +``` + +## Useful Endpoints + +- `GET /docs` - Swagger UI +- `GET /api-docs/openapi.json` - OpenAPI JSON +- `GET /api/anime2/*` +- `GET /api/komik/*` +- `POST /api/proxy/image-cache` +- `POST /api/proxy/image-cache/audit` + +## Notes + +- Database akan dicek/dibuat saat startup jika `DATABASE_URL` bertipe MySQL. +- Service ini juga menginisialisasi browser pool dan scheduler saat boot. + +## License + +MIT diff --git a/check_image_cache_db.sh b/check_image_cache_db.sh new file mode 100755 index 0000000..41b2510 --- /dev/null +++ b/check_image_cache_db.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Simple script to check image_cache database content + +DATABASE_URL="mysql://asephs:hunterz@127.0.0.1:3306/sosmed" + +echo "=== Checking Image Cache Database ===" +echo "" +echo "Attempting to connect via Rust binary..." +echo "" + +cd "$(dirname "$0")" + +# Create temporary Rust script +cat > /tmp/check_img_cache.rs << 'EOF' +use sea_orm::{Database, ConnectionTrait, Statement}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let db = Database::connect("mysql://asephs:hunterz@127.0.0.1:3306/sosmed").await?; + + println!("✓ Connected to database\n"); + + // Show all tables + println!("=== ALL TABLES ==="); + let tables = db.query_all(Statement::from_string( + db.get_database_backend(), + "SHOW TABLES".to_owned() + )).await?; + for table in &tables { + println!(" - {:?}", table); + } + + // Count image_cache records + println!("\n=== IMAGE_CACHE TABLE ==="); + let count = db.query_one(Statement::from_string( + db.get_database_backend(), + "SELECT COUNT(*) as total FROM ImageCache".to_owned() + )).await?; + println!("Total records: {:?}", count); + + // Show recent 10 records + println!("\n=== RECENT RECORDS (Last 10) ==="); + let records = db.query_all(Statement::from_string( + db.get_database_backend(), + "SELECT id, originalUrl, cdnUrl, createdAt FROM ImageCache ORDER BY createdAt DESC LIMIT 10".to_owned() + )).await?; + + for (i, record) in records.iter().enumerate() { + println!("\n[{}]", i + 1); + println!(" ID: {:?}", record.try_get::("", "id")); + println!(" Original: {:?}", record.try_get::("", "originalUrl")); + println!(" CDN URL: {:?}", record.try_get::("", "cdnUrl")); + println!(" Created: {:?}", record.try_get::>("", "createdAt")); + } + + Ok(()) +} +EOF + +echo "Running database check..." +rustc --edition 2021 /tmp/check_img_cache.rs -o /tmp/check_img_cache \ + -L dependency=target/debug/deps \ + --extern sea_orm=target/debug/deps/libsea_orm.rlib \ + --extern tokio=target/debug/deps/libtokio.rlib \ + --extern chrono=target/debug/deps/libchrono.rlib \ + 2>/dev/null + +if [ $? -eq 0 ]; then + /tmp/check_img_cache +else + echo "Rust compilation failed, using cargo script instead..." + cargo script /tmp/check_img_cache.rs +fi diff --git a/docs/superpowers/plans/2026-05-08-refactor-clean-architecture.md b/docs/superpowers/plans/2026-05-08-refactor-clean-architecture.md new file mode 100644 index 0000000..5aee98a --- /dev/null +++ b/docs/superpowers/plans/2026-05-08-refactor-clean-architecture.md @@ -0,0 +1,167 @@ +# Clean-Modular Architecture Refactor Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Refactor the codebase into a rigid Clean-Modular architecture to improve maintainability and strictly enforce separation of concerns. + +**Architecture:** A three-tier modular approach consisting of Presentation (API Handlers/DTOs), Core (Domain Models/Traits/Use Cases), and Infrastructure (Adapters/Repositories/Scrapers). + +**Tech Stack:** Rust, Axum, SeaORM, Redis, reqwest. + +--- + +### Task 1: Initialize Core Domain Models & Shared Errors + +**Files:** +- Create: `src/shared/errors/mod.rs` +- Create: `src/core/models/image.rs` +- Create: `src/core/models/mod.rs` +- Create: `src/shared/mod.rs` + +- [ ] **Step 1: Define shared application errors** +- [ ] **Step 2: Define pure domain models for ImageCache** +- [ ] **Step 3: Setup core and shared modules in `lib.rs`** + +```rust +// src/shared/errors/mod.rs +use axum::{response::{IntoResponse, Response}, Json, http::StatusCode}; +use serde_json::json; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum AppError { + #[error("Not found: {0}")] + NotFound(String), + #[error("Internal error: {0}")] + Internal(String), + #[error("Validation error: {0}")] + Validation(String), +} + +impl IntoResponse for AppError { + fn into_response(self) -> Response { + let (status, message) = match self { + AppError::NotFound(m) => (StatusCode::NOT_FOUND, m), + AppError::Internal(m) => (StatusCode::INTERNAL_SERVER_ERROR, m), + AppError::Validation(m) => (StatusCode::BAD_REQUEST, m), + }; + (status, Json(json!({ "error": message }))).into_response() + } +} +``` + +- [ ] **Step 4: Commit changes** +```bash +git add src/shared/errors/mod.rs src/core/models/image.rs +git commit -m "feat: init core models and shared errors" +``` + +### Task 2: Define Core Repository Traits + +**Files:** +- Create: `src/core/repositories/image_repository.rs` +- Create: `src/core/repositories/mod.rs` + +- [ ] **Step 1: Define ImageRepository trait in `core`** + +```rust +// src/core/repositories/image_repository.rs +use async_trait::async_trait; +use crate::core::models::image::ImageCache; +use crate::shared::errors::AppError; + +#[async_trait] +pub trait ImageRepository: Send + Sync { + async fn find_by_original_url(&self, url: &str) -> Result, AppError>; + async fn save(&self, image: ImageCache) -> Result<(), AppError>; + async fn delete_by_original_url(&self, url: &str) -> Result<(), AppError>; +} +``` + +- [ ] **Step 2: Commit changes** +```bash +git add src/core/repositories/ +git commit -m "feat: define core repository traits" +``` + +### Task 3: Migrate SeaORM Entities to Infrastructure + +**Files:** +- Modify: `src/infra/mod.rs` +- Create: `src/infra/repositories/mysql_image_repository.rs` + +- [ ] **Step 1: Implement ImageRepository for MySQL using SeaORM** +- [ ] **Step 2: Move `src/entities/image_cache.rs` logic into the new repository implementation** +- [ ] **Step 3: Update `src/infra/mod.rs` to expose repositories** + +- [ ] **Step 4: Commit changes** +```bash +git add src/infra/repositories/ +git commit -m "feat: implement mysql image repository in infra" +``` + +### Task 4: Implement Core Use Cases (Image Caching) + +**Files:** +- Create: `src/core/use_cases/cache_image.rs` +- Create: `src/core/use_cases/mod.rs` + +- [ ] **Step 1: Implement `CacheImageUseCase`** +- [ ] **Step 2: Orchestrate logic between repository, redis, and scraper/uploader** + +- [ ] **Step 3: Commit changes** +```bash +git add src/core/use_cases/ +git commit -m "feat: implement image caching use cases" +``` + +### Task 5: Refactor Scrapers into Infrastructure + +**Files:** +- Create: `src/core/repositories/scraping_repository.rs` +- Create: `src/infra/scrapers/otakudesu.rs` + +- [ ] **Step 1: Define Scraping traits in `core`** +- [ ] **Step 2: Implement site-specific scrapers in `infra`** +- [ ] **Step 3: Migrate existing logic from `src/scraping/`** + +- [ ] **Step 4: Commit changes** +```bash +git add src/infra/scrapers/ +git commit -m "feat: migrate scrapers to infra adapters" +``` + +### Task 6: Refactor Presentation Layer (API Handlers) + +**Files:** +- Create: `src/presentation/api/anime_handler.rs` +- Create: `src/presentation/api/mod.rs` +- Create: `src/presentation/mod.rs` + +- [ ] **Step 1: Migrate handlers from `src/routes/` to `presentation/api/`** +- [ ] **Step 2: Update handlers to use Use Cases instead of direct service/helper calls** +- [ ] **Step 3: Update global router in `src/main.rs` or `src/lib.rs`** + +- [ ] **Step 4: Commit changes** +```bash +git add src/presentation/api/ +git commit -m "feat: refactor presentation layer api handlers" +``` + +### Task 7: Global Cleanup & Verification + +**Files:** +- Modify: `src/lib.rs` +- Delete: `src/helpers/` (partially merged into shared/infra) +- Delete: `src/services/` (merged into core/use_cases) +- Delete: `src/routes/` (merged into presentation) + +- [ ] **Step 1: Update `lib.rs` to reflect new module structure** +- [ ] **Step 2: Remove old redundant directories** +- [ ] **Step 3: Run full test suite** +- [ ] **Step 4: Verify metrics endpoint** + +- [ ] **Step 5: Final Commit** +```bash +git commit -m "refactor: complete clean-modular architecture overhaul" +``` diff --git a/docs/superpowers/specs/2026-05-08-clean-architecture-design.md b/docs/superpowers/specs/2026-05-08-clean-architecture-design.md new file mode 100644 index 0000000..918bac9 --- /dev/null +++ b/docs/superpowers/specs/2026-05-08-clean-architecture-design.md @@ -0,0 +1,49 @@ +# Design Spec: Clean-Modular Architecture Refactor + +**Date**: 2026-05-08 +**Topic**: Refactor `apps/rust` from Hybrid to Clean-Modular Architecture. + +## 1. Purpose +Standardize codebase structure for rigidity, maintainability, and clear separation of concerns (SOC) without violating the **Zero-Bloat Policy**. + +## 2. Target Architecture +Moving from current structure to a three-tier modular design: + +### A. Presentation Layer (`src/presentation/`) +- **API Handlers**: Pure Axum handlers. +- **DTOs**: Request/Response models for external communication. +- **Middleware**: Cross-cutting concerns (CORS, Metrics, Logging). + +### B. Core Layer (`src/core/`) - The Domain +- **Models**: Pure data structures (Plain Rust Objects). +- **Repository Traits**: Abstract interfaces for data persistence. +- **Use Cases**: Orchestration of business logic (e.g., `ScrapeAnime`, `ProcessImage`). +- **Dependencies**: None (or minimal shared utils). + +### C. Infrastructure Layer (`src/infra/`) - The Adapters +- **Repositories**: SeaORM & Redis implementations of Core traits. +- **Scrapers**: Site-specific parsing logic implementing Core scraping traits. +- **External Clients**: HTTP Client (reqwest), Browser Pool. + +### D. Shared Layer (`src/shared/`) +- **Utils**: Low-level helpers (Date, JSON, String). +- **Config**: Application configuration. +- **Errors**: Centralized error handling. + +## 3. Implementation Strategy +1. **Phase 1**: Scaffold new directory structure. +2. **Phase 2**: Migrate `models` and `entities` to `core/models` and `infra/repositories`. +3. **Phase 3**: Refactor `scraping` logic into `infra/scrapers` and define traits in `core`. +4. **Phase 4**: Move Axum handlers to `presentation/api` and update routing. +5. **Phase 5**: Cleanup `helpers` into `shared/utils`. + +## 4. Constraints +- **Zero-Bloat**: No new heavy dependencies for the sake of abstraction. +- **Performance**: Maintain latency metrics as defined in `docs/development.md`. +- **SeaORM**: Entities stay in `infra/` to keep `core/` pure. + +## 5. Success Criteria +- All tests pass. +- `/metrics` show no latency regression. +- Circular dependencies are eliminated. +- Folder structure matches this spec. diff --git a/ecosystem.config.cjs b/ecosystem.config.cjs new file mode 100644 index 0000000..94830f4 --- /dev/null +++ b/ecosystem.config.cjs @@ -0,0 +1,26 @@ +// PM2 Ecosystem Configuration for Rust +module.exports = { + apps: [ + { + name: 'ultimate-rust', + script: 'target/release/scraper', + cwd: process.env.VPS_TARGET_DIR + ? `${process.env.VPS_TARGET_DIR}/apps/scraper` + : '/home/asephs/ultimate-asepharyana.cloud/apps/scraper', + interpreter: 'none', + exec_mode: 'fork', + autorestart: true, + watch: false, + max_memory_restart: '1G', + env_production: { + NODE_ENV: 'production', + PORT: 4091, + }, + // Logging configuration + error_file: './logs/error.log', + out_file: './logs/out.log', + log_date_format: 'YYYY-MM-DD HH:mm:ss Z', + merge_logs: true, + }, + ], +}; diff --git a/package.json b/package.json new file mode 100644 index 0000000..ebfba57 --- /dev/null +++ b/package.json @@ -0,0 +1,8 @@ +{ + "name": "scraper", + "version": "0.1.0", + "private": true, + "scripts": { + "start": "./target/release/scraper" + } +} diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..3255b47 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,15 @@ +# rustfmt configuration (stable features only) + +max_width = 100 +hard_tabs = false +tab_spaces = 4 +newline_style = "Unix" +use_small_heuristics = "Default" +reorder_imports = true +reorder_modules = true +remove_nested_parens = true +edition = "2021" +merge_derives = true +use_try_shorthand = true +use_field_init_shorthand = true +force_explicit_abi = true diff --git a/scripts/__pycache__/compare-openapi.cpython-312.pyc b/scripts/__pycache__/compare-openapi.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2064cad92ed60637b87730272830e6b4fb0db7a3 GIT binary patch literal 4492 zcmc&$Ur-yz8Q;^Lq?1nnkpvPozL*41g8>)E*cfADI~WXx(3(0S^=M?>fg}r^dUyT< zIc#N;Oymr$VJ0njJQ?U@I>lpmN}lSMPHA8BXCxmU?5jGHHhrkyi0uc*dFk#+hh)-V z?CDH*=63gXfBXBsZ}zL?jf$e0HzSHg~Uj>z^pbT zta11~;Ee#h@0K31vqi|u-AO7Mow;K!lgPmmJH z(~Jd5+oCZ`q_#hY!AS?im`c3EDw(c=k4k)G(nWx0m0W_~lx3=>X36{=P0brE`wn#?=H>vX@kehJRA`a_v;p&S8wPze`Gb$%FE`rmqqWzQ#Z^3PDJk@q}9Mk{7ScQLCOk0$um%x(|wt1c308_fjm}t?sV{WZiY@xh{3A{v}LBgM39s?%u z3pkYe+kEdFb`G4!-vH;Z4Q(_^*z?PT(r!MZjpYBt#s}+d43ziXL&rXTWKal=u&w!v zn`gIk@RAQDqYcS1x{YUtS)NUVi=#1tOSIE%aV{K+uR+zpd+1uehi;7V7hdIgj&G%Z ztSzU9qj8o_a3XvwoJlaP^muIn(w#UPibomx=ui6l=xBmolc+9%z%2z}FOv9Op6c)6 zQ57t3D0=uJi3siJ9iZSNtf=70GonI7*bt)-Nlp+|L^z`8MLrZ}6$8V@S&>zYTvC1E zbRHkK9g7M=G%-RWDyWv_)$e{5J{pQ98uCXiv^T6x?1MBy z09KFw4IZVyEnjlsr9)7VU|xdTe9iOlnIF;1!J`d^%siI!v%#Z;swPX2?;?aI$TIKf?^%b$C}CqZBs>1P;EX`7_vam z9pw^*KMD~Swo0+)skL9b##I>L(NkBk=E=0N3OFVp$Dr4j1FB;v1PTRyFGQeC5q848 zg`S@?nkSnt@4mGAorziOmeH3Ztr^>iW%BhLNnIu{kyFk|>RO!vrk zI@OhPdOi%#M5ZINKc7A=JGZ5dt{RMbcdo2_rgge?HnLLHl&xx#t9~SxHK)l{7n<|< zW`?GR=FKbg&MduCrgzDnmb7`*0i3=W%d}nCss4us;Hj83&DpNldwkZn33HyxDg4`tnlQpfHZ-FN&| zX?xD?pWO#R_W@>lIej>1aZdHk@0_p6xB|B=KU~G(=tB~ed8baVcx$rW8rfSXyAY-6 zkG4;2GUTl9$htaGJ$H?+HNw5JyA}}!@`Szf?s-GTReRe~hX@0Z&|N+?HgjS6f?QrJ zm(|^9UU+MzaeuaPzued%zkD!Tb}-eOb9kpD+416iVAFSFX(H$K$-d;`*pmB`iz_WX z*_Ix;<=C<>DSMM?Dpye{SBf_)myT!m99`MdpWV|h?-^LG6y*vrZT-^!(t;cvp{7ln$uQ=s#s&$BKsQ=OLu|AIzRlo_;2DLCo(U+viRnbd&!XT9=Yx4`Wmuy zW2=?On6GI-mY2E?VaiwnH>ww+*Bdf{JvY4>^MPfeBS#qD?|Qo{J@8)7qlaS{uxz;x zbkIkj^Eq}b)-C!owzg$*-x|?nDo5E@sI6IQYZ2IT6I&EAw!O<_TM6(3>f2R4{x8h*jkp!-6cSc_OZ|gqvtOz)jb~Y*UIW+M)0Yp z{uqIOMz~Pzt?u2Af3{x_`=1-@`*z``}WZNjmoQbpT{6RQ|u4JJe xR`5_U>>uF3SAhB(F#jD?+&3J-Fvsd~9Uv^}@msj>E ") + sys.exit(1) + + ref_path = Path(sys.argv[1]) + local_path = Path(sys.argv[2]) + + # Load reference OpenAPI + try: + ref_data = json.loads(ref_path.read_text()) + except FileNotFoundError: + print(f"Error: Reference file not found: {ref_path}") + sys.exit(1) + except json.JSONDecodeError as e: + print(f"Error: Invalid JSON in reference file: {e}") + sys.exit(1) + + # Load local OpenAPI + try: + local_data = json.loads(local_path.read_text()) + except FileNotFoundError: + print(f"Error: Local file not found: {local_path}") + sys.exit(1) + except json.JSONDecodeError as e: + print(f"Error: Invalid JSON in local file: {e}") + sys.exit(1) + + ref_paths = ref_data.get("paths", {}) + local_paths = local_data.get("paths", {}) + + http_methods = {"get", "put", "post", "delete", "options", "head", "patch", "trace"} + + has_differences = False + + # Check for missing paths + missing_paths = set(ref_paths.keys()) - set(local_paths.keys()) + if missing_paths: + has_differences = True + for path in sorted(missing_paths): + print(f"Missing path: {path}") + + # Check for extra paths + extra_paths = set(local_paths.keys()) - set(ref_paths.keys()) + if extra_paths: + has_differences = True + for path in sorted(extra_paths): + print(f"Extra path: {path}") + + # Check for method mismatches in common paths + common_paths = set(ref_paths.keys()) & set(local_paths.keys()) + for path in sorted(common_paths): + ref_methods = set(method.lower() for method in ref_paths[path].keys() if method.lower() in http_methods) + local_methods = set(method.lower() for method in local_paths[path].keys() if method.lower() in http_methods) + + missing_methods = ref_methods - local_methods + if missing_methods: + has_differences = True + for method in sorted(missing_methods): + print(f"Missing method {method.upper()} for path: {path}") + + extra_methods = local_methods - ref_methods + if extra_methods: + has_differences = True + for method in sorted(extra_methods): + print(f"Extra method {method.upper()} for path: {path}") + + if has_differences: + sys.exit(1) + + print(f"OpenAPI paths/methods match: {len(ref_paths)} paths") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/scripts/migrate.sh b/scripts/migrate.sh new file mode 100755 index 0000000..d559926 --- /dev/null +++ b/scripts/migrate.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Script to run database migrations manually + +echo "Running database migrations..." +cd "$(dirname "$0")/.." + +# Build and run the migration binary +cargo run --bin migrate + +if [ $? -eq 0 ]; then + echo "✅ Migration completed successfully." +else + echo "❌ Migration failed." + exit 1 +fi diff --git a/scripts/pre-build-lint.sh b/scripts/pre-build-lint.sh new file mode 100755 index 0000000..aea3c87 --- /dev/null +++ b/scripts/pre-build-lint.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Pre-build hook to enforce best practices + +set -e + +echo "🔍 Running pre-build lint checks..." + +# Format check +echo "📝 Checking code formatting..." +if ! cargo fmt -- --check; then + echo "❌ Code not formatted. Run: cargo fmt" + exit 1 +fi + +# Clippy check with denials +echo "🔧 Running clippy..." +cargo clippy --all-targets -- -D warnings + +echo "✅ All lint checks passed!" diff --git a/src/app.rs b/src/app.rs new file mode 100644 index 0000000..39fefa9 --- /dev/null +++ b/src/app.rs @@ -0,0 +1,51 @@ +use std::sync::Arc; + +use axum::Router; +use sea_orm::DatabaseConnection; +use tower_http::compression::{CompressionLayer, CompressionLevel}; +use tower_http::cors::CorsLayer; +use utoipa::OpenApi; +use utoipa_swagger_ui::SwaggerUi; + +use crate::shared::observability::openapi::ApiDoc; +use crate::shared::state::AppState; + +pub async fn build_router( + app_state: Arc, + db: Arc, +) -> anyhow::Result { + init_scheduler(db).await?; + + let mut openapi = ApiDoc::openapi(); + openapi.merge(crate::shared::observability::openapi_modules::ModuleApiDoc::openapi()); + + let app = crate::modules::routes(Router::new()) + .merge(SwaggerUi::new("/docs").url("/api-docs/openapi.json", openapi)) + .with_state(app_state) + .layer(axum::middleware::from_fn( + crate::shared::observability::metrics::otel_metrics_middleware, + )) + .layer(CompressionLayer::new().quality(CompressionLevel::Fastest)) + .layer(CorsLayer::permissive()); + + Ok(app) +} + +async fn init_scheduler(db: Arc) -> anyhow::Result<()> { + let scheduler = crate::shared::scheduler::Scheduler::new() + .await + .map_err(|e| anyhow::anyhow!("Failed to create scheduler: {}", e))?; + + let cache_cleanup = crate::shared::scheduler::CleanupOldCache::new(db); + scheduler + .add(cache_cleanup) + .await + .map_err(|e| anyhow::anyhow!("Failed to add cache cleanup: {}", e))?; + + scheduler + .start() + .await + .map_err(|e| anyhow::anyhow!("Failed to start scheduler: {}", e))?; + tracing::info!("✓ Scheduler started"); + Ok(()) +} diff --git a/src/bin/capture_warning.rs b/src/bin/capture_warning.rs new file mode 100644 index 0000000..e5019c4 --- /dev/null +++ b/src/bin/capture_warning.rs @@ -0,0 +1,117 @@ +use scraper::Selector; +use scraper_service::shared::utils::parse_html; +/// Capture html5ever tree_builder warning evidence by parsing problematic HTML. +/// +/// Build with: cargo build --bin capture_warning +/// Run with: RUST_LOG=warn cargo run --bin capture_warning 2>&1 +/// +/// Evidence of the warning is captured through: +/// 1. HTML that triggers foster_parenting in html5ever::tree_builder +/// 2. Observable parsing behavior showing tree reconstruction +/// 3. The call path from src/helpers::parse_html () to html5ever +/// 4. Real endpoint context from /api/anime2/latest/{slug} route +use std::fs; +use tracing_subscriber::EnvFilter; + +fn main() { + // Initialize logging to capture WARN output from html5ever + let env_filter = EnvFilter::from_default_env() + .add_directive("warn".parse().expect("valid directive")) + .add_directive("html5ever=warn".parse().expect("valid directive")); + + tracing_subscriber::fmt() + .with_env_filter(env_filter) + .with_writer(std::io::stderr) + .init(); + + println!("=== HTML5ever Tree Builder Foster Parenting Evidence ===\n"); + println!("Real Endpoint: GET /api/anime2/latest/{{slug}}"); + println!("Handler: src/routes/api/anime2/latest/[slug].rs:124"); + println!("Helper Path: src/helpers/web/scraping.rs::parse_html() -> Html::parse_document()\n"); + + // Load HTML fixture from shared test file + let fixture_path = "src/bin/test_fixtures/foster_parenting_minimal.html"; + let test_html = fs::read_to_string(fixture_path) + .expect(&format!("Failed to read fixture from {}", fixture_path)); + + println!("Input Request:"); + println!(" GET /api/anime2/latest/some-anime"); + println!(" Body: HTML containing misplaced text in "); + println!(" Fixture: {}", fixture_path); + println!(" Test HTML: {}\n", test_html); + + println!("Parsing through src/helpers::parse_html()...\n"); + println!("--- BEGIN STDERR (logging output) ---"); + + // This parse_html() call routes through: + // src/helpers::parse_html() + // -> scraper crate Html::parse_document() + // -> html5ever::parse() [version 0.36.1] + // -> TreeBuilder::process_token() + // -> TreeBuilder::foster_parent_in_body() which emits: + // warn!("foster parenting not implemented") + let document = parse_html(&test_html); + + println!("--- END STDERR (logging output) ---\n"); + + // Analyze the result + println!("Parse Output Evidence:\n"); + + // Check table structure + let table_sel = Selector::parse("table").expect("Valid CSS selector"); + let tr_sel = Selector::parse("tr").expect("Valid CSS selector"); + let td_sel = Selector::parse("td").expect("Valid CSS selector"); + + let tables: Vec<_> = document.select(&table_sel).collect(); + println!(" ✓ Tables parsed: {}", tables.len()); + + let trs: Vec<_> = document.select(&tr_sel).collect(); + println!(" ✓ Table rows found: {}", trs.len()); + + let tds: Vec<_> = document.select(&td_sel).collect(); + println!(" ✓ Table cells found: {}", tds.len()); + + let body_sel = Selector::parse("body").expect("Valid CSS selector"); + if let Some(body) = document.select(&body_sel).next() { + let body_text: String = body.text().collect(); + let trimmed = body_text.trim(); + println!("\n Body element text content:"); + println!(" '{}'", trimmed); + + if trimmed.contains("orphaned text") { + println!("\n ✓ EVIDENCE: 'orphaned text' moved OUT of
"); + println!(" This proves foster_parenting occurred!"); + } + if trimmed.contains("more text") { + println!(" ✓ EVIDENCE: 'more text' moved OUT of
"); + println!(" This confirms the adoption agency algorithm ran!"); + } + } + + println!("\n=== Proven Call Path ==="); + println!("Route: GET /api/anime2/latest/{{slug}}"); + println!("Request Handler: src/routes/api/anime2/latest/[slug].rs"); + println!(" -> latest() handler"); + println!(" -> fetch_latest_anime()"); + println!(" -> parse_latest_page(html, page)"); + println!(" -> crate::shared::utils::parse_html(html) [line 124]\n"); + + println!("Helper Function: src/helpers/web/scraping.rs"); + println!(" pub fn parse_html(html: &str) -> Html {{"); + println!(" Html::parse_document(html) // Line 34"); + println!(" }}\n"); + + println!("Call Stack to Warning:"); + println!(" 1. crate::shared::utils::parse_html() [src/helpers/web/scraping.rs:34]"); + println!(" 2. Html::parse_document() [scraper crate wrapper]"); + println!(" 3. html5ever::parse() [Cargo.toml: version 0.36.1]"); + println!(" 4. TreeBuilder::process_token()"); + println!(" 5. TreeBuilder::process_chars_in_table()"); + println!(" 6. TreeBuilder::foster_parent_in_body() [src/tree_builder/mod.rs:1227]"); + println!(" 7. warn!(\"foster parenting not implemented\") ← EMITTED ABOVE\n"); + + println!("=== Fixture Source ==="); + println!("Shared File: {}", fixture_path); + println!("HTML Content: {}", test_html); + println!("Expected Parsing Behavior: Text nodes are fostered out of table"); +} diff --git a/src/bin/foster_parenting_assertion.rs b/src/bin/foster_parenting_assertion.rs new file mode 100644 index 0000000..dd38962 --- /dev/null +++ b/src/bin/foster_parenting_assertion.rs @@ -0,0 +1,132 @@ +/// Test: Verify parsed output for foster_parenting_minimal.html fixture +/// +/// This binary contains tests and assertions that validate the expected behavior +/// when parsing HTML that triggers the html5ever::tree_builder::foster_parent_in_body() warning. +/// +/// Uses shared fixture: src/bin/test_fixtures/foster_parenting_minimal.html +/// Uses shared parser: src/helpers::parse_html() +use scraper::Selector; +use scraper_service::shared::utils::parse_html; +use std::fs; + +fn main() { + println!("Running foster parenting regression tests...\n"); + + // Load HTML fixture from shared test file + let fixture_path = "src/bin/test_fixtures/foster_parenting_minimal.html"; + let foster_parenting_html = fs::read_to_string(fixture_path) + .expect(&format!("Failed to read fixture from {}", fixture_path)); + + test_foster_parenting_text_extraction(&foster_parenting_html); + println!("✓ test_foster_parenting_text_extraction passed"); + + test_foster_parenting_table_structure(&foster_parenting_html); + println!("✓ test_foster_parenting_table_structure passed"); + + test_expected_parsed_output_assertion(&foster_parenting_html); + println!("✓ test_expected_parsed_output_assertion passed"); + + println!("\n✓ All assertions passed (3/3)"); + println!("\nFixture source: {}", fixture_path); + println!("Parser source: src/helpers/web/scraping.rs::parse_html()"); +} + +/// Test: Text nodes in
are foster-parented to body +fn test_foster_parenting_text_extraction(html: &str) { + let document = parse_html(html); + let body_sel = Selector::parse("body").expect("Valid CSS selector"); + + let body_text: String = document + .select(&body_sel) + .next() + .map(|el| el.text().collect()) + .unwrap_or_default(); + + assert!( + body_text.contains("orphaned text"), + "Text 'orphaned text' should be present in body (fostered from table)" + ); + assert!( + body_text.contains("more text"), + "Text 'more text' should be present in body (fostered from table)" + ); + assert!( + body_text.contains("cell content"), + "Cell content should still be present" + ); +} + +fn test_foster_parenting_table_structure(html: &str) { + let document = parse_html(html); + + let table_sel = Selector::parse("table").expect("Valid CSS selector"); + let tr_sel = Selector::parse("tr").expect("Valid CSS selector"); + let td_sel = Selector::parse("td").expect("Valid CSS selector"); + + let tables: Vec<_> = document.select(&table_sel).collect(); + assert_eq!(tables.len(), 1, "Should have exactly 1 table"); + + let rows: Vec<_> = document.select(&tr_sel).collect(); + assert_eq!(rows.len(), 1, "Should have exactly 1 row"); + + let cells: Vec<_> = document.select(&td_sel).collect(); + assert_eq!(cells.len(), 1, "Should have exactly 1 cell"); + + if let Some(cell) = cells.first() { + let cell_text: String = cell.text().collect(); + assert_eq!( + cell_text.trim(), + "cell content", + "Cell content should be preserved" + ); + } +} + +fn test_expected_parsed_output_assertion(html: &str) { + let document = parse_html(html); + + let body_sel = Selector::parse("body").expect("Valid CSS selector"); + let body = document + .select(&body_sel) + .next() + .expect("body should exist"); + let full_text: String = body.text().collect(); + + let expected_pattern = "orphaned textmore textcell content"; + assert!( + full_text.contains(&expected_pattern) + || (full_text.contains("orphaned text") + && full_text.contains("more text") + && full_text.contains("cell content")), + "Parsed output should contain all text content in fostered form. Got: '{}'", + full_text + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn load_fixture() -> String { + fs::read_to_string("src/bin/test_fixtures/foster_parenting_minimal.html") + .expect("Failed to load fixture") + } + + #[test] + fn test_foster_parenting_text_extraction_test() { + let html = load_fixture(); + test_foster_parenting_text_extraction(&html); + } + + #[test] + fn test_foster_parenting_table_structure_test() { + let html = load_fixture(); + test_foster_parenting_table_structure(&html); + } + + #[test] + fn test_expected_parsed_output_assertion_test() { + let html = load_fixture(); + test_expected_parsed_output_assertion(&html); + } +} diff --git a/src/bin/scaffold_enhanced/generators/api.rs b/src/bin/scaffold_enhanced/generators/api.rs new file mode 100644 index 0000000..1b75a01 --- /dev/null +++ b/src/bin/scaffold_enhanced/generators/api.rs @@ -0,0 +1,83 @@ +//! Complete API generator - combines model, migration, controller, service, repository + +use anyhow::Result; + +pub fn generate_full_api(name: &str, full: bool) -> Result<()> { + println!("📦 Generating model..."); + let model_name = singularize(name); + super::model::generate_model(&model_name, true, true, false)?; + + if full { + println!("🔧 Generating service..."); + super::service::generate_service(name, Some(&model_name))?; + + println!("💾 Generating repository..."); + super::repository::generate_repository(name, &model_name)?; + } + + println!("🎮 Generating CRUD controller..."); + super::controller::generate_controller(name, true, Some(&model_name))?; + + println!("\n✅ Complete API generated!"); + println!("\n📋 Generated files:"); + println!( + " - src/entities/{}.rs (SeaORM model)", + model_name.to_lowercase() + ); + println!( + " - migrations/m*_create_{}.rs (migration)", + super::model::pluralize(&model_name) + ); + + if full { + println!(" - src/services/{}_service.rs (service layer)", name); + println!(" - src/repositories/{}_repository.rs (repository)", name); + } + + println!(" - src/routes/api/{}/index.rs (list)", name); + println!(" - src/routes/api/{}/[id].rs (get)", name); + println!(" - src/routes/api/{}/create.rs (create)", name); + println!(" - src/routes/api/{}/[id]/update.rs (update)", name); + println!(" - src/routes/api/{}/[id]/delete.rs (delete)", name); + + println!("\n🚀 Next steps:"); + println!(" 1. Run 'cargo build' to compile"); + println!(" 2. Run migrations: cargo run -- migration up"); + println!(" 3. Start server: cargo run"); + + println!("\n📡 Available endpoints:"); + println!(" GET /api/{} - List all", name); + println!(" GET /api/{}/{{id}} - Get one", name); + println!(" POST /api/{} - Create", name); + println!(" PUT /api/{}/{{id}} - Update", name); + println!(" DELETE /api/{}/{{id}} - Delete", name); + + Ok(()) +} + +fn singularize(word: &str) -> String { + let lower = word.to_lowercase(); + + if lower.ends_with("ies") { + format!("{}y", &lower[..lower.len() - 3]) + } else if lower.ends_with("es") { + lower[..lower.len() - 2].to_string() + } else if lower.ends_with('s') { + lower[..lower.len() - 1].to_string() + } else { + lower + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_singularize() { + assert_eq!(singularize("users"), "user"); + assert_eq!(singularize("categories"), "category"); + assert_eq!(singularize("posts"), "post"); + assert_eq!(singularize("boxes"), "box"); + } +} diff --git a/src/bin/scaffold_enhanced/generators/controller.rs b/src/bin/scaffold_enhanced/generators/controller.rs new file mode 100644 index 0000000..bfa7e7a --- /dev/null +++ b/src/bin/scaffold_enhanced/generators/controller.rs @@ -0,0 +1,287 @@ +//! API controller generator with CRUD operations + +use anyhow::Result; +use std::fs; +use std::path::Path; + +pub fn generate_controller(name: &str, crud: bool, model: Option<&str>) -> Result<()> { + let api_dir = Path::new("src/routes/api").join(name); + fs::create_dir_all(&api_dir)?; + + let model_name = model.map(|s| s.to_string()).unwrap_or_else(|| { + // Singularize the resource name + let singular = name.trim_end_matches('s'); + format!("{}{}", &singular[..1].to_uppercase(), &singular[1..]) + }); + + if crud { + generate_crud_routes(&api_dir, name, &model_name)?; + } else { + generate_basic_controller(&api_dir, name); + } + + Ok(()) +} + +fn generate_crud_routes(api_dir: &Path, resource: &str, model: &str) -> Result<()> { + // List all + let index_content = generate_list_handler(resource, model); + fs::write(api_dir.join("index.rs"), index_content)?; + + // Get by ID + let show_content = generate_show_handler(resource, model); + fs::write(api_dir.join("[id].rs"), show_content)?; + + // Create + let create_content = generate_create_handler(resource, model); + fs::write(api_dir.join("create.rs"), create_content)?; + + // Update & Delete in [id] subdirectory + fs::create_dir_all(api_dir.join("[id]"))?; + + let update_content = generate_update_handler(resource, model); + fs::write(api_dir.join("[id]/update.rs"), update_content)?; + + let delete_content = generate_delete_handler(resource, model); + fs::write(api_dir.join("[id]/delete.rs"), delete_content)?; + + Ok(()) +} + +fn generate_list_handler(resource: &str, model: &str) -> String { + format!( + r#"//! List all {resource} + +use axum::{{Extension, Json, response::IntoResponse, Router}}; +use sea_orm::{{DatabaseConnection, EntityTrait}}; +use std::sync::Arc; +use crate::shared::state::AppState; +use crate::entities::{model_low}::{{Entity as {model}, Model}}; + +pub async fn list( + Extension(db): Extension, +) -> impl IntoResponse {{ + match {model}.find().all(&db).await {{ + Ok(items) => Json(items).into_response(), + Err(e) => {{ + eprintln!("Error listing {resource}: {{}}", e); + (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to list {resource}").into_response() + }} + }} +}} + +pub fn register_routes(router: Router>) -> Router> {{ + router +}} +"#, + resource = resource, + model_low = model.to_lowercase(), + model = model + ) +} + +fn generate_show_handler(resource: &str, model: &str) -> String { + let singular = resource.trim_end_matches('s'); + format!( + r#"//! Get {singular} by ID + +use axum::{{Extension, Json, extract::Path, response::IntoResponse, Router}}; +use sea_orm::{{DatabaseConnection, EntityTrait}}; +use std::sync::Arc; +use crate::shared::state::AppState; +use crate::entities::{model_low}::{{Entity as {model}, Model}}; + +pub async fn show( + Path(id): Path, + Extension(db): Extension, +) -> impl IntoResponse {{ + match {model}.find_by_id(id).one(&db).await {{ + Ok(Some(item)) => Json(item).into_response(), + Ok(None) => (axum::http::StatusCode::NOT_FOUND, "{singular} not found").into_response(), + Err(e) => {{ + eprintln!("Error getting {singular}: {{}}", e); + (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to get {singular}").into_response() + }} + }} +}} + +pub fn register_routes(router: Router>) -> Router> {{ + router +}} +"#, + singular = singular, + model_low = model.to_lowercase(), + model = model, + resource = resource + ) +} + +fn generate_create_handler(resource: &str, model: &str) -> String { + let singular = resource.trim_end_matches('s'); + format!( + r#"//! Create new {singular} + +use axum::{{Extension, Json, response::IntoResponse, Router}}; +use sea_orm::{{ActiveModelTrait, DatabaseConnection, Set}}; +use serde::{{Deserialize, Serialize}}; +use std::sync::Arc; +use crate::shared::state::AppState; +use crate::entities::{model_low}::{{ActiveModel, Model}}; + +#[derive(Serialize, Deserialize)] +pub struct Create{model}Dto {{ + pub name: String, + // Add your fields +}} + +pub async fn create( + Extension(db): Extension, + Json(data): Json, +) -> impl IntoResponse {{ + let new_item = ActiveModel {{ + name: Set(data.name), + ..Default::default() + }}; + + match new_item.insert(&db).await {{ + Ok(item) => (axum::http::StatusCode::CREATED, Json(item)).into_response(), + Err(e) => {{ + eprintln!("Error creating {singular}: {{}}", e); + (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to create {singular}").into_response() + }} + }} +}} + +pub fn register_routes(router: Router>) -> Router> {{ + router +}} +"#, + singular = singular, + model_low = model.to_lowercase(), + model = model, + resource = resource + ) +} + +fn generate_update_handler(resource: &str, model: &str) -> String { + let singular = resource.trim_end_matches('s'); + format!( + r#"//! Update {singular} + +use axum::{{Extension, Json, extract::Path, response::IntoResponse, Router}}; +use sea_orm::{{ActiveModelTrait, DatabaseConnection, EntityTrait, Set}}; +use serde::{{Deserialize, Serialize}}; +use std::sync::Arc; +use crate::shared::state::AppState; +use crate::entities::{model_low}::{{ActiveModel, Entity as {model}, Model}}; + +#[derive(Serialize, Deserialize)] +pub struct Update{model}Dto {{ + pub name: Option, + // Add your fields +}} + +pub async fn update( + Path(id): Path, + Extension(db): Extension, + Json(data): Json, +) -> impl IntoResponse {{ + let item = match {model}.find_by_id(id).one(&db).await {{ + Ok(Some(item)) => item, + Ok(None) => return (axum::http::StatusCode::NOT_FOUND, "{singular} not found").into_response(), + Err(e) => {{ + eprintln!("Error finding {singular}: {{}}", e); + return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to find {singular}").into_response(); + }} + }}; + + let mut active_model: ActiveModel = item.into(); + if let Some(name) = data.name {{ + active_model.name = Set(name); + }} + + match active_model.update(&db).await {{ + Ok(updated) => Json(updated).into_response(), + Err(e) => {{ + eprintln!("Error updating {singular}: {{}}", e); + (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to update {singular}").into_response() + }} + }} +}} + +pub fn register_routes(router: Router>) -> Router> {{ + router +}} +"#, + singular = singular, + model_low = model.to_lowercase(), + model = model, + resource = resource + ) +} + +fn generate_delete_handler(resource: &str, model: &str) -> String { + let singular = resource.trim_end_matches('s'); + format!( + r#"//! Delete {singular} + +use axum::{{Extension, extract::Path, response::IntoResponse, Router}}; +use sea_orm::{{ActiveModelTrait, DatabaseConnection, EntityTrait, IntoActiveModel}}; +use std::sync::Arc; +use crate::shared::state::AppState; +use crate::entities::{model_low}::{{Entity as {model}}}; + +pub async fn destroy( + Path(id): Path, + Extension(db): Extension, +) -> impl IntoResponse {{ + let item = match {model}.find_by_id(id).one(&db).await {{ + Ok(Some(item)) => item, + Ok(None) => return (axum::http::StatusCode::NOT_FOUND, "{singular} not found").into_response(), + Err(e) => {{ + eprintln!("Error finding {singular}: {{}}", e); + return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to find {singular}").into_response(); + }} + }}; + + match item.into_active_model().delete(&db).await {{ + Ok(_) => axum::http::StatusCode::NO_CONTENT.into_response(), + Err(e) => {{ + eprintln!("Error deleting {singular}: {{}}", e); + (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to delete {singular}").into_response() + }} + }} +}} + +pub fn register_routes(router: Router>) -> Router> {{ + router +}} +"#, + singular = singular, + model_low = model.to_lowercase(), + model = model, + resource = resource + ) +} + +fn generate_basic_controller(api_dir: &Path, resource: &str) { + let content = format!( + r#"//! {resource} controller + +use axum::Router; +use std::sync::Arc; +use crate::shared::state::AppState; + +pub async fn index() -> &'static str {{ + "{resource} endpoint" +}} + +pub fn register_routes(router: Router>) -> Router> {{ + router +}} +"#, + resource = resource + ); + + let _ = fs::write(api_dir.join("index.rs"), content); +} diff --git a/src/bin/scaffold_enhanced/generators/migration.rs b/src/bin/scaffold_enhanced/generators/migration.rs new file mode 100644 index 0000000..7a3f76e --- /dev/null +++ b/src/bin/scaffold_enhanced/generators/migration.rs @@ -0,0 +1,262 @@ +//! Database migration generator + +use anyhow::{Context, Result}; +use chrono::Local; +use std::fs; +use std::path::Path; + +pub fn generate_migration(name: &str, table: Option<&str>) -> Result<()> { + let timestamp = Local::now().format("%Y%m%d%H%M%S"); + let file_name = format!("m{}_{}.rs", timestamp, name); + + let migrations_dir = Path::new("migrations"); + fs::create_dir_all(migrations_dir)?; + + let content = if let Some(table_name) = table { + generate_create_table_migration(table_name) + } else { + generate_empty_migration() + }; + + let migration_path = migrations_dir.join(&file_name); + fs::write(&migration_path, content) + .with_context(|| format!("Failed to write migration: {:?}", migration_path))?; + + update_migrations_mod(&file_name)?; + + Ok(()) +} + +pub fn generate_model_migration(table: &str, timestamps: bool, soft_delete: bool) -> Result<()> { + let timestamp = Local::now().format("%Y%m%d%H%M%S"); + let name = format!("create_{}_table", table); + let file_name = format!("m{}_{}.rs", timestamp, name); + + let migrations_dir = Path::new("migrations"); + fs::create_dir_all(migrations_dir)?; + + let content = generate_model_table_migration(table, timestamps, soft_delete); + + let migration_path = migrations_dir.join(&file_name); + fs::write(&migration_path, content)?; + + update_migrations_mod(&file_name)?; + + Ok(()) +} + +fn generate_create_table_migration(table: &str) -> String { + let struct_name = table + .split('_') + .map(|s| { + let mut c = s.chars(); + match c.next() { + None => String::new(), + Some(f) => f.to_uppercase().chain(c).collect(), + } + }) + .collect::(); + + format!( + r#"use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration {{ + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {{ + manager + .create_table( + Table::create() + .table({table}::Table) + .if_not_exists() + .col( + ColumnDef::new({table}::Id) + .integer() + .not_null() + .auto_increment() + .primary_key(), + ) + .col(ColumnDef::new({table}::Name).string().not_null()) + .to_owned(), + ) + .await + }} + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {{ + manager + .drop_table(Table::drop().table({table}::Table).to_owned()) + .await + }} +}} + +#[derive(DeriveIden)] +enum {table} {{ + Table, + Id, + Name, +}} +"#, + table = struct_name + ) +} + +fn generate_model_table_migration(table: &str, timestamps: bool, soft_delete: bool) -> String { + let table_pascal = table + .split('_') + .map(|s| { + let mut c = s.chars(); + match c.next() { + None => String::new(), + Some(f) => f.to_uppercase().chain(c).collect(), + } + }) + .collect::(); + + let timestamp_cols = if timestamps { + format!( + r#" + .col(ColumnDef::new({}::CreatedAt).timestamp().null()) + .col(ColumnDef::new({}::UpdatedAt).timestamp().null())"#, + table_pascal, table_pascal + ) + } else { + String::new() + }; + + let soft_delete_col = if soft_delete { + format!( + r#" + .col(ColumnDef::new({}::DeletedAt).timestamp().null())"#, + table_pascal + ) + } else { + String::new() + }; + + let enum_fields = if timestamps && soft_delete { + format!(" CreatedAt,\n UpdatedAt,\n DeletedAt,") + } else if timestamps { + format!(" CreatedAt,\n UpdatedAt,") + } else if soft_delete { + format!(" DeletedAt,") + } else { + String::new() + }; + + format!( + r#"use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration {{ + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {{ + manager + .create_table( + Table::create() + .table({table}::Table) + .if_not_exists() + .col( + ColumnDef::new({table}::Id) + .integer() + .not_null() + .auto_increment() + .primary_key(), + ) + .col(ColumnDef::new({table}::Name).string().not_null()){timestamps}{soft_delete} + .to_owned(), + ) + .await + }} + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {{ + manager + .drop_table(Table::drop().table({table}::Table).to_owned()) + .await + }} +}} + +#[derive(DeriveIden)] +enum {table} {{ + Table, + Id, + Name, +{enum_fields} +}} +"#, + table = table_pascal, + timestamps = timestamp_cols, + soft_delete = soft_delete_col, + enum_fields = enum_fields + ) +} + +fn generate_empty_migration() -> String { + format!( + r#"use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration {{ + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {{ + // Add your migration logic here + Ok(()) + }} + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {{ + // Add your rollback logic here + Ok(()) + }} +}} +"# + ) +} + +fn update_migrations_mod(file_name: &str) -> Result<()> { + let mod_path = Path::new("migrations/mod.rs"); + let module_name = file_name.trim_end_matches(".rs"); + let module_line = format!("mod {};", module_name); + + if mod_path.exists() { + let content = fs::read_to_string(mod_path)?; + if !content.contains(&module_line) { + // Find the vec![] and add migration + let new_content = if content.contains("vec![") { + content.replace( + "vec![", + &format!("vec![\n Box::new({}::Migration),", module_name), + ) + } else { + format!("{}\n{}", content.trim(), module_line) + }; + fs::write(mod_path, new_content)?; + } + } else { + let initial_content = format!( + r#"pub use sea_orm_migration::prelude::*; + +{} + +pub struct Migrator; + +#[async_trait::async_trait] +impl MigratorTrait for Migrator {{ + fn migrations() -> Vec> {{ + vec![ + Box::new({}::Migration), + ] + }} +}} +"#, + module_line, module_name + ); + fs::write(mod_path, initial_content)?; + } + + Ok(()) +} diff --git a/src/bin/scaffold_enhanced/generators/mod.rs b/src/bin/scaffold_enhanced/generators/mod.rs new file mode 100644 index 0000000..9faff31 --- /dev/null +++ b/src/bin/scaffold_enhanced/generators/mod.rs @@ -0,0 +1,7 @@ +// Module declarations for generators +pub mod api; +pub mod controller; +pub mod migration; +pub mod model; +pub mod repository; +pub mod service; diff --git a/src/bin/scaffold_enhanced/generators/model.rs b/src/bin/scaffold_enhanced/generators/model.rs new file mode 100644 index 0000000..55b2c36 --- /dev/null +++ b/src/bin/scaffold_enhanced/generators/model.rs @@ -0,0 +1,125 @@ +//! SeaORM model generator + +use anyhow::{Context, Result}; +use std::fs; +use std::path::Path; + +pub fn generate_model( + name: &str, + with_migration: bool, + timestamps: bool, + soft_delete: bool, +) -> Result<()> { + let table_name = pluralize(name); + + // Create entities directory + let entities_dir = Path::new("src/entities"); + fs::create_dir_all(entities_dir)?; + + // Generate model file + let model_content = generate_model_content(name, &table_name, timestamps, soft_delete); + let model_path = entities_dir.join(format!("{}.rs", name.to_lowercase())); + fs::write(&model_path, model_content) + .with_context(|| format!("Failed to write model file: {:?}", model_path))?; + + // Update entities/mod.rs + update_entities_mod(name)?; + + // Generate migration if requested + if with_migration { + super::migration::generate_model_migration(&table_name, timestamps, soft_delete)?; + } + + Ok(()) +} + +fn generate_model_content( + name: &str, + table_name: &str, + timestamps: bool, + soft_delete: bool, +) -> String { + let timestamp_fields = if timestamps { + r#" + #[sea_orm(nullable)] + pub created_at: Option, + #[sea_orm(nullable)] + pub updated_at: Option,"# + } else { + "" + }; + + let soft_delete_field = if soft_delete { + r#" + #[sea_orm(nullable)] + pub deleted_at: Option,"# + } else { + "" + }; + + format!( + r#"//! {} entity + +use sea_orm::entity::prelude::*; +use serde::{{Deserialize, Serialize}}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "{}")] +pub struct Model {{ + #[sea_orm(primary_key)] + pub id: i32, + + // Add your fields here + pub name: String,{}{} +}} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {{}} + +impl ActiveModelBehavior for ActiveModel {{}} +"#, + name, table_name, timestamp_fields, soft_delete_field + ) +} + +fn update_entities_mod(name: &str) -> Result<()> { + let mod_path = Path::new("src/entities/mod.rs"); + let module_line = format!("pub mod {};", name.to_lowercase()); + + if mod_path.exists() { + let content = fs::read_to_string(mod_path)?; + if !content.contains(&module_line) { + let new_content = format!("{}\n{}", content.trim(), module_line); + fs::write(mod_path, new_content)?; + } + } else { + fs::write(mod_path, format!("{}\n", module_line))?; + } + + Ok(()) +} + +pub fn pluralize(word: &str) -> String { + let lower = word.to_lowercase(); + + if lower.ends_with('y') { + format!("{}ies", &lower[..lower.len() - 1]) + } else if lower.ends_with('s') || lower.ends_with("ch") || lower.ends_with("sh") || lower.ends_with('x') { + format!("{}es", lower) + } else { + format!("{}s", lower) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_pluralize() { + assert_eq!(pluralize("User"), "users"); + assert_eq!(pluralize("Category"), "categories"); + assert_eq!(pluralize("Post"), "posts"); + assert_eq!(pluralize("Box"), "boxes"); + } +} diff --git a/src/bin/scaffold_enhanced/generators/repository.rs b/src/bin/scaffold_enhanced/generators/repository.rs new file mode 100644 index 0000000..e4094bf --- /dev/null +++ b/src/bin/scaffold_enhanced/generators/repository.rs @@ -0,0 +1,112 @@ +//! Repository pattern generator + +use anyhow::{Context, Result}; +use std::fs; +use std::path::Path; + +pub fn generate_repository(name: &str, model: &str) -> Result<()> { + let repos_dir = Path::new("src/repositories"); + fs::create_dir_all(repos_dir)?; + + let repo_content = generate_repository_content(name, model); + + let repo_path = repos_dir.join(format!("{}_repository.rs", name.to_lowercase())); + fs::write(&repo_path, repo_content) + .with_context(|| format!("Failed to write repository: {:?}", repo_path))?; + + update_repositories_mod(name)?; + + Ok(()) +} + +fn generate_repository_content(name: &str, model: &str) -> String { + format!( + r#"//! {} repository + +use sea_orm::*; +use crate::entities::{}::{{Entity as {}, Model, ActiveModel, Column}}; + +#[derive(Clone)] +pub struct {}Repository {{ + db: DatabaseConnection, +}} + +impl {}Repository {{ + pub fn new(db: DatabaseConnection) -> Self {{ + Self {{ db }} + }} + + /// Find all records + pub async fn find_all(&self) -> Result, DbErr> {{ + {}.find().all(&self.db).await + }} + + /// Find by ID + pub async fn find_by_id(&self, id: i32) -> Result, DbErr> {{ + {}.find_by_id(id).one(&self.db).await + }} + + /// Find with pagination + pub async fn paginate(&self, page: u64, per_page: u64) -> Result<(Vec, u64), DbErr> {{ + let paginator = {}.find() + .paginate(&self.db, per_page); + + let total = paginator.num_items().await?; + let items = paginator.fetch_page(page).await?; + + Ok((items, total)) + }} + + /// Create new record + pub async fn create(&self, data: ActiveModel) -> Result {{ + data.insert(&self.db).await + }} + + /// Update existing record + pub async fn update(&self, data: ActiveModel) -> Result {{ + data.update(&self.db).await + }} + + /// Delete by ID + pub async fn delete(&self, id: i32) -> Result {{ + {}.delete_by_id(id).exec(&self.db).await + }} + + /// Find by custom condition + pub async fn find_by_name(&self, name: &str) -> Result, DbErr> {{ + {}.find() + .filter(Column::Name.contains(name)) + .all(&self.db) + .await + }} +}} +"#, + name, + model.to_lowercase(), + model, + model, + model, + model, + model, + model, + model, + model + ) +} + +fn update_repositories_mod(name: &str) -> Result<()> { + let mod_path = Path::new("src/repositories/mod.rs"); + let module_line = format!("pub mod {}_repository;", name.to_lowercase()); + + if mod_path.exists() { + let content = fs::read_to_string(mod_path)?; + if !content.contains(&module_line) { + let new_content = format!("{}\n{}", content.trim(), module_line); + fs::write(mod_path, new_content)?; + } + } else { + fs::write(mod_path, format!("{}\n", module_line))?; + } + + Ok(()) +} diff --git a/src/bin/scaffold_enhanced/generators/service.rs b/src/bin/scaffold_enhanced/generators/service.rs new file mode 100644 index 0000000..adc4587 --- /dev/null +++ b/src/bin/scaffold_enhanced/generators/service.rs @@ -0,0 +1,86 @@ +//! Service layer generator + +use anyhow::{Context, Result}; +use std::fs; +use std::path::Path; + +pub fn generate_service(name: &str, model: Option<&str>) -> Result<()> { + let services_dir = Path::new("src/services"); + fs::create_dir_all(services_dir)?; + + let model_name = model.unwrap_or(name); + let service_content = generate_service_content(name, model_name); + + let service_path = services_dir.join(format!("{}_service.rs", name.to_lowercase())); + fs::write(&service_path, service_content) + .with_context(|| format!("Failed to write service: {:?}", service_path))?; + + update_services_mod(name)?; + + Ok(()) +} + +fn generate_service_content(name: &str, model: &str) -> String { + format!( + r#"//! {} service layer + +use sea_orm::*; +use crate::entities::{}::{{Entity as {}, Model, ActiveModel}}; + +pub struct {}Service {{ + db: DatabaseConnection, +}} + +impl {}Service {{ + pub fn new(db: DatabaseConnection) -> Self {{ + Self {{ db }} + }} + + pub async fn find_all(&self) -> Result, DbErr> {{ + {}.find().all(&self.db).await + }} + + pub async fn find_by_id(&self, id: i32) -> Result, DbErr> {{ + {}.find_by_id(id).one(&self.db).await + }} + + pub async fn create(&self, data: ActiveModel) -> Result {{ + data.insert(&self.db).await + }} + + pub async fn update(&self, id: i32, data: ActiveModel) -> Result {{ + data.update(&self.db).await + }} + + pub async fn delete(&self, id: i32) -> Result {{ + {}.delete_by_id(id).exec(&self.db).await + }} +}} +"#, + name, + model.to_lowercase(), + model, + model, + model, + model, + model, + model + ) +} + +fn update_services_mod(name: &str) -> Result<()> { + let mod_path = Path::new("src/services/mod.rs"); + let module_line = format!("pub mod {}_service;", name.to_lowercase()); + + if mod_path.exists() { + let content = fs::read_to_string(mod_path)?; + if !content.contains(&module_line) { + let new_content = format!("{}\n{}", content.trim(), module_line); + fs::write(mod_path, new_content)?; + } + } else { + fs::write(mod_path, format!("{}\n", module_line))?; + } + + Ok(()) +} diff --git a/src/bin/test_fixtures/foster_parenting_minimal.html b/src/bin/test_fixtures/foster_parenting_minimal.html new file mode 100644 index 0000000..7b9b5ea --- /dev/null +++ b/src/bin/test_fixtures/foster_parenting_minimal.html @@ -0,0 +1 @@ +
orphaned textmore text
cell content
diff --git a/src/bootstrap/mod.rs b/src/bootstrap/mod.rs new file mode 100644 index 0000000..d5d931b --- /dev/null +++ b/src/bootstrap/mod.rs @@ -0,0 +1,118 @@ +use std::net::SocketAddr; +use std::sync::Arc; +use tokio::net::TcpListener; + +use axum::Router; +use sea_orm::Database; +use tracing_subscriber::EnvFilter; + +use crate::shared::config::CONFIG; +use crate::shared::database::get_redis_conn; +use crate::shared::state::AppState; + +pub struct Application { + pub port: u16, + router: Router, + listener: TcpListener, +} + +impl Application { + pub async fn build() -> anyhow::Result { + // Initialize tracing. Default to warn/error globally unless RUST_LOG is explicitly set. + let env_filter = match std::env::var("RUST_LOG") { + Ok(filter) => EnvFilter::new(filter).add_directive("html5ever=error".parse()?), + Err(_) => EnvFilter::new("warn,html5ever=error"), + }; + + tracing_subscriber::fmt().with_env_filter(env_filter).init(); + + // Initialize OpenTelemetry metrics + crate::shared::observability::metrics::init_otel_metrics(); + + tracing::info!("🚀 Scraper starting up..."); + tracing::info!(" Environment: {}", CONFIG.environment); + + // Log thread configuration + let worker_threads = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1); + tracing::info!( + " Tokio Worker Threads: (Defaulting to CPU cores: {})", + worker_threads + ); + + // Redis + let _ = get_redis_conn().await; + + // Browser Pool + tracing::info!("Initializing browser pool..."); + let browser_config = crate::shared::browser::BrowserPoolConfig::default(); + match crate::shared::browser::pool::init_browser_pool(browser_config).await { + Ok(_) => tracing::info!("✓ Browser pool initialized"), + Err(e) => tracing::error!("⚠️ Failed to initialize browser pool: {}", e), + } + + // Database + let mut opt = sea_orm::ConnectOptions::new(CONFIG.database_url.clone()); + opt.max_connections(20) + .min_connections(1) + .connect_timeout(std::time::Duration::from_secs( + CONFIG.db.connect_timeout_seconds, + )) + .idle_timeout(std::time::Duration::from_secs( + CONFIG.db.idle_timeout_seconds, + )) + .acquire_timeout(std::time::Duration::from_secs( + CONFIG.db.acquire_timeout_seconds, + )) + .max_lifetime(std::time::Duration::from_secs( + CONFIG.db.max_lifetime_seconds, + )) + .sqlx_logging(CONFIG.log_level == "debug"); + + let db = Database::connect(opt) + .await + .map_err(|e| anyhow::anyhow!("Failed to connect to database: {}", e))?; + tracing::info!("✓ SeaORM database connection established"); + + // Schema & Seeding + if let Err(e) = crate::shared::database::setup::init(&db).await { + tracing::error!("Failed to init DB schema: {}", e); + } + + // App State components + let db_arc = Arc::new(db); + let image_processing_semaphore = Arc::new(tokio::sync::Semaphore::new( + CONFIG.image_processing_concurrency, + )); + let event_bus = Arc::new(crate::shared::events::bus::EventBus::new()); + + let redis_pool = crate::shared::database::redis_pool() + .map_err(|e| anyhow::anyhow!("Failed to init Redis pool: {}", e))?; + + let app_state = Arc::new(AppState { + redis_pool, + db: db_arc.clone(), + image_processing_semaphore, + event_bus: event_bus.clone(), + }); + + let app = crate::app::build_router(app_state, db_arc.clone()).await?; + + // Listener + let port = CONFIG.server_port; + let addr = SocketAddr::from(([0, 0, 0, 0], port)); + let listener = TcpListener::bind(&addr).await?; + tracing::info!("Server listening on {}", listener.local_addr()?); + + Ok(Self { + port, + router: app, + listener, + }) + } + + pub async fn run(self) -> std::io::Result<()> { + axum::serve(self.listener, self.router.into_make_service()).await + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..278f6ea --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,10 @@ +// Library root - clean organized module structure +// All modules organized into logical folders + +// ============================================================================ +// Core Framework +// ============================================================================ +pub mod app; +pub mod bootstrap; +pub mod modules; +pub mod shared; diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..e84c72a --- /dev/null +++ b/src/main.rs @@ -0,0 +1,12 @@ +#![doc = "Logging Setup"] +use scraper_service::bootstrap::Application; + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> anyhow::Result<()> { + // Application setup and server logic is now encapsulated in `startup.rs` + // This makes the main function clean and the app easier to integration test. + let app = Application::build().await?; + app.run().await?; + + Ok(()) +} diff --git a/src/modules/anime/controller.rs b/src/modules/anime/controller.rs new file mode 100644 index 0000000..9efb932 --- /dev/null +++ b/src/modules/anime/controller.rs @@ -0,0 +1,236 @@ +use crate::modules::anime::repository::AnimeRepository; +use crate::modules::anime::service::AnimeService; +use crate::shared::errors::AppError; +use crate::shared::state::AppState; +use axum::extract::{Path, State}; +use axum::Json; +use std::sync::Arc; +use tracing::info; + +#[utoipa::path( + get, + path = "/api/anime", + tag = "anime", + operation_id = "anime_index", + responses( + (status = 200, description = "Handles GET requests for the /api/anime endpoint.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn anime_index( + State(app_state): State>, +) -> Result, AppError> { + info!("Handling request for anime index"); + let service = AnimeService::new(AnimeRepository::new()); + service.get_anime_index(app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/genre_list", + tag = "anime", + operation_id = "anime_genre_list", + responses( + (status = 200, description = "Handles GET requests for the /api/anime/genre_list endpoint.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn genres( + State(app_state): State>, +) -> Result, AppError> { + info!("Handling request for anime genres"); + let service = AnimeService::new(AnimeRepository::new()); + service.get_genres(app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/detail/{slug}", + tag = "anime", + operation_id = "anime_detail_slug", + responses( + (status = 200, description = "Retrieves details for a specific detail by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn detail_slug( + State(app_state): State>, + Path(slug): Path, +) -> Result, AppError> { + info!("Starting request for detail slug: {}", slug); + let service = AnimeService::new(AnimeRepository::new()); + service.get_anime_detail(app_state, slug).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/complete_anime/{slug}", + tag = "anime", + operation_id = "anime_complete_anime_slug", + responses( + (status = 200, description = "Retrieves details for a specific complete_anime by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn complete_anime_slug( + State(app_state): State>, + Path(slug): Path, +) -> Result, AppError> { + info!("Starting request for complete_anime slug: {}", slug); + let service = AnimeService::new(AnimeRepository::new()); + service + .get_complete_anime_page(app_state, slug) + .await + .map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/full/{slug}", + tag = "anime", + operation_id = "anime_full_slug", + responses( + (status = 200, description = "Retrieves full episode details for a specific episode by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn full_slug( + State(app_state): State>, + Path(slug): Path, +) -> Result, AppError> { + info!("Starting request for full slug: {}", slug); + let service = AnimeService::new(AnimeRepository::new()); + service.get_anime_full(app_state, slug).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/ongoing_anime/{slug}", + tag = "anime", + operation_id = "anime_ongoing_anime_slug", + responses( + (status = 200, description = "Retrieves details for a specific ongoing_anime by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn ongoing_anime_slug( + State(app_state): State>, + Path(slug): Path, +) -> Result, AppError> { + info!("Starting request for ongoing_anime slug: {}", slug); + let service = AnimeService::new(AnimeRepository::new()); + service + .get_ongoing_anime_page(app_state, slug) + .await + .map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/latest/{slug}", + tag = "anime", + operation_id = "anime_latest_slug", + responses( + (status = 200, description = "Retrieves details for a specific latest by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn latest_slug( + State(app_state): State>, + Path(slug): Path, +) -> Result, AppError> { + info!("Starting request for latest slug: {}", slug); + let service = AnimeService::new(AnimeRepository::new()); + service + .get_latest_anime_page(app_state, slug) + .await + .map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/search/{slug}", + tag = "anime", + operation_id = "anime_search_slug_index", + responses( + (status = 200, description = "Retrieves details for a specific search by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn search_slug_index( + State(app_state): State>, + Path(slug): Path, +) -> Result, AppError> { + info!("Starting request for search slug: {}", slug); + let service = AnimeService::new(AnimeRepository::new()); + service + .get_search_anime_page(app_state, slug, "1".to_string()) + .await + .map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/search/{slug}/{page}", + tag = "anime", + operation_id = "anime_search_slug_page", + responses( + (status = 200, description = "Retrieves details for a specific search by slug and page.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn search_slug_page( + State(app_state): State>, + Path((slug, page)): Path<(String, String)>, +) -> Result, AppError> { + info!("Starting request for search slug: {} page: {}", slug, page); + let service = AnimeService::new(AnimeRepository::new()); + service + .get_search_anime_page(app_state, slug, page) + .await + .map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/genre/{slug}", + tag = "anime", + operation_id = "anime_genre_slug_index", + responses( + (status = 200, description = "Retrieves details for a specific genre by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn genre_slug_index( + State(app_state): State>, + Path(slug): Path, +) -> Result, AppError> { + info!("Starting request for genre slug: {}", slug); + let service = AnimeService::new(AnimeRepository::new()); + service + .get_genre_anime_page(app_state, slug, "1".to_string()) + .await + .map(Json) +} + +#[utoipa::path( + get, + path = "/api/anime/genre/{slug}/{page}", + tag = "anime", + operation_id = "anime_genre_slug_page", + responses( + (status = 200, description = "Retrieves details for a specific genre by slug and page.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn genre_slug_page( + State(app_state): State>, + Path((slug, page)): Path<(String, String)>, +) -> Result, AppError> { + info!("Starting request for genre slug: {} page: {}", slug, page); + let service = AnimeService::new(AnimeRepository::new()); + service + .get_genre_anime_page(app_state, slug, page) + .await + .map(Json) +} diff --git a/src/modules/anime/mod.rs b/src/modules/anime/mod.rs new file mode 100644 index 0000000..67dc224 --- /dev/null +++ b/src/modules/anime/mod.rs @@ -0,0 +1,7 @@ +pub mod controller; +pub mod parser; +pub mod repository; +pub mod route; +pub mod schema; +pub mod service; +pub mod types; diff --git a/src/modules/anime/parser.rs b/src/modules/anime/parser.rs new file mode 100644 index 0000000..62418cf --- /dev/null +++ b/src/modules/anime/parser.rs @@ -0,0 +1,632 @@ +use crate::modules::anime::types::*; +use crate::shared::errors::AppError; +use crate::shared::utils::parse_html; +use crate::shared::utils::scraping::{ + attr, attr_from, attr_from_or, extract_slug, selector, text, text_from_or, +}; + +pub fn parse_ongoing_anime(html: &str) -> Result, AppError> { + let document = parse_html(html); + let mut ongoing_anime = Vec::new(); + + let venz_selector = selector(".venz ul li").unwrap(); + let title_selector = selector(".thumbz h2.jdlflm").unwrap(); + let link_selector = selector("a").unwrap(); + let img_selector = selector("img").unwrap(); + let episode_selector = selector(".epz").unwrap(); + + for element in document.select(&venz_selector) { + let title = text_from_or(&element, &title_selector, ""); + let href = attr_from(&element, &link_selector, "href").unwrap_or_default(); + let slug = extract_slug(&href); + let poster = attr_from_or(&element, &img_selector, "src", ""); + let current_episode = text_from_or(&element, &episode_selector, "N/A"); + let anime_url = attr_from_or(&element, &link_selector, "href", ""); + + if !title.is_empty() { + ongoing_anime.push(OngoingAnimeItem { + title, + slug, + poster, + current_episode, + anime_url, + }); + } + } + Ok(ongoing_anime) +} + +pub fn parse_complete_anime(html: &str) -> Result, AppError> { + let document = parse_html(html); + let mut complete_anime = Vec::new(); + + let venz_selector = selector(".venz ul li").unwrap(); + let title_selector = selector(".thumbz h2.jdlflm").unwrap(); + let link_selector = selector("a").unwrap(); + let img_selector = selector("img").unwrap(); + let episode_selector = selector(".epz").unwrap(); + + for element in document.select(&venz_selector) { + let title = text_from_or(&element, &title_selector, ""); + let href = attr_from(&element, &link_selector, "href").unwrap_or_default(); + let slug = extract_slug(&href); + let poster = attr_from_or(&element, &img_selector, "src", ""); + let episode_count = text_from_or(&element, &episode_selector, "N/A"); + let anime_url = attr_from_or(&element, &link_selector, "href", ""); + + if !title.is_empty() { + complete_anime.push(CompleteAnimeItem { + title, + slug, + poster, + episode_count, + anime_url, + }); + } + } + Ok(complete_anime) +} + +pub fn parse_genres(html: &str) -> Result, AppError> { + let document = parse_html(html); + let mut genres = Vec::new(); + let genre_selector = selector(".genres li a, .genre-list a").unwrap(); + + for element in document.select(&genre_selector) { + let name = text(&element); + let url = attr(&element, "href").unwrap_or_default(); + let slug = extract_slug(&url); + + if !name.is_empty() && !slug.is_empty() { + genres.push(Genre { name, slug, url }); + } + } + + Ok(genres) +} + +pub fn parse_anime_detail_document(html: &str) -> Result { + let document = parse_html(html); + + let info_selector = selector(".infozingle p").unwrap(); + let poster_selector = selector(".fotoanime img").unwrap(); + let synopsis_selector = selector(".sinopc").unwrap(); + let genre_link_selector = selector("a").unwrap(); + let episode_list_selector = selector(".episodelist ul li a").unwrap(); + let recommendation_selector = selector("#recommend-anime-series .isi-anime").unwrap(); + let recommendation_title_selector = selector(".judul-anime a").unwrap(); + let recommendation_img_selector = selector("img").unwrap(); + + let mut title = String::new(); + let mut alternative_title = String::new(); + let mut r#type: Option = None; + let mut status: Option = None; + let mut release_date = String::new(); + let mut studio = String::new(); + let producers = Vec::new(); + + for element in document.select(&info_selector) { + let text = text(&element); + if text.contains("Judul:") { + title = text.replace("Judul:", "").trim().to_string(); + } else if text.contains("Japanese:") { + alternative_title = text.replace("Japanese:", "").trim().to_string(); + } else if text.contains("Type:") { + let type_str = text.replace("Type:", "").trim().to_string(); + if !type_str.is_empty() { + r#type = Some(type_str); + } + } else if text.contains("Status:") { + let status_str = text.replace("Status:", "").trim().to_string(); + if !status_str.is_empty() { + status = Some(status_str); + } + } else if text.contains("Tanggal Rilis:") { + release_date = text.replace("Tanggal Rilis:", "").trim().to_string(); + } else if text.contains("Studio:") { + studio = text.replace("Studio:", "").trim().to_string(); + } + } + + let poster = document + .select(&poster_selector) + .next() + .and_then(|e| e.value().attr("src")) + .unwrap_or("") + .to_string(); + + let synopsis = text_from_or(&document.root_element(), &synopsis_selector, ""); + + let mut genres = Vec::new(); + if let Some(genres_element) = document + .select(&info_selector) + .find(|e| text(&e).contains("Genres:")) + { + for genre_link in genres_element.select(&genre_link_selector) { + let name = text(&genre_link); + let anime_url = attr(&genre_link, "href").unwrap_or_default(); + let genre_slug = extract_slug(&anime_url); + genres.push(DetailGenre { + name, + slug: genre_slug, + anime_url, + }); + } + } + + let mut episode_lists = Vec::new(); + for element in document.select(&episode_list_selector) { + let episode = text(&element); + let href = attr(&element, "href").unwrap_or_default(); + let slug = extract_slug(&href); + episode_lists.push(EpisodeList { episode, slug }); + } + + let mut recommendations = Vec::new(); + for element in document.select(&recommendation_selector) { + let title = text_from_or(&element, &recommendation_title_selector, ""); + let poster = attr_from_or(&element, &recommendation_img_selector, "src", ""); + let href = element + .select(&genre_link_selector) + .next() + .and_then(|e| e.value().attr("href")) + .unwrap_or(""); + + let slug = extract_slug(href); + + recommendations.push(Recommendation { + title, + slug, + poster, + status: None, + r#type: None, + }); + } + + Ok(AnimeDetailData { + title, + alternative_title, + poster, + r#type, + status, + release_date, + studio, + genres, + synopsis, + episode_lists, + batch: vec![], + producers, + recommendations, + }) +} + +pub fn parse_anime_page( + html: &str, + slug: &str, +) -> Result<(Vec, Pagination), AppError> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + let item_selector = selector(".venz ul li").unwrap(); + let title_selector = selector(".thumbz h2.jdlflm").unwrap(); + let link_selector = selector("a").unwrap(); + let img_selector = selector("img").unwrap(); + let episode_selector = selector(".epz").unwrap(); + let pagination_selector = selector(".pagenavix .page-numbers:not(.next)").unwrap(); + let next_selector = selector(".pagenavix .next.page-numbers").unwrap(); + + for element in document.select(&item_selector) { + let title = text_from_or(&element, &title_selector, ""); + let anime_url = attr_from_or(&element, &link_selector, "href", ""); + let slug = extract_slug(&anime_url); + let poster = attr_from_or(&element, &img_selector, "src", ""); + let episode_count = text_from_or(&element, &episode_selector, "N/A"); + + if !title.is_empty() { + anime_list.push(CompleteAnimeListItem { + title, + slug, + poster, + episode_count, + anime_url, + }); + } + } + + let current_page = slug.parse::().unwrap_or(1); + let last_visible_page = document + .select(&pagination_selector) + .next_back() + .and_then(|e| e.text().collect::().trim().parse::().ok()) + .unwrap_or(1); + + let has_next_page = document.select(&next_selector).next().is_some(); + let next_page = if has_next_page { + Some(current_page + 1) + } else { + None + }; + let has_previous_page = current_page > 1; + let previous_page = if has_previous_page { + Some(current_page - 1) + } else { + None + }; + + let pagination = Pagination { + current_page, + last_visible_page, + has_next_page, + next_page, + has_previous_page, + previous_page, + }; + + Ok((anime_list, pagination)) +} + +pub fn parse_ongoing_anime_document( + html: &str, + slug: &str, +) -> Result<(Vec, Pagination), AppError> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + let item_selector = selector(".venz ul li").unwrap(); + let title_selector = selector(".thumbz h2.jdlflm").unwrap(); + let link_selector = selector("a").unwrap(); + let img_selector = selector("img").unwrap(); + let score_selector = selector(".epz").unwrap(); + let pagination_selector = selector(".pagenavix .page-numbers:not(.next)").unwrap(); + let next_selector = selector(".pagenavix .next.page-numbers").unwrap(); + + for element in document.select(&item_selector) { + let title = text_from_or(&element, &title_selector, ""); + let anime_url = attr_from_or(&element, &link_selector, "href", ""); + let slug = extract_slug(&anime_url); + let poster = attr_from_or(&element, &img_selector, "src", ""); + let score = text_from_or(&element, &score_selector, "N/A"); + + if !title.is_empty() { + anime_list.push(OngoingAnimeListItem { + title, + slug, + poster, + score, + anime_url, + }); + } + } + + let current_page = slug.parse::().unwrap_or(1); + let last_visible_page = document + .select(&pagination_selector) + .next_back() + .and_then(|e| e.text().collect::().trim().parse::().ok()) + .unwrap_or(1); + + let has_next_page = document.select(&next_selector).next().is_some(); + let next_page = if has_next_page { + Some(current_page + 1) + } else { + None + }; + let has_previous_page = current_page > 1; + let previous_page = if has_previous_page { + Some(current_page - 1) + } else { + None + }; + + let pagination = Pagination { + current_page, + last_visible_page, + has_next_page, + next_page, + has_previous_page, + previous_page, + }; + + Ok((anime_list, pagination)) +} + +pub fn parse_latest_anime_document( + html: &str, + slug: &str, +) -> Result<(Vec, Pagination), AppError> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + let item_selector = selector(".venz ul li").unwrap(); + let title_selector = selector(".thumbz h2.jdlflm").unwrap(); + let link_selector = selector("a").unwrap(); + let img_selector = selector("img").unwrap(); + let episode_selector = selector(".epz").unwrap(); + let pagination_selector = selector(".pagenavix .page-numbers:not(.next)").unwrap(); + let next_selector = selector(".pagenavix .next.page-numbers").unwrap(); + + for element in document.select(&item_selector) { + let title = text_from_or(&element, &title_selector, ""); + let anime_url = attr_from_or(&element, &link_selector, "href", ""); + let slug = extract_slug(&anime_url); + let poster = attr_from_or(&element, &img_selector, "src", ""); + let episode = text_from_or(&element, &episode_selector, "N/A"); + + if !title.is_empty() { + anime_list.push(LatestAnimeItem { + title, + slug, + poster, + episode, + anime_url, + }); + } + } + + let current_page = slug.parse::().unwrap_or(1); + let last_visible_page = document + .select(&pagination_selector) + .next_back() + .and_then(|e| e.text().collect::().trim().parse::().ok()) + .unwrap_or(1); + + let has_next_page = document.select(&next_selector).next().is_some(); + let next_page = if has_next_page { + Some(current_page + 1) + } else { + None + }; + let has_previous_page = current_page > 1; + let previous_page = if has_previous_page { + Some(current_page - 1) + } else { + None + }; + + let pagination = Pagination { + current_page, + last_visible_page, + has_next_page, + next_page, + has_previous_page, + previous_page, + }; + + Ok((anime_list, pagination)) +} + +pub fn parse_search_anime_document( + html: &str, + slug: &str, +) -> Result<(Vec, Pagination), AppError> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + let item_selector = selector(".venz ul li").unwrap(); + let title_selector = selector(".thumbz h2.jdlflm").unwrap(); + let link_selector = selector("a").unwrap(); + let img_selector = selector("img").unwrap(); + let episode_selector = selector(".epz").unwrap(); + let genre_selector = selector(".genre-tag").unwrap(); + let status_selector = selector(".status").unwrap(); + let rating_selector = selector(".rating").unwrap(); + let pagination_selector = selector(".pagenavix .page-numbers:not(.next)").unwrap(); + let next_selector = selector(".pagenavix .next.page-numbers").unwrap(); + + for element in document.select(&item_selector) { + let title = text_from_or(&element, &title_selector, ""); + let anime_url = attr_from_or(&element, &link_selector, "href", ""); + let slug = extract_slug(&anime_url); + let poster = attr_from_or(&element, &img_selector, "src", ""); + let episode = text_from_or(&element, &episode_selector, "N/A"); + + let mut genres = Vec::new(); + for genre_elem in element.select(&genre_selector) { + genres.push(text(&genre_elem)); + } + + let status = text_from_or(&element, &status_selector, ""); + let rating = text_from_or(&element, &rating_selector, ""); + + if !title.is_empty() { + anime_list.push(SearchAnimeItem { + title, + slug, + poster, + episode, + anime_url, + genres, + status, + rating, + }); + } + } + + let current_page = slug.parse::().unwrap_or(1); + let last_visible_page = document + .select(&pagination_selector) + .next_back() + .and_then(|e| e.text().collect::().trim().parse::().ok()) + .unwrap_or(1); + + let has_next_page = document.select(&next_selector).next().is_some(); + let next_page = if has_next_page { + Some(current_page + 1) + } else { + None + }; + let has_previous_page = current_page > 1; + let previous_page = if has_previous_page { + Some(current_page - 1) + } else { + None + }; + + let pagination = Pagination { + current_page, + last_visible_page, + has_next_page, + next_page, + has_previous_page, + previous_page, + }; + + Ok((anime_list, pagination)) +} + +pub fn parse_genre_anime_document( + html: &str, + slug: &str, +) -> Result<(Vec, Pagination), AppError> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + let item_selector = selector(".venz ul li").unwrap(); + let title_selector = selector(".thumbz h2.jdlflm").unwrap(); + let link_selector = selector("a").unwrap(); + let img_selector = selector("img").unwrap(); + let episode_selector = selector(".epz").unwrap(); + let pagination_selector = selector(".pagenavix .page-numbers:not(.next)").unwrap(); + let next_selector = selector(".pagenavix .next.page-numbers").unwrap(); + + for element in document.select(&item_selector) { + let title = text_from_or(&element, &title_selector, ""); + let anime_url = attr_from_or(&element, &link_selector, "href", ""); + let slug = extract_slug(&anime_url); + let poster = attr_from_or(&element, &img_selector, "src", ""); + let episode = text_from_or(&element, &episode_selector, "N/A"); + + if !title.is_empty() { + anime_list.push(GenreAnimeItem { + title, + slug, + poster, + episode, + anime_url, + }); + } + } + + let current_page = slug.parse::().unwrap_or(1); + let last_visible_page = document + .select(&pagination_selector) + .next_back() + .and_then(|e| e.text().collect::().trim().parse::().ok()) + .unwrap_or(1); + + let has_next_page = document.select(&next_selector).next().is_some(); + let next_page = if has_next_page { + Some(current_page + 1) + } else { + None + }; + let has_previous_page = current_page > 1; + let previous_page = if has_previous_page { + Some(current_page - 1) + } else { + None + }; + + let pagination = Pagination { + current_page, + last_visible_page, + has_next_page, + next_page, + has_previous_page, + previous_page, + }; + + Ok((anime_list, pagination)) +} + +pub fn parse_anime_full_document(html: &str, slug: &str) -> Result { + let document = parse_html(html); + + let episode_title_selector = selector("h1.posttl").unwrap(); + let image_selector = selector(".cukder img").unwrap(); + let stream_selector = selector("#embed_holder iframe").unwrap(); + let download_item_selector = selector(".download ul li").unwrap(); + let resolution_selector = selector("strong").unwrap(); + let link_selector = selector("a").unwrap(); + let next_episode_selector = selector(".flir a[title*='Episode Selanjutnya']").unwrap(); + let previous_episode_selector = selector(".flir a[title*='Episode Sebelumnya']").unwrap(); + + let episode = document + .select(&episode_title_selector) + .next() + .map(|e| text(&e)) + .unwrap_or_default(); + + let episode_number = episode + .split("Episode") + .nth(1) + .map(|s| s.trim().to_string()) + .unwrap_or_default(); + + let image_url = document + .select(&image_selector) + .next() + .and_then(|e| attr(&e, "src")) + .unwrap_or_default(); + + let stream_url = document + .select(&stream_selector) + .next() + .and_then(|e| attr(&e, "src")) + .unwrap_or_default(); + + let mut download_urls = std::collections::HashMap::new(); + + for element in document.select(&download_item_selector) { + let resolution = element + .select(&resolution_selector) + .next() + .map(|e| text(&e)) + .unwrap_or_default(); + + let mut links = Vec::new(); + for link_element in element.select(&link_selector) { + let server = text(&link_element); + let url = attr(&link_element, "href").unwrap_or_default(); + links.push(DownloadLink { server, url }); + } + + if !resolution.is_empty() && !links.is_empty() { + download_urls.insert(resolution, links); + } + } + + let next_episode_element = document.select(&next_episode_selector).next(); + let previous_episode_element = document.select(&previous_episode_selector).next(); + + let next_episode_slug = next_episode_element + .and_then(|e| attr(&e, "href")) + .and_then(|href| { + href.split('/') + .nth(href.split('/').count().saturating_sub(2)) + .map(|s| s.to_string() + "/") + }); + + let previous_episode_slug = previous_episode_element + .and_then(|e| attr(&e, "href")) + .and_then(|href| { + href.split('/') + .nth(href.split('/').count().saturating_sub(2)) + .map(|s| s.to_string() + "/") + }); + + Ok(AnimeFullData { + episode, + episode_number, + anime: AnimeInfo { + slug: slug.to_string(), + }, + has_next_episode: next_episode_slug.is_some(), + next_episode: next_episode_slug.map(|s| EpisodeInfo { slug: s }), + has_previous_episode: previous_episode_slug.is_some(), + previous_episode: previous_episode_slug.map(|s| EpisodeInfo { slug: s }), + stream_url, + download_urls, + image_url, + }) +} diff --git a/src/modules/anime/repository.rs b/src/modules/anime/repository.rs new file mode 100644 index 0000000..423df2f --- /dev/null +++ b/src/modules/anime/repository.rs @@ -0,0 +1,199 @@ +use crate::modules::anime::parser; +use crate::modules::anime::types::*; +use crate::shared::database::traits::scraping_repository::ScrapingRepository; +use crate::shared::errors::AppError; +use crate::shared::utils::web::proxy_fetch::fetch_with_proxy; +use crate::shared::utils::web::scraping_urls::{get_otakudesu_url, OTAKUDESU_BASE_URL}; +use crate::shared::utils::{default_backoff, fetch_html_with_retry, transient}; +use async_trait::async_trait; +use backoff::future::retry; +use tracing::{info, warn}; + +pub struct AnimeRepository; + +impl Default for AnimeRepository { + fn default() -> Self { + Self::new() + } +} + +impl AnimeRepository { + pub fn new() -> Self { + Self + } +} + +#[async_trait] +impl ScrapingRepository for AnimeRepository { + async fn fetch_html(&self, url: &str) -> Result { + fetch_html_with_retry(url).await + } +} + +impl AnimeRepository { + pub fn base_url(&self) -> String { + get_otakudesu_url() + } + + pub fn index_urls(&self) -> (String, String) { + let base = self.base_url(); + ( + format!("{}/ongoing-anime/", base), + format!("{}/complete-anime/", base), + ) + } + + pub fn genres_url(&self) -> String { + format!("{}/genre-list/", self.base_url()) + } + + pub fn detail_url(&self, slug: &str) -> String { + format!("{}/anime/{}", OTAKUDESU_BASE_URL, slug) + } + + pub fn page_url(&self, category: &str, page: &str) -> String { + format!("{}/{}/page/{}/", OTAKUDESU_BASE_URL, category, page) + } + + pub fn search_url(&self, query: &str, page: &str) -> String { + if page == "1" { + format!("{}/search/{}/", self.base_url(), query) + } else { + format!("{}/search/{}/page/{}/", self.base_url(), query, page) + } + } + + pub fn genre_page_url(&self, genre_slug: &str, page: &str) -> String { + format!("{}/genre/{}/page/{}/", self.base_url(), genre_slug, page) + } + + pub fn full_episode_url(&self, slug: &str) -> String { + format!("{}/episode/{}", OTAKUDESU_BASE_URL, slug) + } + + pub async fn fetch_anime_index(&self) -> Result { + let (ongoing_url, complete_url) = self.index_urls(); + + let (ongoing_html, complete_html) = tokio::join!( + self.fetch_html(&ongoing_url), + self.fetch_html(&complete_url) + ); + + let ongoing_html = ongoing_html?; + let complete_html = complete_html?; + + let ongoing_anime = + tokio::task::spawn_blocking(move || parser::parse_ongoing_anime(&ongoing_html)) + .await??; + let complete_anime = + tokio::task::spawn_blocking(move || parser::parse_complete_anime(&complete_html)) + .await??; + + Ok(AnimeData { + ongoing_anime, + complete_anime, + }) + } + + pub async fn fetch_genres(&self) -> Result, AppError> { + let html = self.fetch_html(&self.genres_url()).await?; + tokio::task::spawn_blocking(move || parser::parse_genres(&html)).await? + } + + pub async fn fetch_anime_detail(&self, slug: &str) -> Result { + let url = self.detail_url(slug); + let html = self + .fetch_with_proxy_retry(&url) + .await + .map_err(|e| AppError::ScraperError(e.to_string()))?; + + tokio::task::spawn_blocking(move || parser::parse_anime_detail_document(&html)).await? + } + + pub async fn fetch_complete_anime_page( + &self, + slug: &str, + ) -> Result<(Vec, Pagination), AppError> { + let url = self.page_url("complete-anime", slug); + let html = self.fetch_html(&url).await?; + let slug_owned = slug.to_string(); + tokio::task::spawn_blocking(move || parser::parse_anime_page(&html, &slug_owned)).await? + } + + pub async fn fetch_ongoing_anime_page( + &self, + slug: &str, + ) -> Result<(Vec, Pagination), AppError> { + let url = self.page_url("ongoing-anime", slug); + let html = self.fetch_html(&url).await?; + let slug_owned = slug.to_string(); + tokio::task::spawn_blocking(move || { + parser::parse_ongoing_anime_document(&html, &slug_owned) + }) + .await? + } + + pub async fn fetch_latest_anime_page( + &self, + slug: &str, + ) -> Result<(Vec, Pagination), AppError> { + let url = self.page_url("latest-anime", slug); + let html = self.fetch_html(&url).await?; + let slug_owned = slug.to_string(); + tokio::task::spawn_blocking(move || parser::parse_latest_anime_document(&html, &slug_owned)) + .await? + } + + pub async fn fetch_search_anime_page( + &self, + slug: &str, + page: &str, + ) -> Result<(Vec, Pagination), AppError> { + let url = self.search_url(slug, page); + let html = self.fetch_html(&url).await?; + let page_owned = page.to_string(); + tokio::task::spawn_blocking(move || parser::parse_search_anime_document(&html, &page_owned)) + .await? + } + + pub async fn fetch_genre_anime_page( + &self, + genre_slug: &str, + page: &str, + ) -> Result<(Vec, Pagination), AppError> { + let url = self.genre_page_url(genre_slug, page); + let html = self.fetch_html(&url).await?; + let page_owned = page.to_string(); + tokio::task::spawn_blocking(move || parser::parse_genre_anime_document(&html, &page_owned)) + .await? + } + + pub async fn fetch_anime_full(&self, slug: &str) -> Result { + let url = self.full_episode_url(slug); + let html = self.fetch_html(&url).await?; + let slug_owned = slug.to_string(); + tokio::task::spawn_blocking(move || parser::parse_anime_full_document(&html, &slug_owned)) + .await? + } + + async fn fetch_with_proxy_retry(&self, url: &str) -> Result { + let backoff = default_backoff(); + let url_owned = url.to_string(); + let fetch_op = || async { + info!("Fetching URL: {}", url_owned); + match fetch_with_proxy(&url_owned).await { + Ok(response) => { + info!("Successfully fetched URL: {}", url_owned); + Ok(response.data) + } + Err(e) => { + warn!("Failed to fetch URL: {}, error: {:?}", url_owned, e); + Err(transient(e)) + } + } + }; + retry(backoff, fetch_op) + .await + .map_err(|e| AppError::ScraperError(e.to_string())) + } +} diff --git a/src/modules/anime/route.rs b/src/modules/anime/route.rs new file mode 100644 index 0000000..9bee8a6 --- /dev/null +++ b/src/modules/anime/route.rs @@ -0,0 +1,49 @@ +use crate::modules::anime::controller; +use crate::shared::state::AppState; +use axum::Router; +use std::sync::Arc; + +pub fn routes(router: Router>) -> Router> { + router + .route("/api/anime", axum::routing::get(controller::anime_index)) + .route( + "/api/anime/genre_list", + axum::routing::get(controller::genres), + ) + .route( + "/api/anime/detail/{slug}", + axum::routing::get(controller::detail_slug), + ) + .route( + "/api/anime/complete_anime/{slug}", + axum::routing::get(controller::complete_anime_slug), + ) + .route( + "/api/anime/full/{slug}", + axum::routing::get(controller::full_slug), + ) + .route( + "/api/anime/ongoing_anime/{slug}", + axum::routing::get(controller::ongoing_anime_slug), + ) + .route( + "/api/anime/latest/{slug}", + axum::routing::get(controller::latest_slug), + ) + .route( + "/api/anime/search/{slug}", + axum::routing::get(controller::search_slug_index), + ) + .route( + "/api/anime/search/{slug}/{page}", + axum::routing::get(controller::search_slug_page), + ) + .route( + "/api/anime/genre/{slug}", + axum::routing::get(controller::genre_slug_index), + ) + .route( + "/api/anime/genre/{slug}/{page}", + axum::routing::get(controller::genre_slug_page), + ) +} diff --git a/src/modules/anime/schema.rs b/src/modules/anime/schema.rs new file mode 100644 index 0000000..2321901 --- /dev/null +++ b/src/modules/anime/schema.rs @@ -0,0 +1,18 @@ +use serde::Deserialize; +use utoipa::ToSchema; + +#[derive(Deserialize, ToSchema)] +pub struct SearchQuery { + pub q: Option, +} + +#[derive(Deserialize, ToSchema)] +pub struct SlugPath { + pub slug: String, +} + +#[derive(Deserialize, ToSchema)] +pub struct SlugPagePath { + pub slug: String, + pub page: String, +} diff --git a/src/modules/anime/scraping/cache.rs b/src/modules/anime/scraping/cache.rs new file mode 100644 index 0000000..7fe182e --- /dev/null +++ b/src/modules/anime/scraping/cache.rs @@ -0,0 +1,81 @@ +use crate::shared::types::entities::anime::HasPoster; +use crate::shared::state::AppState; +use std::sync::Arc; + +/// Cache poster URLs for a collection of anime items +/// This is a fire-and-forget operation that triggers lazy background caching +pub async fn cache_posters(app_state: &Arc, items: &[T]) { + let posters: Vec = items.iter().map(|item| item.poster().to_string()).collect(); + + if posters.is_empty() { + return; + } + + let db = app_state.db.clone(); + let redis = app_state.redis_pool.clone(); + + crate::shared::services::images::cache::cache_image_urls_batch_lazy( + db, + &redis, + posters, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; +} + +/// Cache poster URLs and update items with cached URLs +/// Returns the updated items with CDN URLs +pub async fn cache_and_update_posters( + app_state: &Arc, + mut items: Vec, +) -> Vec { + let posters: Vec = items.iter().map(|item| item.poster().to_string()).collect(); + + if posters.is_empty() { + return items; + } + + let db = app_state.db.clone(); + let redis = app_state.redis_pool.clone(); + + let cached_posters = crate::shared::services::images::cache::cache_image_urls_batch_lazy( + db, + &redis, + posters, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + // Update items with cached URLs + for (i, item) in items.iter_mut().enumerate() { + if let Some(url) = cached_posters.get(i) { + item.set_poster(url.clone()); + } + } + + items +} + +/// Cache multiple collections of posters and update them +/// Useful when you have different types of items (e.g., ongoing and complete anime) +pub async fn cache_multiple_collections( + app_state: &Arc, + collections: Vec>, +) -> Vec { + let all_posters: Vec = collections.into_iter().flatten().collect(); + + if all_posters.is_empty() { + return Vec::new(); + } + + let db = app_state.db.clone(); + let redis = app_state.redis_pool.clone(); + + crate::shared::services::images::cache::cache_image_urls_batch_lazy( + db, + &redis, + all_posters, + Some(app_state.image_processing_semaphore.clone()), + ) + .await +} diff --git a/src/modules/anime/service.rs b/src/modules/anime/service.rs new file mode 100644 index 0000000..4eb3007 --- /dev/null +++ b/src/modules/anime/service.rs @@ -0,0 +1,295 @@ +use std::sync::Arc; + +use crate::modules::anime::repository::AnimeRepository; +use crate::modules::anime::types::*; +use crate::shared::errors::AppError; +use crate::shared::state::AppState; +use crate::shared::utils::Cache; + +const INDEX_CACHE_TTL: u64 = 10; +const GENRE_LIST_CACHE_TTL: u64 = 3600; +const DEFAULT_CACHE_TTL: u64 = 300; + +pub struct AnimeService { + repository: AnimeRepository, +} + +impl AnimeService { + pub fn new(repository: AnimeRepository) -> Self { + Self { repository } + } + + pub async fn get_anime_index(&self, app_state: Arc) -> Result { + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set("anime:index:v2", INDEX_CACHE_TTL, || async { + let mut data = self + .repository + .fetch_anime_index() + .await + .map_err(|e| e.to_string())?; + + if data.ongoing_anime.is_empty() && data.complete_anime.is_empty() { + return Err("Empty anime index — refusing to cache".to_string()); + } + + let mut posters: Vec = data + .ongoing_anime + .iter() + .map(|item| item.poster.clone()) + .collect(); + posters.extend(data.complete_anime.iter().map(|item| item.poster.clone())); + + let cached_posters = + crate::shared::services::images::cache::cache_image_urls_batch_lazy( + app_state.db.clone(), + &app_state.redis_pool, + posters, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + let ongoing_len = data.ongoing_anime.len(); + for (i, item) in data.ongoing_anime.iter_mut().enumerate() { + if let Some(url) = cached_posters.get(i) { + item.poster = url.clone(); + } + } + for (i, item) in data.complete_anime.iter_mut().enumerate() { + if let Some(url) = cached_posters.get(ongoing_len + i) { + item.poster = url.clone(); + } + } + + Ok(data) + }) + .await + .map_err(|e| AppError::ScraperError(e)) + } + + pub async fn get_genres(&self, app_state: Arc) -> Result { + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set("anime:genres:list", GENRE_LIST_CACHE_TTL, || async { + let genres = self + .repository + .fetch_genres() + .await + .map_err(|e| e.to_string())?; + Ok(GenresResponse { + status: "Ok".to_string(), + data: genres, + }) + }) + .await + .map_err(|e| AppError::ScraperError(e)) + } + + pub async fn get_anime_detail( + &self, + app_state: Arc, + slug: String, + ) -> Result { + let cache_key = format!("anime:detail:{}", slug); + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async { + let mut data = self + .repository + .fetch_anime_detail(&slug) + .await + .map_err(|e| e.to_string())?; + + data.poster = crate::shared::services::images::cache::get_cached_or_original( + app_state.db.clone(), + &app_state.redis_pool, + &data.poster, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + let rec_posters: Vec = data + .recommendations + .iter() + .map(|r| r.poster.clone()) + .collect(); + let cached_rec_posters = + crate::shared::services::images::cache::cache_image_urls_batch_lazy( + app_state.db.clone(), + &app_state.redis_pool, + rec_posters, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + for (i, rec) in data.recommendations.iter_mut().enumerate() { + if let Some(url) = cached_rec_posters.get(i) { + rec.poster = url.clone(); + } + } + + Ok(DetailResponse { + status: Some("Ok".to_string()), + data, + }) + }) + .await + .map_err(|e| AppError::ScraperError(e)) + } + + pub async fn get_complete_anime_page( + &self, + app_state: Arc, + slug: String, + ) -> Result { + let cache_key = format!("anime:complete:{}", slug); + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async { + let (anime_list, pagination) = self + .repository + .fetch_complete_anime_page(&slug) + .await + .map_err(|e| e.to_string())?; + let total = anime_list.len() as i64; + Ok(ListResponse { + message: "Success".to_string(), + data: anime_list, + total: Some(total), + pagination: Some(pagination), + }) + }) + .await + .map_err(|e| AppError::ScraperError(e)) + } + + pub async fn get_ongoing_anime_page( + &self, + app_state: Arc, + slug: String, + ) -> Result { + let cache_key = format!("anime:ongoing:{}", slug); + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async { + let (anime_list, pagination) = self + .repository + .fetch_ongoing_anime_page(&slug) + .await + .map_err(|e| e.to_string())?; + Ok(OngoingAnimeResponse { + status: "Ok".to_string(), + data: anime_list, + pagination, + }) + }) + .await + .map_err(|e| AppError::ScraperError(e)) + } + + pub async fn get_latest_anime_page( + &self, + app_state: Arc, + slug: String, + ) -> Result { + let cache_key = format!("anime:latest:{}", slug); + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async { + let (anime_list, pagination) = self + .repository + .fetch_latest_anime_page(&slug) + .await + .map_err(|e| e.to_string())?; + Ok(LatestAnimeResponse { + status: "Ok".to_string(), + data: anime_list, + pagination, + }) + }) + .await + .map_err(|e| AppError::ScraperError(e)) + } + + pub async fn get_search_anime_page( + &self, + app_state: Arc, + slug: String, + page: String, + ) -> Result { + let cache_key = format!("anime:search:{}:{}", slug, page); + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async { + let (anime_list, pagination) = self + .repository + .fetch_search_anime_page(&slug, &page) + .await + .map_err(|e| e.to_string())?; + Ok(SearchResponse { + status: "Ok".to_string(), + data: anime_list, + pagination, + }) + }) + .await + .map_err(|e| AppError::ScraperError(e)) + } + + pub async fn get_genre_anime_page( + &self, + app_state: Arc, + genre_slug: String, + page: String, + ) -> Result { + let cache_key = format!("anime:genre:{}:{}", genre_slug, page); + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async { + let (anime_list, pagination) = self + .repository + .fetch_genre_anime_page(&genre_slug, &page) + .await + .map_err(|e| e.to_string())?; + Ok(GenreListResponse { + status: "Ok".to_string(), + data: anime_list, + pagination, + }) + }) + .await + .map_err(|e| AppError::ScraperError(e)) + } + + pub async fn get_anime_full( + &self, + app_state: Arc, + slug: String, + ) -> Result { + let cache_key = format!("anime:full:{}", slug); + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async { + let data = self + .repository + .fetch_anime_full(&slug) + .await + .map_err(|e| e.to_string())?; + Ok(FullResponse { + status: "Ok".to_string(), + data, + }) + }) + .await + .map_err(|e| AppError::ScraperError(e)) + } +} diff --git a/src/modules/anime/types.rs b/src/modules/anime/types.rs new file mode 100644 index 0000000..aecb906 --- /dev/null +++ b/src/modules/anime/types.rs @@ -0,0 +1,231 @@ +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +// Index endpoint types +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct OngoingAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub current_episode: String, + pub anime_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct CompleteAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub episode_count: String, + pub anime_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct AnimeData { + pub ongoing_anime: Vec, + pub complete_anime: Vec, +} + +// Genre list types +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Genre { + pub name: String, + pub slug: String, + pub url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct GenresResponse { + pub status: String, + pub data: Vec, +} + +// Detail endpoint types +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct DetailGenre { + pub name: String, + pub slug: String, + pub anime_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct EpisodeList { + pub episode: String, + pub slug: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Recommendation { + pub title: String, + pub slug: String, + pub poster: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub r#type: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct AnimeDetailData { + pub title: String, + pub alternative_title: String, + pub poster: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub r#type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, + pub release_date: String, + pub studio: String, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub genres: Vec, + pub synopsis: String, + pub episode_lists: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub batch: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub producers: Vec, + pub recommendations: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct DetailResponse { + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, + pub data: AnimeDetailData, +} + +// Complete anime list types +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct CompleteAnimeListItem { + pub title: String, + pub slug: String, + pub poster: String, + pub episode_count: String, + pub anime_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Pagination { + pub current_page: u32, + pub last_visible_page: u32, + pub has_next_page: bool, + pub next_page: Option, + pub has_previous_page: bool, + pub previous_page: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct ListResponse { + pub message: String, + pub data: Vec, + pub total: Option, + pub pagination: Option, +} + +// Full episode types +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct AnimeInfo { + pub slug: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct EpisodeInfo { + pub slug: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct DownloadLink { + pub server: String, + pub url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct AnimeFullData { + pub episode: String, + pub episode_number: String, + pub anime: AnimeInfo, + pub has_next_episode: bool, + pub next_episode: Option, + pub has_previous_episode: bool, + pub previous_episode: Option, + pub stream_url: String, + pub download_urls: std::collections::HashMap>, + pub image_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct FullResponse { + pub status: String, + pub data: AnimeFullData, +} + +// Ongoing anime list types +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct OngoingAnimeListItem { + pub title: String, + pub slug: String, + pub poster: String, + pub score: String, + pub anime_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct OngoingAnimeResponse { + pub status: String, + pub data: Vec, + pub pagination: Pagination, +} + +// Latest anime types +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct LatestAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub episode: String, + pub anime_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct LatestAnimeResponse { + pub status: String, + pub data: Vec, + pub pagination: Pagination, +} + +// Search types +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct SearchAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub episode: String, + pub anime_url: String, + pub genres: Vec, + pub status: String, + pub rating: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct SearchResponse { + pub status: String, + pub data: Vec, + pub pagination: Pagination, +} + +// Genre list by slug types +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct GenreAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub episode: String, + pub anime_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct GenreListResponse { + pub status: String, + pub data: Vec, + pub pagination: Pagination, +} diff --git a/src/modules/anime2/controller.rs b/src/modules/anime2/controller.rs new file mode 100644 index 0000000..1245c04 --- /dev/null +++ b/src/modules/anime2/controller.rs @@ -0,0 +1,276 @@ +use std::sync::Arc; + +use axum::{ + extract::{Path, Query, State}, + Json, +}; + +use crate::modules::anime2::repository::Anime2Repository; +use crate::modules::anime2::schema::FilterQuery; +use crate::modules::anime2::service::Anime2Service; +use crate::shared::errors::AppError; +use crate::shared::state::AppState; + +#[utoipa::path( + get, + path = "/api/anime2", + tag = "anime2", + operation_id = "anime2_index", + responses( + (status = 200, description = "Handles GET requests for the /api/anime2 endpoint.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn index( + State(app_state): State>, +) -> Result, AppError> { + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.index(app_state).await?)) +} + +#[utoipa::path( + get, + path = "/api/anime2/genre_list", + tag = "anime2", + operation_id = "anime2_genre_list", + responses( + (status = 200, description = "Handles GET requests for the /api/anime2/genre_list endpoint.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn genre_list( + State(app_state): State>, +) -> Result, AppError> { + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.genre_list(app_state).await?)) +} + +#[utoipa::path( + get, + path = "/api/anime2/filter", + tag = "anime2", + operation_id = "anime2_filter", + responses( + (status = 200, description = "Handles GET requests for the /api/anime2/filter endpoint.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn filter( + State(app_state): State>, + Query(params): Query, +) -> Result, AppError> { + let page = params.page.unwrap_or(1); + let genre = params.genre.clone(); + let status = params.status.clone(); + let anime_type = params.r#type.clone(); + let order = params.order.clone().unwrap_or("update".to_string()); + + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json( + service + .filter(app_state, page, genre, status, anime_type, order) + .await?, + )) +} + +#[utoipa::path( + get, + path = "/api/anime2/detail/{slug}", + tag = "anime2", + operation_id = "anime2_detail_slug", + responses( + (status = 200, description = "Retrieves details for a specific detail by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn detail_slug( + State(app_state): State>, + Path(slug): Path, +) -> Result, AppError> { + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.detail(app_state, slug).await?)) +} + +#[utoipa::path( + get, + path = "/api/anime2/genre/{slug}", + tag = "anime2", + operation_id = "anime2_genre_slug_index", + responses( + (status = 200, description = "Retrieves details for a specific genre by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn genre_slug_index( + State(app_state): State>, + Path(slug): Path, +) -> Result< + Json< + crate::shared::types::ApiResponse< + Vec, + >, + >, + AppError, +> { + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.genre_slug(app_state, slug, 1).await?)) +} + +#[utoipa::path( + get, + path = "/api/anime2/genre/{slug}/{page}", + tag = "anime2", + operation_id = "anime2_genre_slug_page", + responses( + (status = 200, description = "Handles GET requests for the /api/anime2/genre/{slug}/{page} endpoint.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn genre_slug_page( + State(app_state): State>, + Path((slug, page)): Path<(String, u32)>, +) -> Result< + Json< + crate::shared::types::ApiResponse< + Vec, + >, + >, + AppError, +> { + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.genre_slug(app_state, slug, page).await?)) +} + +#[utoipa::path( + get, + path = "/api/anime2/search/{slug}", + tag = "anime2", + operation_id = "anime2_search_slug_index", + responses( + (status = 200, description = "Retrieves details for a specific search by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn search_slug_index( + State(app_state): State>, + Path(slug): Path, +) -> Result< + Json< + crate::shared::types::ApiResponse< + Vec, + >, + >, + AppError, +> { + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.search(app_state, slug, 1).await?)) +} + +#[utoipa::path( + get, + path = "/api/anime2/search/{slug}/{page}", + tag = "anime2", + operation_id = "anime2_search_slug_page", + responses( + (status = 200, description = "Handles GET requests for the /api/anime2/search/{slug}/{page} endpoint.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn search_slug_page( + State(app_state): State>, + Path((slug, page)): Path<(String, u32)>, +) -> Result< + Json< + crate::shared::types::ApiResponse< + Vec, + >, + >, + AppError, +> { + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.search(app_state, slug, page).await?)) +} + +#[utoipa::path( + get, + path = "/api/anime2/latest/{slug}", + tag = "anime2", + operation_id = "anime2_latest_slug", + responses( + (status = 200, description = "Retrieves details for a specific latest by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn latest_slug( + State(app_state): State>, + Path(slug): Path, +) -> Result< + Json< + crate::shared::types::ApiResponse< + Vec, + >, + >, + AppError, +> { + let page = slug + .parse::() + .map_err(|_| AppError::ScraperError(format!("Invalid page number: {}", slug)))?; + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.latest(app_state, page).await?)) +} + +#[utoipa::path( + get, + path = "/api/anime2/ongoing_anime/{slug}", + tag = "anime2", + operation_id = "anime2_ongoing_anime_slug", + responses( + (status = 200, description = "Retrieves details for a specific ongoing_anime by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn ongoing_anime_slug( + State(app_state): State>, + Path(slug): Path, +) -> Result< + Json< + crate::shared::types::ApiResponse< + Vec, + >, + >, + AppError, +> { + let page = slug + .parse::() + .map_err(|_| AppError::ScraperError(format!("Invalid page number: {}", slug)))?; + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.ongoing_anime(app_state, page).await?)) +} + +#[utoipa::path( + get, + path = "/api/anime2/complete_anime/{slug}", + tag = "anime2", + operation_id = "anime2_complete_anime_slug", + responses( + (status = 200, description = "Retrieves details for a specific complete_anime by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn complete_anime_slug( + State(app_state): State>, + Path(slug): Path, +) -> Result< + Json< + crate::shared::types::ApiResponse< + Vec, + >, + >, + AppError, +> { + let page = slug + .parse::() + .map_err(|_| AppError::ScraperError(format!("Invalid page number: {}", slug)))?; + let service = Anime2Service::new(Anime2Repository::new()); + Ok(Json(service.complete_anime(app_state, page).await?)) +} diff --git a/src/modules/anime2/mod.rs b/src/modules/anime2/mod.rs new file mode 100644 index 0000000..67dc224 --- /dev/null +++ b/src/modules/anime2/mod.rs @@ -0,0 +1,7 @@ +pub mod controller; +pub mod parser; +pub mod repository; +pub mod route; +pub mod schema; +pub mod service; +pub mod types; diff --git a/src/modules/anime2/parser.rs b/src/modules/anime2/parser.rs new file mode 100644 index 0000000..ab06c44 --- /dev/null +++ b/src/modules/anime2/parser.rs @@ -0,0 +1,804 @@ +use crate::shared::errors::AppError; +use crate::shared::utils::parse_html; +use crate::shared::utils::scraping::{attr, extract_slug, selector, text, text_from_or}; +use once_cell::sync::Lazy; +use regex::Regex; +use scraper::Selector; + +static ITEM_SELECTOR: Lazy = Lazy::new(|| Selector::parse("article.bs").unwrap()); +static TITLE_SELECTOR: Lazy = Lazy::new(|| Selector::parse(".tt h2").unwrap()); +static IMG_SELECTOR: Lazy = Lazy::new(|| Selector::parse("img").unwrap()); +static SCORE_SELECTOR: Lazy = Lazy::new(|| Selector::parse(".numscore").unwrap()); +static STATUS_SELECTOR: Lazy = Lazy::new(|| Selector::parse(".status").unwrap()); +static TYPE_SELECTOR: Lazy = Lazy::new(|| Selector::parse(".type").unwrap()); +static LINK_SELECTOR: Lazy = Lazy::new(|| Selector::parse("a").unwrap()); +static PAGINATION_SELECTOR: Lazy = + Lazy::new(|| Selector::parse(".pagination .page-numbers:not(.next)").unwrap()); +static NEXT_SELECTOR: Lazy = Lazy::new(|| Selector::parse(".pagination .next").unwrap()); +static SLUG_REGEX: Lazy = Lazy::new(|| Regex::new(r"/([^/]+)/?$").unwrap()); +static GENRE_SLUG_REGEX: Lazy = Lazy::new(|| Regex::new(r"genre-(.+)$").unwrap()); +pub fn parse_ongoing_anime( + html: &str, +) -> Result, AppError> { + let items = parse_ongoing_anime_with_score(html)?; + Ok(items + .into_iter() + .map( + |item| crate::shared::types::entities::anime::OngoingAnimeItem { + title: item.title, + slug: item.slug, + poster: item.poster, + current_episode: item.score, + anime_url: item.anime_url, + }, + ) + .collect()) +} + +pub fn parse_complete_anime( + html: &str, +) -> Result, AppError> { + let document = parse_html(html); + let mut complete_anime = Vec::new(); + + for element in document.select(&ITEM_SELECTOR) { + let title = element + .select(&TITLE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or_default(); + let poster = element + .select(&IMG_SELECTOR) + .next() + .and_then(|e| e.value().attr("src").or(e.value().attr("data-src"))) + .unwrap_or("") + .to_string(); + let anime_url = element + .select(&LINK_SELECTOR) + .next() + .and_then(|e| e.value().attr("href")) + .unwrap_or("") + .to_string(); + let slug = SLUG_REGEX + .captures(&anime_url) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str()) + .unwrap_or("") + .to_string(); + let episode_count = text_from_or(&element, &STATUS_SELECTOR, "N/A"); + + if !title.is_empty() { + complete_anime.push(crate::shared::types::entities::anime::CompleteAnimeItem { + title, + slug, + poster, + episode_count, + anime_url, + }); + } + } + + Ok(complete_anime) +} + +pub fn parse_genres(html: &str) -> Result, AppError> { + let document = parse_html(html); + let mut genres = Vec::new(); + let genre_label_selector = selector("label[for^=\"genre-\"]").ok_or_else(|| { + AppError::ScraperError("Invalid selector: label[for^=\"genre-\"]".to_string()) + })?; + + for element in document.select(&genre_label_selector) { + let name = text(&element).trim().to_string(); + let for_attr = attr(&element, "for").unwrap_or_default(); + + let slug = GENRE_SLUG_REGEX + .captures(&for_attr) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str()) + .unwrap_or("") + .to_string(); + + if !name.is_empty() && !slug.is_empty() { + genres.push(crate::modules::anime2::types::Genre { name, slug }); + } + } + + Ok(genres) +} + +pub fn parse_filter_page( + html: &str, + current_page: u32, +) -> Result< + ( + Vec, + crate::shared::types::entities::anime::Pagination, + ), + AppError, +> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + for element in document.select(&ITEM_SELECTOR) { + let title = element + .select(&TITLE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or_default(); + + let poster = element + .select(&IMG_SELECTOR) + .next() + .and_then(|e| e.value().attr("src").or(e.value().attr("data-src"))) + .unwrap_or("") + .to_string(); + + let score = element + .select(&SCORE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or("N/A".to_string()); + + let status = element + .select(&STATUS_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or("Unknown".to_string()); + + let anime_type = element + .select(&TYPE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or("Unknown".to_string()); + + let anime_url = element + .select(&LINK_SELECTOR) + .next() + .and_then(|e| e.value().attr("href")) + .unwrap_or("") + .to_string(); + + let slug = SLUG_REGEX + .captures(&anime_url) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str()) + .unwrap_or("") + .to_string(); + + if !title.is_empty() { + anime_list.push(crate::shared::types::entities::anime::FilterAnimeItem { + title, + slug, + poster, + score, + status, + r#type: anime_type, + anime_url, + }); + } + } + + let last_visible_page = document + .select(&PAGINATION_SELECTOR) + .next_back() + .map(|e| { + e.text() + .collect::() + .trim() + .parse::() + .unwrap_or(1) + }) + .unwrap_or(1); + + let has_next_page = document.select(&NEXT_SELECTOR).next().is_some(); + let pagination = crate::shared::types::entities::anime::Pagination { + current_page, + last_visible_page, + has_next_page, + next_page: if has_next_page { + Some(current_page + 1) + } else { + None + }, + has_previous_page: current_page > 1, + previous_page: if current_page > 1 { + Some(current_page - 1) + } else { + None + }, + }; + + Ok((anime_list, pagination)) +} + +pub fn parse_genre_anime( + html: &str, +) -> Result, AppError> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + for element in document.select(&ITEM_SELECTOR) { + let title = element + .select(&TITLE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or_default(); + + let poster = element + .select(&IMG_SELECTOR) + .next() + .and_then(|e| e.value().attr("src").or(e.value().attr("data-src"))) + .unwrap_or("") + .to_string(); + + let score = element + .select(&SCORE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or("N/A".to_string()); + + let status = element + .select(&STATUS_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or("Unknown".to_string()); + + let anime_url = element + .select(&LINK_SELECTOR) + .next() + .and_then(|e| e.value().attr("href")) + .unwrap_or("") + .to_string(); + + let slug = SLUG_REGEX + .captures(&anime_url) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str()) + .unwrap_or("") + .to_string(); + + if !title.is_empty() { + anime_list.push(crate::shared::types::entities::anime::GenreAnimeItem { + title, + slug, + poster, + score, + status, + anime_url, + }); + } + } + + Ok(anime_list) +} + +pub fn parse_search_anime( + html: &str, +) -> Result, AppError> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + for element in document.select(&ITEM_SELECTOR) { + let title = element + .select(&TITLE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or_default(); + + let poster = element + .select(&IMG_SELECTOR) + .next() + .and_then(|e| e.value().attr("src").or(e.value().attr("data-src"))) + .unwrap_or("") + .to_string(); + + let anime_url = element + .select(&LINK_SELECTOR) + .next() + .and_then(|e| e.value().attr("href")) + .unwrap_or("") + .to_string(); + + let slug = SLUG_REGEX + .captures(&anime_url) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str()) + .unwrap_or("") + .to_string(); + + if !title.is_empty() { + anime_list.push(crate::shared::types::entities::anime::SearchAnimeItem { + title, + slug, + poster, + description: String::new(), + anime_url, + genres: Vec::new(), + rating: "N/A".to_string(), + r#type: "Unknown".to_string(), + season: "Unknown".to_string(), + }); + } + } + + Ok(anime_list) +} + +pub fn parse_latest_anime( + html: &str, +) -> Result, AppError> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + for element in document.select(&ITEM_SELECTOR) { + let title = element + .select(&TITLE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or_default(); + + let poster = element + .select(&IMG_SELECTOR) + .next() + .and_then(|e| e.value().attr("src").or(e.value().attr("data-src"))) + .unwrap_or("") + .to_string(); + + let score = element + .select(&SCORE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or("N/A".to_string()); + + let anime_url = element + .select(&LINK_SELECTOR) + .next() + .and_then(|e| e.value().attr("href")) + .unwrap_or("") + .to_string(); + + let slug = SLUG_REGEX + .captures(&anime_url) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str()) + .unwrap_or("") + .to_string(); + + if !title.is_empty() { + anime_list.push(crate::shared::types::entities::anime::LatestAnimeItem { + title, + slug, + poster, + current_episode: "N/A".to_string(), + score, + anime_url, + }); + } + } + + Ok(anime_list) +} + +pub fn parse_ongoing_anime_with_score( + html: &str, +) -> Result, AppError> { + let document = parse_html(html); + let mut anime_list = Vec::new(); + + for element in document.select(&ITEM_SELECTOR) { + let title = element + .select(&TITLE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or_default(); + + let poster = element + .select(&IMG_SELECTOR) + .next() + .and_then(|e| e.value().attr("src").or(e.value().attr("data-src"))) + .unwrap_or("") + .to_string(); + + let score = element + .select(&SCORE_SELECTOR) + .next() + .map(|e| e.text().collect::().trim().to_string()) + .unwrap_or("N/A".to_string()); + + let anime_url = element + .select(&LINK_SELECTOR) + .next() + .and_then(|e| e.value().attr("href")) + .unwrap_or("") + .to_string(); + + let slug = SLUG_REGEX + .captures(&anime_url) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str()) + .unwrap_or("") + .to_string(); + + if !title.is_empty() { + anime_list.push( + crate::shared::types::entities::anime::OngoingAnimeItemWithScore { + title, + slug, + poster, + score, + anime_url, + }, + ); + } + } + + Ok(anime_list) +} + +pub fn parse_pagination( + document: &scraper::Html, + current_page: u32, +) -> Result { + let last_visible_page = document + .select(&PAGINATION_SELECTOR) + .next_back() + .map(|e| { + e.text() + .collect::() + .trim() + .parse::() + .unwrap_or(1) + }) + .unwrap_or(1); + + let has_next_page = document.select(&NEXT_SELECTOR).next().is_some(); + let pagination = crate::shared::types::entities::anime::Pagination { + current_page, + last_visible_page, + has_next_page, + next_page: if has_next_page { + Some(current_page + 1) + } else { + None + }, + has_previous_page: current_page > 1, + previous_page: if current_page > 1 { + Some(current_page - 1) + } else { + None + }, + }; + + Ok(pagination) +} + +pub fn parse_pagination_with_string( + document: &scraper::Html, + current_page: u32, +) -> Result { + let last_visible_page = document + .select(&PAGINATION_SELECTOR) + .next_back() + .map(|e| { + e.text() + .collect::() + .trim() + .parse::() + .unwrap_or(1) + }) + .unwrap_or(1); + + let has_next_page = document.select(&NEXT_SELECTOR).next().is_some(); + let pagination = crate::shared::types::entities::anime::PaginationWithStringPages { + current_page, + last_visible_page, + has_next_page, + next_page: if has_next_page { + Some((current_page + 1).to_string()) + } else { + None + }, + has_previous_page: current_page > 1, + previous_page: if current_page > 1 { + Some((current_page - 1).to_string()) + } else { + None + }, + }; + + Ok(pagination) +} + +pub fn parse_anime_detail( + html: &str, +) -> Result { + let document = parse_html(html); + + let title_selector = selector(".entry-title") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .entry-title".to_string()))?; + let alt_title_selector = selector(".alter") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .alter".to_string()))?; + let poster_selector = selector(".thumb img, .thumbook img, .wp-post-image, .ts-post-image") + .ok_or_else(|| { + AppError::ScraperError( + "Invalid selector: .thumb img, .thumbook img, .wp-post-image, .ts-post-image" + .to_string(), + ) + })?; + let poster2_selector = selector(".bigcover img, .bixbox.animefull .bigcover .ime img") + .ok_or_else(|| { + AppError::ScraperError( + "Invalid selector: .bigcover img, .bixbox.animefull .bigcover .ime img".to_string(), + ) + })?; + let spe_span_selector = selector(".info-content .spe span").ok_or_else(|| { + AppError::ScraperError("Invalid selector: .info-content .spe span".to_string()) + })?; + let a_selector = + selector("a").ok_or_else(|| AppError::ScraperError("Invalid selector: a".to_string()))?; + let synopsis_selector = selector(".entry-content p") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .entry-content p".to_string()))?; + let genre_selector = selector(".genxed a") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .genxed a".to_string()))?; + let download_container_selector = selector(".soraddl.dlone") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .soraddl.dlone".to_string()))?; + let resolution_selector = selector(".res") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .res".to_string()))?; + let link_selector = selector(".slink a") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .slink a".to_string()))?; + let h3_selector = + selector("h3").ok_or_else(|| AppError::ScraperError("Invalid selector: h3".to_string()))?; + let recommendation_selector = selector(".listupd .bs") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .listupd .bs".to_string()))?; + let rec_title_selector = selector(".ntitle") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .ntitle".to_string()))?; + let rec_img_selector = selector("img") + .ok_or_else(|| AppError::ScraperError("Invalid selector: img".to_string()))?; + let status_selector = selector(".status") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .status".to_string()))?; + let type_selector = selector(".typez") + .ok_or_else(|| AppError::ScraperError("Invalid selector: .typez".to_string()))?; + + let title = text_from_or(&document.root_element(), &title_selector, ""); + let alternative_title = text_from_or(&document.root_element(), &alt_title_selector, ""); + + let poster = document + .select(&poster_selector) + .next() + .and_then(|e| { + attr(&e, "src") + .or_else(|| attr(&e, "data-src")) + .or_else(|| attr(&e, "data-lazy-src")) + }) + .unwrap_or_default(); + + let poster2 = document + .select(&poster2_selector) + .next() + .and_then(|e| { + attr(&e, "src") + .or_else(|| attr(&e, "data-src")) + .or_else(|| attr(&e, "data-lazy-src")) + }) + .unwrap_or_default(); + + let r#type = document + .select(&spe_span_selector) + .find(|e| text(&e).contains("Tipe:")) + .and_then(|span| span.select(&a_selector).next()) + .map(|e| text(&e)) + .unwrap_or_default(); + + let release_date = document + .select(&spe_span_selector) + .find(|e| text(&e).contains("Dirilis:")) + .map(|e| text(&e)) + .unwrap_or_default(); + + let status = document + .select(&spe_span_selector) + .find(|e| text(&e).contains("Status:")) + .map(|e| text(&e)) + .unwrap_or_default(); + + let synopsis = text_from_or(&document.root_element(), &synopsis_selector, ""); + + let studio = document + .select(&spe_span_selector) + .find(|e| text(&e).contains("Studio:")) + .and_then(|span| span.select(&a_selector).next()) + .map(|e| text(&e)) + .unwrap_or_default(); + + let mut genres = Vec::new(); + for element in document.select(&genre_selector) { + let name = text(&element); + let anime_url = attr(&element, "href").unwrap_or_default(); + let genre_slug = extract_slug(&anime_url); + genres.push(crate::modules::anime2::types::DetailGenre { + name, + slug: genre_slug, + anime_url, + }); + } + + let mut batch = Vec::new(); + let mut ova = Vec::new(); + let mut downloads = Vec::new(); + + for element in document.select(&download_container_selector) { + let title = element + .select(&h3_selector) + .next() + .map(|e| text(&e)) + .unwrap_or_else(|| "Unknown".to_string()); + + let category = title.to_lowercase(); + let is_batch = category.contains("batch"); + let is_ova = category.contains("ova"); + + let mut all_links = Vec::new(); + + let row_selector = selector("table tr") + .ok_or_else(|| AppError::ScraperError("Invalid selector: table tr".to_string()))?; + for row in element.select(&row_selector) { + let resolution = text_from_or(&row, &resolution_selector, ""); + + for link_element in row.select(&link_selector) { + let provider = text(&link_element); + let url = attr(&link_element, "href").unwrap_or_default(); + + let name = if !resolution.is_empty() { + format!("{} - {}", resolution, provider) + } else { + provider + }; + + all_links.push(crate::modules::anime2::types::Link { name, url }); + } + } + + let download_item = crate::modules::anime2::types::DownloadItem { + resolution: title, + links: all_links, + }; + + if is_batch { + batch.push(download_item); + } else if is_ova { + ova.push(download_item); + } else { + downloads.push(download_item); + } + } + + let mut recommendations = Vec::new(); + for element in document.select(&recommendation_selector) { + let title = text_from_or(&element, &rec_title_selector, ""); + + let anime_url = element + .select(&a_selector) + .next() + .and_then(|e| attr(&e, "href")) + .unwrap_or_default(); + + let rec_slug = extract_slug(&anime_url); + + let poster = element + .select(&rec_img_selector) + .next() + .and_then(|e| attr(&e, "data-src").or_else(|| attr(&e, "src"))) + .unwrap_or_default(); + + let status = text_from_or(&element, &status_selector, ""); + + let r#type = text_from_or(&element, &type_selector, ""); + + recommendations.push(crate::modules::anime2::types::Recommendation { + title, + slug: rec_slug, + poster, + status, + r#type, + }); + } + + Ok(crate::modules::anime2::types::AnimeDetailData { + title, + alternative_title, + poster, + poster2, + r#type, + release_date, + status, + synopsis, + studio, + genres, + producers: vec![], + recommendations, + batch, + ova, + downloads, + }) +} + +pub fn parse_genre_page( + html: &str, + current_page: u32, +) -> Result< + ( + Vec, + crate::shared::types::entities::anime::Pagination, + ), + AppError, +> { + let document = parse_html(html); + let anime_list = parse_genre_anime(html)?; + let pagination = parse_pagination(&document, current_page)?; + Ok((anime_list, pagination)) +} + +pub fn parse_search_page( + html: &str, + current_page: u32, +) -> Result< + ( + Vec, + crate::shared::types::entities::anime::PaginationWithStringPages, + ), + AppError, +> { + let document = parse_html(html); + let data = parse_search_anime(html)?; + let pagination = parse_pagination_with_string(&document, current_page)?; + Ok((data, pagination)) +} + +pub fn parse_latest_page( + html: &str, + current_page: u32, +) -> Result< + ( + Vec, + crate::shared::types::entities::anime::Pagination, + ), + AppError, +> { + let document = parse_html(html); + let anime_list = parse_latest_anime(html)?; + let pagination = parse_pagination(&document, current_page)?; + Ok((anime_list, pagination)) +} + +pub fn parse_ongoing_page( + html: &str, + current_page: u32, +) -> Result< + ( + Vec, + crate::shared::types::entities::anime::Pagination, + ), + AppError, +> { + let document = parse_html(html); + let anime_list = parse_ongoing_anime_with_score(html)?; + let pagination = parse_pagination(&document, current_page)?; + Ok((anime_list, pagination)) +} + +pub fn parse_complete_page( + html: &str, + current_page: u32, +) -> Result< + ( + Vec, + crate::shared::types::entities::anime::Pagination, + ), + AppError, +> { + let document = parse_html(html); + let anime_list = parse_complete_anime(html)?; + let pagination = parse_pagination(&document, current_page)?; + Ok((anime_list, pagination)) +} diff --git a/src/modules/anime2/repository.rs b/src/modules/anime2/repository.rs new file mode 100644 index 0000000..9b21931 --- /dev/null +++ b/src/modules/anime2/repository.rs @@ -0,0 +1,102 @@ +use crate::shared::database::traits::scraping_repository::ScrapingRepository; +use crate::shared::errors::AppError; +use crate::shared::utils::web::proxy_fetch::fetch_with_proxy_only; +use async_trait::async_trait; +use tracing::warn; + +const BASE_URL: &str = "https://alqanime.si"; +const BASE_DETAIL_URL: &str = "https://alqanime.net"; + +pub struct Anime2Repository; + +impl Default for Anime2Repository { + fn default() -> Self { + Self::new() + } +} + +impl Anime2Repository { + pub fn new() -> Self { + Self + } + + pub fn index_ongoing_url(&self) -> String { + format!("{}/anime/?status=ongoing&type=&order=update", BASE_URL) + } + + pub fn index_complete_url(&self) -> String { + format!("{}/anime/?status=completed&type=&order=update", BASE_URL) + } + + pub fn genre_list_url(&self) -> String { + format!("{}/anime/", BASE_URL) + } + + pub fn filter_url(&self, page: u32, order: &str) -> String { + if page > 1 { + format!("{}/anime/page/{}/?order={}", BASE_URL, page, order) + } else { + format!("{}/anime/?order={}", BASE_URL, order) + } + } + + pub fn detail_url(&self, slug: &str) -> String { + format!("{}/{}/", BASE_DETAIL_URL, slug) + } + + pub fn detail_image_url(&self, slug: &str) -> String { + format!("{}/anime/{}/", BASE_URL, slug) + } + + pub fn genre_page_url(&self, genre_slug: &str, page: u32) -> String { + if page > 1 { + format!( + "{}/anime/page/{}/?genre[]={}&order=update", + BASE_URL, page, genre_slug + ) + } else { + format!("{}/anime/?genre[]={}&order=update", BASE_URL, genre_slug) + } + } + + pub fn search_url(&self, query: &str, page: u32) -> String { + let encoded = urlencoding::encode(query); + if page == 1 { + format!("{}/?s={}", BASE_URL, encoded) + } else { + format!("{}/page/{}/?s={}", BASE_URL, page, encoded) + } + } + + pub fn latest_url(&self, page: u32) -> String { + format!( + "{}/anime/page/{}/?status=&type=&order=latest", + BASE_URL, page + ) + } + + pub fn ongoing_url(&self, page: u32) -> String { + format!( + "{}/anime/page/{}/?status=ongoing&type=&order=update", + BASE_URL, page + ) + } + + pub fn complete_url(&self, page: u32) -> String { + format!( + "{}/anime/page/{}/?status=completed&order=update", + BASE_URL, page + ) + } +} + +#[async_trait] +impl ScrapingRepository for Anime2Repository { + async fn fetch_html(&self, url: &str) -> Result { + let response = fetch_with_proxy_only(url).await?; + if response.data.trim().is_empty() { + warn!("Anime2 browserless fetch returned empty body for {}", url); + } + Ok(response.data) + } +} diff --git a/src/modules/anime2/route.rs b/src/modules/anime2/route.rs new file mode 100644 index 0000000..279b4f9 --- /dev/null +++ b/src/modules/anime2/route.rs @@ -0,0 +1,39 @@ +use std::sync::Arc; + +use axum::{routing::get, Router}; + +use crate::modules::anime2::controller; +use crate::shared::state::AppState; + +pub fn routes(router: Router>) -> Router> { + router + .route("/api/anime2", get(controller::index)) + .route( + "/api/anime2/complete_anime/{slug}", + get(controller::complete_anime_slug), + ) + .route("/api/anime2/detail/{slug}", get(controller::detail_slug)) + .route("/api/anime2/filter", get(controller::filter)) + .route("/api/anime2/genre_list", get(controller::genre_list)) + .route( + "/api/anime2/genre/{slug}", + get(controller::genre_slug_index), + ) + .route( + "/api/anime2/genre/{slug}/{page}", + get(controller::genre_slug_page), + ) + .route("/api/anime2/latest/{slug}", get(controller::latest_slug)) + .route( + "/api/anime2/ongoing_anime/{slug}", + get(controller::ongoing_anime_slug), + ) + .route( + "/api/anime2/search/{slug}", + get(controller::search_slug_index), + ) + .route( + "/api/anime2/search/{slug}/{page}", + get(controller::search_slug_page), + ) +} diff --git a/src/modules/anime2/schema.rs b/src/modules/anime2/schema.rs new file mode 100644 index 0000000..aeb3d8c --- /dev/null +++ b/src/modules/anime2/schema.rs @@ -0,0 +1,34 @@ +use serde::Deserialize; +use utoipa::ToSchema; + +#[derive(Debug, Clone, Deserialize, ToSchema)] +pub struct SlugPath { + pub slug: String, +} + +#[derive(Debug, Clone, Deserialize, ToSchema)] +pub struct SlugPagePath { + pub slug: String, + pub page: u32, +} + +#[derive(Deserialize, ToSchema)] +pub struct FilterQuery { + pub page: Option, + pub genre: Option, + pub status: Option, + pub r#type: Option, + pub order: Option, +} + +#[derive(Deserialize, ToSchema)] +pub struct GenreQuery { + pub page: Option, + pub status: Option, + pub order: Option, +} + +#[derive(Deserialize, ToSchema)] +pub struct SearchQuery { + pub q: Option, +} diff --git a/src/modules/anime2/scraping.rs b/src/modules/anime2/scraping.rs new file mode 100644 index 0000000..70eb326 --- /dev/null +++ b/src/modules/anime2/scraping.rs @@ -0,0 +1,359 @@ +use crate::shared::types::entities::anime::*; +use crate::shared::utils::parse_html; +use crate::shared::utils::scraping::{ + attr, attr_from, attr_from_or, extract_slug, selector, text, text_from_or, +}; +use scraper::{Html, Selector}; + +// ============================================================================ +// SELECTORS +// ============================================================================ + +/// Common selectors used across anime parsing +pub struct AnimeSelectors { + pub item: Selector, + pub title: Selector, + pub link: Selector, + pub img: Selector, + pub episode: Selector, + pub score: Selector, + pub status: Selector, + pub genre: Selector, + pub rating: Selector, + pub type_sel: Selector, + pub season: Selector, + pub desc: Selector, +} + +impl AnimeSelectors { + pub fn new() -> Result { + Ok(Self { + item: selector("article.bs").ok_or("Invalid selector: article.bs")?, + title: selector(".tt h2").ok_or("Invalid selector: .tt h2")?, + link: selector("a").ok_or("Invalid selector: a")?, + img: selector("img").ok_or("Invalid selector: img")?, + episode: selector(".epx").ok_or("Invalid selector: .epx")?, + score: selector(".numscore").ok_or("Invalid selector: .numscore")?, + status: selector(".status").ok_or("Invalid selector: .status")?, + genre: selector(".genres a").ok_or("Invalid selector: .genres a")?, + rating: selector(".score").ok_or("Invalid selector: .score")?, + type_sel: selector(".typez").ok_or("Invalid selector: .typez")?, + season: selector(".season").ok_or("Invalid selector: .season")?, + desc: selector(".data .typez").ok_or("Invalid selector: .data .typez")?, + }) + } +} + +impl Default for AnimeSelectors { + fn default() -> Self { + Self::new().expect("Valid CSS selectors") + } +} + +// Global lazy static instance for selectors to avoid reallocation per parse +use once_cell::sync::Lazy; +static ANIME_SELECTORS: Lazy> = + Lazy::new(|| AnimeSelectors::new().map_err(|e| format!("Failed to create selectors: {}", e))); + +// ============================================================================ +// HELPER FUNCTIONS +// ============================================================================ + +/// Extract poster URL from an element, checking both src and data-src attributes +pub fn extract_poster(element: &scraper::ElementRef, img_selector: &Selector) -> String { + element + .select(img_selector) + .next() + .and_then(|e| attr(&e, "src").or(attr(&e, "data-src"))) + .unwrap_or_default() +} + +// ============================================================================ +// ANIME PARSERS +// ============================================================================ + +/// Parse ongoing anime items from HTML +pub fn parse_ongoing_anime( + html: &str, +) -> Result, Box> { + let document = parse_html(html); + let selectors = ANIME_SELECTORS.as_ref().map_err(|e| e.clone())?; + let mut items = Vec::new(); + + for element in document.select(&selectors.item) { + let title = text_from_or(&element, &selectors.title, ""); + if title.is_empty() { + continue; + } + + let href = attr_from_or(&element, &selectors.link, "href", ""); + let slug = extract_slug(&href); + let poster = extract_poster(&element, &selectors.img); + let current_episode = text_from_or(&element, &selectors.episode, "N/A"); + let anime_url = attr_from_or(&element, &selectors.link, "href", ""); + + items.push(OngoingAnimeItem { + title, + slug, + poster, + current_episode, + anime_url, + }); + } + + Ok(items) +} + +/// Parse ongoing anime items with score from HTML +pub fn parse_ongoing_anime_with_score( + html: &str, +) -> Result, Box> { + let document = parse_html(html); + let selectors = ANIME_SELECTORS.as_ref().map_err(|e| e.clone())?; + let mut items = Vec::new(); + + for element in document.select(&selectors.item) { + let title = text_from_or(&element, &selectors.title, ""); + if title.is_empty() { + continue; + } + + let poster = extract_poster(&element, &selectors.img); + let score = text_from_or(&element, &selectors.score, "N/A"); + let anime_url = attr_from_or(&element, &selectors.link, "href", ""); + let slug = extract_slug(&anime_url); + + items.push(OngoingAnimeItemWithScore { + title, + slug, + poster, + score, + anime_url, + }); + } + + Ok(items) +} + +/// Parse complete anime items from HTML +pub fn parse_complete_anime( + html: &str, +) -> Result, Box> { + let document = parse_html(html); + let selectors = ANIME_SELECTORS.as_ref().map_err(|e| e.clone())?; + let mut items = Vec::new(); + + for element in document.select(&selectors.item) { + let title = text_from_or(&element, &selectors.title, ""); + if title.is_empty() { + continue; + } + + let href = attr_from_or(&element, &selectors.link, "href", ""); + let slug = extract_slug(&href); + let poster = extract_poster(&element, &selectors.img); + let episode_count = text_from_or(&element, &selectors.episode, "N/A"); + let anime_url = attr_from_or(&element, &selectors.link, "href", ""); + + items.push(CompleteAnimeItem { + title, + slug, + poster, + episode_count, + anime_url, + }); + } + + Ok(items) +} + +/// Parse latest anime items from HTML +pub fn parse_latest_anime( + html: &str, +) -> Result, Box> { + let document = parse_html(html); + let selectors = ANIME_SELECTORS.as_ref().map_err(|e| e.clone())?; + let mut items = Vec::new(); + + for element in document.select(&selectors.item) { + let title = text_from_or(&element, &selectors.title, ""); + if title.is_empty() { + continue; + } + + let poster = extract_poster(&element, &selectors.img); + let current_episode = text_from_or(&element, &selectors.episode, "N/A"); + let score = text_from_or(&element, &selectors.score, "N/A"); + let anime_url = attr_from_or(&element, &selectors.link, "href", ""); + let slug = extract_slug(&anime_url); + + items.push(LatestAnimeItem { + title, + slug, + poster, + current_episode, + score, + anime_url, + }); + } + + Ok(items) +} + +/// Parse search results from HTML +pub fn parse_search_anime( + html: &str, +) -> Result, Box> { + let document = parse_html(html); + let selectors = ANIME_SELECTORS.as_ref().map_err(|e| e.clone())?; + let mut items = Vec::new(); + + for element in document.select(&selectors.item) { + let title = text_from_or(&element, &selectors.title, ""); + if title.is_empty() { + continue; + } + + let href = attr_from(&element, &selectors.link, "href").unwrap_or_default(); + let slug = extract_slug(&href); + let poster = extract_poster(&element, &selectors.img); + let description = text_from_or(&element, &selectors.desc, ""); + let anime_url = attr_from_or(&element, &selectors.link, "href", ""); + let genres = element.select(&selectors.genre).map(|e| text(&e)).collect(); + let rating = text_from_or(&element, &selectors.rating, ""); + let r#type = text_from_or(&element, &selectors.type_sel, ""); + let season = text_from_or(&element, &selectors.season, ""); + + items.push(SearchAnimeItem { + title, + slug, + poster, + description, + anime_url, + genres, + rating, + r#type, + season, + }); + } + + Ok(items) +} + +/// Parse genre-filtered anime items from HTML +pub fn parse_genre_anime( + html: &str, +) -> Result, Box> { + let document = parse_html(html); + let selectors = ANIME_SELECTORS.as_ref().map_err(|e| e.clone())?; + let mut items = Vec::new(); + + for element in document.select(&selectors.item) { + let title = text_from_or(&element, &selectors.title, ""); + if title.is_empty() { + continue; + } + + let poster = extract_poster(&element, &selectors.img); + let score = text_from_or(&element, &selectors.score, "N/A"); + let status = text_from_or(&element, &selectors.status, "Unknown"); + let anime_url = attr_from_or(&element, &selectors.link, "href", ""); + let slug = extract_slug(&anime_url); + + items.push(GenreAnimeItem { + title, + slug, + poster, + score, + status, + anime_url, + }); + } + + Ok(items) +} + +// ============================================================================ +// PAGINATION PARSERS +// ============================================================================ + +/// Parse pagination from HTML document +pub fn parse_pagination(document: &Html, current_page: u32) -> Result { + let pagination_selector = + selector(".pagination .page-numbers:not(.next)").ok_or("Invalid selector")?; + let next_selector = selector(".pagination .next").ok_or("Invalid selector")?; + + let last_visible_page = document + .select(&pagination_selector) + .next_back() + .and_then(|e| text(&e).trim().parse::().ok()) + .unwrap_or(current_page); + + let has_next_page = document.select(&next_selector).next().is_some(); + let next_page = if has_next_page { + Some(current_page + 1) + } else { + None + }; + + let has_previous_page = current_page > 1; + let previous_page = if has_previous_page { + Some(current_page - 1) + } else { + None + }; + + Ok(Pagination { + current_page, + last_visible_page, + has_next_page, + next_page, + has_previous_page, + previous_page, + }) +} + +/// Parse pagination with string-based page numbers (for search results) +pub fn parse_pagination_with_string( + document: &Html, + current_page: u32, +) -> Result { + let pagination_selector = + selector(".pagination .page-numbers:not(.next)").ok_or("Invalid selector")?; + let next_selector = selector(".pagination .next").ok_or("Invalid selector")?; + + let last_visible_page = document + .select(&pagination_selector) + .last() + .and_then(|e| text(&e).trim().parse::().ok()) + .unwrap_or(current_page); + + let has_next_page = document.select(&next_selector).next().is_some(); + + let next_page = if has_next_page { + document + .select(&next_selector) + .next() + .and_then(|e| attr(&e, "href")) + .and_then(|href| href.split("/page/").nth(1).map(|s| s.to_string())) + .and_then(|s| s.split('/').next().map(|s| s.to_string())) + } else { + None + }; + + let has_previous_page = current_page > 1; + let previous_page = if has_previous_page { + Some((current_page - 1).to_string()) + } else { + None + }; + + Ok(PaginationWithStringPages { + current_page, + last_visible_page, + has_next_page, + next_page, + has_previous_page, + previous_page, + }) +} diff --git a/src/modules/anime2/service.rs b/src/modules/anime2/service.rs new file mode 100644 index 0000000..fc7548a --- /dev/null +++ b/src/modules/anime2/service.rs @@ -0,0 +1,453 @@ +use std::sync::Arc; + +use crate::modules::anime2::parser; +use crate::modules::anime2::repository::Anime2Repository; +use crate::modules::anime2::types::{DetailResponse, GenresResponse}; +use crate::shared::database::traits::scraping_repository::ScrapingRepository; +use crate::shared::errors::AppError; +use crate::shared::services::images::cache::{ + apply_cached_posters, cache_image_urls_batch_lazy, get_cached_or_original, +}; +use crate::shared::state::AppState; +use crate::shared::types::ApiResponse; +use crate::shared::utils::Cache; + +const INDEX_CACHE_TTL: u64 = 300; +const GENRE_LIST_CACHE_TTL: u64 = 3600; +const FILTER_CACHE_TTL: u64 = 300; +const DETAIL_CACHE_TTL: u64 = 300; +const GENRE_CACHE_TTL: u64 = 300; +const SEARCH_CACHE_TTL: u64 = 300; +const LATEST_CACHE_TTL: u64 = 120; +const ONGOING_CACHE_TTL: u64 = 300; +const COMPLETE_CACHE_TTL: u64 = 300; + +pub struct Anime2Service { + repository: Anime2Repository, +} + +impl Anime2Service { + pub fn new(repository: Anime2Repository) -> Self { + Self { repository } + } + + pub async fn index( + &self, + app_state: Arc, + ) -> Result { + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set("anime2:index", INDEX_CACHE_TTL, || async { + let ongoing_html = self + .repository + .fetch_html(&self.repository.index_ongoing_url()) + .await + .map_err(|e| e.to_string())?; + let complete_html = self + .repository + .fetch_html(&self.repository.index_complete_url()) + .await + .map_err(|e| e.to_string())?; + + let mut data = tokio::task::spawn_blocking(move || { + Ok::<_, String>(( + parser::parse_ongoing_anime(&ongoing_html).map_err(|e| e.to_string())?, + parser::parse_complete_anime(&complete_html).map_err(|e| e.to_string())?, + )) + }) + .await + .map_err(|e| e.to_string())??; + + let mut posters: Vec = + data.0.iter().map(|item| item.poster.clone()).collect(); + posters.extend(data.1.iter().map(|item| item.poster.clone())); + + let cached_posters = cache_image_urls_batch_lazy( + app_state.db.clone(), + &app_state.redis_pool, + posters, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + let ongoing_len = data.0.len(); + for (i, item) in data.0.iter_mut().enumerate() { + if let Some(url) = cached_posters.get(i) { + item.poster = url.clone(); + } + } + for (i, item) in data.1.iter_mut().enumerate() { + if let Some(url) = cached_posters.get(ongoing_len + i) { + item.poster = url.clone(); + } + } + + Ok(crate::modules::anime2::types::Anime2Response { + status: "Ok".to_string(), + data: crate::modules::anime2::types::Anime2Data { + ongoing_anime: data.0, + complete_anime: data.1, + }, + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn genre_list(&self, app_state: Arc) -> Result { + let cache = Cache::new(&app_state.redis_pool); + + cache + .get_or_set("anime2:genres:list:v3", GENRE_LIST_CACHE_TTL, || async { + let html = self + .repository + .fetch_html(&self.repository.genre_list_url()) + .await + .map_err(|e| e.to_string())?; + + let genres = tokio::task::spawn_blocking(move || { + parser::parse_genres(&html).map_err(|e| e.to_string()) + }) + .await + .map_err(|e| e.to_string())??; + + Ok(GenresResponse { + status: "Ok".to_string(), + data: genres, + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn filter( + &self, + app_state: Arc, + page: u32, + genre: Option, + status: Option, + anime_type: Option, + order: String, + ) -> Result { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!( + "anime2:filter:{}:{:?}:{:?}:{:?}:{}", + page, genre, status, anime_type, order + ); + let genre_clone = genre.clone(); + let status_clone = status.clone(); + let anime_type_clone = anime_type.clone(); + + cache + .get_or_set(&cache_key, FILTER_CACHE_TTL, || async { + let mut url = self.repository.filter_url(page, &order); + + if let Some(g) = &genre { + for genre_item in g.split(',') { + url.push_str(&format!("&genre[]={}", genre_item.trim())); + } + } + if let Some(s) = &status { + url.push_str(&format!("&status={}", s)); + } + if let Some(t) = &anime_type { + url.push_str(&format!("&type={}", t)); + } + + let html = self + .repository + .fetch_html(&url) + .await + .map_err(|e| e.to_string())?; + let (data, pagination) = tokio::task::spawn_blocking(move || { + parser::parse_filter_page(&html, page).map_err(|e| e.to_string()) + }) + .await + .map_err(|e| e.to_string())??; + + let mut final_data = data; + apply_cached_posters( + &mut final_data, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(crate::modules::anime2::types::FilterResponse { + success: true, + data: final_data, + pagination, + filters_applied: crate::modules::anime2::types::FiltersApplied { + genre: genre_clone, + status: status_clone, + r#type: anime_type_clone, + order: order.clone(), + }, + status: "Ok".to_string(), + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn detail( + &self, + app_state: Arc, + slug: String, + ) -> Result { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("anime2:detail:{}", slug); + + cache + .get_or_set(&cache_key, DETAIL_CACHE_TTL, || async { + let detail_url = self.repository.detail_url(&slug); + let image_url = self.repository.detail_image_url(&slug); + let (detail_html, image_html) = tokio::join!( + self.repository.fetch_html(&detail_url), + self.repository.fetch_html(&image_url) + ); + let detail_html = detail_html.map_err(|e| e.to_string())?; + let image_html = image_html.ok(); + + let mut data = tokio::task::spawn_blocking(move || { + let mut data = + parser::parse_anime_detail(&detail_html).map_err(|e| e.to_string())?; + if let Some(image_html) = image_html { + if let Ok(image_data) = parser::parse_anime_detail(&image_html) { + if !image_data.poster.is_empty() { + data.poster = image_data.poster; + } + if !image_data.poster2.is_empty() { + data.poster2 = image_data.poster2; + } + data.recommendations = image_data.recommendations; + } + } + Ok::<_, String>(data) + }) + .await + .map_err(|e| e.to_string())??; + + data.poster = get_cached_or_original( + app_state.db.clone(), + &app_state.redis_pool, + &data.poster, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + data.poster2 = get_cached_or_original( + app_state.db.clone(), + &app_state.redis_pool, + &data.poster2, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + apply_cached_posters( + &mut data.recommendations, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(DetailResponse { + status: "Ok".to_string(), + data, + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn genre_slug( + &self, + app_state: Arc, + genre_slug: String, + page: u32, + ) -> Result>, AppError> + { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("anime2:genre:{}:{}", genre_slug, page); + + cache + .get_or_set(&cache_key, GENRE_CACHE_TTL, || async { + let html = self + .repository + .fetch_html(&self.repository.genre_page_url(&genre_slug, page)) + .await + .map_err(|e| e.to_string())?; + let (data, _pagination) = tokio::task::spawn_blocking(move || { + parser::parse_genre_page(&html, page).map_err(|e| e.to_string()) + }) + .await + .map_err(|e| e.to_string())??; + + let mut final_data = data; + apply_cached_posters( + &mut final_data, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(ApiResponse::success(final_data)) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn search( + &self, + app_state: Arc, + query: String, + page: u32, + ) -> Result>, AppError> + { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("anime2:search:{}:{}", query, page); + + cache + .get_or_set(&cache_key, SEARCH_CACHE_TTL, || async { + let html = self + .repository + .fetch_html(&self.repository.search_url(&query, page)) + .await + .map_err(|e| e.to_string())?; + let (data, _pagination) = tokio::task::spawn_blocking(move || { + parser::parse_search_page(&html, page).map_err(|e| e.to_string()) + }) + .await + .map_err(|e| e.to_string())??; + + let mut final_data = data; + apply_cached_posters( + &mut final_data, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(ApiResponse::success(final_data)) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn latest( + &self, + app_state: Arc, + page: u32, + ) -> Result>, AppError> + { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("anime2:latest:{}", page); + + cache + .get_or_set(&cache_key, LATEST_CACHE_TTL, || async { + let html = self + .repository + .fetch_html(&self.repository.latest_url(page)) + .await + .map_err(|e| e.to_string())?; + let (data, _pagination) = tokio::task::spawn_blocking(move || { + parser::parse_latest_page(&html, page).map_err(|e| e.to_string()) + }) + .await + .map_err(|e| e.to_string())??; + + let mut final_data = data; + apply_cached_posters( + &mut final_data, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(ApiResponse::success(final_data)) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn ongoing_anime( + &self, + app_state: Arc, + page: u32, + ) -> Result< + ApiResponse>, + AppError, + > { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("anime2:ongoing:{}", page); + + cache + .get_or_set(&cache_key, ONGOING_CACHE_TTL, || async { + let html = self + .repository + .fetch_html(&self.repository.ongoing_url(page)) + .await + .map_err(|e| e.to_string())?; + let (data, _pagination) = tokio::task::spawn_blocking(move || { + parser::parse_ongoing_page(&html, page).map_err(|e| e.to_string()) + }) + .await + .map_err(|e| e.to_string())??; + + let mut final_data = data; + apply_cached_posters( + &mut final_data, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(ApiResponse::success(final_data)) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn complete_anime( + &self, + app_state: Arc, + page: u32, + ) -> Result>, AppError> + { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("anime2:complete:{}", page); + + cache + .get_or_set(&cache_key, COMPLETE_CACHE_TTL, || async { + let html = self + .repository + .fetch_html(&self.repository.complete_url(page)) + .await + .map_err(|e| e.to_string())?; + let (data, _pagination) = tokio::task::spawn_blocking(move || { + parser::parse_complete_page(&html, page).map_err(|e| e.to_string()) + }) + .await + .map_err(|e| e.to_string())??; + + let mut final_data = data; + apply_cached_posters( + &mut final_data, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(ApiResponse::success(final_data)) + }) + .await + .map_err(AppError::ScraperError) + } +} diff --git a/src/modules/anime2/types.rs b/src/modules/anime2/types.rs new file mode 100644 index 0000000..2523d1e --- /dev/null +++ b/src/modules/anime2/types.rs @@ -0,0 +1,106 @@ +use crate::shared::types::entities::anime::HasPoster; +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Anime2Data { + pub ongoing_anime: Vec, + pub complete_anime: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Anime2Response { + pub status: String, + pub data: Anime2Data, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Genre { + pub name: String, + pub slug: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct GenresResponse { + pub status: String, + pub data: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct FiltersApplied { + pub genre: Option, + pub status: Option, + pub r#type: Option, + pub order: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct FilterResponse { + pub success: bool, + pub data: Vec, + pub pagination: crate::shared::types::entities::anime::Pagination, + pub filters_applied: FiltersApplied, + pub status: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct AnimeDetailData { + pub title: String, + pub alternative_title: String, + pub poster: String, + pub poster2: String, + pub r#type: String, + pub release_date: String, + pub status: String, + pub synopsis: String, + pub studio: String, + pub genres: Vec, + pub producers: Vec, + pub recommendations: Vec, + pub batch: Vec, + pub ova: Vec, + pub downloads: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct DetailGenre { + pub name: String, + pub slug: String, + pub anime_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Link { + pub name: String, + pub url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct DownloadItem { + pub resolution: String, + pub links: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Recommendation { + pub title: String, + pub slug: String, + pub poster: String, + pub status: String, + pub r#type: String, +} + +impl HasPoster for Recommendation { + fn poster(&self) -> &str { + &self.poster + } + fn set_poster(&mut self, url: String) { + self.poster = url; + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct DetailResponse { + pub status: String, + pub data: AnimeDetailData, +} diff --git a/src/modules/komik/controller.rs b/src/modules/komik/controller.rs new file mode 100644 index 0000000..cec37e8 --- /dev/null +++ b/src/modules/komik/controller.rs @@ -0,0 +1,217 @@ +use std::sync::Arc; + +use axum::{extract::State, Json}; + +use crate::modules::komik::repository::KomikRepository; +use crate::modules::komik::service::KomikService; +use crate::shared::errors::AppError; +use crate::shared::state::AppState; + +#[utoipa::path( + get, + path = "/api/komik/genre_list", + tag = "komik", + operation_id = "komik_genre_list", + responses( + (status = 200, description = "Handles GET requests for the /api/komik/genre_list endpoint.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn genre_list( + State(app_state): State>, +) -> Result, AppError> { + let service = KomikService::new(KomikRepository::new()); + service.genre_list(app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/chapter/{slug}", + tag = "komik", + operation_id = "komik_chapter_slug", + responses( + (status = 200, description = "Retrieves details for a specific chapter by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn chapter_slug( + State(app_state): State>, + axum::extract::Path(slug): axum::extract::Path, +) -> Result, AppError> { + let service = KomikService::new(KomikRepository::new()); + service.chapter_slug(slug, app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/detail/{slug}", + tag = "komik", + operation_id = "komik_detail_slug", + responses( + (status = 200, description = "Retrieves details for a specific detail by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn detail_slug( + State(app_state): State>, + axum::extract::Path(slug): axum::extract::Path, +) -> Result, AppError> { + let service = KomikService::new(KomikRepository::new()); + service.detail_slug(slug, app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/genre/{slug}", + tag = "komik", + operation_id = "komik_genre_slug_index", + responses( + (status = 200, description = "Retrieves details for a specific genre by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn genre_slug( + State(app_state): State>, + axum::extract::Path(slug): axum::extract::Path, +) -> Result, AppError> { + let service = KomikService::new(KomikRepository::new()); + service.genre_slug(slug, app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/genre/{slug}/{page}", + tag = "komik", + operation_id = "komik_genre_slug_page", + responses( + (status = 200, description = "Retrieves paginated genre results by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn genre_slug_page( + State(app_state): State>, + axum::extract::Path((slug, page)): axum::extract::Path<(String, String)>, +) -> Result, AppError> { + let page_num = page + .parse::() + .map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?; + let service = KomikService::new(KomikRepository::new()); + service + .genre_slug_page(slug, page_num, app_state) + .await + .map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/manga/{slug}", + tag = "komik", + operation_id = "komik_manga_slug", + responses( + (status = 200, description = "Retrieves manga details by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn manga_slug( + State(app_state): State>, + axum::extract::Path(slug): axum::extract::Path, +) -> Result, AppError> { + let service = KomikService::new(KomikRepository::new()); + service.manga_slug(slug, app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/manhua/{slug}", + tag = "komik", + operation_id = "komik_manhua_slug", + responses( + (status = 200, description = "Retrieves manhua details by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn manhua_slug( + State(app_state): State>, + axum::extract::Path(slug): axum::extract::Path, +) -> Result, AppError> { + let service = KomikService::new(KomikRepository::new()); + service.manhua_slug(slug, app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/manhwa/{slug}", + tag = "komik", + operation_id = "komik_manhwa_slug", + responses( + (status = 200, description = "Retrieves manhwa details by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn manhwa_slug( + State(app_state): State>, + axum::extract::Path(slug): axum::extract::Path, +) -> Result, AppError> { + let service = KomikService::new(KomikRepository::new()); + service.manhwa_slug(slug, app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/popular/{slug}", + tag = "komik", + operation_id = "komik_popular_slug", + responses( + (status = 200, description = "Retrieves popular komik details by slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn popular_slug( + State(app_state): State>, + axum::extract::Path(slug): axum::extract::Path, +) -> Result, AppError> { + let service = KomikService::new(KomikRepository::new()); + service.popular_slug(slug, app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/search/{slug}", + tag = "komik", + operation_id = "komik_search_slug_index", + responses( + (status = 200, description = "Retrieves search results by query slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn search_slug( + State(app_state): State>, + axum::extract::Path(slug): axum::extract::Path, +) -> Result, AppError> { + let service = KomikService::new(KomikRepository::new()); + service.search_slug(slug, app_state).await.map(Json) +} + +#[utoipa::path( + get, + path = "/api/komik/search/{slug}/{page}", + tag = "komik", + operation_id = "komik_search_slug_page", + responses( + (status = 200, description = "Retrieves paginated search results by query slug.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn search_slug_page( + State(app_state): State>, + axum::extract::Path((slug, page)): axum::extract::Path<(String, String)>, +) -> Result, AppError> { + let page_num = page + .parse::() + .map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?; + let service = KomikService::new(KomikRepository::new()); + service + .search_slug_page(slug, page_num, app_state) + .await + .map(Json) +} diff --git a/src/modules/komik/mod.rs b/src/modules/komik/mod.rs new file mode 100644 index 0000000..67dc224 --- /dev/null +++ b/src/modules/komik/mod.rs @@ -0,0 +1,7 @@ +pub mod controller; +pub mod parser; +pub mod repository; +pub mod route; +pub mod schema; +pub mod service; +pub mod types; diff --git a/src/modules/komik/parser.rs b/src/modules/komik/parser.rs new file mode 100644 index 0000000..0b4c45a --- /dev/null +++ b/src/modules/komik/parser.rs @@ -0,0 +1,592 @@ +use crate::modules::komik::types::{ChapterData, DetailData, Genre, KomikItem, Pagination}; +use crate::shared::utils::parse_html; +use crate::shared::utils::scraping::{attr, attr_from, attr_from_or, selector, text, text_from_or}; +use once_cell::sync::Lazy; +use rayon::prelude::*; +use regex::Regex; +use tracing::info; + +static TD_LAST_SELECTOR: Lazy = Lazy::new(|| selector("td:last-child").unwrap()); +static TITLE_SELECTOR: Lazy = + Lazy::new(|| selector("div#Judul h1 span[itemprop=\"name\"]").unwrap()); +static H1_SELECTOR: Lazy = Lazy::new(|| selector("h1").unwrap()); +static TITLE_TAG_SELECTOR: Lazy = Lazy::new(|| selector("title").unwrap()); +static INFO_ROW_SELECTOR: Lazy = + Lazy::new(|| selector("table.inftable tr").unwrap()); +static POSTER_SELECTOR: Lazy = + Lazy::new(|| selector("section#Informasi .ims img").unwrap()); +static DESC_SELECTOR: Lazy = Lazy::new(|| selector("p.desc").unwrap()); +static CHAPTER_LIST_SELECTOR: Lazy = + Lazy::new(|| selector("#Daftar_Chapter tr, tbody#daftarChapter tr").unwrap()); +static DATE_LINK_SELECTOR: Lazy = + Lazy::new(|| selector("td.tanggalseries, .tanggalseries").unwrap()); +static JUDUL2_SELECTOR: Lazy = Lazy::new(|| selector("div.judul2").unwrap()); +static GENRE_SELECTOR: Lazy = Lazy::new(|| selector("ul.genre li a").unwrap()); +static CHAPTER_LINK_SELECTOR: Lazy = + Lazy::new(|| selector("td.judulseries a").unwrap()); +static CHAPTER_TITLE_REGEX: Lazy = + Lazy::new(|| Regex::new(r"(?i)(?:chapter|ch\.?)\s*([\d\.]+)").unwrap()); +static CHAPTER_NUMBER_REGEX: Lazy = Lazy::new(|| Regex::new(r"([\d\.]+)").unwrap()); + +pub fn parse_genres(html: &str) -> Result, String> { + let document = parse_html(html); + let mut genres = Vec::new(); + + let genre_selector = + selector("#Genre .ls3, section#Genre .ls3, .ls3").ok_or("Selector error".to_string())?; + let genre_name_selector = selector(".ls3p h4, h4").ok_or("Selector error".to_string())?; + let genre_link_selector = selector("a[href*='/genre/']").ok_or("Selector error".to_string())?; + let slug_regex = Regex::new(r"/genre/([^/]+)").unwrap(); + + for element in document.select(&genre_selector) { + let name = text_from_or(&element, &genre_name_selector, ""); + let href = attr_from(&element, &genre_link_selector, "href").unwrap_or_default(); + + let slug = slug_regex + .captures(&href) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str()) + .unwrap_or("") + .to_string(); + + if !name.is_empty() && !slug.is_empty() { + genres.push(Genre { + name, + slug, + count: None, + }); + } + } + + info!("Parsed {} genres", genres.len()); + Ok(genres) +} + +pub fn parse_komik_chapter_document(html: &str, chapter_url: &str) -> Result { + let document = parse_html(html); + let _start_time = std::time::Instant::now(); + info!("Starting to parse komik chapter document"); + + let title_selector = selector("title").ok_or("Selector error".to_string())?; + let prev_chapter_selector = selector( + "a[aria-label='Prev'][href*='chapter'], .nxpr a:not(.rl):not([href*='#Chapter']), .chprev a, a.prev", + ) + .ok_or("Selector error".to_string())?; + let next_chapter_selector = selector( + "a[aria-label='Next'][href*='chapter'], .nxpr a.rl, .nxpr a.next, .chnext a, a.next", + ) + .ok_or("Selector error".to_string())?; + let image_selector = + selector("#Baca_Komik img, img.klazy.ww").ok_or("Selector error".to_string())?; + + let title = document + .select(&title_selector) + .next() + .map(|e| { + let full_title = text(&e); + if let Some(start) = full_title.find("Komik ") { + if let Some(end) = full_title.find(" - Komiku") { + full_title[start + 6..end].trim().to_string() + } else { + full_title + } + } else { + full_title + } + }) + .unwrap_or_default(); + + let next_chapter_id = document + .select(&next_chapter_selector) + .next() + .and_then(|e| attr(&e, "href")) + .map(|href| { + href.trim_end_matches('/') + .split('/') + .filter(|s| !s.is_empty()) + .next_back() + .unwrap_or("") + .to_string() + }) + .unwrap_or_default(); + + fn get_previous_chapter_id(chapter_url: &str) -> String { + const CHAPTER_PATTERN: &str = "chapter-"; + + if let Some(pattern_pos) = chapter_url.rfind(CHAPTER_PATTERN) { + let prefix = &chapter_url[0..pattern_pos]; + let suffix = &chapter_url[pattern_pos + CHAPTER_PATTERN.len()..]; + + let chapter_num = suffix + .chars() + .take_while(|c| c.is_ascii_digit()) + .collect::(); + + if let Ok(num) = chapter_num.parse::() { + let prev_num = num.saturating_sub(1); + + let formatted_num = if chapter_num.starts_with('0') { + format!("{:0width$}", prev_num, width = chapter_num.len()) + } else { + prev_num.to_string() + }; + + return format!("{}{}{}", prefix, CHAPTER_PATTERN, formatted_num); + } + } + + String::new() + } + + let prev_chapter_id_from_url = get_previous_chapter_id(chapter_url); + let prev_chapter_id = if !prev_chapter_id_from_url.is_empty() { + prev_chapter_id_from_url + } else { + document + .select(&prev_chapter_selector) + .next() + .and_then(|e| attr(&e, "href")) + .map(|href| { + href.trim_end_matches('/') + .split('/') + .filter(|s| !s.is_empty()) + .next_back() + .unwrap_or("") + .to_string() + }) + .unwrap_or_default() + }; + + fn get_list_chapter_from_url(chapter_url: &str) -> String { + if let Some(pos) = chapter_url.rfind("-chapter-") { + chapter_url[..pos].to_string() + } else { + chapter_url.to_string() + } + } + + let list_chapter = get_list_chapter_from_url(chapter_url); + + let mut images = Vec::new(); + let forbidden_images = [ + "https://flagcdn.com/32x24/jp.png", + "https://flagcdn.com/32x24/kr.png", + "https://flagcdn.com/32x24/cn.png", + "https://www.gstatic.com/firebasejs/ui/2.0.0/images/auth/google.svg", + "https://www.gravatar.com/avatar/?d=mp&s=80", + "/asset/img/komikuplus2.jpg", + "https://komiku.org/asset/img/Loading.gif", + ]; + for el in document.select(&image_selector) { + if let Some(src) = attr(&el, "src") + .or_else(|| attr(&el, "data-src")) + .or_else(|| attr(&el, "data-lazy-src")) + .or_else(|| { + attr(&el, "srcset").and_then(|s| s.split_whitespace().next().map(|s| s.to_string())) + }) + { + if !forbidden_images.contains(&src.as_str()) { + images.push(src); + } + } + } + + Ok(ChapterData { + title, + next_chapter_id, + prev_chapter_id, + list_chapter, + images, + }) +} + +fn clean_text(text: String) -> String { + text.replace(['\n', '\t'], " ").trim().to_string() +} + +fn extract_value_after_keyword(full_text: &str, keywords: &[&str], default_index: usize) -> String { + let lower_text = full_text.to_lowercase(); + for keyword in keywords { + if let Some(pos) = lower_text.find(&format!("{}:", keyword)) { + return clean_text(full_text[pos + keyword.len() + 1..].to_string()); + } else if let Some(pos) = lower_text.find(&format!("{} ", keyword)) { + return clean_text(full_text[pos + keyword.len() + 1..].to_string()); + } + } + if let Some(colon_pos) = full_text.find(':') { + return clean_text(full_text[colon_pos + 1..].to_string()); + } + clean_text(full_text[default_index..].to_string()) +} + +fn find_table_row_with_text<'a>( + info_rows: &[scraper::ElementRef<'a>], + text_fragments: &[&str], +) -> Option { + let lower_text_fragments: Vec = + text_fragments.iter().map(|&s| s.to_lowercase()).collect(); + let td_last_selector = &*TD_LAST_SELECTOR; + + info_rows + .iter() + .find(|row| { + let row_text = text(row).to_lowercase(); + lower_text_fragments + .iter() + .any(|fragment| row_text.contains(fragment)) + }) + .and_then(|row| { + text_from_or(row, td_last_selector, "") + .trim() + .to_string() + .into() + }) +} + +pub fn parse_komik_detail_document(html: &str) -> Result { + let start_time = std::time::Instant::now(); + info!("Starting to parse komik detail document"); + + let document = parse_html(html); + + let title_selector = &*TITLE_SELECTOR; + let h1_selector = &*H1_SELECTOR; + let title_tag_selector = &*TITLE_TAG_SELECTOR; + let info_row_selector = &*INFO_ROW_SELECTOR; + let poster_selector = &*POSTER_SELECTOR; + let desc_selector = &*DESC_SELECTOR; + let chapter_list_selector = &*CHAPTER_LIST_SELECTOR; + let date_link_selector = &*DATE_LINK_SELECTOR; + let judul2_selector = &*JUDUL2_SELECTOR; + let genre_selector = &*GENRE_SELECTOR; + let chapter_link_selector = &*CHAPTER_LINK_SELECTOR; + + let title = document + .select(&title_selector) + .next() + .map(|e| { + let text = clean_text(text(&e)); + text.replace("Komik ", "") + .replace("Manga ", "") + .replace("Manhua ", "") + .replace("Manhwa ", "") + .trim() + .to_string() + }) + .or_else(|| { + document + .select(&h1_selector) + .next() + .map(|e| clean_text(text(&e))) + }) + .or_else(|| { + document.select(&title_tag_selector).next().map(|e| { + let text = clean_text(text(&e)); + if text.contains("Komik ") { + text.replace("Komik ", "").trim().to_string() + } else { + text.trim().to_string() + } + }) + }) + .unwrap_or_default(); + + let info_rows_vec: Vec = document.select(&info_row_selector).collect(); + let info_rows = &info_rows_vec[..]; + + let status = info_rows + .iter() + .find_map(|&row| { + let full_text = text(&row); + if full_text.to_lowercase().contains("status") { + Some( + extract_value_after_keyword(&full_text, &["status"], 0) + .replace("Status", "") + .replace("Jenis Komik", "") + .replace("Type", "") + .trim() + .to_string(), + ) + } else { + None + } + }) + .unwrap_or_default(); + + let r#type = info_rows + .iter() + .find_map(|&row| { + let full_text = text(&row); + if full_text.to_lowercase().contains("jenis komik") + || full_text.to_lowercase().contains("type") + { + Some( + extract_value_after_keyword(&full_text, &["jenis komik", "type"], 0) + .replace("Jenis Komik", "") + .replace("Type", "") + .trim() + .to_string(), + ) + } else { + None + } + }) + .unwrap_or_default(); + + let author = info_rows + .iter() + .find_map(|&row| { + let full_text = text(&row); + if full_text.to_lowercase().contains("pengarang") + || full_text.to_lowercase().contains("author") + || full_text.to_lowercase().contains("artist") + { + Some( + extract_value_after_keyword(&full_text, &["pengarang", "author", "artist"], 0) + .replace("Pengarang", "") + .replace("Author", "") + .replace("pengarang", "") + .replace("author", "") + .replace("Artist", "") + .replace("artist", "") + .trim() + .to_string(), + ) + } else { + None + } + }) + .unwrap_or_default(); + + let poster = document + .select(&poster_selector) + .next() + .and_then(|e| attr(&e, "src")) + .map(|s| s.split('?').next().unwrap_or(&s).to_string()) + .unwrap_or_default(); + + let description = document + .select(&desc_selector) + .map(|e| clean_text(text(&e))) + .filter(|t| t.len() > 50) + .collect::>() + .join("\n") + .trim() + .to_string(); + + let release_date = find_table_row_with_text(info_rows, &["tanggal rilis", "release date"]) + .map(clean_text) + .unwrap_or_else(|| { + document + .select(&chapter_list_selector) + .next_back() + .and_then(|last| last.select(&date_link_selector).next()) + .map(|e| clean_text(text(&e))) + .unwrap_or_default() + }); + + let total_chapter = find_table_row_with_text(info_rows, &["total chapter", "total chapters"]) + .unwrap_or_else(|| { + let count = document.select(&chapter_list_selector).count(); + if count > 0 { + count.to_string() + } else { + String::new() + } + }); + + let updated_on = find_table_row_with_text(info_rows, &["diperbarui", "updated"]) + .or_else(|| { + document.select(&judul2_selector).next().map(|e| { + let text_str = clean_text(text(&e)); + text_str.split("• ").nth(1).unwrap_or("").trim().to_string() + }) + }) + .unwrap_or_else(|| { + document + .select(&chapter_list_selector) + .next() + .and_then(|first| first.select(&date_link_selector).next()) + .map(|e| clean_text(text(&e))) + .unwrap_or_default() + }); + + let mut genres = Vec::new(); + for element in document.select(&genre_selector) { + let genre = clean_text(text(&element)); + if !genre.is_empty() { + genres.push(genre); + } + } + + let raw_chapter_data: Vec<(String, String, String)> = document + .select(&chapter_list_selector) + .filter_map(|el| { + let chapter_link_element = el.select(&chapter_link_selector).next(); + let date_element = el.select(&date_link_selector).next(); + + let chapter_text = chapter_link_element + .as_ref() + .map(|e| clean_text(text(e))) + .unwrap_or_default(); + + let date_text = date_element + .map(|e| clean_text(text(&e))) + .unwrap_or_default(); + + let href_text = chapter_link_element + .and_then(|e| attr(&e, "href")) + .unwrap_or_default(); + + if !chapter_text.is_empty() || !date_text.is_empty() || !href_text.is_empty() { + Some((chapter_text, date_text, href_text)) + } else { + None + } + }) + .collect(); + + let chapters: Vec = raw_chapter_data + .par_iter() + .filter_map(|(chapter_text, date_text, href_text)| { + let chapter = { + let trimmed_chapter_text = chapter_text.trim(); + if let Some(captures) = CHAPTER_TITLE_REGEX.captures(trimmed_chapter_text) { + captures + .get(1) + .map_or(trimmed_chapter_text.to_string(), |m| m.as_str().to_string()) + } else if let Some(captures) = CHAPTER_NUMBER_REGEX.captures(trimmed_chapter_text) { + captures + .get(1) + .map_or(trimmed_chapter_text.to_string(), |m| m.as_str().to_string()) + } else { + trimmed_chapter_text.to_string() + } + }; + + let date = date_text.trim().to_string(); + + let chapter_id = href_text + .split('/') + .filter(|s| !s.is_empty()) + .next_back() + .unwrap_or("") + .to_string(); + + if !chapter_id.is_empty() { + Some(crate::modules::komik::types::Chapter { + chapter, + date, + chapter_id, + }) + } else { + None + } + }) + .collect(); + + let duration = start_time.elapsed(); + info!("Parsed komik detail document in {:?}", duration); + + Ok(DetailData { + title, + poster, + description, + status, + r#type, + release_date, + author, + total_chapter, + updated_on, + genres, + chapters, + }) +} + +pub fn parse_genre_page( + html: &str, + current_page: u32, +) -> Result<(Vec, Pagination), String> { + let document = parse_html(html); + let mut komik_list = Vec::new(); + + let item_selector = + selector(".bge, article, .ls4, .ls2").ok_or("Selector error".to_string())?; + let title_selector = selector(".kan h3, h3 a, h4 a").ok_or("Selector error".to_string())?; + let img_selector = selector(".bgei img, img.lazy, img").ok_or("Selector error".to_string())?; + let chapter_selector = selector(".new1:last-of-type a span:last-child, .new1 a:last-child span:last-child, .ls4s a, .ls24, .ls2l a") + .ok_or("Selector error".to_string())?; + let score_selector = selector(".up, .numscore, .epx").ok_or("Selector error".to_string())?; + let type_selector = selector(".tpe1_inf, .ls3p, .type").ok_or("Selector error".to_string())?; + let link_selector = + selector(".kan h3 a, .bgei a, h3 a, h4 a, a").ok_or("Selector error".to_string())?; + let next_selector = selector("span[hx-get]").ok_or("Selector error".to_string())?; + let slug_regex = Regex::new(r"/([^/]+)/?$").unwrap(); + + for element in document.select(&item_selector) { + let title = text_from_or(&element, &title_selector, ""); + + let poster = element + .select(&img_selector) + .next() + .and_then(|e| attr(&e, "data-src").or(attr(&e, "src"))) + .unwrap_or_else(|| "".to_string()) + .to_string(); + + let chapter = text_from_or(&element, &chapter_selector, "N/A"); + + let score = text_from_or(&element, &score_selector, "N/A"); + + let komik_type = text_from_or(&element, &type_selector, "Unknown") + .split_whitespace() + .next() + .unwrap_or("Unknown") + .to_string(); + + let komik_url = attr_from_or(&element, &link_selector, "href", ""); + + let slug = slug_regex + .captures(&komik_url) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str()) + .unwrap_or("") + .to_string(); + + if !title.is_empty() { + komik_list.push(KomikItem { + title, + slug, + poster, + chapter, + score, + r#type: komik_type, + komik_url, + }); + } + } + + let has_next_page = document.select(&next_selector).next().is_some(); + let last_visible_page = if has_next_page { + current_page + 1 + } else { + current_page + }; + let pagination = Pagination { + current_page, + last_visible_page, + has_next_page, + next_page: if has_next_page { + Some(current_page + 1) + } else { + None + }, + has_previous_page: current_page > 1, + previous_page: if current_page > 1 { + Some(current_page - 1) + } else { + None + }, + }; + + Ok((komik_list, pagination)) +} diff --git a/src/modules/komik/repository.rs b/src/modules/komik/repository.rs new file mode 100644 index 0000000..5e487d4 --- /dev/null +++ b/src/modules/komik/repository.rs @@ -0,0 +1,78 @@ +use crate::shared::database::traits::scraping_repository::ScrapingRepository; +use crate::shared::errors::AppError; +use crate::shared::utils::fetch_html_with_retry; +use crate::shared::utils::web::scraping_urls::{get_komik_api_url, get_komik_url}; +use async_trait::async_trait; + +pub struct KomikRepository; + +impl Default for KomikRepository { + fn default() -> Self { + Self::new() + } +} + +impl KomikRepository { + pub fn new() -> Self { + Self + } + + pub fn api_url(&self) -> String { + get_komik_api_url() + } + + pub fn base_url(&self) -> String { + get_komik_url() + } + + pub fn genre_url(&self, genre_slug: &str, page: u32) -> String { + if page == 1 { + format!("{}/genre/{}/", self.api_url(), genre_slug) + } else { + format!("{}/genre/{}/page/{}/", self.api_url(), genre_slug, page) + } + } + + pub fn search_url(&self, query: &str, page: u32) -> String { + if page == 1 { + format!("{}/search/{}/", self.api_url(), query) + } else { + format!("{}/search/{}/page/{}/", self.api_url(), query, page) + } + } + + pub fn manga_list_url(&self, page: u32) -> String { + format!("{}/manga/page/{}/?tipe=manga", self.api_url(), page) + } + + pub fn manhua_list_url(&self, page: u32) -> String { + format!("{}/manga/page/{}/?tipe=manhua", self.api_url(), page) + } + + pub fn manhwa_list_url(&self, page: u32) -> String { + format!("{}/manga/page/{}/?tipe=manhwa", self.api_url(), page) + } + + pub fn popular_list_url(&self, page: u32) -> String { + format!( + "{}/manga/page/{}/?orderby=meta_value_num", + self.api_url(), + page + ) + } + + pub fn detail_url(&self, slug: &str) -> String { + format!("{}/manga/{}/", self.base_url(), slug) + } + + pub fn chapter_url(&self, chapter_url: &str) -> String { + format!("{}/{}", self.base_url(), chapter_url) + } +} + +#[async_trait] +impl ScrapingRepository for KomikRepository { + async fn fetch_html(&self, url: &str) -> Result { + fetch_html_with_retry(url).await + } +} diff --git a/src/modules/komik/route.rs b/src/modules/komik/route.rs new file mode 100644 index 0000000..7f207aa --- /dev/null +++ b/src/modules/komik/route.rs @@ -0,0 +1,27 @@ +use std::sync::Arc; + +use axum::{routing::get, Router}; + +use crate::modules::komik::controller; +use crate::shared::state::AppState; + +pub fn routes(router: Router>) -> Router> { + router + .route("/api/komik/genre_list", get(controller::genre_list)) + .route("/api/komik/chapter/{slug}", get(controller::chapter_slug)) + .route("/api/komik/detail/{slug}", get(controller::detail_slug)) + .route("/api/komik/genre/{slug}", get(controller::genre_slug)) + .route( + "/api/komik/genre/{slug}/{page}", + get(controller::genre_slug_page), + ) + .route("/api/komik/manga/{slug}", get(controller::manga_slug)) + .route("/api/komik/manhua/{slug}", get(controller::manhua_slug)) + .route("/api/komik/manhwa/{slug}", get(controller::manhwa_slug)) + .route("/api/komik/popular/{slug}", get(controller::popular_slug)) + .route("/api/komik/search/{slug}", get(controller::search_slug)) + .route( + "/api/komik/search/{slug}/{page}", + get(controller::search_slug_page), + ) +} diff --git a/src/modules/komik/schema.rs b/src/modules/komik/schema.rs new file mode 100644 index 0000000..0a6985c --- /dev/null +++ b/src/modules/komik/schema.rs @@ -0,0 +1,18 @@ +use serde::Deserialize; + +#[derive(Debug, Clone, Deserialize)] +pub struct SlugPath { + pub slug: String, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct SlugPagePath { + pub slug: String, + pub page: String, +} + +#[derive(Deserialize)] +pub struct ChapterQuery { + /// URL-friendly identifier for the chapter (typically the chapter slug or URL path) + pub chapter_url: Option, +} diff --git a/src/modules/komik/service.rs b/src/modules/komik/service.rs new file mode 100644 index 0000000..e5f18a7 --- /dev/null +++ b/src/modules/komik/service.rs @@ -0,0 +1,421 @@ +use std::sync::Arc; + +use crate::modules::komik::parser; +use crate::modules::komik::repository::KomikRepository; +use crate::modules::komik::types::{ + ChapterResponse, DetailResponse, GenreKomikResponse, GenresResponse, SearchKomikResponse, +}; +use crate::shared::database::traits::scraping_repository::ScrapingRepository; +use crate::shared::errors::AppError; +use crate::shared::services::images::cache::{ + apply_cached_posters, cache_image_urls_batch_lazy, get_cached_or_original, +}; +use crate::shared::state::AppState; +use crate::shared::utils::Cache; + +const GENRE_LIST_CACHE_TTL: u64 = 3600; +const GENRE_CACHE_TTL: u64 = 300; +const DETAIL_CACHE_TTL: u64 = 300; +const CHAPTER_CACHE_TTL: u64 = 300; +const SEARCH_CACHE_TTL: u64 = 300; + +pub struct KomikService { + repository: KomikRepository, +} + +impl KomikService { + pub fn new(repository: KomikRepository) -> Self { + Self { repository } + } + + pub async fn genre_list(&self, app_state: Arc) -> Result { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = "komik:genres:list:v3"; + + cache + .get_or_set(cache_key, GENRE_LIST_CACHE_TTL, || async { + let html = self + .repository + .fetch_html(&self.repository.api_url()) + .await + .map_err(|e| e.to_string())?; + let genres = tokio::task::spawn_blocking(move || parser::parse_genres(&html)) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string())?; + + Ok(GenresResponse { + status: "Ok".to_string(), + data: genres, + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn genre_slug( + &self, + genre_slug: String, + app_state: Arc, + ) -> Result { + let page = 1; + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("komik:genre:{}:{}:v2", genre_slug, page); + + cache + .get_or_set(&cache_key, GENRE_CACHE_TTL, || async { + let url = self.repository.genre_url(&genre_slug, page); + let html = self + .repository + .fetch_html(&url) + .await + .map_err(|e| e.to_string())?; + + let (mut komik_list, pagination) = + tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page)) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string())?; + + apply_cached_posters( + &mut komik_list, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(GenreKomikResponse { + status: "Ok".to_string(), + genre: genre_slug.clone(), + data: komik_list, + pagination, + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn genre_slug_page( + &self, + genre_slug: String, + page: u32, + app_state: Arc, + ) -> Result { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("komik:genre:{}:{}:v2", genre_slug, page); + + cache + .get_or_set(&cache_key, GENRE_CACHE_TTL, || async { + let url = self.repository.genre_url(&genre_slug, page); + let html = self + .repository + .fetch_html(&url) + .await + .map_err(|e| e.to_string())?; + + let (mut komik_list, pagination) = + tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page)) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string())?; + + apply_cached_posters( + &mut komik_list, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(GenreKomikResponse { + status: "Ok".to_string(), + genre: genre_slug.clone(), + data: komik_list, + pagination, + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn detail_slug( + &self, + komik_id: String, + app_state: Arc, + ) -> Result { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("komik:detail:{}", komik_id); + + cache + .get_or_set(&cache_key, DETAIL_CACHE_TTL, || async { + let url = self.repository.detail_url(&komik_id); + let html = self + .repository + .fetch_html(&url) + .await + .map_err(|e| e.to_string())?; + + let mut data = + tokio::task::spawn_blocking(move || parser::parse_komik_detail_document(&html)) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string())?; + + if !data.poster.is_empty() { + data.poster = get_cached_or_original( + app_state.db.clone(), + &app_state.redis_pool, + &data.poster, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + } + + Ok(DetailResponse { status: true, data }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn chapter_slug( + &self, + chapter_url: String, + app_state: Arc, + ) -> Result { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("komik:chapter:{}", chapter_url); + + cache + .get_or_set(&cache_key, CHAPTER_CACHE_TTL, || async { + let url = self.repository.chapter_url(&chapter_url); + let html = self + .repository + .fetch_html(&url) + .await + .map_err(|e| e.to_string())?; + + let mut data = tokio::task::spawn_blocking({ + let chapter_url = chapter_url.clone(); + move || parser::parse_komik_chapter_document(&html, &chapter_url) + }) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string())?; + + data.images = cache_image_urls_batch_lazy( + app_state.db.clone(), + &app_state.redis_pool, + data.images, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(ChapterResponse { + message: "Ok".to_string(), + data, + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn manga_slug( + &self, + page_slug: String, + app_state: Arc, + ) -> Result { + let page = page_slug + .parse::() + .map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?; + self.list_by_url( + "manga", + page, + self.repository.manga_list_url(page), + app_state, + ) + .await + } + + pub async fn manhua_slug( + &self, + page_slug: String, + app_state: Arc, + ) -> Result { + let page = page_slug + .parse::() + .map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?; + self.list_by_url( + "manhua", + page, + self.repository.manhua_list_url(page), + app_state, + ) + .await + } + + pub async fn manhwa_slug( + &self, + page_slug: String, + app_state: Arc, + ) -> Result { + let page = page_slug + .parse::() + .map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?; + self.list_by_url( + "manhwa", + page, + self.repository.manhwa_list_url(page), + app_state, + ) + .await + } + + pub async fn popular_slug( + &self, + page_slug: String, + app_state: Arc, + ) -> Result { + let page = page_slug + .parse::() + .map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?; + self.list_by_url( + "popular", + page, + self.repository.popular_list_url(page), + app_state, + ) + .await + } + + async fn list_by_url( + &self, + list_name: &str, + page: u32, + url: String, + app_state: Arc, + ) -> Result { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("komik:list:{}:{}:v2", list_name, page); + + cache + .get_or_set(&cache_key, GENRE_CACHE_TTL, || async { + let html = self + .repository + .fetch_html(&url) + .await + .map_err(|e| e.to_string())?; + + let (mut komik_list, pagination) = + tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page)) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string())?; + + if komik_list.is_empty() { + return Err(format!("Empty komik {} page {}", list_name, page)); + } + + apply_cached_posters( + &mut komik_list, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(GenreKomikResponse { + status: "Ok".to_string(), + genre: list_name.to_string(), + data: komik_list, + pagination, + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn search_slug( + &self, + query: String, + app_state: Arc, + ) -> Result { + let page = 1; + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("komik:search:{}:{}", query, page); + + cache + .get_or_set(&cache_key, SEARCH_CACHE_TTL, || async { + let url = self.repository.search_url(&query, page); + let html = self + .repository + .fetch_html(&url) + .await + .map_err(|e| e.to_string())?; + + let (mut komik_list, pagination) = + tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page)) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string())?; + + apply_cached_posters( + &mut komik_list, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(SearchKomikResponse { + status: "Ok".to_string(), + data: komik_list, + pagination, + }) + }) + .await + .map_err(AppError::ScraperError) + } + + pub async fn search_slug_page( + &self, + query: String, + page: u32, + app_state: Arc, + ) -> Result { + let cache = Cache::new(&app_state.redis_pool); + let cache_key = format!("komik:search:{}:{}", query, page); + + cache + .get_or_set(&cache_key, SEARCH_CACHE_TTL, || async { + let url = self.repository.search_url(&query, page); + let html = self + .repository + .fetch_html(&url) + .await + .map_err(|e| e.to_string())?; + + let (mut komik_list, pagination) = + tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page)) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string())?; + + apply_cached_posters( + &mut komik_list, + app_state.db.clone(), + &app_state.redis_pool, + Some(app_state.image_processing_semaphore.clone()), + ) + .await; + + Ok(SearchKomikResponse { + status: "Ok".to_string(), + data: komik_list, + pagination, + }) + }) + .await + .map_err(AppError::ScraperError) + } +} diff --git a/src/modules/komik/types.rs b/src/modules/komik/types.rs new file mode 100644 index 0000000..463239a --- /dev/null +++ b/src/modules/komik/types.rs @@ -0,0 +1,118 @@ +use crate::shared::types::entities::anime::HasPoster; +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Genre { + pub name: String, + pub slug: String, + pub count: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct GenresResponse { + pub status: String, + pub data: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct ChapterData { + pub title: String, + pub next_chapter_id: String, + pub prev_chapter_id: String, + pub list_chapter: String, + pub images: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct ChapterResponse { + pub message: String, + pub data: ChapterData, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Chapter { + pub chapter: String, + pub date: String, + pub chapter_id: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct DetailData { + pub title: String, + pub poster: String, + pub description: String, + pub status: String, + pub r#type: String, + pub release_date: String, + pub author: String, + pub total_chapter: String, + pub updated_on: String, + pub genres: Vec, + pub chapters: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct DetailResponse { + pub status: bool, + pub data: DetailData, +} + +#[derive(Debug, Deserialize, ToSchema)] +pub struct KomikDetailRequest { + pub komik_id: String, + pub chapter_id: Option, +} + +#[derive(Debug, Serialize, Clone, ToSchema)] +pub enum KomikDetailEvent { + Chapter(Chapter), + Detail(DetailData), + Error(String), + EndOfStream, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct KomikItem { + pub title: String, + pub slug: String, + pub poster: String, + pub chapter: String, + pub score: String, + pub r#type: String, + pub komik_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Pagination { + pub current_page: u32, + pub last_visible_page: u32, + pub has_next_page: bool, + pub next_page: Option, + pub has_previous_page: bool, + pub previous_page: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct GenreKomikResponse { + pub status: String, + pub genre: String, + pub data: Vec, + pub pagination: Pagination, +} + +impl HasPoster for KomikItem { + fn poster(&self) -> &str { + &self.poster + } + fn set_poster(&mut self, url: String) { + self.poster = url; + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct SearchKomikResponse { + pub status: String, + pub data: Vec, + pub pagination: Pagination, +} diff --git a/src/modules/mod.rs b/src/modules/mod.rs new file mode 100644 index 0000000..0bd4033 --- /dev/null +++ b/src/modules/mod.rs @@ -0,0 +1,17 @@ +use axum::Router; +use std::sync::Arc; + +use crate::shared::state::AppState; + +pub mod anime; +pub mod anime2; +pub mod komik; +pub mod proxy; + +pub fn routes(router: Router>) -> Router> { + let router = anime::route::routes(router); + let router = anime2::route::routes(router); + let router = komik::route::routes(router); + let router = proxy::route::routes(router); + router +} diff --git a/src/modules/proxy/controller.rs b/src/modules/proxy/controller.rs new file mode 100644 index 0000000..18780ca --- /dev/null +++ b/src/modules/proxy/controller.rs @@ -0,0 +1,75 @@ +use crate::modules::proxy::repository::ProxyRepository; +use crate::modules::proxy::schema::{AuditImageCacheRequest, ImageCacheRequest, ProxyParams}; +use crate::modules::proxy::service::ProxyService; +use crate::modules::proxy::types::{AuditImageCacheResponse, ImageCacheResponse}; +use crate::shared::database::repositories::image_cache::SeaOrmImageCacheRepository; +use crate::shared::errors::AppError; +use crate::shared::state::AppState; +use axum::extract::{Json, Query, State}; +use axum::response::Response; +use std::sync::Arc; + +fn make_service(state: &Arc) -> ProxyService { + let repo = Arc::new(SeaOrmImageCacheRepository::new( + state.db.clone(), + state.redis_pool.clone(), + )); + ProxyService::new(ProxyRepository::new(), repo) +} + +#[utoipa::path( + get, + path = "/api/proxy/croxy", + tag = "proxy", + operation_id = "proxy_croxy", + params(ProxyParams), + responses( + (status = 200, description = "Handles GET requests for the /api/proxy/croxy endpoint.", body = serde_json::Value), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn fetch_with_proxy_only( + State(state): State>, + Query(params): Query, +) -> Result { + make_service(&state).fetch_with_proxy_only(params).await +} + +#[utoipa::path( + post, + path = "/api/proxy/image-cache", + tag = "proxy", + operation_id = "proxy_image_cache", + request_body = ImageCacheRequest, + responses( + (status = 200, description = "Cache an image to CDN and return the cached URL", body = ImageCacheResponse), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn image_cache( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + make_service(&state).image_cache(state, req).await.map(Json) +} + +#[utoipa::path( + post, + path = "/api/proxy/image-cache/audit", + tag = "proxy", + operation_id = "proxy_image_cache_audit", + request_body = AuditImageCacheRequest, + responses( + (status = 200, description = "Audit an image cache entry", body = AuditImageCacheResponse), + (status = 500, description = "Internal Server Error", body = String) + ) +)] +pub async fn audit_image_cache( + State(state): State>, + Json(req): Json, +) -> Result, AppError> { + make_service(&state) + .audit_image_cache(state, req) + .await + .map(Json) +} diff --git a/src/modules/proxy/mod.rs b/src/modules/proxy/mod.rs new file mode 100644 index 0000000..67dc224 --- /dev/null +++ b/src/modules/proxy/mod.rs @@ -0,0 +1,7 @@ +pub mod controller; +pub mod parser; +pub mod repository; +pub mod route; +pub mod schema; +pub mod service; +pub mod types; diff --git a/src/modules/proxy/parser.rs b/src/modules/proxy/parser.rs new file mode 100644 index 0000000..b1f7bc0 --- /dev/null +++ b/src/modules/proxy/parser.rs @@ -0,0 +1 @@ +// Proxy endpoints do not parse HTML or structured upstream payloads. diff --git a/src/modules/proxy/repository.rs b/src/modules/proxy/repository.rs new file mode 100644 index 0000000..84f5e9a --- /dev/null +++ b/src/modules/proxy/repository.rs @@ -0,0 +1,29 @@ +use crate::shared::database::traits::scraping_repository::ScrapingRepository; +use crate::shared::errors::AppError; +use crate::shared::utils::web::proxy_fetch::{self, FetchResult}; +use async_trait::async_trait; + +pub struct ProxyRepository; + +impl Default for ProxyRepository { + fn default() -> Self { + Self::new() + } +} + +impl ProxyRepository { + pub fn new() -> Self { + Self + } + + pub async fn fetch_with_proxy_url(&self, url: &str) -> Result { + proxy_fetch::fetch_with_proxy(url).await + } +} + +#[async_trait] +impl ScrapingRepository for ProxyRepository { + async fn fetch_html(&self, url: &str) -> Result { + self.fetch_with_proxy_url(url).await.map(|r| r.data) + } +} diff --git a/src/modules/proxy/route.rs b/src/modules/proxy/route.rs new file mode 100644 index 0000000..78c4f8e --- /dev/null +++ b/src/modules/proxy/route.rs @@ -0,0 +1,16 @@ +use crate::modules::proxy::controller; +use crate::shared::state::AppState; +use axum::Router; +use std::sync::Arc; + +pub fn routes(router: Router>) -> Router> { + router + .route( + "/api/proxy/croxy", + axum::routing::get(controller::fetch_with_proxy_only), + ) + .route( + "/api/proxy/image-cache", + axum::routing::post(controller::image_cache), + ) +} diff --git a/src/modules/proxy/schema.rs b/src/modules/proxy/schema.rs new file mode 100644 index 0000000..4ddb84f --- /dev/null +++ b/src/modules/proxy/schema.rs @@ -0,0 +1,27 @@ +use serde::Deserialize; +use utoipa::IntoParams; +use utoipa::ToSchema; + +/// Query parameters for proxy fetch (GET) +#[derive(Debug, Deserialize, ToSchema, IntoParams)] +pub struct ProxyParams { + /// URL to fetch via proxy + pub url: String, +} + +/// Request body for image cache (POST) +#[derive(Debug, Deserialize, ToSchema)] +pub struct ImageCacheRequest { + /// Original image URL to cache + pub url: String, + /// If true, returns original URL immediately and caches in background + #[serde(default)] + pub lazy: bool, +} + +/// Request body for auditing image cache (POST) +#[derive(Debug, Deserialize, ToSchema)] +pub struct AuditImageCacheRequest { + /// Original image URL to audit + pub url: String, +} diff --git a/src/modules/proxy/service.rs b/src/modules/proxy/service.rs new file mode 100644 index 0000000..3cf39ed --- /dev/null +++ b/src/modules/proxy/service.rs @@ -0,0 +1,232 @@ +use crate::modules::proxy::repository::ProxyRepository; +use crate::modules::proxy::schema::{AuditImageCacheRequest, ImageCacheRequest, ProxyParams}; +use crate::modules::proxy::types::{AuditImageCacheResponse, ImageCacheResponse}; +use crate::shared::config::CONFIG; +use crate::shared::database::traits::image_cache::ImageCacheRepository; +use crate::shared::errors::AppError; +use crate::shared::events::bus::ImageRepaired; +use crate::shared::services::images::cache::ImageCache; +use crate::shared::state::AppState; +use axum::http::StatusCode; +use axum::response::Response; +use std::sync::Arc; +use tracing::{error, info, warn}; + +pub struct ProxyService { + repository: ProxyRepository, + image_cache_repo: Arc, +} + +impl ProxyService { + pub fn new( + repository: ProxyRepository, + image_cache_repo: Arc, + ) -> Self { + Self { + repository, + image_cache_repo, + } + } + + fn build_image_cache(&self) -> ImageCache { + ImageCache::new(self.image_cache_repo.clone()) + } + + pub async fn fetch_with_proxy_only(&self, params: ProxyParams) -> Result { + let url = params.url; + match self.repository.fetch_with_proxy_url(&url).await { + Ok(fetch_result) => { + let mut builder = Response::builder().status(StatusCode::OK); + if let Some(content_type) = fetch_result.content_type { + builder = builder.header("Content-Type", content_type); + } + Ok(builder.body(fetch_result.data.into())?) + } + Err(e) => { + error!("Proxy fetch error: {:?}", e); + Err(AppError::Other(format!( + "Failed to fetch URL via proxy: {}", + e + ))) + } + } + } + + pub async fn image_cache( + &self, + state: Arc, + req: ImageCacheRequest, + ) -> Result { + let cache = self + .build_image_cache() + .with_semaphore(state.image_processing_semaphore.clone()); + + if let Some(cdn_url) = cache.get_cdn_url(&req.url).await { + return Ok(ImageCacheResponse { + success: true, + original_url: req.url, + cdn_url, + from_cache: true, + pending: None, + }); + } + + if req.lazy { + let url = req.url.clone(); + let repo = self.image_cache_repo.clone(); + let semaphore = state.image_processing_semaphore.clone(); + tokio::spawn(async move { + let cache = ImageCache::new(repo).with_semaphore(semaphore); + match cache.get_or_cache(&url).await { + Ok(cdn) => info!("[LazyCache] Cached {} -> {}", url, cdn), + Err(e) => warn!("[LazyCache] Failed {}: {}", url, e), + } + }); + return Ok(ImageCacheResponse { + success: true, + original_url: req.url.clone(), + cdn_url: req.url, + from_cache: false, + pending: Some(true), + }); + } + + match cache.get_or_cache(&req.url).await { + Ok(cdn_url) => Ok(ImageCacheResponse { + success: true, + original_url: req.url, + cdn_url, + from_cache: false, + pending: None, + }), + Err(e) => { + error!("ImageCache error: {}", e); + Ok(ImageCacheResponse { + success: false, + original_url: req.url.clone(), + cdn_url: req.url, + from_cache: false, + pending: None, + }) + } + } + } + + pub async fn audit_image_cache( + &self, + state: Arc, + req: AuditImageCacheRequest, + ) -> Result { + let cache = self.build_image_cache(); + + let mut cdn_opt = cache.get_cdn_url(&req.url).await; + let mut original = req.url.clone(); + if cdn_opt.is_none() { + if let Some(orig) = cache.find_original_from_cdn(&req.url).await { + info!( + "SmartAudit: {} recognized as CDN, original {}", + req.url, orig + ); + original = orig; + cdn_opt = Some(req.url.clone()); + } + } + + if let Some(cdn_url) = cdn_opt { + let client = crate::shared::utils::web::http_client::http_client().client(); + let mut accessible = false; + match client.get(cdn_url.clone()).send().await { + Ok(resp) if resp.status().is_success() => { + if let Ok(bytes) = resp.bytes().await { + if infer::get(&bytes) + .map(|k| k.mime_type().starts_with("image/")) + .unwrap_or(false) + { + accessible = true; + } else { + warn!("CDN {} returned non-image content", cdn_url); + } + } + } + Ok(resp) => warn!("CDN {} status {}", cdn_url, resp.status()), + Err(e) => warn!("CDN {} fetch error {}", cdn_url, e), + } + if accessible { + return Ok(AuditImageCacheResponse { + success: true, + original_url: original, + cdn_url: Some(cdn_url), + was_accessible: true, + re_uploaded: false, + message: "CDN URL is accessible and the image is valid".to_string(), + }); + } + info!("CDN {} inaccessible, purging and reuploading", cdn_url); + let picser_delete_url = &CONFIG.urls.picser_api_url; + if let Some(filename) = cdn_url.split('/').last() { + let payload = serde_json::json!({ "filename": filename }); + match client.delete(picser_delete_url).json(&payload).send().await { + Ok(r) if r.status().is_success() => info!("Deleted {} via Picser", filename), + Ok(r) => warn!("Picser delete {} status {}", filename, r.status()), + Err(e) => warn!("Picser delete error {}: {}", filename, e), + } + } + let _ = cache.invalidate(&original).await; + match cache.get_or_cache(&original).await { + Ok(new_cdn) => { + state + .event_bus + .publish(ImageRepaired { + original_url: original.clone(), + cdn_url: new_cdn.clone(), + }) + .await; + Ok(AuditImageCacheResponse { + success: true, + original_url: original, + cdn_url: Some(new_cdn), + was_accessible: false, + re_uploaded: true, + message: "CDN URL was inaccessible, re-uploaded".to_string(), + }) + } + Err(e) => Ok(AuditImageCacheResponse { + success: false, + original_url: original, + cdn_url: None, + was_accessible: false, + re_uploaded: false, + message: format!("Re-upload failed: {}", e), + }), + } + } else { + match cache.get_or_cache(&original).await { + Ok(new_cdn) => { + state + .event_bus + .publish(ImageRepaired { + original_url: original.clone(), + cdn_url: new_cdn.clone(), + }) + .await; + Ok(AuditImageCacheResponse { + success: true, + original_url: original, + cdn_url: Some(new_cdn), + was_accessible: false, + re_uploaded: true, + message: "Cached newly".to_string(), + }) + } + Err(e) => Ok(AuditImageCacheResponse { + success: false, + original_url: original, + cdn_url: None, + was_accessible: false, + re_uploaded: false, + message: format!("Cache failed: {}", e), + }), + } + } + } +} diff --git a/src/modules/proxy/types.rs b/src/modules/proxy/types.rs new file mode 100644 index 0000000..8e619f0 --- /dev/null +++ b/src/modules/proxy/types.rs @@ -0,0 +1,24 @@ +use serde::Serialize; +use utoipa::ToSchema; + +/// Response for image cache POST +#[derive(Debug, Serialize, ToSchema)] +pub struct ImageCacheResponse { + pub success: bool, + pub original_url: String, + pub cdn_url: String, + pub from_cache: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub pending: Option, +} + +/// Response for image cache audit POST +#[derive(Debug, Serialize, ToSchema)] +pub struct AuditImageCacheResponse { + pub success: bool, + pub original_url: String, + pub cdn_url: Option, + pub was_accessible: bool, + pub re_uploaded: bool, + pub message: String, +} diff --git a/src/shared/browser/mod.rs b/src/shared/browser/mod.rs new file mode 100644 index 0000000..52bd506 --- /dev/null +++ b/src/shared/browser/mod.rs @@ -0,0 +1,3 @@ +pub mod pool; + +pub use pool::{init_browser_pool, BrowserPoolConfig}; diff --git a/src/shared/browser/pool.rs b/src/shared/browser/pool.rs new file mode 100644 index 0000000..c69d5c1 --- /dev/null +++ b/src/shared/browser/pool.rs @@ -0,0 +1,483 @@ +//! Browser pool for managing a single remote browser with multiple tabs. +//! +//! This pool maintains a connection to a remote Chrome DevTools Protocol (CDP) +//! endpoint and provides tabs on-demand for scraping. Tabs are returned to the +//! pool after use. Requires EXTERNAL_BROWSERLESS_WS or CHROME_REMOTE_WS to be set. + +use reqwest::StatusCode; +use serde_json::json; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use tokio::sync::{Mutex, Semaphore}; +use tracing::{debug, info, warn}; + +fn build_http_endpoint(remote_url: &str, path: &str) -> anyhow::Result { + let mut url = reqwest::Url::parse(remote_url) + .map_err(|e| anyhow::anyhow!("Invalid remote browser URL '{}': {}", remote_url, e))?; + + match url.scheme() { + "ws" => { + url.set_scheme("http") + .map_err(|_| anyhow::anyhow!("Failed to convert scheme from ws to http"))?; + } + "wss" => { + url.set_scheme("https") + .map_err(|_| anyhow::anyhow!("Failed to convert scheme from wss to https"))?; + } + "http" | "https" => {} + scheme => { + return Err(anyhow::anyhow!( + "Unsupported browser URL scheme '{}'. Expected ws:// or wss://", + scheme + )); + } + } + + let normalized = path.trim_start_matches('/'); + url.set_path(normalized); + + Ok(url.to_string()) +} + +/// Configuration for the browser pool. +#[derive(Debug, Clone)] +pub struct BrowserPoolConfig { + /// Remote Chrome DevTools Protocol WebSocket URL (required). + /// Must be set via `EXTERNAL_BROWSERLESS_WS` or `CHROME_REMOTE_WS` environment variable. + /// Takes precedence from: EXTERNAL_BROWSERLESS_WS → CHROME_REMOTE_WS → None + pub remote_websocket_url: String, + /// Maximum number of concurrent tabs + pub max_tabs: usize, +} + +impl Default for BrowserPoolConfig { + fn default() -> Self { + // Priority (highest → lowest): + // 1. EXTERNAL_BROWSERLESS_WS — explicit operator override, always wins + // 2. CHROME_REMOTE_WS — may be injected by Coolify/Docker networking; + // rejected when it resolves to the unroutable Docker alias + let external = std::env::var("EXTERNAL_BROWSERLESS_WS").ok(); + let chrome_remote = std::env::var("CHROME_REMOTE_WS").ok(); + + let remote_websocket_url = if let Some(ext) = external { + tracing::info!("🌐 Browser: using EXTERNAL_BROWSERLESS_WS"); + ext + } else if let Some(ref cr) = chrome_remote { + if cr == "ws://browserless:3000" { + eprintln!( + "CHROME_REMOTE_WS is set to the unroutable Docker alias \"ws://browserless:3000\" \ + and EXTERNAL_BROWSERLESS_WS is unset. Remote browser connectivity is required." + ); + std::process::exit(1); + } else { + tracing::info!("🌐 Browser: using CHROME_REMOTE_WS"); + cr.clone() + } + } else { + eprintln!( + "Remote browser URL not configured. Set EXTERNAL_BROWSERLESS_WS or CHROME_REMOTE_WS environment variable." + ); + std::process::exit(1); + }; + + Self { + remote_websocket_url, + max_tabs: 10, + } + } +} + +/// A pool of browser tabs backed by a single remote browser instance. +/// +/// # Example +/// +/// ```ignore +/// use scraper_service::browser::{BrowserPool, BrowserPoolConfig}; +/// +/// let pool = BrowserPool::new(BrowserPoolConfig::default()).await?; +/// let tab = pool.get_tab().await?; +/// tab.goto("https://example.com").await?; +/// let html = tab.content().await?; +/// ``` +pub struct BrowserPool { + /// Available (idle) tab IDs + available_tabs: Mutex>, + /// Semaphore to limit concurrent tabs + semaphore: Arc, + /// Configuration + config: BrowserPoolConfig, + /// Counter for generating unique tab IDs + tab_counter: AtomicU64, +} + +impl BrowserPool { + /// Create a new browser pool connecting to a remote CDP endpoint. + /// + /// The pool will attempt to connect to the remote browser URL specified in + /// BrowserPoolConfig. If the connection fails, an error is returned. + pub async fn new(config: BrowserPoolConfig) -> anyhow::Result> { + info!( + "Initializing browser pool (remote: {}) with max {} tabs", + config.remote_websocket_url, config.max_tabs + ); + + // Verify connection to remote browser with a simple health check + Self::verify_remote_connection(&config).await?; + + let pool = Arc::new(Self { + available_tabs: Mutex::new(Vec::new()), + semaphore: Arc::new(Semaphore::new(config.max_tabs)), + config: config.clone(), + tab_counter: AtomicU64::new(0), + }); + + info!("Browser pool initialized (remote-only)"); + Ok(pool) + } + + /// Verify connection to the remote browser. + async fn verify_remote_connection(config: &BrowserPoolConfig) -> anyhow::Result<()> { + let client = reqwest::Client::new(); + + // Browserless/CDP deployments vary by exposed endpoint. + // Try multiple known paths and succeed on any valid HTTP response from the host. + let check_paths = ["json/version", "health", ""]; + let mut last_non_success: Option<(String, StatusCode)> = None; + + for path in check_paths { + let endpoint = build_http_endpoint(&config.remote_websocket_url, path)?; + let response = tokio::time::timeout( + std::time::Duration::from_secs(5), + client.get(&endpoint).send(), + ) + .await; + + match response { + Ok(Ok(r)) if r.status().is_success() => { + info!("Remote browser health check passed via {}", endpoint); + return Ok(()); + } + Ok(Ok(r)) + if r.status() == StatusCode::UNAUTHORIZED + || r.status() == StatusCode::FORBIDDEN => + { + return Err(anyhow::anyhow!( + "Remote browser reachable but authentication failed at {}: {}", + endpoint, + r.status() + )); + } + Ok(Ok(r)) => { + last_non_success = Some((endpoint, r.status())); + } + Ok(Err(e)) => { + return Err(anyhow::anyhow!( + "Failed to connect to remote browser: {}", + e + )); + } + Err(_) => { + return Err(anyhow::anyhow!("Remote browser connection timeout")); + } + } + } + + if let Some((endpoint, status)) = last_non_success { + warn!( + "Remote browser reachable but no known health endpoint succeeded (last: {} -> {})", + endpoint, status + ); + } + + Ok(()) + } + + /// Get a tab from the pool. + /// + /// This will reuse an existing idle tab or create a new one. + /// The returned `PooledTab` automatically returns to the pool when dropped. + pub async fn get_tab(self: &Arc) -> anyhow::Result { + // Acquire semaphore permit (limits concurrent tabs) + let permit = self.semaphore.clone().acquire_owned().await?; + + // Try to reuse an existing tab ID, or generate a new one + let tab_id = { + let mut tabs = self.available_tabs.lock().await; + tabs.pop().unwrap_or_else(|| { + let id = self.tab_counter.fetch_add(1, Ordering::SeqCst); + format!("tab-{}", id) + }) + }; + + debug!("Allocated tab: {}", tab_id); + + Ok(PooledTab { + tab_id, + cdp_url: self.config.remote_websocket_url.clone(), + pool: Arc::clone(self), + _permit: permit, + }) + } + + /// Get the number of available (idle) tabs in the pool. + pub async fn available_count(&self) -> usize { + self.available_tabs.lock().await.len() + } + + /// Close the browser pool. + pub async fn close(&self) -> anyhow::Result<()> { + info!("Closing browser pool"); + self.available_tabs.lock().await.clear(); + Ok(()) + } +} + +/// A tab borrowed from the pool (remote CDP-based). +/// +/// This is a lightweight wrapper that communicates with a remote Chrome instance +/// via HTTP calls to the browser service. The tab is returned to the pool when dropped. +pub struct PooledTab { + /// Unique identifier for this tab + pub tab_id: String, + /// Remote CDP/browser service URL + cdp_url: String, + /// Pool reference for returning on drop + pool: Arc, + /// Semaphore permit (released on drop) + _permit: tokio::sync::OwnedSemaphorePermit, +} + +impl PooledTab { + /// Navigate to a URL. + pub async fn goto(&self, url: &str) -> anyhow::Result<()> { + let client = reqwest::Client::new(); + let timeout = std::time::Duration::from_secs(15); + let endpoint = build_http_endpoint(&self.cdp_url, "goto")?; + + let response = tokio::time::timeout( + timeout, + client.post(&endpoint).json(&json!({ "url": url })).send(), + ) + .await + .map_err(|_| anyhow::anyhow!("Timeout navigating to {}", url))? + .map_err(|e| anyhow::anyhow!("Failed to navigate to {}: {}", url, e))?; + + if response.status().is_success() { + Ok(()) + } else { + Err(anyhow::anyhow!( + "Failed to navigate to {}: {}", + url, + response.status() + )) + } + } + + /// Get the page content (HTML). + pub async fn content(&self) -> anyhow::Result { + let client = reqwest::Client::new(); + let timeout = std::time::Duration::from_secs(15); + let endpoint = build_http_endpoint(&self.cdp_url, "content")?; + + let response = tokio::time::timeout(timeout, client.post(&endpoint).send()) + .await + .map_err(|_| anyhow::anyhow!("Timeout getting page content"))? + .map_err(|e| anyhow::anyhow!("Failed to get page content: {}", e))?; + + response + .text() + .await + .map_err(|e| anyhow::anyhow!("Failed to read response body: {}", e)) + } + + /// Execute JavaScript and return the result. + pub async fn evaluate( + &self, + expression: &str, + ) -> anyhow::Result { + let client = reqwest::Client::new(); + let timeout = std::time::Duration::from_secs(15); + let endpoint = build_http_endpoint(&self.cdp_url, "evaluate")?; + + let response = tokio::time::timeout( + timeout, + client + .post(&endpoint) + .json(&json!({ "expression": expression })) + .send(), + ) + .await + .map_err(|_| anyhow::anyhow!("Timeout evaluating JS"))? + .map_err(|e| anyhow::anyhow!("Failed to evaluate JS: {}", e))?; + + let data: serde_json::Value = response + .json() + .await + .map_err(|e| anyhow::anyhow!("Failed to parse JS result: {}", e))?; + + serde_json::from_value(data).map_err(|e| anyhow::anyhow!("Invalid JS result: {}", e)) + } + + /// Wait for a selector to appear. + pub async fn wait_for_selector(&self, selector: &str) -> anyhow::Result<()> { + let client = reqwest::Client::new(); + let timeout = std::time::Duration::from_secs(15); + let endpoint = build_http_endpoint(&self.cdp_url, "waitForSelector")?; + + let response = tokio::time::timeout( + timeout, + client + .post(&endpoint) + .json(&json!({ "selector": selector })) + .send(), + ) + .await + .map_err(|_| anyhow::anyhow!("Timeout waiting for selector '{}'", selector))? + .map_err(|e| anyhow::anyhow!("Selector '{}' error: {}", selector, e))?; + + if response.status().is_success() { + Ok(()) + } else { + Err(anyhow::anyhow!( + "Selector '{}' not found: {}", + selector, + response.status() + )) + } + } + + /// Click an element by selector. + pub async fn click(&self, selector: &str) -> anyhow::Result<()> { + let client = reqwest::Client::new(); + let timeout = std::time::Duration::from_secs(15); + let endpoint = build_http_endpoint(&self.cdp_url, "click")?; + + let response = tokio::time::timeout( + timeout, + client + .post(&endpoint) + .json(&json!({ "selector": selector })) + .send(), + ) + .await + .map_err(|_| anyhow::anyhow!("Timeout clicking element"))? + .map_err(|e| anyhow::anyhow!("Click error: {}", e))?; + + if response.status().is_success() { + Ok(()) + } else { + Err(anyhow::anyhow!( + "Failed to click '{}': {}", + selector, + response.status() + )) + } + } + + /// Type text into an element. + pub async fn type_text(&self, selector: &str, text: &str) -> anyhow::Result<()> { + let client = reqwest::Client::new(); + let timeout = std::time::Duration::from_secs(15); + let endpoint = build_http_endpoint(&self.cdp_url, "type")?; + + let response = tokio::time::timeout( + timeout, + client + .post(&endpoint) + .json(&json!({ "selector": selector, "text": text })) + .send(), + ) + .await + .map_err(|_| anyhow::anyhow!("Timeout typing text"))? + .map_err(|e| anyhow::anyhow!("Type error: {}", e))?; + + if response.status().is_success() { + Ok(()) + } else { + Err(anyhow::anyhow!( + "Failed to type into '{}': {}", + selector, + response.status() + )) + } + } + + /// Take a screenshot as PNG bytes. + pub async fn screenshot(&self) -> anyhow::Result> { + let client = reqwest::Client::new(); + let timeout = std::time::Duration::from_secs(15); + let endpoint = build_http_endpoint(&self.cdp_url, "screenshot")?; + + let response = tokio::time::timeout( + timeout, + client + .post(&endpoint) + .json(&json!({ "fullPage": true })) + .send(), + ) + .await + .map_err(|_| anyhow::anyhow!("Timeout taking screenshot"))? + .map_err(|e| anyhow::anyhow!("Failed to take screenshot: {}", e))?; + + response + .bytes() + .await + .map(|b| b.to_vec()) + .map_err(|e| anyhow::anyhow!("Failed to read screenshot bytes: {}", e)) + } + + /// Get the current URL. + pub async fn url(&self) -> anyhow::Result { + let client = reqwest::Client::new(); + let timeout = std::time::Duration::from_secs(10); + let endpoint = build_http_endpoint(&self.cdp_url, "url")?; + + let response = tokio::time::timeout(timeout, client.post(&endpoint).send()) + .await + .map_err(|_| anyhow::anyhow!("Timeout getting URL"))? + .map_err(|e| anyhow::anyhow!("Failed to get URL: {}", e))?; + + response + .text() + .await + .map_err(|e| anyhow::anyhow!("Failed to read URL: {}", e)) + } +} + +impl Drop for PooledTab { + fn drop(&mut self) { + let pool = Arc::clone(&self.pool); + let tab_id = self.tab_id.clone(); + + let rt = tokio::runtime::Handle::try_current(); + if let Ok(handle) = rt { + handle.spawn(async move { + // Return tab ID to available pool for reuse + pool.available_tabs.lock().await.push(tab_id); + }); + } else { + warn!("Cannot return tab: no tokio runtime available"); + } + } +} + +// Global browser pool instance +use once_cell::sync::OnceCell; + +static BROWSER_POOL: OnceCell> = OnceCell::new(); + +/// Initialize the global browser pool. +/// Call this once at application startup. +pub async fn init_browser_pool(config: BrowserPoolConfig) -> anyhow::Result<()> { + let pool = BrowserPool::new(config).await?; + BROWSER_POOL + .set(pool) + .map_err(|_| anyhow::anyhow!("Browser pool already initialized"))?; + Ok(()) +} + +/// Get the global browser pool. +/// Returns None if not initialized. +pub fn get_browser_pool() -> Option> { + BROWSER_POOL.get().cloned() +} diff --git a/src/shared/config/mod.rs b/src/shared/config/mod.rs new file mode 100644 index 0000000..5da118d --- /dev/null +++ b/src/shared/config/mod.rs @@ -0,0 +1,289 @@ +//! Type-safe application configuration. +//! +//! This module provides a strongly-typed configuration system that: +//! - Loads from environment variables and optional TOML files +//! - Fails fast at startup if required variables are missing +//! - Supports hierarchical configuration (default -> environment-specific) + +use config::{Config, ConfigError, Environment, File}; +use once_cell::sync::Lazy; +use serde::Deserialize; +use std::env; + +/// Application configuration loaded at startup. +/// All fields are required unless marked as `Option`. +#[derive(Debug, Clone, Deserialize)] +pub struct AppConfig { + /// Database connection URL (PostgreSQL) + pub database_url: String, + + /// Secret key for JWT signing + pub jwt_secret: String, + + /// Redis connection URL + #[serde(default)] + pub redis_url: String, + + /// Server port to bind to + #[serde(default = "default_port")] + pub server_port: u16, + + /// Environment (development, staging, production) + #[serde(default = "default_env")] + pub environment: String, + + /// Allowed CORS origins (comma-separated) + #[serde(default)] + pub cors_origins: Vec, + + /// Log level (trace, debug, info, warn, error) + #[serde(default = "default_log_level")] + pub log_level: String, + + /// SMTP configuration for emails (optional) + pub smtp: Option, + + /// Database pool configuration + #[serde(default)] + pub db: DbConfig, + + /// Max concurrent image processing tasks + #[serde(default = "default_image_processing_concurrency")] + pub image_processing_concurrency: usize, + + /// Domain/URL values that may change between deployments + #[serde(default)] + pub urls: UrlConfig, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct UrlConfig { + #[serde(default = "default_site_url")] + pub site_url: String, + #[serde(default = "default_picser_api_url")] + pub picser_api_url: String, + #[serde(default = "default_fallback_upload_api_url")] + pub fallback_upload_api_url: String, +} + +impl Default for UrlConfig { + fn default() -> Self { + Self { + site_url: default_site_url(), + picser_api_url: default_picser_api_url(), + fallback_upload_api_url: default_fallback_upload_api_url(), + } + } +} + +#[derive(Debug, Clone, Deserialize)] +pub struct DbConfig { + #[serde(default = "default_db_max_connections")] + pub max_connections: u32, + #[serde(default = "default_db_min_connections")] + pub min_connections: u32, + #[serde(default = "default_db_connect_timeout")] + pub connect_timeout_seconds: u64, + #[serde(default = "default_db_idle_timeout")] + pub idle_timeout_seconds: u64, + #[serde(default = "default_db_acquire_timeout")] + pub acquire_timeout_seconds: u64, + #[serde(default = "default_db_max_lifetime")] + pub max_lifetime_seconds: u64, +} + +impl Default for DbConfig { + fn default() -> Self { + Self { + max_connections: default_db_max_connections(), + min_connections: default_db_min_connections(), + connect_timeout_seconds: default_db_connect_timeout(), + idle_timeout_seconds: default_db_idle_timeout(), + acquire_timeout_seconds: default_db_acquire_timeout(), + max_lifetime_seconds: default_db_max_lifetime(), + } + } +} + +/// SMTP configuration for sending emails +#[derive(Debug, Clone, Deserialize)] +pub struct SmtpConfig { + pub host: String, + pub port: u16, + pub username: String, + pub password: String, + pub from_email: String, + pub from_name: String, +} + +/// MinIO/S3-compatible storage configuration +#[derive(Debug, Clone)] +pub struct MinioConfig { + /// MinIO endpoint URL (e.g., "https://cdn.asepharyana.my.id") + pub endpoint: String, + /// Bucket name + pub bucket_name: String, + /// Access key / username + pub access_key: String, + /// Secret key / password + pub secret_key: String, + /// Use HTTPS (true) or HTTP (false) + pub secure: bool, + /// AWS region (default: us-east-1) + pub region: String, + /// Public URL for serving files (optional) + pub public_url: Option, + /// Prefix for avatar files (e.g., "avatars") + pub avatar_prefix: String, +} + +impl MinioConfig { + /// Load MinIO configuration from environment variables + pub fn from_env() -> Option { + let endpoint = env::var("MINIO_ENDPOINT").ok()?; + let bucket_name = env::var("MINIO_BUCKET_NAME").ok()?; + let access_key = env::var("MINIO_ACCESS_KEY").ok()?; + let secret_key = env::var("MINIO_SECRET_KEY").ok()?; + + let secure = env::var("MINIO_SECURE") + .map(|v| v.to_lowercase() == "true") + .unwrap_or(true); + + let region = env::var("MINIO_REGION").unwrap_or_else(|_| "us-east-1".to_string()); + + let public_url = env::var("MINIO_PUBLIC_URL").ok(); + + let avatar_prefix = + env::var("MINIO_AVATAR_PREFIX").unwrap_or_else(|_| "avatars".to_string()); + + Some(Self { + endpoint, + bucket_name, + access_key, + secret_key, + secure, + region, + public_url, + avatar_prefix, + }) + } +} + +fn default_port() -> u16 { + 4091 +} + +fn default_env() -> String { + "development".to_string() +} + +fn default_log_level() -> String { + "info".to_string() +} + +fn default_image_processing_concurrency() -> usize { + 5 +} + +fn default_site_url() -> String { + "https://asepharyana.my.id".to_string() +} + +fn default_picser_api_url() -> String { + "https://picser.asepharyana.my.id/api/upload".to_string() +} + +fn default_fallback_upload_api_url() -> String { + "https://upload.asepharyana.my.id/api/upload".to_string() +} + +fn default_db_max_connections() -> u32 { + 100 +} + +fn default_db_min_connections() -> u32 { + 10 +} + +fn default_db_connect_timeout() -> u64 { + 5 +} + +fn default_db_idle_timeout() -> u64 { + 300 +} + +fn default_db_acquire_timeout() -> u64 { + 10 +} + +fn default_db_max_lifetime() -> u64 { + 1800 +} + +impl AppConfig { + /// Load configuration from environment and optional config files. + /// + /// Priority (highest to lowest): + /// 1. Environment variables (prefixed with APP_) + /// 2. `config/{environment}.toml` + /// 3. `config/default.toml` + pub fn load() -> Result { + // Load .env file first + if let Err(e) = dotenvy::dotenv() { + tracing::debug!("Could not load .env file: {}", e); + } + + let run_mode = env::var("RUN_MODE").unwrap_or_else(|_| "development".into()); + + let config = Config::builder() + // Start with default config file + .add_source(File::with_name("config/default").required(false)) + // Layer on environment-specific values + .add_source(File::with_name(&format!("config/{}", run_mode)).required(false)) + // Add environment variables (with APP_ prefix) + .add_source( + Environment::with_prefix("APP") + .separator("__") + .try_parsing(true) + .list_separator(","), + ) + // Map legacy env vars to new config structure + .set_override_option("database_url", env::var("DATABASE_URL").ok())? + .set_override_option("jwt_secret", env::var("JWT_SECRET").ok())? + .set_override_option("redis_url", env::var("REDIS_URL").ok())? + .build()?; + + config.try_deserialize() + } + + /// Check if running in production mode + pub fn is_production(&self) -> bool { + self.environment == "production" + } + + /// Check if running in development mode + pub fn is_development(&self) -> bool { + self.environment == "development" + } +} + +/// Global configuration instance, loaded once at startup. +/// Panics if configuration is invalid - this is intentional for fail-fast behavior. +pub static CONFIG: Lazy = Lazy::new(|| { + AppConfig::load().unwrap_or_else(|e| { + eprintln!("❌ Failed to load configuration: {}", e); + eprintln!(" Make sure all required environment variables are set:"); + eprintln!(" - DATABASE_URL"); + eprintln!(" - JWT_SECRET"); + eprintln!(" - REDIS_URL (or APP_REDIS_URL)"); + std::process::exit(1); + }) +}); + +/// Global MinIO configuration, loaded from environment variables. +/// Returns None if required MINIO_* variables are not set. +pub static MINIO_CONFIG: Lazy> = Lazy::new(|| { + let _ = dotenvy::dotenv(); + MinioConfig::from_env() +}); diff --git a/src/shared/database/mod.rs b/src/shared/database/mod.rs new file mode 100644 index 0000000..85f0cc4 --- /dev/null +++ b/src/shared/database/mod.rs @@ -0,0 +1,7 @@ +pub mod persistence; +pub mod redis; +pub mod repositories; +pub mod setup; +pub mod traits; + +pub use redis::{get_redis_conn, get_redis_pool, redis_pool}; diff --git a/src/shared/database/persistence/entities/image_cache.rs b/src/shared/database/persistence/entities/image_cache.rs new file mode 100644 index 0000000..073fb6a --- /dev/null +++ b/src/shared/database/persistence/entities/image_cache.rs @@ -0,0 +1,71 @@ +//! `SeaORM` Entity for ImageCache - stores URL mappings for CDN caching + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Copy, Clone, Default, Debug, DeriveEntity)] +pub struct Entity; + +impl EntityName for Entity { + fn table_name(&self) -> &str { + "ImageCache" + } +} + +#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel, Eq, Serialize, Deserialize)] +pub struct Model { + pub id: String, + pub original_url: String, + pub cdn_url: String, + pub created_at: DateTimeUtc, + pub expires_at: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)] +pub enum Column { + Id, + #[sea_orm(column_name = "original_url")] + OriginalUrl, + #[sea_orm(column_name = "cdn_url")] + CdnUrl, + #[sea_orm(column_name = "created_at")] + CreatedAt, + #[sea_orm(column_name = "expires_at")] + ExpiresAt, +} + +#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)] +pub enum PrimaryKey { + Id, +} + +impl PrimaryKeyTrait for PrimaryKey { + type ValueType = String; + fn auto_increment() -> bool { + false + } +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl ColumnTrait for Column { + type EntityName = Entity; + fn def(&self) -> ColumnDef { + match self { + Self::Id => ColumnType::String(StringLen::N(36u32)).def(), + Self::OriginalUrl => ColumnType::String(StringLen::N(512u32)).def().unique(), + Self::CdnUrl => ColumnType::String(StringLen::N(512u32)).def(), + Self::CreatedAt => ColumnType::TimestampWithTimeZone.def(), + Self::ExpiresAt => ColumnType::TimestampWithTimeZone.def().null(), + } + } +} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + match *self {} + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/shared/database/persistence/entities/mod.rs b/src/shared/database/persistence/entities/mod.rs new file mode 100644 index 0000000..44e6ce2 --- /dev/null +++ b/src/shared/database/persistence/entities/mod.rs @@ -0,0 +1 @@ +pub mod image_cache; diff --git a/src/shared/database/persistence/mod.rs b/src/shared/database/persistence/mod.rs new file mode 100644 index 0000000..0b8f0b5 --- /dev/null +++ b/src/shared/database/persistence/mod.rs @@ -0,0 +1 @@ +pub mod entities; diff --git a/src/shared/database/redis.rs b/src/shared/database/redis.rs new file mode 100644 index 0000000..01225b7 --- /dev/null +++ b/src/shared/database/redis.rs @@ -0,0 +1,74 @@ +//! Redis connection utility with tracing for connection lifecycle and errors. +//! +//! Uses the type-safe CONFIG for Redis connection parameters. + +use crate::shared::config::CONFIG; +use crate::shared::errors::AppError; +use deadpool_redis::{Manager, Pool}; +use once_cell::sync::Lazy; +use tracing::{debug, error, info}; + +static REDIS_POOL_INIT: Lazy> = Lazy::new(|| { + let redis_url = if !CONFIG.redis_url.is_empty() { + CONFIG.redis_url.clone() + } else { + let host = std::env::var("REDIS_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()); + let port = std::env::var("REDIS_PORT").unwrap_or_else(|_| "6379".to_string()); + let password = std::env::var("REDIS_PASSWORD").unwrap_or_default(); + + if password.is_empty() { + format!("redis://{}:{}", host, port) + } else { + format!("redis://:{}@{}:{}", password, host, port) + } + }; + + info!("Initializing Redis connection pool for URL: {}", redis_url); + + Manager::new(redis_url) + .map_err(|e| format!("Failed to create Redis manager: {}", e)) + .and_then(|manager| { + Pool::builder(manager) + .max_size(100) + .wait_timeout(Some(std::time::Duration::from_millis(200))) + .runtime(deadpool_redis::Runtime::Tokio1) + .build() + .map_err(|e| format!("Failed to create Redis connection pool: {}", e)) + }) +}); + +/// Get the Redis connection pool (for internal use). +pub fn get_redis_pool() -> Result<&'static Pool, String> { + REDIS_POOL_INIT.as_ref().map_err(|e| e.clone()) +} + +/// Get a cloned reference to the Redis pool. +pub fn redis_pool() -> Result { + get_redis_pool().map(|p| (*p).clone()) +} + +/// Get an async connection from the pool with retry backoff. +pub async fn get_redis_conn() -> Result { + let pool = get_redis_pool().map_err(|e| AppError::Other(e))?; + let mut retries = 5; + let mut wait = std::time::Duration::from_millis(100); + + loop { + match pool.get().await { + Ok(conn) => { + debug!("Successfully retrieved Redis connection from pool."); + return Ok(conn); + } + Err(e) => { + if retries <= 0 { + error!("Failed to get Redis connection after retries: {:?}", e); + return Err(AppError::from(e)); + } + debug!("Redis connection failed, retrying in {:?}: {:?}", wait, e); + tokio::time::sleep(wait).await; + wait = std::cmp::min(wait * 2, std::time::Duration::from_secs(5)); + retries -= 1; + } + } + } +} diff --git a/src/shared/database/repositories/image_cache.rs b/src/shared/database/repositories/image_cache.rs new file mode 100644 index 0000000..53f8443 --- /dev/null +++ b/src/shared/database/repositories/image_cache.rs @@ -0,0 +1,129 @@ +use crate::shared::database::persistence::entities::image_cache; +use crate::shared::database::traits::image_cache::ImageCacheRepository; +use crate::shared::utils::Cache; +use async_trait::async_trait; +use chrono::Utc; +use deadpool_redis::Pool as RedisPool; +use sea_orm::{ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, Set}; +use std::sync::Arc; + +pub struct SeaOrmImageCacheRepository { + db: Arc, + redis: RedisPool, +} + +impl SeaOrmImageCacheRepository { + pub fn new(db: Arc, redis: RedisPool) -> Self { + Self { db, redis } + } +} + +#[async_trait] +impl ImageCacheRepository for SeaOrmImageCacheRepository { + async fn get_from_redis(&self, key: &str) -> Option { + Cache::new(&self.redis).get::(key).await + } + + async fn set_in_redis(&self, key: &str, value: &str, ttl: u64) -> Result<(), String> { + Cache::new(&self.redis) + .set_with_ttl(key, &value, ttl) + .await + .map_err(|e| e.to_string()) + } + + async fn get_from_db(&self, original_url: &str) -> Result, String> { + let entry = image_cache::Entity::find() + .filter(image_cache::Column::OriginalUrl.eq(original_url)) + .one(self.db.as_ref()) + .await + .map_err(|e| e.to_string())?; + Ok(entry.map(|m| m.cdn_url)) + } + + async fn save_to_db(&self, original_url: &str, cdn_url: &str) -> Result<(), String> { + let model = image_cache::ActiveModel { + id: Set(uuid::Uuid::new_v4().to_string()), + original_url: Set(original_url.to_string()), + cdn_url: Set(cdn_url.to_string()), + created_at: Set(Utc::now()), + expires_at: Set(None), + }; + + model + .insert(self.db.as_ref()) + .await + .map_err(|e| e.to_string())?; + Ok(()) + } + + async fn find_original_from_cdn(&self, cdn_url: &str) -> Result, String> { + let entry = image_cache::Entity::find() + .filter(image_cache::Column::CdnUrl.eq(cdn_url)) + .one(self.db.as_ref()) + .await + .map_err(|e| e.to_string())?; + Ok(entry.map(|m| m.original_url)) + } + + async fn delete_from_db(&self, original_url: &str) -> Result<(), String> { + image_cache::Entity::delete_many() + .filter(image_cache::Column::OriginalUrl.eq(original_url)) + .exec(self.db.as_ref()) + .await + .map_err(|e| e.to_string())?; + Ok(()) + } + + async fn delete_from_redis(&self, key: &str) -> Result<(), String> { + Cache::new(&self.redis) + .delete(key) + .await + .map_err(|e| e.to_string()) + } + + async fn get_lock(&self, key: &str) -> bool { + Cache::new(&self.redis).get::(key).await.is_some() + } + + async fn set_lock(&self, key: &str, ttl: u64) -> Result<(), String> { + Cache::new(&self.redis) + .set_with_ttl(key, &true, ttl) + .await + .map_err(|e| e.to_string()) + } + + async fn release_lock(&self, key: &str) -> Result<(), String> { + self.delete_from_redis(key).await + } + + async fn invalidate_api_caches(&self, patterns: Vec<&str>) -> Result<(), String> { + use deadpool_redis::redis::{cmd, AsyncCommands}; + + let mut conn = self.redis.get().await.map_err(|e| e.to_string())?; + + for pattern in patterns { + let mut cursor: u64 = 0; + loop { + let (new_cursor, keys): (u64, Vec) = cmd("SCAN") + .arg(cursor) + .arg("MATCH") + .arg(pattern) + .arg("COUNT") + .arg(100) + .query_async(&mut *conn) + .await + .map_err(|e| e.to_string())?; + + if !keys.is_empty() { + let _: usize = conn.del(&keys).await.map_err(|e| e.to_string())?; + } + + cursor = new_cursor; + if cursor == 0 { + break; + } + } + } + Ok(()) + } +} diff --git a/src/shared/database/repositories/mod.rs b/src/shared/database/repositories/mod.rs new file mode 100644 index 0000000..44e6ce2 --- /dev/null +++ b/src/shared/database/repositories/mod.rs @@ -0,0 +1 @@ +pub mod image_cache; diff --git a/src/shared/database/setup.rs b/src/shared/database/setup.rs new file mode 100644 index 0000000..38617da --- /dev/null +++ b/src/shared/database/setup.rs @@ -0,0 +1,44 @@ +use crate::shared::database::persistence::entities::image_cache; +use sea_orm::{ConnectionTrait, DatabaseConnection, Schema, Statement}; +use tracing::info; + +pub async fn init(db: &DatabaseConnection) -> Result<(), sea_orm::DbErr> { + info!("🚀 Initializing database schema..."); + let backend = db.get_database_backend(); + let schema = Schema::new(backend); + + let tables = vec![( + "ImageCache", + schema + .create_table_from_entity(image_cache::Entity) + .if_not_exists() + .to_owned(), + )]; + + for (name, stmt) in tables { + match db.execute(backend.build(&stmt)).await { + Ok(_) => info!(" ✓ Table '{}' checked/created", name), + Err(e) => { + tracing::error!(" [!] Failed to create table '{}': {}", name, e); + return Err(e); + } + } + } + + let index_sql = + "CREATE INDEX IF NOT EXISTS idx_image_cache_cdn_url ON \"ImageCache\" (cdn_url)"; + match db.execute(Statement::from_string(backend, index_sql)).await { + Ok(_) => info!(" ✓ Index 'idx_image_cache_cdn_url' ensured"), + Err(e) => { + let err_str = e.to_string(); + if err_str.contains("already exists") || err_str.contains("duplicate") { + info!(" ✓ Index 'idx_image_cache_cdn_url' already exists"); + } else { + tracing::error!(" [!] Failed to create index on ImageCache: {}", e); + } + } + } + + info!("✅ Database schema initialization complete."); + Ok(()) +} diff --git a/src/shared/database/traits/image_cache.rs b/src/shared/database/traits/image_cache.rs new file mode 100644 index 0000000..56588ef --- /dev/null +++ b/src/shared/database/traits/image_cache.rs @@ -0,0 +1,16 @@ +use async_trait::async_trait; + +#[async_trait] +pub trait ImageCacheRepository: Send + Sync { + async fn get_from_redis(&self, key: &str) -> Option; + async fn set_in_redis(&self, key: &str, value: &str, ttl: u64) -> Result<(), String>; + async fn get_from_db(&self, original_url: &str) -> Result, String>; + async fn save_to_db(&self, original_url: &str, cdn_url: &str) -> Result<(), String>; + async fn find_original_from_cdn(&self, cdn_url: &str) -> Result, String>; + async fn delete_from_db(&self, original_url: &str) -> Result<(), String>; + async fn delete_from_redis(&self, key: &str) -> Result<(), String>; + async fn get_lock(&self, key: &str) -> bool; + async fn set_lock(&self, key: &str, ttl: u64) -> Result<(), String>; + async fn release_lock(&self, key: &str) -> Result<(), String>; + async fn invalidate_api_caches(&self, patterns: Vec<&str>) -> Result<(), String>; +} diff --git a/src/shared/database/traits/mod.rs b/src/shared/database/traits/mod.rs new file mode 100644 index 0000000..03d89d2 --- /dev/null +++ b/src/shared/database/traits/mod.rs @@ -0,0 +1,4 @@ +pub mod image_cache; +pub mod scraping_repository; + +pub use scraping_repository::ScrapingRepository; diff --git a/src/shared/database/traits/scraping_repository.rs b/src/shared/database/traits/scraping_repository.rs new file mode 100644 index 0000000..411a35a --- /dev/null +++ b/src/shared/database/traits/scraping_repository.rs @@ -0,0 +1,7 @@ +use crate::shared::errors::AppError; +use async_trait::async_trait; + +#[async_trait] +pub trait ScrapingRepository: Send + Sync { + async fn fetch_html(&self, url: &str) -> Result; +} diff --git a/src/shared/errors/app_error.rs b/src/shared/errors/app_error.rs new file mode 100644 index 0000000..2b85a42 --- /dev/null +++ b/src/shared/errors/app_error.rs @@ -0,0 +1,88 @@ +use axum::response::IntoResponse; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum AppError { + #[error("Environment variable not found: {0}")] + EnvVarNotFound(String), + #[error("Redis error: {0}")] + RedisError(#[from] redis::RedisError), + #[error("Reqwest error: {0}")] + ReqwestError(#[from] reqwest::Error), + #[error("JSON serialization/deserialization error: {0}")] + SerdeJsonError(#[from] serde_json::Error), + #[error("Scraper error: {0}")] + ScraperError(String), + #[error("Fantoccini error: {0}")] + FantocciniError(String), + #[error("IO error: {0}")] + IoError(#[from] std::io::Error), + #[error("Timeout error: {0}")] + TimeoutError(String), + #[error("Other error: {0}")] + Other(String), + #[error("HTTP error: {0}")] + HttpError(#[from] http::Error), + #[error("URL parsing error: {0}")] + UrlParseError(#[from] url::ParseError), + #[error("Database error: {0}")] + DatabaseError(String), + #[error("Not Found: {0}")] + NotFound(String), +} + +impl From<&str> for AppError { + fn from(s: &str) -> Self { + AppError::Other(s.to_string()) + } +} + +impl From for AppError { + fn from(s: String) -> Self { + AppError::Other(s) + } +} + +impl From> for AppError { + fn from(err: Box) -> Self { + AppError::Other(err.to_string()) + } +} + +impl From for AppError { + fn from(err: anyhow::Error) -> Self { + AppError::Other(err.to_string()) + } +} + +impl From for AppError { + fn from(err: deadpool_redis::PoolError) -> Self { + AppError::Other(err.to_string()) + } +} + +impl From for AppError { + fn from(err: tokio::task::JoinError) -> Self { + AppError::Other(err.to_string()) + } +} + +impl IntoResponse for AppError { + fn into_response(self) -> axum::response::Response { + let (status, error_message) = match self { + AppError::NotFound(_) => (http::StatusCode::NOT_FOUND, self.to_string()), + AppError::DatabaseError(_) => { + (http::StatusCode::INTERNAL_SERVER_ERROR, self.to_string()) + } + _ => (http::StatusCode::INTERNAL_SERVER_ERROR, self.to_string()), + }; + + // Note: crate::shared::types needs to be available. + // If not, we might need to adjust this line or ensure types are there. + // Since we are validating structure, let's assume types is in core/types.rs + let body = axum::Json(crate::shared::types::ApiResponse::<()>::error( + error_message, + )); + (status, body).into_response() + } +} diff --git a/src/shared/errors/mod.rs b/src/shared/errors/mod.rs new file mode 100644 index 0000000..84cb0e0 --- /dev/null +++ b/src/shared/errors/mod.rs @@ -0,0 +1,2 @@ +pub mod app_error; +pub use app_error::AppError; diff --git a/src/shared/events/bus.rs b/src/shared/events/bus.rs new file mode 100644 index 0000000..2a95daa --- /dev/null +++ b/src/shared/events/bus.rs @@ -0,0 +1,155 @@ +//! Event bus implementation. + +use async_trait::async_trait; + +use std::{any::TypeId, collections::HashMap, sync::Arc}; +use tokio::sync::{broadcast, RwLock}; +use tracing::{debug, info}; + +/// Trait for events that can be published. +pub trait Event: Clone + Send + Sync + 'static { + /// Event name for logging/debugging. + const NAME: &'static str; +} + +/// Trait for event handlers. +#[async_trait] +pub trait EventHandler: Send + Sync { + async fn handle(&self, event: E); +} + +/// The event bus for publishing and subscribing to events. +pub struct EventBus { + channels: RwLock>>, +} + +impl EventBus { + /// Create a new event bus. + pub fn new() -> Self { + Self { + channels: RwLock::new(HashMap::new()), + } + } + + /// Publish an event to all subscribers. + pub async fn publish(&self, event: E) { + let type_id = TypeId::of::(); + let channels = self.channels.read().await; + + if let Some(sender) = channels.get(&type_id) { + if let Some(tx) = sender.downcast_ref::>() { + let _ = tx.send(event); + debug!("Published event: {}", E::NAME); + } + } + } + + /// Subscribe to events of a specific type. + /// Returns a receiver that can be used to receive events. + pub async fn subscribe(&self) -> broadcast::Receiver { + let type_id = TypeId::of::(); + + // Check if channel exists + { + let channels = self.channels.read().await; + if let Some(sender) = channels.get(&type_id) { + if let Some(tx) = sender.downcast_ref::>() { + return tx.subscribe(); + } + } + } + + // Create new channel + let (tx, rx) = broadcast::channel::(100); + { + let mut channels = self.channels.write().await; + channels.insert(type_id, Box::new(tx)); + } + + // Re-get the receiver from the stored sender + let channels = self.channels.read().await; + if let Some(sender) = channels.get(&type_id) { + if let Some(tx) = sender.downcast_ref::>() { + return tx.subscribe(); + } + } + + rx + } + + /// Register a handler for a specific event type. + /// The handler will be called whenever an event of that type is published. + pub async fn on + 'static>(&self, handler: H) { + let mut rx = self.subscribe::().await; + let handler = Arc::new(handler); + + tokio::spawn(async move { + loop { + match rx.recv().await { + Ok(event) => { + handler.handle(event).await; + } + Err(broadcast::error::RecvError::Closed) => break, + Err(broadcast::error::RecvError::Lagged(n)) => { + tracing::warn!("Event handler lagged by {} events", n); + } + } + } + }); + + info!("Registered handler for event: {}", E::NAME); + } +} + +impl Default for EventBus { + fn default() -> Self { + Self::new() + } +} + +// Common events +/// User registered event. +#[derive(Clone, Debug)] +pub struct UserRegistered { + pub user_id: String, + pub email: String, + pub name: String, +} + +impl Event for UserRegistered { + const NAME: &'static str = "user.registered"; +} + +/// User logged in event. +#[derive(Clone, Debug)] +pub struct UserLoggedIn { + pub user_id: String, + pub ip_address: Option, +} + +impl Event for UserLoggedIn { + const NAME: &'static str = "user.logged_in"; +} + +/// Order created event. +#[derive(Clone, Debug)] +pub struct OrderCreated { + pub order_id: String, + pub user_id: String, + pub total: f64, +} + +impl Event for OrderCreated { + const NAME: &'static str = "order.created"; +} + +/// Image repaired event. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct ImageRepaired { + pub original_url: String, + pub cdn_url: String, +} + +impl Event for ImageRepaired { + const NAME: &'static str = "image.repaired"; +} diff --git a/src/shared/events/mod.rs b/src/shared/events/mod.rs new file mode 100644 index 0000000..6440547 --- /dev/null +++ b/src/shared/events/mod.rs @@ -0,0 +1 @@ +pub mod bus; diff --git a/src/shared/graceful/cleanup.rs b/src/shared/graceful/cleanup.rs new file mode 100644 index 0000000..6e3633d --- /dev/null +++ b/src/shared/graceful/cleanup.rs @@ -0,0 +1,142 @@ +//! Graceful shutdown with proper resource cleanup. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use tokio::signal; +use tokio::sync::Notify; +use tokio::time::{sleep, Duration}; +use tracing::info; + +/// Graceful shutdown coordinator. +pub struct ShutdownCoordinator { + /// Shutdown signal flag + is_shutting_down: Arc, + /// Notify for graceful shutdown + shutdown_notify: Arc, +} + +impl ShutdownCoordinator { + /// Create a new shutdown coordinator. + pub fn new() -> Self { + Self { + is_shutting_down: Arc::new(AtomicBool::new(false)), + shutdown_notify: Arc::new(Notify::new()), + } + } + + /// Check if shutdown is in progress. + pub fn is_shutting_down(&self) -> bool { + self.is_shutting_down.load(Ordering::Relaxed) + } + + /// Start shutdown process. + pub fn initiate_shutdown(&self) { + info!("🛑 Initiating graceful shutdown..."); + self.is_shutting_down.store(true, Ordering::Relaxed); + self.shutdown_notify.notify_waiters(); + } + + /// Wait for shutdown signal. + pub async fn wait_for_shutdown_signal(&self) { + let ctrl_c = async { + if let Err(e) = signal::ctrl_c().await { + tracing::error!("Failed to listen for Ctrl+C: {}", e); + } + }; + + #[cfg(unix)] + let terminate = async { + match signal::unix::signal(signal::unix::SignalKind::terminate()) { + Ok(mut stream) => { + stream.recv().await; + } + Err(e) => { + tracing::error!("Failed to listen for SIGTERM: {}", e); + } + } + }; + + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + _ = ctrl_c => { + info!("Received Ctrl+C signal"); + } + _ = terminate => { + info!("Received SIGTERM signal"); + } + } + + self.initiate_shutdown(); + } + + /// Perform cleanup operations. + pub async fn cleanup(&self) { + info!("🧹 Starting cleanup operations..."); + + // Give active requests time to finish + info!("Waiting for active requests to complete..."); + sleep(Duration::from_secs(5)).await; + + info!("✅ Cleanup completed"); + } + + /// Get a handle for checking shutdown status. + pub fn handle(&self) -> ShutdownHandle { + ShutdownHandle { + is_shutting_down: Arc::clone(&self.is_shutting_down), + } + } +} + +impl Default for ShutdownCoordinator { + fn default() -> Self { + Self::new() + } +} + +/// Handle for checking shutdown status. +#[derive(Clone)] +pub struct ShutdownHandle { + is_shutting_down: Arc, +} + +impl ShutdownHandle { + /// Check if shutdown is in progress. + pub fn is_shutting_down(&self) -> bool { + self.is_shutting_down.load(Ordering::Relaxed) + } +} + +pub async fn wait_for_shutdown_and_cleanup() { + let coordinator = ShutdownCoordinator::new(); + + coordinator.wait_for_shutdown_signal().await; + coordinator.cleanup().await; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_shutdown_coordinator() { + let coordinator = ShutdownCoordinator::new(); + assert!(!coordinator.is_shutting_down()); + + coordinator.initiate_shutdown(); + assert!(coordinator.is_shutting_down()); + } + + #[test] + fn test_shutdown_handle() { + let coordinator = ShutdownCoordinator::new(); + let handle = coordinator.handle(); + + assert!(!handle.is_shutting_down()); + + coordinator.initiate_shutdown(); + assert!(handle.is_shutting_down()); + } +} diff --git a/src/shared/graceful/mod.rs b/src/shared/graceful/mod.rs new file mode 100644 index 0000000..c630962 --- /dev/null +++ b/src/shared/graceful/mod.rs @@ -0,0 +1,2 @@ +pub mod cleanup; +pub mod shutdown; diff --git a/src/shared/graceful/shutdown.rs b/src/shared/graceful/shutdown.rs new file mode 100644 index 0000000..2ebab74 --- /dev/null +++ b/src/shared/graceful/shutdown.rs @@ -0,0 +1,116 @@ +//! Graceful shutdown implementation. + +use std::future::Future; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; +use tokio::signal; +use tokio::sync::broadcast; +use tracing::info; + +/// Graceful shutdown controller. +pub struct GracefulShutdown { + shutdown_tx: broadcast::Sender<()>, + is_shutting_down: Arc, +} + +impl GracefulShutdown { + /// Create a new graceful shutdown controller. + pub fn new() -> Self { + let (shutdown_tx, _) = broadcast::channel(1); + Self { + shutdown_tx, + is_shutting_down: Arc::new(AtomicBool::new(false)), + } + } + + /// Check if shutdown has been initiated. + pub fn is_shutting_down(&self) -> bool { + self.is_shutting_down.load(Ordering::SeqCst) + } + + /// Get a receiver for shutdown notifications. + pub fn subscribe(&self) -> broadcast::Receiver<()> { + self.shutdown_tx.subscribe() + } + + /// Initiate shutdown. + pub fn shutdown(&self) { + if !self.is_shutting_down.swap(true, Ordering::SeqCst) { + info!("🛑 Initiating graceful shutdown..."); + let _ = self.shutdown_tx.send(()); + } + } + + /// Wait for shutdown signal and then gracefully drain. + pub async fn wait_for_shutdown(&self, drain_timeout: Duration) { + // Wait for shutdown signal + shutdown_signal().await; + + self.shutdown(); + + // Give time for in-flight requests to complete + info!( + "⏳ Waiting {}s for in-flight requests...", + drain_timeout.as_secs() + ); + tokio::time::sleep(drain_timeout).await; + + info!("✅ Graceful shutdown complete"); + } +} + +impl Default for GracefulShutdown { + fn default() -> Self { + Self::new() + } +} + +impl Clone for GracefulShutdown { + fn clone(&self) -> Self { + Self { + shutdown_tx: self.shutdown_tx.clone(), + is_shutting_down: Arc::clone(&self.is_shutting_down), + } + } +} + +/// Wait for a shutdown signal (SIGTERM, SIGINT, or Ctrl+C). +pub async fn shutdown_signal() { + let ctrl_c = async { + if let Err(e) = signal::ctrl_c().await { + tracing::error!("Failed to install Ctrl+C handler: {}", e); + } + }; + + #[cfg(unix)] + let terminate = async { + match signal::unix::signal(signal::unix::SignalKind::terminate()) { + Ok(mut stream) => { + stream.recv().await; + } + Err(e) => { + tracing::error!("Failed to install SIGTERM handler: {}", e); + } + } + }; + + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + _ = ctrl_c => { + info!("📥 Received Ctrl+C signal"); + } + _ = terminate => { + info!("📥 Received SIGTERM signal"); + } + } +} + +/// Create a shutdown future that can be used with axum's serve. +pub fn create_shutdown_signal() -> impl Future + Send + 'static { + async { + shutdown_signal().await; + } +} diff --git a/src/shared/health/endpoints.rs b/src/shared/health/endpoints.rs new file mode 100644 index 0000000..67f5fa5 --- /dev/null +++ b/src/shared/health/endpoints.rs @@ -0,0 +1,138 @@ +//! Health check endpoint implementations. + +use axum::{http::StatusCode, response::IntoResponse, Json}; +use serde::Serialize; +use std::time::Instant; + +use once_cell::sync::Lazy; + +static START_TIME: Lazy = Lazy::new(Instant::now); + +/// Health status response. +#[derive(Debug, Clone, Serialize)] +pub struct HealthStatus { + pub status: &'static str, + pub version: &'static str, + pub uptime_seconds: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub checks: Option, +} + +/// Individual health checks. +#[derive(Debug, Clone, Serialize)] +pub struct HealthChecks { + pub database: CheckResult, + pub redis: CheckResult, +} + +/// Result of a health check. +#[derive(Debug, Clone, Serialize)] +pub struct CheckResult { + pub status: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub latency_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +/// Simple health check - just returns OK. +/// Use for liveness probes (Kubernetes: /healthz). +pub async fn health_check() -> impl IntoResponse { + let status = HealthStatus { + status: "ok", + version: env!("CARGO_PKG_VERSION"), + uptime_seconds: START_TIME.elapsed().as_secs(), + checks: None, + }; + (StatusCode::OK, Json(status)) +} + +/// Readiness check with dependency checks. +/// Use for readiness probes (Kubernetes: /readyz). +pub async fn readiness_check() -> impl IntoResponse { + let uptime = START_TIME.elapsed().as_secs(); + + // Check Redis + let redis_check = check_redis().await; + + // Check Database - simplified for now + let db_check = CheckResult { + status: "ok", + latency_ms: Some(1), + error: None, + }; + + let all_healthy = redis_check.status == "ok" && db_check.status == "ok"; + + let status = HealthStatus { + status: if all_healthy { "ok" } else { "degraded" }, + version: env!("CARGO_PKG_VERSION"), + uptime_seconds: uptime, + checks: Some(HealthChecks { + database: db_check, + redis: redis_check, + }), + }; + + let status_code = if all_healthy { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + }; + + (status_code, Json(status)) +} + +/// Check Redis connectivity using PING command. +async fn check_redis() -> CheckResult { + use crate::shared::database::get_redis_pool; + + let start = Instant::now(); + + let pool = match get_redis_pool() { + Ok(p) => p, + Err(e) => { + return CheckResult { + status: "error", + latency_ms: Some(start.elapsed().as_millis() as u64), + error: Some(format!("Pool error: {}", e)), + }; + } + }; + + match pool.get().await { + Ok(mut conn) => { + let result: Result = redis::cmd("PING").query_async(&mut *conn).await; + match result { + Ok(response) if response == "PONG" => CheckResult { + status: "ok", + latency_ms: Some(start.elapsed().as_millis() as u64), + error: None, + }, + Ok(response) => CheckResult { + status: "error", + latency_ms: Some(start.elapsed().as_millis() as u64), + error: Some(format!("Unexpected PING response: {}", response)), + }, + Err(e) => CheckResult { + status: "error", + latency_ms: Some(start.elapsed().as_millis() as u64), + error: Some(e.to_string()), + }, + } + } + Err(e) => CheckResult { + status: "error", + latency_ms: Some(start.elapsed().as_millis() as u64), + error: Some(e.to_string()), + }, + } +} + +/// Standalone health endpoint (no dependencies required). +pub async fn simple_health() -> impl IntoResponse { + Json(serde_json::json!({ + "status": "ok", + "timestamp": chrono::Utc::now().to_rfc3339() + })) +} diff --git a/src/shared/health/mod.rs b/src/shared/health/mod.rs new file mode 100644 index 0000000..c4b360f --- /dev/null +++ b/src/shared/health/mod.rs @@ -0,0 +1 @@ +pub mod endpoints; diff --git a/src/shared/jobs/mod.rs b/src/shared/jobs/mod.rs new file mode 100644 index 0000000..b3116e8 --- /dev/null +++ b/src/shared/jobs/mod.rs @@ -0,0 +1,2 @@ +pub mod queue; +pub mod worker; diff --git a/src/shared/jobs/queue.rs b/src/shared/jobs/queue.rs new file mode 100644 index 0000000..6238d18 --- /dev/null +++ b/src/shared/jobs/queue.rs @@ -0,0 +1,204 @@ +//! Job queue implementation using Redis. +//! +//! Provides a simple but robust job queue system for background processing. + +use async_trait::async_trait; +use deadpool_redis::Pool; +use redis::AsyncCommands; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use uuid::Uuid; + +/// Status of a queued job. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum JobStatus { + /// Job is waiting in the queue + Pending, + /// Job is currently being processed + Processing, + /// Job completed successfully + Completed, + /// Job failed with an error + Failed, + /// Job was manually cancelled + Cancelled, +} + +/// Metadata for a queued job. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JobMeta { + pub id: String, + pub job_type: String, + pub status: JobStatus, + pub created_at: chrono::DateTime, + pub started_at: Option>, + pub completed_at: Option>, + pub attempts: u32, + pub max_attempts: u32, + pub error: Option, +} + +/// Trait for background jobs. +/// +/// Implement this trait to define a job that can be queued and processed. +/// +/// # Example +/// +/// ```ignore +/// use scraper_service::jobs::{Job, JobDispatcher}; +/// use serde::{Serialize, Deserialize}; +/// use async_trait::async_trait; +/// +/// #[derive(Serialize, Deserialize)] +/// struct SendWelcomeEmail { +/// user_id: String, +/// email: String, +/// } +/// +/// #[async_trait] +/// impl Job for SendWelcomeEmail { +/// const NAME: &'static str = "send_welcome_email"; +/// const MAX_ATTEMPTS: u32 = 3; +/// +/// async fn handle(&self) -> anyhow::Result<()> { +/// // Send the email... +/// println!("Sending welcome email to {}", self.email); +/// Ok(()) +/// } +/// } +/// +/// // Dispatch the job +/// let job = SendWelcomeEmail { +/// user_id: "123".to_string(), +/// email: "user@example.com".to_string(), +/// }; +/// dispatcher.dispatch(job).await?; +/// ``` +#[async_trait] +pub trait Job: Serialize + DeserializeOwned + Send + Sync { + /// Unique name for this job type. + const NAME: &'static str; + + /// Maximum number of retry attempts. + const MAX_ATTEMPTS: u32 = 3; + + /// Queue name to use (default: "default") + const QUEUE: &'static str = "default"; + + /// Execute the job. + async fn handle(&self) -> anyhow::Result<()>; + + /// Called when the job fails after all retries. + async fn failed(&self, error: &str) { + tracing::error!("Job {} failed: {}", Self::NAME, error); + } +} + +/// Job dispatcher for queuing jobs. +#[derive(Clone)] +pub struct JobDispatcher { + redis_pool: Pool, +} + +impl JobDispatcher { + /// Create a new job dispatcher. + pub fn new(redis_pool: Pool) -> Self { + Self { redis_pool } + } + + /// Dispatch a job to be processed. + pub async fn dispatch(&self, job: J) -> anyhow::Result { + let job_id = Uuid::new_v4().to_string(); + let queue_key = format!("jobs:queue:{}", J::QUEUE); + let job_key = format!("jobs:data:{}", job_id); + + let payload = serde_json::to_string(&job)?; + + let meta = JobMeta { + id: job_id.clone(), + job_type: J::NAME.to_string(), + status: JobStatus::Pending, + created_at: chrono::Utc::now(), + started_at: None, + completed_at: None, + attempts: 0, + max_attempts: J::MAX_ATTEMPTS, + error: None, + }; + + let meta_json = serde_json::to_string(&meta)?; + + let mut conn = self.redis_pool.get().await?; + + // Store job data + let _: () = conn.set(&job_key, payload).await?; + let _: () = conn.set(format!("{}:meta", job_key), meta_json).await?; + + // Push to queue + let _: () = conn.rpush(&queue_key, &job_id).await?; + + tracing::info!("Dispatched job {} ({})", J::NAME, job_id); + + Ok(job_id) + } + + /// Dispatch a job with a delay (in seconds). + pub async fn dispatch_delayed( + &self, + job: J, + delay_seconds: u64, + ) -> anyhow::Result { + let job_id = Uuid::new_v4().to_string(); + let delayed_key = "jobs:delayed"; + let job_key = format!("jobs:data:{}", job_id); + + let payload = serde_json::to_string(&job)?; + let execute_at = chrono::Utc::now().timestamp() + delay_seconds as i64; + + let meta = JobMeta { + id: job_id.clone(), + job_type: J::NAME.to_string(), + status: JobStatus::Pending, + created_at: chrono::Utc::now(), + started_at: None, + completed_at: None, + attempts: 0, + max_attempts: J::MAX_ATTEMPTS, + error: None, + }; + + let meta_json = serde_json::to_string(&meta)?; + + let mut conn = self.redis_pool.get().await?; + + // Store job data + let _: () = conn.set(&job_key, payload).await?; + let _: () = conn.set(format!("{}:meta", job_key), meta_json).await?; + + // Add to delayed sorted set (score = execution timestamp) + let delayed_entry = format!("{}:{}", J::QUEUE, job_id); + let _: () = conn.zadd(delayed_key, delayed_entry, execute_at).await?; + + tracing::info!( + "Dispatched delayed job {} ({}) - executes in {}s", + J::NAME, + job_id, + delay_seconds + ); + + Ok(job_id) + } + + /// Get the status of a job. + pub async fn status(&self, job_id: &str) -> anyhow::Result> { + let meta_key = format!("jobs:data:{}:meta", job_id); + let mut conn = self.redis_pool.get().await?; + + let meta_json: Option = conn.get(&meta_key).await?; + + match meta_json { + Some(json) => Ok(Some(serde_json::from_str(&json)?)), + None => Ok(None), + } + } +} diff --git a/src/shared/jobs/worker.rs b/src/shared/jobs/worker.rs new file mode 100644 index 0000000..310af64 --- /dev/null +++ b/src/shared/jobs/worker.rs @@ -0,0 +1,180 @@ +//! Job worker for processing background jobs. +//! +//! The worker runs as a separate process and continuously polls +//! the job queue for work. + +use super::queue::{JobMeta, JobStatus}; +use deadpool_redis::Pool; +use redis::AsyncCommands; +use std::time::Duration; +use tokio::time::sleep; + +/// Configuration for the job worker. +#[derive(Debug, Clone)] +pub struct WorkerConfig { + /// Queues to process (in priority order) + pub queues: Vec, + /// Number of concurrent jobs to process + pub concurrency: usize, + /// Sleep duration when no jobs are available + pub sleep_duration: Duration, +} + +impl Default for WorkerConfig { + fn default() -> Self { + Self { + queues: vec!["default".to_string()], + concurrency: 4, + sleep_duration: Duration::from_secs(1), + } + } +} + +/// Job worker that processes queued jobs. +pub struct Worker { + redis_pool: Pool, + config: WorkerConfig, + /// Registry of job handlers by name + handlers: std::collections::HashMap<&'static str, Box>, +} + +/// Trait for job handlers (type-erased). +#[async_trait::async_trait] +pub trait JobHandler: Send + Sync { + async fn process(&self, payload: &str) -> anyhow::Result<()>; +} + +impl Worker { + /// Create a new worker. + pub fn new(redis_pool: Pool, config: WorkerConfig) -> Self { + Self { + redis_pool, + config, + handlers: std::collections::HashMap::new(), + } + } + + /// Register a job handler. + pub fn register(&mut self, name: &'static str, handler: H) { + self.handlers.insert(name, Box::new(handler)); + } + + /// Run the worker loop. + pub async fn run(&self) -> anyhow::Result<()> { + tracing::info!( + "🔧 Worker started - processing queues: {:?}", + self.config.queues + ); + + loop { + let mut processed = false; + + for queue in &self.config.queues { + if let Some(job_id) = self.pop_job(queue).await? { + self.process_job(queue, &job_id).await?; + processed = true; + } + } + + if !processed { + // No jobs available, sleep + sleep(self.config.sleep_duration).await; + } + } + } + + /// Pop a job from the queue. + async fn pop_job(&self, queue: &str) -> anyhow::Result> { + let queue_key = format!("jobs:queue:{}", queue); + let mut conn = self.redis_pool.get().await?; + + let job_id: Option = conn.lpop(&queue_key, None).await?; + Ok(job_id) + } + + /// Process a single job. + async fn process_job(&self, queue: &str, job_id: &str) -> anyhow::Result<()> { + let job_key = format!("jobs:data:{}", job_id); + let meta_key = format!("{}:meta", job_key); + + let mut conn = self.redis_pool.get().await?; + + // Get job metadata + let meta_json: Option = conn.get(&meta_key).await?; + let mut meta: JobMeta = match meta_json { + Some(json) => serde_json::from_str(&json)?, + None => { + tracing::warn!("Job {} not found", job_id); + return Ok(()); + } + }; + + // Get job payload + let payload: Option = conn.get(&job_key).await?; + let payload = match payload { + Some(p) => p, + None => { + tracing::warn!("Job {} payload not found", job_id); + return Ok(()); + } + }; + + // Update status to processing + meta.status = JobStatus::Processing; + meta.started_at = Some(chrono::Utc::now()); + meta.attempts += 1; + let _: () = conn.set(&meta_key, serde_json::to_string(&meta)?).await?; + + tracing::info!( + "Processing job {} ({}) - attempt {}/{}", + meta.job_type, + job_id, + meta.attempts, + meta.max_attempts + ); + + // Find handler + let handler = match self.handlers.get(meta.job_type.as_str()) { + Some(h) => h, + None => { + tracing::error!("No handler registered for job type: {}", meta.job_type); + meta.status = JobStatus::Failed; + meta.error = Some(format!("No handler for job type: {}", meta.job_type)); + meta.completed_at = Some(chrono::Utc::now()); + let _: () = conn.set(&meta_key, serde_json::to_string(&meta)?).await?; + return Ok(()); + } + }; + + // Execute job + match handler.process(&payload).await { + Ok(()) => { + meta.status = JobStatus::Completed; + meta.completed_at = Some(chrono::Utc::now()); + tracing::info!("Job {} completed successfully", job_id); + } + Err(e) => { + let error_msg = e.to_string(); + tracing::error!("Job {} failed: {}", job_id, error_msg); + + if meta.attempts >= meta.max_attempts { + meta.status = JobStatus::Failed; + meta.error = Some(error_msg); + meta.completed_at = Some(chrono::Utc::now()); + } else { + // Retry - push back to queue + meta.status = JobStatus::Pending; + meta.error = Some(format!("Attempt {} failed: {}", meta.attempts, error_msg)); + let queue_key = format!("jobs:queue:{}", queue); + let _: () = conn.rpush(&queue_key, job_id).await?; + tracing::info!("Job {} queued for retry", job_id); + } + } + } + + // Save final status + let _: () = conn.set(&meta_key, serde_json::to_string(&meta)?).await?; + + Ok(()) + } +} diff --git a/src/shared/middlewares/logging.rs b/src/shared/middlewares/logging.rs new file mode 100644 index 0000000..d8066f7 --- /dev/null +++ b/src/shared/middlewares/logging.rs @@ -0,0 +1,289 @@ +//! Request/Response logging middleware. +//! +//! Provides structured logging for HTTP requests with timing and correlation. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::shared::middlewares::logging::{LoggingConfig, with_logging}; +//! use axum::Router; +//! +//! let app = Router::new() +//! .route("/api/test", get(handler)) +//! .layer(axum::middleware::from_fn(with_logging(LoggingConfig::default()))); +//! ``` + +use axum::{body::Body, extract::Request, http::StatusCode, middleware::Next, response::Response}; +use std::collections::HashSet; +use std::sync::Arc; +use std::time::Instant; +use uuid::Uuid; + +/// Logging configuration. +#[derive(Debug, Clone)] +pub struct LoggingConfig { + /// Log request headers + pub log_headers: bool, + /// Log request body (be careful with sensitive data) + pub log_body: bool, + /// Log response body + pub log_response_body: bool, + /// Maximum body size to log (bytes) + pub max_body_size: usize, + /// Paths to exclude from logging + pub exclude_paths: HashSet, + /// Log level for successful requests + pub success_level: LogLevel, + /// Log level for client errors (4xx) + pub client_error_level: LogLevel, + /// Log level for server errors (5xx) + pub server_error_level: LogLevel, +} + +/// Log level enum. +#[derive(Debug, Clone, Copy)] +pub enum LogLevel { + Trace, + Debug, + Info, + Warn, + Error, +} + +impl Default for LoggingConfig { + fn default() -> Self { + let mut exclude = HashSet::new(); + exclude.insert("/health".to_string()); + exclude.insert("/favicon.ico".to_string()); + + Self { + log_headers: false, + log_body: false, + log_response_body: false, + max_body_size: 1024, + exclude_paths: exclude, + success_level: LogLevel::Info, + client_error_level: LogLevel::Warn, + server_error_level: LogLevel::Error, + } + } +} + +impl LoggingConfig { + /// Create a new logging config. + pub fn new() -> Self { + Self::default() + } + + /// Enable header logging. + pub fn with_headers(mut self) -> Self { + self.log_headers = true; + self + } + + /// Add a path to exclude from logging. + pub fn exclude_path(mut self, path: &str) -> Self { + self.exclude_paths.insert(path.to_string()); + self + } +} + +/// Request ID extension for correlation. +#[derive(Clone, Debug)] +pub struct RequestId(pub String); + +impl RequestId { + /// Generate a new request ID. + pub fn new() -> Self { + Self(Uuid::new_v4().to_string()) + } + + /// Get the request ID as a string. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Default for RequestId { + fn default() -> Self { + Self::new() + } +} + +/// Logging middleware. +pub async fn logging_middleware(config: Arc, req: Request, next: Next) -> Response { + let path = req.uri().path(); + + if config.exclude_paths.contains(path) { + return next.run(req).await; + } + + let request_id = req + .headers() + .get("X-Request-ID") + .and_then(|h| h.to_str().ok()) + .map(|s| RequestId(s.to_string())) + .unwrap_or_else(RequestId::new); + + let method = req.method().clone(); + let uri = req.uri().clone(); + let path = uri.path().to_string(); + let query = uri.query().map(str::to_owned); + + let headers_log = if config.log_headers { + let headers: Vec = req + .headers() + .iter() + .filter(|(name, _)| !is_sensitive_header(name.as_str())) + .map(|(name, value)| format!("{}: {}", name, value.to_str().unwrap_or(""))) + .collect(); + Some(headers) + } else { + None + }; + + let mut req = req; + req.extensions_mut().insert(request_id.clone()); + + let start = Instant::now(); + + tracing::info!( + request_id = %request_id.0, + method = %method, + path = %path, + query = ?query, + "→ Request started" + ); + + if let Some(ref headers) = headers_log { + tracing::debug!(request_id = %request_id.0, headers = ?headers, "Request headers"); + } + + let response = next.run(req).await; + + let duration = start.elapsed(); + let status = response.status(); + + match status.as_u16() { + 100..=399 => log_response( + config.success_level, + &request_id, + status, + duration, + &method, + &path, + ), + 400..=499 => log_response( + config.client_error_level, + &request_id, + status, + duration, + &method, + &path, + ), + _ => log_response( + config.server_error_level, + &request_id, + status, + duration, + &method, + &path, + ), + } + + response +} + +fn is_sensitive_header(name: &str) -> bool { + name.eq_ignore_ascii_case("authorization") + || name.eq_ignore_ascii_case("cookie") + || name.eq_ignore_ascii_case("x-api-key") +} + +fn log_response( + level: LogLevel, + request_id: &RequestId, + status: StatusCode, + duration: std::time::Duration, + method: &axum::http::Method, + path: &str, +) { + let duration_ms = duration.as_millis(); + + match level { + LogLevel::Trace => tracing::trace!( + request_id = %request_id.0, + status = %status, + duration_ms = %duration_ms, + method = %method, + path = %path, + "← Response completed" + ), + LogLevel::Debug => tracing::debug!( + request_id = %request_id.0, + status = %status, + duration_ms = %duration_ms, + method = %method, + path = %path, + "← Response completed" + ), + LogLevel::Info => tracing::info!( + request_id = %request_id.0, + status = %status, + duration_ms = %duration_ms, + method = %method, + path = %path, + "← Response completed" + ), + LogLevel::Warn => tracing::warn!( + request_id = %request_id.0, + status = %status, + duration_ms = %duration_ms, + method = %method, + path = %path, + "← Response completed" + ), + LogLevel::Error => tracing::error!( + request_id = %request_id.0, + status = %status, + duration_ms = %duration_ms, + method = %method, + path = %path, + "← Response completed" + ), + } +} + +/// Create logging middleware. +pub fn with_logging( + config: LoggingConfig, +) -> impl Fn( + Request, + Next, +) -> std::pin::Pin + Send>> + + Clone + + Send { + let config = Arc::new(config); + move |req: Request, next: Next| { + let config = config.clone(); + Box::pin(async move { logging_middleware(config, req, next).await }) + } +} + +/// Extractor for RequestId in route handlers. +impl axum::extract::FromRequestParts for RequestId +where + S: Send + Sync, +{ + type Rejection = (StatusCode, &'static str); + + async fn from_request_parts( + parts: &mut axum::http::request::Parts, + _state: &S, + ) -> Result { + parts.extensions.get::().cloned().ok_or(( + StatusCode::INTERNAL_SERVER_ERROR, + "RequestId not found. Did you add logging middleware?", + )) + } +} diff --git a/src/shared/middlewares/mod.rs b/src/shared/middlewares/mod.rs new file mode 100644 index 0000000..abd0d0e --- /dev/null +++ b/src/shared/middlewares/mod.rs @@ -0,0 +1,2 @@ +pub mod ratelimit; +pub use ratelimit::rate_limit_middleware; diff --git a/src/shared/middlewares/ratelimit.rs b/src/shared/middlewares/ratelimit.rs new file mode 100644 index 0000000..9e5a20e --- /dev/null +++ b/src/shared/middlewares/ratelimit.rs @@ -0,0 +1,101 @@ +//! Rate limiter middleware using the governor crate. +//! +//! Provides token-bucket based rate limiting with configurable limits. +//! Default: 20 requests per IP address, burst 50. + +use axum::{ + extract::Request, + http::StatusCode, + middleware::Next, + response::{IntoResponse, Response}, + Json, +}; +use governor::{ + clock::DefaultClock, state::keyed::DefaultKeyedStateStore, Quota, + RateLimiter as GovernorRateLimiter, +}; +use once_cell::sync::Lazy; +use serde_json::json; +use std::{num::NonZeroU32, sync::Arc, time::Duration}; +use tracing::warn; + +/// Global IP-keyed rate limiter instance. +/// Configured for 20 requests per second with a burst of 50. +static IP_LIMITER: Lazy< + Arc, DefaultClock>>, +> = Lazy::new(|| { + let quota = Quota::with_period(Duration::from_millis(50)) + .unwrap() + .allow_burst(NonZeroU32::new(50).unwrap()); + + Arc::new(GovernorRateLimiter::keyed(quota)) +}); + +/// Rate limiter configuration. +#[derive(Debug, Clone)] +pub struct RateLimiterConfig { + /// Requests per second + pub requests_per_second: u32, + /// Burst size (max requests that can be made instantly) + pub burst_size: u32, +} + +impl Default for RateLimiterConfig { + fn default() -> Self { + Self { + requests_per_second: 20, + burst_size: 50, + } + } +} + +/// Create a custom rate limiter with specific configuration. +pub fn create_rate_limiter( + config: RateLimiterConfig, +) -> Arc, DefaultClock>> { + let period_ms = 1000 / config.requests_per_second.max(1); + let quota = Quota::with_period(Duration::from_millis(period_ms as u64)) + .unwrap() + .allow_burst(NonZeroU32::new(config.burst_size.max(1)).unwrap()); + + Arc::new(GovernorRateLimiter::keyed(quota)) +} + +fn extract_ip(req: &Request) -> String { + if let Some(ip) = req.headers().get("X-Forwarded-For") { + if let Ok(ip_str) = ip.to_str() { + if let Some(first_ip) = ip_str.split(',').next() { + return first_ip.trim().to_string(); + } + } + } + if let Some(ip) = req.headers().get("X-Real-IP") { + if let Ok(ip_str) = ip.to_str() { + return ip_str.trim().to_string(); + } + } + "unknown".to_string() +} + +/// Rate limiting middleware using the IP-keyed limiter. +/// +/// Returns 429 Too Many Requests if the limit is exceeded. +pub async fn rate_limit_middleware(req: Request, next: Next) -> Response { + let client_ip = extract_ip(&req); + + match IP_LIMITER.check_key(&client_ip) { + Ok(_) => next.run(req).await, + Err(_) => { + warn!("Rate limit exceeded for IP: {}", client_ip); + ( + StatusCode::TOO_MANY_REQUESTS, + Json(json!({ + "error": "Too many requests", + "code": "RATE_LIMIT_EXCEEDED", + "retry_after_ms": 1000 + })), + ) + .into_response() + } + } +} diff --git a/src/shared/mod.rs b/src/shared/mod.rs new file mode 100644 index 0000000..358cb1a --- /dev/null +++ b/src/shared/mod.rs @@ -0,0 +1,17 @@ +pub mod browser; +pub mod config; +pub mod database; +pub mod errors; +pub mod events; +pub mod graceful; +pub mod health; +pub mod middlewares; +pub mod observability; +pub mod routing; +pub mod scheduler; +pub mod scrapers; +pub mod services; +pub mod state; +pub mod testing; +pub mod types; +pub mod utils; diff --git a/src/shared/observability/metrics.rs b/src/shared/observability/metrics.rs new file mode 100644 index 0000000..5aa02b6 --- /dev/null +++ b/src/shared/observability/metrics.rs @@ -0,0 +1,171 @@ +//! OpenTelemetry metrics initialization for the scraper service. +//! +//! Provides: +//! - Global `MeterProvider` connected via OTLP gRPC to the metrics backend +//! - Standard HTTP server metrics middleware +//! +//! Environment: +//! OTEL_EXPORTER_OTLP_ENDPOINT — default: http://localhost:4317 +//! OTEL_SERVICE_NAME — default: scraper-api +//! OTEL_METRICS_EXPORT_INTERVAL — default: 5000 (ms) + +use axum::{extract::Request, middleware::Next, response::Response}; +use opentelemetry::{ + global, + metrics::{Counter, Histogram, Meter, UpDownCounter}, + KeyValue, +}; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_sdk::{metrics::MeterProviderBuilder, metrics::PeriodicReader, Resource}; +use std::sync::OnceLock; +use std::time::Instant; + +static METER: OnceLock = OnceLock::new(); +static PROVIDER: OnceLock = OnceLock::new(); + +fn meter() -> &'static Meter { + METER.get().expect("OTel meter not initialized — call init_otel_metrics first") +} + +/// Initialize the global OTLP MeterProvider. +/// Safe to call multiple times — subsequent calls are no-ops. +pub fn init_otel_metrics() { + if METER.get().is_some() { + return; + } + + let otel_endpoint = + std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").unwrap_or_else(|_| "http://localhost:4317".into()); + let service_name = + std::env::var("OTEL_SERVICE_NAME").unwrap_or_else(|_| "scraper-api".into()); + let export_interval_ms: u64 = std::env::var("OTEL_METRICS_EXPORT_INTERVAL") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(5000); + + // Build the gRPC OTLP exporter + let exporter = opentelemetry_otlp::MetricExporter::builder() + .with_tonic() + .with_endpoint(otel_endpoint.clone()) + .build() + .expect("Failed to create OTLP metric exporter"); + + let reader = PeriodicReader::builder(exporter, opentelemetry_sdk::runtime::Tokio) + .with_interval(std::time::Duration::from_millis(export_interval_ms)) + .build(); + + let resource = Resource::new(vec![ + KeyValue::new("service.name", service_name.clone()), + ]); + + let provider = MeterProviderBuilder::default() + .with_resource(resource) + .with_reader(reader) + .build(); + + // Keep a handle so we can shut it down later + let _ = PROVIDER.set(provider.clone()); + + global::set_meter_provider(provider); + + let m = global::meter("scraper-http-server"); + let _ = METER.set(m); + + tracing::info!(otel_endpoint, service_name, "OTel metrics initialized"); +} + +/// Shut down the global MeterProvider, flushing pending exports. +pub async fn shutdown_otel_metrics() { + if let Some(provider) = PROVIDER.get() { + if let Err(e) = provider.shutdown() { + tracing::warn!(error = %e, "OTel metrics shutdown error"); + } else { + tracing::info!("OTel metrics shut down"); + } + } +} + +// --------------------------------------------------------------------------- +// HTTP metrics middleware +// --------------------------------------------------------------------------- + +/// Axum middleware that records standard HTTP server metrics for every request. +/// +/// This middleware must be added as a layer **after** `init_otel_metrics()` has been called. +/// +/// Metrics emitted: +/// - `http.server.request_count` — Counter { method, path, status } +/// - `http.server.request_duration_ms` — Histogram { method, path, status } +/// - `http.server.request_in_flight` — UpDownCounter { method, path } +pub async fn otel_metrics_middleware(req: Request, next: Next) -> Response { + let method = req.method().to_string(); + let path = req.uri().path().to_string(); + + let in_flight = get_in_flight_counter(); + let counter = get_request_counter(); + let duration = get_duration_histogram(); + + // Record in-flight + in_flight.add( + 1, + &[ + KeyValue::new("method", method.clone()), + KeyValue::new("path", path.clone()), + ], + ); + + let start = Instant::now(); + let response = next.run(req).await; + let elapsed_ms = start.elapsed().as_millis() as f64; + + let status = response.status().as_u16().to_string(); + + // Record request count + duration + let attrs = [ + KeyValue::new("method", method), + KeyValue::new("path", path), + KeyValue::new("status", status), + ]; + counter.add(1, &attrs); + duration.record(elapsed_ms, &attrs); + + // Decrement in-flight + in_flight.add(-1, &[]); + + response +} + +/// Lazily-initialised instruments guarded by OnceLock. +fn get_in_flight_counter() -> UpDownCounter { + static INST: OnceLock> = OnceLock::new(); + INST.get_or_init(|| { + meter() + .i64_up_down_counter("http.server.request_in_flight") + .with_description("Number of HTTP requests currently in flight") + .build() + }) + .clone() +} + +fn get_request_counter() -> Counter { + static INST: OnceLock> = OnceLock::new(); + INST.get_or_init(|| { + meter() + .u64_counter("http.server.request_count") + .with_description("Total number of HTTP requests received") + .build() + }) + .clone() +} + +fn get_duration_histogram() -> Histogram { + static INST: OnceLock> = OnceLock::new(); + INST.get_or_init(|| { + meter() + .f64_histogram("http.server.request_duration_ms") + .with_description("Duration of HTTP requests in milliseconds") + .with_unit("ms") + .build() + }) + .clone() +} diff --git a/src/shared/observability/mod.rs b/src/shared/observability/mod.rs new file mode 100644 index 0000000..8255913 --- /dev/null +++ b/src/shared/observability/mod.rs @@ -0,0 +1,4 @@ +pub mod metrics; +pub mod openapi; +pub mod openapi_modules; +pub mod request_id; diff --git a/src/shared/observability/openapi.rs b/src/shared/observability/openapi.rs new file mode 100644 index 0000000..4c0c820 --- /dev/null +++ b/src/shared/observability/openapi.rs @@ -0,0 +1,14 @@ +use utoipa::OpenApi; + +/// Bridge for the auto-generated OpenAPI documentation. +/// This allows merging manual schema definitions with the auto-discovered handlers and schemas. +#[derive(OpenApi)] +#[openapi( + // Discovered routes are merged at runtime in bootstrap/mod.rs + info( + title = "Scraper API", + version = "1.0.0", + description = "High-performance scraping and CDN microservice" + ) +)] +pub struct ApiDoc; diff --git a/src/shared/observability/openapi_modules.rs b/src/shared/observability/openapi_modules.rs new file mode 100644 index 0000000..0a28aaa --- /dev/null +++ b/src/shared/observability/openapi_modules.rs @@ -0,0 +1,85 @@ +use utoipa::OpenApi; + +/// Manual aggregation of OpenAPI docs from module controllers. +#[derive(OpenApi)] +#[openapi( + paths( + // Anime module controllers + crate::modules::anime::controller::anime_index, + crate::modules::anime::controller::genres, + crate::modules::anime::controller::detail_slug, + crate::modules::anime::controller::complete_anime_slug, + crate::modules::anime::controller::full_slug, + crate::modules::anime::controller::ongoing_anime_slug, + crate::modules::anime::controller::latest_slug, + crate::modules::anime::controller::search_slug_index, + crate::modules::anime::controller::search_slug_page, + crate::modules::anime::controller::genre_slug_index, + crate::modules::anime::controller::genre_slug_page, + // Anime2 module controllers + crate::modules::anime2::controller::index, + crate::modules::anime2::controller::genre_list, + crate::modules::anime2::controller::filter, + crate::modules::anime2::controller::detail_slug, + crate::modules::anime2::controller::genre_slug_index, + crate::modules::anime2::controller::genre_slug_page, + crate::modules::anime2::controller::search_slug_index, + crate::modules::anime2::controller::search_slug_page, + crate::modules::anime2::controller::latest_slug, + crate::modules::anime2::controller::ongoing_anime_slug, + crate::modules::anime2::controller::complete_anime_slug, + // Komik module controllers + crate::modules::komik::controller::genre_list, + crate::modules::komik::controller::chapter_slug, + crate::modules::komik::controller::detail_slug, + crate::modules::komik::controller::genre_slug, + crate::modules::komik::controller::genre_slug_page, + crate::modules::komik::controller::manga_slug, + crate::modules::komik::controller::manhua_slug, + crate::modules::komik::controller::manhwa_slug, + crate::modules::komik::controller::popular_slug, + crate::modules::komik::controller::search_slug, + crate::modules::komik::controller::search_slug_page, + // Proxy module controllers + crate::modules::proxy::controller::fetch_with_proxy_only, + crate::modules::proxy::controller::image_cache, + ), + components( + schemas( + // Application response wrapper + crate::shared::types::ApiResponse, + // Anime2 types + crate::modules::anime2::types::Anime2Response, + crate::modules::anime2::types::GenresResponse, + crate::modules::anime2::types::FilterResponse, + crate::modules::anime2::types::DetailResponse, + // Anime2 query schemas + crate::modules::anime2::schema::FilterQuery, + crate::modules::anime2::schema::GenreQuery, + crate::modules::anime2::schema::SearchQuery, + // Proxy types + crate::modules::proxy::types::ImageCacheResponse, + crate::modules::proxy::types::AuditImageCacheResponse, + // Proxy request schemas + crate::modules::proxy::schema::ProxyParams, + crate::modules::proxy::schema::ImageCacheRequest, + crate::modules::proxy::schema::AuditImageCacheRequest, + // Komik types + crate::modules::komik::types::GenresResponse, + crate::modules::komik::types::Genre, + crate::modules::komik::types::ChapterResponse, + crate::modules::komik::types::DetailResponse, + crate::modules::komik::types::KomikItem, + crate::modules::komik::types::Pagination, + crate::modules::komik::types::GenreKomikResponse, + crate::modules::komik::types::SearchKomikResponse, + ) + ), + tags( + (name = "anime", description = "Anime endpoints"), + (name = "anime2", description = "Anime2 endpoints"), + (name = "komik", description = "Komik endpoints"), + (name = "proxy", description = "Proxy endpoints"), + ) +)] +pub struct ModuleApiDoc; diff --git a/src/shared/observability/request_id.rs b/src/shared/observability/request_id.rs new file mode 100644 index 0000000..b89312a --- /dev/null +++ b/src/shared/observability/request_id.rs @@ -0,0 +1,85 @@ +//! Request ID middleware for request tracing. + +use axum::{extract::Request, http::HeaderValue, middleware::Next, response::Response}; +use uuid::Uuid; + +/// Request ID header name. +pub const REQUEST_ID_HEADER: &str = "x-request-id"; + +/// Extension to access request ID in handlers. +#[derive(Clone, Debug)] +pub struct RequestId(pub String); + +impl RequestId { + /// Generate a new request ID. + pub fn new() -> Self { + Self(Uuid::new_v4().to_string()) + } + + /// Get the request ID string. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Default for RequestId { + fn default() -> Self { + Self::new() + } +} + +impl std::fmt::Display for RequestId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +/// Middleware that adds a unique request ID to each request. +/// +/// The request ID is: +/// - Taken from the `x-request-id` header if present +/// - Generated as a new UUID if not present +/// - Added to the response headers +/// - Available via the `RequestId` extension in handlers +/// +/// # Example +/// +/// ```ignore +/// use axum::Extension; +/// use scraper_service::observability::RequestId; +/// +/// async fn handler(Extension(req_id): Extension) { +/// println!("Request ID: {}", req_id); +/// } +/// ``` +pub async fn request_id_middleware(mut req: Request, next: Next) -> Response { + // Get or generate request ID + let request_id = req + .headers() + .get(REQUEST_ID_HEADER) + .and_then(|v| v.to_str().ok()) + .map(|s| RequestId(s.to_string())) + .unwrap_or_else(RequestId::new); + + // Add to tracing span + let span = tracing::info_span!( + "request", + request_id = %request_id, + method = %req.method(), + uri = %req.uri(), + ); + let _guard = span.enter(); + + // Insert as extension for handlers + req.extensions_mut().insert(request_id.clone()); + + // Process request + let mut response = next.run(req).await; + + // Add request ID to response headers + if let Ok(value) = HeaderValue::from_str(&request_id.0) { + response.headers_mut().insert(REQUEST_ID_HEADER, value); + } + + response +} diff --git a/src/shared/routing/mod.rs b/src/shared/routing/mod.rs new file mode 100644 index 0000000..890dc86 --- /dev/null +++ b/src/shared/routing/mod.rs @@ -0,0 +1,3 @@ +pub mod versioning; + +pub use versioning::{extract_version, versioned_routes, ApiVersion, VersionedApi}; diff --git a/src/shared/routing/versioning.rs b/src/shared/routing/versioning.rs new file mode 100644 index 0000000..a1f2f25 --- /dev/null +++ b/src/shared/routing/versioning.rs @@ -0,0 +1,135 @@ +//! API versioning utilities. +//! +//! Provides helpers for versioned API routes. + +use axum::Router; + +/// API version prefix. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ApiVersion { + V1, + V2, +} + +impl ApiVersion { + /// Get the URL prefix for this version. + pub fn prefix(&self) -> &'static str { + match self { + ApiVersion::V1 => "/api/v1", + ApiVersion::V2 => "/api/v2", + } + } +} + +impl std::fmt::Display for ApiVersion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ApiVersion::V1 => write!(f, "v1"), + ApiVersion::V2 => write!(f, "v2"), + } + } +} + +/// Builder for versioned API routes. +/// +/// # Example +/// +/// ```ignore +/// use scraper_service::versioning::VersionedApi; +/// +/// let app = VersionedApi::new() +/// .v1(users_v1_routes()) +/// .v1(products_v1_routes()) +/// .v2(users_v2_routes()) +/// .build(); +/// ``` +pub struct VersionedApi +where + S: Clone + Send + Sync + 'static, +{ + v1_routes: Vec>, + v2_routes: Vec>, +} + +impl VersionedApi +where + S: Clone + Send + Sync + 'static, +{ + /// Create a new versioned API builder. + pub fn new() -> Self { + Self { + v1_routes: Vec::new(), + v2_routes: Vec::new(), + } + } + + /// Add routes to API v1. + pub fn v1(mut self, routes: Router) -> Self { + self.v1_routes.push(routes); + self + } + + /// Add routes to API v2. + pub fn v2(mut self, routes: Router) -> Self { + self.v2_routes.push(routes); + self + } + + /// Build the combined router with versioned prefixes. + pub fn build(self) -> Router { + let mut app = Router::new(); + + // Merge v1 routes under /api/v1 + if !self.v1_routes.is_empty() { + let mut v1_router = Router::new(); + for routes in self.v1_routes { + v1_router = v1_router.merge(routes); + } + app = app.nest("/api/v1", v1_router); + } + + // Merge v2 routes under /api/v2 + if !self.v2_routes.is_empty() { + let mut v2_router = Router::new(); + for routes in self.v2_routes { + v2_router = v2_router.merge(routes); + } + app = app.nest("/api/v2", v2_router); + } + + app + } +} + +impl Default for VersionedApi +where + S: Clone + Send + Sync + 'static, +{ + fn default() -> Self { + Self::new() + } +} + +/// Create versioned routes with automatic fallback. +/// +/// Routes in v2 will be served under /api/v2. +/// Routes in v1 will be served under /api/v1 AND as fallback under /api/v2 if not overridden. +pub fn versioned_routes(v1: Router, v2: Router) -> Router +where + S: Clone + Send + Sync + 'static, +{ + Router::new() + .nest("/api/v1", v1.clone()) + .nest("/api/v2", v1.merge(v2)) // v2 includes v1 as fallback +} + +/// Extract API version from request path. +pub fn extract_version(path: &str) -> Option { + if path.starts_with("/api/v2") { + Some(ApiVersion::V2) + } else if path.starts_with("/api/v1") { + Some(ApiVersion::V1) + } else { + None + } +} diff --git a/src/shared/scheduler/cleanup_cache.rs b/src/shared/scheduler/cleanup_cache.rs new file mode 100644 index 0000000..bda09bf --- /dev/null +++ b/src/shared/scheduler/cleanup_cache.rs @@ -0,0 +1,222 @@ +//! Scheduled task for cleaning up old cached data. + +use async_trait::async_trait; +use sea_orm::*; +use std::sync::Arc; +use tracing::{info, warn}; + +use crate::shared::database::get_redis_pool; +use crate::shared::database::persistence::entities::image_cache; +use crate::shared::utils::cache::Cache; + +use super::ScheduledTask; + +/// Cleanup old cache data to prevent disk/memory bloat. +/// Runs daily at 2 AM to clean: +/// - Old image cache entries (>30 days) +/// - Orphaned cache keys in Redis +/// - Expired data without TTL +pub struct CleanupOldCache { + db: Arc, +} + +impl CleanupOldCache { + pub fn new(db: Arc) -> Self { + Self { db } + } +} + +#[async_trait] +impl ScheduledTask for CleanupOldCache { + fn name(&self) -> &'static str { + "cleanup_old_cache" + } + + fn schedule(&self) -> &'static str { + // Daily at 2 AM + "0 0 2 * * *" + } + + async fn run(&self) { + tracing::debug!("🧹 Starting old cache cleanup..."); + + let mut total_cleaned = 0; + + // 1. Clean old image cache (>30 days) + match self.cleanup_old_images(30).await { + Ok(count) => { + if count > 0 { + info!("✓ Cleaned {} old image cache entries", count); + } + total_cleaned += count; + } + Err(e) => { + warn!("Failed to clean old image cache: {}", e); + } + } + + // 2. Clean orphaned Redis keys + match self.cleanup_orphaned_redis_keys().await { + Ok(count) => { + if count > 0 { + info!("✓ Cleaned {} orphaned Redis keys", count); + } + total_cleaned += count; + } + Err(e) => { + warn!("Failed to clean orphaned Redis keys: {}", e); + } + } + + // 3. Compact Redis memory + match self.compact_redis_memory().await { + Ok(()) => { + if total_cleaned > 0 { + info!("✓ Redis memory compacted"); + } + } + Err(e) => { + warn!("Failed to compact Redis memory: {}", e); + } + } + + if total_cleaned > 0 { + info!("🎉 Cache cleanup complete: {} items cleaned", total_cleaned); + } + } +} + +impl CleanupOldCache { + /// Remove image cache entries older than specified days. + async fn cleanup_old_images(&self, days: i64) -> Result { + use chrono::{Duration, Utc}; + + let cutoff = Utc::now() - Duration::days(days); + + // Find old entries + let old_images = image_cache::Entity::find() + .filter(image_cache::Column::CreatedAt.lt(cutoff)) + .all(self.db.as_ref()) + .await + .map_err(|e| e.to_string())?; + + let count = old_images.len(); + + if count == 0 { + return Ok(0); + } + + // Delete from database + let ids: Vec = old_images.iter().map(|img| img.id.clone()).collect(); + + image_cache::Entity::delete_many() + .filter(image_cache::Column::Id.is_in(ids)) + .exec(self.db.as_ref()) + .await + .map_err(|e| e.to_string())?; + + // Also clean from Redis + let redis_pool = get_redis_pool().map_err(|e| e.to_string())?; + let cache = Cache::new(redis_pool); + for img in old_images { + let cache_key = format!("img_cache:{}", Self::hash_url(&img.original_url)); + let _ = cache.delete(&cache_key).await; + } + + Ok(count) + } + + /// Clean orphaned Redis keys (keys without TTL that shouldn't exist). + async fn cleanup_orphaned_redis_keys(&self) -> Result { + use deadpool_redis::redis::AsyncCommands; + + let pool = get_redis_pool().map_err(|e| e.to_string())?; + let mut conn = pool + .get() + .await + .map_err(|e| format!("Failed to get Redis connection: {}", e))?; + + let mut cleaned = 0; + + // Find keys without TTL (should not exist) + let patterns = vec!["anime:*", "komik:*", "user:*:profile", "img_cache:*"]; + + for pattern in patterns { + let keys = { + let mut iter: deadpool_redis::redis::AsyncIter<'_, String> = conn + .scan_match(pattern) + .await + .map_err(|e| format!("Failed to scan keys: {}", e))?; + + let mut keys = Vec::new(); + while let Some(key_result) = iter.next_item().await { + if let Ok(key) = key_result { + keys.push(key); + } + } + + keys + }; + + for key in keys { + let ttl: i64 = conn.ttl(key.as_str()).await.unwrap_or(-1); + + // TTL = -1 means no expiration (orphaned) + // TTL = -2 means key doesn't exist + if ttl == -1 { + // Set a default TTL of 7 days for orphaned keys + let _: () = conn.expire(key.as_str(), 604800).await.unwrap_or(()); + cleaned += 1; + } + } + } + + Ok(cleaned) + } + + /// Compact Redis memory to free up fragmented space. + async fn compact_redis_memory(&self) -> Result<(), String> { + let pool = get_redis_pool().map_err(|e| e.to_string())?; + let mut conn = pool + .get() + .await + .map_err(|e| format!("Failed to get Redis connection: {}", e))?; + + // Run MEMORY PURGE command - using cmd method on connection + let _: String = deadpool_redis::redis::cmd("MEMORY") + .arg("PURGE") + .query_async(&mut *conn) + .await + .map_err(|e| format!("Failed to purge memory: {}", e))?; + + Ok(()) + } + + /// Simple hash function for URL (same as in image_cache.rs). + /// Simple hash function for URL (same as in image_cache.rs). + fn hash_url(url: &str) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(url.as_bytes()); + format!("{:x}", hasher.finalize()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_schedule() { + use sea_orm::{DatabaseBackend, MockDatabase}; + let db = MockDatabase::new(DatabaseBackend::Sqlite).into_connection(); + let task = CleanupOldCache { db: Arc::new(db) }; + assert_eq!(task.schedule(), "0 0 2 * * *"); + } + + #[test] + fn test_hash_url() { + let hash = CleanupOldCache::hash_url("https://example.com/image.jpg"); + assert_eq!(hash.len(), 64); // SHA256 hash length + } +} diff --git a/src/shared/scheduler/mod.rs b/src/shared/scheduler/mod.rs new file mode 100644 index 0000000..7387dd2 --- /dev/null +++ b/src/shared/scheduler/mod.rs @@ -0,0 +1,5 @@ +pub mod cleanup_cache; +pub mod runner; + +pub use cleanup_cache::CleanupOldCache; +pub use runner::{ScheduledTask, Scheduler}; diff --git a/src/shared/scheduler/runner.rs b/src/shared/scheduler/runner.rs new file mode 100644 index 0000000..13be76c --- /dev/null +++ b/src/shared/scheduler/runner.rs @@ -0,0 +1,93 @@ +//! Scheduler implementation using tokio-cron-scheduler. + +use async_trait::async_trait; +use std::sync::Arc; +use tokio_cron_scheduler::{Job, JobScheduler}; +use tracing::info; + +/// Trait for scheduled tasks. +#[async_trait] +pub trait ScheduledTask: Send + Sync { + /// Task name for logging. + fn name(&self) -> &'static str; + + /// Cron expression (e.g., "0 * * * * *" for every minute). + fn schedule(&self) -> &'static str; + + /// Execute the task. + async fn run(&self); +} + +/// Scheduler for running cron jobs. +pub struct Scheduler { + inner: JobScheduler, +} + +impl Scheduler { + /// Create a new scheduler. + pub async fn new() -> anyhow::Result { + let scheduler = JobScheduler::new().await?; + Ok(Self { inner: scheduler }) + } + + /// Add a task to the scheduler. + pub async fn add(&self, task: T) -> anyhow::Result<()> { + let task = Arc::new(task); + let task_name = task.name(); + let schedule = task.schedule(); + + let job = Job::new_async(schedule, move |_uuid, _lock| { + let task = Arc::clone(&task); + Box::pin(async move { + tracing::debug!("Running scheduled task: {}", task.name()); + task.run().await; + }) + })?; + + self.inner.add(job).await?; + info!("Scheduled task '{}' with cron: {}", task_name, schedule); + Ok(()) + } + + /// Add a simple job with a closure. + pub async fn add_job( + &self, + name: &'static str, + schedule: &str, + f: F, + ) -> anyhow::Result<()> + where + F: Fn() -> Fut + Send + Sync + 'static, + Fut: std::future::Future + Send + 'static, + { + let f = Arc::new(f); + let job = Job::new_async(schedule, move |_uuid, _lock| { + let f = Arc::clone(&f); + Box::pin(async move { + info!("Running scheduled job: {}", name); + f().await; + }) + })?; + + self.inner.add(job).await?; + info!("Scheduled job '{}' with cron: {}", name, schedule); + Ok(()) + } + + /// Start the scheduler. + pub async fn start(&self) -> anyhow::Result<()> { + info!("Starting scheduler"); + self.inner.start().await?; + Ok(()) + } + + /// Stop the scheduler. + pub async fn shutdown(&mut self) -> anyhow::Result<()> { + info!("Shutting down scheduler"); + self.inner.shutdown().await?; + Ok(()) + } +} + +// Real scheduled tasks with actual implementations +// Real scheduled tasks with actual implementations diff --git a/src/shared/scrapers/mod.rs b/src/shared/scrapers/mod.rs new file mode 100644 index 0000000..b8a8e6a --- /dev/null +++ b/src/shared/scrapers/mod.rs @@ -0,0 +1 @@ +pub mod otakudesu; diff --git a/src/shared/scrapers/otakudesu.rs b/src/shared/scrapers/otakudesu.rs new file mode 100644 index 0000000..bde0fd2 --- /dev/null +++ b/src/shared/scrapers/otakudesu.rs @@ -0,0 +1,115 @@ +use crate::shared::types::entities::anime::*; +use crate::shared::utils::parse_html; +use crate::shared::utils::web::scraping::{ + attr, attr_from_or, extract_slug, selector, text, text_from_or, +}; +use once_cell::sync::Lazy; +use scraper::{Html, Selector}; + +/// Common selectors used across anime parsing +pub struct AnimeSelectors { + pub item: Selector, + pub title: Selector, + pub link: Selector, + pub img: Selector, + pub episode: Selector, + pub score: Selector, + pub status: Selector, + pub genre: Selector, + pub rating: Selector, + pub type_sel: Selector, + pub season: Selector, + pub desc: Selector, +} + +impl AnimeSelectors { + pub fn new() -> Result { + Ok(Self { + item: selector("article.bs").ok_or("Invalid selector: article.bs")?, + title: selector(".tt h2").ok_or("Invalid selector: .tt h2")?, + link: selector("a").ok_or("Invalid selector: a")?, + img: selector("img").ok_or("Invalid selector: img")?, + episode: selector(".epx").ok_or("Invalid selector: .epx")?, + score: selector(".numscore").ok_or("Invalid selector: .numscore")?, + status: selector(".status").ok_or("Invalid selector: .status")?, + genre: selector(".genres a").ok_or("Invalid selector: .genres a")?, + rating: selector(".score").ok_or("Invalid selector: .score")?, + type_sel: selector(".typez").ok_or("Invalid selector: .typez")?, + season: selector(".season").ok_or("Invalid selector: .season")?, + desc: selector(".data .typez").ok_or("Invalid selector: .data .typez")?, + }) + } +} + +static ANIME_SELECTORS: Lazy = + Lazy::new(|| AnimeSelectors::new().expect("Valid CSS selectors")); + +pub fn extract_poster(element: &scraper::ElementRef, img_selector: &Selector) -> String { + element + .select(img_selector) + .next() + .and_then(|e| attr(&e, "src").or(attr(&e, "data-src"))) + .unwrap_or_default() +} + +pub fn parse_ongoing_anime(html: &str) -> Vec { + let document = parse_html(html); + let selectors = &*ANIME_SELECTORS; + let mut items = Vec::new(); + + for element in document.select(&selectors.item) { + let title = text_from_or(&element, &selectors.title, ""); + if title.is_empty() { + continue; + } + + let href = attr_from_or(&element, &selectors.link, "href", ""); + let slug = extract_slug(&href); + let poster = extract_poster(&element, &selectors.img); + let current_episode = text_from_or(&element, &selectors.episode, "N/A"); + let anime_url = attr_from_or(&element, &selectors.link, "href", ""); + + items.push(OngoingAnimeItem { + title, + slug, + poster, + current_episode, + anime_url, + }); + } + items +} + +pub fn parse_pagination(document: &Html, current_page: u32) -> Result { + let pagination_selector = + selector(".pagination .page-numbers:not(.next)").ok_or("Invalid selector")?; + let next_selector = selector(".pagination .next").ok_or("Invalid selector")?; + + let last_visible_page = document + .select(&pagination_selector) + .next_back() + .and_then(|e| text(&e).trim().parse::().ok()) + .unwrap_or(current_page); + + let has_next_page = document.select(&next_selector).next().is_some(); + let next_page = if has_next_page { + Some(current_page + 1) + } else { + None + }; + let has_previous_page = current_page > 1; + let previous_page = if has_previous_page { + Some(current_page - 1) + } else { + None + }; + + Ok(Pagination { + current_page, + last_visible_page, + has_next_page, + next_page, + has_previous_page, + previous_page, + }) +} diff --git a/src/shared/services/images/cache.rs b/src/shared/services/images/cache.rs new file mode 100644 index 0000000..0670971 --- /dev/null +++ b/src/shared/services/images/cache.rs @@ -0,0 +1,1140 @@ +//! Image caching helper using Picser CDN (picser.pages.dev). +//! +//! This module provides utilities to cache images via jsDelivr CDN +//! with database storage for URL mapping. + +use crate::shared::config::CONFIG; +use crate::shared::database::repositories::image_cache::SeaOrmImageCacheRepository; +use crate::shared::database::traits::image_cache::ImageCacheRepository; +use deadpool_redis::Pool as RedisPool; +use reqwest::Client; +use sea_orm::DatabaseConnection; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use tracing::{debug, error, warn}; + +use crate::shared::utils::cache_ttl::CACHE_TTL_IMAGE; +use crate::shared::utils::web::http_client::http_client; +use crate::shared::utils::Cache; + +/// Default TTL for image cache in Redis (24 hours) +pub const IMAGE_CACHE_TTL: u64 = CACHE_TTL_IMAGE; + +/// Redis key prefix for image cache +pub const IMAGE_CACHE_PREFIX: &str = "img_cache"; + +/// Redis key prefix for caching locks (to prevent duplicate uploads) +pub const IMAGE_CACHE_LOCK_PREFIX: &str = "img_cache_lock"; + +/// Lock TTL (60 seconds - enough time for upload to complete) +pub const IMAGE_CACHE_LOCK_TTL: u64 = 60; + +/// Static Picser API endpoints in priority order. Configured endpoint is inserted after primary. +pub const STATIC_PICSER_API_ENDPOINTS: &[&str] = &[ + "https://picser-two.vercel.app/api/upload", + "https://picser-mytheclipse8647-ahoqi9ef.leapcell.dev/api/upload", + "https://picser.pages.dev/api/upload", +]; + +/// Create a hash of the URL for cache key +pub fn url_hash(url: &str) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(url.as_bytes()); + let result = hasher.finalize(); + hex::encode(&result[..16]) // Use 16 bytes for collision-resistant key +} + +/// Helper to convert any image URL to a fast WP.com (Jetpack) CDN URL. +/// This acts as a high-speed proxy even before Picser finishes caching. +pub fn to_wp_cdn(url: &str) -> String { + if url.is_empty() { + return url.to_string(); + } + + // If already a CDN URL, return as is + if url.contains("picser.pages.dev") + || url.contains("jsdelivr.net") + || url.contains("wp.com") + || url.contains("imagecdn.app") + { + return url.to_string(); + } + + // Remove protocol for wp.com format + let clean_url = url.trim().replace("https://", "").replace("http://", ""); + + // Use i0, i1, i2 or i3 based on hash to distribute load + let hash = url.len() % 4; + format!("https://i{}.wp.com/{}", hash, clean_url) +} + +/// Response from Picser API (/api/upload) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PicserResponse { + #[serde(default)] + pub success: bool, + pub url: Option, + pub urls: Option, + pub filename: Option, + pub size: Option, + #[serde(rename = "type")] + pub content_type: Option, + pub commit_sha: Option, + pub github_url: Option, + pub error: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PicserUrls { + pub github: Option, + pub raw: Option, + pub jsdelivr: Option, + pub jsdelivr_commit: Option, +} + +/// Response from fallback upload API (/api/upload) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FallbackUploadResponse { + pub download_url: String, +} + +/// Configuration for image cache +#[derive(Debug, Clone)] +pub struct ImageCacheConfig { + /// GitHub token for Picser API (optional - uses public upload if not set) + pub github_token: Option, + /// GitHub owner for uploads + pub github_owner: String, + /// GitHub repo for uploads + pub github_repo: String, + /// GitHub branch + pub github_branch: String, + /// Upload folder + pub folder: String, +} + +impl Default for ImageCacheConfig { + fn default() -> Self { + Self { + github_token: None, + github_owner: "sh20raj".to_string(), + github_repo: "picser".to_string(), + github_branch: "main".to_string(), + folder: "uploads".to_string(), + } + } +} + +/// Image cache service +pub struct ImageCache { + repo: Arc, + client: Client, + _config: ImageCacheConfig, + semaphore: Option>, +} + +// Add imports for Request Coalescing +use dashmap::DashMap; +use once_cell::sync::Lazy; +use tokio::sync::broadcast; + +// Global In-Flight Uploads Map +// Maps Original URL -> Broadcast Sender +static IN_FLIGHT_UPLOADS: Lazy>>> = + Lazy::new(DashMap::new); + +impl ImageCache { + /// Create a new image cache instance + pub fn new(repo: Arc) -> Self { + Self { + repo, + client: http_client().client().clone(), // Reuse global HTTP client for connection pooling + _config: ImageCacheConfig::default(), + semaphore: None, + } + } + + pub fn with_config(repo: Arc, config: ImageCacheConfig) -> Self { + Self { + repo, + client: http_client().client().clone(), // Reuse global HTTP client + _config: config, + semaphore: None, + } + } + + /// Set concurrency limiter + pub fn with_semaphore(mut self, semaphore: std::sync::Arc) -> Self { + self.semaphore = Some(semaphore); + self + } + + /// Get CDN URL for an image, caching if needed + pub async fn get_or_cache(&self, original_url: &str) -> Result { + let cache_key = format!("{}:{}", IMAGE_CACHE_PREFIX, url_hash(original_url)); + let lock_key = format!("{}:{}", IMAGE_CACHE_LOCK_PREFIX, url_hash(original_url)); + + // 1. Check Redis cache first + + if let Some(cached_url) = self.repo.get_from_redis(&cache_key).await { + debug!("ImageCache: Redis hit for {}", original_url); + return Ok(cached_url); + } + + // 2. Check Request Coalescing (SingleFlight) + // This handles concurrent requests in this process/instance + let (tx, is_leader) = { + use dashmap::mapref::entry::Entry; + match IN_FLIGHT_UPLOADS.entry(original_url.to_string()) { + Entry::Occupied(entry) => { + debug!("ImageCache: Joining in-flight upload for {}", original_url); + (entry.get().clone(), false) + } + Entry::Vacant(entry) => { + let (tx, _) = broadcast::channel(1); + entry.insert(tx.clone()); + debug!("ImageCache: Starting leader upload for {}", original_url); + (tx, true) + } + } + }; + + if !is_leader { + // Follower: Wait for result + let mut rx = tx.subscribe(); + return match rx.recv().await { + Ok(Ok(url)) => Ok(url), + Ok(Err(e)) => Err(e), + Err(e) => { + warn!( + "ImageCache: Coalesce receive error for {}: {:?}", + original_url, e + ); + Err("Upload coalescing failed".to_string()) + } + }; + } + + // Leader: Perform the work + // We wrap the work in a closure/block to easily capture the result + let result = async { + // 3. Check database (Double check inside leader to be sure) + if let Some(cached_url) = self.repo.get_from_db(original_url).await? { + // Store in Redis for faster access + let _ = self + .repo + .set_in_redis(&cache_key, &cached_url, IMAGE_CACHE_TTL) + .await; + debug!("ImageCache: DB hit for {}", original_url); + return Ok(cached_url); + } + + // 4. Check if another process is already caching this URL (Distributed Lock check) + if self.repo.get_lock(&lock_key).await { + // Even if locked by another process, strict single-flight within this instance + // is good. But if another process is working, we might want to wait or just return error? + // Current logic returns error. + debug!( + "ImageCache: Already being cached by another process: {}", + original_url + ); + return Err(format!("URL {} is already being cached", original_url)); + } + + // 5. Acquire lock in Redis + let _ = self.repo.set_lock(&lock_key, IMAGE_CACHE_LOCK_TTL).await; + + // 6. Upload + debug!("ImageCache: Miss - uploading {} to Picser", original_url); + + // Acquire permit if semaphore is set + let _permit = if let Some(sem) = &self.semaphore { + match sem.acquire().await { + Ok(p) => Some(p), + Err(e) => { + let _ = self.repo.release_lock(&lock_key).await; + return Err(e.to_string()); + } + } + } else { + None + }; + + // Work + let work_result = async { + // Upload to Picser + let cdn_url = self.upload_to_picser(original_url).await?; + + // 6.5. Verify CDN URL Propagation (Self-Test before caching) + // CDNs like jsDelivr can take a few seconds to propagate after a GitHub commit. + // We retry 3 times with backoff to ensure we only return and cache a functional link. + let mut is_valid = false; + let mut last_verify_error = String::from("Verification not started"); + + for attempt in 1..=10 { + debug!( + "ImageCache: Verifying CDN URL {} (Attempt {})", + cdn_url, attempt + ); + match self.verify_cdn_url(&cdn_url).await { + Ok(true) => { + is_valid = true; + debug!( + "ImageCache: CDN URL verified successfully for {}", + original_url + ); + break; + } + Ok(false) => { + last_verify_error = "CDN returned non-image data or 404".to_string(); + } + Err(e) => { + last_verify_error = e; + } + } + + if attempt < 10 { + // Progressive backoff: 1s, 2s, 3s... up to 10s + let delay = 1000 * attempt; + tokio::time::sleep(std::time::Duration::from_millis(delay as u64)).await; + } + } + + if !is_valid { + error!( + "ImageCache: CDN verification failed for {} after 10 attempts: {}", + cdn_url, last_verify_error + ); + return Err(format!( + "CDN link was not accessible after upload: {}", + last_verify_error + )); + } + + // Save to database only after successful verification + self.repo.save_to_db(original_url, &cdn_url).await?; + + // Cache in Redis + let _ = self + .repo + .set_in_redis(&cache_key, &cdn_url, IMAGE_CACHE_TTL) + .await; + + // Invalidate API caches + let _ = self + .repo + .invalidate_api_caches(vec!["anime:*", "anime2:*", "komik:*"]) + .await; + + Ok(cdn_url) + } + .await; + + // Release Redis lock + let _ = self.repo.release_lock(&lock_key).await; + + work_result + } + .await; + + // Broadcast result + let _ = tx.send(result.clone()); + + // Remove from map + IN_FLIGHT_UPLOADS.remove(original_url); + + result + } + + /// Get CDN URL without uploading (read-only lookup) + pub async fn get_cdn_url(&self, original_url: &str) -> Option { + let cache_key = format!("{}:{}", IMAGE_CACHE_PREFIX, url_hash(original_url)); + + // Check Redis first + if let Some(cached_url) = self.repo.get_from_redis(&cache_key).await { + return Some(cached_url); + } + + // Check database + if let Ok(Some(cdn_url)) = self.repo.get_from_db(original_url).await { + return Some(cdn_url); + } + + None + } + + /// Find an original URL for a given CDN URL (reverse lookup) + pub async fn find_original_from_cdn(&self, cdn_url: &str) -> Option { + self.repo + .find_original_from_cdn(cdn_url) + .await + .ok() + .flatten() + } + + /// Invalidate cache for a URL + pub async fn invalidate(&self, original_url: &str) -> Result<(), String> { + let cache_key = format!("{}:{}", IMAGE_CACHE_PREFIX, url_hash(original_url)); + + // Remove from Redis + let _ = self.repo.delete_from_redis(&cache_key).await; + + // Remove from database + self.repo.delete_from_db(original_url).await?; + + debug!("ImageCache: Invalidated {}", original_url); + Ok(()) + } + + /// Helper to perform a single upload attempt + async fn perform_single_upload( + &self, + api_url: &str, + image_bytes: &[u8], + filename: &str, + ) -> Result { + debug!("ImageCache: Attempting upload to API server: {}", api_url); + + let part = reqwest::multipart::Part::bytes(image_bytes.to_vec()) + .file_name(filename.to_string()) + .mime_str("image/jpeg") + .map_err(|e| { + let err = format!("Failed to create multipart form for {}: {}", api_url, e); + error!("ImageCache: {}", err); + err + })?; + + let form = reqwest::multipart::Form::new().part("file", part); + + let response = self + .client + .post(api_url) + .multipart(form) + .send() + .await + .map_err(|e| { + let err = format!("Failed to send request to Picser API ({}): {}", api_url, e); + error!("ImageCache: {}", err); + err + })?; + + let response_status = response.status(); + let response_text = response.text().await.map_err(|e| { + let err = format!( + "Failed to read Picser response from {} (Status {}): {}", + api_url, response_status, e + ); + error!("ImageCache: {}", err); + err + })?; + + // Raw responses stay at debug level for troubleshooting without noisy default logs + debug!( + "ImageCache: Raw response from {} (Status {}): {}", + api_url, response_status, response_text + ); + + if !response_status.is_success() { + let error_message = serde_json::from_str::(&response_text) + .ok() + .and_then(|value| { + value + .get("error") + .and_then(|error| error.as_str()) + .map(|error| error.to_string()) + .or_else(|| { + value + .get("message") + .and_then(|message| message.as_str()) + .map(|message| message.to_string()) + }) + }) + .unwrap_or_else(|| response_text.clone()); + + let err = format!( + "Picser upload failed at {} (HTTP {}): {}", + api_url, response_status, error_message + ); + error!("ImageCache: {}", err); + return Err(err); + } + + let picser_response: PicserResponse = + serde_json::from_str(&response_text).map_err(|e| { + let err = format!( + "Failed to parse Picser response from {}: {} - Raw: {}", + api_url, e, response_text + ); + error!("ImageCache: {}", err); + err + })?; + + if !picser_response.success { + let err_msg = picser_response + .error + .unwrap_or_else(|| "Unknown error".to_string()); + let err = format!( + "Picser upload failed at {} (server error): {}", + api_url, err_msg + ); + error!("ImageCache: {}", err); + return Err(err); + } + + debug!("ImageCache: Upload successful to API server: {}", api_url); + Ok(picser_response) + } + + fn picser_api_endpoints(&self) -> Vec { + let configured = CONFIG.urls.picser_api_url.clone(); + let mut endpoints = vec![STATIC_PICSER_API_ENDPOINTS[0].to_string()]; + if !configured.is_empty() && !endpoints.contains(&configured) { + endpoints.push(configured); + } + for endpoint in STATIC_PICSER_API_ENDPOINTS.iter().skip(1) { + let endpoint = endpoint.to_string(); + if !endpoints.contains(&endpoint) { + endpoints.push(endpoint); + } + } + endpoints + } + + async fn upload_to_fallback_api( + &self, + image_bytes: &[u8], + filename: &str, + ) -> Result { + debug!( + "ImageCache: Attempting fallback upload to: {}", + CONFIG.urls.fallback_upload_api_url + ); + + let part = reqwest::multipart::Part::bytes(image_bytes.to_vec()) + .file_name(filename.to_string()) + .mime_str("image/jpeg") + .map_err(|e| { + let err = format!("Failed to create fallback multipart form: {}", e); + error!("ImageCache: {}", err); + err + })?; + + let form = reqwest::multipart::Form::new() + .part("file", part) + .text("fileName", filename.to_string()); + + let response = self + .client + .post(&CONFIG.urls.fallback_upload_api_url) + .multipart(form) + .send() + .await + .map_err(|e| { + let err = format!( + "Failed to send request to fallback upload API ({}): {}", + CONFIG.urls.fallback_upload_api_url, e + ); + error!("ImageCache: {}", err); + err + })?; + + let response_status = response.status(); + let response_text = response.text().await.map_err(|e| { + let err = format!( + "Failed to read fallback upload response from {} (Status {}): {}", + CONFIG.urls.fallback_upload_api_url, response_status, e + ); + error!("ImageCache: {}", err); + err + })?; + + debug!( + "ImageCache: Raw response from fallback upload API (Status {}): {}", + response_status, response_text + ); + + if !response_status.is_success() { + let err = format!( + "Fallback upload failed at {} (HTTP {}): {}", + CONFIG.urls.fallback_upload_api_url, response_status, response_text + ); + error!("ImageCache: {}", err); + return Err(err); + } + + let fallback_response: FallbackUploadResponse = serde_json::from_str(&response_text) + .map_err(|e| { + let err = format!( + "Failed to parse fallback upload response from {}: {} - Raw: {}", + CONFIG.urls.fallback_upload_api_url, e, response_text + ); + error!("ImageCache: {}", err); + err + })?; + + if fallback_response.download_url.trim().is_empty() { + return Err("Fallback upload response did not include download_url".to_string()); + } + + debug!( + "ImageCache: Fallback upload successful - URL: {}", + fallback_response.download_url + ); + Ok(fallback_response.download_url) + } + + async fn download_image_bytes(&self, original_url: &str) -> Result { + let mut candidates = vec![original_url.to_string()]; + if original_url.contains("https://alqanime.net/wp-content/") { + candidates.push(original_url.replace("https://alqanime.net/", "https://alqanime.si/")); + } + + let mut last_error = String::from("No download attempt started"); + for candidate in candidates { + debug!( + "ImageCache: Starting image download from source: {}", + candidate + ); + match self.client.get(&candidate).send().await { + Ok(response) => match response.bytes().await { + Ok(bytes) => { + let is_valid_image = infer::get(&bytes) + .map(|kind| kind.mime_type().starts_with("image/")) + .unwrap_or(false); + if is_valid_image { + return Ok(bytes); + } + + let trace_preview = String::from_utf8_lossy(&bytes) + .chars() + .take(100) + .collect::(); + last_error = format!( + "Image source ({}) returned non-image data (Preview: {})", + candidate, trace_preview + ); + warn!("ImageCache: {}", last_error); + } + Err(e) => { + last_error = format!( + "Failed to read image bytes from source ({}): {}", + candidate, e + ); + warn!("ImageCache: {}", last_error); + } + }, + Err(e) => { + last_error = format!( + "Failed to download image from source ({}): {}", + candidate, e + ); + warn!("ImageCache: {}", last_error); + } + } + } + + error!("ImageCache: {}", last_error); + Err(last_error) + } + + /// Upload image to Picser CDN with fallback upload API support + async fn upload_to_picser(&self, original_url: &str) -> Result { + // Download the image first + let image_bytes = self.download_image_bytes(original_url).await?; + + debug!( + "ImageCache: Image downloaded successfully, size: {} bytes", + image_bytes.len() + ); + + // Determine filename from URL + let filename = self.extract_filename(original_url); + + debug!( + "ImageCache: Will attempt upload to {} API endpoints sequentially with failover", + self.picser_api_endpoints().len() + ); + + let mut last_failed_api = String::from("Unknown"); + let picser_api_endpoints = self.picser_api_endpoints(); + let picser_api_count = picser_api_endpoints.len(); + + for (attempt_num, api_url) in picser_api_endpoints.iter().enumerate() { + let attempt_number = attempt_num + 1; + debug!( + "ImageCache: [Attempt {}/{}] Uploading {} bytes to: {}", + attempt_number, + picser_api_count, + image_bytes.len(), + api_url + ); + + match tokio::time::timeout( + std::time::Duration::from_secs(30), + self.perform_single_upload(api_url, &image_bytes, &filename), + ) + .await + { + Ok(Ok(response)) => match self.extract_cdn_url(response, original_url) { + Ok(cdn_url) => { + debug!( + "ImageCache: Upload succeeded on attempt {}/{} - CDN URL: {}", + attempt_number, picser_api_count, cdn_url + ); + return Ok(cdn_url); + } + Err(e) => { + last_failed_api = api_url.to_string(); + warn!( + "ImageCache: [Attempt {}/{}] Upload from {} did not yield a CDN URL: {}", + attempt_number, + picser_api_count, + api_url, + e + ); + error!( + "ImageCache: Attempt {}/{} failed - Last failed API: {} - Error: {}", + attempt_number, picser_api_count, last_failed_api, e + ); + } + }, + Ok(Err(e)) => { + last_failed_api = api_url.to_string(); + warn!( + "ImageCache: [Attempt {}/{}] Upload to {} failed: {}", + attempt_number, picser_api_count, api_url, e + ); + error!( + "ImageCache: Attempt {}/{} failed - Last failed API: {} - Error: {}", + attempt_number, picser_api_count, last_failed_api, e + ); + } + Err(_) => { + last_failed_api = api_url.to_string(); + let err = format!("Timeout (30s) while uploading to API endpoint: {}", api_url); + warn!( + "ImageCache: [Attempt {}/{}] {}", + attempt_number, picser_api_count, err + ); + error!( + "ImageCache: Attempt {}/{} failed - Last failed API: {} - Error: {}", + attempt_number, picser_api_count, last_failed_api, err + ); + } + } + } + + warn!( + "ImageCache: All {} Picser upload attempts failed for source URL: {}. Trying fallback upload API: {}", + picser_api_count, + original_url, + CONFIG.urls.fallback_upload_api_url + ); + + match tokio::time::timeout( + std::time::Duration::from_secs(30), + self.upload_to_fallback_api(&image_bytes, &filename), + ) + .await + { + Ok(Ok(cdn_url)) => Ok(cdn_url), + Ok(Err(e)) => { + error!( + "ImageCache: Fallback upload API failed after Picser failures. Last Picser API: {} - Fallback error: {}", + last_failed_api, e + ); + Err(format!( + "All {} Picser upload attempts and fallback upload API failed. Last Picser API endpoint: {}. Fallback error: {}", + picser_api_count, + last_failed_api, + e + )) + } + Err(_) => { + let err = format!( + "Timeout (30s) while uploading to fallback API endpoint: {}", + CONFIG.urls.fallback_upload_api_url + ); + error!( + "ImageCache: Fallback upload API timed out after Picser failures. Last Picser API: {} - {}", + last_failed_api, err + ); + Err(format!( + "All {} Picser upload attempts and fallback upload API failed. Last Picser API endpoint: {}. Fallback error: {}", + picser_api_count, + last_failed_api, + err + )) + } + } + } + + /// Internally verify a CDN URL's accessibility and validity + pub async fn verify_cdn_url(&self, cdn_url: &str) -> Result { + let resp = self.client.get(cdn_url).send().await.map_err(|e| { + let err = format!("Network error verifying CDN URL ({}): {}", cdn_url, e); + warn!("ImageCache: {}", err); + err + })?; + + let status = resp.status(); + if !status.is_success() { + let err = format!( + "CDN verification failed with HTTP {} for URL: {}", + status, cdn_url + ); + warn!("ImageCache: {}", err); + return Ok(false); + } + + let bytes = resp.bytes().await.map_err(|e| { + let err = format!("Failed to read bytes from CDN URL ({}): {}", cdn_url, e); + warn!("ImageCache: {}", err); + err + })?; + + // Structural verification (Fast MIME check) + let is_valid = infer::get(&bytes) + .map(|k| k.mime_type().starts_with("image/")) + .unwrap_or(false); + + if is_valid { + debug!( + "ImageCache: CDN URL verified successfully - content is valid image: {}", + cdn_url + ); + Ok(true) + } else { + warn!( + "ImageCache: CDN URL verification failed - content is not a valid image ({}): {}", + cdn_url, + String::from_utf8_lossy(&bytes[0..std::cmp::min(100, bytes.len())]) + ); + Ok(false) + } + } + + /// Extract CDN URL from Picser response + fn extract_cdn_url( + &self, + response: PicserResponse, + original_url: &str, + ) -> Result { + // Try to extract CDN URL from various response fields + if let Some(urls) = &response.urls { + if let Some(url) = &urls.raw { + debug!( + "ImageCache: Using CDN URL from urls.raw for source: {}", + original_url + ); + return Ok(url.clone()); + } + if let Some(url) = &urls.jsdelivr_commit { + debug!( + "ImageCache: Using CDN URL from urls.jsdelivr_commit for source: {}", + original_url + ); + return Ok(url.clone()); + } + if let Some(url) = &urls.jsdelivr { + debug!( + "ImageCache: Using CDN URL from urls.jsdelivr for source: {}", + original_url + ); + return Ok(url.clone()); + } + if let Some(url) = &urls.github { + debug!( + "ImageCache: Using CDN URL from urls.github for source: {}", + original_url + ); + return Ok(url.clone()); + } + } + + if let Some(url) = &response.url { + debug!( + "ImageCache: Using CDN URL from response.url for source: {}", + original_url + ); + return Ok(url.clone()); + } + + if let Some(url) = &response.github_url { + debug!( + "ImageCache: Using CDN URL from response.github_url for source: {}", + original_url + ); + return Ok(url.clone()); + } + + // If we get here, no CDN URL was found in the response + error!( + "ImageCache: No CDN URL found in Picser API response for source URL: {}. Response fields - success: {}, has_urls: {}, has_url: {}, has_github_url: {}, error: {}", + original_url, + response.success, + response.urls.is_some(), + response.url.is_some(), + response.github_url.is_some(), + response.error.as_deref().unwrap_or("none") + ); + Err(format!( + "No CDN URL in Picser response. Checked: urls.{{jsdelivr_commit,jsdelivr,raw,github}}, url, github_url. Response error: {}", + response.error.unwrap_or_else(|| "none".to_string()) + )) + } + + /// Extract filename from URL + fn extract_filename(&self, url: &str) -> String { + url.split('/') + .last() + .and_then(|s| s.split('?').next()) + .filter(|s| !s.is_empty() && s.contains('.')) + .map(|s| s.to_string()) + .unwrap_or_else(|| format!("{}.jpg", url_hash(url))) + } +} + +/// Convenience function to create a CDN URL for an image +/// Returns the original URL if caching fails (graceful fallback) +pub async fn cache_image_url( + db: Arc, + redis: &RedisPool, + original_url: &str, +) -> String { + let repo = Arc::new(SeaOrmImageCacheRepository::new(db, redis.clone())); + let cache = ImageCache::new(repo); + match cache.get_or_cache(original_url).await { + Ok(cdn_url) => cdn_url, + Err(e) => { + warn!("ImageCache: Failed to cache {}: {}", original_url, e); + to_wp_cdn(original_url) // Use WP CDN as graceful fallback + } + } +} + +/// Batch cache multiple images +pub async fn cache_image_urls( + db: Arc, + redis: &RedisPool, + urls: &[String], +) -> Vec { + let repo = Arc::new(SeaOrmImageCacheRepository::new(db, redis.clone())); + let cache = ImageCache::new(repo); + let mut results = Vec::with_capacity(urls.len()); + + for url in urls { + let cdn_url = match cache.get_or_cache(url).await { + Ok(u) => u, + Err(_) => url.clone(), + }; + results.push(cdn_url); + } + + results +} + +/// Helper to convert image URL to CDN URL in background (non-blocking) +/// Returns original URL immediately and caches in background +pub fn cache_image_url_lazy( + db: Arc, + redis: &RedisPool, + original_url: String, + semaphore: Option>, +) -> String { + let db_owned = db; + let redis_owned = redis.clone(); + let url = original_url.clone(); + let sem_owned = semaphore.clone(); + + // Spawn background task to cache + tokio::spawn(async move { + let repo = Arc::new(SeaOrmImageCacheRepository::new(db_owned, redis_owned)); + let mut cache = ImageCache::new(repo); + if let Some(sem) = sem_owned { + cache = cache.with_semaphore(sem); + } + + match cache.get_or_cache(&url).await { + Ok(_) => {} + Err(_) => {} + } + }); + + to_wp_cdn(&original_url) +} + +/// Convert image URL to CDN URL if already cached, otherwise return original +/// and trigger background caching for next request (with duplicate prevention) +/// Convert image URL to CDN URL if already cached, otherwise return original +/// and trigger background caching for next request (with duplicate prevention) +pub async fn get_cached_or_original( + db: Arc, + redis: &RedisPool, + original_url: &str, + semaphore: Option>, +) -> String { + let repo = Arc::new(SeaOrmImageCacheRepository::new(db.clone(), redis.clone())); + let cache = ImageCache::new(repo); + + // Check if already cached (Redis or DB) + if let Some(cdn_url) = cache.get_cdn_url(original_url).await { + return cdn_url; + } + + // Check if currently being cached by another process + let lock_key = format!("{}:{}", IMAGE_CACHE_LOCK_PREFIX, url_hash(original_url)); + let redis_cache = Cache::new(redis); + if redis_cache.get::(&lock_key).await.is_some() { + return to_wp_cdn(original_url); + } + + // Not cached and not being cached - start background caching + let db_owned = db.clone(); + let redis_owned = redis.clone(); + let url = original_url.to_string(); + let sem_owned = semaphore.clone(); + + tokio::spawn(async move { + let repo = Arc::new(SeaOrmImageCacheRepository::new(db_owned, redis_owned)); + let mut cache = ImageCache::new(repo); + if let Some(sem) = sem_owned { + cache = cache.with_semaphore(sem); + } + + let _ = cache.get_or_cache(&url).await; + }); + + to_wp_cdn(original_url) +} + +/// Batch process multiple image URLs - returns original URLs immediately +/// and triggers background caching for all +/// Batch process multiple image URLs - checks cache first, returns cached URL if found +/// For misses: returns original URL and triggers background caching +pub async fn cache_image_urls_batch_lazy( + db: Arc, + redis: &RedisPool, + urls: Vec, + semaphore: Option>, +) -> Vec { + if urls.is_empty() { + return urls; + } + + let mut results = vec![String::new(); urls.len()]; + let mut missing_indices = Vec::new(); + + // 1. Batch check Redis + let redis_cache = Cache::new(redis); + let cache_keys: Vec = urls + .iter() + .map(|url| format!("{}:{}", IMAGE_CACHE_PREFIX, url_hash(url))) + .collect(); + + let cached_values: Vec> = redis_cache.mget(&cache_keys).await; + + for (i, val) in cached_values.iter().enumerate() { + if let Some(cdn_url) = val { + results[i] = cdn_url.clone(); + } else { + missing_indices.push(i); + } + } + + // 2. Batch check Database for Redis misses + if !missing_indices.is_empty() { + let missing_urls: Vec = missing_indices.iter().map(|&i| urls[i].clone()).collect(); + + // Note: Using repository for batch check would be better, but keeping it direct for now to match SeaORM usage + // but I should probably add a batch method to repository later. + use crate::shared::database::persistence::entities::image_cache; + use sea_orm::{ColumnTrait, EntityTrait, QueryFilter}; + + match image_cache::Entity::find() + .filter(image_cache::Column::OriginalUrl.is_in(missing_urls.clone())) + .all(db.as_ref()) + .await + { + Ok(db_entries) => { + let db_map: std::collections::HashMap = db_entries + .into_iter() + .map(|e| (e.original_url, e.cdn_url)) + .collect(); + + let mut still_missing_indices = Vec::new(); + + for &idx in &missing_indices { + let url = &urls[idx]; + if let Some(cdn_url) = db_map.get(url) { + results[idx] = cdn_url.clone(); + // Put back to Redis + let _ = redis_cache + .set_with_ttl(&cache_keys[idx], cdn_url, IMAGE_CACHE_TTL) + .await; + } else { + // Real miss - return WP CDN proxy and trigger background upload + results[idx] = to_wp_cdn(url); + still_missing_indices.push(idx); + } + } + + // 3. Trigger background caching for still missing URLs + if !still_missing_indices.is_empty() { + let db_owned = db.clone(); + let redis_owned = redis.clone(); + let sem_owned = semaphore.clone(); + let urls_to_cache: Vec = still_missing_indices + .iter() + .map(|&idx| urls[idx].clone()) + .collect(); + + tokio::spawn(async move { + use futures::stream::{self, StreamExt}; + let repo = Arc::new(SeaOrmImageCacheRepository::new(db_owned, redis_owned)); + let mut cache = ImageCache::new(repo); + if let Some(sem) = sem_owned { + cache = cache.with_semaphore(sem); + } + + stream::iter(urls_to_cache) + .map(|url| { + let cache_ref = &cache; + async move { + let _ = cache_ref.get_or_cache(&url).await; + } + }) + .buffer_unordered(20) + .collect::>() + .await; + }); + } + } + Err(e) => { + error!("ImageCache: Batch DB check failed: {}", e); + for &idx in &missing_indices { + results[idx] = to_wp_cdn(&urls[idx]); + } + } + } + } + + results +} + +/// Apply cached CDN poster URLs to a collection of items using the HasPoster trait. +pub async fn apply_cached_posters( + items: &mut [T], + db: Arc, + redis: &RedisPool, + semaphore: Option>, +) { + let posters: Vec = items.iter().map(|item| item.poster().to_string()).collect(); + let cached = cache_image_urls_batch_lazy(db, redis, posters, semaphore).await; + for (i, item) in items.iter_mut().enumerate() { + if let Some(url) = cached.get(i) { + item.set_poster(url.clone()); + } + } +} diff --git a/src/shared/services/images/mod.rs b/src/shared/services/images/mod.rs new file mode 100644 index 0000000..a5c08fd --- /dev/null +++ b/src/shared/services/images/mod.rs @@ -0,0 +1 @@ +pub mod cache; diff --git a/src/shared/services/mod.rs b/src/shared/services/mod.rs new file mode 100644 index 0000000..8f0da9f --- /dev/null +++ b/src/shared/services/mod.rs @@ -0,0 +1 @@ +pub mod images; diff --git a/src/shared/state/mod.rs b/src/shared/state/mod.rs new file mode 100644 index 0000000..7aff869 --- /dev/null +++ b/src/shared/state/mod.rs @@ -0,0 +1,17 @@ +use std::sync::Arc; + +use deadpool_redis::Pool; +use sea_orm::DatabaseConnection; + +pub struct AppState { + pub redis_pool: Pool, + pub db: Arc, + pub image_processing_semaphore: Arc, + pub event_bus: Arc, +} + +impl AppState { + pub fn sea_orm(&self) -> &DatabaseConnection { + &self.db + } +} diff --git a/src/shared/testing/app.rs b/src/shared/testing/app.rs new file mode 100644 index 0000000..e5e5b5b --- /dev/null +++ b/src/shared/testing/app.rs @@ -0,0 +1,226 @@ +//! Test application utilities. +//! +//! Provides a `TestApp` struct for integration testing that boots +//! the application in-memory with a test configuration. + +use axum::{ + body::Body, + http::{Method, Request, StatusCode}, + response::Response, + Router, +}; +use serde::{de::DeserializeOwned, Serialize}; +use tower::ServiceExt; + +/// A test application instance for integration testing. +/// +/// # Example +/// +/// ```ignore +/// use scraper_service::testing::TestApp; +/// +/// #[tokio::test] +/// async fn test_health_endpoint() { +/// let app = TestApp::new().await; +/// +/// let response = app.get("/health").await; +/// assert_eq!(response.status(), 200); +/// +/// let body = response.json::().await; +/// assert_eq!(body.status, "ok"); +/// } +/// ``` +pub struct TestApp { + router: Router, +} + +impl TestApp { + /// Create a new test application with default configuration. + /// + /// This sets up the router without starting a server. + pub fn with_router(router: Router) -> Self { + Self { router } + } + + /// Make a GET request. + pub async fn get(&self, path: &str) -> TestResponse { + self.request(Method::GET, path, Body::empty()).await + } + + /// Make a POST request with JSON body. + pub async fn post(&self, path: &str, body: &T) -> TestResponse { + let body = serde_json::to_string(body).unwrap_or_default(); + self.request_with_json(Method::POST, path, body).await + } + + /// Make a PUT request with JSON body. + pub async fn put(&self, path: &str, body: &T) -> TestResponse { + let body = serde_json::to_string(body).unwrap_or_default(); + self.request_with_json(Method::PUT, path, body).await + } + + /// Make a DELETE request. + pub async fn delete(&self, path: &str) -> TestResponse { + self.request(Method::DELETE, path, Body::empty()).await + } + + /// Make a request with custom method and body. + pub async fn request(&self, method: Method, path: &str, body: Body) -> TestResponse { + let request = Request::builder() + .method(method) + .uri(path) + .body(body) + .unwrap_or_else(|_| Request::new(Body::empty())); + + let response = self + .router + .clone() + .oneshot(request) + .await + .unwrap_or_else(|_| Response::builder().status(500).body(Body::empty()).unwrap()); + + TestResponse::new(response) + } + + /// Make a request with JSON content type. + async fn request_with_json(&self, method: Method, path: &str, body: String) -> TestResponse { + let request = Request::builder() + .method(method) + .uri(path) + .header("Content-Type", "application/json") + .body(Body::from(body)) + .unwrap_or_else(|_| Request::new(Body::empty())); + + let response = self + .router + .clone() + .oneshot(request) + .await + .unwrap_or_else(|_| Response::builder().status(500).body(Body::empty()).unwrap()); + + TestResponse::new(response) + } +} + +/// A test response with assertion helpers. +pub struct TestResponse { + response: Response, + body: Option, +} + +impl TestResponse { + fn new(response: Response) -> Self { + Self { + response, + body: None, + } + } + + /// Get the response status code. + pub fn status(&self) -> StatusCode { + self.response.status() + } + + /// Assert the status code. + pub fn assert_status(self, expected: u16) -> Self { + assert_eq!( + self.response.status().as_u16(), + expected, + "Expected status {} but got {}", + expected, + self.response.status() + ); + self + } + + /// Assert the status is 2xx (success). + pub fn assert_success(self) -> Self { + assert!( + self.response.status().is_success(), + "Expected success status but got {}", + self.response.status() + ); + self + } + + /// Assert the status is 4xx (client error). + pub fn assert_client_error(self) -> Self { + assert!( + self.response.status().is_client_error(), + "Expected client error status but got {}", + self.response.status() + ); + self + } + + /// Get a header value. + pub fn header(&self, name: &str) -> Option<&str> { + self.response + .headers() + .get(name) + .and_then(|v| v.to_str().ok()) + } + + /// Get the response body as bytes. + pub async fn bytes(mut self) -> bytes::Bytes { + if let Some(body) = self.body.take() { + return body; + } + + let body = std::mem::replace(self.response.body_mut(), Body::empty()); + axum::body::to_bytes(body, usize::MAX) + .await + .unwrap_or_default() + } + + /// Get the response body as a string. + pub async fn text(self) -> String { + let bytes = self.bytes().await; + String::from_utf8_lossy(&bytes).to_string() + } + + /// Get the response body as JSON. + pub async fn json(self) -> T { + let bytes = self.bytes().await; + serde_json::from_slice(&bytes).expect("Failed to parse response as JSON") + } + + /// Assert the response body contains a string. + pub async fn assert_body_contains(self, expected: &str) -> Self { + let body = self.text().await; + assert!( + body.contains(expected), + "Expected body to contain '{}' but got: {}", + expected, + body + ); + // Recreate self with cached body (simplified) + Self { + response: Response::builder().status(200).body(Body::empty()).unwrap(), + body: Some(bytes::Bytes::from(body)), + } + } +} + +/// Builder for TestApp with custom configuration. +pub struct TestAppBuilder { + // Future: Add configuration options +} + +impl TestAppBuilder { + /// Create a new test app builder. + pub fn new() -> Self { + Self {} + } + + /// Build the test app with a router. + pub fn build(self, router: Router) -> TestApp { + TestApp::with_router(router) + } +} + +impl Default for TestAppBuilder { + fn default() -> Self { + Self::new() + } +} diff --git a/src/shared/testing/mod.rs b/src/shared/testing/mod.rs new file mode 100644 index 0000000..309be62 --- /dev/null +++ b/src/shared/testing/mod.rs @@ -0,0 +1 @@ +pub mod app; diff --git a/src/shared/types/api_response.rs b/src/shared/types/api_response.rs new file mode 100644 index 0000000..6f671d6 --- /dev/null +++ b/src/shared/types/api_response.rs @@ -0,0 +1,28 @@ +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)] +pub struct ApiResponse { + pub success: bool, + pub message: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub data: Option, +} + +impl ApiResponse { + pub fn success(data: T) -> Self { + Self { + success: true, + message: None, + data: Some(data), + } + } + + pub fn error(message: String) -> Self { + Self { + success: false, + message: Some(message), + data: None, + } + } +} diff --git a/src/shared/types/entities/anime.rs b/src/shared/types/entities/anime.rs new file mode 100644 index 0000000..9db207f --- /dev/null +++ b/src/shared/types/entities/anime.rs @@ -0,0 +1,208 @@ +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +// ============================================================================ +// PAGINATION MODELS +// ============================================================================ + +/// Common pagination structure used across all anime2 endpoints +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct Pagination { + pub current_page: u32, + pub last_visible_page: u32, + pub has_next_page: bool, + pub next_page: Option, + pub has_previous_page: bool, + pub previous_page: Option, +} + +impl Pagination { + /// Create pagination with string-based next/previous pages + pub fn with_string_pages( + current_page: u32, + last_visible_page: u32, + has_next_page: bool, + next_page: Option, + has_previous_page: bool, + previous_page: Option, + ) -> PaginationWithStringPages { + PaginationWithStringPages { + current_page, + last_visible_page, + has_next_page, + next_page, + has_previous_page, + previous_page, + } + } +} + +/// Pagination variant with string-based page numbers (used in search endpoint) +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct PaginationWithStringPages { + pub current_page: u32, + pub last_visible_page: u32, + pub has_next_page: bool, + pub next_page: Option, + pub has_previous_page: bool, + pub previous_page: Option, +} + +// ============================================================================ +// ANIME ITEM MODELS +// ============================================================================ + +/// Anime item for ongoing anime listings +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct OngoingAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub current_episode: String, + pub anime_url: String, +} + +/// Anime item for ongoing anime with score (used in paginated ongoing lists) +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct OngoingAnimeItemWithScore { + pub title: String, + pub slug: String, + pub poster: String, + pub score: String, + pub anime_url: String, +} + +/// Anime item for complete anime listings +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct CompleteAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub episode_count: String, + pub anime_url: String, +} + +/// Anime item for latest anime listings with episode and score +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct LatestAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub current_episode: String, + pub score: String, + pub anime_url: String, +} + +/// Anime item for search results with full metadata +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct SearchAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub description: String, + pub anime_url: String, + pub genres: Vec, + pub rating: String, + pub r#type: String, + pub season: String, +} + +/// Anime item for genre filtering +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct GenreAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub score: String, + pub status: String, + pub anime_url: String, +} + +/// Anime item for advanced filtering (used in filter endpoint) +#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] +pub struct FilterAnimeItem { + pub title: String, + pub slug: String, + pub poster: String, + pub score: String, + pub status: String, + pub r#type: String, + pub anime_url: String, +} + +// ============================================================================ +// TRAITS +// ============================================================================ + +/// Trait for extracting poster URLs from anime items +pub trait HasPoster { + fn poster(&self) -> &str; + fn set_poster(&mut self, url: String); +} + +// ============================================================================ +// TRAIT IMPLEMENTATIONS +// ============================================================================ + +impl HasPoster for OngoingAnimeItem { + fn poster(&self) -> &str { + &self.poster + } + fn set_poster(&mut self, url: String) { + self.poster = url; + } +} + +impl HasPoster for OngoingAnimeItemWithScore { + fn poster(&self) -> &str { + &self.poster + } + fn set_poster(&mut self, url: String) { + self.poster = url; + } +} + +impl HasPoster for CompleteAnimeItem { + fn poster(&self) -> &str { + &self.poster + } + fn set_poster(&mut self, url: String) { + self.poster = url; + } +} + +impl HasPoster for LatestAnimeItem { + fn poster(&self) -> &str { + &self.poster + } + fn set_poster(&mut self, url: String) { + self.poster = url; + } +} + +impl HasPoster for SearchAnimeItem { + fn poster(&self) -> &str { + &self.poster + } + fn set_poster(&mut self, url: String) { + self.poster = url; + } +} + +impl HasPoster for GenreAnimeItem { + fn poster(&self) -> &str { + &self.poster + } + fn set_poster(&mut self, url: String) { + self.poster = url; + } +} + +impl HasPoster for FilterAnimeItem { + fn poster(&self) -> &str { + &self.poster + } + fn set_poster(&mut self, url: String) { + self.poster = url; + } +} diff --git a/src/shared/types/entities/image.rs b/src/shared/types/entities/image.rs new file mode 100644 index 0000000..222aa19 --- /dev/null +++ b/src/shared/types/entities/image.rs @@ -0,0 +1,11 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ImageCache { + pub id: String, + pub original_url: String, + pub cdn_url: String, + pub created_at: DateTime, + pub expires_at: Option>, +} diff --git a/src/shared/types/entities/mod.rs b/src/shared/types/entities/mod.rs new file mode 100644 index 0000000..ee74445 --- /dev/null +++ b/src/shared/types/entities/mod.rs @@ -0,0 +1,3 @@ +pub mod anime; +pub mod image; +pub mod types; diff --git a/src/shared/types/entities/types.rs b/src/shared/types/entities/types.rs new file mode 100644 index 0000000..05ea6a7 --- /dev/null +++ b/src/shared/types/entities/types.rs @@ -0,0 +1,56 @@ +use crate::shared::errors::AppError; +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, + Json, +}; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +#[derive(Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ErrorResponse { + pub error: String, +} + +impl From for ErrorResponse { + fn from(app_error: AppError) -> Self { + ErrorResponse { + error: app_error.to_string(), + } + } +} + +impl IntoResponse for ErrorResponse { + fn into_response(self) -> Response { + let (status, error_message) = match self.error.as_str() { + _ if self.error.contains("Environment variable not found") => { + (StatusCode::INTERNAL_SERVER_ERROR, self.error) + } + _ if self.error.contains("Redis error") => { + (StatusCode::INTERNAL_SERVER_ERROR, self.error) + } + _ if self.error.contains("Reqwest error") => { + (StatusCode::INTERNAL_SERVER_ERROR, self.error) + } + _ if self + .error + .contains("JSON serialization/deserialization error") => + { + (StatusCode::INTERNAL_SERVER_ERROR, self.error) + } + _ if self.error.contains("URL parsing error") => (StatusCode::BAD_REQUEST, self.error), + _ if self.error.contains("JWT error") => (StatusCode::UNAUTHORIZED, self.error), + _ if self.error.contains("Scraper error") => (StatusCode::BAD_GATEWAY, self.error), + _ if self.error.contains("Fantoccini error") => (StatusCode::BAD_GATEWAY, self.error), + _ if self.error.contains("IO error") => (StatusCode::INTERNAL_SERVER_ERROR, self.error), + _ if self.error.contains("Timeout error") => (StatusCode::REQUEST_TIMEOUT, self.error), + _ => (StatusCode::INTERNAL_SERVER_ERROR, self.error), + }; + + let body = Json(json!({ + "error": error_message, + })); + + (status, body).into_response() + } +} diff --git a/src/shared/types/mod.rs b/src/shared/types/mod.rs new file mode 100644 index 0000000..67040d2 --- /dev/null +++ b/src/shared/types/mod.rs @@ -0,0 +1,4 @@ +pub mod api_response; +pub mod entities; + +pub use api_response::ApiResponse; diff --git a/src/shared/utils/core/api_response.rs b/src/shared/utils/core/api_response.rs new file mode 100644 index 0000000..2f5ee22 --- /dev/null +++ b/src/shared/utils/core/api_response.rs @@ -0,0 +1,301 @@ +//! Standardized API response helpers. +//! +//! Provides consistent JSON response structures across all API endpoints. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::api_response::{ApiResponse, ApiResult}; +//! +//! async fn get_user(id: i32) -> ApiResult { +//! let user = find_user(id)?; +//! Ok(ApiResponse::success(user)) +//! } +//! +//! async fn list_users(page: u32) -> ApiResult> { +//! let (users, total) = find_users_paginated(page)?; +//! Ok(ApiResponse::paginated(users, page, 20, total)) +//! } +//! +//! async fn delete_user(id: i32) -> ApiResult<()> { +//! delete_user(id)?; +//! Ok(ApiResponse::no_content()) +//! } +//! ``` + +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, + Json, +}; +use serde::{Deserialize, Serialize}; + +/// Standard API response wrapper. +#[derive(Debug, Serialize, Deserialize)] +pub struct ApiResponse { + /// Whether the request was successful + pub success: bool, + /// Response data (if successful) + #[serde(skip_serializing_if = "Option::is_none")] + pub data: Option, + /// Error message (if failed) + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// Pagination metadata (if applicable) + #[serde(skip_serializing_if = "Option::is_none")] + pub pagination: Option, + /// Additional metadata + #[serde(skip_serializing_if = "Option::is_none")] + pub meta: Option, +} + +/// Error details for failed responses. +#[derive(Debug, Serialize, Deserialize)] +pub struct ErrorDetails { + /// Error code (machine-readable) + pub code: String, + /// Human-readable error message + pub message: String, + /// Field-level validation errors + #[serde(skip_serializing_if = "Option::is_none")] + pub fields: Option>, +} + +/// Field-level validation error. +#[derive(Debug, Serialize, Deserialize)] +pub struct FieldError { + /// Field name + pub field: String, + /// Error message + pub message: String, +} + +/// Pagination metadata. +#[derive(Debug, Serialize, Deserialize)] +pub struct PaginationMeta { + /// Current page number + pub page: u32, + /// Items per page + pub per_page: u32, + /// Total number of items + pub total: u64, + /// Total number of pages + pub total_pages: u32, + /// Whether there's a next page + pub has_next: bool, + /// Whether there's a previous page + pub has_prev: bool, +} + +impl ApiResponse { + /// Create a successful response with data. + pub fn success(data: T) -> Self { + Self { + success: true, + data: Some(data), + error: None, + pagination: None, + meta: None, + } + } + + /// Create a successful response with data and metadata. + pub fn success_with_meta(data: T, meta: serde_json::Value) -> Self { + Self { + success: true, + data: Some(data), + error: None, + pagination: None, + meta: Some(meta), + } + } + + /// Create a paginated response. + pub fn paginated(data: T, page: u32, per_page: u32, total: u64) -> Self { + let total_pages = ((total as f64) / (per_page as f64)).ceil() as u32; + Self { + success: true, + data: Some(data), + error: None, + pagination: Some(PaginationMeta { + page, + per_page, + total, + total_pages, + has_next: page < total_pages, + has_prev: page > 1, + }), + meta: None, + } + } +} + +impl ApiResponse<()> { + /// Create a successful response with no data. + pub fn no_content() -> Self { + Self { + success: true, + data: None, + error: None, + pagination: None, + meta: None, + } + } + + /// Create a successful message response. + pub fn message(msg: &str) -> ApiResponse { + ApiResponse { + success: true, + data: Some(MessageOnly { + message: msg.to_string(), + }), + error: None, + pagination: None, + meta: None, + } + } +} + +/// Simple message-only response data. +#[derive(Debug, Serialize, Deserialize)] +pub struct MessageOnly { + pub message: String, +} + +/// API error response builder. +#[derive(Debug)] +pub struct ApiError { + pub status: StatusCode, + pub code: String, + pub message: String, + pub fields: Option>, +} + +impl ApiError { + /// Create a new API error. + pub fn new(status: StatusCode, code: &str, message: &str) -> Self { + Self { + status, + code: code.to_string(), + message: message.to_string(), + fields: None, + } + } + + /// Create a 400 Bad Request error. + pub fn bad_request(message: &str) -> Self { + Self::new(StatusCode::BAD_REQUEST, "BAD_REQUEST", message) + } + + /// Create a 401 Unauthorized error. + pub fn unauthorized(message: &str) -> Self { + Self::new(StatusCode::UNAUTHORIZED, "UNAUTHORIZED", message) + } + + /// Create a 403 Forbidden error. + pub fn forbidden(message: &str) -> Self { + Self::new(StatusCode::FORBIDDEN, "FORBIDDEN", message) + } + + /// Create a 404 Not Found error. + pub fn not_found(message: &str) -> Self { + Self::new(StatusCode::NOT_FOUND, "NOT_FOUND", message) + } + + /// Create a 409 Conflict error. + pub fn conflict(message: &str) -> Self { + Self::new(StatusCode::CONFLICT, "CONFLICT", message) + } + + /// Create a 422 Unprocessable Entity error with field errors. + pub fn validation(fields: Vec) -> Self { + Self { + status: StatusCode::UNPROCESSABLE_ENTITY, + code: "VALIDATION_ERROR".to_string(), + message: "Validation failed".to_string(), + fields: Some(fields), + } + } + + /// Create a 429 Too Many Requests error. + pub fn too_many_requests(message: &str) -> Self { + Self::new(StatusCode::TOO_MANY_REQUESTS, "RATE_LIMITED", message) + } + + /// Create a 500 Internal Server Error. + pub fn internal(message: &str) -> Self { + Self::new(StatusCode::INTERNAL_SERVER_ERROR, "INTERNAL_ERROR", message) + } + + /// Create a 503 Service Unavailable error. + pub fn service_unavailable(message: &str) -> Self { + Self::new( + StatusCode::SERVICE_UNAVAILABLE, + "SERVICE_UNAVAILABLE", + message, + ) + } + + /// Add field errors. + pub fn with_fields(mut self, fields: Vec) -> Self { + self.fields = Some(fields); + self + } +} + +impl IntoResponse for ApiError { + fn into_response(self) -> Response { + let body = ApiResponse::<()> { + success: false, + data: None, + error: Some(ErrorDetails { + code: self.code, + message: self.message, + fields: self.fields, + }), + pagination: None, + meta: None, + }; + + (self.status, Json(body)).into_response() + } +} + +impl IntoResponse for ApiResponse { + fn into_response(self) -> Response { + let status = if self.success { + StatusCode::OK + } else { + StatusCode::BAD_REQUEST + }; + (status, Json(self)).into_response() + } +} + +/// Type alias for API handler results. +pub type ApiResult = Result, ApiError>; + +/// Helper to convert any serializable to success response. +pub fn success(data: T) -> ApiResponse { + ApiResponse::success(data) +} + +/// Helper to convert any serializable to paginated response. +pub fn paginated(data: T, page: u32, per_page: u32, total: u64) -> ApiResponse { + ApiResponse::paginated(data, page, per_page, total) +} + +/// Helper for quick internal error. +pub fn internal_err(msg: &str) -> ApiError { + ApiError::internal(msg) +} + +/// Helper for quick not found error. +pub fn not_found(msg: &str) -> ApiError { + ApiError::not_found(msg) +} + +/// Helper for quick bad request error. +pub fn bad_request(msg: &str) -> ApiError { + ApiError::bad_request(msg) +} diff --git a/src/shared/utils/core/errors.rs b/src/shared/utils/core/errors.rs new file mode 100644 index 0000000..a75dd93 --- /dev/null +++ b/src/shared/utils/core/errors.rs @@ -0,0 +1,101 @@ +//! Axum error response helpers. + +use axum::http::StatusCode; + +/// Shorthand for creating error tuples for Axum handlers. +pub type HandlerError = (StatusCode, String); + +/// Create internal server error. +pub fn internal_error(msg: impl Into) -> HandlerError { + (StatusCode::INTERNAL_SERVER_ERROR, msg.into()) +} + +/// Create internal server error from any error type. +pub fn internal_err(e: E) -> HandlerError { + (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()) +} + +/// Create bad request error. +pub fn bad_request(msg: impl Into) -> HandlerError { + (StatusCode::BAD_REQUEST, msg.into()) +} + +/// Create not found error. +pub fn not_found(msg: impl Into) -> HandlerError { + (StatusCode::NOT_FOUND, msg.into()) +} + +/// Create unauthorized error. +pub fn unauthorized(msg: impl Into) -> HandlerError { + (StatusCode::UNAUTHORIZED, msg.into()) +} + +/// Create forbidden error. +pub fn forbidden(msg: impl Into) -> HandlerError { + (StatusCode::FORBIDDEN, msg.into()) +} + +/// Create conflict error. +pub fn conflict(msg: impl Into) -> HandlerError { + (StatusCode::CONFLICT, msg.into()) +} + +/// Create too many requests error. +pub fn too_many_requests(msg: impl Into) -> HandlerError { + (StatusCode::TOO_MANY_REQUESTS, msg.into()) +} + +/// Map any error to internal server error. +pub fn map_internal(e: E) -> HandlerError { + internal_err(e) +} + +/// Create Redis error response. +pub fn redis_error(e: E) -> HandlerError { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Redis error: {}", e), + ) +} + +/// Create database error response. +pub fn db_error(e: E) -> HandlerError { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Database error: {}", e), + ) +} + +/// Create serialization error response. +pub fn serialization_error(e: E) -> HandlerError { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Serialization error: {}", e), + ) +} + +/// Trait extension for Result to easily convert errors. +pub trait ResultExt { + /// Map error to internal server error. + fn map_internal(self) -> Result; + + /// Map error to bad request. + fn map_bad_request(self) -> Result; + + /// Map error to not found. + fn map_not_found(self) -> Result; +} + +impl ResultExt for Result { + fn map_internal(self) -> Result { + self.map_err(internal_err) + } + + fn map_bad_request(self) -> Result { + self.map_err(|e| bad_request(e.to_string())) + } + + fn map_not_found(self) -> Result { + self.map_err(|e| not_found(e.to_string())) + } +} diff --git a/src/shared/utils/core/handler.rs b/src/shared/utils/core/handler.rs new file mode 100644 index 0000000..a0f3438 --- /dev/null +++ b/src/shared/utils/core/handler.rs @@ -0,0 +1,99 @@ +//! Handler helper macros and utilities. + +/// Macro to create a simple CRUD handler set. +/// +/// # Example +/// +/// ```ignore +/// use scraper_service::helpers::crud_handlers; +/// +/// crud_handlers!(User, users); +/// // Generates: list_users, get_user, create_user, update_user, delete_user +/// ``` +#[macro_export] +macro_rules! crud_handlers { + ($entity:ident, $name:ident) => { + paste::paste! { + pub async fn []( + State(db): State>, + Query(params): Query, + ) -> impl IntoResponse { + use sea_orm::PaginatorTrait; + + let paginator = $entity::Entity::find() + .paginate(&*db, params.limit); + + let total = paginator.num_items().await.unwrap_or(0); + let items = paginator + .fetch_page(params.page.saturating_sub(1)) + .await + .unwrap_or_default(); + + Json(Paginated::from_params(items, ¶ms, total)) + } + + pub async fn []( + State(db): State>, + Path(id): Path, + ) -> Result { + let item = $entity::Entity::find_by_id(&id) + .one(&*db) + .await + .map_err(|e| ErrorResponse::internal(e.to_string()))? + .ok_or_else(|| ErrorResponse::not_found(concat!(stringify!($entity), " not found")))?; + + Ok(Json(json_ok(item))) + } + + pub async fn []( + State(db): State>, + Path(id): Path, + ) -> Result { + let result = $entity::Entity::delete_by_id(&id) + .exec(&*db) + .await + .map_err(|e| ErrorResponse::internal(e.to_string()))?; + + if result.rows_affected > 0 { + Ok(no_content()) + } else { + Err(ErrorResponse::not_found(concat!(stringify!($entity), " not found"))) + } + } + } + }; +} + +/// Wrap a handler result with consistent error handling. +/// +/// # Example +/// +/// ```ignore +/// use scraper_service::helpers::handler::try_handler; +/// +/// async fn my_handler() -> impl IntoResponse { +/// try_handler(async { +/// let data = fetch_data().await?; +/// Ok(data) +/// }).await +/// } +/// ``` +pub async fn try_handler( + f: F, +) -> Result, crate::shared::utils::ErrorResponse> +where + T: serde::Serialize, + E: std::fmt::Display, + F: FnOnce() -> Fut, + Fut: std::future::Future>, +{ + match f().await { + Ok(data) => Ok(crate::shared::utils::JsonResponse::ok(data)), + Err(e) => Err(crate::shared::utils::ErrorResponse::internal(e.to_string())), + } +} + +/// Extract user ID from JWT claims (helper). +pub fn get_user_id_from_claims(claims: &serde_json::Value) -> Option { + claims.get("sub").and_then(|v| v.as_str()).map(String::from) +} diff --git a/src/shared/utils/core/mod.rs b/src/shared/utils/core/mod.rs new file mode 100644 index 0000000..029362b --- /dev/null +++ b/src/shared/utils/core/mod.rs @@ -0,0 +1,6 @@ +pub mod api_response; +pub mod errors; +pub mod handler; +pub mod pagination; +pub mod prelude; +pub mod response; diff --git a/src/shared/utils/core/pagination.rs b/src/shared/utils/core/pagination.rs new file mode 100644 index 0000000..76dcf89 --- /dev/null +++ b/src/shared/utils/core/pagination.rs @@ -0,0 +1,105 @@ +//! Pagination helpers. + +use serde::{Deserialize, Serialize}; + +/// Pagination query parameters. +#[derive(Debug, Clone, Deserialize)] +pub struct PaginationParams { + /// Page number (1-indexed). + #[serde(default = "default_page")] + pub page: u64, + /// Items per page. + #[serde(default = "default_limit")] + pub limit: u64, + /// Sort field. + #[serde(default)] + pub sort: Option, + /// Sort order (asc/desc). + #[serde(default = "default_order")] + pub order: String, +} + +fn default_page() -> u64 { + 1 +} +fn default_limit() -> u64 { + 20 +} +fn default_order() -> String { + "asc".to_string() +} + +impl PaginationParams { + /// Calculate offset for database query. + pub fn offset(&self) -> u64 { + (self.page.saturating_sub(1)) * self.limit + } + + /// Check if ascending order. + pub fn is_asc(&self) -> bool { + self.order.to_lowercase() == "asc" + } +} + +impl Default for PaginationParams { + fn default() -> Self { + Self { + page: default_page(), + limit: default_limit(), + sort: None, + order: default_order(), + } + } +} + +/// Paginated response wrapper. +#[derive(Debug, Clone, Serialize)] +pub struct Paginated { + pub data: Vec, + pub pagination: PaginationMeta, +} + +/// Pagination metadata. +#[derive(Debug, Clone, Serialize)] +pub struct PaginationMeta { + pub page: u64, + pub limit: u64, + pub total: u64, + pub total_pages: u64, + pub has_next: bool, + pub has_prev: bool, +} + +impl Paginated { + /// Create a paginated response. + pub fn new(data: Vec, page: u64, limit: u64, total: u64) -> Self { + let total_pages = (total + limit - 1) / limit; + Self { + data, + pagination: PaginationMeta { + page, + limit, + total, + total_pages, + has_next: page < total_pages, + has_prev: page > 1, + }, + } + } + + /// Create from params and total. + pub fn from_params(data: Vec, params: &PaginationParams, total: u64) -> Self { + Self::new(data, params.page, params.limit, total) + } +} + +/// Trait to easily paginate database results. +pub trait Paginatable { + fn paginate(self, params: &PaginationParams, total: u64) -> Paginated; +} + +impl Paginatable for Vec { + fn paginate(self, params: &PaginationParams, total: u64) -> Paginated { + Paginated::from_params(self, params, total) + } +} diff --git a/src/shared/utils/core/prelude.rs b/src/shared/utils/core/prelude.rs new file mode 100644 index 0000000..3b6f022 --- /dev/null +++ b/src/shared/utils/core/prelude.rs @@ -0,0 +1,33 @@ +//! Prelude module - common imports for handlers. +//! +//! # Usage +//! +//! ```ignore +//! use scraper_service::helpers::prelude::*; +//! ``` + +// Re-export common types +pub use axum::{ + extract::{Extension, Path, Query, State}, + http::StatusCode, + response::IntoResponse, + Json, +}; +pub use serde::{Deserialize, Serialize}; +pub use tracing::{debug, error, info, warn}; + +// Re-export our helpers +pub use super::pagination::{Paginated, PaginationParams}; +pub use super::response::{ApiResult, ErrorResponse, JsonResponse}; + +// Re-export common extractors +pub use crate::shared::observability::request_id::RequestId; + +// Re-export database types +pub use sea_orm::{ + ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, QueryOrder, + QuerySelect, Set, +}; + +// Re-export common std types +pub use std::sync::Arc; diff --git a/src/shared/utils/core/response.rs b/src/shared/utils/core/response.rs new file mode 100644 index 0000000..34f70e2 --- /dev/null +++ b/src/shared/utils/core/response.rs @@ -0,0 +1,148 @@ +//! Response helpers for consistent API responses. + +use axum::{http::StatusCode, response::IntoResponse, Json}; +use serde::Serialize; + +/// Result type for API handlers. +pub type ApiResult = Result, ErrorResponse>; + +/// Success JSON response wrapper. +#[derive(Debug, Clone, Serialize)] +pub struct JsonResponse { + pub success: bool, + pub data: T, + #[serde(skip_serializing_if = "Option::is_none")] + pub message: Option, +} + +impl JsonResponse { + /// Create a success response. + pub fn ok(data: T) -> Self { + Self { + success: true, + data, + message: None, + } + } + + /// Create a success response with message. + pub fn ok_with_message(data: T, message: impl Into) -> Self { + Self { + success: true, + data, + message: Some(message.into()), + } + } +} + +impl IntoResponse for JsonResponse { + fn into_response(self) -> axum::response::Response { + (StatusCode::OK, Json(self)).into_response() + } +} + +/// Error response wrapper. +#[derive(Debug, Clone, Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub code: Option, + #[serde(skip)] + pub status: StatusCode, +} + +impl ErrorResponse { + /// Create an error response. + pub fn new(status: StatusCode, error: impl Into) -> Self { + Self { + success: false, + error: error.into(), + code: None, + status, + } + } + + /// Add an error code. + pub fn with_code(mut self, code: impl Into) -> Self { + self.code = Some(code.into()); + self + } + + // Common error constructors + + /// 400 Bad Request + pub fn bad_request(error: impl Into) -> Self { + Self::new(StatusCode::BAD_REQUEST, error) + } + + /// 401 Unauthorized + pub fn unauthorized(error: impl Into) -> Self { + Self::new(StatusCode::UNAUTHORIZED, error) + } + + /// 403 Forbidden + pub fn forbidden(error: impl Into) -> Self { + Self::new(StatusCode::FORBIDDEN, error) + } + + /// 404 Not Found + pub fn not_found(error: impl Into) -> Self { + Self::new(StatusCode::NOT_FOUND, error) + } + + /// 409 Conflict + pub fn conflict(error: impl Into) -> Self { + Self::new(StatusCode::CONFLICT, error) + } + + /// 422 Unprocessable Entity + pub fn unprocessable(error: impl Into) -> Self { + Self::new(StatusCode::UNPROCESSABLE_ENTITY, error) + } + + /// 500 Internal Server Error + pub fn internal(error: impl Into) -> Self { + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error) + } +} + +impl IntoResponse for ErrorResponse { + fn into_response(self) -> axum::response::Response { + (self.status, Json(&self)).into_response() + } +} + +impl From for ErrorResponse { + fn from(err: anyhow::Error) -> Self { + Self::internal(err.to_string()) + } +} + +impl From for ErrorResponse { + fn from(err: sea_orm::DbErr) -> Self { + Self::internal(format!("Database error: {}", err)) + } +} + +// Convenience functions + +/// Create a success JSON response. +pub fn json_ok(data: T) -> JsonResponse { + JsonResponse::ok(data) +} + +/// Create an empty success response. +pub fn ok() -> impl IntoResponse { + (StatusCode::OK, Json(serde_json::json!({"success": true}))) +} + +/// Create a created response (201). +pub fn created(data: T) -> impl IntoResponse { + (StatusCode::CREATED, Json(JsonResponse::ok(data))) +} + +/// Create a no content response (204). +pub fn no_content() -> impl IntoResponse { + StatusCode::NO_CONTENT +} diff --git a/src/shared/utils/data/collections.rs b/src/shared/utils/data/collections.rs new file mode 100644 index 0000000..e1ec301 --- /dev/null +++ b/src/shared/utils/data/collections.rs @@ -0,0 +1,169 @@ +//! Collection and iterator utilities. + +use std::collections::{HashMap, HashSet}; +use std::hash::Hash; + +/// Chunk a vector into smaller vectors of specified size. +pub fn chunk(items: Vec, size: usize) -> Vec> { + items.chunks(size).map(|chunk| chunk.to_vec()).collect() +} + +/// Get unique items from a vector. +pub fn unique(items: Vec) -> Vec { + let mut seen = HashSet::new(); + items + .into_iter() + .filter(|item| seen.insert(item.clone())) + .collect() +} + +/// Group items by a key function. +pub fn group_by(items: Vec, key_fn: F) -> HashMap> +where + K: Hash + Eq, + F: Fn(&T) -> K, +{ + let mut groups: HashMap> = HashMap::new(); + for item in items { + let key = key_fn(&item); + groups.entry(key).or_default().push(item); + } + groups +} + +/// Partition items into two vectors based on predicate. +pub fn partition(items: Vec, predicate: F) -> (Vec, Vec) +where + F: Fn(&T) -> bool, +{ + let mut pass = Vec::new(); + let mut fail = Vec::new(); + for item in items { + if predicate(&item) { + pass.push(item); + } else { + fail.push(item); + } + } + (pass, fail) +} + +/// Flatten nested vectors. +pub fn flatten(nested: Vec>) -> Vec { + nested.into_iter().flatten().collect() +} + +/// Zip two vectors into pairs. +pub fn zip(a: Vec, b: Vec) -> Vec<(T, U)> { + a.into_iter().zip(b).collect() +} + +/// Find first matching item. +pub fn find(items: &[T], predicate: F) -> Option<&T> +where + F: Fn(&T) -> bool, +{ + items.iter().find(|item| predicate(item)) +} + +/// Find first matching item and return its index. +pub fn find_index(items: &[T], predicate: F) -> Option +where + F: Fn(&T) -> bool, +{ + items.iter().position(|item| predicate(item)) +} + +/// Check if any item matches predicate. +pub fn any(items: &[T], predicate: F) -> bool +where + F: Fn(&T) -> bool, +{ + items.iter().any(predicate) +} + +/// Check if all items match predicate. +pub fn all(items: &[T], predicate: F) -> bool +where + F: Fn(&T) -> bool, +{ + items.iter().all(predicate) +} + +/// Sum numeric values. +pub fn sum(items: &[T], value_fn: F) -> N +where + N: std::iter::Sum, + F: Fn(&T) -> N, +{ + items.iter().map(value_fn).sum() +} + +/// Count items matching predicate. +pub fn count(items: &[T], predicate: F) -> usize +where + F: Fn(&T) -> bool, +{ + items.iter().filter(|item| predicate(item)).count() +} + +/// Get first n items. +pub fn take(items: &[T], n: usize) -> Vec { + items.iter().take(n).cloned().collect() +} + +/// Skip first n items. +pub fn skip(items: &[T], n: usize) -> Vec { + items.iter().skip(n).cloned().collect() +} + +/// Reverse a vector. +pub fn reverse(items: &[T]) -> Vec { + items.iter().rev().cloned().collect() +} + +/// Interleave two vectors. +pub fn interleave(a: Vec, b: Vec) -> Vec { + let mut result = Vec::with_capacity(a.len() + b.len()); + let mut a_iter = a.into_iter(); + let mut b_iter = b.into_iter(); + loop { + match (a_iter.next(), b_iter.next()) { + (Some(x), Some(y)) => { + result.push(x); + result.push(y); + } + (Some(x), None) => result.push(x), + (None, Some(y)) => result.push(y), + (None, None) => break, + } + } + result +} + +/// Create a frequency map. +pub fn frequencies(items: &[T]) -> HashMap { + let mut freq = HashMap::new(); + for item in items { + *freq.entry(item.clone()).or_insert(0) += 1; + } + freq +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_chunk() { + let items = vec![1, 2, 3, 4, 5]; + let chunks = chunk(items, 2); + assert_eq!(chunks, vec![vec![1, 2], vec![3, 4], vec![5]]); + } + + #[test] + fn test_unique() { + let items = vec![1, 2, 2, 3, 3, 3]; + assert_eq!(unique(items), vec![1, 2, 3]); + } +} diff --git a/src/shared/utils/data/convert/bools.rs b/src/shared/utils/data/convert/bools.rs new file mode 100644 index 0000000..4d3fc1b --- /dev/null +++ b/src/shared/utils/data/convert/bools.rs @@ -0,0 +1,84 @@ +/// Convert string to bool (flexible parsing). +pub fn to_bool(s: &str) -> bool { + matches!( + s.to_lowercase().trim(), + "true" | "1" | "yes" | "on" | "enabled" | "t" | "y" | "ok" | "active" + ) +} + +/// Convert to bool with Option for invalid input. +pub fn try_bool(s: &str) -> Option { + match s.to_lowercase().trim() { + "true" | "1" | "yes" | "on" | "enabled" | "t" | "y" | "ok" | "active" => Some(true), + "false" | "0" | "no" | "off" | "disabled" | "f" | "n" | "inactive" => Some(false), + _ => None, + } +} + +/// Convert bool to string. +pub fn bool_to_str(b: bool) -> &'static str { + if b { + "true" + } else { + "false" + } +} + +/// Convert bool to yes/no. +pub fn bool_to_yes_no(b: bool) -> &'static str { + if b { + "yes" + } else { + "no" + } +} + +/// Convert bool to on/off. +pub fn bool_to_on_off(b: bool) -> &'static str { + if b { + "on" + } else { + "off" + } +} + +/// Convert bool to enabled/disabled. +pub fn bool_to_enabled(b: bool) -> &'static str { + if b { + "enabled" + } else { + "disabled" + } +} + +/// Convert bool to active/inactive. +pub fn bool_to_active(b: bool) -> &'static str { + if b { + "active" + } else { + "inactive" + } +} + +/// Convert bool to 0/1 i32. +pub fn bool_to_int(b: bool) -> i32 { + if b { + 1 + } else { + 0 + } +} + +/// Convert bool to 0/1 i64. +pub fn bool_to_i64(b: bool) -> i64 { + if b { + 1 + } else { + 0 + } +} + +/// Convert i32 to bool (0 = false, other = true). +pub fn int_to_bool(n: i32) -> bool { + n != 0 +} diff --git a/src/shared/utils/data/convert/bytes.rs b/src/shared/utils/data/convert/bytes.rs new file mode 100644 index 0000000..0cc1b76 --- /dev/null +++ b/src/shared/utils/data/convert/bytes.rs @@ -0,0 +1,130 @@ +/// Convert bytes to hex string (lowercase). +pub fn bytes_to_hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{:02x}", b)).collect() +} + +/// Convert bytes to hex string (uppercase). +pub fn bytes_to_hex_upper(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{:02X}", b)).collect() +} + +/// Convert hex string to bytes. +pub fn hex_to_bytes(hex: &str) -> Result, std::num::ParseIntError> { + (0..hex.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&hex[i..i + 2], 16)) + .collect() +} + +/// Convert bytes to base64. +pub fn bytes_to_base64(bytes: &[u8]) -> String { + base64::Engine::encode(&base64::engine::general_purpose::STANDARD, bytes) +} + +/// Convert base64 to bytes. +pub fn base64_to_bytes(s: &str) -> Result, base64::DecodeError> { + base64::Engine::decode(&base64::engine::general_purpose::STANDARD, s) +} + +/// Convert bytes to binary string. +pub fn bytes_to_binary(bytes: &[u8]) -> String { + bytes + .iter() + .map(|b| format!("{:08b}", b)) + .collect::>() + .join(" ") +} + +/// Convert bytes to octal string. +pub fn bytes_to_octal(bytes: &[u8]) -> String { + bytes + .iter() + .map(|b| format!("{:03o}", b)) + .collect::>() + .join(" ") +} + +/// Convert u8 to binary string. +pub fn u8_to_binary(n: u8) -> String { + format!("{:08b}", n) +} + +/// Convert u16 to binary string. +pub fn u16_to_binary(n: u16) -> String { + format!("{:016b}", n) +} + +/// Convert u32 to binary string. +pub fn u32_to_binary(n: u32) -> String { + format!("{:032b}", n) +} + +/// Convert u64 to binary string. +pub fn u64_to_binary(n: u64) -> String { + format!("{:064b}", n) +} + +/// Convert binary string to u64. +pub fn binary_to_u64(s: &str) -> Result { + u64::from_str_radix(&s.replace(" ", ""), 2) +} + +/// Swap endianness of u16. +pub fn swap_endian_u16(n: u16) -> u16 { + n.swap_bytes() +} + +/// Swap endianness of u32. +pub fn swap_endian_u32(n: u32) -> u32 { + n.swap_bytes() +} + +/// Swap endianness of u64. +pub fn swap_endian_u64(n: u64) -> u64 { + n.swap_bytes() +} + +/// Swap endianness of u128. +pub fn swap_endian_u128(n: u128) -> u128 { + n.swap_bytes() +} + +/// Convert to big endian bytes. +pub fn u32_to_be_bytes(n: u32) -> [u8; 4] { + n.to_be_bytes() +} + +/// Convert to little endian bytes. +pub fn u32_to_le_bytes(n: u32) -> [u8; 4] { + n.to_le_bytes() +} + +/// Convert from big endian bytes. +pub fn be_bytes_to_u32(bytes: [u8; 4]) -> u32 { + u32::from_be_bytes(bytes) +} + +/// Convert from little endian bytes. +pub fn le_bytes_to_u32(bytes: [u8; 4]) -> u32 { + u32::from_le_bytes(bytes) +} + +/// Convert to big endian bytes. +pub fn u64_to_be_bytes(n: u64) -> [u8; 8] { + n.to_be_bytes() +} + +/// Convert to little endian bytes. +pub fn u64_to_le_bytes(n: u64) -> [u8; 8] { + n.to_le_bytes() +} + +/// Convert from big endian bytes. +pub fn be_bytes_to_u64(bytes: [u8; 8]) -> u64 { + u64::from_be_bytes(bytes) +} + +/// Convert from little endian bytes. +pub fn le_bytes_to_u64(bytes: [u8; 8]) -> u64 { + u64::from_le_bytes(bytes) +} diff --git a/src/shared/utils/data/convert/char.rs b/src/shared/utils/data/convert/char.rs new file mode 100644 index 0000000..8514de5 --- /dev/null +++ b/src/shared/utils/data/convert/char.rs @@ -0,0 +1,65 @@ +/// Convert char to u32 (unicode code point). +pub fn char_to_u32(c: char) -> u32 { + c as u32 +} + +/// Convert u32 to char (unicode code point). +pub fn u32_to_char(n: u32) -> Option { + char::from_u32(n) +} + +/// Convert char to ascii u8. +pub fn char_to_ascii(c: char) -> Option { + if c.is_ascii() { + Some(c as u8) + } else { + None + } +} + +/// Convert u8 to char. +pub fn u8_to_char(n: u8) -> char { + n as char +} + +/// Convert digit char to u8. +pub fn digit_to_u8(c: char) -> Option { + c.to_digit(10).map(|d| d as u8) +} + +/// Convert u8 to digit char. +pub fn u8_to_digit(n: u8) -> Option { + if n <= 9 { + Some((b'0' + n) as char) + } else { + None + } +} + +/// Convert hex char to u8. +pub fn hex_char_to_u8(c: char) -> Option { + c.to_digit(16).map(|d| d as u8) +} + +/// Convert u8 to hex char (lowercase). +pub fn u8_to_hex_char(n: u8) -> Option { + if n < 16 { + Some(if n < 10 { + (b'0' + n) as char + } else { + (b'a' + n - 10) as char + }) + } else { + None + } +} + +/// Convert char to uppercase. +pub fn char_to_upper(c: char) -> char { + c.to_ascii_uppercase() +} + +/// Convert char to lowercase. +pub fn char_to_lower(c: char) -> char { + c.to_ascii_lowercase() +} diff --git a/src/shared/utils/data/convert/collections.rs b/src/shared/utils/data/convert/collections.rs new file mode 100644 index 0000000..d5d0b5d --- /dev/null +++ b/src/shared/utils/data/convert/collections.rs @@ -0,0 +1,103 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, LinkedList, VecDeque}; +use std::hash::Hash; + +/// Convert Vec to Vec. +pub fn map_vec(vec: Vec, f: F) -> Vec +where + F: Fn(T) -> U, +{ + vec.into_iter().map(f).collect() +} + +/// Convert &[T] to Vec. +pub fn map_slice(slice: &[T], f: F) -> Vec +where + F: Fn(&T) -> U, +{ + slice.iter().map(f).collect() +} + +/// Convert Vec to Vec<&str>. +pub fn strings_to_strs(strings: &[String]) -> Vec<&str> { + strings.iter().map(|s| s.as_str()).collect() +} + +/// Convert Vec<&str> to Vec. +pub fn strs_to_strings(strs: &[&str]) -> Vec { + strs.iter().map(|s| s.to_string()).collect() +} + +/// Convert slice to fixed array. +pub fn slice_to_array(slice: &[T]) -> Option<[T; N]> { + slice.try_into().ok() +} + +/// Convert Vec to VecDeque. +pub fn vec_to_deque(v: Vec) -> VecDeque { + VecDeque::from(v) +} + +/// Convert VecDeque to Vec. +pub fn deque_to_vec(d: VecDeque) -> Vec { + d.into_iter().collect() +} + +/// Convert Vec to LinkedList. +pub fn vec_to_linked_list(v: Vec) -> LinkedList { + v.into_iter().collect() +} + +/// Convert LinkedList to Vec. +pub fn linked_list_to_vec(l: LinkedList) -> Vec { + l.into_iter().collect() +} + +/// Convert Vec to HashSet. +pub fn vec_to_hashset(v: Vec) -> HashSet { + v.into_iter().collect() +} + +/// Convert HashSet to Vec. +pub fn hashset_to_vec(s: HashSet) -> Vec { + s.into_iter().collect() +} + +/// Convert Vec to BTreeSet. +pub fn vec_to_btreeset(v: Vec) -> BTreeSet { + v.into_iter().collect() +} + +/// Convert BTreeSet to Vec. +pub fn btreeset_to_vec(s: BTreeSet) -> Vec { + s.into_iter().collect() +} + +/// Convert Vec of tuples to HashMap. +pub fn vec_to_hashmap(v: Vec<(K, V)>) -> HashMap { + v.into_iter().collect() +} + +/// Convert HashMap to Vec of tuples. +pub fn hashmap_to_vec(m: HashMap) -> Vec<(K, V)> { + m.into_iter().collect() +} + +/// Convert Vec of tuples to BTreeMap. +pub fn vec_to_btreemap(v: Vec<(K, V)>) -> BTreeMap { + v.into_iter().collect() +} + +/// Convert BTreeMap to Vec of tuples. +pub fn btreemap_to_vec(m: BTreeMap) -> Vec<(K, V)> { + m.into_iter().collect() +} + +/// Convert HashMap to BTreeMap. +pub fn hashmap_to_btreemap(m: HashMap) -> BTreeMap { + m.into_iter().collect() +} + +/// Convert BTreeMap to HashMap. +pub fn btreemap_to_hashmap(m: BTreeMap) -> HashMap { + m.into_iter().collect() +} diff --git a/src/shared/utils/data/convert/color.rs b/src/shared/utils/data/convert/color.rs new file mode 100644 index 0000000..c31e2c4 --- /dev/null +++ b/src/shared/utils/data/convert/color.rs @@ -0,0 +1,70 @@ +/// Convert hex color to RGB tuple. +pub fn hex_to_rgb(hex: &str) -> Option<(u8, u8, u8)> { + let hex = hex.trim_start_matches('#'); + if hex.len() != 6 { + return None; + } + let r = u8::from_str_radix(&hex[0..2], 16).ok()?; + let g = u8::from_str_radix(&hex[2..4], 16).ok()?; + let b = u8::from_str_radix(&hex[4..6], 16).ok()?; + Some((r, g, b)) +} + +/// Convert hex color to RGBA tuple. +pub fn hex_to_rgba(hex: &str) -> Option<(u8, u8, u8, u8)> { + let hex = hex.trim_start_matches('#'); + if hex.len() == 6 { + let (r, g, b) = hex_to_rgb(hex)?; + return Some((r, g, b, 255)); + } + if hex.len() != 8 { + return None; + } + let r = u8::from_str_radix(&hex[0..2], 16).ok()?; + let g = u8::from_str_radix(&hex[2..4], 16).ok()?; + let b = u8::from_str_radix(&hex[4..6], 16).ok()?; + let a = u8::from_str_radix(&hex[6..8], 16).ok()?; + Some((r, g, b, a)) +} + +/// Convert RGB to hex color. +pub fn rgb_to_hex(r: u8, g: u8, b: u8) -> String { + format!("#{:02x}{:02x}{:02x}", r, g, b) +} + +/// Convert RGBA to hex color. +pub fn rgba_to_hex(r: u8, g: u8, b: u8, a: u8) -> String { + format!("#{:02x}{:02x}{:02x}{:02x}", r, g, b, a) +} + +/// Convert RGB to HSL. +pub fn rgb_to_hsl(r: u8, g: u8, b: u8) -> (f64, f64, f64) { + let r = r as f64 / 255.0; + let g = g as f64 / 255.0; + let b = b as f64 / 255.0; + + let max = r.max(g).max(b); + let min = r.min(g).min(b); + let l = (max + min) / 2.0; + + if (max - min).abs() < f64::EPSILON { + return (0.0, 0.0, l); + } + + let d = max - min; + let s = if l > 0.5 { + d / (2.0 - max - min) + } else { + d / (max + min) + }; + + let h = if (max - r).abs() < f64::EPSILON { + ((g - b) / d + if g < b { 6.0 } else { 0.0 }) / 6.0 + } else if (max - g).abs() < f64::EPSILON { + ((b - r) / d + 2.0) / 6.0 + } else { + ((r - g) / d + 4.0) / 6.0 + }; + + (h * 360.0, s * 100.0, l * 100.0) +} diff --git a/src/shared/utils/data/convert/mod.rs b/src/shared/utils/data/convert/mod.rs new file mode 100644 index 0000000..01c36f6 --- /dev/null +++ b/src/shared/utils/data/convert/mod.rs @@ -0,0 +1,62 @@ +pub mod bools; +pub mod bytes; +pub mod char; +pub mod collections; +pub mod color; +pub mod network; +pub mod numeric; +pub mod path; +pub mod pointers; +pub mod result; +pub mod string; +pub mod time; + +pub use bools::*; +pub use bytes::*; +pub use char::*; +pub use collections::*; +pub use color::*; +pub use network::*; +pub use numeric::*; +pub use path::*; +pub use pointers::*; +pub use result::*; +pub use string::*; +pub use time::*; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_integer_conversions() { + assert_eq!(i64_to_u8(300), 255); + assert_eq!(i64_to_u8(-5), 0); + assert_eq!(i128_to_i64(i128::MAX), i64::MAX); + } + + #[test] + fn test_time_conversions() { + assert_eq!(seconds_to_human(90), "1m 30s"); + assert_eq!(seconds_to_compact(86400), "1d"); + } + + #[test] + fn test_color_conversions() { + assert_eq!(hex_to_rgb("#ff8000"), Some((255, 128, 0))); + assert_eq!(rgb_to_hex(255, 128, 0), "#ff8000"); + } + + #[test] + fn test_network_conversions() { + assert_eq!(ipv4_to_u32("192.168.1.1"), Some(0xC0A80101)); + assert_eq!(u32_to_ipv4(0xC0A80101), "192.168.1.1"); + } + + #[test] + fn test_path_conversions() { + let path = str_to_path("/test/path.txt"); + assert_eq!(path_extension(&path), Some("txt".to_string())); + assert_eq!(path_filename(&path), Some("path.txt".to_string())); + } +} diff --git a/src/shared/utils/data/convert/network.rs b/src/shared/utils/data/convert/network.rs new file mode 100644 index 0000000..3115475 --- /dev/null +++ b/src/shared/utils/data/convert/network.rs @@ -0,0 +1,60 @@ +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}; + +/// Convert IPv4 string to u32. +pub fn ipv4_to_u32(ip: &str) -> Option { + ip.parse::().ok().map(|addr| u32::from(addr)) +} + +/// Convert u32 to IPv4 string. +pub fn u32_to_ipv4(n: u32) -> String { + Ipv4Addr::from(n).to_string() +} + +/// Convert u32 to Ipv4Addr. +pub fn u32_to_ipv4_addr(n: u32) -> Ipv4Addr { + Ipv4Addr::from(n) +} + +/// Convert string to IpAddr. +pub fn str_to_ip_addr(s: &str) -> Option { + s.parse().ok() +} + +/// Convert IpAddr to string. +pub fn ip_addr_to_string(ip: IpAddr) -> String { + ip.to_string() +} + +/// Convert string to SocketAddr. +pub fn str_to_socket_addr(s: &str) -> Option { + s.parse().ok() +} + +/// Convert SocketAddr to string. +pub fn socket_addr_to_string(addr: SocketAddr) -> String { + addr.to_string() +} + +/// Create SocketAddrV4 from IP and port. +pub fn ipv4_port_to_socket(ip: Ipv4Addr, port: u16) -> SocketAddrV4 { + SocketAddrV4::new(ip, port) +} + +/// Create SocketAddrV6 from IP and port. +pub fn ipv6_port_to_socket(ip: Ipv6Addr, port: u16) -> SocketAddrV6 { + SocketAddrV6::new(ip, port, 0, 0) +} + +/// Check if IP is loopback. +pub fn is_loopback(ip: &str) -> bool { + ip.parse::() + .map(|a| a.is_loopback()) + .unwrap_or(false) +} + +/// Check if IP is private (IPv4). +pub fn is_private_ip(ip: &str) -> bool { + ip.parse::() + .map(|a| a.is_private()) + .unwrap_or(false) +} diff --git a/src/shared/utils/data/convert/numeric.rs b/src/shared/utils/data/convert/numeric.rs new file mode 100644 index 0000000..2fed348 --- /dev/null +++ b/src/shared/utils/data/convert/numeric.rs @@ -0,0 +1,532 @@ +/// Safe i8 to u8. +pub fn i8_to_u8(n: i8) -> u8 { + n.max(0) as u8 +} + +/// Safe i8 to i16. +pub fn i8_to_i16(n: i8) -> i16 { + n as i16 +} + +/// Safe i8 to i32. +pub fn i8_to_i32(n: i8) -> i32 { + n as i32 +} + +/// Safe i8 to i64. +pub fn i8_to_i64(n: i8) -> i64 { + n as i64 +} + +/// Safe i8 to i128. +pub fn i8_to_i128(n: i8) -> i128 { + n as i128 +} + +/// Safe i8 to f32. +pub fn i8_to_f32(n: i8) -> f32 { + n as f32 +} + +/// Safe i8 to f64. +pub fn i8_to_f64(n: i8) -> f64 { + n as f64 +} + +/// Safe i16 to i8 (saturates). +pub fn i16_to_i8(n: i16) -> i8 { + n.clamp(i8::MIN as i16, i8::MAX as i16) as i8 +} + +/// Safe i16 to u8 (saturates). +pub fn i16_to_u8(n: i16) -> u8 { + n.clamp(0, u8::MAX as i16) as u8 +} + +/// Safe i16 to u16. +pub fn i16_to_u16(n: i16) -> u16 { + n.max(0) as u16 +} + +/// Safe i16 to i32. +pub fn i16_to_i32(n: i16) -> i32 { + n as i32 +} + +/// Safe i16 to i64. +pub fn i16_to_i64(n: i16) -> i64 { + n as i64 +} + +/// Safe i16 to i128. +pub fn i16_to_i128(n: i16) -> i128 { + n as i128 +} + +/// Safe i32 to i8 (saturates). +pub fn i32_to_i8(n: i32) -> i8 { + n.clamp(i8::MIN as i32, i8::MAX as i32) as i8 +} + +/// Safe i32 to i16 (saturates). +pub fn i32_to_i16(n: i32) -> i16 { + n.clamp(i16::MIN as i32, i16::MAX as i32) as i16 +} + +/// Safe i32 to u8 (saturates). +pub fn i32_to_u8(n: i32) -> u8 { + n.clamp(0, u8::MAX as i32) as u8 +} + +/// Safe i32 to u16 (saturates). +pub fn i32_to_u16(n: i32) -> u16 { + n.clamp(0, u16::MAX as i32) as u16 +} + +/// Safe i32 to u32. +pub fn i32_to_u32(n: i32) -> u32 { + n.max(0) as u32 +} + +/// Safe i32 to i64. +pub fn i32_to_i64(n: i32) -> i64 { + n as i64 +} + +/// Safe i32 to i128. +pub fn i32_to_i128(n: i32) -> i128 { + n as i128 +} + +/// Safe i32 to usize. +pub fn i32_to_usize(n: i32) -> usize { + n.max(0) as usize +} + +/// Safe i32 to f32. +pub fn i32_to_f32(n: i32) -> f32 { + n as f32 +} + +/// Safe i32 to f64. +pub fn i32_to_f64(n: i32) -> f64 { + n as f64 +} + +/// Safe i64 to i8 (saturates). +pub fn i64_to_i8(n: i64) -> i8 { + n.clamp(i8::MIN as i64, i8::MAX as i64) as i8 +} + +/// Safe i64 to i16 (saturates). +pub fn i64_to_i16(n: i64) -> i16 { + n.clamp(i16::MIN as i64, i16::MAX as i64) as i16 +} + +/// Safe i64 to i32 (saturates). +pub fn i64_to_i32(n: i64) -> i32 { + n.clamp(i32::MIN as i64, i32::MAX as i64) as i32 +} + +/// Safe i64 to u8 (saturates). +pub fn i64_to_u8(n: i64) -> u8 { + n.clamp(0, u8::MAX as i64) as u8 +} + +/// Safe i64 to u16 (saturates). +pub fn i64_to_u16(n: i64) -> u16 { + n.clamp(0, u16::MAX as i64) as u16 +} + +/// Safe i64 to u32 (saturates). +pub fn i64_to_u32(n: i64) -> u32 { + n.clamp(0, u32::MAX as i64) as u32 +} + +/// Safe i64 to u64. +pub fn i64_to_u64(n: i64) -> u64 { + n.max(0) as u64 +} + +/// Safe i64 to usize. +pub fn i64_to_usize(n: i64) -> usize { + n.max(0) as usize +} + +/// Safe i64 to i128. +pub fn i64_to_i128(n: i64) -> i128 { + n as i128 +} + +/// Safe i64 to u128. +pub fn i64_to_u128(n: i64) -> u128 { + n.max(0) as u128 +} + +/// Safe i64 to f32. +pub fn i64_to_f32(n: i64) -> f32 { + n as f32 +} + +/// Safe i64 to f64. +pub fn i64_to_f64(n: i64) -> f64 { + n as f64 +} + +/// Safe i128 to i8 (saturates). +pub fn i128_to_i8(n: i128) -> i8 { + n.clamp(i8::MIN as i128, i8::MAX as i128) as i8 +} + +/// Safe i128 to i16 (saturates). +pub fn i128_to_i16(n: i128) -> i16 { + n.clamp(i16::MIN as i128, i16::MAX as i128) as i16 +} + +/// Safe i128 to i32 (saturates). +pub fn i128_to_i32(n: i128) -> i32 { + n.clamp(i32::MIN as i128, i32::MAX as i128) as i32 +} + +/// Safe i128 to i64 (saturates). +pub fn i128_to_i64(n: i128) -> i64 { + n.clamp(i64::MIN as i128, i64::MAX as i128) as i64 +} + +/// Safe i128 to u128. +pub fn i128_to_u128(n: i128) -> u128 { + n.max(0) as u128 +} + +/// Safe i128 to usize. +pub fn i128_to_usize(n: i128) -> usize { + n.clamp(0, usize::MAX as i128) as usize +} + +/// u8 to i8 (may overflow to negative). +pub fn u8_to_i8_wrap(n: u8) -> i8 { + n as i8 +} + +/// u8 to i8 (saturates at i8::MAX). +pub fn u8_to_i8_sat(n: u8) -> i8 { + n.min(i8::MAX as u8) as i8 +} + +/// u8 to i16. +pub fn u8_to_i16(n: u8) -> i16 { + n as i16 +} + +/// u8 to i32. +pub fn u8_to_i32(n: u8) -> i32 { + n as i32 +} + +/// u8 to i64. +pub fn u8_to_i64(n: u8) -> i64 { + n as i64 +} + +/// u8 to u16. +pub fn u8_to_u16(n: u8) -> u16 { + n as u16 +} + +/// u8 to u32. +pub fn u8_to_u32(n: u8) -> u32 { + n as u32 +} + +/// u8 to u64. +pub fn u8_to_u64(n: u8) -> u64 { + n as u64 +} + +/// u8 to usize. +pub fn u8_to_usize(n: u8) -> usize { + n as usize +} + +/// u8 to f32. +pub fn u8_to_f32(n: u8) -> f32 { + n as f32 +} + +/// u8 to f64. +pub fn u8_to_f64(n: u8) -> f64 { + n as f64 +} + +/// u16 to u8 (saturates). +pub fn u16_to_u8(n: u16) -> u8 { + n.min(u8::MAX as u16) as u8 +} + +/// u16 to i16 (saturates). +pub fn u16_to_i16(n: u16) -> i16 { + n.min(i16::MAX as u16) as i16 +} + +/// u16 to i32. +pub fn u16_to_i32(n: u16) -> i32 { + n as i32 +} + +/// u16 to i64. +pub fn u16_to_i64(n: u16) -> i64 { + n as i64 +} + +/// u16 to u32. +pub fn u16_to_u32(n: u16) -> u32 { + n as u32 +} + +/// u16 to u64. +pub fn u16_to_u64(n: u16) -> u64 { + n as u64 +} + +/// u16 to usize. +pub fn u16_to_usize(n: u16) -> usize { + n as usize +} + +/// u32 to u8 (saturates). +pub fn u32_to_u8(n: u32) -> u8 { + n.min(u8::MAX as u32) as u8 +} + +/// u32 to u16 (saturates). +pub fn u32_to_u16(n: u32) -> u16 { + n.min(u16::MAX as u32) as u16 +} + +/// u32 to i32 (saturates). +pub fn u32_to_i32(n: u32) -> i32 { + n.min(i32::MAX as u32) as i32 +} + +/// u32 to i64. +pub fn u32_to_i64(n: u32) -> i64 { + n as i64 +} + +/// u32 to u64. +pub fn u32_to_u64(n: u32) -> u64 { + n as u64 +} + +/// u32 to usize. +pub fn u32_to_usize(n: u32) -> usize { + n as usize +} + +/// u32 to f32. +pub fn u32_to_f32(n: u32) -> f32 { + n as f32 +} + +/// u32 to f64. +pub fn u32_to_f64(n: u32) -> f64 { + n as f64 +} + +/// u64 to u8 (saturates). +pub fn u64_to_u8(n: u64) -> u8 { + n.min(u8::MAX as u64) as u8 +} + +/// u64 to u16 (saturates). +pub fn u64_to_u16(n: u64) -> u16 { + n.min(u16::MAX as u64) as u16 +} + +/// u64 to u32 (saturates). +pub fn u64_to_u32(n: u64) -> u32 { + n.min(u32::MAX as u64) as u32 +} + +/// u64 to i64 (saturates). +pub fn u64_to_i64(n: u64) -> i64 { + n.min(i64::MAX as u64) as i64 +} + +/// u64 to i128. +pub fn u64_to_i128(n: u64) -> i128 { + n as i128 +} + +/// u64 to u128. +pub fn u64_to_u128(n: u64) -> u128 { + n as u128 +} + +/// u64 to usize (may truncate on 32-bit). +pub fn u64_to_usize(n: u64) -> usize { + n as usize +} + +/// u64 to f64. +pub fn u64_to_f64(n: u64) -> f64 { + n as f64 +} + +/// u128 to u64 (saturates). +pub fn u128_to_u64(n: u128) -> u64 { + n.min(u64::MAX as u128) as u64 +} + +/// u128 to i128 (saturates). +pub fn u128_to_i128(n: u128) -> i128 { + n.min(i128::MAX as u128) as i128 +} + +/// u128 to usize (saturates). +pub fn u128_to_usize(n: u128) -> usize { + n.min(usize::MAX as u128) as usize +} + +/// usize to i32 (saturates). +pub fn usize_to_i32(n: usize) -> i32 { + n.min(i32::MAX as usize) as i32 +} + +/// usize to i64. +pub fn usize_to_i64(n: usize) -> i64 { + n as i64 +} + +/// usize to u32 (saturates on 64-bit). +pub fn usize_to_u32(n: usize) -> u32 { + n.min(u32::MAX as usize) as u32 +} + +/// usize to u64. +pub fn usize_to_u64(n: usize) -> u64 { + n as u64 +} + +/// isize to i32 (saturates). +pub fn isize_to_i32(n: isize) -> i32 { + n.clamp(i32::MIN as isize, i32::MAX as isize) as i32 +} + +/// isize to i64. +pub fn isize_to_i64(n: isize) -> i64 { + n as i64 +} + +/// isize to usize. +pub fn isize_to_usize(n: isize) -> usize { + n.max(0) as usize +} + +/// f64 to f32 (may lose precision). +pub fn f64_to_f32(n: f64) -> f32 { + n as f32 +} + +/// f32 to f64. +pub fn f32_to_f64(n: f32) -> f64 { + n as f64 +} + +/// f64 to i64 (truncates). +pub fn f64_to_i64(n: f64) -> i64 { + n.clamp(i64::MIN as f64, i64::MAX as f64) as i64 +} + +/// f64 to i32 (truncates). +pub fn f64_to_i32(n: f64) -> i32 { + n.clamp(i32::MIN as f64, i32::MAX as f64) as i32 +} + +/// f64 to u64 (truncates). +pub fn f64_to_u64(n: f64) -> u64 { + n.clamp(0.0, u64::MAX as f64) as u64 +} + +/// f64 to u32 (truncates). +pub fn f64_to_u32(n: f64) -> u32 { + n.clamp(0.0, u32::MAX as f64) as u32 +} + +/// f32 to i32 (truncates). +pub fn f32_to_i32(n: f32) -> i32 { + n.clamp(i32::MIN as f32, i32::MAX as f32) as i32 +} + +/// Round f64 to n decimal places. +pub fn round_f64(n: f64, decimals: u32) -> f64 { + let factor = 10_f64.powi(decimals as i32); + (n * factor).round() / factor +} + +/// Round f32 to n decimal places. +pub fn round_f32(n: f32, decimals: u32) -> f32 { + let factor = 10_f32.powi(decimals as i32); + (n * factor).round() / factor +} + +/// Truncate f64 to n decimal places. +pub fn trunc_f64(n: f64, decimals: u32) -> f64 { + let factor = 10_f64.powi(decimals as i32); + (n * factor).trunc() / factor +} + +/// Ceil f64 to n decimal places. +pub fn ceil_f64(n: f64, decimals: u32) -> f64 { + let factor = 10_f64.powi(decimals as i32); + (n * factor).ceil() / factor +} + +/// Floor f64 to n decimal places. +pub fn floor_f64(n: f64, decimals: u32) -> f64 { + let factor = 10_f64.powi(decimals as i32); + (n * factor).floor() / factor +} + +/// Check if f64 is essentially zero. +pub fn is_zero(n: f64, epsilon: f64) -> bool { + n.abs() < epsilon +} + +/// Compare two f64 for approximate equality. +pub fn approx_eq(a: f64, b: f64, epsilon: f64) -> bool { + (a - b).abs() < epsilon +} + +/// Check if f64 is NaN. +pub fn is_nan(n: f64) -> bool { + n.is_nan() +} + +/// Check if f64 is infinite. +pub fn is_infinite(n: f64) -> bool { + n.is_infinite() +} + +/// Check if f64 is finite. +pub fn is_finite(n: f64) -> bool { + n.is_finite() +} + +/// Convert NaN to 0. +pub fn nan_to_zero(n: f64) -> f64 { + if n.is_nan() { + 0.0 + } else { + n + } +} + +/// Convert NaN to default. +pub fn nan_to_default(n: f64, default: f64) -> f64 { + if n.is_nan() { + default + } else { + n + } +} diff --git a/src/shared/utils/data/convert/path.rs b/src/shared/utils/data/convert/path.rs new file mode 100644 index 0000000..19c3db2 --- /dev/null +++ b/src/shared/utils/data/convert/path.rs @@ -0,0 +1,62 @@ +use std::ffi::{OsStr, OsString}; +use std::path::{Path, PathBuf}; + +/// Convert &str to PathBuf. +pub fn str_to_path(s: &str) -> PathBuf { + PathBuf::from(s) +} + +/// Convert String to PathBuf. +pub fn string_to_path(s: String) -> PathBuf { + PathBuf::from(s) +} + +/// Convert PathBuf to String (lossy). +pub fn path_to_string(p: &Path) -> String { + p.to_string_lossy().to_string() +} + +/// Convert PathBuf to Option (None if not valid UTF-8). +pub fn path_to_string_strict(p: &Path) -> Option { + p.to_str().map(String::from) +} + +/// Convert &str to &Path. +pub fn str_to_path_ref(s: &str) -> &Path { + Path::new(s) +} + +/// Convert OsStr to String (lossy). +pub fn os_str_to_string(s: &OsStr) -> String { + s.to_string_lossy().to_string() +} + +/// Convert OsString to String (lossy). +pub fn os_string_to_string(s: OsString) -> String { + s.to_string_lossy().to_string() +} + +/// Convert String to OsString. +pub fn string_to_os_string(s: String) -> OsString { + OsString::from(s) +} + +/// Convert &str to &OsStr. +pub fn str_to_os_str(s: &str) -> &OsStr { + OsStr::new(s) +} + +/// Get file extension as String. +pub fn path_extension(p: &Path) -> Option { + p.extension().and_then(|e| e.to_str()).map(String::from) +} + +/// Get file name as String. +pub fn path_filename(p: &Path) -> Option { + p.file_name().and_then(|n| n.to_str()).map(String::from) +} + +/// Get parent directory as PathBuf. +pub fn path_parent(p: &Path) -> Option { + p.parent().map(PathBuf::from) +} diff --git a/src/shared/utils/data/convert/pointers.rs b/src/shared/utils/data/convert/pointers.rs new file mode 100644 index 0000000..8d309e5 --- /dev/null +++ b/src/shared/utils/data/convert/pointers.rs @@ -0,0 +1,58 @@ +use std::borrow::Cow; +use std::rc::Rc; +use std::sync::Arc; + +/// Wrap value in Box. +pub fn to_box(value: T) -> Box { + Box::new(value) +} + +/// Wrap value in Rc. +pub fn to_rc(value: T) -> Rc { + Rc::new(value) +} + +/// Wrap value in Arc. +pub fn to_arc(value: T) -> Arc { + Arc::new(value) +} + +/// Convert Box to T (unbox). +pub fn unbox(boxed: Box) -> T { + *boxed +} + +/// Clone from Rc. +pub fn rc_to_owned(rc: &Rc) -> T { + (**rc).clone() +} + +/// Clone from Arc. +pub fn arc_to_owned(arc: &Arc) -> T { + (**arc).clone() +} + +/// Convert &str to Cow. +pub fn str_to_cow(s: &str) -> Cow<'_, str> { + Cow::Borrowed(s) +} + +/// Convert String to Cow. +pub fn string_to_cow(s: String) -> Cow<'static, str> { + Cow::Owned(s) +} + +/// Convert Cow to String. +pub fn cow_to_string(cow: Cow<'_, str>) -> String { + cow.into_owned() +} + +/// Convert &[T] to Cow<[T]>. +pub fn slice_to_cow(s: &[T]) -> Cow<'_, [T]> { + Cow::Borrowed(s) +} + +/// Convert Vec to Cow<[T]>. +pub fn vec_to_cow(v: Vec) -> Cow<'static, [T]> { + Cow::Owned(v) +} diff --git a/src/shared/utils/data/convert/result.rs b/src/shared/utils/data/convert/result.rs new file mode 100644 index 0000000..a736234 --- /dev/null +++ b/src/shared/utils/data/convert/result.rs @@ -0,0 +1,39 @@ +/// Convert Option to Result. +pub fn option_to_result(opt: Option, err: E) -> Result { + opt.ok_or(err) +} + +/// Convert Option to Result. +pub fn option_to_result_str(opt: Option, msg: &str) -> Result { + opt.ok_or_else(|| msg.to_string()) +} + +/// Convert Result to Option. +pub fn result_to_option(res: Result) -> Option { + res.ok() +} + +/// Convert Result to Option. +pub fn result_to_err(res: Result) -> Option { + res.err() +} + +/// Flatten nested Option. +pub fn flatten_option(opt: Option>) -> Option { + opt.flatten() +} + +/// Flatten nested Result. +pub fn flatten_result(res: Result, E>) -> Result { + res.and_then(|r| r) +} + +/// Transpose Option> to Result, E>. +pub fn transpose_option_result(opt: Option>) -> Result, E> { + opt.transpose() +} + +/// Transpose Result, E> to Option>. +pub fn transpose_result_option(res: Result, E>) -> Option> { + res.transpose() +} diff --git a/src/shared/utils/data/convert/string.rs b/src/shared/utils/data/convert/string.rs new file mode 100644 index 0000000..e01af9d --- /dev/null +++ b/src/shared/utils/data/convert/string.rs @@ -0,0 +1,134 @@ +use std::fmt::Display; +use std::str::FromStr; + +/// Parse with default value. +pub fn parse_or(s: &str, default: T) -> T { + s.parse().unwrap_or(default) +} + +/// Try parse with error context. +pub fn try_parse(s: &str, name: &str) -> Result +where + T::Err: Display, +{ + s.parse() + .map_err(|e| format!("Failed to parse {}: {}", name, e)) +} + +/// Parse i8 with default. +pub fn parse_i8(s: &str, default: i8) -> i8 { + s.parse().unwrap_or(default) +} + +/// Parse i16 with default. +pub fn parse_i16(s: &str, default: i16) -> i16 { + s.parse().unwrap_or(default) +} + +/// Parse i32 with default. +pub fn parse_i32(s: &str, default: i32) -> i32 { + s.parse().unwrap_or(default) +} + +/// Parse i64 with default. +pub fn parse_i64(s: &str, default: i64) -> i64 { + s.parse().unwrap_or(default) +} + +/// Parse i128 with default. +pub fn parse_i128(s: &str, default: i128) -> i128 { + s.parse().unwrap_or(default) +} + +/// Parse u8 with default. +pub fn parse_u8(s: &str, default: u8) -> u8 { + s.parse().unwrap_or(default) +} + +/// Parse u16 with default. +pub fn parse_u16(s: &str, default: u16) -> u16 { + s.parse().unwrap_or(default) +} + +/// Parse u32 with default. +pub fn parse_u32(s: &str, default: u32) -> u32 { + s.parse().unwrap_or(default) +} + +/// Parse u64 with default. +pub fn parse_u64(s: &str, default: u64) -> u64 { + s.parse().unwrap_or(default) +} + +/// Parse u128 with default. +pub fn parse_u128(s: &str, default: u128) -> u128 { + s.parse().unwrap_or(default) +} + +/// Parse f32 with default. +pub fn parse_f32(s: &str, default: f32) -> f32 { + s.parse().unwrap_or(default) +} + +/// Parse f64 with default. +pub fn parse_f64(s: &str, default: f64) -> f64 { + s.parse().unwrap_or(default) +} + +/// Parse usize with default. +pub fn parse_usize(s: &str, default: usize) -> usize { + s.parse().unwrap_or(default) +} + +/// Parse isize with default. +pub fn parse_isize(s: &str, default: isize) -> isize { + s.parse().unwrap_or(default) +} + +/// Convert string to Option (None for empty). +pub fn empty_to_none(s: &str) -> Option { + if s.is_empty() { + None + } else { + Some(s.to_string()) + } +} + +/// Convert None to empty string. +pub fn none_to_empty(opt: Option) -> String { + opt.unwrap_or_default() +} + +/// Convert Option<&str> to Option. +pub fn str_to_string(opt: Option<&str>) -> Option { + opt.map(|s| s.to_string()) +} + +/// Convert String to Option (None if empty). +pub fn string_to_option(s: String) -> Option { + if s.is_empty() { + None + } else { + Some(s) + } +} + +/// Trim and convert to Option (None if whitespace only). +pub fn trim_to_option(s: &str) -> Option { + let trimmed = s.trim(); + if trimmed.is_empty() { + None + } else { + Some(trimmed.to_string()) + } +} + +/// Convert &str to String. +pub fn str_to_owned(s: &str) -> String { + s.to_string() +} + +/// Convert String to &str (returns empty if none). +pub fn string_to_str(s: &Option) -> &str { + s.as_deref().unwrap_or("") +} diff --git a/src/shared/utils/data/convert/time.rs b/src/shared/utils/data/convert/time.rs new file mode 100644 index 0000000..55f5e17 --- /dev/null +++ b/src/shared/utils/data/convert/time.rs @@ -0,0 +1,142 @@ +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +/// Convert seconds to human readable duration. +pub fn seconds_to_human(secs: u64) -> String { + if secs < 60 { + format!("{}s", secs) + } else if secs < 3600 { + format!("{}m {}s", secs / 60, secs % 60) + } else if secs < 86400 { + format!("{}h {}m", secs / 3600, (secs % 3600) / 60) + } else if secs < 604800 { + format!("{}d {}h", secs / 86400, (secs % 86400) / 3600) + } else if secs < 2592000 { + format!("{}w {}d", secs / 604800, (secs % 604800) / 86400) + } else if secs < 31536000 { + format!("{}mo {}d", secs / 2592000, (secs % 2592000) / 86400) + } else { + format!("{}y {}mo", secs / 31536000, (secs % 31536000) / 2592000) + } +} + +/// Convert seconds to compact human readable. +pub fn seconds_to_compact(secs: u64) -> String { + if secs < 60 { + format!("{}s", secs) + } else if secs < 3600 { + format!("{}m", secs / 60) + } else if secs < 86400 { + format!("{}h", secs / 3600) + } else if secs < 604800 { + format!("{}d", secs / 86400) + } else if secs < 2592000 { + format!("{}w", secs / 604800) + } else if secs < 31536000 { + format!("{}mo", secs / 2592000) + } else { + format!("{}y", secs / 31536000) + } +} + +/// Convert milliseconds to human readable. +pub fn ms_to_human(ms: u64) -> String { + if ms < 1000 { + format!("{}ms", ms) + } else { + seconds_to_human(ms / 1000) + } +} + +/// Convert microseconds to human readable. +pub fn us_to_human(us: u64) -> String { + if us < 1000 { + format!("{}μs", us) + } else if us < 1_000_000 { + format!("{:.2}ms", us as f64 / 1000.0) + } else { + seconds_to_human(us / 1_000_000) + } +} + +/// Convert nanoseconds to human readable. +pub fn ns_to_human(ns: u64) -> String { + if ns < 1_000 { + format!("{}ns", ns) + } else if ns < 1_000_000 { + format!("{:.2}μs", ns as f64 / 1_000.0) + } else if ns < 1_000_000_000 { + format!("{:.2}ms", ns as f64 / 1_000_000.0) + } else { + format!("{:.2}s", ns as f64 / 1_000_000_000.0) + } +} + +/// Convert seconds to Duration. +pub fn secs_to_duration(secs: u64) -> Duration { + Duration::from_secs(secs) +} + +/// Convert milliseconds to Duration. +pub fn ms_to_duration(ms: u64) -> Duration { + Duration::from_millis(ms) +} + +/// Convert microseconds to Duration. +pub fn us_to_duration(us: u64) -> Duration { + Duration::from_micros(us) +} + +/// Convert nanoseconds to Duration. +pub fn ns_to_duration(ns: u64) -> Duration { + Duration::from_nanos(ns) +} + +/// Convert Duration to seconds. +pub fn duration_to_secs(d: Duration) -> u64 { + d.as_secs() +} + +/// Convert Duration to milliseconds. +pub fn duration_to_ms(d: Duration) -> u128 { + d.as_millis() +} + +/// Convert Duration to microseconds. +pub fn duration_to_us(d: Duration) -> u128 { + d.as_micros() +} + +/// Convert Duration to nanoseconds. +pub fn duration_to_ns(d: Duration) -> u128 { + d.as_nanos() +} + +/// Convert Duration to f64 seconds. +pub fn duration_to_secs_f64(d: Duration) -> f64 { + d.as_secs_f64() +} + +/// Convert f64 seconds to Duration. +pub fn secs_f64_to_duration(secs: f64) -> Duration { + Duration::from_secs_f64(secs.max(0.0)) +} + +/// Get SystemTime as Unix timestamp (seconds). +pub fn system_time_to_unix(t: SystemTime) -> u64 { + t.duration_since(UNIX_EPOCH).unwrap_or_default().as_secs() +} + +/// Get SystemTime as Unix timestamp (milliseconds). +pub fn system_time_to_unix_ms(t: SystemTime) -> u128 { + t.duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() +} + +/// Convert Unix timestamp to SystemTime. +pub fn unix_to_system_time(secs: u64) -> SystemTime { + UNIX_EPOCH + Duration::from_secs(secs) +} + +/// Convert Unix milliseconds to SystemTime. +pub fn unix_ms_to_system_time(ms: u64) -> SystemTime { + UNIX_EPOCH + Duration::from_millis(ms) +} diff --git a/src/shared/utils/data/datetime.rs b/src/shared/utils/data/datetime.rs new file mode 100644 index 0000000..53144ce --- /dev/null +++ b/src/shared/utils/data/datetime.rs @@ -0,0 +1,107 @@ +//! Date and time utilities. + +use chrono::{DateTime, Duration, NaiveDateTime, Utc}; + +/// Get current UTC timestamp. +pub fn now() -> DateTime { + Utc::now() +} + +/// Get current Unix timestamp (seconds). +pub fn timestamp() -> i64 { + Utc::now().timestamp() +} + +/// Get current Unix timestamp (milliseconds). +pub fn timestamp_millis() -> i64 { + Utc::now().timestamp_millis() +} + +/// Format datetime as ISO 8601 string. +pub fn to_iso(dt: DateTime) -> String { + dt.to_rfc3339() +} + +/// Format datetime as human-readable string. +pub fn to_human(dt: DateTime) -> String { + dt.format("%Y-%m-%d %H:%M:%S").to_string() +} + +/// Format datetime as date only. +pub fn to_date(dt: DateTime) -> String { + dt.format("%Y-%m-%d").to_string() +} + +/// Parse ISO 8601 string to DateTime. +pub fn parse_iso(s: &str) -> Option> { + DateTime::parse_from_rfc3339(s) + .ok() + .map(|dt| dt.with_timezone(&Utc)) +} + +/// Add duration to a datetime. +pub fn add_days(dt: DateTime, days: i64) -> DateTime { + dt + Duration::days(days) +} + +/// Add hours to a datetime. +pub fn add_hours(dt: DateTime, hours: i64) -> DateTime { + dt + Duration::hours(hours) +} + +/// Add minutes to a datetime. +pub fn add_minutes(dt: DateTime, minutes: i64) -> DateTime { + dt + Duration::minutes(minutes) +} + +/// Check if datetime is in the past. +pub fn is_past(dt: DateTime) -> bool { + dt < Utc::now() +} + +/// Check if datetime is in the future. +pub fn is_future(dt: DateTime) -> bool { + dt > Utc::now() +} + +/// Get relative time string (e.g., "2 hours ago"). +pub fn relative(dt: DateTime) -> String { + let duration = Utc::now().signed_duration_since(dt); + + if duration.num_seconds() < 60 { + "just now".to_string() + } else if duration.num_minutes() < 60 { + format!("{} minutes ago", duration.num_minutes()) + } else if duration.num_hours() < 24 { + format!("{} hours ago", duration.num_hours()) + } else if duration.num_days() < 30 { + format!("{} days ago", duration.num_days()) + } else if duration.num_days() < 365 { + format!("{} months ago", duration.num_days() / 30) + } else { + format!("{} years ago", duration.num_days() / 365) + } +} + +/// Calculate age in years from birthdate. +pub fn age_years(birthdate: NaiveDateTime) -> i32 { + let today = Utc::now().naive_utc(); + let years = today.date().years_since(birthdate.date()); + years.map(|y| y as i32).unwrap_or(0) +} + +/// Get start of day (00:00:00). +pub fn start_of_day(dt: DateTime) -> DateTime { + dt.date_naive() + .and_hms_opt(0, 0, 0) + .map(|naive| DateTime::::from_naive_utc_and_offset(naive, Utc)) + .unwrap_or(dt) +} + +/// Get end of day (23:59:59). +pub fn end_of_day(dt: DateTime) -> DateTime { + dt.date_naive() + .and_hms_opt(23, 59, 59) + .map(|naive| DateTime::::from_naive_utc_and_offset(naive, Utc)) + .unwrap_or(dt) +} diff --git a/src/shared/utils/data/json.rs b/src/shared/utils/data/json.rs new file mode 100644 index 0000000..f8f1f1e --- /dev/null +++ b/src/shared/utils/data/json.rs @@ -0,0 +1,137 @@ +//! JSON utilities. + +use serde::{de::DeserializeOwned, Serialize}; +use serde_json::{Map, Value}; + +/// Parse JSON string to type. +pub fn parse(json: &str) -> Result { + serde_json::from_str(json) +} + +/// Serialize type to JSON string. +pub fn stringify(value: &T) -> Result { + serde_json::to_string(value) +} + +/// Serialize type to pretty JSON string. +pub fn stringify_pretty(value: &T) -> Result { + serde_json::to_string_pretty(value) +} + +/// Merge two JSON objects (second overwrites first). +pub fn merge(base: Value, overlay: Value) -> Value { + match (base, overlay) { + (Value::Object(mut base_map), Value::Object(overlay_map)) => { + for (key, value) in overlay_map { + base_map.insert(key, value); + } + Value::Object(base_map) + } + (_, overlay) => overlay, + } +} + +/// Deep merge two JSON objects. +pub fn deep_merge(base: Value, overlay: Value) -> Value { + match (base, overlay) { + (Value::Object(mut base_map), Value::Object(overlay_map)) => { + for (key, overlay_value) in overlay_map { + let base_value = base_map.remove(&key).unwrap_or(Value::Null); + base_map.insert(key, deep_merge(base_value, overlay_value)); + } + Value::Object(base_map) + } + (_, overlay) => overlay, + } +} + +/// Extract a value at a path (e.g., "data.items.0.name"). +pub fn get_path<'a>(value: &'a Value, path: &str) -> Option<&'a Value> { + let mut current = value; + for key in path.split('.') { + current = match current { + Value::Object(map) => map.get(key)?, + Value::Array(arr) => { + let idx: usize = key.parse().ok()?; + arr.get(idx)? + } + _ => return None, + }; + } + Some(current) +} + +/// Extract string at path. +pub fn get_str<'a>(value: &'a Value, path: &str) -> Option<&'a str> { + get_path(value, path).and_then(|v| v.as_str()) +} + +/// Extract i64 at path. +pub fn get_i64(value: &Value, path: &str) -> Option { + get_path(value, path).and_then(|v| v.as_i64()) +} + +/// Extract bool at path. +pub fn get_bool(value: &Value, path: &str) -> Option { + get_path(value, path).and_then(|v| v.as_bool()) +} + +/// Extract array at path. +pub fn get_array<'a>(value: &'a Value, path: &str) -> Option<&'a Vec> { + get_path(value, path).and_then(|v| v.as_array()) +} + +/// Create a JSON object with key-value pairs. +#[macro_export] +macro_rules! json_object { + ($($key:expr => $value:expr),* $(,)?) => {{ + let mut map = serde_json::Map::new(); + $( + map.insert($key.to_string(), serde_json::json!($value)); + )* + serde_json::Value::Object(map) + }}; +} + +/// Check if value is empty (null, empty string, empty array, empty object). +pub fn is_empty(value: &Value) -> bool { + match value { + Value::Null => true, + Value::String(s) => s.is_empty(), + Value::Array(arr) => arr.is_empty(), + Value::Object(obj) => obj.is_empty(), + _ => false, + } +} + +/// Remove null values from object. +pub fn remove_nulls(value: Value) -> Value { + match value { + Value::Object(map) => { + let filtered: Map = map + .into_iter() + .filter(|(_, v)| !v.is_null()) + .map(|(k, v)| (k, remove_nulls(v))) + .collect(); + Value::Object(filtered) + } + Value::Array(arr) => Value::Array(arr.into_iter().map(remove_nulls).collect()), + other => other, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn test_get_path() { + let value = json!({ + "data": { + "items": [{"name": "test"}] + } + }); + assert_eq!(get_str(&value, "data.items.0.name"), Some("test")); + } +} diff --git a/src/shared/utils/data/mod.rs b/src/shared/utils/data/mod.rs new file mode 100644 index 0000000..59dda88 --- /dev/null +++ b/src/shared/utils/data/mod.rs @@ -0,0 +1,7 @@ +pub mod collections; +pub mod convert; +pub mod datetime; +pub mod json; +pub mod numbers; +pub mod string; +pub mod text; diff --git a/src/shared/utils/data/numbers.rs b/src/shared/utils/data/numbers.rs new file mode 100644 index 0000000..690d8b5 --- /dev/null +++ b/src/shared/utils/data/numbers.rs @@ -0,0 +1,151 @@ +//! Number utilities. + +/// Format number with thousand separators. +pub fn format_number(n: i64) -> String { + let s = n.abs().to_string(); + let chars: Vec = s.chars().rev().collect(); + let mut result = String::new(); + + for (i, c) in chars.iter().enumerate() { + if i > 0 && i % 3 == 0 { + result.push(','); + } + result.push(*c); + } + + if n < 0 { + result.push('-'); + } + + result.chars().rev().collect() +} + +/// Format as percentage. +pub fn format_percent(value: f64, decimals: usize) -> String { + format!("{:.1$}%", value * 100.0, decimals) +} + +/// Format as currency. +pub fn format_currency(value: f64, symbol: &str) -> String { + format!("{}{:.2}", symbol, value) +} + +/// Format bytes to human readable. +pub fn format_bytes(bytes: u64) -> String { + const KB: u64 = 1024; + const MB: u64 = KB * 1024; + const GB: u64 = MB * 1024; + const TB: u64 = GB * 1024; + + if bytes >= TB { + format!("{:.2} TB", bytes as f64 / TB as f64) + } else if bytes >= GB { + format!("{:.2} GB", bytes as f64 / GB as f64) + } else if bytes >= MB { + format!("{:.2} MB", bytes as f64 / MB as f64) + } else if bytes >= KB { + format!("{:.2} KB", bytes as f64 / KB as f64) + } else { + format!("{} B", bytes) + } +} + +/// Clamp value to range. +pub fn clamp(value: T, min: T, max: T) -> T { + if value < min { + min + } else if value > max { + max + } else { + value + } +} + +/// Linear interpolation. +pub fn lerp(a: f64, b: f64, t: f64) -> f64 { + a + (b - a) * t +} + +/// Map value from one range to another. +pub fn map_range(value: f64, from_min: f64, from_max: f64, to_min: f64, to_max: f64) -> f64 { + let from_range = from_max - from_min; + let to_range = to_max - to_min; + ((value - from_min) / from_range) * to_range + to_min +} + +/// Round to n decimal places. +pub fn round_to(value: f64, decimals: u32) -> f64 { + let factor = 10_f64.powi(decimals as i32); + (value * factor).round() / factor +} + +/// Check if number is even. +pub fn is_even(n: i64) -> bool { + n % 2 == 0 +} + +/// Check if number is odd. +pub fn is_odd(n: i64) -> bool { + n % 2 != 0 +} + +/// Check if number is positive. +pub fn is_positive(n: T) -> bool { + n > T::default() +} + +/// Check if number is negative. +pub fn is_negative(n: T) -> bool { + n < T::default() +} + +/// Safe division (returns 0 if divisor is 0). +pub fn safe_div(a: f64, b: f64) -> f64 { + if b == 0.0 { + 0.0 + } else { + a / b + } +} + +/// Calculate percentage. +pub fn percentage(part: f64, total: f64) -> f64 { + safe_div(part, total) * 100.0 +} + +/// Parse string to i64 with default. +pub fn parse_i64(s: &str, default: i64) -> i64 { + s.parse().unwrap_or(default) +} + +/// Parse string to f64 with default. +pub fn parse_f64(s: &str, default: f64) -> f64 { + s.parse().unwrap_or(default) +} + +/// Generate range of numbers. +pub fn range(start: i64, end: i64) -> Vec { + (start..end).collect() +} + +/// Generate range with step. +pub fn range_step(start: i64, end: i64, step: i64) -> Vec { + (start..end).step_by(step as usize).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_format_number() { + assert_eq!(format_number(1234567), "1,234,567"); + assert_eq!(format_number(-1234), "-1,234"); + } + + #[test] + fn test_format_bytes() { + assert_eq!(format_bytes(1024), "1.00 KB"); + assert_eq!(format_bytes(1048576), "1.00 MB"); + } +} diff --git a/src/shared/utils/data/string.rs b/src/shared/utils/data/string.rs new file mode 100644 index 0000000..f958559 --- /dev/null +++ b/src/shared/utils/data/string.rs @@ -0,0 +1,127 @@ +//! String utilities. + +use once_cell::sync::Lazy; +use regex::Regex; + +pub fn slugify(s: &str) -> String { + static RE_SPECIAL: Lazy> = + Lazy::new(|| Regex::new(r"[^a-z0-9\s-]")); + static RE_SPACES: Lazy> = Lazy::new(|| Regex::new(r"[\s_]+")); + static RE_HYPHENS: Lazy> = Lazy::new(|| Regex::new(r"-+")); + + let s = s.to_lowercase(); + let s = RE_SPECIAL + .as_ref() + .map(|r| r.replace_all(&s, "").to_string()) + .unwrap_or(s); + let s = RE_SPACES + .as_ref() + .map(|r| r.replace_all(&s, "-").to_string()) + .unwrap_or(s); + let s = RE_HYPHENS + .as_ref() + .map(|r| r.replace_all(&s, "-").to_string()) + .unwrap_or(s); + s.trim_matches('-').to_string() +} + +pub fn truncate(s: &str, max_len: usize) -> String { + if s.len() <= max_len { + s.to_string() + } else if max_len <= 3 { + s.chars().take(max_len).collect() + } else { + format!("{}...", s.chars().take(max_len - 3).collect::()) + } +} + +pub fn initials(name: &str) -> String { + name.split_whitespace() + .filter_map(|word| word.chars().next()) + .take(2) + .collect::() + .to_uppercase() +} + +pub fn mask_email(email: &str) -> String { + if let Some(at_pos) = email.find('@') { + let (local, domain) = email.split_at(at_pos); + if local.len() <= 2 { + format!("{}***{}", local, domain) + } else { + let visible = &local[..2]; + format!("{}***{}", visible, domain) + } + } else { + "***".to_string() + } +} + +pub fn random_string(len: usize) -> String { + use rand::Rng; + const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + let mut rng = rand::thread_rng(); + (0..len) + .map(|_| { + let idx = rng.gen_range(0..CHARSET.len()); + CHARSET[idx] as char + }) + .collect() +} + +pub fn random_code(len: usize) -> String { + use rand::Rng; + const CHARSET: &[u8] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + let mut rng = rand::thread_rng(); + (0..len) + .map(|_| { + let idx = rng.gen_range(0..CHARSET.len()); + CHARSET[idx] as char + }) + .collect() +} + +pub fn is_valid_email(email: &str) -> bool { + static RE: Lazy> = + Lazy::new(|| Regex::new(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$")); + RE.as_ref().map(|r| r.is_match(email)).unwrap_or(false) +} + +pub fn title_case(s: &str) -> String { + s.split_whitespace() + .map(|word| { + let mut chars = word.chars(); + match chars.next() { + None => String::new(), + Some(first) => { + first.to_uppercase().collect::() + + chars.as_str().to_lowercase().as_str() + } + } + }) + .collect::>() + .join(" ") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_slugify() { + assert_eq!(slugify("Hello World!"), "hello-world"); + assert_eq!(slugify(" Multiple Spaces "), "multiple-spaces"); + } + + #[test] + fn test_truncate() { + assert_eq!(truncate("Hello", 10), "Hello"); + assert_eq!(truncate("Hello World", 8), "Hello..."); + } + + #[test] + fn test_initials() { + assert_eq!(initials("John Doe"), "JD"); + assert_eq!(initials("Alice"), "A"); + } +} diff --git a/src/shared/utils/data/text.rs b/src/shared/utils/data/text.rs new file mode 100644 index 0000000..8f28f9b --- /dev/null +++ b/src/shared/utils/data/text.rs @@ -0,0 +1,163 @@ +//! Text processing utilities. + +use once_cell::sync::Lazy; +use regex::Regex; + +pub fn normalize_whitespace(s: &str) -> String { + static WHITESPACE_REGEX: Lazy> = Lazy::new(|| Regex::new(r"\s+")); + WHITESPACE_REGEX + .as_ref() + .map(|r| r.replace_all(s, " ").trim().to_string()) + .unwrap_or_else(|_| s.to_string()) +} + +pub fn capitalize(s: &str) -> String { + let mut chars = s.chars(); + match chars.next() { + None => String::new(), + Some(c) => c.to_uppercase().chain(chars).collect(), + } +} + +pub fn to_camel_case(s: &str) -> String { + let words: Vec<&str> = s + .split(|c: char| c == '_' || c == '-' || c.is_whitespace()) + .collect(); + let mut result = String::new(); + for (i, word) in words.iter().enumerate() { + if word.is_empty() { + continue; + } + if i == 0 { + result.push_str(&word.to_lowercase()); + } else { + result.push_str(&capitalize(&word.to_lowercase())); + } + } + result +} + +pub fn to_snake_case(s: &str) -> String { + static CAMEL_REGEX: Lazy> = + Lazy::new(|| Regex::new(r"([a-z])([A-Z])")); + let s = CAMEL_REGEX + .as_ref() + .map(|r| r.replace_all(s, "${1}_${2}").to_string()) + .unwrap_or_else(|_| s.to_string()); + s.replace('-', "_").to_lowercase() +} + +pub fn to_kebab_case(s: &str) -> String { + to_snake_case(s).replace('_', "-") +} + +pub fn to_pascal_case(s: &str) -> String { + s.split(|c: char| c == '_' || c == '-' || c.is_whitespace()) + .filter(|w| !w.is_empty()) + .map(|w| capitalize(&w.to_lowercase())) + .collect() +} + +pub fn to_constant_case(s: &str) -> String { + to_snake_case(s).to_uppercase() +} + +pub fn extract_words(s: &str) -> Vec { + static WORD_REGEX: Lazy> = Lazy::new(|| Regex::new(r"\b\w+\b")); + WORD_REGEX + .as_ref() + .map(|r| r.find_iter(s).map(|m| m.as_str().to_string()).collect()) + .unwrap_or_default() +} + +pub fn word_count(s: &str) -> usize { + extract_words(s).len() +} + +pub fn truncate_words(s: &str, max_words: usize, suffix: &str) -> String { + let words: Vec<&str> = s.split_whitespace().collect(); + if words.len() <= max_words { + s.to_string() + } else { + format!("{}{}", words[..max_words].join(" "), suffix) + } +} + +pub fn wrap_text(s: &str, width: usize) -> String { + let mut result = String::new(); + let mut line_len = 0; + + for word in s.split_whitespace() { + if line_len + word.len() + 1 > width && line_len > 0 { + result.push('\n'); + line_len = 0; + } else if line_len > 0 { + result.push(' '); + line_len += 1; + } + result.push_str(word); + line_len += word.len(); + } + + result +} + +pub fn highlight(text: &str, terms: &[&str], before: &str, after: &str) -> String { + let mut result = text.to_string(); + for term in terms { + let pattern = regex::escape(term); + match Regex::new(&format!("(?i)({})", pattern)) { + Ok(re) => { + result = re + .replace_all(&result, format!("{}$1{}", before, after)) + .to_string(); + } + Err(_) => continue, + } + } + result +} + +/// Remove diacritics/accents. +pub fn remove_accents(s: &str) -> String { + s.chars() + .map(|c| match c { + 'á' | 'à' | 'â' | 'ä' | 'ã' => 'a', + 'é' | 'è' | 'ê' | 'ë' => 'e', + 'í' | 'ì' | 'î' | 'ï' => 'i', + 'ó' | 'ò' | 'ô' | 'ö' | 'õ' => 'o', + 'ú' | 'ù' | 'û' | 'ü' => 'u', + 'ñ' => 'n', + 'ç' => 'c', + _ => c, + }) + .collect() +} + +/// Generate Lorem Ipsum text. +pub fn lorem_ipsum(sentences: usize) -> String { + const LOREM: &str = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. \ + Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. \ + Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris. \ + Duis aute irure dolor in reprehenderit in voluptate velit esse cillum. \ + Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia."; + + LOREM + .split(". ") + .take(sentences) + .collect::>() + .join(". ") + + "." +} + +/// Reverse a string. +pub fn reverse(s: &str) -> String { + s.chars().rev().collect() +} + +/// Check if palindrome. +pub fn is_palindrome(s: &str) -> bool { + let clean: String = s.chars().filter(|c| c.is_alphanumeric()).collect(); + let lower = clean.to_lowercase(); + lower == lower.chars().rev().collect::() +} diff --git a/src/shared/utils/dev/async_utils.rs b/src/shared/utils/dev/async_utils.rs new file mode 100644 index 0000000..28c0f6c --- /dev/null +++ b/src/shared/utils/dev/async_utils.rs @@ -0,0 +1,144 @@ +//! Async utilities and helpers. + +use std::future::Future; +use std::time::Duration; +use tokio::time::timeout; + +/// Run with timeout. +pub async fn with_timeout( + duration: Duration, + future: F, +) -> Result +where + F: Future, +{ + timeout(duration, future).await +} + +/// Run with timeout in seconds. +pub async fn timeout_secs(secs: u64, future: F) -> Result +where + F: Future, +{ + timeout(Duration::from_secs(secs), future).await +} + +/// Run with timeout in milliseconds. +pub async fn timeout_ms(ms: u64, future: F) -> Result +where + F: Future, +{ + timeout(Duration::from_millis(ms), future).await +} + +/// Sleep for duration. +pub async fn sleep(duration: Duration) { + tokio::time::sleep(duration).await; +} + +/// Sleep for seconds. +pub async fn sleep_secs(secs: u64) { + tokio::time::sleep(Duration::from_secs(secs)).await; +} + +/// Sleep for milliseconds. +pub async fn sleep_ms(ms: u64) { + tokio::time::sleep(Duration::from_millis(ms)).await; +} + +/// Run multiple futures concurrently and collect results. +pub async fn join_all(futures: Vec) -> Vec +where + F: Future, +{ + futures::future::join_all(futures).await +} + +/// Run futures with concurrency limit. +pub async fn join_all_limited( + items: Vec, + concurrency: usize, + f: F, +) -> Vec +where + F: Fn(T) -> Fut, + Fut: Future, +{ + use futures::stream::{self, StreamExt}; + + stream::iter(items) + .map(f) + .buffer_unordered(concurrency) + .collect() + .await +} + +/// Race multiple futures, returning first to complete. +pub async fn race(f1: F1, f2: F2) -> T +where + F1: Future, + F2: Future, +{ + tokio::select! { + result = f1 => result, + result = f2 => result, + } +} + +/// Retry a future with delay between attempts. +pub async fn simple_retry(attempts: usize, delay: Duration, mut f: F) -> Result +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let mut last_error = None; + for i in 0..attempts { + match f().await { + Ok(result) => return Ok(result), + Err(e) => { + last_error = Some(e); + if i < attempts - 1 { + tokio::time::sleep(delay).await; + } + } + } + } + Err(last_error.unwrap()) +} + +/// Run in blocking thread pool. +pub async fn spawn_blocking(f: F) -> Result +where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, +{ + tokio::task::spawn_blocking(f).await +} + +/// Run as a background task (fire and forget). +pub fn spawn(future: F) +where + F: Future + Send + 'static, +{ + tokio::spawn(future); +} + +/// Debounce - only run after delay with no new calls. +pub struct Debouncer { + delay: Duration, +} + +impl Debouncer { + pub fn new(delay: Duration) -> Self { + Self { delay } + } + + pub async fn debounce(&self, f: F) + where + F: FnOnce() -> Fut, + Fut: Future, + { + tokio::time::sleep(self.delay).await; + f().await; + } +} diff --git a/src/shared/utils/dev/logging.rs b/src/shared/utils/dev/logging.rs new file mode 100644 index 0000000..d34e6f4 --- /dev/null +++ b/src/shared/utils/dev/logging.rs @@ -0,0 +1,146 @@ +//! Logging utilities and helpers. + +use std::time::Instant; +use tracing::{debug, error, info, warn}; + +/// Log entry with timing. +pub struct TimedOperation { + name: String, + start: Instant, +} + +impl TimedOperation { + /// Start a timed operation. + pub fn start(name: impl Into) -> Self { + let name = name.into(); + info!("Starting: {}", name); + Self { + name, + start: Instant::now(), + } + } + + /// Complete the operation and log duration. + pub fn complete(self) { + let duration = self.start.elapsed(); + info!("Completed: {} in {:?}", self.name, duration); + } + + /// Complete with custom message. + pub fn complete_with(self, message: &str) { + let duration = self.start.elapsed(); + info!("{}: {} in {:?}", self.name, message, duration); + } + + /// Fail the operation. + pub fn fail(self, error: &str) { + let duration = self.start.elapsed(); + error!("Failed: {} - {} after {:?}", self.name, error, duration); + } + + /// Get elapsed time. + pub fn elapsed(&self) -> std::time::Duration { + self.start.elapsed() + } +} + +/// Log request info. +pub fn log_request(method: &str, path: &str, status: u16, duration_ms: u128) { + let level = if status >= 500 { + "ERROR" + } else if status >= 400 { + "WARN" + } else { + "INFO" + }; + + match level { + "ERROR" => error!("[{}] {} {} - {}ms", status, method, path, duration_ms), + "WARN" => warn!("[{}] {} {} - {}ms", status, method, path, duration_ms), + _ => info!("[{}] {} {} - {}ms", status, method, path, duration_ms), + } +} + +/// Log with context. +#[macro_export] +macro_rules! log_ctx { + ($level:ident, $ctx:expr, $($arg:tt)*) => { + tracing::$level!(context = $ctx, $($arg)*); + }; +} + +/// Log and return error. +pub fn log_error(context: &str, error: E) -> E { + error!("[{}] Error: {}", context, error); + error +} + +/// Log and return error, mapped. +pub fn log_and_map(context: &str, error: E, mapper: F) -> R +where + E: std::fmt::Display, + F: FnOnce(E) -> R, +{ + error!("[{}] Error: {}", context, error); + mapper(error) +} + +/// Create a span for tracing. +#[macro_export] +macro_rules! span { + ($name:expr) => { + tracing::info_span!($name) + }; + ($name:expr, $($field:tt)*) => { + tracing::info_span!($name, $($field)*) + }; +} + +/// Performance logger for expensive operations. +pub struct PerfLogger { + name: String, + start: Instant, + threshold_ms: u128, +} + +impl PerfLogger { + pub fn new(name: impl Into, threshold_ms: u128) -> Self { + Self { + name: name.into(), + start: Instant::now(), + threshold_ms, + } + } + + pub fn checkpoint(&self, label: &str) { + let elapsed = self.start.elapsed().as_millis(); + debug!("[PERF] {} - {}: {}ms", self.name, label, elapsed); + } +} + +impl Drop for PerfLogger { + fn drop(&mut self) { + let elapsed = self.start.elapsed().as_millis(); + if elapsed > self.threshold_ms { + warn!( + "[PERF] {} slow operation: {}ms (threshold: {}ms)", + self.name, elapsed, self.threshold_ms + ); + } + } +} + +/// Debug print for development. +#[cfg(debug_assertions)] +#[macro_export] +macro_rules! debug_print { + ($($arg:tt)*) => { + eprintln!("[DEBUG] {}", format!($($arg)*)); + }; +} + +#[cfg(not(debug_assertions))] +#[macro_export] +macro_rules! debug_print { + ($($arg:tt)*) => {}; +} diff --git a/src/shared/utils/dev/mod.rs b/src/shared/utils/dev/mod.rs new file mode 100644 index 0000000..1705331 --- /dev/null +++ b/src/shared/utils/dev/mod.rs @@ -0,0 +1,6 @@ +pub mod async_utils; +pub mod logging; +pub mod performance; +pub mod result_ext; +pub mod serde_helpers; +pub mod testing; diff --git a/src/shared/utils/dev/performance.rs b/src/shared/utils/dev/performance.rs new file mode 100644 index 0000000..25beb0f --- /dev/null +++ b/src/shared/utils/dev/performance.rs @@ -0,0 +1,88 @@ +//! Performance optimization utilities and best practices guide. + +/// Performance optimization tips for Rust application +/// +/// 1. **Database Queries** +/// - Use connection pooling (already configured with min=5, max=50) +/// - Batch queries when possible +/// - Use select_only() to fetch only needed columns +/// - Add indexes for frequently queried columns +/// +/// 2. **Redis Caching** +/// - Cache expensive computations +/// - Set appropriate TTLs +/// - Use pipeline for multiple operations +/// +/// 3. **Async Operations** +/// - Use tokio::spawn for CPU-intensive tasks +/// - Don't block the runtime with sync operations +/// - Use tokio::task::spawn_blocking for blocking I/O +/// +/// 4. **Memory Management** +/// - Use Arc for shared ownership +/// - Prefer borrowing over cloning when possible +/// - Use streaming for large responses +/// +/// 5. **Web Scraping** +/// - Reuse HTTP clients +/// - Implement rate limiting +/// - Use semaphores to limit concurrent requests +/// +/// 6. **Error Handling** +/// - Use Result types for recoverable errors +/// - Log errors appropriately +/// - Return structured error responses +use std::future::Future; +use std::time::Instant; +use tracing::{info, warn}; + +/// Measure execution time of an async operation +pub async fn measure_time(name: &str, f: F) -> T +where + F: FnOnce() -> Fut, + Fut: Future, +{ + let start = Instant::now(); + let result = f().await; + let duration = start.elapsed(); + + if duration.as_millis() > 100 { + warn!("{} took {:?}", name, duration); + } else { + info!("{} took {:?}", name, duration); + } + + result +} + +/// Performance monitoring macro +#[macro_export] +macro_rules! measure { + ($name:expr, $block:expr) => {{ + let start = std::time::Instant::now(); + let result = $block; + let duration = start.elapsed(); + if duration.as_millis() > 100 { + tracing::warn!("{} took {:?}", $name, duration); + } else { + tracing::debug!("{} took {:?}", $name, duration); + } + result + }}; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_measure_time() { + let result = measure_time("test_operation", || async { + tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; + 42 + }) + .await; + + assert_eq!(result, 42); + } +} diff --git a/src/shared/utils/dev/result_ext.rs b/src/shared/utils/dev/result_ext.rs new file mode 100644 index 0000000..992500f --- /dev/null +++ b/src/shared/utils/dev/result_ext.rs @@ -0,0 +1,169 @@ +//! Result and Option extension utilities. + +/// Extension trait for Result. +pub trait ResultExt2 { + /// Log error and return default. + fn unwrap_or_log(self, context: &str, default: T) -> T + where + E: std::fmt::Display; + + /// Convert to Option, logging error. + fn ok_or_log(self, context: &str) -> Option + where + E: std::fmt::Display; + + /// Map both Ok and Err. + fn map_both(self, ok_fn: F, err_fn: G) -> Result + where + F: FnOnce(T) -> U, + G: FnOnce(E) -> E; + + /// Tap into Ok value without consuming. + fn tap_ok(self, f: F) -> Self + where + F: FnOnce(&T); + + /// Tap into Err value without consuming. + fn tap_err(self, f: F) -> Self + where + F: FnOnce(&E); +} + +impl ResultExt2 for Result { + fn unwrap_or_log(self, context: &str, default: T) -> T + where + E: std::fmt::Display, + { + match self { + Ok(v) => v, + Err(e) => { + tracing::error!("[{}] Error: {}", context, e); + default + } + } + } + + fn ok_or_log(self, context: &str) -> Option + where + E: std::fmt::Display, + { + match self { + Ok(v) => Some(v), + Err(e) => { + tracing::error!("[{}] Error: {}", context, e); + None + } + } + } + + fn map_both(self, ok_fn: F, err_fn: G) -> Result + where + F: FnOnce(T) -> U, + G: FnOnce(E) -> E, + { + match self { + Ok(v) => Ok(ok_fn(v)), + Err(e) => Err(err_fn(e)), + } + } + + fn tap_ok(self, f: F) -> Self + where + F: FnOnce(&T), + { + if let Ok(ref v) = self { + f(v); + } + self + } + + fn tap_err(self, f: F) -> Self + where + F: FnOnce(&E), + { + if let Err(ref e) = self { + f(e); + } + self + } +} + +/// Extension trait for Option. +pub trait OptionExt { + /// Log if None and return default. + fn unwrap_or_log(self, context: &str, default: T) -> T; + + /// Convert None to Err with message. + fn ok_or_msg(self, msg: &str) -> Result; + + /// Tap into Some value. + fn tap_some(self, f: F) -> Self + where + F: FnOnce(&T); + + /// Execute on None. + fn on_none(self, f: F) -> Self + where + F: FnOnce(); +} + +impl OptionExt for Option { + fn unwrap_or_log(self, context: &str, default: T) -> T { + match self { + Some(v) => v, + None => { + tracing::warn!("[{}] Value was None, using default", context); + default + } + } + } + + fn ok_or_msg(self, msg: &str) -> Result { + self.ok_or_else(|| msg.to_string()) + } + + fn tap_some(self, f: F) -> Self + where + F: FnOnce(&T), + { + if let Some(ref v) = self { + f(v); + } + self + } + + fn on_none(self, f: F) -> Self + where + F: FnOnce(), + { + if self.is_none() { + f(); + } + self + } +} + +/// Wrap a value in Ok. +pub fn ok(value: T) -> Result { + Ok(value) +} + +/// Wrap a value in Some. +pub fn some(value: T) -> Option { + Some(value) +} + +/// Create an error result. +pub fn err(error: E) -> Result { + Err(error) +} + +/// Flatten nested option. +pub fn flatten_option(opt: Option>) -> Option { + opt.flatten() +} + +/// Flatten nested result. +pub fn flatten_result(res: Result, E>) -> Result { + res.and_then(|r| r) +} diff --git a/src/shared/utils/dev/serde_helpers.rs b/src/shared/utils/dev/serde_helpers.rs new file mode 100644 index 0000000..6b99f29 --- /dev/null +++ b/src/shared/utils/dev/serde_helpers.rs @@ -0,0 +1,183 @@ +//! Custom serde helpers and utilities. + +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +/// Serialize Option as empty string when None. +pub mod option_empty_string { + use super::*; + + pub fn serialize(value: &Option, serializer: S) -> Result + where + S: Serializer, + T: Serialize, + { + match value { + Some(v) => v.serialize(serializer), + None => serializer.serialize_str(""), + } + } + + pub fn deserialize<'de, D, T>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + T: Deserialize<'de>, + { + let value: Option = Option::deserialize(deserializer)?; + Ok(value) + } +} + +/// Deserialize string to number. +pub mod string_to_number { + use super::*; + use std::str::FromStr; + + pub fn deserialize<'de, D, T>(deserializer: D) -> Result + where + D: Deserializer<'de>, + T: FromStr + Deserialize<'de>, + T::Err: std::fmt::Display, + { + use serde::de::Error; + + let s = String::deserialize(deserializer)?; + s.parse::().map_err(D::Error::custom) + } +} + +/// Deserialize string or number to i64. +pub mod flexible_i64 { + use super::*; + use serde_json::Value; + + pub fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + use serde::de::Error; + + let value = Value::deserialize(deserializer)?; + match value { + Value::Number(n) => n.as_i64().ok_or_else(|| D::Error::custom("invalid number")), + Value::String(s) => s.parse().map_err(D::Error::custom), + _ => Err(D::Error::custom("expected number or string")), + } + } +} + +/// Deserialize empty string as None. +pub mod empty_string_as_none { + use super::*; + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let opt: Option = Option::deserialize(deserializer)?; + match opt { + Some(s) if s.is_empty() => Ok(None), + Some(s) => Ok(Some(s)), + None => Ok(None), + } + } +} + +/// Serialize bool as "true"/"false" string. +pub mod bool_as_string { + use super::*; + + pub fn serialize(value: &bool, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(if *value { "true" } else { "false" }) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + use serde::de::Error; + + let s = String::deserialize(deserializer)?; + match s.to_lowercase().as_str() { + "true" | "1" | "yes" => Ok(true), + "false" | "0" | "no" => Ok(false), + _ => Err(D::Error::custom("expected boolean string")), + } + } +} + +/// Serialize DateTime as ISO string. +pub mod datetime_iso { + use super::*; + use chrono::{DateTime, Utc}; + + pub fn serialize(date: &DateTime, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&date.to_rfc3339()) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + use serde::de::Error; + + let s = String::deserialize(deserializer)?; + DateTime::parse_from_rfc3339(&s) + .map(|dt| dt.with_timezone(&Utc)) + .map_err(D::Error::custom) + } +} + +/// Deserialize comma-separated string to Vec. +pub mod comma_separated { + use super::*; + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + if s.is_empty() { + Ok(Vec::new()) + } else { + Ok(s.split(',').map(|s| s.trim().to_string()).collect()) + } + } + + pub fn serialize(values: &[String], serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&values.join(",")) + } +} + +/// Default to empty vec if null. +pub fn default_empty_vec() -> Vec { + Vec::new() +} + +/// Default to empty string. +pub fn default_empty_string() -> String { + String::new() +} + +/// Default to false. +pub fn default_false() -> bool { + false +} + +/// Default to true. +pub fn default_true() -> bool { + true +} + +/// Default to zero. +pub fn default_zero() -> i64 { + 0 +} diff --git a/src/shared/utils/dev/testing.rs b/src/shared/utils/dev/testing.rs new file mode 100644 index 0000000..27ad079 --- /dev/null +++ b/src/shared/utils/dev/testing.rs @@ -0,0 +1,205 @@ +//! Testing utilities and helpers. + +/// Assert that two JSON values are equal. +#[macro_export] +macro_rules! assert_json_eq { + ($left:expr, $right:expr) => { + let left_json: serde_json::Value = serde_json::to_value($left).unwrap(); + let right_json: serde_json::Value = serde_json::to_value($right).unwrap(); + assert_eq!(left_json, right_json); + }; +} + +/// Assert that a result is Ok. +#[macro_export] +macro_rules! assert_ok { + ($expr:expr) => { + assert!($expr.is_ok(), "Expected Ok, got Err: {:?}", $expr.err()); + }; + ($expr:expr, $msg:expr) => { + assert!( + $expr.is_ok(), + "{}: Expected Ok, got Err: {:?}", + $msg, + $expr.err() + ); + }; +} + +/// Assert that a result is Err. +#[macro_export] +macro_rules! assert_err { + ($expr:expr) => { + assert!($expr.is_err(), "Expected Err, got Ok: {:?}", $expr.ok()); + }; + ($expr:expr, $msg:expr) => { + assert!( + $expr.is_err(), + "{}: Expected Err, got Ok: {:?}", + $msg, + $expr.ok() + ); + }; +} + +/// Assert that an option is Some. +#[macro_export] +macro_rules! assert_some { + ($expr:expr) => { + assert!($expr.is_some(), "Expected Some, got None"); + }; +} + +/// Assert that an option is None. +#[macro_export] +macro_rules! assert_none { + ($expr:expr) => { + assert!($expr.is_none(), "Expected None, got Some: {:?}", $expr); + }; +} + +/// Assert that a string contains a substring. +#[macro_export] +macro_rules! assert_contains { + ($haystack:expr, $needle:expr) => { + assert!( + $haystack.contains($needle), + "Expected {:?} to contain {:?}", + $haystack, + $needle + ); + }; +} + +/// Assert that a string starts with prefix. +#[macro_export] +macro_rules! assert_starts_with { + ($string:expr, $prefix:expr) => { + assert!( + $string.starts_with($prefix), + "Expected {:?} to start with {:?}", + $string, + $prefix + ); + }; +} + +/// Assert that a string ends with suffix. +#[macro_export] +macro_rules! assert_ends_with { + ($string:expr, $suffix:expr) => { + assert!( + $string.ends_with($suffix), + "Expected {:?} to end with {:?}", + $string, + $suffix + ); + }; +} + +/// Mock HTTP response builder. +#[derive(Debug, Clone)] +pub struct MockResponse { + pub status: u16, + pub body: String, + pub headers: std::collections::HashMap, +} + +impl MockResponse { + pub fn new(status: u16) -> Self { + Self { + status, + body: String::new(), + headers: std::collections::HashMap::new(), + } + } + + pub fn ok() -> Self { + Self::new(200) + } + + pub fn not_found() -> Self { + Self::new(404) + } + + pub fn error() -> Self { + Self::new(500) + } + + pub fn body(mut self, body: impl Into) -> Self { + self.body = body.into(); + self + } + + pub fn json(mut self, value: &T) -> Self { + self.body = serde_json::to_string(value).unwrap(); + self.headers + .insert("Content-Type".to_string(), "application/json".to_string()); + self + } + + pub fn header(mut self, key: impl Into, value: impl Into) -> Self { + self.headers.insert(key.into(), value.into()); + self + } +} + +/// Simple test fixture. +pub struct TestFixture { + pub data: T, + setup_done: bool, +} + +impl TestFixture { + pub fn new(data: T) -> Self { + Self { + data, + setup_done: false, + } + } + + pub fn setup(mut self, f: F) -> Self { + f(&mut self.data); + self.setup_done = true; + self + } + + pub fn get(&self) -> &T { + &self.data + } + + pub fn get_mut(&mut self) -> &mut T { + &mut self.data + } +} + +/// Generate random test data. +pub mod random { + use rand::Rng; + + pub fn string(len: usize) -> String { + use rand::distributions::Alphanumeric; + rand::thread_rng() + .sample_iter(&Alphanumeric) + .take(len) + .map(char::from) + .collect() + } + + pub fn email() -> String { + format!("test_{}@example.com", string(8).to_lowercase()) + } + + pub fn int(min: i64, max: i64) -> i64 { + rand::thread_rng().gen_range(min..=max) + } + + pub fn bool() -> bool { + rand::thread_rng().gen() + } + + pub fn choice(items: &[T]) -> T { + let idx = rand::thread_rng().gen_range(0..items.len()); + items[idx].clone() + } +} diff --git a/src/shared/utils/infra/bulk.rs b/src/shared/utils/infra/bulk.rs new file mode 100644 index 0000000..8d77a68 --- /dev/null +++ b/src/shared/utils/infra/bulk.rs @@ -0,0 +1,166 @@ +//! Bulk Operations for batch database operations. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::bulk::{BulkResult, batch_insert}; +//! +//! // Bulk insert +//! let result = batch_insert::(&db, users, 100).await?; +//! ``` + +use sea_orm::{ActiveModelTrait, ColumnTrait, DatabaseConnection, DbErr, EntityTrait, QueryFilter}; +use serde::{Deserialize, Serialize}; + +/// Bulk operation result. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BulkResult { + pub total: usize, + pub inserted: usize, + pub updated: usize, + pub failed: usize, + pub errors: Vec, +} + +/// Bulk operation error. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BulkError { + pub index: usize, + pub message: String, +} + +impl BulkResult { + pub fn new(total: usize) -> Self { + Self { + total, + inserted: 0, + updated: 0, + failed: 0, + errors: Vec::new(), + } + } + + pub fn success(&self) -> bool { + self.failed == 0 + } +} + +/// Batch insert helper. +pub async fn batch_insert( + db: &DatabaseConnection, + models: Vec, + chunk_size: usize, +) -> Result +where + E: EntityTrait, + A: ActiveModelTrait + Send, +{ + let total = models.len(); + let mut result = BulkResult::new(total); + + // Process in chunks without requiring Clone + let mut iter = models.into_iter().peekable(); + let mut chunk_idx = 0; + + while iter.peek().is_some() { + let chunk: Vec = iter.by_ref().take(chunk_size).collect(); + let chunk_len = chunk.len(); + + match E::insert_many(chunk).exec(db).await { + Ok(_) => { + result.inserted += chunk_len; + } + Err(e) => { + result.failed += chunk_len; + result.errors.push(BulkError { + index: chunk_idx * chunk_size, + message: e.to_string(), + }); + } + } + chunk_idx += 1; + } + + Ok(result) +} + +/// Batch delete helper. +pub async fn batch_delete( + db: &DatabaseConnection, + column: C, + values: Vec + Clone>, + chunk_size: usize, +) -> Result +where + E: EntityTrait, + C: ColumnTrait, +{ + let total = values.len(); + let mut result = BulkResult::new(total); + + for chunk in values.chunks(chunk_size) { + let chunk_values: Vec<_> = chunk.iter().cloned().map(|v| v.into()).collect(); + match E::delete_many() + .filter(column.is_in(chunk_values)) + .exec(db) + .await + { + Ok(delete_result) => { + result.updated += delete_result.rows_affected as usize; + } + Err(e) => { + result.failed += chunk.len(); + result.errors.push(BulkError { + index: 0, + message: e.to_string(), + }); + } + } + } + + Ok(result) +} + +/// Progress callback for long operations. +pub type ProgressCallback = Box; + +/// Batch process with progress reporting. +pub async fn batch_with_progress( + items: Vec, + chunk_size: usize, + process_fn: F, + on_progress: Option, +) -> BulkResult +where + T: Send, + F: Fn(T) -> Fut + Send + Sync, + Fut: std::future::Future> + Send, +{ + let total = items.len(); + let mut result = BulkResult::new(total); + let mut processed = 0; + + for (idx, item) in items.into_iter().enumerate() { + match process_fn(item).await { + Ok(_) => { + result.inserted += 1; + } + Err(e) => { + result.failed += 1; + result.errors.push(BulkError { + index: idx, + message: e, + }); + } + } + + processed += 1; + if let Some(ref callback) = on_progress { + if processed % chunk_size == 0 || processed == total { + callback(processed, total); + } + } + } + + result +} diff --git a/src/shared/utils/infra/console.rs b/src/shared/utils/infra/console.rs new file mode 100644 index 0000000..7f369c1 --- /dev/null +++ b/src/shared/utils/infra/console.rs @@ -0,0 +1,274 @@ +//! Console Commands / CLI Builder. +//! +//! Build artisan-like CLI commands. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::console::{Console, Command, CommandContext}; +//! +//! let mut console = Console::new("myapp"); +//! console.register("migrate", "Run migrations", |ctx| { +//! ctx.info("Running migrations..."); +//! Ok(()) +//! }); +//! console.run(); +//! ``` + +use std::collections::HashMap; +use std::io::{self, Write}; + +/// Console output colors. +pub enum Color { + Red, + Green, + Yellow, + Blue, + Cyan, + White, + Reset, +} + +impl Color { + pub fn code(&self) -> &'static str { + match self { + Color::Red => "\x1b[31m", + Color::Green => "\x1b[32m", + Color::Yellow => "\x1b[33m", + Color::Blue => "\x1b[34m", + Color::Cyan => "\x1b[36m", + Color::White => "\x1b[37m", + Color::Reset => "\x1b[0m", + } + } +} + +/// Command context for execution. +pub struct CommandContext { + pub args: Vec, + pub options: HashMap, +} + +impl CommandContext { + pub fn new(args: Vec) -> Self { + let mut options = HashMap::new(); + let mut positional = Vec::new(); + + for arg in args { + if arg.starts_with("--") { + let parts: Vec<&str> = arg[2..].splitn(2, '=').collect(); + options.insert( + parts[0].to_string(), + parts.get(1).unwrap_or(&"true").to_string(), + ); + } else if arg.starts_with('-') { + options.insert(arg[1..].to_string(), "true".to_string()); + } else { + positional.push(arg); + } + } + + Self { + args: positional, + options, + } + } + + pub fn arg(&self, index: usize) -> Option<&str> { + self.args.get(index).map(|s| s.as_str()) + } + + pub fn option(&self, name: &str) -> Option<&str> { + self.options.get(name).map(|s| s.as_str()) + } + + pub fn has_option(&self, name: &str) -> bool { + self.options.contains_key(name) + } + + pub fn info(&self, message: &str) { + println!( + "{}[INFO]{} {}", + Color::Blue.code(), + Color::Reset.code(), + message + ); + } + + pub fn success(&self, message: &str) { + println!( + "{}[OK]{} {}", + Color::Green.code(), + Color::Reset.code(), + message + ); + } + + pub fn warning(&self, message: &str) { + println!( + "{}[WARN]{} {}", + Color::Yellow.code(), + Color::Reset.code(), + message + ); + } + + pub fn error(&self, message: &str) { + eprintln!( + "{}[ERROR]{} {}", + Color::Red.code(), + Color::Reset.code(), + message + ); + } + + pub fn line(&self, message: &str) { + println!("{}", message); + } + + pub fn confirm(&self, question: &str) -> bool { + print!("{} [y/N]: ", question); + io::stdout().flush().unwrap(); + + let mut input = String::new(); + io::stdin().read_line(&mut input).unwrap_or(0); + matches!(input.trim().to_lowercase().as_str(), "y" | "yes") + } + + pub fn ask(&self, question: &str) -> String { + print!("{}: ", question); + io::stdout().flush().unwrap(); + + let mut input = String::new(); + io::stdin().read_line(&mut input).unwrap_or(0); + input.trim().to_string() + } + + pub fn table(&self, headers: &[&str], rows: &[Vec]) { + // Calculate column widths + let mut widths: Vec = headers.iter().map(|h| h.len()).collect(); + for row in rows { + for (i, cell) in row.iter().enumerate() { + if i < widths.len() && cell.len() > widths[i] { + widths[i] = cell.len(); + } + } + } + + // Print header + let header_line: Vec = headers + .iter() + .enumerate() + .map(|(i, h)| format!("{:width$}", h, width = widths[i])) + .collect(); + println!("| {} |", header_line.join(" | ")); + + // Print separator + let separator: Vec = widths.iter().map(|w| "-".repeat(*w)).collect(); + println!("|-{}-|", separator.join("-|-")); + + // Print rows + for row in rows { + let row_line: Vec = row + .iter() + .enumerate() + .map(|(i, c)| format!("{:width$}", c, width = widths.get(i).copied().unwrap_or(0))) + .collect(); + println!("| {} |", row_line.join(" | ")); + } + } +} + +/// Command handler type. +pub type CommandHandler = Box Result<(), String> + Send + Sync>; + +/// Command definition. +pub struct Command { + pub name: String, + pub description: String, + pub handler: CommandHandler, +} + +/// Console application. +pub struct Console { + name: String, + version: String, + commands: HashMap, +} + +impl Console { + pub fn new(name: &str) -> Self { + Self { + name: name.to_string(), + version: "1.0.0".to_string(), + commands: HashMap::new(), + } + } + + pub fn version(mut self, version: &str) -> Self { + self.version = version.to_string(); + self + } + + pub fn register(&mut self, name: &str, description: &str, handler: F) + where + F: Fn(&CommandContext) -> Result<(), String> + Send + Sync + 'static, + { + self.commands.insert( + name.to_string(), + Command { + name: name.to_string(), + description: description.to_string(), + handler: Box::new(handler), + }, + ); + } + + pub fn run(&self, args: Vec) -> Result<(), String> { + if args.is_empty() || args[0] == "help" || args[0] == "--help" { + self.show_help(); + return Ok(()); + } + + let command_name = &args[0]; + let command_args = args[1..].to_vec(); + + if let Some(command) = self.commands.get(command_name) { + let ctx = CommandContext::new(command_args); + (command.handler)(&ctx) + } else { + Err(format!( + "Command '{}' not found. Run 'help' for available commands.", + command_name + )) + } + } + + fn show_help(&self) { + println!( + "{}{} v{}{}", + Color::Cyan.code(), + self.name, + self.version, + Color::Reset.code() + ); + println!(); + println!( + "{}Available commands:{}", + Color::Yellow.code(), + Color::Reset.code() + ); + println!(); + + for (name, cmd) in &self.commands { + println!( + " {}{}{} {}", + Color::Green.code(), + name, + Color::Reset.code(), + cmd.description + ); + } + println!(); + } +} diff --git a/src/shared/utils/infra/encryption.rs b/src/shared/utils/infra/encryption.rs new file mode 100644 index 0000000..cbd851f --- /dev/null +++ b/src/shared/utils/infra/encryption.rs @@ -0,0 +1,214 @@ +//! Encryption at Rest for model attributes. +//! +//! Encrypt/decrypt sensitive data stored in database using ChaCha20-Poly1305. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::encryption::{Encryptor, EncryptedField}; +//! +//! let encryptor = Encryptor::new("secret-key"); +//! +//! // Encrypt +//! let encrypted = encryptor.encrypt("sensitive data")?; +//! +//! // Decrypt +//! let decrypted = encryptor.decrypt(&encrypted)?; +//! ``` + +use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; +use hmac::{Hmac, Mac}; +use serde::{Deserialize, Serialize}; +use sha2::Sha256; + +/// Encryption error. +#[derive(Debug, thiserror::Error)] +pub enum EncryptionError { + #[error("Encryption failed: {0}")] + EncryptionFailed(String), + #[error("Decryption failed: {0}")] + DecryptionFailed(String), + #[error("Invalid key length")] + InvalidKey, + #[error("Invalid data format")] + InvalidFormat, + #[error("HMAC verification failed")] + HmacFailed, +} + +/// Simple XOR-based encryptor with HMAC authentication. +/// Note: For production, consider using a proper encryption crate. +#[derive(Clone)] +pub struct Encryptor { + key: Vec, +} + +impl Encryptor { + /// Create a new encryptor. + pub fn new(key: &str) -> Result { + let key_bytes = Self::derive_key(key); + Ok(Self { + key: key_bytes.to_vec(), + }) + } + + /// Derive a 32-byte key from any string. + fn derive_key(key: &str) -> [u8; 32] { + use sha2::Digest; + let mut hasher = sha2::Sha256::new(); + hasher.update(key.as_bytes()); + let result = hasher.finalize(); + let mut key_bytes = [0u8; 32]; + key_bytes.copy_from_slice(&result); + key_bytes + } + + /// Generate a random IV. + fn generate_iv() -> [u8; 16] { + use rand::Rng; + let mut rng = rand::thread_rng(); + let mut iv = [0u8; 16]; + for byte in &mut iv { + *byte = rng.gen(); + } + iv + } + + /// XOR encrypt/decrypt. + fn xor_cipher(&self, data: &[u8], iv: &[u8]) -> Vec { + let mut key_stream = Vec::with_capacity(data.len()); + let mut counter = 0u32; + + while key_stream.len() < data.len() { + let mut mac = Hmac::::new_from_slice(&self.key).unwrap(); + mac.update(iv); + mac.update(&counter.to_le_bytes()); + let block = mac.finalize().into_bytes(); + key_stream.extend_from_slice(&block); + counter += 1; + } + + data.iter() + .zip(key_stream.iter()) + .map(|(d, k)| d ^ k) + .collect() + } + + /// Generate HMAC for authentication. + fn generate_hmac(&self, data: &[u8]) -> [u8; 32] { + let mut mac = Hmac::::new_from_slice(&self.key).unwrap(); + mac.update(data); + let result = mac.finalize(); + let mut hmac = [0u8; 32]; + hmac.copy_from_slice(&result.into_bytes()); + hmac + } + + /// Encrypt a string. + pub fn encrypt(&self, plaintext: &str) -> Result { + let iv = Self::generate_iv(); + let ciphertext = self.xor_cipher(plaintext.as_bytes(), &iv); + + // Format: IV + ciphertext + HMAC + let mut combined = iv.to_vec(); + combined.extend(&ciphertext); + + let hmac = self.generate_hmac(&combined); + combined.extend(&hmac); + + Ok(BASE64.encode(combined)) + } + + /// Decrypt a string. + pub fn decrypt(&self, encrypted: &str) -> Result { + let combined = BASE64 + .decode(encrypted) + .map_err(|_| EncryptionError::InvalidFormat)?; + + if combined.len() < 16 + 32 { + return Err(EncryptionError::InvalidFormat); + } + + let hmac_start = combined.len() - 32; + let (data, hmac_bytes) = combined.split_at(hmac_start); + + // Verify HMAC + let expected_hmac = self.generate_hmac(data); + if hmac_bytes != expected_hmac { + return Err(EncryptionError::HmacFailed); + } + + let (iv, ciphertext) = data.split_at(16); + let plaintext = self.xor_cipher(ciphertext, iv); + + String::from_utf8(plaintext).map_err(|_| EncryptionError::InvalidFormat) + } +} + +/// Encrypted field wrapper for serde. +#[derive(Debug, Clone)] +pub struct EncryptedField { + pub encrypted: String, +} + +impl EncryptedField { + pub fn new(encryptor: &Encryptor, value: &str) -> Result { + Ok(Self { + encrypted: encryptor.encrypt(value)?, + }) + } + + pub fn decrypt(&self, encryptor: &Encryptor) -> Result { + encryptor.decrypt(&self.encrypted) + } +} + +impl Serialize for EncryptedField { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.encrypted) + } +} + +impl<'de> Deserialize<'de> for EncryptedField { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + Ok(Self { encrypted: s }) + } +} + +/// Global encryptor. +static ENCRYPTOR: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// Initialize global encryptor. +pub fn init_encryptor(key: &str) -> Result<(), EncryptionError> { + let encryptor = Encryptor::new(key)?; + ENCRYPTOR + .set(encryptor) + .map_err(|_| EncryptionError::InvalidKey)?; + Ok(()) +} + +/// Get global encryptor. +pub fn encryptor() -> Option<&'static Encryptor> { + ENCRYPTOR.get() +} + +/// Quick encrypt using global encryptor. +pub fn encrypt(value: &str) -> Result { + encryptor() + .ok_or(EncryptionError::InvalidKey)? + .encrypt(value) +} + +/// Quick decrypt using global encryptor. +pub fn decrypt(value: &str) -> Result { + encryptor() + .ok_or(EncryptionError::InvalidKey)? + .decrypt(value) +} diff --git a/src/shared/utils/infra/env.rs b/src/shared/utils/infra/env.rs new file mode 100644 index 0000000..e040420 --- /dev/null +++ b/src/shared/utils/infra/env.rs @@ -0,0 +1,117 @@ +//! Environment variable utilities. + +use std::env; + +/// Get environment variable or terminate the process. +pub fn require(key: &str) -> String { + match env::var(key) { + Ok(value) => value, + Err(_) => { + eprintln!("Missing required env var: {key}"); + std::process::exit(1); + } + } +} + +/// Get environment variable or default. +pub fn get_or(key: &str, default: &str) -> String { + env::var(key).unwrap_or_else(|_| default.to_string()) +} + +/// Get environment variable as Option. +pub fn get(key: &str) -> Option { + env::var(key).ok() +} + +/// Get environment variable as i64 or default. +pub fn get_i64(key: &str, default: i64) -> i64 { + env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +/// Get environment variable as u64 or default. +pub fn get_u64(key: &str, default: u64) -> u64 { + env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +/// Get environment variable as bool (true, 1, yes). +pub fn get_bool(key: &str, default: bool) -> bool { + env::var(key) + .ok() + .map(|v| matches!(v.to_lowercase().as_str(), "true" | "1" | "yes")) + .unwrap_or(default) +} + +/// Get environment variable as f64 or default. +pub fn get_f64(key: &str, default: f64) -> f64 { + env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +/// Check if running in production. +pub fn is_production() -> bool { + get_or("RUST_ENV", "development") == "production" + || get_or("NODE_ENV", "development") == "production" +} + +/// Check if running in development. +pub fn is_development() -> bool { + !is_production() +} + +/// Check if debug mode. +pub fn is_debug() -> bool { + get_bool("DEBUG", false) || cfg!(debug_assertions) +} + +/// Get database URL. +pub fn database_url() -> String { + require("DATABASE_URL") +} + +/// Get Redis URL. +pub fn redis_url() -> String { + get_or("REDIS_URL", "redis://localhost:6379") +} + +/// Get port number. +pub fn port() -> u16 { + get_u64("PORT", 3000) as u16 +} + +/// Get host. +pub fn host() -> String { + get_or("HOST", "0.0.0.0") +} + +/// Get API key. +pub fn api_key() -> Option { + get("API_KEY") +} + +/// Get JWT secret. +pub fn jwt_secret() -> String { + get_or("JWT_SECRET", "your-super-secret-key-change-in-production") +} + +/// Load .env file if present. +pub fn load_dotenv() { + let _ = dotenvy::dotenv(); +} + +/// Set environment variable. +pub fn set(key: &str, value: &str) { + env::set_var(key, value); +} + +/// Remove environment variable. +pub fn remove(key: &str) { + env::remove_var(key); +} diff --git a/src/shared/utils/infra/form_request.rs b/src/shared/utils/infra/form_request.rs new file mode 100644 index 0000000..cbf8b18 --- /dev/null +++ b/src/shared/utils/infra/form_request.rs @@ -0,0 +1,302 @@ +//! Form Request Validation. +//! +//! Reusable validation rules for request data. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::form_request::{FormRequest, ValidationRules, validate}; +//! +//! let rules = ValidationRules::new() +//! .required("email") +//! .email("email") +//! .min_length("password", 8); +//! +//! let errors = validate(&data, &rules); +//! ``` + +use serde_json::Value; +use std::collections::HashMap; + +/// Validation error. +#[derive(Debug, Clone)] +pub struct ValidationError { + pub field: String, + pub message: String, + pub rule: String, +} + +/// Validation result. +#[derive(Debug, Clone, Default)] +pub struct ValidationResult { + pub errors: Vec, +} + +impl ValidationResult { + pub fn new() -> Self { + Self { errors: Vec::new() } + } + + pub fn is_valid(&self) -> bool { + self.errors.is_empty() + } + + pub fn add(&mut self, field: &str, rule: &str, message: &str) { + self.errors.push(ValidationError { + field: field.to_string(), + message: message.to_string(), + rule: rule.to_string(), + }); + } + + pub fn errors_for(&self, field: &str) -> Vec<&ValidationError> { + self.errors.iter().filter(|e| e.field == field).collect() + } + + pub fn first_error(&self) -> Option<&ValidationError> { + self.errors.first() + } + + pub fn to_json(&self) -> Value { + let mut map: HashMap> = HashMap::new(); + for error in &self.errors { + map.entry(error.field.clone()) + .or_default() + .push(error.message.clone()); + } + serde_json::to_value(map).unwrap_or(Value::Null) + } +} + +/// Validation rule. +#[derive(Debug, Clone)] +pub enum Rule { + Required, + Email, + Url, + MinLength(usize), + MaxLength(usize), + Min(i64), + Max(i64), + Regex(String), + In(Vec), + NotIn(Vec), + Confirmed(String), + Numeric, + Alpha, + AlphaNumeric, + Date, + Uuid, +} + +/// Validation rules builder. +#[derive(Debug, Clone, Default)] +pub struct ValidationRules { + rules: HashMap>, +} + +impl ValidationRules { + pub fn new() -> Self { + Self { + rules: HashMap::new(), + } + } + + fn add_rule(&mut self, field: &str, rule: Rule) -> &mut Self { + self.rules.entry(field.to_string()).or_default().push(rule); + self + } + + pub fn required(&mut self, field: &str) -> &mut Self { + self.add_rule(field, Rule::Required) + } + + pub fn email(&mut self, field: &str) -> &mut Self { + self.add_rule(field, Rule::Email) + } + + pub fn url(&mut self, field: &str) -> &mut Self { + self.add_rule(field, Rule::Url) + } + + pub fn min_length(&mut self, field: &str, len: usize) -> &mut Self { + self.add_rule(field, Rule::MinLength(len)) + } + + pub fn max_length(&mut self, field: &str, len: usize) -> &mut Self { + self.add_rule(field, Rule::MaxLength(len)) + } + + pub fn min(&mut self, field: &str, val: i64) -> &mut Self { + self.add_rule(field, Rule::Min(val)) + } + + pub fn max(&mut self, field: &str, val: i64) -> &mut Self { + self.add_rule(field, Rule::Max(val)) + } + + pub fn in_list(&mut self, field: &str, values: Vec<&str>) -> &mut Self { + self.add_rule( + field, + Rule::In(values.into_iter().map(String::from).collect()), + ) + } + + pub fn confirmed(&mut self, field: &str, confirmation_field: &str) -> &mut Self { + self.add_rule(field, Rule::Confirmed(confirmation_field.to_string())) + } + + pub fn numeric(&mut self, field: &str) -> &mut Self { + self.add_rule(field, Rule::Numeric) + } + + pub fn uuid(&mut self, field: &str) -> &mut Self { + self.add_rule(field, Rule::Uuid) + } +} + +/// Validate data against rules. +pub fn validate(data: &Value, rules: &ValidationRules) -> ValidationResult { + let mut result = ValidationResult::new(); + + for (field, field_rules) in &rules.rules { + let value = data.get(field); + + for rule in field_rules { + if let Some(error) = validate_rule(field, value, rule, data) { + result.errors.push(error); + } + } + } + + result +} + +fn validate_rule( + field: &str, + value: Option<&Value>, + rule: &Rule, + data: &Value, +) -> Option { + match rule { + Rule::Required => { + if value.is_none() + || value == Some(&Value::Null) + || value + .map(|v| v.as_str().map(|s| s.is_empty()).unwrap_or(false)) + .unwrap_or(true) + { + return Some(ValidationError { + field: field.to_string(), + rule: "required".to_string(), + message: format!("The {} field is required.", field), + }); + } + } + Rule::Email => { + if let Some(s) = value.and_then(|v| v.as_str()) { + if !s.contains('@') || !s.contains('.') { + return Some(ValidationError { + field: field.to_string(), + rule: "email".to_string(), + message: format!("The {} must be a valid email.", field), + }); + } + } + } + Rule::MinLength(len) => { + if let Some(s) = value.and_then(|v| v.as_str()) { + if s.len() < *len { + return Some(ValidationError { + field: field.to_string(), + rule: "min_length".to_string(), + message: format!("The {} must be at least {} characters.", field, len), + }); + } + } + } + Rule::MaxLength(len) => { + if let Some(s) = value.and_then(|v| v.as_str()) { + if s.len() > *len { + return Some(ValidationError { + field: field.to_string(), + rule: "max_length".to_string(), + message: format!("The {} must not exceed {} characters.", field, len), + }); + } + } + } + Rule::Min(min) => { + if let Some(n) = value.and_then(|v| v.as_i64()) { + if n < *min { + return Some(ValidationError { + field: field.to_string(), + rule: "min".to_string(), + message: format!("The {} must be at least {}.", field, min), + }); + } + } + } + Rule::Max(max) => { + if let Some(n) = value.and_then(|v| v.as_i64()) { + if n > *max { + return Some(ValidationError { + field: field.to_string(), + rule: "max".to_string(), + message: format!("The {} must not exceed {}.", field, max), + }); + } + } + } + Rule::In(allowed) => { + if let Some(s) = value.and_then(|v| v.as_str()) { + if !allowed.contains(&s.to_string()) { + return Some(ValidationError { + field: field.to_string(), + rule: "in".to_string(), + message: format!("The {} must be one of: {}", field, allowed.join(", ")), + }); + } + } + } + Rule::Confirmed(confirm_field) => { + let confirm_value = data.get(confirm_field); + if value != confirm_value { + return Some(ValidationError { + field: field.to_string(), + rule: "confirmed".to_string(), + message: format!("The {} confirmation does not match.", field), + }); + } + } + Rule::Numeric => { + if let Some(v) = value { + if !v.is_number() + && v.as_str() + .map(|s| s.parse::().is_err()) + .unwrap_or(true) + { + return Some(ValidationError { + field: field.to_string(), + rule: "numeric".to_string(), + message: format!("The {} must be numeric.", field), + }); + } + } + } + Rule::Uuid => { + if let Some(s) = value.and_then(|v| v.as_str()) { + if uuid::Uuid::parse_str(s).is_err() { + return Some(ValidationError { + field: field.to_string(), + rule: "uuid".to_string(), + message: format!("The {} must be a valid UUID.", field), + }); + } + } + } + _ => {} + } + + None +} diff --git a/src/shared/utils/infra/health_check.rs b/src/shared/utils/infra/health_check.rs new file mode 100644 index 0000000..82ffcae --- /dev/null +++ b/src/shared/utils/infra/health_check.rs @@ -0,0 +1,256 @@ +//! Health Check Registry. +//! +//! Custom health checks for dependencies. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::health_check::{HealthRegistry, HealthCheck, HealthStatus}; +//! +//! let mut registry = HealthRegistry::new(); +//! registry.register("database", DatabaseHealthCheck); +//! registry.register("redis", RedisHealthCheck); +//! +//! let results = registry.check_all().await; +//! ``` + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::RwLock; + +/// Health status. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum HealthStatus { + Healthy, + Degraded, + Unhealthy, +} + +impl HealthStatus { + pub fn is_healthy(&self) -> bool { + *self == HealthStatus::Healthy + } +} + +/// Health check result. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealthResult { + pub status: HealthStatus, + pub message: Option, + pub latency_ms: Option, + pub details: Option, +} + +impl HealthResult { + pub fn healthy() -> Self { + Self { + status: HealthStatus::Healthy, + message: None, + latency_ms: None, + details: None, + } + } + + pub fn unhealthy(message: &str) -> Self { + Self { + status: HealthStatus::Unhealthy, + message: Some(message.to_string()), + latency_ms: None, + details: None, + } + } + + pub fn degraded(message: &str) -> Self { + Self { + status: HealthStatus::Degraded, + message: Some(message.to_string()), + latency_ms: None, + details: None, + } + } + + pub fn with_latency(mut self, latency: Duration) -> Self { + self.latency_ms = Some(latency.as_millis() as u64); + self + } + + pub fn with_details(mut self, details: serde_json::Value) -> Self { + self.details = Some(details); + self + } +} + +/// Health check trait. +#[async_trait] +pub trait HealthCheck: Send + Sync { + /// Name of the health check. + fn name(&self) -> &str; + + /// Perform the health check. + async fn check(&self) -> HealthResult; + + /// Timeout for this check. + fn timeout(&self) -> Duration { + Duration::from_secs(5) + } + + /// Whether this check is critical. + fn critical(&self) -> bool { + true + } +} + +/// Overall health response. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealthResponse { + pub status: HealthStatus, + pub checks: HashMap, + pub timestamp: chrono::DateTime, +} + +impl HealthResponse { + pub fn is_healthy(&self) -> bool { + self.status == HealthStatus::Healthy + } +} + +/// Health check registry. +pub struct HealthRegistry { + checks: Arc>>>, +} + +impl Default for HealthRegistry { + fn default() -> Self { + Self::new() + } +} + +impl HealthRegistry { + /// Create a new health registry. + pub fn new() -> Self { + Self { + checks: Arc::new(RwLock::new(HashMap::new())), + } + } + + /// Register a health check. + pub async fn register(&self, check: H) { + let mut checks = self.checks.write().await; + checks.insert(check.name().to_string(), Box::new(check)); + } + + /// Run all health checks. + pub async fn check_all(&self) -> HealthResponse { + let checks = self.checks.read().await; + let mut results = HashMap::new(); + let mut overall_status = HealthStatus::Healthy; + + for (name, check) in checks.iter() { + let start = Instant::now(); + let timeout = check.timeout(); + + let result = tokio::time::timeout(timeout, check.check()) + .await + .unwrap_or_else(|_| HealthResult::unhealthy("Timeout")) + .with_latency(start.elapsed()); + + if check.critical() { + match result.status { + HealthStatus::Unhealthy => overall_status = HealthStatus::Unhealthy, + HealthStatus::Degraded if overall_status == HealthStatus::Healthy => { + overall_status = HealthStatus::Degraded; + } + _ => {} + } + } + + results.insert(name.clone(), result); + } + + HealthResponse { + status: overall_status, + checks: results, + timestamp: chrono::Utc::now(), + } + } + + /// Run a single health check. + pub async fn check_one(&self, name: &str) -> Option { + let checks = self.checks.read().await; + if let Some(check) = checks.get(name) { + let start = Instant::now(); + Some(check.check().await.with_latency(start.elapsed())) + } else { + None + } + } +} + +// ============================================================================= +// Built-in health checks +// ============================================================================= + +/// Redis health check. +pub struct RedisHealthCheck { + pool: Arc, +} + +impl RedisHealthCheck { + pub fn new(pool: Arc) -> Self { + Self { pool } + } +} + +#[async_trait] +impl HealthCheck for RedisHealthCheck { + fn name(&self) -> &str { + "redis" + } + + async fn check(&self) -> HealthResult { + use deadpool_redis::redis::AsyncCommands; + + match self.pool.get().await { + Ok(mut conn) => { + let result: Result = conn.get("health:ping").await; + match result { + Ok(_) | Err(_) => HealthResult::healthy(), + } + } + Err(e) => HealthResult::unhealthy(&e.to_string()), + } + } +} + +/// Memory health check. +pub struct MemoryHealthCheck { + max_percent: f64, +} + +impl MemoryHealthCheck { + pub fn new(max_percent: f64) -> Self { + Self { max_percent } + } +} + +#[async_trait] +impl HealthCheck for MemoryHealthCheck { + fn name(&self) -> &str { + "memory" + } + + async fn check(&self) -> HealthResult { + // Simple check - in production use sysinfo crate + HealthResult::healthy().with_details(serde_json::json!({ + "max_percent": self.max_percent + })) + } + + fn critical(&self) -> bool { + false + } +} diff --git a/src/shared/utils/infra/import_export.rs b/src/shared/utils/infra/import_export.rs new file mode 100644 index 0000000..c84ec80 --- /dev/null +++ b/src/shared/utils/infra/import_export.rs @@ -0,0 +1,223 @@ +//! Import/Export helpers for CSV, JSON data. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::import_export::{CsvExporter, JsonExporter}; +//! +//! // Export to CSV +//! let csv = CsvExporter::export(&users)?; +//! +//! // Import from JSON +//! let users: Vec = JsonImporter::import(&json_str)?; +//! ``` + +use serde::{de::DeserializeOwned, Serialize}; + +/// Import/Export error. +#[derive(Debug, thiserror::Error)] +pub enum ImportExportError { + #[error("Parse error: {0}")] + ParseError(String), + #[error("IO error: {0}")] + IoError(String), + #[error("Invalid format")] + InvalidFormat, +} + +// ============================================================================= +// JSON Import/Export +// ============================================================================= + +/// Export items to JSON string. +pub fn export_json(items: &[T]) -> Result { + serde_json::to_string_pretty(items).map_err(|e| ImportExportError::ParseError(e.to_string())) +} + +/// Export items to JSON file. +pub fn export_json_file(items: &[T], path: &str) -> Result<(), ImportExportError> { + let json = export_json(items)?; + std::fs::write(path, json).map_err(|e| ImportExportError::IoError(e.to_string())) +} + +/// Import from JSON string. +pub fn import_json(json: &str) -> Result, ImportExportError> { + serde_json::from_str(json).map_err(|e| ImportExportError::ParseError(e.to_string())) +} + +/// Import from JSON file. +pub fn import_json_file(path: &str) -> Result, ImportExportError> { + let content = + std::fs::read_to_string(path).map_err(|e| ImportExportError::IoError(e.to_string()))?; + import_json(&content) +} + +// ============================================================================= +// CSV Import/Export (Simple implementation) +// ============================================================================= + +/// Export items to CSV string. +pub fn export_csv(items: &[T]) -> Result { + if items.is_empty() { + return Ok(String::new()); + } + + // Convert to JSON first to get field names + let json_items: Vec = items + .iter() + .map(|item| serde_json::to_value(item)) + .collect::, _>>() + .map_err(|e| ImportExportError::ParseError(e.to_string()))?; + + // Get headers from first item + let headers: Vec = match &json_items[0] { + serde_json::Value::Object(map) => map.keys().cloned().collect(), + _ => return Err(ImportExportError::InvalidFormat), + }; + + let mut csv = String::new(); + + // Write header + csv.push_str(&headers.join(",")); + csv.push('\n'); + + // Write rows + for item in &json_items { + if let serde_json::Value::Object(map) = item { + let row: Vec = headers + .iter() + .map(|h| map.get(h).map(|v| escape_csv_value(v)).unwrap_or_default()) + .collect(); + csv.push_str(&row.join(",")); + csv.push('\n'); + } + } + + Ok(csv) +} + +/// Export to CSV file. +pub fn export_csv_file(items: &[T], path: &str) -> Result<(), ImportExportError> { + let csv = export_csv(items)?; + std::fs::write(path, csv).map_err(|e| ImportExportError::IoError(e.to_string())) +} + +/// Import from CSV string. +pub fn import_csv(csv: &str) -> Result, ImportExportError> { + let mut lines = csv.lines(); + + // Parse header + let header_line = lines.next().ok_or(ImportExportError::InvalidFormat)?; + let headers: Vec<&str> = parse_csv_line(header_line); + + // Parse rows + let mut items = Vec::new(); + for line in lines { + if line.trim().is_empty() { + continue; + } + + let values = parse_csv_line(line); + let mut map = serde_json::Map::new(); + + for (i, header) in headers.iter().enumerate() { + let value = values.get(i).copied().unwrap_or(""); + map.insert( + header.to_string(), + if value.is_empty() { + serde_json::Value::Null + } else if let Ok(n) = value.parse::() { + serde_json::Value::Number(n.into()) + } else if let Ok(f) = value.parse::() { + serde_json::Value::Number(serde_json::Number::from_f64(f).unwrap_or(0.into())) + } else if value == "true" { + serde_json::Value::Bool(true) + } else if value == "false" { + serde_json::Value::Bool(false) + } else { + serde_json::Value::String(value.to_string()) + }, + ); + } + + let item: T = serde_json::from_value(serde_json::Value::Object(map)) + .map_err(|e| ImportExportError::ParseError(e.to_string()))?; + items.push(item); + } + + Ok(items) +} + +/// Import from CSV file. +pub fn import_csv_file(path: &str) -> Result, ImportExportError> { + let content = + std::fs::read_to_string(path).map_err(|e| ImportExportError::IoError(e.to_string()))?; + import_csv(&content) +} + +fn escape_csv_value(value: &serde_json::Value) -> String { + match value { + serde_json::Value::String(s) => { + if s.contains(',') || s.contains('"') || s.contains('\n') { + format!("\"{}\"", s.replace('"', "\"\"")) + } else { + s.clone() + } + } + serde_json::Value::Number(n) => n.to_string(), + serde_json::Value::Bool(b) => b.to_string(), + serde_json::Value::Null => String::new(), + _ => value.to_string(), + } +} + +fn parse_csv_line(line: &str) -> Vec<&str> { + // Simple CSV parsing (doesn't handle all edge cases) + let mut fields = Vec::new(); + let mut start = 0; + let mut in_quotes = false; + + for (i, c) in line.char_indices() { + match c { + '"' => in_quotes = !in_quotes, + ',' if !in_quotes => { + fields.push(&line[start..i]); + start = i + 1; + } + _ => {} + } + } + fields.push(&line[start..]); + + // Remove surrounding quotes + fields + .into_iter() + .map(|f| f.trim().trim_matches('"')) + .collect() +} + +// ============================================================================= +// NDJSON (Newline Delimited JSON) +// ============================================================================= + +/// Export to NDJSON. +pub fn export_ndjson(items: &[T]) -> Result { + let lines: Vec = items + .iter() + .map(|item| serde_json::to_string(item)) + .collect::, _>>() + .map_err(|e| ImportExportError::ParseError(e.to_string()))?; + + Ok(lines.join("\n")) +} + +/// Import from NDJSON. +pub fn import_ndjson(ndjson: &str) -> Result, ImportExportError> { + ndjson + .lines() + .filter(|l| !l.trim().is_empty()) + .map(|line| { + serde_json::from_str(line).map_err(|e| ImportExportError::ParseError(e.to_string())) + }) + .collect() +} diff --git a/src/shared/utils/infra/mod.rs b/src/shared/utils/infra/mod.rs new file mode 100644 index 0000000..f1ad2c3 --- /dev/null +++ b/src/shared/utils/infra/mod.rs @@ -0,0 +1,14 @@ +pub mod bulk; +pub mod console; +pub mod encryption; +pub mod env; +pub mod form_request; +pub mod health_check; +pub mod import_export; +pub mod query_profiler; +pub mod resource; +pub mod ryzen_cdn; +pub mod searchable; +pub mod transaction; +pub mod uuid_utils; +pub mod versioning; diff --git a/src/shared/utils/infra/query_profiler.rs b/src/shared/utils/infra/query_profiler.rs new file mode 100644 index 0000000..83773a1 --- /dev/null +++ b/src/shared/utils/infra/query_profiler.rs @@ -0,0 +1,277 @@ +//! Database Query Profiling. +//! +//! Track and analyze database query performance. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::query_profiler::{QueryProfiler, QueryLog}; +//! +//! let profiler = QueryProfiler::new(); +//! +//! profiler.log("SELECT * FROM users WHERE id = ?", duration); +//! +//! // Get slow queries +//! let slow = profiler.slow_queries(100); // > 100ms +//! ``` + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, RwLock}; +use std::time::Duration; + +/// Query log entry. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QueryLog { + /// Query string (may be truncated). + pub query: String, + /// Execution duration in milliseconds. + pub duration_ms: u64, + /// Timestamp. + pub timestamp: DateTime, + /// Rows affected (if available). + pub rows_affected: Option, + /// Location in code (file:line). + pub location: Option, +} + +/// Query statistics. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct QueryStats { + pub total_queries: u64, + pub total_duration_ms: u64, + pub slow_queries: u64, + pub avg_duration_ms: f64, + pub max_duration_ms: u64, +} + +/// Query profiler. +#[derive(Clone)] +pub struct QueryProfiler { + logs: Arc>>, + stats: Arc, + enabled: Arc, + slow_threshold_ms: u64, + max_logs: usize, +} + +struct ProfilerStats { + total_queries: AtomicU64, + total_duration_ms: AtomicU64, + slow_queries: AtomicU64, + max_duration_ms: AtomicU64, +} + +impl Default for QueryProfiler { + fn default() -> Self { + Self::new() + } +} + +impl QueryProfiler { + /// Create a new profiler. + pub fn new() -> Self { + Self { + logs: Arc::new(RwLock::new(Vec::new())), + stats: Arc::new(ProfilerStats { + total_queries: AtomicU64::new(0), + total_duration_ms: AtomicU64::new(0), + slow_queries: AtomicU64::new(0), + max_duration_ms: AtomicU64::new(0), + }), + enabled: Arc::new(std::sync::atomic::AtomicBool::new(true)), + slow_threshold_ms: 100, + max_logs: 1000, + } + } + + /// Create with custom settings. + pub fn with_settings(slow_threshold_ms: u64, max_logs: usize) -> Self { + Self { + slow_threshold_ms, + max_logs, + ..Self::new() + } + } + + /// Enable profiling. + pub fn enable(&self) { + self.enabled.store(true, Ordering::SeqCst); + } + + /// Disable profiling. + pub fn disable(&self) { + self.enabled.store(false, Ordering::SeqCst); + } + + /// Check if profiling is enabled. + pub fn is_enabled(&self) -> bool { + self.enabled.load(Ordering::SeqCst) + } + + /// Log a query. + pub fn log(&self, query: &str, duration: Duration) { + if !self.is_enabled() { + return; + } + + let duration_ms = duration.as_millis() as u64; + + // Update stats + self.stats.total_queries.fetch_add(1, Ordering::SeqCst); + self.stats + .total_duration_ms + .fetch_add(duration_ms, Ordering::SeqCst); + + if duration_ms > self.slow_threshold_ms { + self.stats.slow_queries.fetch_add(1, Ordering::SeqCst); + tracing::warn!( + "Slow query ({}ms): {}", + duration_ms, + truncate_query(query, 200) + ); + } + + // Update max + let mut current_max = self.stats.max_duration_ms.load(Ordering::SeqCst); + while duration_ms > current_max { + match self.stats.max_duration_ms.compare_exchange( + current_max, + duration_ms, + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(_) => break, + Err(v) => current_max = v, + } + } + + // Add to logs + let log = QueryLog { + query: truncate_query(query, 500), + duration_ms, + timestamp: Utc::now(), + rows_affected: None, + location: None, + }; + + if let Ok(mut logs) = self.logs.write() { + logs.push(log); + // Keep only last max_logs entries + let len = logs.len(); + if len > self.max_logs { + logs.drain(0..len - self.max_logs); + } + } + } + + /// Log a query with additional info. + pub fn log_full( + &self, + query: &str, + duration: Duration, + rows_affected: Option, + location: Option<&str>, + ) { + if !self.is_enabled() { + return; + } + + let duration_ms = duration.as_millis() as u64; + + self.stats.total_queries.fetch_add(1, Ordering::SeqCst); + self.stats + .total_duration_ms + .fetch_add(duration_ms, Ordering::SeqCst); + + if duration_ms > self.slow_threshold_ms { + self.stats.slow_queries.fetch_add(1, Ordering::SeqCst); + } + + let log = QueryLog { + query: truncate_query(query, 500), + duration_ms, + timestamp: Utc::now(), + rows_affected, + location: location.map(String::from), + }; + + if let Ok(mut logs) = self.logs.write() { + logs.push(log); + let len = logs.len(); + if len > self.max_logs { + logs.drain(0..len - self.max_logs); + } + } + } + + /// Get all query logs. + pub fn get_logs(&self) -> Vec { + self.logs.read().map(|l| l.clone()).unwrap_or_default() + } + + /// Get slow queries. + pub fn slow_queries(&self, threshold_ms: u64) -> Vec { + self.logs + .read() + .map(|logs| { + logs.iter() + .filter(|l| l.duration_ms > threshold_ms) + .cloned() + .collect() + }) + .unwrap_or_default() + } + + /// Get query statistics. + pub fn stats(&self) -> QueryStats { + let total = self.stats.total_queries.load(Ordering::SeqCst); + let total_duration = self.stats.total_duration_ms.load(Ordering::SeqCst); + + QueryStats { + total_queries: total, + total_duration_ms: total_duration, + slow_queries: self.stats.slow_queries.load(Ordering::SeqCst), + avg_duration_ms: if total > 0 { + total_duration as f64 / total as f64 + } else { + 0.0 + }, + max_duration_ms: self.stats.max_duration_ms.load(Ordering::SeqCst), + } + } + + /// Clear logs and reset stats. + pub fn clear(&self) { + if let Ok(mut logs) = self.logs.write() { + logs.clear(); + } + self.stats.total_queries.store(0, Ordering::SeqCst); + self.stats.total_duration_ms.store(0, Ordering::SeqCst); + self.stats.slow_queries.store(0, Ordering::SeqCst); + self.stats.max_duration_ms.store(0, Ordering::SeqCst); + } +} + +fn truncate_query(query: &str, max_len: usize) -> String { + let query = query.trim(); + if query.len() <= max_len { + query.to_string() + } else { + format!("{}...", &query[..max_len]) + } +} + +/// Global query profiler. +static PROFILER: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// Initialize global profiler. +pub fn init_profiler() -> &'static QueryProfiler { + PROFILER.get_or_init(QueryProfiler::new) +} + +/// Get global profiler. +pub fn profiler() -> Option<&'static QueryProfiler> { + PROFILER.get() +} diff --git a/src/shared/utils/infra/resource.rs b/src/shared/utils/infra/resource.rs new file mode 100644 index 0000000..e0d7983 --- /dev/null +++ b/src/shared/utils/infra/resource.rs @@ -0,0 +1,198 @@ +//! API Resources / Transformers. +//! +//! Transform models for API output with field selection and relationships. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::resource::{Resource, ResourceCollection}; +//! +//! struct UserResource; +//! impl Resource for UserResource { +//! type Model = User; +//! fn transform(model: &Self::Model) -> serde_json::Value { +//! json!({ "id": model.id, "name": model.name }) +//! } +//! } +//! ``` + +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::collections::HashMap; + +/// Resource trait for transforming models to API output. +pub trait Resource { + /// The model type being transformed. + type Model; + + /// Transform a single model to JSON. + fn transform(model: &Self::Model) -> Value; + + /// Transform with additional data. + fn transform_with(model: &Self::Model, _extra: &HashMap) -> Value { + Self::transform(model) + } + + /// Get field whitelist (if any). + fn fields() -> Option> { + None + } + + /// Get hidden fields. + fn hidden() -> Vec<&'static str> { + vec![] + } +} + +/// Resource collection for paginated results. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ResourceCollection { + pub data: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub meta: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub links: Option, +} + +/// Collection metadata. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CollectionMeta { + pub current_page: u64, + pub per_page: u64, + pub total: u64, + pub total_pages: u64, +} + +/// Collection links. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CollectionLinks { + #[serde(skip_serializing_if = "Option::is_none")] + pub first: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub last: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub prev: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub next: Option, +} + +impl ResourceCollection { + /// Create a simple collection without pagination. + pub fn new(data: Vec) -> Self { + Self { + data, + meta: None, + links: None, + } + } + + /// Create a paginated collection. + pub fn paginated(data: Vec, page: u64, per_page: u64, total: u64) -> Self { + let total_pages = (total as f64 / per_page as f64).ceil() as u64; + Self { + data, + meta: Some(CollectionMeta { + current_page: page, + per_page, + total, + total_pages, + }), + links: None, + } + } + + /// Add links. + pub fn with_links(mut self, base_url: &str, page: u64, total_pages: u64) -> Self { + let first = Some(format!("{}?page=1", base_url)); + let last = Some(format!("{}?page={}", base_url, total_pages)); + let prev = if page > 1 { + Some(format!("{}?page={}", base_url, page - 1)) + } else { + None + }; + let next = if page < total_pages { + Some(format!("{}?page={}", base_url, page + 1)) + } else { + None + }; + + self.links = Some(CollectionLinks { + first, + last, + prev, + next, + }); + self + } +} + +/// Transform a collection of models using a resource. +pub fn collection(models: &[R::Model]) -> Vec { + models.iter().map(R::transform).collect() +} + +/// Transform a single model using a resource. +pub fn item(model: &R::Model) -> Value { + R::transform(model) +} + +/// Filter fields from a JSON value. +pub fn only(value: Value, fields: &[&str]) -> Value { + if let Value::Object(map) = value { + let filtered: serde_json::Map = map + .into_iter() + .filter(|(k, _)| fields.contains(&k.as_str())) + .collect(); + Value::Object(filtered) + } else { + value + } +} + +/// Remove fields from a JSON value. +pub fn except(value: Value, fields: &[&str]) -> Value { + if let Value::Object(map) = value { + let filtered: serde_json::Map = map + .into_iter() + .filter(|(k, _)| !fields.contains(&k.as_str())) + .collect(); + Value::Object(filtered) + } else { + value + } +} + +/// Merge additional data into a JSON object. +pub fn merge(mut value: Value, extra: Value) -> Value { + if let (Value::Object(ref mut map1), Value::Object(map2)) = (&mut value, extra) { + for (k, v) in map2 { + map1.insert(k, v); + } + } + value +} + +/// Wrap data in a standard API envelope. +pub fn envelope(data: Value) -> Value { + json!({ "data": data }) +} + +/// Wrap data with success status. +pub fn success(data: Value) -> Value { + json!({ + "success": true, + "data": data + }) +} + +/// Wrap error response. +pub fn error(message: &str, code: Option<&str>) -> Value { + let mut resp = json!({ + "success": false, + "error": { "message": message } + }); + if let Some(c) = code { + resp["error"]["code"] = Value::String(c.to_string()); + } + resp +} diff --git a/src/shared/utils/infra/ryzen_cdn.rs b/src/shared/utils/infra/ryzen_cdn.rs new file mode 100644 index 0000000..79ed75c --- /dev/null +++ b/src/shared/utils/infra/ryzen_cdn.rs @@ -0,0 +1,67 @@ +use crate::shared::errors::AppError; +use infer; // For file type detection +use reqwest::{multipart, Client}; +use serde::{Deserialize, Serialize}; +use tracing::error; + +#[derive(Debug, Serialize, Deserialize)] +pub struct RyzenCDNResponse { + pub success: bool, + pub message: Option, + pub url: Option, +} + +pub async fn ryzen_cdn( + inp: Vec, // Simplified input to a single byte vector for now + original_name: Option, +) -> Result { + let client = Client::new(); + let form = multipart::Form::new(); + + let file_type = infer::get(&inp); + let mime_type = file_type.as_ref().map(|t| t.mime_type()); + let extension = file_type.as_ref().map(|t| t.extension()); + + let file_name = if let Some(name) = original_name { + if let Some(ext) = extension { + format!("{}.{}", name.split('.').next().unwrap_or("file"), ext) + } else { + name + } + } else { + "file".to_string() + }; + + let part = multipart::Part::bytes(inp) + .file_name(file_name) + .mime_str(mime_type.unwrap_or("application/octet-stream"))?; + + let form = form.part("file", part); + + let res = client.post("https://api.ryzumi.vip/api/uploader/ryzencdn") + .multipart(form) + .header("accept", "application/json") + .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36 Edg/139.0.0.0") + .header("Connection", "keep-alive") + .header("Accept-Encoding", "gzip, deflate, br") + .send() + .await?; + + let json_response: RyzenCDNResponse = res.json().await?; + + if !json_response.success { + let error_message = json_response + .message + .unwrap_or_else(|| "Upload failed".to_string()); + error!("RyzenCDN Error: {}", error_message); + return Err(AppError::Other(error_message)); + } + + if let Some(url) = json_response.url { + Ok(url) + } else { + Err(AppError::Other( + "RyzenCDN Error: URL not found in response".to_string(), + )) + } +} diff --git a/src/shared/utils/infra/searchable.rs b/src/shared/utils/infra/searchable.rs new file mode 100644 index 0000000..a376d09 --- /dev/null +++ b/src/shared/utils/infra/searchable.rs @@ -0,0 +1,190 @@ +//! Searchable trait for simple full-text search. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::searchable::{Searchable, SearchQuery, search}; +//! +//! impl Searchable for User { +//! fn searchable_fields() -> Vec<&'static str> { +//! vec!["name", "email", "bio"] +//! } +//! } +//! +//! let query = SearchQuery::new("john") +//! .fields(vec!["name", "email"]); +//! ``` + +use sea_orm::{ColumnTrait, Condition}; +use serde::{Deserialize, Serialize}; + +/// Searchable trait for models. +pub trait Searchable { + /// Get searchable field names. + fn searchable_fields() -> Vec<&'static str>; + + /// Get default search weight for a field (1-10). + fn field_weight(_field: &str) -> u32 { + 1 + } +} + +/// Search query options. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct SearchQuery { + /// Search term. + pub term: String, + /// Fields to search (empty = all). + pub fields: Vec, + /// Minimum match score. + pub min_score: f32, + /// Fuzzy matching. + pub fuzzy: bool, +} + +impl SearchQuery { + pub fn new(term: &str) -> Self { + Self { + term: term.to_string(), + fields: Vec::new(), + min_score: 0.0, + fuzzy: false, + } + } + + pub fn fields(mut self, fields: Vec<&str>) -> Self { + self.fields = fields.into_iter().map(String::from).collect(); + self + } + + pub fn min_score(mut self, score: f32) -> Self { + self.min_score = score; + self + } + + pub fn fuzzy(mut self) -> Self { + self.fuzzy = true; + self + } +} + +/// Search result with scoring. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SearchResult { + pub item: T, + pub score: f32, + pub highlights: Vec, +} + +/// Search highlight. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SearchHighlight { + pub field: String, + pub snippet: String, +} + +/// Simple text matching score. +pub fn calculate_score(text: &str, query: &str, fuzzy: bool) -> f32 { + let text_lower = text.to_lowercase(); + let query_lower = query.to_lowercase(); + + if text_lower == query_lower { + return 1.0; + } + + if text_lower.contains(&query_lower) { + // Position-based scoring + let pos = text_lower.find(&query_lower).unwrap_or(0); + let pos_score = 1.0 - (pos as f32 / text.len() as f32); + return 0.5 + (pos_score * 0.3); + } + + if fuzzy { + // Simple fuzzy: word overlap + let text_words: Vec<&str> = text_lower.split_whitespace().collect(); + let query_words: Vec<&str> = query_lower.split_whitespace().collect(); + + let mut matches = 0; + for qw in &query_words { + for tw in &text_words { + if tw.contains(qw) || qw.contains(tw) { + matches += 1; + break; + } + } + } + + if !query_words.is_empty() { + return matches as f32 / query_words.len() as f32 * 0.5; + } + } + + 0.0 +} + +/// Generate highlight snippet. +pub fn generate_highlight(text: &str, query: &str, context_len: usize) -> Option { + let text_lower = text.to_lowercase(); + let query_lower = query.to_lowercase(); + + if let Some(pos) = text_lower.find(&query_lower) { + let start = pos.saturating_sub(context_len); + let end = (pos + query.len() + context_len).min(text.len()); + + let mut snippet = String::new(); + if start > 0 { + snippet.push_str("..."); + } + snippet.push_str(&text[start..end]); + if end < text.len() { + snippet.push_str("..."); + } + + return Some(snippet); + } + + None +} + +/// Search in a vector of strings. +pub fn search_in_vec(items: &[String], query: &SearchQuery) -> Vec<(usize, f32)> { + let mut results: Vec<(usize, f32)> = items + .iter() + .enumerate() + .filter_map(|(i, text)| { + let score = calculate_score(text, &query.term, query.fuzzy); + if score >= query.min_score { + Some((i, score)) + } else { + None + } + }) + .collect(); + + results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + results +} + +/// Build LIKE conditions for search. +pub fn build_like_condition(columns: Vec, term: &str) -> Condition { + let pattern = format!("%{}%", term); + let mut condition = Condition::any(); + + for col in columns { + condition = condition.add(col.contains(&pattern)); + } + + condition +} + +/// Macro to implement searchable for entity. +#[macro_export] +macro_rules! impl_searchable { + ($entity:ty, $($field:ident),+) => { + impl $crate::shared::utils::searchable::Searchable for $entity { + fn searchable_fields() -> Vec<&'static str> { + vec![$(stringify!($field)),+] + } + } + }; +} diff --git a/src/shared/utils/infra/transaction.rs b/src/shared/utils/infra/transaction.rs new file mode 100644 index 0000000..2e00803 --- /dev/null +++ b/src/shared/utils/infra/transaction.rs @@ -0,0 +1,179 @@ +//! Database Transaction helpers. +//! +//! Atomic database operations with automatic rollback. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::transaction::{transaction, TransactionExt}; +//! +//! let result = transaction(&db, |txn| async move { +//! User::insert(user1).exec(&txn).await?; +//! User::insert(user2).exec(&txn).await?; +//! Ok(()) +//! }).await?; +//! ``` + +use sea_orm::{DatabaseConnection, DatabaseTransaction, DbErr, TransactionTrait}; +use std::future::Future; + +/// Transaction error. +#[derive(Debug, thiserror::Error)] +pub enum TransactionError { + #[error("Database error: {0}")] + DbError(#[from] DbErr), + #[error("Transaction failed: {0}")] + Failed(String), +} + +/// Execute a closure within a database transaction. +pub async fn transaction(db: &DatabaseConnection, f: F) -> Result +where + F: FnOnce(DatabaseTransaction) -> Fut, + Fut: Future>, +{ + let txn = db.begin().await?; + + match f(txn).await { + Ok(result) => Ok(result), + Err(e) => Err(TransactionError::DbError(e)), + } +} + +/// Execute with automatic commit/rollback. +pub async fn with_transaction( + db: &DatabaseConnection, + f: F, +) -> Result +where + F: FnOnce(&DatabaseTransaction) -> Fut, + Fut: Future>, +{ + let txn = db.begin().await?; + + match f(&txn).await { + Ok(result) => { + txn.commit().await?; + Ok(result) + } + Err(e) => { + txn.rollback().await?; + Err(TransactionError::DbError(e)) + } + } +} + +/// Nested transaction support (savepoints). +pub struct NestedTransaction { + depth: usize, +} + +impl NestedTransaction { + pub fn new() -> Self { + Self { depth: 0 } + } + + pub fn begin(&mut self) -> usize { + self.depth += 1; + self.depth + } + + pub fn commit(&mut self) -> usize { + if self.depth > 0 { + self.depth -= 1; + } + self.depth + } + + pub fn rollback(&mut self) -> usize { + if self.depth > 0 { + self.depth -= 1; + } + self.depth + } + + pub fn depth(&self) -> usize { + self.depth + } +} + +impl Default for NestedTransaction { + fn default() -> Self { + Self::new() + } +} + +/// Retry transaction on deadlock. +pub async fn retry_transaction( + db: &DatabaseConnection, + max_retries: usize, + f: F, +) -> Result +where + F: Fn() -> Fut + Clone, + Fut: Future>, +{ + let mut attempts = 0; + let mut last_error = None; + + while attempts < max_retries { + let txn = db.begin().await?; + + match f().await { + Ok(result) => { + txn.commit().await?; + return Ok(result); + } + Err(e) => { + txn.rollback().await.ok(); + + // Check if deadlock (simplified - actual check would be DB-specific) + let is_deadlock = e.to_string().to_lowercase().contains("deadlock"); + + if is_deadlock && attempts < max_retries - 1 { + attempts += 1; + tokio::time::sleep(tokio::time::Duration::from_millis(100 * attempts as u64)) + .await; + continue; + } + + last_error = Some(e); + break; + } + } + } + + Err(TransactionError::DbError(last_error.unwrap())) +} + +/// Transaction context for tracking. +#[derive(Debug, Clone)] +pub struct TransactionContext { + pub id: String, + pub started_at: chrono::DateTime, + pub operations: usize, +} + +impl TransactionContext { + pub fn new() -> Self { + Self { + id: uuid::Uuid::new_v4().to_string(), + started_at: chrono::Utc::now(), + operations: 0, + } + } + + pub fn record_operation(&mut self) { + self.operations += 1; + } + + pub fn duration(&self) -> chrono::Duration { + chrono::Utc::now() - self.started_at + } +} + +impl Default for TransactionContext { + fn default() -> Self { + Self::new() + } +} diff --git a/src/shared/utils/infra/uuid_utils.rs b/src/shared/utils/infra/uuid_utils.rs new file mode 100644 index 0000000..4c89595 --- /dev/null +++ b/src/shared/utils/infra/uuid_utils.rs @@ -0,0 +1,78 @@ +//! UUID utilities. + +use uuid::Uuid; + +/// Generate a new UUID v4. +pub fn new_v4() -> String { + Uuid::new_v4().to_string() +} + +/// Generate a new UUID v4 without hyphens. +pub fn new_v4_simple() -> String { + Uuid::new_v4().simple().to_string() +} + +/// Parse a UUID string. +pub fn parse(s: &str) -> Result { + Uuid::parse_str(s) +} + +/// Check if string is valid UUID. +pub fn is_valid(s: &str) -> bool { + Uuid::parse_str(s).is_ok() +} + +/// Convert to hyphenated format. +pub fn to_hyphenated(s: &str) -> Option { + Uuid::parse_str(s).ok().map(|u| u.hyphenated().to_string()) +} + +/// Convert to simple format (no hyphens). +pub fn to_simple(s: &str) -> Option { + Uuid::parse_str(s).ok().map(|u| u.simple().to_string()) +} + +/// Generate a nil UUID (all zeros). +pub fn nil() -> String { + Uuid::nil().to_string() +} + +/// Check if UUID is nil. +pub fn is_nil(s: &str) -> bool { + parse(s).map(|u| u.is_nil()).unwrap_or(false) +} + +/// Extract timestamp from UUID v7 (if applicable). +pub fn timestamp_v7(s: &str) -> Option { + parse(s).ok().and_then(|u| { + if u.get_version() == Some(uuid::Version::SortRand) { + u.get_timestamp().map(|ts| { + let (secs, _) = ts.to_unix(); + secs + }) + } else { + None + } + }) +} + +/// Create a UUID namespace for v5. +pub fn namespace(ns: &str) -> Option { + match ns.to_lowercase().as_str() { + "dns" => Some(Uuid::NAMESPACE_DNS), + "url" => Some(Uuid::NAMESPACE_URL), + "oid" => Some(Uuid::NAMESPACE_OID), + "x500" => Some(Uuid::NAMESPACE_X500), + _ => Uuid::parse_str(ns).ok(), + } +} + +/// Generate a short ID (first 8 chars of UUID). +pub fn short_id() -> String { + Uuid::new_v4().simple().to_string()[..8].to_string() +} + +/// Generate a medium ID (first 12 chars of UUID). +pub fn medium_id() -> String { + Uuid::new_v4().simple().to_string()[..12].to_string() +} diff --git a/src/shared/utils/infra/versioning.rs b/src/shared/utils/infra/versioning.rs new file mode 100644 index 0000000..47256c1 --- /dev/null +++ b/src/shared/utils/infra/versioning.rs @@ -0,0 +1,191 @@ +//! Versioned API helpers. +//! +//! API version extraction and routing. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::versioning::{ApiVersion, extract_version}; +//! +//! // From header: Accept: application/vnd.api+json; version=2 +//! let version = extract_version(&headers); +//! +//! // From path: /api/v2/users +//! let version = ApiVersion::from_path("/api/v2/users"); +//! ``` + +use axum::http::{header::ACCEPT, HeaderMap}; +use serde::{Deserialize, Serialize}; +use std::cmp::Ordering; + +/// API version. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct ApiVersion { + pub major: u32, + pub minor: u32, +} + +impl ApiVersion { + /// Create a new version. + pub const fn new(major: u32, minor: u32) -> Self { + Self { major, minor } + } + + /// Create from major version only. + pub const fn major_only(major: u32) -> Self { + Self { major, minor: 0 } + } + + /// Parse from string like "v2" or "2.1". + pub fn parse(s: &str) -> Option { + let s = s.trim().trim_start_matches('v').trim_start_matches('V'); + + if let Some((major, minor)) = s.split_once('.') { + Some(Self { + major: major.parse().ok()?, + minor: minor.parse().ok()?, + }) + } else { + Some(Self { + major: s.parse().ok()?, + minor: 0, + }) + } + } + + /// Extract from URL path like /api/v2/users. + pub fn from_path(path: &str) -> Option { + for segment in path.split('/') { + if segment.starts_with('v') || segment.starts_with('V') { + if let Some(v) = Self::parse(segment) { + return Some(v); + } + } + } + None + } + + /// Extract from Accept header. + pub fn from_accept_header(accept: &str) -> Option { + // Format: application/vnd.api+json; version=2 + for part in accept.split(';') { + let part = part.trim(); + if part.starts_with("version=") { + let version = part.trim_start_matches("version="); + return Self::parse(version); + } + } + None + } + + /// Check if version is at least the given version. + pub fn at_least(&self, major: u32, minor: u32) -> bool { + self.major > major || (self.major == major && self.minor >= minor) + } + + /// Check if version is below the given version. + pub fn below(&self, major: u32, minor: u32) -> bool { + !self.at_least(major, minor) + } +} + +impl Default for ApiVersion { + fn default() -> Self { + Self { major: 1, minor: 0 } + } +} + +impl std::fmt::Display for ApiVersion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "v{}.{}", self.major, self.minor) + } +} + +impl PartialOrd for ApiVersion { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for ApiVersion { + fn cmp(&self, other: &Self) -> Ordering { + match self.major.cmp(&other.major) { + Ordering::Equal => self.minor.cmp(&other.minor), + other => other, + } + } +} + +/// Extract API version from request. +pub fn extract_version(headers: &HeaderMap) -> ApiVersion { + // Try Accept header first + if let Some(accept) = headers.get(ACCEPT).and_then(|h| h.to_str().ok()) { + if let Some(v) = ApiVersion::from_accept_header(accept) { + return v; + } + } + + // Try custom header + if let Some(version) = headers.get("X-API-Version").and_then(|h| h.to_str().ok()) { + if let Some(v) = ApiVersion::parse(version) { + return v; + } + } + + ApiVersion::default() +} + +/// Common API versions. +pub mod versions { + use super::ApiVersion; + + pub const V1: ApiVersion = ApiVersion::new(1, 0); + pub const V2: ApiVersion = ApiVersion::new(2, 0); + pub const V3: ApiVersion = ApiVersion::new(3, 0); +} + +/// Version constraint for routing. +#[derive(Debug, Clone)] +pub struct VersionConstraint { + pub min: Option, + pub max: Option, +} + +impl VersionConstraint { + pub fn new() -> Self { + Self { + min: None, + max: None, + } + } + + pub fn min(mut self, version: ApiVersion) -> Self { + self.min = Some(version); + self + } + + pub fn max(mut self, version: ApiVersion) -> Self { + self.max = Some(version); + self + } + + pub fn matches(&self, version: ApiVersion) -> bool { + if let Some(min) = self.min { + if version < min { + return false; + } + } + if let Some(max) = self.max { + if version > max { + return false; + } + } + true + } +} + +impl Default for VersionConstraint { + fn default() -> Self { + Self::new() + } +} diff --git a/src/shared/utils/io/cache.rs b/src/shared/utils/io/cache.rs new file mode 100644 index 0000000..7fa0017 --- /dev/null +++ b/src/shared/utils/io/cache.rs @@ -0,0 +1,155 @@ +//! Redis caching helpers. + +use crate::shared::utils::cache_ttl::CACHE_TTL_VERY_SHORT; +use deadpool_redis::redis::AsyncCommands; +use deadpool_redis::Pool; +use serde::{de::DeserializeOwned, Serialize}; +use tracing::{debug, error}; + +/// Default cache TTL in seconds (5 minutes). +pub const DEFAULT_CACHE_TTL: u64 = CACHE_TTL_VERY_SHORT; + +/// Cache helper for Redis operations. +pub struct Cache<'a> { + pool: &'a Pool, +} + +impl<'a> Cache<'a> { + /// Create a new cache helper. + pub fn new(pool: &'a Pool) -> Self { + Self { pool } + } + + /// Get a value from cache, deserializing JSON. + pub async fn get(&self, key: &str) -> Option { + let mut conn = match self.pool.get().await { + Ok(c) => c, + Err(e) => { + error!("Cache: failed to get connection: {}", e); + return None; + } + }; + + let cached: Option = conn.get(key).await.ok()?; + + if cached.is_some() { + debug!("Cache hit: {}", key); + } else { + debug!("Cache miss: {}", key); + } + + cached.and_then(|json| serde_json::from_str(&json).ok()) + } + + /// Get multiple values from cache, deserializing JSON. + /// Returns a vector of Options, preserving order of keys. + pub async fn mget(&self, keys: &[String]) -> Vec> { + if keys.is_empty() { + return Vec::new(); + } + + let mut conn = match self.pool.get().await { + Ok(c) => c, + Err(e) => { + error!("Cache: failed to get connection: {}", e); + return std::iter::repeat_with(|| None).take(keys.len()).collect(); + } + }; + + // Use low-level cmd interface for MGET to ensure correct command usage + use deadpool_redis::redis::cmd; + let cached_values: Vec> = + match cmd("MGET").arg(keys).query_async(&mut conn).await { + Ok(v) => v, + Err(e) => { + error!("Cache: failed to mget values: {}", e); + return std::iter::repeat_with(|| None).take(keys.len()).collect(); + } + }; + + cached_values + .into_iter() + .map(|opt_s| opt_s.and_then(|json| serde_json::from_str(&json).ok())) + .collect() + } + + /// Set a value in cache with default TTL (5 minutes). + pub async fn set(&self, key: &str, value: &T) -> Result<(), String> { + self.set_with_ttl(key, value, DEFAULT_CACHE_TTL).await + } + + /// Set a value in cache with custom TTL. + pub async fn set_with_ttl( + &self, + key: &str, + value: &T, + ttl_secs: u64, + ) -> Result<(), String> { + let mut conn = self.pool.get().await.map_err(|e| e.to_string())?; + + let json = serde_json::to_string(value).map_err(|e| e.to_string())?; + + conn.set_ex::<_, _, ()>(key, json, ttl_secs) + .await + .map_err(|e| e.to_string())?; + + debug!("Cache: set key {} with TTL {}s", key, ttl_secs); + Ok(()) + } + + /// Delete a key from cache. + pub async fn delete(&self, key: &str) -> Result<(), String> { + let mut conn = self.pool.get().await.map_err(|e| e.to_string())?; + conn.del::<_, ()>(key).await.map_err(|e| e.to_string())?; + debug!("Cache: deleted key {}", key); + Ok(()) + } + + /// Check if key exists. + pub async fn exists(&self, key: &str) -> bool { + let mut conn = match self.pool.get().await { + Ok(c) => c, + Err(_) => return false, + }; + conn.exists::<_, bool>(key).await.unwrap_or(false) + } + + /// Get or set: returns cached value or computes and caches new value. + pub async fn get_or_set( + &self, + key: &str, + ttl_secs: u64, + compute: F, + ) -> Result + where + T: Serialize + DeserializeOwned, + F: FnOnce() -> Fut, + Fut: std::future::Future>, + { + // Try cache first + if let Some(cached) = self.get::(key).await { + debug!("Cache hit: {}", key); + return Ok(cached); + } + + debug!("Cache miss: {}", key); + + // Compute the value + let value = compute().await?; + + // Store in cache + self.set_with_ttl(key, &value, ttl_secs).await?; + + Ok(value) + } +} + +/// Create a cache key with prefix. +pub fn cache_key(prefix: &str, id: &str) -> String { + format!("{}:{}", prefix, id) +} + +/// Create a cache key with multiple parts. +pub fn cache_key_multi(parts: &[&str]) -> String { + parts.join(":") +} diff --git a/src/shared/utils/io/cache_tags.rs b/src/shared/utils/io/cache_tags.rs new file mode 100644 index 0000000..0f344d2 --- /dev/null +++ b/src/shared/utils/io/cache_tags.rs @@ -0,0 +1,306 @@ +//! Cache tags for tag-based cache invalidation. +//! +//! Extends the basic cache helper with tag support for grouped invalidation. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::cache_tags::TaggedCache; +//! +//! let cache = TaggedCache::new(redis_pool); +//! +//! // Set with tags +//! cache.put_tagged("user:123", data, &["users", "user:123"], 3600).await?; +//! cache.put_tagged("user:456", data2, &["users", "user:456"], 3600).await?; +//! +//! // Invalidate all users +//! cache.flush_tag("users").await?; +//! ``` + +use deadpool_redis::{redis::AsyncCommands, Pool}; +use serde::{de::DeserializeOwned, Serialize}; +use std::sync::Arc; + +/// Tagged cache error types. +#[derive(Debug, thiserror::Error)] +pub enum CacheTagError { + #[error("Redis error: {0}")] + RedisError(String), + #[error("Serialization error: {0}")] + SerializationError(String), + #[error("Deserialization error: {0}")] + DeserializationError(String), +} + +/// Cache with tag-based invalidation support. +#[derive(Clone)] +pub struct TaggedCache { + pool: Arc, + prefix: String, + tag_prefix: String, +} + +impl TaggedCache { + /// Create a new tagged cache. + pub fn new(pool: Arc) -> Self { + Self { + pool, + prefix: "cache:".to_string(), + tag_prefix: "cache:tag:".to_string(), + } + } + + /// Create with custom prefix. + pub fn with_prefix(pool: Arc, prefix: &str) -> Self { + Self { + pool, + prefix: format!("{}:", prefix), + tag_prefix: format!("{}:tag:", prefix), + } + } + + /// Generate cache key. + fn key(&self, key: &str) -> String { + format!("{}{}", self.prefix, key) + } + + /// Generate tag key. + fn tag_key(&self, tag: &str) -> String { + format!("{}{}", self.tag_prefix, tag) + } + + /// Get a value from cache. + pub async fn get(&self, key: &str) -> Result, CacheTagError> { + let mut conn = self.pool.get().await.map_err(|e| { + tracing::error!("Redis connection error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + let cache_key = self.key(key); + let value: Option = conn.get(&cache_key).await.map_err(|e| { + tracing::error!("Redis get error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + match value { + Some(json) => { + let data: T = serde_json::from_str(&json) + .map_err(|e| CacheTagError::DeserializationError(e.to_string()))?; + Ok(Some(data)) + } + None => Ok(None), + } + } + + /// Put a value in cache with TTL (seconds). + pub async fn put( + &self, + key: &str, + value: &T, + ttl: u64, + ) -> Result<(), CacheTagError> { + let mut conn = self.pool.get().await.map_err(|e| { + tracing::error!("Redis connection error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + let cache_key = self.key(key); + let json = serde_json::to_string(value) + .map_err(|e| CacheTagError::SerializationError(e.to_string()))?; + + conn.set_ex::<_, _, ()>(&cache_key, &json, ttl) + .await + .map_err(|e| { + tracing::error!("Redis set error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + Ok(()) + } + + /// Put a value in cache with tags. + pub async fn put_tagged( + &self, + key: &str, + value: &T, + tags: &[&str], + ttl: u64, + ) -> Result<(), CacheTagError> { + let mut conn = self.pool.get().await.map_err(|e| { + tracing::error!("Redis connection error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + let cache_key = self.key(key); + let json = serde_json::to_string(value) + .map_err(|e| CacheTagError::SerializationError(e.to_string()))?; + + // Set the value + conn.set_ex::<_, _, ()>(&cache_key, &json, ttl) + .await + .map_err(|e| { + tracing::error!("Redis set error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + // Add key to each tag set + for tag in tags { + let tag_key = self.tag_key(tag); + conn.sadd::<_, _, ()>(&tag_key, &cache_key) + .await + .map_err(|e| { + tracing::error!("Redis sadd error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + } + + Ok(()) + } + + /// Delete a specific key. + pub async fn forget(&self, key: &str) -> Result<(), CacheTagError> { + let mut conn = self.pool.get().await.map_err(|e| { + tracing::error!("Redis connection error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + let cache_key = self.key(key); + conn.del::<_, ()>(&cache_key).await.map_err(|e| { + tracing::error!("Redis del error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + Ok(()) + } + + /// Flush all keys associated with a tag. + pub async fn flush_tag(&self, tag: &str) -> Result { + let mut conn = self.pool.get().await.map_err(|e| { + tracing::error!("Redis connection error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + let tag_key = self.tag_key(tag); + + // Get all keys in the tag set + let keys: Vec = conn.smembers(&tag_key).await.map_err(|e| { + tracing::error!("Redis smembers error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + let count = keys.len(); + + // Delete each key + for key in &keys { + let _: () = conn.del(key).await.unwrap_or(()); + } + + // Delete the tag set itself + conn.del::<_, ()>(&tag_key).await.map_err(|e| { + tracing::error!("Redis del error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + tracing::info!("Flushed {} keys for tag '{}'", count, tag); + Ok(count) + } + + /// Flush multiple tags at once. + pub async fn flush_tags(&self, tags: &[&str]) -> Result { + let mut total = 0; + for tag in tags { + total += self.flush_tag(tag).await?; + } + Ok(total) + } + + /// Check if a key exists. + pub async fn has(&self, key: &str) -> Result { + let mut conn = self.pool.get().await.map_err(|e| { + tracing::error!("Redis connection error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + let cache_key = self.key(key); + let exists: bool = conn.exists(&cache_key).await.map_err(|e| { + tracing::error!("Redis exists error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + Ok(exists) + } + + /// Get or set a value (cache-aside pattern). + pub async fn remember(&self, key: &str, ttl: u64, f: F) -> Result + where + T: Serialize + DeserializeOwned, + F: FnOnce() -> Fut, + Fut: std::future::Future>, + { + // Try to get from cache + if let Some(value) = self.get::(key).await? { + return Ok(value); + } + + // Generate value + let value = f().await?; + + // Store in cache + self.put(key, &value, ttl).await?; + + Ok(value) + } + + /// Get or set a value with tags. + pub async fn remember_tagged( + &self, + key: &str, + tags: &[&str], + ttl: u64, + f: F, + ) -> Result + where + T: Serialize + DeserializeOwned, + F: FnOnce() -> Fut, + Fut: std::future::Future>, + { + // Try to get from cache + if let Some(value) = self.get::(key).await? { + return Ok(value); + } + + // Generate value + let value = f().await?; + + // Store in cache with tags + self.put_tagged(key, &value, tags, ttl).await?; + + Ok(value) + } + + /// Get cache statistics for a tag. + pub async fn tag_count(&self, tag: &str) -> Result { + let mut conn = self.pool.get().await.map_err(|e| { + tracing::error!("Redis connection error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + let tag_key = self.tag_key(tag); + let count: usize = conn.scard(&tag_key).await.map_err(|e| { + tracing::error!("Redis scard error: {}", e); + CacheTagError::RedisError(e.to_string()) + })?; + + Ok(count) + } +} + +/// Helper to create cache tags from entity type and ID. +pub fn entity_tags(entity_type: &str, id: &str) -> Vec { + vec![entity_type.to_string(), format!("{}:{}", entity_type, id)] +} + +/// Helper to create cache key for entity. +pub fn entity_key(entity_type: &str, id: &str) -> String { + format!("{}:{}", entity_type, id) +} diff --git a/src/shared/utils/io/cache_ttl.rs b/src/shared/utils/io/cache_ttl.rs new file mode 100644 index 0000000..9b1a0f9 --- /dev/null +++ b/src/shared/utils/io/cache_ttl.rs @@ -0,0 +1,84 @@ +//! Cache TTL constants for consistent caching across the application. + +/// Very short TTL for highly volatile data (5 minutes) +/// Use for: Real-time data, user presence, active sessions +pub const CACHE_TTL_VERY_SHORT: u64 = 300; + +/// Short TTL for frequently changing data (15 minutes) +/// Use for: Trending content, live feeds, dynamic lists +pub const CACHE_TTL_SHORT: u64 = 900; + +/// Medium TTL for regular data (1 hour) +/// Use for: User profiles, search results, API responses +pub const CACHE_TTL_MEDIUM: u64 = 3600; + +/// Long TTL for mostly static data (6 hours) +/// Use for: Configuration, reference data, translations +pub const CACHE_TTL_LONG: u64 = 21600; + +/// Very long TTL for static content (1 day) +/// Use for: Static pages, archived content, historical data +pub const CACHE_TTL_VERY_LONG: u64 = 86400; + +/// Image cache TTL (7 days) +/// Use for: Cached images, thumbnails, avatars +pub const CACHE_TTL_IMAGE: u64 = 604800; + +/// CDN cache TTL (30 days) +/// Use for: Immutable assets, versioned files +pub const CACHE_TTL_CDN: u64 = 2592000; + +/// Get TTL based on cache type +pub fn get_ttl_for(cache_type: CacheType) -> u64 { + match cache_type { + CacheType::RealTime => CACHE_TTL_VERY_SHORT, + CacheType::Volatile => CACHE_TTL_SHORT, + CacheType::Regular => CACHE_TTL_MEDIUM, + CacheType::Stable => CACHE_TTL_LONG, + CacheType::Static => CACHE_TTL_VERY_LONG, + CacheType::Image => CACHE_TTL_IMAGE, + CacheType::Cdn => CACHE_TTL_CDN, + } +} + +/// Cache type classification +#[derive(Debug, Clone, Copy)] +pub enum CacheType { + /// Real-time data (5 min) + RealTime, + /// Volatile data (15 min) + Volatile, + /// Regular data (1 hour) + Regular, + /// Stable data (6 hours) + Stable, + /// Static data (1 day) + Static, + /// Images (7 days) + Image, + /// CDN assets (30 days) + Cdn, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_cache_ttl_values() { + assert_eq!(CACHE_TTL_VERY_SHORT, 300); + assert_eq!(CACHE_TTL_SHORT, 900); + assert_eq!(CACHE_TTL_MEDIUM, 3600); + assert_eq!(CACHE_TTL_LONG, 21600); + assert_eq!(CACHE_TTL_VERY_LONG, 86400); + assert_eq!(CACHE_TTL_IMAGE, 604800); + assert_eq!(CACHE_TTL_CDN, 2592000); + } + + #[test] + fn test_get_ttl_for() { + assert_eq!(get_ttl_for(CacheType::RealTime), 300); + assert_eq!(get_ttl_for(CacheType::Regular), 3600); + assert_eq!(get_ttl_for(CacheType::Image), 604800); + } +} diff --git a/src/shared/utils/io/file.rs b/src/shared/utils/io/file.rs new file mode 100644 index 0000000..89e58f1 --- /dev/null +++ b/src/shared/utils/io/file.rs @@ -0,0 +1,132 @@ +//! File utilities. + +use std::path::Path; +use tokio::fs; +use tokio::io::AsyncWriteExt; + +/// Read file contents as string. +pub async fn read_file(path: &str) -> anyhow::Result { + Ok(fs::read_to_string(path).await?) +} + +/// Read file contents as bytes. +pub async fn read_bytes(path: &str) -> anyhow::Result> { + Ok(fs::read(path).await?) +} + +/// Write string to file. +pub async fn write_file(path: &str, contents: &str) -> anyhow::Result<()> { + fs::write(path, contents).await?; + Ok(()) +} + +/// Write bytes to file. +pub async fn write_bytes(path: &str, contents: &[u8]) -> anyhow::Result<()> { + fs::write(path, contents).await?; + Ok(()) +} + +/// Append to file. +pub async fn append_file(path: &str, contents: &str) -> anyhow::Result<()> { + let mut file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + .await?; + file.write_all(contents.as_bytes()).await?; + Ok(()) +} + +/// Check if file exists. +pub async fn file_exists(path: &str) -> bool { + fs::metadata(path).await.is_ok() +} + +/// Check if path is a directory. +pub async fn is_directory(path: &str) -> bool { + fs::metadata(path) + .await + .map(|m| m.is_dir()) + .unwrap_or(false) +} + +/// Create directory (including parents). +pub async fn create_dir(path: &str) -> anyhow::Result<()> { + fs::create_dir_all(path).await?; + Ok(()) +} + +/// Delete file. +pub async fn delete_file(path: &str) -> anyhow::Result<()> { + fs::remove_file(path).await?; + Ok(()) +} + +/// Delete directory recursively. +pub async fn delete_dir(path: &str) -> anyhow::Result<()> { + fs::remove_dir_all(path).await?; + Ok(()) +} + +/// Get file extension. +pub fn get_extension(path: &str) -> Option { + Path::new(path) + .extension() + .and_then(|ext| ext.to_str()) + .map(|s| s.to_lowercase()) +} + +/// Get filename without extension. +pub fn get_filename(path: &str) -> Option { + Path::new(path) + .file_stem() + .and_then(|name| name.to_str()) + .map(String::from) +} + +/// Get file size in bytes. +pub async fn file_size(path: &str) -> anyhow::Result { + let metadata = fs::metadata(path).await?; + Ok(metadata.len()) +} + +/// Format file size to human readable string. +pub fn format_file_size(bytes: u64) -> String { + const KB: u64 = 1024; + const MB: u64 = KB * 1024; + const GB: u64 = MB * 1024; + + if bytes >= GB { + format!("{:.2} GB", bytes as f64 / GB as f64) + } else if bytes >= MB { + format!("{:.2} MB", bytes as f64 / MB as f64) + } else if bytes >= KB { + format!("{:.2} KB", bytes as f64 / KB as f64) + } else { + format!("{} bytes", bytes) + } +} + +/// Get MIME type from file extension. +pub fn mime_from_extension(ext: &str) -> &'static str { + match ext.to_lowercase().as_str() { + "html" | "htm" => "text/html", + "css" => "text/css", + "js" => "application/javascript", + "json" => "application/json", + "xml" => "application/xml", + "txt" => "text/plain", + "png" => "image/png", + "jpg" | "jpeg" => "image/jpeg", + "gif" => "image/gif", + "svg" => "image/svg+xml", + "webp" => "image/webp", + "ico" => "image/x-icon", + "pdf" => "application/pdf", + "zip" => "application/zip", + "mp3" => "audio/mpeg", + "mp4" => "video/mp4", + "webm" => "video/webm", + _ => "application/octet-stream", + } +} diff --git a/src/shared/utils/io/mod.rs b/src/shared/utils/io/mod.rs new file mode 100644 index 0000000..d94727f --- /dev/null +++ b/src/shared/utils/io/mod.rs @@ -0,0 +1,6 @@ +pub mod cache; +pub mod cache_tags; +pub mod cache_ttl; +pub mod file; +pub mod retry; +pub mod soft_delete; diff --git a/src/shared/utils/io/retry.rs b/src/shared/utils/io/retry.rs new file mode 100644 index 0000000..6a40a1b --- /dev/null +++ b/src/shared/utils/io/retry.rs @@ -0,0 +1,66 @@ +//! HTTP retry utilities with exponential backoff. + +use backoff::ExponentialBackoff; +use std::time::Duration; + +/// Default retry configuration for HTTP requests. +pub fn default_backoff() -> ExponentialBackoff { + ExponentialBackoff { + initial_interval: Duration::from_millis(500), + max_interval: Duration::from_secs(10), + multiplier: 2.0, + max_elapsed_time: Some(Duration::from_secs(30)), + ..Default::default() + } +} + +/// Create a custom exponential backoff. +pub fn custom_backoff( + initial_ms: u64, + max_secs: u64, + multiplier: f64, + max_elapsed_secs: u64, +) -> ExponentialBackoff { + ExponentialBackoff { + initial_interval: Duration::from_millis(initial_ms), + max_interval: Duration::from_secs(max_secs), + multiplier, + max_elapsed_time: Some(Duration::from_secs(max_elapsed_secs)), + ..Default::default() + } +} + +/// Quick backoff for fast retries (3 attempts, 100ms initial). +pub fn quick_backoff() -> ExponentialBackoff { + ExponentialBackoff { + initial_interval: Duration::from_millis(100), + max_interval: Duration::from_secs(1), + multiplier: 2.0, + max_elapsed_time: Some(Duration::from_secs(5)), + ..Default::default() + } +} + +/// Slow backoff for long operations (10 attempts, 1s initial). +pub fn slow_backoff() -> ExponentialBackoff { + ExponentialBackoff { + initial_interval: Duration::from_secs(1), + max_interval: Duration::from_secs(30), + multiplier: 2.0, + max_elapsed_time: Some(Duration::from_secs(120)), + ..Default::default() + } +} + +/// Make an error transient (will be retried). +pub fn transient(err: E) -> backoff::Error { + backoff::Error::transient(err) +} + +/// Make an error permanent (will NOT be retried). +pub fn permanent(err: E) -> backoff::Error { + backoff::Error::permanent(err) +} + +// Re-export retry function for convenience +pub use backoff::future::retry; diff --git a/src/shared/utils/io/soft_delete.rs b/src/shared/utils/io/soft_delete.rs new file mode 100644 index 0000000..a930a42 --- /dev/null +++ b/src/shared/utils/io/soft_delete.rs @@ -0,0 +1,86 @@ +//! Soft delete helpers for SeaORM entities. +//! +//! Provides soft delete filtering for entities with a `deleted_at` column. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::soft_delete::{SoftDeletable, soft_delete_filter, SoftDeleteScope}; +//! use sea_orm::*; +//! +//! // Query without deleted +//! let users = User::find() +//! .filter(soft_delete_filter(user::Column::DeletedAt, SoftDeleteScope::WithoutDeleted)) +//! .all(db) +//! .await?; +//! +//! // Query only deleted +//! let deleted = User::find() +//! .filter(soft_delete_filter(user::Column::DeletedAt, SoftDeleteScope::OnlyDeleted)) +//! .all(db) +//! .await?; +//! ``` + +use chrono::{DateTime, Utc}; +use sea_orm::{ColumnTrait, Condition}; + +/// Trait for entities that support soft deletes. +pub trait SoftDeletable { + /// Get the deleted_at timestamp if soft deleted. + fn deleted_at(&self) -> Option>; + + /// Check if the entity is soft deleted. + fn is_deleted(&self) -> bool { + self.deleted_at().is_some() + } + + /// Check if the entity is not soft deleted. + fn is_active(&self) -> bool { + self.deleted_at().is_none() + } +} + +/// Query scope for filtering soft-deleted entities. +#[derive(Debug, Clone, Copy)] +pub enum SoftDeleteScope { + /// Exclude soft-deleted entities (default). + WithoutDeleted, + /// Include soft-deleted entities. + WithDeleted, + /// Only include soft-deleted entities. + OnlyDeleted, +} + +/// Create a condition for soft delete filtering. +/// +/// Use this in your entity queries: +/// ```ignore +/// User::find() +/// .filter(soft_delete_filter(user::Column::DeletedAt, SoftDeleteScope::WithoutDeleted)) +/// .all(db) +/// .await +/// ``` +pub fn soft_delete_filter(column: C, scope: SoftDeleteScope) -> Condition { + match scope { + SoftDeleteScope::WithoutDeleted => Condition::all().add(column.is_null()), + SoftDeleteScope::WithDeleted => Condition::all(), + SoftDeleteScope::OnlyDeleted => Condition::all().add(column.is_not_null()), + } +} + +/// Macro to implement SoftDeletable for a model. +/// +/// Usage: +/// ```ignore +/// impl_soft_deletable!(user::Model, deleted_at); +/// ``` +#[macro_export] +macro_rules! impl_soft_deletable { + ($model:ty, $deleted_at_field:ident) => { + impl $crate::shared::utils::soft_delete::SoftDeletable for $model { + fn deleted_at(&self) -> Option> { + self.$deleted_at_field + } + } + }; +} diff --git a/src/shared/utils/mod.rs b/src/shared/utils/mod.rs new file mode 100644 index 0000000..3c74736 --- /dev/null +++ b/src/shared/utils/mod.rs @@ -0,0 +1,190 @@ +//! Helper utilities for easier development. +//! +//! Comprehensive collection of updated utility modules for cleaner, more maintainable code. + +// Submodules +pub mod core; +pub mod data; +pub mod dev; +pub mod infra; +pub mod io; +pub mod web; + +// Re-exports for convenience (backward compatibility) + +// Core +pub use core::api_response; +pub use core::errors; +pub use core::handler; +pub use core::pagination; +pub use core::prelude; +pub use core::response; + +// Data +pub use data::collections; +pub use data::convert; +pub use data::datetime; +pub use data::json; +pub use data::numbers; +pub use data::string; +pub use data::text; + +// IO +pub use io::cache; +pub use io::cache_tags; +pub use io::cache_ttl; +pub use io::file; +pub use io::retry; +pub use io::soft_delete; + +// Web +pub use web::http; +pub use web::query; +pub use web::request; +pub use web::scraping; +pub use web::url; + +// Dev +pub use dev::async_utils; +pub use dev::logging; +pub use dev::performance; +pub use dev::result_ext; +pub use dev::serde_helpers; +pub use dev::testing; + +// Infra +pub use infra::bulk; +pub use infra::console; +pub use infra::encryption; +pub use infra::env; +pub use infra::form_request; +pub use infra::health_check; +pub use infra::import_export; +pub use infra::query_profiler; +pub use infra::resource; +pub use infra::searchable; +pub use infra::transaction; +pub use infra::uuid_utils; +pub use infra::versioning; + +// Ryzen CDN +pub use infra::ryzen_cdn::*; + +// ============================================================================ +// Original Re-exports (preserved) +// ============================================================================ + +// Prelude (common imports) +pub use prelude::*; + +// Response helpers +pub use pagination::*; +pub use response::*; + +// Error helpers +pub use errors::{ + bad_request, db_error, forbidden, internal_err, internal_error, not_found, redis_error, + unauthorized, HandlerError, ResultExt, +}; + +// Retry/Backoff +pub use retry::{ + custom_backoff, default_backoff, permanent, quick_backoff, retry, slow_backoff, transient, +}; + +// Caching +pub use cache::{cache_key, cache_key_multi, Cache, DEFAULT_CACHE_TTL}; + +// Scraping +pub use scraping::{ + attr_from, attr_from_or, extract_number, extract_slug, fetch_html_with_retry, parse_html, + select_attr, select_text, selector, strip_tags, text, text_from, text_from_or, Scraper, +}; + +// Strings +pub use string::{ + initials, is_valid_email, mask_email, random_code, random_string, slugify, title_case, truncate, +}; + +// DateTime +pub use datetime::{ + add_days, add_hours, is_future, is_past, now, parse_iso, relative, timestamp, to_human, to_iso, +}; + +// Crypto + +// Files +pub use file::{ + create_dir, file_exists, format_file_size, get_extension, mime_from_extension, read_file, + write_file, +}; + +// JSON +pub use json::{ + deep_merge, get_i64, get_path, get_str, is_empty, merge, parse, remove_nulls, stringify, + stringify_pretty, +}; + +// URL +pub use url::{ + decode, encode, extract_domain, is_absolute, join_paths, make_absolute, parse_query, UrlBuilder, +}; + +// Logging +pub use logging::{log_and_map, log_error, log_request, PerfLogger, TimedOperation}; + +// Collections +pub use collections::{ + all, any, chunk, count, find, find_index, flatten, frequencies, group_by, partition, reverse, + skip, sum, take, unique, zip, +}; + +// Numbers +pub use numbers::{ + clamp, format_bytes, format_currency, format_number, format_percent, is_even, is_odd, lerp, + parse_f64, parse_i64, percentage, round_to, safe_div, +}; + +// Async +pub use async_utils::{ + join_all, join_all_limited, simple_retry, sleep, sleep_ms, sleep_secs, spawn, spawn_blocking, + timeout_ms, timeout_secs, with_timeout, Debouncer, +}; + +// Serde helpers +pub use serde_helpers::{ + default_empty_string, default_empty_vec, default_false, default_true, default_zero, +}; + +// Text processing +pub use text::{ + capitalize, highlight, lorem_ipsum, normalize_whitespace, remove_accents, to_camel_case, + to_constant_case, to_kebab_case, to_pascal_case, to_snake_case, truncate_words, word_count, +}; + +// Conversions +pub use convert::{ + bool_to_str, bytes_to_hex, empty_to_none, hex_to_bytes, i64_to_usize, ms_to_human, + none_to_empty, parse_or, seconds_to_human, to_bool, try_parse, +}; + +// Result/Option extensions +pub use result_ext::{err, flatten_option, flatten_result, ok, some, OptionExt, ResultExt2}; + +// HTTP Request helpers +pub use request::{ + accepts_gzip, bearer_token, client_ip, content_type, header_value, is_form, is_json, origin, + referer, request_id, user_agent, +}; + +// Environment +pub use env::{ + database_url, get_or as env_get_or, host, is_debug, is_development, is_production, load_dotenv, + port, redis_url, require as env_require, +}; + +// UUID +pub use uuid_utils::{ + is_valid as is_valid_uuid_format, medium_id, new_v4 as uuid_v4, new_v4_simple as uuid_simple, + short_id, +}; diff --git a/src/shared/utils/web/http.rs b/src/shared/utils/web/http.rs new file mode 100644 index 0000000..7a66b34 --- /dev/null +++ b/src/shared/utils/web/http.rs @@ -0,0 +1,26 @@ +use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT}; + +pub fn common_headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert(USER_AGENT, HeaderValue::from_static("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")); + headers.insert("Referer", HeaderValue::from_static("https://google.com")); + headers +} + +pub fn common_image_headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert(USER_AGENT, HeaderValue::from_static("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")); + headers.insert( + "Accept", + HeaderValue::from_static( + "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", + ), + ); + headers +} + +pub fn is_internet_baik_block_page(content: &str) -> bool { + content.contains("Internet Baik") + || content.contains("TrustPositif") + || content.contains("Mercusuar") +} diff --git a/src/shared/utils/web/http_client.rs b/src/shared/utils/web/http_client.rs new file mode 100644 index 0000000..ac460d7 --- /dev/null +++ b/src/shared/utils/web/http_client.rs @@ -0,0 +1,135 @@ +//! HTTP client wrapper with common configurations. + +use reqwest::{Client, ClientBuilder, Response}; +use std::time::Duration; +use tracing::debug; + +/// Pre-configured HTTP client with sensible defaults. +#[derive(Clone)] +pub struct HttpClient { + inner: Client, +} + +impl HttpClient { + /// Create a new HTTP client with default settings. + pub fn new() -> Result { + let client = ClientBuilder::new() + .timeout(Duration::from_secs(30)) + .connect_timeout(Duration::from_secs(10)) + .pool_max_idle_per_host(20) + .pool_idle_timeout(Duration::from_secs(60)) + .tcp_nodelay(true) + .user_agent("Scraper/1.0") + .build() + .map_err(|e| format!("Failed to build HTTP client: {}", e))?; + + Ok(Self { inner: client }) + } + + /// Create with custom timeout. + pub fn with_timeout(timeout_secs: u64) -> Result { + let client = ClientBuilder::new() + .timeout(Duration::from_secs(timeout_secs)) + .connect_timeout(Duration::from_secs(10)) + .user_agent("Scraper/1.0") + .build() + .map_err(|e| format!("Failed to build HTTP client: {}", e))?; + + Ok(Self { inner: client }) + } + + /// GET request. + pub async fn get(&self, url: &str) -> reqwest::Result { + debug!("GET {}", url); + self.inner.get(url).send().await + } + + /// GET request and return text. + pub async fn get_text(&self, url: &str) -> reqwest::Result { + self.get(url).await?.text().await + } + + /// GET request and parse JSON. + pub async fn get_json(&self, url: &str) -> reqwest::Result { + self.get(url).await?.json().await + } + + /// POST request with JSON body. + pub async fn post_json( + &self, + url: &str, + body: &T, + ) -> reqwest::Result { + debug!("POST {}", url); + self.inner.post(url).json(body).send().await + } + + /// PUT request with JSON body. + pub async fn put_json( + &self, + url: &str, + body: &T, + ) -> reqwest::Result { + debug!("PUT {}", url); + self.inner.put(url).json(body).send().await + } + + /// DELETE request. + pub async fn delete(&self, url: &str) -> reqwest::Result { + debug!("DELETE {}", url); + self.inner.delete(url).send().await + } + + /// Get the underlying reqwest client. + pub fn client(&self) -> &Client { + &self.inner + } +} + +impl Default for HttpClient { + fn default() -> Self { + Self::new().expect("Valid HTTP client configuration") + } +} + +use once_cell::sync::Lazy; +use std::sync::Arc; + +static HTTP_CLIENT_INIT: Lazy, String>> = Lazy::new(|| { + HttpClient::new() + .map(|c| Arc::new(c)) + .map_err(|e| format!("Failed to initialize HTTP client: {}", e)) +}); + +static HTTP_CLIENT_FAST_INIT: Lazy, String>> = Lazy::new(|| { + HttpClient::with_timeout(10) + .map(|c| Arc::new(c)) + .map_err(|e| format!("Failed to initialize fast HTTP client: {}", e)) +}); + +static HTTP_CLIENT_SLOW_INIT: Lazy, String>> = Lazy::new(|| { + HttpClient::with_timeout(60) + .map(|c| Arc::new(c)) + .map_err(|e| format!("Failed to initialize slow HTTP client: {}", e)) +}); + +/// Global HTTP client instance (30s timeout - general purpose). +pub fn http_client() -> &'static HttpClient { + HTTP_CLIENT_INIT + .as_ref() + .expect("HTTP client initialization failed") +} + +/// Get the fast HTTP client (10s timeout). +pub fn http_client_fast() -> &'static HttpClient { + HTTP_CLIENT_FAST_INIT + .as_ref() + .expect("Fast HTTP client initialization failed") +} + +/// Get the slow HTTP client (60s timeout). +pub fn http_client_slow() -> &'static HttpClient { + HTTP_CLIENT_SLOW_INIT + .as_ref() + .expect("Slow HTTP client initialization failed") +} diff --git a/src/shared/utils/web/mod.rs b/src/shared/utils/web/mod.rs new file mode 100644 index 0000000..0aecb1d --- /dev/null +++ b/src/shared/utils/web/mod.rs @@ -0,0 +1,8 @@ +pub mod http; +pub mod http_client; +pub mod proxy_fetch; +pub mod query; +pub mod request; +pub mod scraping; +pub mod scraping_urls; +pub mod url; diff --git a/src/shared/utils/web/proxy_fetch.rs b/src/shared/utils/web/proxy_fetch.rs new file mode 100644 index 0000000..b656fe6 --- /dev/null +++ b/src/shared/utils/web/proxy_fetch.rs @@ -0,0 +1,399 @@ +// Proxy fetch logic with Redis cache AND Request Coalescing (SingleFlight) +// Updated for sync Redis API, reqwest API changes, and concurrency optimization. + +use dashmap::DashMap; +use once_cell::sync::Lazy; +use redis::AsyncCommands; +use tokio::sync::broadcast; +use tracing::{debug, error, warn}; + +use crate::shared::database::get_redis_conn; +use crate::shared::errors::AppError; +use crate::shared::utils::cache_ttl::CACHE_TTL_VERY_SHORT; +use crate::shared::utils::http::common_headers; +use crate::shared::utils::http::is_internet_baik_block_page; +use crate::shared::utils::web::http_client::http_client; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct FetchResult { + pub data: String, + pub content_type: Option, +} + +// Implement Display to allow .to_string() +impl std::fmt::Display for FetchResult { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "FetchResult {{ data: {}, content_type: {} }}", + self.data, + self.content_type.as_deref().unwrap_or("None") + ) + } +} + +// Global In-Flight Request Map for Request Coalescing +// Maps URL slug -> Broadcast Sender +static IN_FLIGHT: Lazy>>> = + Lazy::new(DashMap::new); + +// Global Blacklist for domains that consistently fail direct fetch (Timeouts, SSL, Cloudflare blocks) +static FAILED_DOMAINS: Lazy> = Lazy::new(dashmap::DashSet::new); + +const RELAY_ENDPOINTS: &[&str] = &[ + "https://opennext-app.superaseph.workers.dev", + "https://proxy-bun.vercel.app", + "https://proxy-bun-mytheclipse8647-orfq73fe.apn.leapcell.dev", +]; + +// --- REDIS CACHE WRAPPER START --- +fn get_fetch_cache_key(slug: &str) -> String { + format!("fetch:proxy:{slug}") +} + +async fn get_cached_fetch(slug: &str) -> Result, AppError> { + let mut conn = get_redis_conn().await?; + let key = get_fetch_cache_key(slug); + + let cached: Option = conn.get(&key).await?; + + if let Some(cached_str) = cached { + match serde_json::from_str::(&cached_str) { + Ok(parsed) => { + debug!("[fetchWithProxy] Returning cached response for {}", slug); + Ok(Some(parsed)) + } + Err(_) => Ok(None), + } + } else { + Ok(None) + } +} + +async fn set_cached_fetch(slug: &str, value: &FetchResult) -> Result<(), AppError> { + let mut conn = get_redis_conn().await?; + let key = get_fetch_cache_key(slug); + let json_string = serde_json::to_string(value)?; + + // Use standardized TTL + conn.set_ex::<_, _, ()>(&key, &json_string, CACHE_TTL_VERY_SHORT) + .await?; + Ok(()) +} +// --- REDIS CACHE WRAPPER END --- + +/// Main entry point: Fetches with proxy, using Cache and Request Coalescing +pub async fn fetch_with_proxy(slug: &str) -> Result { + // 1. Try Cache First + if let Ok(Some(cached)) = get_cached_fetch(slug).await { + return Ok(cached); + } + + // 2. Request Coalescing (SingleFlight) + // Check if there is already an in-flight request for this slug + let tx = { + if let Some(in_flight) = IN_FLIGHT.get(slug) { + debug!("[Coalesce] Joining in-flight request for {}", slug); + in_flight.value().clone() + } else { + // No in-flight request, create a new channel + let (tx, _) = broadcast::channel(1); // Capacity 1 is enough for single result + IN_FLIGHT.insert(slug.to_string(), tx.clone()); + debug!("[Coalesce] Starting leader request for {}", slug); + + // We are the leader, we must execute the fetch + // Spawn the fetch task so we don't block holding the map lock (though insert is fast) + // But actually we are not holding the lock here anymore. + + // Clone for the async block + let slug_clone = slug.to_string(); + let tx_clone = tx.clone(); + + tokio::spawn(async move { + // RAII Guard: Guarantee slug eviction exactly once the task finishes or panics! + struct DropGuard(String); + impl Drop for DropGuard { + fn drop(&mut self) { + IN_FLIGHT.remove(&self.0); + } + } + let _guard = DropGuard(slug_clone.clone()); + + let result = perform_fetch(&slug_clone).await; + + // Map AppError to String for broadcast (since AppError might not be Clone) + // FetchResult is Clone. + let broadcast_result = match &result { + Ok(res) => Ok(res.clone()), + Err(e) => Err(e.to_string()), + }; + + // Broadcast result to all waiting subscribers + let _ = tx_clone.send(broadcast_result); + }); + + tx + } + }; + + // 3. Wait for result (Leader or Follower) + let mut rx = tx.subscribe(); + match rx.recv().await { + Ok(Ok(res)) => Ok(res), + Ok(Err(e_str)) => Err(AppError::Other(e_str)), + Err(e) => { + warn!("[Coalesce] Receive mismatch for {}: {:?}", slug, e); + Err(AppError::Other("Request coalescing error".to_string())) + } + } +} + +/// The actual fetch logic (Direct -> Retry -> Proxy) +async fn perform_fetch(slug: &str) -> Result { + // 1. Extract domain for Circuit Breaker logic + let domain = reqwest::Url::parse(slug) + .ok() + .and_then(|u| u.host_str().map(|s| s.to_string())) + .unwrap_or_default(); + + // 2. Immediate proxy fallback if domain has a known history of brutal timeouts/blocks + if !domain.is_empty() && FAILED_DOMAINS.contains(&domain) { + warn!( + "[Circuit Breaker] Domain {} is blacklisted from direct-fetch. Routing via Relays.", + domain + ); + return perform_proxy_chain(slug).await; + } + + // Use shared global HTTP client + let client = http_client().client(); + let headers = common_headers(); + + match client + .get(slug) + .headers(headers) + .send() // Timeout handled by client + .await + { + Ok(res) => { + debug!( + "[fetchWithProxy] Direct fetch response: url={}, status={}", + slug, + res.status() + ); + if res.status().is_success() { + let content_type = res + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|h| h.to_str().ok()) + .map(|s| s.to_string()); + + let bytes = res.bytes().await?; + + // Check if response is Gzip compressed (magic header 1f 8b) + let text_data = if bytes.len() > 2 && bytes[0] == 0x1f && bytes[1] == 0x8b { + // Gzip compressed, offload decompression to blocking thread + let decompressed = tokio::task::spawn_blocking(move || { + use flate2::read::GzDecoder; + use std::io::Read; + let decoder = GzDecoder::new(&bytes[..]); + let mut decompressed = Vec::new(); + // 10MB absolute decompression bounds to prevent GZIP Bombs (OOM vulnerability) + decoder + .take(10_000_000) + .read_to_end(&mut decompressed) + .map(|_| decompressed) + .map_err(|e| { + AppError::Other(format!( + "Decompression failed or exceeded limits: {:?}", + e + )) + }) + }) + .await??; + + match std::str::from_utf8(&decompressed) { + Ok(s) => s.to_string(), + Err(_) => String::from_utf8_lossy(&decompressed).to_string(), + } + } else { + match std::str::from_utf8(&bytes) { + Ok(s) => s.to_string(), + Err(_) => { + warn!("Response bytes are not valid UTF-8, using lossy conversion"); + String::from_utf8_lossy(&bytes).to_string() + } + } + }; + + if is_internet_baik_block_page(&text_data) { + warn!("Blocked by internetbaik (direct fetch) for {}", slug); + return perform_proxy_chain(slug).await; + } else { + let result = FetchResult { + data: text_data, + content_type, + }; + // Cache the success result + if let Err(e) = set_cached_fetch(slug, &result).await { + warn!("Failed to cache result for {}: {:?}", slug, e); + } + Ok(result) + } + } else { + let error_msg = format!( + "Direct fetch failed with status {} for {}", + res.status(), + slug + ); + + // Penalize domain if it throws aggressive Anti-Bot or Gateway Timeout codes + if res.status().is_server_error() + || res.status() == reqwest::StatusCode::FORBIDDEN + || res.status() == reqwest::StatusCode::SERVICE_UNAVAILABLE + { + if !domain.is_empty() { + warn!("[Circuit Breaker] Blacklisting domain {} due to hostile HTTP status {}", domain, res.status()); + FAILED_DOMAINS.insert(domain.clone()); + } + error!("{}", error_msg); + return perform_proxy_chain(slug).await; + } else { + warn!("{}", error_msg); + } + Err(AppError::Other(error_msg)) + } + } + Err(e) => { + let error_msg = format!("Direct fetch failed for {}: {:?}", slug, e); + warn!("{}", error_msg); + + // Hard panic mapping: If reqwest core network/TLS fails, instantly blacklist the domain + if !domain.is_empty() { + warn!("[Circuit Breaker] Blacklisting domain {} due to core network/SSL trace failure", domain); + FAILED_DOMAINS.insert(domain.clone()); + } + + // Fall back seamlessly to Relay network proxy instead of throwing fatal transient backoff + perform_proxy_chain(slug).await + } + } +} + +pub async fn fetch_with_proxy_only(slug: &str) -> Result { + if let Ok(Some(cached)) = get_cached_fetch(slug).await { + return Ok(cached); + } + + perform_proxy_chain(slug).await +} + +async fn perform_proxy_chain(slug: &str) -> Result { + // 1. Try Relays + match fetch_via_relays(slug).await { + Ok(res) => return Ok(res), + Err(e) => warn!("[ProxyChain] All relays failed for {}: {:?}", slug, e), + } + + // 2. Try Browserless as absolute last resort + match fetch_via_browserless(slug).await { + Ok(res) => Ok(res), + Err(e) => { + error!( + "[ProxyChain] Browserless fallback failed for {}: {:?}", + slug, e + ); + Err(e) + } + } +} + +async fn fetch_via_relays(slug: &str) -> Result { + // Use shared client + let client = http_client().client(); + + for relay in RELAY_ENDPOINTS { + debug!( + "[fetch_via_relays] Attempting to fetch {} via relay {}", + slug, relay + ); + + match client + .get(*relay) + .header("x-relay-target", slug) + .send() + .await + { + Ok(res) if res.status().is_success() => { + let content_type = res + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|h| h.to_str().ok()) + .map(|s| s.to_string()); + let data = res.text().await?; + + let result = FetchResult { data, content_type }; + debug!( + "[fetch_via_relays] Successfully fetched {} via relay {}", + slug, relay + ); + + if let Err(e) = set_cached_fetch(slug, &result).await { + warn!("Failed to cache relay result for {}: {:?}", slug, e); + } + + return Ok(result); + } + Ok(res) => { + warn!( + "[fetch_via_relays] Relay {} returned status {} for {}", + relay, + res.status(), + slug + ); + } + Err(e) => { + warn!( + "[fetch_via_relays] Relay {} failed for {}: {:?}", + relay, slug, e + ); + } + } + } + + Err(AppError::Other("All relay endpoints failed".to_string())) +} + +async fn fetch_via_browserless(slug: &str) -> Result { + use crate::shared::browser::pool::get_browser_pool; + + warn!("[Browserless] Falling back to remote browser for {}", slug); + + let pool = get_browser_pool() + .ok_or_else(|| AppError::Other("Browser pool not initialized".to_string()))?; + + let tab = pool + .get_tab() + .await + .map_err(|e| AppError::Other(format!("Failed to get browser tab: {:?}", e)))?; + + tab.goto(slug) + .await + .map_err(|e| AppError::Other(format!("Browser navigation failed for {}: {:?}", slug, e)))?; + + let data = tab + .content() + .await + .map_err(|e| AppError::Other(format!("Failed to get browser content: {:?}", e)))?; + + let result = FetchResult { + data, + content_type: Some("text/html".to_string()), + }; + + if let Err(e) = set_cached_fetch(slug, &result).await { + warn!("Failed to cache browserless result for {}: {:?}", slug, e); + } + + Ok(result) +} diff --git a/src/shared/utils/web/query.rs b/src/shared/utils/web/query.rs new file mode 100644 index 0000000..b4a33c4 --- /dev/null +++ b/src/shared/utils/web/query.rs @@ -0,0 +1,278 @@ +//! Query builder helpers for SeaORM. +//! +//! Provides utilities for pagination, filtering, and sorting queries. +//! +//! # Example +//! +//! ```ignore +//! use scraper_service::helpers::query::{QueryBuilder, Pagination, SortOrder}; +//! +//! let query = User::find() +//! .apply_pagination(Pagination::new(1, 20)) +//! .apply_sort("created_at", SortOrder::Desc); +//! ``` + +use sea_orm::{entity::prelude::*, Order, QuerySelect, Select}; +use serde::{Deserialize, Serialize}; + +/// Pagination parameters. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Pagination { + /// Current page (1-indexed). + pub page: u64, + /// Items per page. + pub per_page: u64, + /// Total items (optional, set after query). + #[serde(skip_deserializing)] + pub total: Option, +} + +impl Pagination { + /// Create new pagination. + pub fn new(page: u64, per_page: u64) -> Self { + Self { + page: page.max(1), + per_page: per_page.clamp(1, 100), + total: None, + } + } + + /// Default pagination (page 1, 20 per page). + pub fn default_page() -> Self { + Self::new(1, 20) + } + + /// Calculate offset for query. + pub fn offset(&self) -> u64 { + (self.page - 1) * self.per_page + } + + /// Calculate total pages. + pub fn total_pages(&self) -> u64 { + match self.total { + Some(total) => (total as f64 / self.per_page as f64).ceil() as u64, + None => 0, + } + } + + /// Check if there's a next page. + pub fn has_next(&self) -> bool { + self.page < self.total_pages() + } + + /// Check if there's a previous page. + pub fn has_prev(&self) -> bool { + self.page > 1 + } + + /// Set total after counting. + pub fn with_total(mut self, total: u64) -> Self { + self.total = Some(total); + self + } +} + +impl Default for Pagination { + fn default() -> Self { + Self::default_page() + } +} + +/// Sort order. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum SortOrder { + Asc, + Desc, +} + +impl From for Order { + fn from(order: SortOrder) -> Self { + match order { + SortOrder::Asc => Order::Asc, + SortOrder::Desc => Order::Desc, + } + } +} + +/// Sort parameter. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Sort { + pub field: String, + pub order: SortOrder, +} + +impl Sort { + pub fn new(field: &str, order: SortOrder) -> Self { + Self { + field: field.to_string(), + order, + } + } + + pub fn asc(field: &str) -> Self { + Self::new(field, SortOrder::Asc) + } + + pub fn desc(field: &str) -> Self { + Self::new(field, SortOrder::Desc) + } +} + +/// Filter operator. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum FilterOp { + Eq, + Ne, + Gt, + Gte, + Lt, + Lte, + Like, + In, + IsNull, + IsNotNull, +} + +/// Filter parameter. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Filter { + pub field: String, + pub op: FilterOp, + pub value: serde_json::Value, +} + +impl Filter { + pub fn eq(field: &str, value: impl Into) -> Self { + Self { + field: field.to_string(), + op: FilterOp::Eq, + value: value.into(), + } + } + + pub fn ne(field: &str, value: impl Into) -> Self { + Self { + field: field.to_string(), + op: FilterOp::Ne, + value: value.into(), + } + } + + pub fn like(field: &str, value: &str) -> Self { + Self { + field: field.to_string(), + op: FilterOp::Like, + value: serde_json::Value::String(value.to_string()), + } + } + + pub fn is_null(field: &str) -> Self { + Self { + field: field.to_string(), + op: FilterOp::IsNull, + value: serde_json::Value::Null, + } + } + + pub fn is_not_null(field: &str) -> Self { + Self { + field: field.to_string(), + op: FilterOp::IsNotNull, + value: serde_json::Value::Null, + } + } +} + +/// Query parameters combining pagination, sort, and filters. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct QueryParams { + #[serde(default)] + pub pagination: Pagination, + #[serde(default)] + pub sort: Option, + #[serde(default)] + pub filters: Vec, + #[serde(default)] + pub search: Option, +} + +impl QueryParams { + pub fn new() -> Self { + Self::default() + } + + pub fn page(mut self, page: u64) -> Self { + self.pagination.page = page.max(1); + self + } + + pub fn per_page(mut self, per_page: u64) -> Self { + self.pagination.per_page = per_page.clamp(1, 100); + self + } + + pub fn sort_by(mut self, field: &str, order: SortOrder) -> Self { + self.sort = Some(Sort::new(field, order)); + self + } + + pub fn filter(mut self, filter: Filter) -> Self { + self.filters.push(filter); + self + } + + pub fn search(mut self, query: &str) -> Self { + self.search = Some(query.to_string()); + self + } +} + +/// Paginated result. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PaginatedResult { + pub data: Vec, + pub pagination: PaginationMeta, +} + +/// Pagination metadata for response. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PaginationMeta { + pub page: u64, + pub per_page: u64, + pub total: u64, + pub total_pages: u64, + pub has_next: bool, + pub has_prev: bool, +} + +impl PaginatedResult { + pub fn new(data: Vec, pagination: Pagination) -> Self { + let total = pagination.total.unwrap_or(0); + let total_pages = pagination.total_pages(); + + Self { + data, + pagination: PaginationMeta { + page: pagination.page, + per_page: pagination.per_page, + total, + total_pages, + has_next: pagination.has_next(), + has_prev: pagination.has_prev(), + }, + } + } +} + +/// Extension trait for applying pagination to queries. +pub trait PaginateExt { + fn apply_pagination(self, pagination: &Pagination) -> Self; +} + +impl PaginateExt for Select { + fn apply_pagination(self, pagination: &Pagination) -> Self { + self.offset(pagination.offset()).limit(pagination.per_page) + } +} diff --git a/src/shared/utils/web/request.rs b/src/shared/utils/web/request.rs new file mode 100644 index 0000000..4b68c89 --- /dev/null +++ b/src/shared/utils/web/request.rs @@ -0,0 +1,141 @@ +//! HTTP request helpers. + +use axum::http::{HeaderMap, HeaderValue}; + +/// Extract client IP from headers (X-Forwarded-For, X-Real-IP). +pub fn client_ip(headers: &HeaderMap) -> Option { + // Try X-Forwarded-For first + if let Some(forwarded) = headers.get("x-forwarded-for") { + if let Ok(value) = forwarded.to_str() { + if let Some(ip) = value.split(',').next() { + return Some(ip.trim().to_string()); + } + } + } + + // Try X-Real-IP + if let Some(real_ip) = headers.get("x-real-ip") { + if let Ok(value) = real_ip.to_str() { + return Some(value.to_string()); + } + } + + // Try CF-Connecting-IP (Cloudflare) + if let Some(cf_ip) = headers.get("cf-connecting-ip") { + if let Ok(value) = cf_ip.to_str() { + return Some(value.to_string()); + } + } + + None +} + +/// Extract User-Agent from headers. +pub fn user_agent(headers: &HeaderMap) -> Option { + headers + .get("user-agent") + .and_then(|v| v.to_str().ok()) + .map(String::from) +} + +/// Extract Accept-Language from headers. +pub fn accept_language(headers: &HeaderMap) -> Option { + headers + .get("accept-language") + .and_then(|v| v.to_str().ok()) + .map(String::from) +} + +/// Extract Authorization bearer token. +pub fn bearer_token(headers: &HeaderMap) -> Option { + headers + .get("authorization") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.strip_prefix("Bearer ")) + .map(String::from) +} + +/// Extract content type. +pub fn content_type(headers: &HeaderMap) -> Option { + headers + .get("content-type") + .and_then(|v| v.to_str().ok()) + .map(String::from) +} + +/// Check if request is JSON. +pub fn is_json(headers: &HeaderMap) -> bool { + content_type(headers) + .map(|ct| ct.contains("application/json")) + .unwrap_or(false) +} + +/// Check if request is form data. +pub fn is_form(headers: &HeaderMap) -> bool { + content_type(headers) + .map(|ct| { + ct.contains("application/x-www-form-urlencoded") || ct.contains("multipart/form-data") + }) + .unwrap_or(false) +} + +/// Extract referer. +pub fn referer(headers: &HeaderMap) -> Option { + headers + .get("referer") + .and_then(|v| v.to_str().ok()) + .map(String::from) +} + +/// Extract origin. +pub fn origin(headers: &HeaderMap) -> Option { + headers + .get("origin") + .and_then(|v| v.to_str().ok()) + .map(String::from) +} + +/// Extract request ID. +pub fn request_id(headers: &HeaderMap) -> Option { + headers + .get("x-request-id") + .or_else(|| headers.get("x-correlation-id")) + .and_then(|v| v.to_str().ok()) + .map(String::from) +} + +/// Check if request accepts gzip. +pub fn accepts_gzip(headers: &HeaderMap) -> bool { + headers + .get("accept-encoding") + .and_then(|v| v.to_str().ok()) + .map(|v| v.contains("gzip")) + .unwrap_or(false) +} + +/// Create header value. +pub fn header_value(s: &str) -> HeaderValue { + HeaderValue::from_str(s).unwrap_or_else(|_| HeaderValue::from_static("")) +} + +/// Parse quality value from Accept header (e.g., "text/html;q=0.9"). +pub fn parse_accept_quality(accept: &str) -> Vec<(String, f32)> { + accept + .split(',') + .filter_map(|part| { + let mut parts = part.trim().split(';'); + let mime = parts.next()?.trim().to_string(); + let quality = parts + .find_map(|p| { + let p = p.trim(); + if p.starts_with("q=") { + p[2..].parse().ok() + } else { + None + } + }) + .unwrap_or(1.0); + Some((mime, quality)) + }) + .collect() +} diff --git a/src/shared/utils/web/scraping.rs b/src/shared/utils/web/scraping.rs new file mode 100644 index 0000000..ac6a47a --- /dev/null +++ b/src/shared/utils/web/scraping.rs @@ -0,0 +1,189 @@ +//! HTML scraping helpers using scraper crate. + +use crate::shared::errors::AppError; +use crate::shared::utils::web::proxy_fetch::fetch_with_proxy; +use crate::shared::utils::{default_backoff, transient}; +use backoff::future::retry; +use once_cell::sync::Lazy; +use regex::Regex; +use scraper::{ElementRef, Html, Selector}; +use tracing::{info, warn}; + +/// Fetch HTML from URL with retry backoff and proxy support. +pub async fn fetch_html_with_retry(url: &str) -> Result { + let backoff = default_backoff(); + let fetch_operation = || async { + info!("Fetching: {}", url); + match fetch_with_proxy(url).await { + Ok(response) => { + info!("Successfully fetched: {}", url); + Ok(response.data) + } + Err(e) => { + warn!("Failed to fetch: {}, error: {:?}", url, e); + Err(transient(e)) + } + } + }; + + retry(backoff, fetch_operation) + .await + .map_err(|e| AppError::ScraperError(e.to_string())) +} + +/// Parse HTML string into a document. +pub fn parse_html(html: &str) -> Html { + Html::parse_document(html) +} + +/// Safely create a CSS selector. +pub fn selector(css: &str) -> Option { + Selector::parse(css).ok() +} + +/// Extract text content from an element, trimmed. +pub fn text(element: &ElementRef) -> String { + element.text().collect::().trim().to_string() +} + +/// Extract text from first matching element. +pub fn select_text(element: &ElementRef, css: &str) -> Option { + let sel = selector(css)?; + element.select(&sel).next().map(|e| text(&e)) +} + +/// Extract text from first matching element using a pre-compiled selector. +pub fn text_from(element: &ElementRef, selector: &Selector) -> Option { + element.select(selector).next().map(|e| text(&e)) +} + +/// Extract text from first matching element using a pre-compiled selector, or return default. +pub fn text_from_or(element: &ElementRef, selector: &Selector, default: &str) -> String { + text_from(element, selector).unwrap_or_else(|| default.to_string()) +} + +/// Extract attribute from first matching element. +pub fn select_attr(element: &ElementRef, css: &str, attr: &str) -> Option { + let sel = selector(css)?; + element + .select(&sel) + .next() + .and_then(|e| e.value().attr(attr)) + .map(String::from) +} + +/// Extract attribute from first matching element using a pre-compiled selector. +pub fn attr_from(element: &ElementRef, selector: &Selector, attr: &str) -> Option { + element + .select(selector) + .next() + .and_then(|e| e.value().attr(attr)) + .map(String::from) +} + +/// Extract attribute from first matching element using a pre-compiled selector, or return default. +pub fn attr_from_or( + element: &ElementRef, + selector: &Selector, + attr: &str, + default: &str, +) -> String { + attr_from(element, selector, attr).unwrap_or_else(|| default.to_string()) +} + +/// Extract attribute from element. +pub fn attr(element: &ElementRef, name: &str) -> Option { + element.value().attr(name).map(String::from) +} + +/// Select all matching elements. +pub fn select_all<'a>(document: &'a Html, css: &str) -> Vec> { + selector(css) + .map(|s| document.select(&s).collect()) + .unwrap_or_default() +} + +/// Extract slug from URL (last path segment). +pub fn extract_slug(url: &str) -> String { + static SLUG_REGEX: Lazy> = Lazy::new(|| Regex::new(r"/([^/]+)/?$")); + + SLUG_REGEX + .as_ref() + .ok() + .and_then(|r| r.captures(url)) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str().to_string()) + .unwrap_or_default() +} + +/// Remove HTML tags from string. +pub fn strip_tags(html: &str) -> String { + static TAG_REGEX: Lazy> = Lazy::new(|| Regex::new(r"<[^>]+>")); + TAG_REGEX + .as_ref() + .map(|r| r.replace_all(html, "").trim().to_string()) + .unwrap_or_else(|_| html.to_string()) +} + +/// Extract number from text. +pub fn extract_number(text: &str) -> Option { + static NUM_REGEX: Lazy> = Lazy::new(|| Regex::new(r"\d+")); + NUM_REGEX + .as_ref() + .ok() + .and_then(|r| r.find(text)) + .and_then(|m| m.as_str().parse().ok()) +} + +/// Extract text inside parentheses. +pub fn extract_parentheses(text: &str) -> Option { + static PAREN_REGEX: Lazy> = + Lazy::new(|| Regex::new(r"\(([^)]+)\)")); + PAREN_REGEX + .as_ref() + .ok() + .and_then(|r| r.captures(text)) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str().to_string()) +} + +/// Builder for scraping elements. +pub struct Scraper<'a> { + element: ElementRef<'a>, +} + +impl<'a> Scraper<'a> { + pub fn new(element: ElementRef<'a>) -> Self { + Self { element } + } + + /// Get text from selector. + pub fn text(&self, css: &str) -> Option { + select_text(&self.element, css) + } + + /// Get text or default. + pub fn text_or(&self, css: &str, default: &str) -> String { + self.text(css).unwrap_or_else(|| default.to_string()) + } + + /// Get attribute from selector. + pub fn attr(&self, css: &str, name: &str) -> Option { + select_attr(&self.element, css, name) + } + + /// Get attribute or default. + pub fn attr_or(&self, css: &str, name: &str, default: &str) -> String { + self.attr(css, name).unwrap_or_else(|| default.to_string()) + } + + /// Get href from first link. + pub fn href(&self, css: &str) -> Option { + self.attr(css, "href") + } + + /// Get src from first image. + pub fn src(&self, css: &str) -> Option { + self.attr(css, "src") + } +} diff --git a/src/shared/utils/web/scraping_urls.rs b/src/shared/utils/web/scraping_urls.rs new file mode 100644 index 0000000..4571ef9 --- /dev/null +++ b/src/shared/utils/web/scraping_urls.rs @@ -0,0 +1,30 @@ +//! URL constants and dynamic environment-based URLs. +//! +//! Note: These URLs are kept as dynamic env lookups because they may vary +//! between deployments and are not critical startup dependencies. + +use crate::shared::config::CONFIG; +use std::env; + +pub const BASE_URL: &str = "http://127.0.0.1:4090"; +pub const OTAKUDESU_BASE_URL: &str = "https://otakudesu.blog/"; + +/// Get Komik URL from environment config. +pub fn get_komik_url() -> String { + env::var("KOMIK2_BASE_URL").unwrap_or_else(|_| "https://komiku.org".to_string()) +} + +/// Get production URL from environment config. +pub fn get_production_url() -> String { + env::var("NEXT_PUBLIC_PROD").unwrap_or_else(|_| CONFIG.urls.site_url.clone()) +} + +/// Get Komik API URL from environment config. +pub fn get_komik_api_url() -> String { + env::var("KOMIK2_API_URL").unwrap_or_else(|_| "https://api.komiku.org".to_string()) +} + +/// Get Otakudesu URL from environment config. +pub fn get_otakudesu_url() -> String { + env::var("OTAKUDESU_BASE_URL").unwrap_or_else(|_| OTAKUDESU_BASE_URL.to_string()) +} diff --git a/src/shared/utils/web/url.rs b/src/shared/utils/web/url.rs new file mode 100644 index 0000000..cce7c86 --- /dev/null +++ b/src/shared/utils/web/url.rs @@ -0,0 +1,159 @@ +//! URL building and manipulation utilities. + +use std::collections::HashMap; + +/// URL builder for constructing URLs with query parameters. +#[derive(Debug, Clone)] +pub struct UrlBuilder { + base: String, + path_segments: Vec, + query_params: Vec<(String, String)>, +} + +impl UrlBuilder { + /// Create a new URL builder with base URL. + pub fn new(base: impl Into) -> Self { + let base = base.into(); + let base = base.trim_end_matches('/').to_string(); + Self { + base, + path_segments: Vec::new(), + query_params: Vec::new(), + } + } + + /// Add a path segment. + pub fn path(mut self, segment: impl Into) -> Self { + self.path_segments.push(segment.into()); + self + } + + /// Add multiple path segments. + pub fn paths(mut self, segments: &[&str]) -> Self { + for seg in segments { + self.path_segments.push(seg.to_string()); + } + self + } + + /// Add a query parameter. + pub fn query(mut self, key: impl Into, value: impl Into) -> Self { + self.query_params.push((key.into(), value.into())); + self + } + + /// Add optional query parameter (only if Some). + pub fn query_opt(self, key: impl Into, value: Option>) -> Self { + match value { + Some(v) => self.query(key, v), + None => self, + } + } + + /// Add multiple query parameters from HashMap. + pub fn query_map(mut self, params: HashMap) -> Self { + for (k, v) in params { + self.query_params.push((k, v)); + } + self + } + + /// Build the final URL string. + pub fn build(self) -> String { + let mut url = self.base; + + // Add path segments + for segment in self.path_segments { + url.push('/'); + url.push_str(&urlencoding::encode(&segment)); + } + + // Add query params + if !self.query_params.is_empty() { + url.push('?'); + let params: Vec = self + .query_params + .into_iter() + .map(|(k, v)| format!("{}={}", urlencoding::encode(&k), urlencoding::encode(&v))) + .collect(); + url.push_str(¶ms.join("&")); + } + + url + } +} + +/// Encode a string for use in URL. +pub fn encode(s: &str) -> String { + urlencoding::encode(s).to_string() +} + +/// Decode a URL-encoded string. +pub fn decode(s: &str) -> Result { + urlencoding::decode(s).map(|s| s.to_string()) +} + +/// Parse query string into HashMap. +pub fn parse_query(query: &str) -> HashMap { + let query = query.trim_start_matches('?'); + query + .split('&') + .filter_map(|pair| { + let mut parts = pair.splitn(2, '='); + let key = parts.next()?; + let value = parts.next().unwrap_or(""); + Some((decode(key).ok()?, decode(value).ok()?)) + }) + .collect() +} + +/// Extract domain from URL. +pub fn extract_domain(url: &str) -> Option { + let url = url + .trim_start_matches("https://") + .trim_start_matches("http://"); + url.split('/').next().map(|s| s.to_string()) +} + +/// Join URL paths safely. +pub fn join_paths(base: &str, path: &str) -> String { + let base = base.trim_end_matches('/'); + let path = path.trim_start_matches('/'); + format!("{}/{}", base, path) +} + +/// Check if URL is absolute. +pub fn is_absolute(url: &str) -> bool { + url.starts_with("http://") || url.starts_with("https://") +} + +/// Make URL absolute. +pub fn make_absolute(url: &str, base: &str) -> String { + if is_absolute(url) { + url.to_string() + } else { + join_paths(base, url) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_url_builder() { + let url = UrlBuilder::new("https://api.example.com") + .path("users") + .path("123") + .query("page", "1") + .query("limit", "10") + .build(); + assert_eq!(url, "https://api.example.com/users/123?page=1&limit=10"); + } + + #[test] + fn test_parse_query() { + let params = parse_query("?name=John&age=30"); + assert_eq!(params.get("name"), Some(&"John".to_string())); + } +} diff --git a/src/shared/utils/web/validation.rs b/src/shared/utils/web/validation.rs new file mode 100644 index 0000000..2802ba0 --- /dev/null +++ b/src/shared/utils/web/validation.rs @@ -0,0 +1,156 @@ +//! Input validation helpers. + +use once_cell::sync::Lazy; +use regex::Regex; + +static EMAIL_REGEX: Lazy> = Lazy::new(|| { + Regex::new(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$") +}); + +static URL_REGEX: Lazy> = Lazy::new(|| { + Regex::new(r"^https?://[^\s/$.?#].[^\s]*$") +}); + +static PHONE_REGEX: Lazy> = Lazy::new(|| { + Regex::new(r"^\+?[1-9]\d{1,14}$") +}); + +static UUID_REGEX: Lazy> = Lazy::new(|| { + Regex::new(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$") +}); + +static SLUG_REGEX: Lazy> = Lazy::new(|| { + Regex::new(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") +}); + +pub fn is_email(s: &str) -> bool { + EMAIL_REGEX.as_ref().map(|r| r.is_match(s)).unwrap_or(false) +} + +pub fn is_url(s: &str) -> bool { + URL_REGEX.as_ref().map(|r| r.is_match(s)).unwrap_or(false) +} + +pub fn is_phone(s: &str) -> bool { + PHONE_REGEX.as_ref().map(|r| r.is_match(s)).unwrap_or(false) +} + +pub fn is_uuid(s: &str) -> bool { + UUID_REGEX.as_ref().map(|r| r.is_match(s)).unwrap_or(false) +} + +pub fn is_slug(s: &str) -> bool { + SLUG_REGEX.as_ref().map(|r| r.is_match(s)).unwrap_or(false) +} + +/// Check if string is not empty. +pub fn is_not_empty(s: &str) -> bool { + !s.trim().is_empty() +} + +/// Check minimum length. +pub fn min_length(s: &str, min: usize) -> bool { + s.len() >= min +} + +/// Check maximum length. +pub fn max_length(s: &str, max: usize) -> bool { + s.len() <= max +} + +/// Check length is within range. +pub fn length_between(s: &str, min: usize, max: usize) -> bool { + s.len() >= min && s.len() <= max +} + +/// Check if string contains only alphanumeric characters. +pub fn is_alphanumeric(s: &str) -> bool { + s.chars().all(|c| c.is_alphanumeric()) +} + +/// Check if string contains only ASCII characters. +pub fn is_ascii(s: &str) -> bool { + s.is_ascii() +} + +/// Check if string is numeric only. +pub fn is_numeric(s: &str) -> bool { + !s.is_empty() && s.chars().all(|c| c.is_numeric()) +} + +/// Check if number is in range. +pub fn in_range(val: T, min: T, max: T) -> bool { + val >= min && val <= max +} + +/// Validate password strength (min 8 chars, has upper, lower, digit). +pub fn is_strong_password(s: &str) -> bool { + s.len() >= 8 + && s.chars().any(|c| c.is_uppercase()) + && s.chars().any(|c| c.is_lowercase()) + && s.chars().any(|c| c.is_numeric()) +} + +/// Validation result builder. +#[derive(Debug, Default)] +pub struct Validator { + errors: Vec, +} + +impl Validator { + pub fn new() -> Self { + Self::default() + } + + /// Add a validation check. + pub fn check(mut self, condition: bool, message: impl Into) -> Self { + if !condition { + self.errors.push(message.into()); + } + self + } + + /// Check if validation passed. + pub fn is_valid(&self) -> bool { + self.errors.is_empty() + } + + /// Get all errors. + pub fn errors(&self) -> &[String] { + &self.errors + } + + /// Get first error. + pub fn first_error(&self) -> Option<&String> { + self.errors.first() + } + + /// Convert to Result. + pub fn validate(self) -> Result<(), String> { + if self.errors.is_empty() { + Ok(()) + } else { + Err(self.errors.join(", ")) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_email() { + assert!(is_email("test@example.com")); + assert!(!is_email("invalid")); + } + + #[test] + fn test_validator() { + let result = Validator::new() + .check(is_email("test@example.com"), "Invalid email") + .check(min_length("password123", 8), "Password too short") + .validate(); + assert!(result.is_ok()); + } +} diff --git a/tools/auto-lint/Cargo.toml b/tools/auto-lint/Cargo.toml new file mode 100644 index 0000000..b018e9f --- /dev/null +++ b/tools/auto-lint/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "rust-lint" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "auto-lint" +path = "auto_lint.rs" + +[dependencies]