MythEclipse/asepharyana-hub-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: MythEclipse/asepharyana-hub-scraper:c035601a366691e3841c11a49f541bae3cb0289f
Branches Tags
MythEclipse/asepharyana-hub-scraper:main
..
compare: MythEclipse/asepharyana-hub-scraper:main
Branches Tags
MythEclipse/asepharyana-hub-scraper:main
10 Commits
c035601a36 .. main
Author SHA1 Message Date
mytheclipsebotreview 62aa5b0e52 fix(config): remove list_separator, breaks string fields
Notify Parent Repo / dispatch (push) Canceled after 0s
2026-07-30 22:18:03 +07:00
asepharyana bc782ae4f8 ci: add workflow_dispatch to notify-parent for manual triggering 2026-07-25 18:13:25 +07:00
asepharyana 7640e66c76 chore: deny unused_mut, unreachable_code, trivial_casts, trivial_numeric_casts, explicit_outlives_requirements, unused_labels, unused_braces, noop_method_call, clippy::unnecessary_to_owned 2026-07-22 15:03:13 +07:00
asepharyana 96ce29b0f3 chore: set dead_code lint to deny 2026-07-22 14:52:19 +07:00
asepharyana 5c51ec895f feat: remove croxy proxy and image cache features 2026-07-22 14:51:08 +07:00
asepharyana 891064354b fix: disable extra_float_digits startup param for PG16 compatibility 2026-07-22 14:23:36 +07:00
asepharyana 4cd04410cf chore: remove 17 unused Rust dependencies from scraper 2026-07-22 14:12:58 +07:00
asepharyana cd4e3aee20 feat: add episode list with download URLs to detail endpoint 
- Add AlqEpisode type with episode, title, url, date, download_url
- Add parse_episode_list() to extract episodes from .eplister
- Add parse_episode_download() to extract download URL from episode page
- Add episodes field to AlqDetailData
- Fetch download URLs for latest 5 episodes in parallel
 2026-07-22 14:04:47 +07:00
asepharyana 052c6cfeb8 fix: revert to alqanime.si, use fetch_with_proxy for direct+relay 
alqanime.net is behind Cloudflare challenge - direct fetch returns
challenge page and proxy relays are down. alqanime.si works with
direct fetch.

Also change repository to use fetch_with_proxy (direct+relay fallback)
instead of fetch_with_proxy_only (relay only).
 2026-07-22 13:56:34 +07:00
asepharyana 1a54e8a127 docs: update CLAUDE.md alqanime.si -> alqanime.net 2026-07-22 13:43:11 +07:00
38 changed files with 173 additions and 3113 deletions
Expand all files Collapse all files
.github/workflows/notify-parent.yml
+1
View File
@@ -4,6 +4,7 @@ on:
push: push:
branches: branches:
- main - main
workflow_dispatch:
jobs: jobs:
dispatch: dispatch:
CLAUDE.md
+1 -1
View File
@@ -33,7 +33,7 @@ Request → Router (route.rs) → Controller → Service → Repository → Pars
│ │
├── Redis (L1 cache) ├── Redis (L1 cache)
├── SeaORM/MySQL (L2, image_cache) ├── SeaORM/MySQL (L2, image_cache)
└── External HTTP (alqanime.si, picser CDN) └── External HTTP (alqanime.net, picser CDN)
``` ```
### Directory Layout ### Directory Layout
Cargo.lock
Generated
+11 -488
View File
@@ -64,56 +64,6 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "anstream"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
dependencies = [
"anstyle",
"anstyle-parse",
"anstyle-query",
"anstyle-wincon",
"colorchoice",
"is_terminal_polyfill",
"utf8parse",
]
[[package]]
name = "anstyle"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
[[package]]
name = "anstyle-parse"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
dependencies = [
"utf8parse",
]
[[package]]
name = "anstyle-query"
version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "anstyle-wincon"
version = "3.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
"windows-sys 0.61.2",
]
[[package]] [[package]]
name = "anyhow" name = "anyhow"
version = "1.0.102" version = "1.0.102"
@@ -489,17 +439,6 @@ dependencies = [
"shlex", "shlex",
] ]
[[package]]
name = "cfb"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d38f2da7a0a2c4ccf0065be06397cc26a81f4e528be095826eee9d4adbb8c60f"
dependencies = [
"byteorder",
"fnv",
"uuid",
]
[[package]] [[package]]
name = "cfg-if" name = "cfg-if"
version = "1.0.4" version = "1.0.4"
@@ -526,62 +465,6 @@ dependencies = [
"windows-link", "windows-link",
] ]
[[package]]
name = "chrono-tz"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3"
dependencies = [
"chrono",
"phf 0.12.1",
]
[[package]]
name = "clap"
version = "4.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351"
dependencies = [
"clap_builder",
"clap_derive",
]
[[package]]
name = "clap_builder"
version = "4.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
dependencies = [
"anstream",
"anstyle",
"clap_lex",
"strsim",
]
[[package]]
name = "clap_derive"
version = "4.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a"
dependencies = [
"heck 0.5.0",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "clap_lex"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
[[package]]
name = "colorchoice"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
[[package]] [[package]]
name = "combine" name = "combine"
version = "4.6.7" version = "4.6.7"
@@ -739,17 +622,6 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "croner"
version = "3.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4aa42bcd3d846ebf66e15bd528d1087f75d1c6c1c66ebff626178a106353c576"
dependencies = [
"chrono",
"derive_builder",
"strum 0.27.2",
]
[[package]] [[package]]
name = "crossbeam-deque" name = "crossbeam-deque"
version = "0.8.6" version = "0.8.6"
@@ -809,7 +681,7 @@ dependencies = [
"cssparser-macros", "cssparser-macros",
"dtoa-short", "dtoa-short",
"itoa", "itoa",
"phf 0.13.1", "phf",
"smallvec", "smallvec",
] ]
@@ -823,41 +695,6 @@ dependencies = [
"syn 2.0.117", "syn 2.0.117",
] ]
[[package]]
name = "darling"
version = "0.20.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.20.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn 2.0.117",
]
[[package]]
name = "darling_macro"
version = "0.20.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
dependencies = [
"darling_core",
"quote",
"syn 2.0.117",
]
[[package]] [[package]]
name = "dashmap" name = "dashmap"
version = "6.1.0" version = "6.1.0"
@@ -878,12 +715,6 @@ version = "2.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea"
[[package]]
name = "data-url"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be1e0bca6c3637f992fc1cc7cbc52a78c1ef6db076dbf1059c4323d6a2048376"
[[package]] [[package]]
name = "deadpool" name = "deadpool"
version = "0.12.3" version = "0.12.3"
@@ -949,37 +780,6 @@ dependencies = [
"syn 2.0.117", "syn 2.0.117",
] ]
[[package]]
name = "derive_builder"
version = "0.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947"
dependencies = [
"derive_builder_macro",
]
[[package]]
name = "derive_builder_core"
version = "0.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"
dependencies = [
"darling",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "derive_builder_macro"
version = "0.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
dependencies = [
"derive_builder_core",
"syn 2.0.117",
]
[[package]] [[package]]
name = "derive_more" name = "derive_more"
version = "2.1.1" version = "2.1.1"
@@ -1174,12 +974,6 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "foldhash"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
[[package]] [[package]]
name = "foreign-types" name = "foreign-types"
version = "0.3.2" version = "0.3.2"
@@ -1302,12 +1096,6 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"
[[package]]
name = "futures-timer"
version = "3.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24"
[[package]] [[package]]
name = "futures-util" name = "futures-util"
version = "0.3.32" version = "0.3.32"
@@ -1362,11 +1150,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"js-sys",
"libc", "libc",
"r-efi 5.3.0", "r-efi 5.3.0",
"wasip2", "wasip2",
"wasm-bindgen",
] ]
[[package]] [[package]]
@@ -1388,29 +1174,6 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
[[package]]
name = "governor"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9efcab3c1958580ff1f25a2a41be1668f7603d849bb63af523b208a3cc1223b8"
dependencies = [
"cfg-if",
"dashmap",
"futures-sink",
"futures-timer",
"futures-util",
"getrandom 0.3.4",
"hashbrown 0.16.1",
"nonzero_ext",
"parking_lot",
"portable-atomic",
"quanta",
"rand 0.9.2",
"smallvec",
"spinning_top",
"web-time",
]
[[package]] [[package]]
name = "h2" name = "h2"
version = "0.4.13" version = "0.4.13"
@@ -1453,7 +1216,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
dependencies = [ dependencies = [
"allocator-api2", "allocator-api2",
"equivalent", "equivalent",
"foldhash 0.1.5", "foldhash",
] ]
[[package]] [[package]]
@@ -1461,11 +1224,6 @@ name = "hashbrown"
version = "0.16.1" version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash 0.2.0",
]
[[package]] [[package]]
name = "hashlink" name = "hashlink"
@@ -1792,12 +1550,6 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]] [[package]]
name = "idna" name = "idna"
version = "1.1.0" version = "1.1.0"
@@ -1841,15 +1593,6 @@ dependencies = [
"serde_core", "serde_core",
] ]
[[package]]
name = "infer"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a588916bfdfd92e71cacef98a63d9b1f0d74d6599980d11894290e7ddefffcf7"
dependencies = [
"cfb",
]
[[package]] [[package]]
name = "inherent" name = "inherent"
version = "1.0.13" version = "1.0.13"
@@ -1886,12 +1629,6 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "is_terminal_polyfill"
version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
[[package]] [[package]]
name = "itertools" name = "itertools"
version = "0.14.0" version = "0.14.0"
@@ -2147,12 +1884,6 @@ version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
[[package]]
name = "nonzero_ext"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21"
[[package]] [[package]]
name = "nu-ansi-term" name = "nu-ansi-term"
version = "0.50.3" version = "0.50.3"
@@ -2194,17 +1925,6 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
[[package]]
name = "num-derive"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]] [[package]]
name = "num-integer" name = "num-integer"
version = "0.1.46" version = "0.1.46"
@@ -2251,12 +1971,6 @@ version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
name = "once_cell_polyfill"
version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
[[package]] [[package]]
name = "openssl" name = "openssl"
version = "0.10.76" version = "0.10.76"
@@ -2346,12 +2060,6 @@ dependencies = [
"tonic", "tonic",
] ]
[[package]]
name = "opentelemetry-semantic-conventions"
version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52"
[[package]] [[package]]
name = "opentelemetry_sdk" name = "opentelemetry_sdk"
version = "0.27.1" version = "0.27.1"
@@ -2445,12 +2153,6 @@ dependencies = [
"windows-link", "windows-link",
] ]
[[package]]
name = "paste"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]] [[package]]
name = "pathdiff" name = "pathdiff"
version = "0.2.3" version = "0.2.3"
@@ -2524,15 +2226,6 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "phf"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7"
dependencies = [
"phf_shared 0.12.1",
]
[[package]] [[package]]
name = "phf" name = "phf"
version = "0.13.1" version = "0.13.1"
@@ -2540,7 +2233,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
dependencies = [ dependencies = [
"phf_macros", "phf_macros",
"phf_shared 0.13.1", "phf_shared",
"serde", "serde",
] ]
@@ -2551,7 +2244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1"
dependencies = [ dependencies = [
"phf_generator", "phf_generator",
"phf_shared 0.13.1", "phf_shared",
] ]
[[package]] [[package]]
@@ -2561,7 +2254,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737"
dependencies = [ dependencies = [
"fastrand", "fastrand",
"phf_shared 0.13.1", "phf_shared",
] ]
[[package]] [[package]]
@@ -2571,21 +2264,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef"
dependencies = [ dependencies = [
"phf_generator", "phf_generator",
"phf_shared 0.13.1", "phf_shared",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.117", "syn 2.0.117",
] ]
[[package]]
name = "phf_shared"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981"
dependencies = [
"siphasher",
]
[[package]] [[package]]
name = "phf_shared" name = "phf_shared"
version = "0.13.1" version = "0.13.1"
@@ -2660,12 +2344,6 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
[[package]]
name = "portable-atomic"
version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
[[package]] [[package]]
name = "potential_utf" name = "potential_utf"
version = "0.1.4" version = "0.1.4"
@@ -2802,21 +2480,6 @@ dependencies = [
"syn 1.0.109", "syn 1.0.109",
] ]
[[package]]
name = "quanta"
version = "0.12.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7"
dependencies = [
"crossbeam-utils",
"libc",
"once_cell",
"raw-cpuid",
"wasi",
"web-sys",
"winapi",
]
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.45" version = "1.0.45"
@@ -2903,15 +2566,6 @@ dependencies = [
"getrandom 0.3.4", "getrandom 0.3.4",
] ]
[[package]]
name = "raw-cpuid"
version = "11.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
dependencies = [
"bitflags",
]
[[package]] [[package]]
name = "rayon" name = "rayon"
version = "1.11.0" version = "1.11.0"
@@ -3328,30 +2982,17 @@ dependencies = [
"async-trait", "async-trait",
"axum 0.8.8", "axum 0.8.8",
"backoff", "backoff",
"base64",
"bytes",
"chrono", "chrono",
"clap",
"config", "config",
"dashmap", "dashmap",
"data-url",
"deadpool-redis", "deadpool-redis",
"dotenvy", "dotenvy",
"flate2", "flate2",
"futures", "futures",
"governor",
"hex",
"hmac",
"http", "http",
"infer",
"itertools",
"log",
"mime_guess",
"opentelemetry", "opentelemetry",
"opentelemetry-otlp", "opentelemetry-otlp",
"opentelemetry-semantic-conventions",
"opentelemetry_sdk", "opentelemetry_sdk",
"rand 0.8.5",
"rayon", "rayon",
"redis", "redis",
"regex", "regex",
@@ -3360,25 +3001,16 @@ dependencies = [
"sea-orm", "sea-orm",
"serde", "serde",
"serde_json", "serde_json",
"sha1",
"sha2",
"tempfile",
"thiserror 2.0.18", "thiserror 2.0.18",
"tl",
"tokio", "tokio",
"tokio-cron-scheduler",
"tokio-util",
"tower 0.5.3",
"tower-http", "tower-http",
"tracing", "tracing",
"tracing-subscriber", "tracing-subscriber",
"url", "url",
"urlencoding", "urlencoding",
"utoipa", "utoipa",
"utoipa-axum",
"utoipa-swagger-ui", "utoipa-swagger-ui",
"uuid", "uuid",
"walkdir",
] ]
[[package]] [[package]]
@@ -3416,7 +3048,7 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"sqlx", "sqlx",
"strum 0.26.3", "strum",
"thiserror 2.0.18", "thiserror 2.0.18",
"time", "time",
"tracing", "tracing",
@@ -3510,7 +3142,7 @@ dependencies = [
"derive_more", "derive_more",
"log", "log",
"new_debug_unreachable", "new_debug_unreachable",
"phf 0.13.1", "phf",
"phf_codegen", "phf_codegen",
"precomputed-hash", "precomputed-hash",
"rustc-hash", "rustc-hash",
@@ -3745,15 +3377,6 @@ dependencies = [
"lock_api", "lock_api",
] ]
[[package]]
name = "spinning_top"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300"
dependencies = [
"lock_api",
]
[[package]] [[package]]
name = "spki" name = "spki"
version = "0.7.3" version = "0.7.3"
@@ -3993,7 +3616,7 @@ checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901"
dependencies = [ dependencies = [
"new_debug_unreachable", "new_debug_unreachable",
"parking_lot", "parking_lot",
"phf_shared 0.13.1", "phf_shared",
"precomputed-hash", "precomputed-hash",
] ]
@@ -4004,7 +3627,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69" checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69"
dependencies = [ dependencies = [
"phf_generator", "phf_generator",
"phf_shared 0.13.1", "phf_shared",
"proc-macro2", "proc-macro2",
"quote", "quote",
] ]
@@ -4020,39 +3643,12 @@ dependencies = [
"unicode-properties", "unicode-properties",
] ]
[[package]]
name = "strsim"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]] [[package]]
name = "strum" name = "strum"
version = "0.26.3" version = "0.26.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
[[package]]
name = "strum"
version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf"
dependencies = [
"strum_macros",
]
[[package]]
name = "strum_macros"
version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"
dependencies = [
"heck 0.5.0",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]] [[package]]
name = "subtle" name = "subtle"
version = "2.6.1" version = "2.6.1"
@@ -4266,12 +3862,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tl"
version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b130bd8a58c163224b44e217b4239ca7b927d82bf6cc2fea1fc561d15056e3f7"
[[package]] [[package]]
name = "tokio" name = "tokio"
version = "1.50.0" version = "1.50.0"
@@ -4289,22 +3879,6 @@ dependencies = [
"windows-sys 0.61.2", "windows-sys 0.61.2",
] ]
[[package]]
name = "tokio-cron-scheduler"
version = "0.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f50e41f200fd8ed426489bd356910ede4f053e30cebfbd59ef0f856f0d7432a"
dependencies = [
"chrono",
"chrono-tz",
"croner",
"num-derive",
"num-traits",
"tokio",
"tracing",
"uuid",
]
[[package]] [[package]]
name = "tokio-macros" name = "tokio-macros"
version = "2.6.1" version = "2.6.1"
@@ -4712,12 +4286,6 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
[[package]]
name = "utf8parse"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]] [[package]]
name = "utoipa" name = "utoipa"
version = "5.4.0" version = "5.4.0"
@@ -4730,19 +4298,6 @@ dependencies = [
"utoipa-gen", "utoipa-gen",
] ]
[[package]]
name = "utoipa-axum"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c25bae5bccc842449ec0c5ddc5cbb6a3a1eaeac4503895dc105a1138f8234a0"
dependencies = [
"axum 0.8.8",
"paste",
"tower-layer",
"tower-service",
"utoipa",
]
[[package]] [[package]]
name = "utoipa-gen" name = "utoipa-gen"
version = "5.4.0" version = "5.4.0"
@@ -4965,23 +4520,13 @@ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
[[package]]
name = "web-time"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]] [[package]]
name = "web_atoms" name = "web_atoms"
version = "0.2.3" version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57a9779e9f04d2ac1ce317aee707aa2f6b773afba7b931222bff6983843b1576" checksum = "57a9779e9f04d2ac1ce317aee707aa2f6b773afba7b931222bff6983843b1576"
dependencies = [ dependencies = [
"phf 0.13.1", "phf",
"phf_codegen", "phf_codegen",
"string_cache", "string_cache",
"string_cache_codegen", "string_cache_codegen",
@@ -5015,22 +4560,6 @@ dependencies = [
"wasite", "wasite",
] ]
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]] [[package]]
name = "winapi-util" name = "winapi-util"
version = "0.1.11" version = "0.1.11"
@@ -5040,12 +4569,6 @@ dependencies = [
"windows-sys 0.61.2", "windows-sys 0.61.2",
] ]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]] [[package]]
name = "windows-core" name = "windows-core"
version = "0.62.2" version = "0.62.2"
Cargo.toml
+11 -24
View File
@@ -26,56 +26,33 @@ sea-orm = { version = "1.1.19", features = ["sqlx-postgres", "runtime-tokio-rust
uuid = { version = "1.10.0", features = ["v4", "serde"] } uuid = { version = "1.10.0", features = ["v4", "serde"] }
chrono = { version = "0.4", features = ["serde"] } chrono = { version = "0.4", features = ["serde"] }
bytes = "1.11.0"
futures = "0.3" futures = "0.3"
reqwest = { version = "0.12.28", features = ["json", "stream", "multipart"] } reqwest = { version = "0.12.28", features = ["json", "stream", "multipart"] }
http = "1.4.0" http = "1.4.0"
sha1 = "0.10.6"
data-url = "0.3.2"
base64 = "0.22.1"
tokio-util = { version = "0.7.18", features = ["codec"] }
async-trait = "0.1.89" async-trait = "0.1.89"
regex = "1.12.2" regex = "1.12.2"
infer = "0.19.0"
urlencoding = "2.1" urlencoding = "2.1"
url = "2.5.8" url = "2.5.8"
rand = "0.8"
tempfile = "3.24.0"
mime_guess = "2.0.5"
tower-http = { version = "0.6.8", features = ["fs", "cors", "compression-gzip", "compression-br", "compression-zstd"] } tower-http = { version = "0.6.8", features = ["fs", "cors", "compression-gzip", "compression-br", "compression-zstd"] }
backoff = { version = "0.4", features = ["futures", "tokio"] } backoff = { version = "0.4", features = ["futures", "tokio"] }
dashmap = "6.1" dashmap = "6.1"
deadpool-redis = { version = "0.22.1", features = ["serde"] } deadpool-redis = { version = "0.22.1", features = ["serde"] }
rayon = "1.11" rayon = "1.11"
tl = "0.7.8"
tower = { version = "0.5", features = ["make"] }
scraper = "0.25.0" scraper = "0.25.0"
flate2 = "1.1" flate2 = "1.1"
redis = { version = "0.32.7", features = ["tokio-rustls-comp", "safe_iterators"] } redis = { version = "0.32.7", features = ["tokio-rustls-comp", "safe_iterators"] }
thiserror = "2.0.18" thiserror = "2.0.18"
itertools = "0.14"
clap = { version = "4.5", features = ["derive"] }
config = { version = "0.15.19", features = ["toml"] } config = { version = "0.15.19", features = ["toml"] }
governor = "0.10.4"
tokio-cron-scheduler = "0.15.1"
hex = "0.4.3"
hmac = "0.12.1"
sha2 = "0.10.9"
log = "0.4"
walkdir = "2.5"
utoipa = { version = "5.0", features = ["axum_extras"] } utoipa = { version = "5.0", features = ["axum_extras"] }
utoipa-swagger-ui = { version = "9.0", features = ["axum"] } utoipa-swagger-ui = { version = "9.0", features = ["axum"] }
utoipa-axum = "0.2.0"
# OpenTelemetry metrics # OpenTelemetry metrics
opentelemetry = { version = "0.27", features = ["metrics"] } opentelemetry = { version = "0.27", features = ["metrics"] }
opentelemetry_sdk = { version = "0.27", features = ["metrics", "rt-tokio"] } opentelemetry_sdk = { version = "0.27", features = ["metrics", "rt-tokio"] }
opentelemetry-otlp = { version = "0.27", features = ["metrics"] } opentelemetry-otlp = { version = "0.27", features = ["metrics"] }
opentelemetry-semantic-conventions = "0.27"
# Dependensi yang hanya dibutuhkan untuk build script (build.rs) # Dependensi yang hanya dibutuhkan untuk build script (build.rs)
@@ -99,6 +76,15 @@ unsafe_code = "forbid"
unused_variables = "deny" unused_variables = "deny"
unused_imports = "deny" unused_imports = "deny"
unused_must_use = "deny" unused_must_use = "deny"
dead_code = "deny"
unused_mut = "deny"
unreachable_code = "deny"
trivial_casts = "deny"
trivial_numeric_casts = "deny"
explicit_outlives_requirements = "deny"
unused_labels = "deny"
unused_braces = "deny"
noop_method_call = "deny"
[lints.clippy] [lints.clippy]
unwrap_used = "warn" unwrap_used = "warn"
@@ -106,6 +92,7 @@ expect_used = "warn"
panic = "deny" panic = "deny"
todo = "deny" todo = "deny"
unimplemented = "deny" unimplemented = "deny"
unnecessary_to_owned = "deny"
[features] [features]
# default = ["ffmpeg"] # default = ["ffmpeg"]
src/application/anime/use_cases.rs
+3 -72
View File
@@ -3,18 +3,12 @@
//! Orchestrates repository fetching, caching, and image poster processing. //! Orchestrates repository fetching, caching, and image poster processing.
//! Returns pure domain types — no DTOs. //! Returns pure domain types — no DTOs.
use std::sync::Arc;
use deadpool_redis::Pool; use deadpool_redis::Pool;
use sea_orm::DatabaseConnection;
use crate::domain::entity::anime::*; use crate::domain::entity::anime::*;
use crate::domain::error::*; use crate::domain::error::*;
use crate::infrastructure::cache::redis::Cache; use crate::infrastructure::cache::redis::Cache;
use crate::infrastructure::repository::OtakudesuRepository; use crate::infrastructure::repository::OtakudesuRepository;
use crate::infrastructure::services::images::cache::{
cache_image_urls_batch_lazy, get_cached_or_original,
};
const INDEX_CACHE_TTL: u64 = 10; const INDEX_CACHE_TTL: u64 = 10;
const GENRE_LIST_CACHE_TTL: u64 = 3600; const GENRE_LIST_CACHE_TTL: u64 = 3600;
@@ -23,22 +17,13 @@ const DEFAULT_CACHE_TTL: u64 = 300;
pub struct AnimeUseCases { pub struct AnimeUseCases {
repository: OtakudesuRepository, repository: OtakudesuRepository,
redis_pool: Pool, redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
} }
impl AnimeUseCases { impl AnimeUseCases {
pub fn new( pub fn new(repository: OtakudesuRepository, redis_pool: Pool) -> Self {
repository: OtakudesuRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
) -> Self {
Self { Self {
repository, repository,
redis_pool, redis_pool,
db,
semaphore,
} }
} }
@@ -49,7 +34,7 @@ impl AnimeUseCases {
pub async fn get_anime_index(&self) -> Result<AnimeData, DomainError> { pub async fn get_anime_index(&self) -> Result<AnimeData, DomainError> {
self.cache() self.cache()
.get_or_set("anime:index:v2", INDEX_CACHE_TTL, || async { .get_or_set("anime:index:v2", INDEX_CACHE_TTL, || async {
let mut data = self let data = self
.repository .repository
.fetch_anime_index() .fetch_anime_index()
.await .await
@@ -59,33 +44,6 @@ impl AnimeUseCases {
return Err("Empty anime index — refusing to cache".to_string()); return Err("Empty anime index — refusing to cache".to_string());
} }
let mut posters: Vec<String> = data
.ongoing_anime
.iter()
.map(|item| item.poster.clone())
.collect();
posters.extend(data.complete_anime.iter().map(|item| item.poster.clone()));
let cached_posters = cache_image_urls_batch_lazy(
self.db.clone(),
&self.redis_pool,
posters,
self.semaphore.clone(),
)
.await;
let ongoing_len = data.ongoing_anime.len();
for (i, item) in data.ongoing_anime.iter_mut().enumerate() {
if let Some(url) = cached_posters.get(i) {
item.poster = url.clone();
}
}
for (i, item) in data.complete_anime.iter_mut().enumerate() {
if let Some(url) = cached_posters.get(ongoing_len + i) {
item.poster = url.clone();
}
}
Ok(data) Ok(data)
}) })
.await .await
@@ -108,39 +66,12 @@ impl AnimeUseCases {
let cache_key = format!("anime:detail:{}", slug); let cache_key = format!("anime:detail:{}", slug);
self.cache() self.cache()
.get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async { .get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async {
let mut data = self let data = self
.repository .repository
.fetch_anime_detail(&slug) .fetch_anime_detail(&slug)
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
data.poster = get_cached_or_original(
self.db.clone(),
&self.redis_pool,
&data.poster,
self.semaphore.clone(),
)
.await;
let rec_posters: Vec<String> = data
.recommendations
.iter()
.map(|r| r.poster.clone())
.collect();
let cached_rec_posters = cache_image_urls_batch_lazy(
self.db.clone(),
&self.redis_pool,
rec_posters,
self.semaphore.clone(),
)
.await;
for (i, rec) in data.recommendations.iter_mut().enumerate() {
if let Some(url) = cached_rec_posters.get(i) {
rec.poster = url.clone();
}
}
Ok(data) Ok(data)
}) })
.await .await
src/application/anime2/use_cases.rs
+41 -128
View File
@@ -8,23 +8,17 @@
//! TODO: Once parsers return domain types, replace shared types with //! TODO: Once parsers return domain types, replace shared types with
//! `crate::domain::entity::anime::{GenreAnimeItem, SearchAnimeItem, LatestAnimeItem}`. //! `crate::domain::entity::anime::{GenreAnimeItem, SearchAnimeItem, LatestAnimeItem}`.
use std::sync::Arc;
use deadpool_redis::Pool; use deadpool_redis::Pool;
use sea_orm::DatabaseConnection;
use crate::domain::error::*; use crate::domain::error::*;
use crate::domain::repository::ScrapingRepository; use crate::domain::repository::ScrapingRepository;
use crate::infrastructure::cache::redis::Cache; use crate::infrastructure::cache::redis::Cache;
use crate::infrastructure::repository::AlqanimeRepository; use crate::infrastructure::repository::AlqanimeRepository;
use crate::infrastructure::services::images::cache::{
apply_cached_posters, cache_image_urls_batch_lazy, get_cached_or_original,
};
use crate::infrastructure::repository::parsers::alqanime_parser as parser; use crate::infrastructure::repository::parsers::alqanime_parser as parser;
use crate::domain::entity::anime::{ use crate::domain::entity::anime::{
CompleteAnimeItem, FilterAnimeItem, Genre, GenreAnimeItem, HasPoster, LatestAnimeItem, CompleteAnimeItem, FilterAnimeItem, Genre, GenreAnimeItem, LatestAnimeItem,
OngoingAnimeItemWithScore, Pagination, SearchAnimeItem, OngoingAnimeItemWithScore, Pagination, SearchAnimeItem,
}; };
@@ -37,7 +31,7 @@ pub use crate::domain::entity::anime::{
SearchAnimeItem as Anime2SearchItem, SearchAnimeItem as Anime2SearchItem,
}; };
pub use crate::infrastructure::repository::parsers::alqanime_parser::{ pub use crate::infrastructure::repository::parsers::alqanime_parser::{
AlqDetailData, AlqDownloadItem, AlqLink, AlqRecommendation, AlqDetailData, AlqDownloadItem, AlqEpisode, AlqLink, AlqRecommendation,
}; };
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@@ -56,15 +50,6 @@ pub struct Anime2Item {
pub anime_url: String, pub anime_url: String,
} }
impl HasPoster for Anime2Item {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] #[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct Anime2Data { pub struct Anime2Data {
pub ongoing_anime: Vec<Anime2Item>, pub ongoing_anime: Vec<Anime2Item>,
@@ -123,22 +108,13 @@ const COMPLETE_CACHE_TTL: u64 = 300;
pub struct Anime2UseCases { pub struct Anime2UseCases {
repository: AlqanimeRepository, repository: AlqanimeRepository,
redis_pool: Pool, redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
} }
impl Anime2UseCases { impl Anime2UseCases {
pub fn new( pub fn new(repository: AlqanimeRepository, redis_pool: Pool) -> Self {
repository: AlqanimeRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
) -> Self {
Self { Self {
repository, repository,
redis_pool, redis_pool,
db,
semaphore,
} }
} }
@@ -168,7 +144,7 @@ impl Anime2UseCases {
}) })
.await .await
.map_err(|e| e.to_string())??; .map_err(|e| e.to_string())??;
let mut ongoing: Vec<Anime2Item> = data let ongoing: Vec<Anime2Item> = data
.0 .0
.into_iter() .into_iter()
.map(|item| Anime2Item { .map(|item| Anime2Item {
@@ -182,7 +158,7 @@ impl Anime2UseCases {
}) })
.collect(); .collect();
let mut complete: Vec<Anime2Item> = data let complete: Vec<Anime2Item> = data
.1 .1
.into_iter() .into_iter()
.map(|item| Anime2Item { .map(|item| Anime2Item {
@@ -196,30 +172,6 @@ impl Anime2UseCases {
}) })
.collect(); .collect();
let mut posters: Vec<String> =
ongoing.iter().map(|item| item.poster.clone()).collect();
posters.extend(complete.iter().map(|item| item.poster.clone()));
let cached_posters = cache_image_urls_batch_lazy(
self.db.clone(),
&self.redis_pool,
posters,
self.semaphore.clone(),
)
.await;
let ongoing_len = ongoing.len();
for (i, item) in ongoing.iter_mut().enumerate() {
if let Some(url) = cached_posters.get(i) {
item.poster = url.clone();
}
}
for (i, item) in complete.iter_mut().enumerate() {
if let Some(url) = cached_posters.get(ongoing_len + i) {
item.poster = url.clone();
}
}
Ok(Anime2Response { Ok(Anime2Response {
status: "Ok".to_string(), status: "Ok".to_string(),
data: Anime2Data { data: Anime2Data {
@@ -293,20 +245,12 @@ impl Anime2UseCases {
.fetch_html(&url) .fetch_html(&url)
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let (mut data, pagination) = tokio::task::spawn_blocking(move || { let (data, pagination) = tokio::task::spawn_blocking(move || {
parser::parse_filter_page(&html, page).map_err(|e| e.to_string()) parser::parse_filter_page(&html, page).map_err(|e| e.to_string())
}) })
.await .await
.map_err(|e| e.to_string())??; .map_err(|e| e.to_string())??;
apply_cached_posters(
&mut data,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(FilterResponse { Ok(FilterResponse {
success: true, success: true,
data, data,
@@ -342,28 +286,37 @@ impl Anime2UseCases {
.await .await
.map_err(|e| e.to_string())??; .map_err(|e| e.to_string())??;
data.poster = get_cached_or_original( // Fetch download URLs for the latest episodes
self.db.clone(), let episodes = &mut data.episodes;
&self.redis_pool, if !episodes.is_empty() {
&data.poster, // Limit to latest 5 episodes to keep things fast
self.semaphore.clone(), let latest: Vec<_> = episodes
) .iter_mut()
.await; .take(5)
data.poster2 = get_cached_or_original( .map(|ep| ep.url.clone())
self.db.clone(), .collect();
&self.redis_pool,
&data.poster2,
self.semaphore.clone(),
)
.await;
apply_cached_posters( let results: Vec<_> = futures::future::join_all(latest.iter().map(|url| {
&mut data.recommendations, self.repository.fetch_html(url)
self.db.clone(), }))
&self.redis_pool, .await;
self.semaphore.clone(),
) for (i, result) in results.into_iter().enumerate() {
.await; if let Ok(html) = result {
if let Ok(Some(dl_url)) =
tokio::task::spawn_blocking(move || {
parser::parse_episode_download(&html)
})
.await
.unwrap_or(Ok(None))
{
if let Some(ep) = episodes.get_mut(i) {
ep.download_url = Some(dl_url);
}
}
}
}
}
Ok(DetailResponse { Ok(DetailResponse {
status: "Ok".to_string(), status: "Ok".to_string(),
@@ -388,20 +341,12 @@ impl Anime2UseCases {
.fetch_html(&self.repository.genre_page_url(&genre_slug, page)) .fetch_html(&self.repository.genre_page_url(&genre_slug, page))
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let (mut data, _pagination) = tokio::task::spawn_blocking(move || { let (data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_genre_page(&html, page).map_err(|e| e.to_string()) parser::parse_genre_page(&html, page).map_err(|e| e.to_string())
}) })
.await .await
.map_err(|e| e.to_string())??; .map_err(|e| e.to_string())??;
apply_cached_posters(
&mut data,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(ApiResponse::success(data)) Ok(ApiResponse::success(data))
}) })
.await .await
@@ -422,20 +367,12 @@ impl Anime2UseCases {
.fetch_html(&self.repository.search_url(&query, page)) .fetch_html(&self.repository.search_url(&query, page))
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let (mut data, _pagination) = tokio::task::spawn_blocking(move || { let (data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_search_page(&html, page).map_err(|e| e.to_string()) parser::parse_search_page(&html, page).map_err(|e| e.to_string())
}) })
.await .await
.map_err(|e| e.to_string())??; .map_err(|e| e.to_string())??;
apply_cached_posters(
&mut data,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(ApiResponse::success(data)) Ok(ApiResponse::success(data))
}) })
.await .await
@@ -455,20 +392,12 @@ impl Anime2UseCases {
.fetch_html(&self.repository.latest_url(page)) .fetch_html(&self.repository.latest_url(page))
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let (mut data, _pagination) = tokio::task::spawn_blocking(move || { let (data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_latest_page(&html, page).map_err(|e| e.to_string()) parser::parse_latest_page(&html, page).map_err(|e| e.to_string())
}) })
.await .await
.map_err(|e| e.to_string())??; .map_err(|e| e.to_string())??;
apply_cached_posters(
&mut data,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(ApiResponse::success(data)) Ok(ApiResponse::success(data))
}) })
.await .await
@@ -488,20 +417,12 @@ impl Anime2UseCases {
.fetch_html(&self.repository.ongoing_url(page)) .fetch_html(&self.repository.ongoing_url(page))
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let (mut data, _pagination) = tokio::task::spawn_blocking(move || { let (data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_ongoing_page(&html, page).map_err(|e| e.to_string()) parser::parse_ongoing_page(&html, page).map_err(|e| e.to_string())
}) })
.await .await
.map_err(|e| e.to_string())??; .map_err(|e| e.to_string())??;
apply_cached_posters(
&mut data,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(ApiResponse::success(data)) Ok(ApiResponse::success(data))
}) })
.await .await
@@ -521,20 +442,12 @@ impl Anime2UseCases {
.fetch_html(&self.repository.complete_url(page)) .fetch_html(&self.repository.complete_url(page))
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let (mut data, _pagination) = tokio::task::spawn_blocking(move || { let (data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_complete_page(&html, page).map_err(|e| e.to_string()) parser::parse_complete_page(&html, page).map_err(|e| e.to_string())
}) })
.await .await
.map_err(|e| e.to_string())??; .map_err(|e| e.to_string())??;
apply_cached_posters(
&mut data,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(ApiResponse::success(data)) Ok(ApiResponse::success(data))
}) })
.await .await
src/application/komik/use_cases.rs
+8 -81
View File
@@ -6,10 +6,7 @@
//! `crate::infrastructure::repository::parsers::komik_parser`. //! `crate::infrastructure::repository::parsers::komik_parser`.
//! TODO: Move response DTOs to `crate::presentation::dto::komik`. //! TODO: Move response DTOs to `crate::presentation::dto::komik`.
use std::sync::Arc;
use deadpool_redis::Pool; use deadpool_redis::Pool;
use sea_orm::DatabaseConnection;
use crate::domain::entity::anime::Pagination; use crate::domain::entity::anime::Pagination;
use crate::domain::entity::komik::{ChapterData, DetailData, KomikGenre, KomikItem}; use crate::domain::entity::komik::{ChapterData, DetailData, KomikGenre, KomikItem};
@@ -17,9 +14,6 @@ use crate::domain::error::*;
use crate::domain::repository::ScrapingRepository; use crate::domain::repository::ScrapingRepository;
use crate::infrastructure::cache::redis::Cache; use crate::infrastructure::cache::redis::Cache;
use crate::infrastructure::repository::KomikRepository; use crate::infrastructure::repository::KomikRepository;
use crate::infrastructure::services::images::cache::{
apply_cached_posters, cache_image_urls_batch_lazy, get_cached_or_original,
};
use crate::infrastructure::repository::parsers::komik_parser as parser; use crate::infrastructure::repository::parsers::komik_parser as parser;
@@ -36,22 +30,13 @@ const SEARCH_CACHE_TTL: u64 = 300;
pub struct KomikUseCases { pub struct KomikUseCases {
repository: KomikRepository, repository: KomikRepository,
redis_pool: Pool, redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
} }
impl KomikUseCases { impl KomikUseCases {
pub fn new( pub fn new(repository: KomikRepository, redis_pool: Pool) -> Self {
repository: KomikRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
) -> Self {
Self { Self {
repository, repository,
redis_pool, redis_pool,
db,
semaphore,
} }
} }
@@ -94,20 +79,12 @@ impl KomikUseCases {
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let (mut komik_list, pagination) = let (komik_list, pagination) =
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page)) tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
.await .await
.map_err(|e| e.to_string())? .map_err(|e| e.to_string())?
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
apply_cached_posters(
&mut komik_list,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok((komik_list, pagination)) Ok((komik_list, pagination))
}) })
.await .await
@@ -130,20 +107,12 @@ impl KomikUseCases {
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let (mut komik_list, pagination) = let (komik_list, pagination) =
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page)) tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
.await .await
.map_err(|e| e.to_string())? .map_err(|e| e.to_string())?
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
apply_cached_posters(
&mut komik_list,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok((komik_list, pagination)) Ok((komik_list, pagination))
}) })
.await .await
@@ -162,22 +131,12 @@ impl KomikUseCases {
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let mut data = let data =
tokio::task::spawn_blocking(move || parser::parse_komik_detail_document(&html)) tokio::task::spawn_blocking(move || parser::parse_komik_detail_document(&html))
.await .await
.map_err(|e| e.to_string())? .map_err(|e| e.to_string())?
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
if !data.poster.is_empty() {
data.poster = get_cached_or_original(
self.db.clone(),
&self.redis_pool,
&data.poster,
self.semaphore.clone(),
)
.await;
}
Ok(data) Ok(data)
}) })
.await .await
@@ -196,7 +155,7 @@ impl KomikUseCases {
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let mut data = tokio::task::spawn_blocking({ let data = tokio::task::spawn_blocking({
let chapter_url = chapter_url.clone(); let chapter_url = chapter_url.clone();
move || parser::parse_komik_chapter_document(&html, &chapter_url) move || parser::parse_komik_chapter_document(&html, &chapter_url)
}) })
@@ -204,14 +163,6 @@ impl KomikUseCases {
.map_err(|e| e.to_string())? .map_err(|e| e.to_string())?
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
data.images = cache_image_urls_batch_lazy(
self.db.clone(),
&self.redis_pool,
data.images,
self.semaphore.clone(),
)
.await;
Ok(data) Ok(data)
}) })
.await .await
@@ -278,7 +229,7 @@ impl KomikUseCases {
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let (mut komik_list, pagination) = let (komik_list, pagination) =
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page)) tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
.await .await
.map_err(|e| e.to_string())? .map_err(|e| e.to_string())?
@@ -288,14 +239,6 @@ impl KomikUseCases {
return Err(format!("Empty komik {} page {}", list_name, page)); return Err(format!("Empty komik {} page {}", list_name, page));
} }
apply_cached_posters(
&mut komik_list,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok((komik_list, pagination)) Ok((komik_list, pagination))
}) })
.await .await
@@ -318,20 +261,12 @@ impl KomikUseCases {
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let (mut komik_list, pagination) = let (komik_list, pagination) =
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page)) tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
.await .await
.map_err(|e| e.to_string())? .map_err(|e| e.to_string())?
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
apply_cached_posters(
&mut komik_list,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok((komik_list, pagination)) Ok((komik_list, pagination))
}) })
.await .await
@@ -354,20 +289,12 @@ impl KomikUseCases {
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let (mut komik_list, pagination) = let (komik_list, pagination) =
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page)) tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
.await .await
.map_err(|e| e.to_string())? .map_err(|e| e.to_string())?
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
apply_cached_posters(
&mut komik_list,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok((komik_list, pagination)) Ok((komik_list, pagination))
}) })
.await .await
src/application/proxy/use_cases.rs
-235
View File
@@ -1,236 +1 @@
//! Proxy application use cases. //! Proxy application use cases.
//!
//! Provides proxy fetch, image caching, and audit/repair operations.
//!
//! TODO: Move result types to `crate::presentation::dto::proxy`.
//! TODO: Add event bus integration for ImageRepaired events.
//! TODO: Replace `reqwest::Client::new()` with shared HTTP client from infrastructure.
use std::sync::Arc;
use axum::http::StatusCode;
use axum::response::Response;
use serde::Serialize;
use tracing::{error, info, warn};
use utoipa::ToSchema;
use crate::domain::error::*;
use crate::domain::repository::ImageCacheRepository;
use crate::infrastructure::repository::{ProxyRepository, SeaOrmImageCacheRepository};
use crate::infrastructure::services::images::cache::ImageCache;
// ============================================================================
// Result types — TEMPORARY: move to presentation layer
// ============================================================================
/// Result of caching an image URL.
/// TODO: Move to presentation::dto::proxy
#[derive(Debug, Serialize, ToSchema)]
pub struct ImageCacheResult {
pub success: bool,
pub original_url: String,
pub cdn_url: String,
pub from_cache: bool,
pub pending: Option<bool>,
}
/// Result of auditing/repairing a cached image URL.
/// TODO: Move to presentation::dto::proxy
#[derive(Debug, Serialize, ToSchema)]
pub struct AuditImageCacheResult {
pub success: bool,
pub original_url: String,
pub cdn_url: Option<String>,
pub was_accessible: bool,
pub re_uploaded: bool,
pub message: String,
}
// ============================================================================
// Use case struct
// ============================================================================
pub struct ProxyUseCases {
repository: ProxyRepository,
image_cache_repo: Arc<dyn ImageCacheRepository>,
}
impl ProxyUseCases {
pub fn new(
repository: ProxyRepository,
image_cache_repo: Arc<SeaOrmImageCacheRepository>,
) -> Self {
Self {
repository,
image_cache_repo,
}
}
fn build_image_cache(&self) -> ImageCache {
ImageCache::new(self.image_cache_repo.clone())
}
pub async fn fetch_with_proxy_only(&self, url: String) -> Result<Response, DomainError> {
let fetch_result = self
.repository
.fetch_with_proxy_url(&url)
.await
.map_err(|e| DomainError::Scraping(ScrapingError::Http(e.to_string())))?;
let mut builder = Response::builder().status(StatusCode::OK);
if let Some(content_type) = fetch_result.content_type {
builder = builder.header("Content-Type", content_type);
}
builder
.body(fetch_result.data.into())
.map_err(|e| DomainError::Repository(RepositoryError::Network(e.to_string())))
}
pub async fn image_cache(
&self,
url: String,
lazy: bool,
) -> Result<ImageCacheResult, DomainError> {
let cache = self.build_image_cache();
if let Some(cdn_url) = cache.get_cdn_url(&url).await {
return Ok(ImageCacheResult {
success: true,
original_url: url,
cdn_url,
from_cache: true,
pending: None,
});
}
if lazy {
let repo = self.image_cache_repo.clone();
let url_clone = url.clone();
tokio::spawn(async move {
let cache = ImageCache::new(repo);
match cache.get_or_cache(&url_clone).await {
Ok(cdn) => info!("[LazyCache] Cached {} -> {}", url_clone, cdn),
Err(e) => warn!("[LazyCache] Failed {}: {}", url_clone, e),
}
});
return Ok(ImageCacheResult {
success: true,
original_url: url.clone(),
cdn_url: url,
from_cache: false,
pending: Some(true),
});
}
match cache.get_or_cache(&url).await {
Ok(cdn_url) => Ok(ImageCacheResult {
success: true,
original_url: url,
cdn_url,
from_cache: false,
pending: None,
}),
Err(e) => {
error!("ImageCache error: {}", e);
Ok(ImageCacheResult {
success: false,
original_url: url.clone(),
cdn_url: url,
from_cache: false,
pending: None,
})
}
}
}
pub async fn audit_image_cache(
&self,
url: String,
) -> Result<AuditImageCacheResult, DomainError> {
let cache = self.build_image_cache();
let mut cdn_opt = cache.get_cdn_url(&url).await;
let mut original = url.clone();
if cdn_opt.is_none() {
if let Some(orig) = cache.find_original_from_cdn(&url).await {
info!("SmartAudit: {} recognized as CDN, original {}", url, orig);
original = orig;
cdn_opt = Some(url.clone());
}
}
if let Some(cdn_url) = cdn_opt {
let client = reqwest::Client::new();
let mut accessible = false;
match client.get(&cdn_url).send().await {
Ok(resp) if resp.status().is_success() => {
if let Ok(bytes) = resp.bytes().await {
if infer::get(&bytes)
.map(|k| k.mime_type().starts_with("image/"))
.unwrap_or(false)
{
accessible = true;
} else {
warn!("CDN {} returned non-image content", cdn_url);
}
}
}
Ok(resp) => warn!("CDN {} status {}", cdn_url, resp.status()),
Err(e) => warn!("CDN {} fetch error {}", cdn_url, e),
}
if accessible {
return Ok(AuditImageCacheResult {
success: true,
original_url: original,
cdn_url: Some(cdn_url),
was_accessible: true,
re_uploaded: false,
message: "CDN URL is accessible and the image is valid".to_string(),
});
}
info!("CDN {} inaccessible, purging and reuploading", cdn_url);
let _ = cache.invalidate(&original).await;
match cache.get_or_cache(&original).await {
Ok(new_cdn) => Ok(AuditImageCacheResult {
success: true,
original_url: original,
cdn_url: Some(new_cdn),
was_accessible: false,
re_uploaded: true,
message: "CDN URL was inaccessible, re-uploaded".to_string(),
}),
Err(e) => Ok(AuditImageCacheResult {
success: false,
original_url: original,
cdn_url: None,
was_accessible: false,
re_uploaded: false,
message: format!("Re-upload failed: {}", e),
}),
}
} else {
match cache.get_or_cache(&original).await {
Ok(new_cdn) => Ok(AuditImageCacheResult {
success: true,
original_url: original,
cdn_url: Some(new_cdn),
was_accessible: false,
re_uploaded: true,
message: "Cached newly".to_string(),
}),
Err(e) => Ok(AuditImageCacheResult {
success: false,
original_url: original,
cdn_url: None,
was_accessible: false,
re_uploaded: false,
message: format!("Cache failed: {}", e),
}),
}
}
}
}
src/bootstrap/mod.rs
+2 -12
View File
@@ -62,7 +62,8 @@ impl Application {
.max_lifetime(std::time::Duration::from_secs( .max_lifetime(std::time::Duration::from_secs(
CONFIG.db.max_lifetime_seconds, CONFIG.db.max_lifetime_seconds,
)) ))
.sqlx_logging(CONFIG.log_level == "debug"); .sqlx_logging(CONFIG.log_level == "debug")
.map_sqlx_postgres_opts(|opts| opts.extra_float_digits(None));
let db = Database::connect(opt) let db = Database::connect(opt)
.await .await
@@ -76,26 +77,15 @@ impl Application {
// App State components // App State components
let db_arc = Arc::new(db); let db_arc = Arc::new(db);
let image_processing_semaphore = Arc::new(tokio::sync::Semaphore::new(
CONFIG.image_processing_concurrency,
));
let event_bus = Arc::new(crate::events::bus::EventBus::new()); let event_bus = Arc::new(crate::events::bus::EventBus::new());
let redis_pool = crate::infrastructure::cache::redis_pool::redis_pool() let redis_pool = crate::infrastructure::cache::redis_pool::redis_pool()
.map_err(|e| anyhow::anyhow!("Failed to init Redis pool: {}", e))?; .map_err(|e| anyhow::anyhow!("Failed to init Redis pool: {}", e))?;
use crate::infrastructure::repository::SeaOrmImageCacheRepository;
let image_cache_repo = Arc::new(SeaOrmImageCacheRepository::new(
db_arc.clone(),
redis_pool.clone(),
));
let app_state = Arc::new(AppState { let app_state = Arc::new(AppState {
redis_pool, redis_pool,
db: db_arc.clone(), db: db_arc.clone(),
image_processing_semaphore,
event_bus: event_bus.clone(), event_bus: event_bus.clone(),
image_cache_repo,
}); });
let app = crate::presentation::router::build_router(app_state.clone())?; let app = crate::presentation::router::build_router(app_state.clone())?;
src/bootstrap/setup.rs
+3 -41
View File
@@ -1,47 +1,9 @@
//! Database schema initialization. //! Database schema initialization.
use sea_orm::{ConnectionTrait, DatabaseConnection, Schema, Statement}; use sea_orm::DatabaseConnection;
use tracing::info; use tracing::info;
use crate::infrastructure::persistence::entities::image_cache; pub async fn init(_db: &DatabaseConnection) -> Result<(), sea_orm::DbErr> {
info!("✅ Database schema initialization complete (no tables to create).");
pub async fn init(db: &DatabaseConnection) -> Result<(), sea_orm::DbErr> {
info!("🚀 Initializing database schema...");
let backend = db.get_database_backend();
let schema = Schema::new(backend);
let tables = vec![(
"ImageCache",
schema
.create_table_from_entity(image_cache::Entity)
.if_not_exists()
.to_owned(),
)];
for (name, stmt) in tables {
match db.execute(backend.build(&stmt)).await {
Ok(_) => info!(" ✓ Table '{}' checked/created", name),
Err(e) => {
tracing::error!(" [!] Failed to create table '{}': {}", name, e);
return Err(e);
}
}
}
let index_sql =
"CREATE INDEX IF NOT EXISTS idx_image_cache_cdn_url ON \"ImageCache\" (cdn_url)";
match db.execute(Statement::from_string(backend, index_sql)).await {
Ok(_) => info!(" ✓ Index 'idx_image_cache_cdn_url' ensured"),
Err(e) => {
let err_str = e.to_string();
if err_str.contains("already exists") || err_str.contains("duplicate") {
info!(" ✓ Index 'idx_image_cache_cdn_url' already exists");
} else {
tracing::error!(" [!] Failed to create index on ImageCache: {}", e);
}
}
}
info!("✅ Database schema initialization complete.");
Ok(()) Ok(())
} }
src/config/mod.rs
+14 -24
View File
@@ -32,8 +32,8 @@ pub struct AppConfig {
#[serde(default = "default_env")] #[serde(default = "default_env")]
pub environment: String, pub environment: String,
/// Allowed CORS origins (comma-separated) /// Allowed CORS origins (comma-separated in env var, parsed to Vec)
#[serde(default)] #[serde(default, deserialize_with = "deserialize_comma_separated")]
pub cors_origins: Vec<String>, pub cors_origins: Vec<String>,
/// Log level (trace, debug, info, warn, error) /// Log level (trace, debug, info, warn, error)
@@ -47,10 +47,6 @@ pub struct AppConfig {
#[serde(default)] #[serde(default)]
pub db: DbConfig, pub db: DbConfig,
/// Max concurrent image processing tasks
#[serde(default = "default_image_processing_concurrency")]
pub image_processing_concurrency: usize,
/// Domain/URL values that may change between deployments /// Domain/URL values that may change between deployments
#[serde(default)] #[serde(default)]
pub urls: UrlConfig, pub urls: UrlConfig,
@@ -60,18 +56,12 @@ pub struct AppConfig {
pub struct UrlConfig { pub struct UrlConfig {
#[serde(default = "default_site_url")] #[serde(default = "default_site_url")]
pub site_url: String, pub site_url: String,
#[serde(default = "default_picser_api_url")]
pub picser_api_url: String,
#[serde(default = "default_fallback_upload_api_url")]
pub fallback_upload_api_url: String,
} }
impl Default for UrlConfig { impl Default for UrlConfig {
fn default() -> Self { fn default() -> Self {
Self { Self {
site_url: default_site_url(), site_url: default_site_url(),
picser_api_url: default_picser_api_url(),
fallback_upload_api_url: default_fallback_upload_api_url(),
} }
} }
} }
@@ -181,20 +171,21 @@ fn default_log_level() -> String {
"info".to_string() "info".to_string()
} }
fn default_image_processing_concurrency() -> usize {
5
}
fn default_site_url() -> String { fn default_site_url() -> String {
"https://asepharyana.my.id".to_string() "https://asepharyana.my.id".to_string()
} }
fn default_picser_api_url() -> String { /// Deserialize a comma-separated string into a Vec<String>.
"https://picser.asepharyana.my.id/api/upload".to_string() /// Used for fields like cors_origins that come from a single env var.
} fn deserialize_comma_separated<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error>
where
fn default_fallback_upload_api_url() -> String { D: serde::Deserializer<'de>,
"https://upload.asepharyana.my.id/api/upload".to_string() {
let s = String::deserialize(deserializer)?;
Ok(s.split(',')
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.collect())
} }
fn default_db_max_connections() -> u32 { fn default_db_max_connections() -> u32 {
@@ -245,8 +236,7 @@ impl AppConfig {
.add_source( .add_source(
Environment::with_prefix("APP") Environment::with_prefix("APP")
.separator("__") .separator("__")
.try_parsing(true) .try_parsing(true),
.list_separator(","),
) )
// Map legacy env vars to new config structure // Map legacy env vars to new config structure
.set_override_option("database_url", env::var("DATABASE_URL").ok())? .set_override_option("database_url", env::var("DATABASE_URL").ok())?
src/domain/entity/anime.rs
-111
View File
@@ -290,115 +290,4 @@ pub struct FilterAnimeItem {
pub anime_url: String, pub anime_url: String,
} }
// ============================================================================
// TRAITS
// ============================================================================
/// Trait for types that have a poster image URL
pub trait HasPoster {
fn poster(&self) -> &str;
fn set_poster(&mut self, url: String);
}
// ============================================================================
// TRAIT IMPLEMENTATIONS
// ============================================================================
impl HasPoster for OngoingAnimeItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for CompleteAnimeItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for CompleteAnimeListItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for OngoingAnimeListItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for LatestAnimeItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for SearchAnimeItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for GenreAnimeItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for Recommendation {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for Anime2Item {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for OngoingAnimeItemWithScore {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for FilterAnimeItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
src/domain/entity/komik.rs
-11
View File
@@ -3,8 +3,6 @@
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use utoipa::ToSchema; use utoipa::ToSchema;
use crate::domain::entity::anime::HasPoster;
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] #[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct KomikGenre { pub struct KomikGenre {
pub name: String, pub name: String,
@@ -53,12 +51,3 @@ pub struct KomikItem {
pub r#type: String, pub r#type: String,
pub komik_url: String, pub komik_url: String,
} }
impl HasPoster for KomikItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
src/domain/repository/image_cache.rs
-19
View File
@@ -1,19 +0,0 @@
//! Repository trait for image cache (original → CDN mapping).
use async_trait::async_trait;
/// Repository for managing cached image URL mappings.
#[async_trait]
pub trait ImageCacheRepository: Send + Sync {
async fn get_from_redis(&self, key: &str) -> Option<String>;
async fn set_in_redis(&self, key: &str, value: &str, ttl: u64) -> Result<(), String>;
async fn get_from_db(&self, original_url: &str) -> Result<Option<String>, String>;
async fn save_to_db(&self, original_url: &str, cdn_url: &str) -> Result<(), String>;
async fn find_original_from_cdn(&self, cdn_url: &str) -> Result<Option<String>, String>;
async fn delete_from_db(&self, original_url: &str) -> Result<(), String>;
async fn delete_from_redis(&self, key: &str) -> Result<(), String>;
async fn get_lock(&self, key: &str) -> bool;
async fn set_lock(&self, key: &str, ttl: u64) -> Result<(), String>;
async fn release_lock(&self, key: &str) -> Result<(), String>;
async fn invalidate_api_caches(&self, patterns: Vec<&str>) -> Result<(), String>;
}
src/domain/repository/mod.rs
-2
View File
@@ -1,5 +1,3 @@
pub mod image_cache;
pub mod scraping; pub mod scraping;
pub use image_cache::ImageCacheRepository;
pub use scraping::ScrapingRepository; pub use scraping::ScrapingRepository;
src/events/bus.rs
-9
View File
@@ -143,13 +143,4 @@ impl Event for OrderCreated {
const NAME: &'static str = "order.created"; const NAME: &'static str = "order.created";
} }
/// Image repaired event.
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
pub struct ImageRepaired {
pub original_url: String,
pub cdn_url: String,
}
impl Event for ImageRepaired {
const NAME: &'static str = "image.repaired";
}
src/infrastructure/mod.rs
-2
View File
@@ -1,6 +1,4 @@
pub mod cache; pub mod cache;
pub mod persistence;
pub mod repository; pub mod repository;
pub mod scraping; pub mod scraping;
pub mod services;
pub mod utils; pub mod utils;
src/infrastructure/persistence/entities/image_cache.rs
-71
View File
@@ -1,71 +0,0 @@
//! `SeaORM` Entity for ImageCache - stores URL mappings for CDN caching
use sea_orm::entity::prelude::*;
use serde::{Deserialize, Serialize};
#[derive(Copy, Clone, Default, Debug, DeriveEntity)]
pub struct Entity;
impl EntityName for Entity {
fn table_name(&self) -> &str {
"ImageCache"
}
}
#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel, Eq, Serialize, Deserialize)]
pub struct Model {
pub id: String,
pub original_url: String,
pub cdn_url: String,
pub created_at: DateTimeUtc,
pub expires_at: Option<DateTimeUtc>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
pub enum Column {
Id,
#[sea_orm(column_name = "original_url")]
OriginalUrl,
#[sea_orm(column_name = "cdn_url")]
CdnUrl,
#[sea_orm(column_name = "created_at")]
CreatedAt,
#[sea_orm(column_name = "expires_at")]
ExpiresAt,
}
#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)]
pub enum PrimaryKey {
Id,
}
impl PrimaryKeyTrait for PrimaryKey {
type ValueType = String;
fn auto_increment() -> bool {
false
}
}
#[derive(Copy, Clone, Debug, EnumIter)]
pub enum Relation {}
impl ColumnTrait for Column {
type EntityName = Entity;
fn def(&self) -> ColumnDef {
match self {
Self::Id => ColumnType::String(StringLen::N(36u32)).def(),
Self::OriginalUrl => ColumnType::String(StringLen::N(512u32)).def().unique(),
Self::CdnUrl => ColumnType::String(StringLen::N(512u32)).def(),
Self::CreatedAt => ColumnType::TimestampWithTimeZone.def(),
Self::ExpiresAt => ColumnType::TimestampWithTimeZone.def().null(),
}
}
}
impl RelationTrait for Relation {
fn def(&self) -> RelationDef {
match *self {}
}
}
impl ActiveModelBehavior for ActiveModel {}
src/infrastructure/persistence/entities/mod.rs
-1
View File
@@ -1 +0,0 @@
pub mod image_cache;
src/infrastructure/persistence/mod.rs
-1
View File
@@ -1 +0,0 @@
pub mod entities;
src/infrastructure/repository/alqanime.rs
+4 -4
View File
@@ -5,9 +5,9 @@ use tracing::warn;
use crate::domain::error::ScrapingError; use crate::domain::error::ScrapingError;
use crate::domain::repository::ScrapingRepository; use crate::domain::repository::ScrapingRepository;
use crate::infrastructure::scraping::proxy_fetch::fetch_with_proxy_only; use crate::infrastructure::scraping::proxy_fetch::fetch_with_proxy;
const BASE_URL: &str = "https://alqanime.net"; const BASE_URL: &str = "https://alqanime.si";
pub struct AlqanimeRepository; pub struct AlqanimeRepository;
@@ -37,7 +37,7 @@ impl AlqanimeRepository {
} }
pub fn detail_url(&self, slug: &str) -> String { pub fn detail_url(&self, slug: &str) -> String {
format!("{}/{}/", BASE_URL, slug) format!("{}/anime/{}/", BASE_URL, slug)
} }
pub fn genre_page_url(&self, genre_slug: &str, page: u32) -> String { pub fn genre_page_url(&self, genre_slug: &str, page: u32) -> String {
@@ -85,7 +85,7 @@ impl AlqanimeRepository {
#[async_trait] #[async_trait]
impl ScrapingRepository for AlqanimeRepository { impl ScrapingRepository for AlqanimeRepository {
async fn fetch_html(&self, url: &str) -> Result<String, ScrapingError> { async fn fetch_html(&self, url: &str) -> Result<String, ScrapingError> {
let response = fetch_with_proxy_only(url) let response = fetch_with_proxy(url)
.await .await
.map_err(|e| ScrapingError::Http(format!("Alqanime fetch failed: {}", e)))?; .map_err(|e| ScrapingError::Http(format!("Alqanime fetch failed: {}", e)))?;
if response.data.trim().is_empty() { if response.data.trim().is_empty() {
src/infrastructure/repository/image_cache_seaorm.rs
-131
View File
@@ -1,131 +0,0 @@
//! SeaORM-backed implementation of ImageCacheRepository.
use async_trait::async_trait;
use chrono::Utc;
use deadpool_redis::Pool as RedisPool;
use sea_orm::{ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, Set};
use std::sync::Arc;
use crate::domain::repository::ImageCacheRepository;
use crate::infrastructure::cache::redis::Cache;
use crate::infrastructure::persistence::entities::image_cache;
pub struct SeaOrmImageCacheRepository {
db: Arc<DatabaseConnection>,
redis: RedisPool,
}
impl SeaOrmImageCacheRepository {
pub fn new(db: Arc<DatabaseConnection>, redis: RedisPool) -> Self {
Self { db, redis }
}
}
#[async_trait]
impl ImageCacheRepository for SeaOrmImageCacheRepository {
async fn get_from_redis(&self, key: &str) -> Option<String> {
Cache::new(&self.redis).get::<String>(key).await
}
async fn set_in_redis(&self, key: &str, value: &str, ttl: u64) -> Result<(), String> {
Cache::new(&self.redis)
.set_with_ttl(key, &value, ttl)
.await
.map_err(|e| e.to_string())
}
async fn get_from_db(&self, original_url: &str) -> Result<Option<String>, String> {
let entry = image_cache::Entity::find()
.filter(image_cache::Column::OriginalUrl.eq(original_url))
.one(self.db.as_ref())
.await
.map_err(|e| e.to_string())?;
Ok(entry.map(|m| m.cdn_url))
}
async fn save_to_db(&self, original_url: &str, cdn_url: &str) -> Result<(), String> {
let model = image_cache::ActiveModel {
id: Set(uuid::Uuid::new_v4().to_string()),
original_url: Set(original_url.to_string()),
cdn_url: Set(cdn_url.to_string()),
created_at: Set(Utc::now()),
expires_at: Set(None),
};
model
.insert(self.db.as_ref())
.await
.map_err(|e| e.to_string())?;
Ok(())
}
async fn find_original_from_cdn(&self, cdn_url: &str) -> Result<Option<String>, String> {
let entry = image_cache::Entity::find()
.filter(image_cache::Column::CdnUrl.eq(cdn_url))
.one(self.db.as_ref())
.await
.map_err(|e| e.to_string())?;
Ok(entry.map(|m| m.original_url))
}
async fn delete_from_db(&self, original_url: &str) -> Result<(), String> {
image_cache::Entity::delete_many()
.filter(image_cache::Column::OriginalUrl.eq(original_url))
.exec(self.db.as_ref())
.await
.map_err(|e| e.to_string())?;
Ok(())
}
async fn delete_from_redis(&self, key: &str) -> Result<(), String> {
Cache::new(&self.redis)
.delete(key)
.await
.map_err(|e| e.to_string())
}
async fn get_lock(&self, key: &str) -> bool {
Cache::new(&self.redis).get::<bool>(key).await.is_some()
}
async fn set_lock(&self, key: &str, ttl: u64) -> Result<(), String> {
Cache::new(&self.redis)
.set_with_ttl(key, &true, ttl)
.await
.map_err(|e| e.to_string())
}
async fn release_lock(&self, key: &str) -> Result<(), String> {
self.delete_from_redis(key).await
}
async fn invalidate_api_caches(&self, patterns: Vec<&str>) -> Result<(), String> {
use deadpool_redis::redis::{cmd, AsyncCommands};
let mut conn = self.redis.get().await.map_err(|e| e.to_string())?;
for pattern in patterns {
let mut cursor: u64 = 0;
loop {
let (new_cursor, keys): (u64, Vec<String>) = cmd("SCAN")
.arg(cursor)
.arg("MATCH")
.arg(pattern)
.arg("COUNT")
.arg(100)
.query_async(&mut *conn)
.await
.map_err(|e| e.to_string())?;
if !keys.is_empty() {
let _: usize = conn.del(&keys).await.map_err(|e| e.to_string())?;
}
cursor = new_cursor;
if cursor == 0 {
break;
}
}
}
Ok(())
}
}
src/infrastructure/repository/mod.rs
-2
View File
@@ -1,12 +1,10 @@
pub mod alqanime; pub mod alqanime;
pub mod image_cache_seaorm;
pub mod komik; pub mod komik;
pub mod otakudesu; pub mod otakudesu;
pub mod parsers; pub mod parsers;
pub mod proxy; pub mod proxy;
pub use alqanime::AlqanimeRepository; pub use alqanime::AlqanimeRepository;
pub use image_cache_seaorm::SeaOrmImageCacheRepository;
pub use komik::KomikRepository; pub use komik::KomikRepository;
pub use otakudesu::OtakudesuRepository; pub use otakudesu::OtakudesuRepository;
pub use proxy::ProxyRepository; pub use proxy::ProxyRepository;
src/infrastructure/repository/parsers/alqanime_parser.rs
+69 -12
View File
@@ -1,7 +1,7 @@
use crate::domain::entity::anime::{ use crate::domain::entity::anime::{
CompleteAnimeItem, DetailGenre, FilterAnimeItem, Genre, GenreAnimeItem, HasPoster, CompleteAnimeItem, DetailGenre, FilterAnimeItem, Genre, GenreAnimeItem, LatestAnimeItem,
LatestAnimeItem, OngoingAnimeItem, OngoingAnimeItemWithScore, Pagination, OngoingAnimeItem, OngoingAnimeItemWithScore, Pagination, PaginationWithStringPages,
PaginationWithStringPages, SearchAnimeItem, SearchAnimeItem,
}; };
use crate::domain::error::ScrapingError; use crate::domain::error::ScrapingError;
use crate::infrastructure::scraping::parsing_utils::parse_html; use crate::infrastructure::scraping::parsing_utils::parse_html;
@@ -9,6 +9,16 @@ use crate::infrastructure::scraping::parsing_utils::{
attr, extract_slug, selector, text, text_from_or, attr, extract_slug, selector, text, text_from_or,
}; };
/// An episode entry from the detail page episode list.
#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema, Debug, Clone)]
pub struct AlqEpisode {
pub episode: String,
pub title: String,
pub url: String,
pub date: String,
pub download_url: Option<String>,
}
/// Parser-specific types for Alqanime detail data /// Parser-specific types for Alqanime detail data
#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema, Debug, Clone)] #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema, Debug, Clone)]
pub struct AlqLink { pub struct AlqLink {
@@ -31,15 +41,6 @@ pub struct AlqRecommendation {
pub r#type: String, pub r#type: String,
} }
impl HasPoster for AlqRecommendation {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
use regex::Regex; use regex::Regex;
use scraper::Selector; use scraper::Selector;
use std::sync::LazyLock; use std::sync::LazyLock;
@@ -61,6 +62,7 @@ pub struct AlqDetailData {
pub batch: Vec<AlqDownloadItem>, pub batch: Vec<AlqDownloadItem>,
pub ova: Vec<AlqDownloadItem>, pub ova: Vec<AlqDownloadItem>,
pub downloads: Vec<AlqDownloadItem>, pub downloads: Vec<AlqDownloadItem>,
pub episodes: Vec<AlqEpisode>,
} }
static ITEM_SELECTOR: LazyLock<Selector> = LazyLock::new(|| Selector::parse("article.bs").unwrap()); static ITEM_SELECTOR: LazyLock<Selector> = LazyLock::new(|| Selector::parse("article.bs").unwrap());
@@ -76,6 +78,16 @@ static NEXT_SELECTOR: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".pagination .next").unwrap()); LazyLock::new(|| Selector::parse(".pagination .next").unwrap());
static SLUG_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"/([^/]+)/?$").unwrap()); static SLUG_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"/([^/]+)/?$").unwrap());
static GENRE_SLUG_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"genre-(.+)$").unwrap()); static GENRE_SLUG_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"genre-(.+)$").unwrap());
static EPISODE_LIST_SELECTOR: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".eplister ul li").unwrap());
static EP_NUM_SELECTOR: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".epl-num").unwrap());
static EP_TITLE_SELECTOR: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".epl-title").unwrap());
static EP_DATE_SELECTOR: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".epl-date").unwrap());
static EP_DOWNLOAD_SELECTOR: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(r#"a[rel="nofollow"][aria-label="Download"]"#).unwrap());
pub fn parse_ongoing_anime(html: &str) -> Result<Vec<OngoingAnimeItem>, ScrapingError> { pub fn parse_ongoing_anime(html: &str) -> Result<Vec<OngoingAnimeItem>, ScrapingError> {
let items = parse_ongoing_anime_with_score(html)?; let items = parse_ongoing_anime_with_score(html)?;
Ok(items Ok(items
@@ -556,6 +568,48 @@ pub fn parse_pagination_with_string(
Ok(pagination) Ok(pagination)
} }
/// Parse episode list from the detail page's `.eplister` section.
pub fn parse_episode_list(html: &str) -> Result<Vec<AlqEpisode>, ScrapingError> {
let document = parse_html(html);
let mut episodes = Vec::new();
for element in document.select(&EPISODE_LIST_SELECTOR) {
let episode = text_from_or(&element, &EP_NUM_SELECTOR, "");
let title = text_from_or(&element, &EP_TITLE_SELECTOR, "");
let date = text_from_or(&element, &EP_DATE_SELECTOR, "");
let url = element
.select(&LINK_SELECTOR)
.next()
.and_then(|e| e.value().attr("href"))
.unwrap_or("")
.to_string();
if !title.is_empty() {
episodes.push(AlqEpisode {
episode,
title,
url,
date,
download_url: None,
});
}
}
Ok(episodes)
}
/// Parse download URL from an individual episode page.
pub fn parse_episode_download(html: &str) -> Result<Option<String>, ScrapingError> {
let document = parse_html(html);
let download_url = document
.select(&EP_DOWNLOAD_SELECTOR)
.next()
.and_then(|e| e.value().attr("href"))
.map(|s| s.to_string());
Ok(download_url)
}
pub fn parse_anime_detail(html: &str) -> Result<AlqDetailData, ScrapingError> { pub fn parse_anime_detail(html: &str) -> Result<AlqDetailData, ScrapingError> {
let document = parse_html(html); let document = parse_html(html);
@@ -748,6 +802,8 @@ pub fn parse_anime_detail(html: &str) -> Result<AlqDetailData, ScrapingError> {
}); });
} }
let episodes = parse_episode_list(html).unwrap_or_default();
Ok(AlqDetailData { Ok(AlqDetailData {
title, title,
alternative_title, alternative_title,
@@ -764,6 +820,7 @@ pub fn parse_anime_detail(html: &str) -> Result<AlqDetailData, ScrapingError> {
batch, batch,
ova, ova,
downloads, downloads,
episodes,
}) })
} }
src/infrastructure/services/images/cache.rs
-1142
View File
File diff suppressed because it is too large Load Diff
src/infrastructure/services/images/mod.rs
-1
View File
@@ -1 +0,0 @@
pub mod cache;
src/infrastructure/services/mod.rs
-1
View File
@@ -1 +0,0 @@
pub mod images;
src/lib.rs
-1
View File
@@ -27,4 +27,3 @@ pub mod bootstrap;
pub mod config; pub mod config;
pub mod events; pub mod events;
pub mod observability; pub mod observability;
pub mod scheduler;
src/observability/openapi_modules.rs
-2
View File
@@ -41,8 +41,6 @@ use utoipa::OpenApi;
crate::presentation::handler::komik::search_slug, crate::presentation::handler::komik::search_slug,
crate::presentation::handler::komik::search_slug_page, crate::presentation::handler::komik::search_slug_page,
// Proxy module handlers // Proxy module handlers
crate::presentation::handler::proxy::fetch_with_proxy_only,
crate::presentation::handler::proxy::image_cache,
), ),
components( components(
schemas( schemas(
src/presentation/handler/anime.rs
+1 -6
View File
@@ -78,12 +78,7 @@ pub struct GenreListResponse {
// ============================================================================ // ============================================================================
fn make_use_cases(state: &Arc<AppState>) -> AnimeUseCases { fn make_use_cases(state: &Arc<AppState>) -> AnimeUseCases {
AnimeUseCases::new( AnimeUseCases::new(OtakudesuRepository::new(), state.redis_pool.clone())
OtakudesuRepository::new(),
state.redis_pool.clone(),
state.db.clone(),
Some(state.image_processing_semaphore.clone()),
)
} }
// ============================================================================ // ============================================================================
src/presentation/handler/anime2.rs
+1 -6
View File
@@ -39,12 +39,7 @@ pub struct FilterQuery {
// ============================================================================ // ============================================================================
fn make_use_cases(state: &Arc<AppState>) -> Anime2UseCases { fn make_use_cases(state: &Arc<AppState>) -> Anime2UseCases {
Anime2UseCases::new( Anime2UseCases::new(AlqanimeRepository::new(), state.redis_pool.clone())
AlqanimeRepository::new(),
state.redis_pool.clone(),
state.db.clone(),
Some(state.image_processing_semaphore.clone()),
)
} }
// ============================================================================ // ============================================================================
src/presentation/handler/komik.rs
+1 -6
View File
@@ -25,12 +25,7 @@ use crate::presentation::state::AppState;
// ============================================================================ // ============================================================================
fn make_use_cases(state: &Arc<AppState>) -> KomikUseCases { fn make_use_cases(state: &Arc<AppState>) -> KomikUseCases {
KomikUseCases::new( KomikUseCases::new(KomikRepository::new(), state.redis_pool.clone())
KomikRepository::new(),
state.redis_pool.clone(),
state.db.clone(),
Some(state.image_processing_semaphore.clone()),
)
} }
// ============================================================================ // ============================================================================
src/presentation/handler/proxy.rs
+1 -131
View File
@@ -1,131 +1 @@
//! Proxy and image cache API handlers. //! Proxy handlers.
use std::sync::Arc;
use axum::extract::{Json, Query, State};
use axum::response::Response;
use serde::Deserialize;
use tracing::info;
use utoipa::{IntoParams, ToSchema};
use crate::application::proxy::use_cases::{
AuditImageCacheResult, ImageCacheResult, ProxyUseCases,
};
use crate::infrastructure::repository::image_cache_seaorm::SeaOrmImageCacheRepository;
use crate::infrastructure::repository::ProxyRepository;
use crate::presentation::error::AppError;
use crate::presentation::state::AppState;
// ============================================================================
// Request DTOs
// ============================================================================
/// Query parameters for proxy fetch (GET).
#[derive(Debug, Deserialize, IntoParams, ToSchema)]
pub struct ProxyParams {
/// URL to fetch via proxy.
pub url: String,
}
/// Request body for image cache (POST).
#[derive(Debug, Deserialize, ToSchema)]
pub struct ImageCacheRequest {
/// Original image URL to cache.
pub url: String,
/// If true, returns original URL immediately and caches in background.
#[serde(default)]
pub lazy: bool,
}
/// Request body for auditing image cache (POST).
#[derive(Debug, Deserialize, ToSchema)]
pub struct AuditImageCacheRequest {
/// Original image URL to audit.
pub url: String,
}
// ============================================================================
// Helper
// ============================================================================
fn make_use_cases(state: &Arc<AppState>) -> ProxyUseCases {
let repo = Arc::new(SeaOrmImageCacheRepository::new(
state.db.clone(),
state.redis_pool.clone(),
));
ProxyUseCases::new(ProxyRepository::new(), repo)
}
// ============================================================================
// Handlers
// ============================================================================
/// GET /api/proxy/croxy — Fetch a URL through the proxy and return raw bytes.
#[utoipa::path(
get,
path = "/api/proxy/croxy",
tag = "proxy",
operation_id = "proxy_croxy",
params(ProxyParams),
responses(
(status = 200, description = "Proxied response", body = Vec::<u8>, content_type = "application/octet-stream"),
(status = 500, description = "Internal Server Error"),
)
)]
pub async fn fetch_with_proxy_only(
State(state): State<Arc<AppState>>,
Query(params): Query<ProxyParams>,
) -> Result<Response, AppError> {
info!("Handling proxy fetch for URL: {}", params.url);
let response = make_use_cases(&state)
.fetch_with_proxy_only(params.url)
.await?;
Ok(response)
}
/// POST /api/proxy/image-cache — Cache an image URL to CDN.
#[utoipa::path(
post,
path = "/api/proxy/image-cache",
tag = "proxy",
operation_id = "proxy_image_cache",
request_body = ImageCacheRequest,
responses(
(status = 200, description = "Image cache result", body = ImageCacheResult),
(status = 500, description = "Internal Server Error"),
)
)]
pub async fn image_cache(
State(state): State<Arc<AppState>>,
Json(req): Json<ImageCacheRequest>,
) -> Result<Json<ImageCacheResult>, AppError> {
info!(
"Handling image cache for URL: {} (lazy: {})",
req.url, req.lazy
);
let result = make_use_cases(&state)
.image_cache(req.url, req.lazy)
.await?;
Ok(Json(result))
}
/// POST /api/proxy/image-cache/audit — Audit and repair a cached image.
#[utoipa::path(
post,
path = "/api/proxy/image-cache/audit",
tag = "proxy",
operation_id = "proxy_image_cache_audit",
request_body = AuditImageCacheRequest,
responses(
(status = 200, description = "Audit result", body = AuditImageCacheResult),
(status = 500, description = "Internal Server Error"),
)
)]
pub async fn audit_image_cache(
State(state): State<Arc<AppState>>,
Json(req): Json<AuditImageCacheRequest>,
) -> Result<Json<AuditImageCacheResult>, AppError> {
info!("Handling audit cache for URL: {}", req.url);
let result = make_use_cases(&state).audit_image_cache(req.url).await?;
Ok(Json(result))
}
src/presentation/router.rs
+1 -12
View File
@@ -154,18 +154,7 @@ pub fn build_router(app_state: Arc<AppState>) -> anyhow::Result<Router> {
axum::routing::get(crate::presentation::handler::komik::search_slug_page), axum::routing::get(crate::presentation::handler::komik::search_slug_page),
) )
// Proxy routes // Proxy routes
.route( // (all proxy routes removed)
"/api/proxy/croxy",
axum::routing::get(crate::presentation::handler::proxy::fetch_with_proxy_only),
)
.route(
"/api/proxy/image-cache",
axum::routing::post(crate::presentation::handler::proxy::image_cache),
)
.route(
"/api/proxy/image-cache/audit",
axum::routing::post(crate::presentation::handler::proxy::audit_image_cache),
)
// Health // Health
.route( .route(
"/health", "/health",
src/presentation/state.rs
-3
View File
@@ -6,7 +6,6 @@ use deadpool_redis::Pool;
use sea_orm::DatabaseConnection; use sea_orm::DatabaseConnection;
use crate::events::bus::EventBus; use crate::events::bus::EventBus;
use crate::infrastructure::repository::SeaOrmImageCacheRepository;
/// Shared application state injected into every handler via Axum State. /// Shared application state injected into every handler via Axum State.
/// ///
@@ -16,9 +15,7 @@ use crate::infrastructure::repository::SeaOrmImageCacheRepository;
pub struct AppState { pub struct AppState {
pub redis_pool: Pool, pub redis_pool: Pool,
pub db: Arc<DatabaseConnection>, pub db: Arc<DatabaseConnection>,
pub image_processing_semaphore: Arc<tokio::sync::Semaphore>,
pub event_bus: Arc<EventBus>, pub event_bus: Arc<EventBus>,
pub image_cache_repo: Arc<SeaOrmImageCacheRepository>,
} }
impl AppState { impl AppState {
src/scheduler/cleanup_cache.rs
-222
View File
@@ -1,222 +0,0 @@
//! Scheduled task for cleaning up old cached data.
use async_trait::async_trait;
use sea_orm::*;
use std::sync::Arc;
use tracing::{info, warn};
use crate::infrastructure::cache::redis::Cache;
use crate::infrastructure::cache::redis_pool::get_redis_pool;
use crate::infrastructure::persistence::entities::image_cache;
use super::ScheduledTask;
/// Cleanup old cache data to prevent disk/memory bloat.
/// Runs daily at 2 AM to clean:
/// - Old image cache entries (>30 days)
/// - Orphaned cache keys in Redis
/// - Expired data without TTL
pub struct CleanupOldCache {
db: Arc<DatabaseConnection>,
}
impl CleanupOldCache {
pub fn new(db: Arc<DatabaseConnection>) -> Self {
Self { db }
}
}
#[async_trait]
impl ScheduledTask for CleanupOldCache {
fn name(&self) -> &'static str {
"cleanup_old_cache"
}
fn schedule(&self) -> &'static str {
// Daily at 2 AM
"0 0 2 * * *"
}
async fn run(&self) {
tracing::debug!("🧹 Starting old cache cleanup...");
let mut total_cleaned = 0;
// 1. Clean old image cache (>30 days)
match self.cleanup_old_images(30).await {
Ok(count) => {
if count > 0 {
info!("✓ Cleaned {} old image cache entries", count);
}
total_cleaned += count;
}
Err(e) => {
warn!("Failed to clean old image cache: {}", e);
}
}
// 2. Clean orphaned Redis keys
match self.cleanup_orphaned_redis_keys().await {
Ok(count) => {
if count > 0 {
info!("✓ Cleaned {} orphaned Redis keys", count);
}
total_cleaned += count;
}
Err(e) => {
warn!("Failed to clean orphaned Redis keys: {}", e);
}
}
// 3. Compact Redis memory
match self.compact_redis_memory().await {
Ok(()) => {
if total_cleaned > 0 {
info!("✓ Redis memory compacted");
}
}
Err(e) => {
warn!("Failed to compact Redis memory: {}", e);
}
}
if total_cleaned > 0 {
info!("🎉 Cache cleanup complete: {} items cleaned", total_cleaned);
}
}
}
impl CleanupOldCache {
/// Remove image cache entries older than specified days.
async fn cleanup_old_images(&self, days: i64) -> Result<usize, String> {
use chrono::{Duration, Utc};
let cutoff = Utc::now() - Duration::days(days);
// Find old entries
let old_images = image_cache::Entity::find()
.filter(image_cache::Column::CreatedAt.lt(cutoff))
.all(self.db.as_ref())
.await
.map_err(|e| e.to_string())?;
let count = old_images.len();
if count == 0 {
return Ok(0);
}
// Delete from database
let ids: Vec<String> = old_images.iter().map(|img| img.id.clone()).collect();
image_cache::Entity::delete_many()
.filter(image_cache::Column::Id.is_in(ids))
.exec(self.db.as_ref())
.await
.map_err(|e| e.to_string())?;
// Also clean from Redis
let redis_pool = get_redis_pool().map_err(|e| e.to_string())?;
let cache = Cache::new(redis_pool);
for img in old_images {
let cache_key = format!("img_cache:{}", Self::hash_url(&img.original_url));
let _ = cache.delete(&cache_key).await;
}
Ok(count)
}
/// Clean orphaned Redis keys (keys without TTL that shouldn't exist).
async fn cleanup_orphaned_redis_keys(&self) -> Result<usize, String> {
use deadpool_redis::redis::AsyncCommands;
let pool = get_redis_pool().map_err(|e| e.to_string())?;
let mut conn = pool
.get()
.await
.map_err(|e| format!("Failed to get Redis connection: {}", e))?;
let mut cleaned = 0;
// Find keys without TTL (should not exist)
let patterns = vec!["anime:*", "komik:*", "user:*:profile", "img_cache:*"];
for pattern in patterns {
let keys = {
let mut iter: deadpool_redis::redis::AsyncIter<'_, String> = conn
.scan_match(pattern)
.await
.map_err(|e| format!("Failed to scan keys: {}", e))?;
let mut keys = Vec::new();
while let Some(key_result) = iter.next_item().await {
if let Ok(key) = key_result {
keys.push(key);
}
}
keys
};
for key in keys {
let ttl: i64 = conn.ttl(key.as_str()).await.unwrap_or(-1);
// TTL = -1 means no expiration (orphaned)
// TTL = -2 means key doesn't exist
if ttl == -1 {
// Set a default TTL of 7 days for orphaned keys
let _: () = conn.expire(key.as_str(), 604800).await.unwrap_or(());
cleaned += 1;
}
}
}
Ok(cleaned)
}
/// Compact Redis memory to free up fragmented space.
async fn compact_redis_memory(&self) -> Result<(), String> {
let pool = get_redis_pool().map_err(|e| e.to_string())?;
let mut conn = pool
.get()
.await
.map_err(|e| format!("Failed to get Redis connection: {}", e))?;
// Run MEMORY PURGE command - using cmd method on connection
let _: String = deadpool_redis::redis::cmd("MEMORY")
.arg("PURGE")
.query_async(&mut *conn)
.await
.map_err(|e| format!("Failed to purge memory: {}", e))?;
Ok(())
}
/// Simple hash function for URL (same as in image_cache.rs).
/// Simple hash function for URL (same as in image_cache.rs).
fn hash_url(url: &str) -> String {
use sha2::{Digest, Sha256};
let mut hasher = Sha256::new();
hasher.update(url.as_bytes());
format!("{:x}", hasher.finalize())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_schedule() {
use sea_orm::{DatabaseBackend, MockDatabase};
let db = MockDatabase::new(DatabaseBackend::Sqlite).into_connection();
let task = CleanupOldCache { db: Arc::new(db) };
assert_eq!(task.schedule(), "0 0 2 * * *");
}
#[test]
fn test_hash_url() {
let hash = CleanupOldCache::hash_url("https://example.com/image.jpg");
assert_eq!(hash.len(), 64); // SHA256 hash length
}
}
src/scheduler/mod.rs
-5
View File
@@ -1,5 +0,0 @@
pub mod cleanup_cache;
pub mod runner;
pub use cleanup_cache::CleanupOldCache;
pub use runner::{ScheduledTask, Scheduler};
src/scheduler/runner.rs
-93
View File
@@ -1,93 +0,0 @@
//! Scheduler implementation using tokio-cron-scheduler.
use async_trait::async_trait;
use std::sync::Arc;
use tokio_cron_scheduler::{Job, JobScheduler};
use tracing::info;
/// Trait for scheduled tasks.
#[async_trait]
pub trait ScheduledTask: Send + Sync {
/// Task name for logging.
fn name(&self) -> &'static str;
/// Cron expression (e.g., "0 * * * * *" for every minute).
fn schedule(&self) -> &'static str;
/// Execute the task.
async fn run(&self);
}
/// Scheduler for running cron jobs.
pub struct Scheduler {
inner: JobScheduler,
}
impl Scheduler {
/// Create a new scheduler.
pub async fn new() -> anyhow::Result<Self> {
let scheduler = JobScheduler::new().await?;
Ok(Self { inner: scheduler })
}
/// Add a task to the scheduler.
pub async fn add<T: ScheduledTask + 'static>(&self, task: T) -> anyhow::Result<()> {
let task = Arc::new(task);
let task_name = task.name();
let schedule = task.schedule();
let job = Job::new_async(schedule, move |_uuid, _lock| {
let task = Arc::clone(&task);
Box::pin(async move {
tracing::debug!("Running scheduled task: {}", task.name());
task.run().await;
})
})?;
self.inner.add(job).await?;
info!("Scheduled task '{}' with cron: {}", task_name, schedule);
Ok(())
}
/// Add a simple job with a closure.
pub async fn add_job<F, Fut>(
&self,
name: &'static str,
schedule: &str,
f: F,
) -> anyhow::Result<()>
where
F: Fn() -> Fut + Send + Sync + 'static,
Fut: std::future::Future<Output = ()> + Send + 'static,
{
let f = Arc::new(f);
let job = Job::new_async(schedule, move |_uuid, _lock| {
let f = Arc::clone(&f);
Box::pin(async move {
info!("Running scheduled job: {}", name);
f().await;
})
})?;
self.inner.add(job).await?;
info!("Scheduled job '{}' with cron: {}", name, schedule);
Ok(())
}
/// Start the scheduler.
pub async fn start(&self) -> anyhow::Result<()> {
info!("Starting scheduler");
self.inner.start().await?;
Ok(())
}
/// Stop the scheduler.
pub async fn shutdown(&mut self) -> anyhow::Result<()> {
info!("Shutting down scheduler");
self.inner.shutdown().await?;
Ok(())
}
}
// Real scheduled tasks with actual implementations
// Real scheduled tasks with actual implementations