asepharyana
8b9bdeb56a
Facebook: - Broaden URL validation to accept ALL real FB shapes: /watch/?v=, /reel/:id, /videos/:id (user vids), /video.php, /story.php, fb.watch, and no-www facebook.com. Old regex rejected /reel, /videos, fb.watch. - Add yt-dlp as the PRIMARY provider (authoritative, returns direct fbcdn progressive URLs with real title/format metadata, no junk). Only falls back to downr.org then Playwright. This kills the 'link exists but 404s' bug where Playwright harvested broken bytestart byte-range chunks alongside the one good URL. - Filter Playwright-harvested bytestart= chunk URLs + dedupe. TeraBox: - Add /proxy/terabox streaming endpoint that forwards the resolver's cookie-authenticated stream through the public API. - Rewrite the returned download URL to the public proxy (raw dlink 403s without cookies; TERABOX_PUBLIC_BASE_URL env override). - Fix: server_port env is APP_SERVER_PORT (single underscore); the OLD stale 4091 bind in tests came from APP__SERVER_PORT not mapping. flake: bump scraper-0.1.0 -> 0.1.1
62 lines
1.8 KiB
Nix
62 lines
1.8 KiB
Nix
{
|
|
description = "Asepharyana Scraper — Rust/Axum web scraper & image proxy API";
|
|
|
|
inputs = {
|
|
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
|
|
flake-utils.url = "github:numtide/flake-utils";
|
|
};
|
|
|
|
outputs = { self, nixpkgs, flake-utils }:
|
|
flake-utils.lib.eachSystem [ "x86_64-linux" ] (system:
|
|
let
|
|
pkgs = import nixpkgs {
|
|
inherit system;
|
|
config.allowUnfree = true;
|
|
};
|
|
|
|
scraper = pkgs.stdenv.mkDerivation {
|
|
name = "scraper-0.1.1";
|
|
src = ./.;
|
|
|
|
nativeBuildInputs = with pkgs; [
|
|
cacert curl gcc gnumake openssl pkg-config python3 libclang
|
|
rustc cargo clang cmake zlib
|
|
];
|
|
buildInputs = with pkgs; [ openssl stdenv.cc.cc.lib ];
|
|
|
|
LIBCLANG_PATH = "${pkgs.libclang.lib}/lib";
|
|
LD_LIBRARY_PATH = "${pkgs.libclang.lib}/lib:${pkgs.stdenv.cc.cc.lib}/lib";
|
|
NIX_ENFORCE_PURITY = "0";
|
|
|
|
SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
|
|
|
|
phases = [ "unpackPhase" "buildPhase" "installPhase" ];
|
|
buildPhase = ''
|
|
export HOME="$TMPDIR" CARGO_HOME="$TMPDIR/.cargo-scraper"
|
|
echo "=== Building scraper ==="
|
|
cargo build --release 2>&1
|
|
'';
|
|
installPhase = ''
|
|
mkdir -p $out/bin
|
|
cp target/release/scraper $out/bin/scraper
|
|
cp $src/scrape_media.py $out/bin/scrape_media.py
|
|
chmod +x $out/bin/scrape_media.py
|
|
'';
|
|
};
|
|
in
|
|
{
|
|
packages = {
|
|
inherit scraper;
|
|
default = scraper;
|
|
};
|
|
|
|
apps.scraper = {
|
|
type = "app";
|
|
program = "${scraper}/bin/scraper";
|
|
};
|
|
|
|
devShells.default = pkgs.mkShell {
|
|
buildInputs = with pkgs; [ rustc cargo clang cmake openssl pkg-config ];
|
|
};
|
|
});
|
|
} |