Compare commits
53
Commits
996b7d06ef
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
88e2a3faad | ||
|
|
75b6d0a527 | ||
|
|
0c2ee3b3cf | ||
|
|
12b6a133a7 | ||
|
|
d2e2b450f6 | ||
|
|
17d877e2c6 | ||
|
|
62021397e0 | ||
|
|
ad7747bfd4 | ||
|
|
66fb92e5af | ||
|
|
673c934f37 | ||
|
|
90d6c7dd6f | ||
|
|
2653e23283 | ||
|
|
91ec588a88 | ||
|
|
864d41d8fc | ||
|
|
c14be68ff8 | ||
|
|
811a68821d | ||
|
|
bf616f6790 | ||
|
|
24dfb1c6b1 | ||
|
|
b9a3fcd828 | ||
|
|
f00b944e56 | ||
|
|
73b0630cbd | ||
|
|
d1771720bd | ||
|
|
7dd2a4f0c7 | ||
|
|
fbc1f1822a | ||
|
|
c768b29e9d | ||
|
|
59535d66b8 | ||
|
|
319d2980c5 | ||
|
|
4acfa3d2e9 | ||
|
|
5bd3d286ce | ||
|
|
bfafcc0a20 | ||
|
|
c26ca806d0 | ||
|
|
bc3d1f2e3f | ||
|
|
f95022142d | ||
|
|
bd766792b1 | ||
|
|
d22f788e3e | ||
|
|
1ecab987a6 | ||
|
|
9a4853a484 | ||
|
|
ad917f6675 | ||
|
|
fab91ad69c | ||
|
|
3501d547c0 | ||
|
|
332853f398 | ||
|
|
4c216b1d9f | ||
|
|
3580001b8a | ||
|
|
6d3696d261 | ||
|
|
1484d5265d | ||
|
|
245ea169ad | ||
|
|
2cf6aa3275 | ||
|
|
d0de4de2d5 | ||
|
|
adbf9b5efa | ||
|
|
a986ce1e08 | ||
|
|
5617d0ff35 | ||
|
|
d7d6ae0f0d | ||
|
|
ea31c4c591 |
+7
-8
@@ -1,14 +1,13 @@
|
|||||||
BOT_TOKEN=isi_token_bot_telegram
|
BOT_TOKENS=isi_token_bot_1,isi_token_bot_2,isi_token_bot_3
|
||||||
ADDITIONAL_BOT_TOKENS=token_cadangan_1,token_cadangan_2,token_cadangan_3
|
|
||||||
STORAGE_CHANNEL_ID=-1001234567890
|
STORAGE_CHANNEL_ID=-1001234567890
|
||||||
BASE_URL=https://tele.asepharyana.my.id
|
BASE_URL=https://upload.asepharyana.my.id
|
||||||
DATABASE_URL=postgresql://user:password@localhost:5432/telegram_uploader
|
DATABASE_URL=postgresql://asephs:***@100.121.180.82:6432/uploader
|
||||||
PORT=3000
|
PORT=4000
|
||||||
NODE_ENV=production
|
NODE_ENV=production
|
||||||
LOG_LEVEL=info
|
LOG_LEVEL=info
|
||||||
RATE_LIMIT_WINDOW_MS=60000
|
RATE_LIMIT_WINDOW_MS=60000
|
||||||
RATE_LIMIT_MAX_REQUESTS=30
|
RATE_LIMIT_MAX_REQUESTS=30
|
||||||
# TRUST_PROXY=true # Uncomment when behind reverse proxy (Traefik, Nginx) for correct client IP detection
|
# TRUST_PROXY=true # Uncomment when behind reverse proxy (Caddy, Nginx) for correct client IP detection
|
||||||
|
|
||||||
# S3-compatible API credentials
|
# S3-compatible API credentials
|
||||||
# S3_ACCESS_KEY=filedrop-admin
|
# S3_ACCESS_KEY=filedrop-admin
|
||||||
@@ -17,8 +16,8 @@ RATE_LIMIT_MAX_REQUESTS=30
|
|||||||
# S3_VHOST_DOMAINS=upload.asepharyana.my.id,upload.asepharyana.web.id
|
# S3_VHOST_DOMAINS=upload.asepharyana.my.id,upload.asepharyana.web.id
|
||||||
|
|
||||||
# Telegram-safe internal chunking for large stored files
|
# Telegram-safe internal chunking for large stored files
|
||||||
# Telegram max upload ~49MB, using 48MB for safety
|
# Telegram getFile download limit is 20 MB; guard rejects > 19922944 (19 MB)
|
||||||
# TELEGRAM_CHUNK_SIZE_BYTES=50331648
|
# TELEGRAM_CHUNK_SIZE_BYTES=19922944
|
||||||
# COMPRESS_CHUNKED_UPLOADS=true
|
# COMPRESS_CHUNKED_UPLOADS=true
|
||||||
# CHUNK_COMPRESSION_MIN_SIZE_BYTES=4096
|
# CHUNK_COMPRESSION_MIN_SIZE_BYTES=4096
|
||||||
|
|
||||||
|
|||||||
@@ -1,48 +0,0 @@
|
|||||||
name: Deploy FileDrop
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
deploy:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Check out repository
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Set up Bun
|
|
||||||
uses: oven-sh/setup-bun@v2
|
|
||||||
with:
|
|
||||||
bun-version: latest
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
run: bun install --frozen-lockfile
|
|
||||||
|
|
||||||
- name: Lint
|
|
||||||
run: bun run lint
|
|
||||||
|
|
||||||
- name: Build
|
|
||||||
run: bun run build
|
|
||||||
|
|
||||||
- name: Deploy to VPS
|
|
||||||
shell: bash
|
|
||||||
env:
|
|
||||||
VPS_HOST: ${{ secrets.VPS_HOST }}
|
|
||||||
VPS_USER: ${{ secrets.VPS_USER }}
|
|
||||||
VPS_SSH_KEY_VALUE: ${{ secrets.VPS_SSH_KEY }}
|
|
||||||
PRODUCTION_ENV: ${{ secrets.PRODUCTION_ENV }}
|
|
||||||
ADMIN_PASSWORD: ${{ secrets.ADMIN_PASSWORD }}
|
|
||||||
run: |
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
key_file="${RUNNER_TEMP:-/tmp}/filedrop_deploy_key"
|
|
||||||
printf '%s\n' "$VPS_SSH_KEY_VALUE" > "$key_file"
|
|
||||||
chmod 600 "$key_file"
|
|
||||||
|
|
||||||
printf '%s\n' "$PRODUCTION_ENV" > .env
|
|
||||||
chmod 600 .env
|
|
||||||
|
|
||||||
VPS_SSH_KEY="$key_file" ./deploy.sh --no-build
|
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
name: Build & Deploy (Nix)
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: deploy
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
id-token: write
|
||||||
|
|
||||||
|
env:
|
||||||
|
VPS_HOST: ${{ secrets.VPS_HOST }}
|
||||||
|
VPS_USER: ${{ secrets.VPS_USER }}
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
lint:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
submodules: false
|
||||||
|
|
||||||
|
- uses: oven-sh/setup-bun@v2
|
||||||
|
with:
|
||||||
|
bun-version: latest
|
||||||
|
|
||||||
|
- name: Install deps
|
||||||
|
run: bun install --frozen-lockfile
|
||||||
|
|
||||||
|
- name: Lint (biome)
|
||||||
|
run: bunx biome check src test
|
||||||
|
|
||||||
|
build-and-deploy:
|
||||||
|
needs: lint
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
submodules: false
|
||||||
|
|
||||||
|
- name: Install Nix
|
||||||
|
uses: DeterminateSystems/nix-installer-action@v22
|
||||||
|
with:
|
||||||
|
determinate: false
|
||||||
|
extra-conf: |
|
||||||
|
sandbox = false
|
||||||
|
accept-flake-config = true
|
||||||
|
|
||||||
|
- name: Cache Nix
|
||||||
|
uses: DeterminateSystems/magic-nix-cache-action@v14
|
||||||
|
with:
|
||||||
|
use-flakehub: false
|
||||||
|
|
||||||
|
- name: Build teleuploader
|
||||||
|
id: build
|
||||||
|
run: |
|
||||||
|
nix build .#teleuploader --impure --option sandbox false --print-build-logs
|
||||||
|
STORE_PATH=$(readlink result)
|
||||||
|
echo "store-path=$STORE_PATH" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "Build OK: $STORE_PATH"
|
||||||
|
|
||||||
|
- name: Setup SSH key
|
||||||
|
env:
|
||||||
|
SSH_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||||
|
run: |
|
||||||
|
mkdir -p ~/.ssh
|
||||||
|
echo "$SSH_KEY" > ~/.ssh/id_ed25519
|
||||||
|
chmod 600 ~/.ssh/id_ed25519
|
||||||
|
sed -i 's/\r$//' ~/.ssh/id_ed25519
|
||||||
|
ssh-keygen -y -f ~/.ssh/id_ed25519 >/dev/null 2>&1 || { echo "SSH key invalid"; exit 1; }
|
||||||
|
ssh-keyscan -H "$VPS_HOST" >> ~/.ssh/known_hosts 2>/dev/null
|
||||||
|
|
||||||
|
- name: Deploy teleuploader to VPS
|
||||||
|
run: |
|
||||||
|
STORE_PATH="${{ steps.build.outputs.store-path }}"
|
||||||
|
echo "=== Copying teleuploader: $STORE_PATH ==="
|
||||||
|
nix copy --to "ssh://$VPS_USER@$VPS_HOST" "$STORE_PATH"
|
||||||
|
|
||||||
|
echo "=== Updating profile ==="
|
||||||
|
ssh "$VPS_USER@$VPS_HOST" "sudo /nix/var/nix/profiles/default/bin/nix-env --profile /nix/var/nix/profiles/teleuploader --set '$STORE_PATH'"
|
||||||
|
|
||||||
|
echo "=== Restarting service ==="
|
||||||
|
ssh "$VPS_USER@$VPS_HOST" "sudo systemctl daemon-reload && sudo systemctl restart teleuploader && sleep 3 && sudo systemctl is-active teleuploader"
|
||||||
|
echo "✅ teleuploader deployed"
|
||||||
|
|
||||||
|
cleanup:
|
||||||
|
# Bersihkan sampah Nix di VPS SETELAH deploy: hapus generasi profile lama
|
||||||
|
# + nix store gc. Profil yang sedang dipakai tidak disentuh.
|
||||||
|
needs: build-and-deploy
|
||||||
|
if: always()
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Nix GC on VPS
|
||||||
|
env:
|
||||||
|
VPS_HOST: ${{ secrets.VPS_HOST }}
|
||||||
|
VPS_USER: ${{ secrets.VPS_USER }}
|
||||||
|
SSH_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||||
|
run: |
|
||||||
|
mkdir -p ~/.ssh
|
||||||
|
echo "$SSH_KEY" > ~/.ssh/id_ed25519
|
||||||
|
chmod 600 ~/.ssh/id_ed25519
|
||||||
|
ssh-keyscan -H "$VPS_HOST" >> ~/.ssh/known_hosts 2>/dev/null
|
||||||
|
ssh "$VPS_USER@$VPS_HOST" "sudo /usr/local/bin/nix-gc-vps.sh" || echo "⚠️ Nix GC gagal (non-fatal)"
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
name: Publish to FlakeHub
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main, master]
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
flakehub-publish:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
id-token: write
|
||||||
|
contents: read
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
- uses: DeterminateSystems/determinate-nix-action@main
|
||||||
|
- uses: DeterminateSystems/flakehub-push@main
|
||||||
|
with:
|
||||||
|
visibility: public
|
||||||
|
rolling: true
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
name: Mirror to Gitea
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main, master]
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
mirror:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- name: Mirror to Gitea
|
||||||
|
env:
|
||||||
|
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||||
|
run: |
|
||||||
|
git remote add gitea "https://oauth2:${GITEA_TOKEN}@git.imrnes.team/MythEclipse/TeleUploader.git"
|
||||||
|
git push --mirror gitea
|
||||||
|
echo "✅ Mirrored to Gitea (MythEclipse/TeleUploader)"
|
||||||
@@ -36,3 +36,4 @@ S3_GUIDE.md
|
|||||||
|
|
||||||
# Finder (MacOS) folder config
|
# Finder (MacOS) folder config
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
result
|
||||||
|
|||||||
@@ -20,11 +20,17 @@ Default to using Bun instead of Node.js.
|
|||||||
- Bun.$`ls` instead of execa.
|
- Bun.$`ls` instead of execa.
|
||||||
- Rate limiter lokal dinonaktifkan (`checkRateLimit` di `src/utils/rateLimit.ts` selalu mengembalikan `true`).
|
- Rate limiter lokal dinonaktifkan (`checkRateLimit` di `src/utils/rateLimit.ts` selalu mengembalikan `true`).
|
||||||
- Telegram API memiliki auto-retry otomatis jika mengembalikan error 429 (Too Many Requests) menggunakan pool Telegraf multi-bot di `src/utils/telegram.ts`.
|
- Telegram API memiliki auto-retry otomatis jika mengembalikan error 429 (Too Many Requests) menggunakan pool Telegraf multi-bot di `src/utils/telegram.ts`.
|
||||||
- Multi-bot dikonfigurasi melalui `ADDITIONAL_BOT_TOKENS` (koma terpisah) di `.env` yang digabung dengan `BOT_TOKEN` utama (total 4 bot).
|
|- Multi-bot dikonfigurasi melalui `BOT_TOKENS` (koma terpisah) di `.env` — semua token bot digabung dalam satu variabel.
|
||||||
- Menggunakan mekanisme rotasi instan jika ada bot yang terkena rate limit 429 sebelum memutuskan untuk sleep.
|
- Menggunakan mekanisme rotasi instan jika ada bot yang terkena rate limit 429 sebelum memutuskan untuk sleep.
|
||||||
- Pengiriman berkas ke Telegram dieksekusi secara responsif dan paralel penuh tanpa batas konkurensi/antrian.
|
- Pengiriman berkas ke Telegram dieksekusi secara responsif dan paralel penuh tanpa batas konkurensi/antrian.
|
||||||
- Berkas API upload ditulis secara sementara ke disk `/tmp/teleuploader-*` dan di-stream ke Telegram menggunakan `fs.createReadStream` (RAM-optimized) lalu dihapus otomatis setelah 50ms (timeout aman).
|
- Berkas API upload ditulis secara sementara ke disk `/tmp/teleuploader-*` dan di-stream ke Telegram menggunakan `fs.createReadStream` (RAM-optimized) lalu dihapus otomatis setelah 50ms (timeout aman).
|
||||||
|
|
||||||
|
## Chunk size (TELEGRAM_CHUNK_SIZE_BYTES)
|
||||||
|
|
||||||
|
- Batas keras: Telegram Bot API `getFile` hanya bisa resolve file ≤ 20 MB — di atas itu error `Bad Request: file is too big` dan part tidak bisa di-download.
|
||||||
|
- Guard fail-fast di `src/env.ts`: service MENOLAK start (exit non-zero) jika `TELEGRAM_CHUNK_SIZE_BYTES` > 19922944 (19 MB, margin aman dari limit 20 MB). Konstanta: `TELEGRAM_CHUNK_SIZE_MAX_BYTES` di `src/shared/utils/validation.ts`, juga dipakai `asSafeChunkSize()` di runtime.
|
||||||
|
- Default 19 MB; berlaku untuk chunked storage DAN S3 multipart parts (sama-sama disimpan ke Telegram lalu di-resolve via getFile).
|
||||||
|
|
||||||
## Testing
|
## Testing
|
||||||
|
|
||||||
Use `bun test` to run tests. Jalankan tes secara spesifik (misal `bun test test/rateLimit.test.ts`) untuk menghindari polusi mock antar berkas tes ketika dijalankan bersamaan.
|
Use `bun test` to run tests. Jalankan tes secara spesifik (misal `bun test test/rateLimit.test.ts`) untuk menghindari polusi mock antar berkas tes ketika dijalankan bersamaan.
|
||||||
|
|||||||
+7
-3
@@ -1,3 +1,7 @@
|
|||||||
|
# ⚠️ LEGACY — pembangunan & deploy sekarang DISARANKAN memakai Nix + systemd
|
||||||
|
# (lihat flake.nix + .github/workflows/deploy.yml + Caddy reverse proxy di orangevps).
|
||||||
|
# Dockerfile ini hanya dipertahankan untuk konteks historis / fallback, bukan deploy produksi.
|
||||||
|
|
||||||
# Stage 1: Builder
|
# Stage 1: Builder
|
||||||
FROM oven/bun:alpine AS builder
|
FROM oven/bun:alpine AS builder
|
||||||
|
|
||||||
@@ -21,12 +25,12 @@ WORKDIR /usr/src/app
|
|||||||
# Copy built files, schema, and package.json
|
# Copy built files, schema, and package.json
|
||||||
COPY --from=builder /usr/src/app/dist/index.js ./dist/index.js
|
COPY --from=builder /usr/src/app/dist/index.js ./dist/index.js
|
||||||
COPY --from=builder /usr/src/app/dist/migrate.js ./dist/migrate.js
|
COPY --from=builder /usr/src/app/dist/migrate.js ./dist/migrate.js
|
||||||
COPY --from=builder /usr/src/app/src/home.html ./home.html
|
COPY --from=builder /usr/src/app/src/home.html ./dist/home.html
|
||||||
COPY schema.sql ./
|
COPY schema.sql ./
|
||||||
COPY package.json ./
|
COPY package.json ./
|
||||||
|
|
||||||
# Expose port
|
# Expose port
|
||||||
EXPOSE 3000
|
EXPOSE 4000
|
||||||
|
|
||||||
# Start server
|
# Start server
|
||||||
CMD ["bun", "dist/index.js"]
|
CMD ["bun", "dist/index.js"]
|
||||||
@@ -4,12 +4,11 @@ Backend production-ready untuk upload file ke Telegram yang tersimpan di private
|
|||||||
|
|
||||||
## Setup
|
## Setup
|
||||||
|
|
||||||
1. Install PostgreSQL database
|
1. Siapkan PostgreSQL database (produksi: database `uploader` via PgBouncer pool di `100.121.180.82:6432`)
|
||||||
2. Buat database: `createdb telegram_uploader`
|
2. Setup environment: `cp .env.example .env`
|
||||||
3. Setup environment: `cp .env.example .env`
|
3. Edit `.env` dengan nilai yang sesuai (lihat `DATABASE_URL`, `PORT=4000`)
|
||||||
4. Edit `.env` dengan nilai yang sesuai
|
4. Create table: `bun run db:migrate`
|
||||||
5. Create table: `bun run db:migrate`
|
5. Install dependencies: `bun install`
|
||||||
6. Install dependencies: `bun install`
|
|
||||||
|
|
||||||
## Telegram Private Channel Setup
|
## Telegram Private Channel Setup
|
||||||
|
|
||||||
@@ -24,6 +23,20 @@ bun run dev # Development mode
|
|||||||
bun run start # Production mode
|
bun run start # Production mode
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Deployment (Produksi — Nix + systemd)
|
||||||
|
|
||||||
|
> Infra lama berbasis Docker + Traefik sudah dihapus dari orangevps (2026-08-02).
|
||||||
|
|
||||||
|
- **Host**: orangevps
|
||||||
|
- **Service**: systemd unit `teleuploader` (env via `/etc/teleuploader/env` / BWS secrets)
|
||||||
|
- **Build**: Nix flake (`flake.nix`) — `nix build .#teleuploader` → `nix copy` → `systemctl restart teleuploader`
|
||||||
|
- **CI**: `.github/workflows/deploy.yml` (Gitea Actions / GitHub Actions)
|
||||||
|
- **Port**: `4000` (`PORT` env)
|
||||||
|
- **Domain**: `https://upload.asepharyana.my.id`
|
||||||
|
- **Reverse proxy**: Caddy (bukan Traefik/Docker)
|
||||||
|
- **Database**: `postgresql://asephs:***@100.121.180.82:6432/uploader` (PgBouncer pool di imrnes, **bukan** 5432/localhost)
|
||||||
|
- `deploy.sh` & `Dockerfile` & `docker-compose.yml` bersifat **legacy** — jangan dipakai untuk deploy produksi.
|
||||||
|
|
||||||
## API Endpoints
|
## API Endpoints
|
||||||
|
|
||||||
- `POST /api/upload` - Upload file
|
- `POST /api/upload` - Upload file
|
||||||
@@ -34,7 +47,7 @@ bun run start # Production mode
|
|||||||
## FAQ
|
## FAQ
|
||||||
|
|
||||||
**URL permanen maksudnya apa?**
|
**URL permanen maksudnya apa?**
|
||||||
URL backend tetap permanen: `https://tele.asepharyana.my.id/f/{public_id}`
|
URL backend tetap permanen: `https://upload.asepharyana.my.id/f/{public_id}`
|
||||||
Ini berarti URL service Anda fix, bukan jaminan file Telegram abadi.
|
Ini berarti URL service Anda fix, bukan jaminan file Telegram abadi.
|
||||||
|
|
||||||
## Testing
|
## Testing
|
||||||
|
|||||||
@@ -1,4 +1,9 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
# ⚠️ LEGACY — Script deploy lama berbasis Docker. Sejak 2026-08-02 Docker dihapus
|
||||||
|
# dari orangevps; produksi kini memakai Nix + systemd (lihat flake.nix, dan CI
|
||||||
|
# .github/workflows/deploy.yml yang menjalankan `nix build` → `nix copy` →
|
||||||
|
# `systemctl restart teleuploader`). Berkas ini hanya dipertahankan sebagai
|
||||||
|
# referensi historis — JANGAN dipakai untuk deploy produksi.
|
||||||
# ─── FileDrop Deploy Script ──────────────────────────────────────────────────
|
# ─── FileDrop Deploy Script ──────────────────────────────────────────────────
|
||||||
# Builds the Bun app locally and deploys to the VPS via Docker.
|
# Builds the Bun app locally and deploys to the VPS via Docker.
|
||||||
#
|
#
|
||||||
|
|||||||
+7
-4
@@ -1,19 +1,22 @@
|
|||||||
|
# ⚠️ LEGACY — Docker/Traefik sudah DIHAPUS dari VPS produksi (orangevps).
|
||||||
|
# Deploy sekarang Nix + systemd (flake.nix + .github/workflows/deploy.yml) dengan
|
||||||
|
# Caddy reverse proxy. Berkas ini hanya dipertahankan sebagai referensi historis.
|
||||||
services:
|
services:
|
||||||
app:
|
app:
|
||||||
build: .
|
build: .
|
||||||
container_name: filedrop-app
|
container_name: filedrop-app
|
||||||
restart: always
|
restart: always
|
||||||
environment:
|
environment:
|
||||||
|
- BOT_TOKENS=${BOT_TOKENS}
|
||||||
- BOT_TOKEN=${BOT_TOKEN}
|
- BOT_TOKEN=${BOT_TOKEN}
|
||||||
- ADDITIONAL_BOT_TOKENS=${ADDITIONAL_BOT_TOKENS:-}
|
- ADDITIONAL_BOT_TOKENS=${ADDITIONAL_BOT_TOKENS:-}
|
||||||
- STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID}
|
- STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID}
|
||||||
- BASE_URL=${BASE_URL}
|
- BASE_URL=${BASE_URL}
|
||||||
- DATABASE_URL=${DATABASE_URL}
|
- DATABASE_URL=${DATABASE_URL}
|
||||||
- PORT=3000
|
- PORT=4000
|
||||||
- NODE_ENV=production
|
- NODE_ENV=production
|
||||||
- LOG_LEVEL=info
|
- LOG_LEVEL=info
|
||||||
- TRUST_PROXY=true
|
- TRUST_PROXY=true
|
||||||
- UPLOAD_CONCURRENCY=${UPLOAD_CONCURRENCY:-8}
|
|
||||||
- BATCH_MAX_ITEMS=${BATCH_MAX_ITEMS:-20}
|
- BATCH_MAX_ITEMS=${BATCH_MAX_ITEMS:-20}
|
||||||
- BATCH_MAX_SIZE_BYTES=${BATCH_MAX_SIZE_BYTES:-524288000}
|
- BATCH_MAX_SIZE_BYTES=${BATCH_MAX_SIZE_BYTES:-524288000}
|
||||||
- MAX_REQUEST_BODY_BYTES=${MAX_REQUEST_BODY_BYTES:-2147483648}
|
- MAX_REQUEST_BODY_BYTES=${MAX_REQUEST_BODY_BYTES:-2147483648}
|
||||||
@@ -46,7 +49,7 @@ services:
|
|||||||
healthcheck:
|
healthcheck:
|
||||||
test:
|
test:
|
||||||
- CMD-SHELL
|
- CMD-SHELL
|
||||||
- "bun -e \"fetch('http://localhost:3000/health').then(r => r.status === 200 ? process.exit(0) : process.exit(1))\""
|
- "bun -e \"fetch('http://localhost:4000/health').then(r => r.status === 200 ? process.exit(0) : process.exit(1))\""
|
||||||
interval: 30s
|
interval: 30s
|
||||||
timeout: 10s
|
timeout: 10s
|
||||||
retries: 3
|
retries: 3
|
||||||
@@ -59,7 +62,7 @@ services:
|
|||||||
- "traefik.http.routers.filedrop.entrypoints=websecure"
|
- "traefik.http.routers.filedrop.entrypoints=websecure"
|
||||||
- "traefik.http.routers.filedrop.tls=true"
|
- "traefik.http.routers.filedrop.tls=true"
|
||||||
- "traefik.http.routers.filedrop.tls.certresolver=cloudflare"
|
- "traefik.http.routers.filedrop.tls.certresolver=cloudflare"
|
||||||
- "traefik.http.services.filedrop.loadbalancer.server.port=3000"
|
- "traefik.http.services.filedrop.loadbalancer.server.port=4000"
|
||||||
- "traefik.http.middlewares.filedrop-rl.ratelimit.average=300"
|
- "traefik.http.middlewares.filedrop-rl.ratelimit.average=300"
|
||||||
- "traefik.http.middlewares.filedrop-rl.ratelimit.burst=100"
|
- "traefik.http.middlewares.filedrop-rl.ratelimit.burst=100"
|
||||||
- "traefik.http.middlewares.filedrop-rl.ratelimit.period=1m"
|
- "traefik.http.middlewares.filedrop-rl.ratelimit.period=1m"
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# Telegram Bot Uploader Backend Implementation Plan
|
# Telegram Bot Uploader Backend Implementation Plan
|
||||||
|
|
||||||
|
> ⚠️ **LEGACY** — Dokumen historis (2026-05-17). Port & infrastruktur sudah berubah:
|
||||||
|
> produksi kini berjalan di port `4000` (Nix + systemd + Caddy, domain `upload.asepharyana.my.id`)
|
||||||
|
> dan database via PgBouncer pool `100.121.180.82:6432` (bukan port 5432, bukan localhost).
|
||||||
|
|
||||||
|
|
||||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
**Goal:** Production-ready backend for Telegram file uploader with dual upload methods (bot + HTTP API), PostgreSQL storage, and redirect-based downloads.
|
**Goal:** Production-ready backend for Telegram file uploader with dual upload methods (bot + HTTP API), PostgreSQL storage, and redirect-based downloads.
|
||||||
@@ -75,9 +80,9 @@ schema.sql
|
|||||||
```bash
|
```bash
|
||||||
BOT_TOKEN=isi_token_bot_telegram
|
BOT_TOKEN=isi_token_bot_telegram
|
||||||
STORAGE_CHANNEL_ID=-1001234567890
|
STORAGE_CHANNEL_ID=-1001234567890
|
||||||
BASE_URL=https://tele.asepharyana.my.id
|
BASE_URL=https://upload.asepharyana.my.id
|
||||||
DATABASE_URL=postgresql://user:password@localhost:5432/telegram_uploader
|
DATABASE_URL=postgresql://asephs:***@100.121.180.82:6432/uploader
|
||||||
PORT=3000
|
PORT=4000
|
||||||
NODE_ENV=production
|
NODE_ENV=production
|
||||||
LOG_LEVEL=info
|
LOG_LEVEL=info
|
||||||
RATE_LIMIT_WINDOW_MS=60000
|
RATE_LIMIT_WINDOW_MS=60000
|
||||||
@@ -148,7 +153,7 @@ bun run start # Production mode
|
|||||||
## FAQ
|
## FAQ
|
||||||
|
|
||||||
**URL permanen maksudnya apa?**
|
**URL permanen maksudnya apa?**
|
||||||
URL backend tetap permanen: `https://tele.asepharyana.my.id/f/{public_id}`
|
URL backend tetap permanen: `https://upload.asepharyana.my.id/f/{public_id}`
|
||||||
Ini berarti URL service Anda fix, bukan jaminan file Telegram abadi.
|
Ini berarti URL service Anda fix, bukan jaminan file Telegram abadi.
|
||||||
|
|
||||||
## Testing
|
## Testing
|
||||||
@@ -250,7 +255,7 @@ export const config = {
|
|||||||
storageChatId: parseInt(process.env.STORAGE_CHANNEL_ID, 10),
|
storageChatId: parseInt(process.env.STORAGE_CHANNEL_ID, 10),
|
||||||
baseUrl: process.env.BASE_URL,
|
baseUrl: process.env.BASE_URL,
|
||||||
databaseUrl: process.env.DATABASE_URL,
|
databaseUrl: process.env.DATABASE_URL,
|
||||||
port: parseInt(process.env.PORT, 10) || 3000,
|
port: parseInt(process.env.PORT, 10) || 4000,
|
||||||
nodeEnv: process.env.NODE_ENV || 'development',
|
nodeEnv: process.env.NODE_ENV || 'development',
|
||||||
logLevel: process.env.LOG_LEVEL || 'info',
|
logLevel: process.env.LOG_LEVEL || 'info',
|
||||||
rateLimitWindowMs: parseInt(process.env.RATE_LIMIT_WINDOW_MS, 10) || 60000,
|
rateLimitWindowMs: parseInt(process.env.RATE_LIMIT_WINDOW_MS, 10) || 60000,
|
||||||
@@ -1009,14 +1014,14 @@ bun run dev
|
|||||||
- [ ] **Step 2: Test health endpoint**
|
- [ ] **Step 2: Test health endpoint**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl http://localhost:3000/health
|
curl http://localhost:4000/health
|
||||||
# Expected: {"status":"ok"}
|
# Expected: {"status":"ok"}
|
||||||
```
|
```
|
||||||
|
|
||||||
- [ ] **Step 3: Upload test file via HTTP API (multipart)**
|
- [ ] **Step 3: Upload test file via HTTP API (multipart)**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -X POST http://localhost:3000/api/upload \
|
curl -X POST http://localhost:4000/api/upload \
|
||||||
-F "file=@/path/to/testfile.txt" \
|
-F "file=@/path/to/testfile.txt" \
|
||||||
-F "fileName=test.txt"
|
-F "fileName=test.txt"
|
||||||
```
|
```
|
||||||
@@ -1024,14 +1029,14 @@ curl -X POST http://localhost:3000/api/upload \
|
|||||||
- [ ] **Step 4: Check file info endpoint**
|
- [ ] **Step 4: Check file info endpoint**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl http://localhost:3000/file/{public_id}/info
|
curl http://localhost:4000/file/{public_id}/info
|
||||||
# Expected: JSON with file metadata
|
# Expected: JSON with file metadata
|
||||||
```
|
```
|
||||||
|
|
||||||
- [ ] **Step 5: Download redirect**
|
- [ ] **Step 5: Download redirect**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -I http://localhost:3000/f/{public_id}
|
curl -I http://localhost:4000/f/{public_id}
|
||||||
# Expected: HTTP 302 with Location header to Telegram CDN
|
# Expected: HTTP 302 with Location header to Telegram CDN
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -1047,7 +1052,7 @@ curl -I http://localhost:3000/f/{public_id}
|
|||||||
- [ ] **Step 7: Test error handling (file too large)**
|
- [ ] **Step 7: Test error handling (file too large)**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -X POST http://localhost:3000/api/upload \
|
curl -X POST http://localhost:4000/api/upload \
|
||||||
-F "file=@/dev/null" \
|
-F "file=@/dev/null" \
|
||||||
-H "Content-Length: 10000000000"
|
-H "Content-Length: 10000000000"
|
||||||
# Expected: HTTP 400 with error message
|
# Expected: HTTP 400 with error message
|
||||||
@@ -1057,7 +1062,7 @@ curl -X POST http://localhost:3000/api/upload \
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Send 31 requests within 1 minute
|
# Send 31 requests within 1 minute
|
||||||
for i in {1..31}; do curl http://localhost:3000/f/{public_id} & done
|
for i in {1..31}; do curl http://localhost:4000/f/{public_id} & done
|
||||||
wait
|
wait
|
||||||
# Expected: First 30 succeed, last one returns 429
|
# Expected: First 30 succeed, last one returns 429
|
||||||
```
|
```
|
||||||
@@ -1066,7 +1071,7 @@ wait
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# In terminal 1: bun run dev
|
# In terminal 1: bun run dev
|
||||||
# In terminal 2: curl http://localhost:3000/health && sleep 0.1 && curl http://localhost:3000/health
|
# In terminal 2: curl http://localhost:4000/health && sleep 0.1 && curl http://localhost:4000/health
|
||||||
# Send SIGINT to server (Ctrl+C in terminal 1)
|
# Send SIGINT to server (Ctrl+C in terminal 1)
|
||||||
# Check if server stops cleanly, logs show shutdown sequence
|
# Check if server stops cleanly, logs show shutdown sequence
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -1,5 +1,11 @@
|
|||||||
# Docker, Traefik, and GitHub Actions Deployment Plan
|
# Docker, Traefik, and GitHub Actions Deployment Plan
|
||||||
|
|
||||||
|
> ⚠️ **LEGACY** — Dokumen historis (2026-05-18). Arsitektur Docker + Traefik +
|
||||||
|
> GitHub Actions sudah digantikan (2026-08-02) oleh Nix + systemd + Caddy di
|
||||||
|
> orangevps: port `4000`, domain `upload.asepharyana.my.id`, database via
|
||||||
|
> PgBouncer pool `100.121.180.82:6432` (bukan 5432/localhost).
|
||||||
|
|
||||||
|
|
||||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
**Goal:** Containerize TeleUploader using Bun, configure Traefik labels for routing `upload.asepharyana.my.id`, and set up full GitHub Actions CI/CD to VPS `45.127.35.244`.
|
**Goal:** Containerize TeleUploader using Bun, configure Traefik labels for routing `upload.asepharyana.my.id`, and set up full GitHub Actions CI/CD to VPS `45.127.35.244`.
|
||||||
@@ -57,7 +63,7 @@ WORKDIR /usr/src/app
|
|||||||
|
|
||||||
# Set production environment variables
|
# Set production environment variables
|
||||||
ENV NODE_ENV=production
|
ENV NODE_ENV=production
|
||||||
ENV PORT=3000
|
ENV PORT=4000
|
||||||
|
|
||||||
# Copy necessary files from builder and repo
|
# Copy necessary files from builder and repo
|
||||||
COPY --from=builder /usr/src/app/dist/index.js ./dist/index.js
|
COPY --from=builder /usr/src/app/dist/index.js ./dist/index.js
|
||||||
@@ -65,7 +71,7 @@ COPY --from=builder /usr/src/app/package.json ./package.json
|
|||||||
COPY schema.sql ./schema.sql
|
COPY schema.sql ./schema.sql
|
||||||
|
|
||||||
# Expose server port
|
# Expose server port
|
||||||
EXPOSE 3000
|
EXPOSE 4000
|
||||||
|
|
||||||
# Start server
|
# Start server
|
||||||
CMD ["bun", "dist/index.js"]
|
CMD ["bun", "dist/index.js"]
|
||||||
@@ -104,7 +110,7 @@ services:
|
|||||||
- STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID}
|
- STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID}
|
||||||
- BASE_URL=${BASE_URL}
|
- BASE_URL=${BASE_URL}
|
||||||
- DATABASE_URL=${DATABASE_URL}
|
- DATABASE_URL=${DATABASE_URL}
|
||||||
- PORT=3000
|
- PORT=4000
|
||||||
- NODE_ENV=production
|
- NODE_ENV=production
|
||||||
- LOG_LEVEL=info
|
- LOG_LEVEL=info
|
||||||
networks:
|
networks:
|
||||||
@@ -115,7 +121,7 @@ services:
|
|||||||
- "traefik.http.routers.teleuploader.entrypoints=websecure"
|
- "traefik.http.routers.teleuploader.entrypoints=websecure"
|
||||||
- "traefik.http.routers.teleuploader.tls=true"
|
- "traefik.http.routers.teleuploader.tls=true"
|
||||||
- "traefik.http.routers.teleuploader.tls.certresolver=letsencrypt"
|
- "traefik.http.routers.teleuploader.tls.certresolver=letsencrypt"
|
||||||
- "traefik.http.services.teleuploader.loadbalancer.server.port=3000"
|
- "traefik.http.services.teleuploader.loadbalancer.server.port=4000"
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
app-shared-net:
|
app-shared-net:
|
||||||
@@ -166,9 +172,9 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
BOT_TOKEN: "mock_token"
|
BOT_TOKEN: "mock_token"
|
||||||
STORAGE_CHANNEL_ID: "123456"
|
STORAGE_CHANNEL_ID: "123456"
|
||||||
BASE_URL: "http://localhost:3000"
|
BASE_URL: "http://localhost:4000"
|
||||||
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/postgres"
|
DATABASE_URL: "postgresql://asephs:***@100.121.180.82:6432/postgres"
|
||||||
PORT: "3000"
|
PORT: "4000"
|
||||||
run: bun run test
|
run: bun run test
|
||||||
|
|
||||||
build-and-push:
|
build-and-push:
|
||||||
@@ -223,7 +229,7 @@ jobs:
|
|||||||
- STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID}
|
- STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID}
|
||||||
- BASE_URL=${BASE_URL}
|
- BASE_URL=${BASE_URL}
|
||||||
- DATABASE_URL=${DATABASE_URL}
|
- DATABASE_URL=${DATABASE_URL}
|
||||||
- PORT=3000
|
- PORT=4000
|
||||||
- NODE_ENV=production
|
- NODE_ENV=production
|
||||||
- LOG_LEVEL=info
|
- LOG_LEVEL=info
|
||||||
networks:
|
networks:
|
||||||
@@ -234,7 +240,7 @@ jobs:
|
|||||||
- "traefik.http.routers.teleuploader.entrypoints=websecure"
|
- "traefik.http.routers.teleuploader.entrypoints=websecure"
|
||||||
- "traefik.http.routers.teleuploader.tls=true"
|
- "traefik.http.routers.teleuploader.tls=true"
|
||||||
- "traefik.http.routers.teleuploader.tls.certresolver=letsencrypt"
|
- "traefik.http.routers.teleuploader.tls.certresolver=letsencrypt"
|
||||||
- "traefik.http.services.teleuploader.loadbalancer.server.port=3000"
|
- "traefik.http.services.teleuploader.loadbalancer.server.port=4000"
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
app-shared-net:
|
app-shared-net:
|
||||||
@@ -248,7 +254,7 @@ jobs:
|
|||||||
STORAGE_CHANNEL_ID=${{ secrets.STORAGE_CHANNEL_ID }}
|
STORAGE_CHANNEL_ID=${{ secrets.STORAGE_CHANNEL_ID }}
|
||||||
BASE_URL=${{ secrets.BASE_URL }}
|
BASE_URL=${{ secrets.BASE_URL }}
|
||||||
DATABASE_URL=${{ secrets.DATABASE_URL }}
|
DATABASE_URL=${{ secrets.DATABASE_URL }}
|
||||||
PORT=3000
|
PORT=4000
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
# Pull latest docker image
|
# Pull latest docker image
|
||||||
|
|||||||
@@ -1,5 +1,11 @@
|
|||||||
# S3-Compatible TeleUploader Implementation Plan
|
# S3-Compatible TeleUploader Implementation Plan
|
||||||
|
|
||||||
|
> ⚠️ **LEGACY** — Dokumen historis (2026-07-06). Port & infrastruktur sudah berubah:
|
||||||
|
> produksi kini berjalan di port `4000` (Nix + systemd + Caddy, domain
|
||||||
|
> `upload.asepharyana.my.id`), database via PgBouncer pool `100.121.180.82:6432`
|
||||||
|
> (bukan 5432/localhost). Contoh kode di bawah memakai `localhost:4000` untuk dev.
|
||||||
|
|
||||||
|
|
||||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
**Goal:** Transform TeleUploader into an S3-compatible storage server (Telegram-backed) with a web file manager UI.
|
**Goal:** Transform TeleUploader into an S3-compatible storage server (Telegram-backed) with a web file manager UI.
|
||||||
@@ -2675,7 +2681,7 @@ describe('S3 Bucket Operations', () => {
|
|||||||
process.env.S3_DEFAULT_REGION = 'us-east-1';
|
process.env.S3_DEFAULT_REGION = 'us-east-1';
|
||||||
process.env.BOT_TOKEN = '123456:ABC-DEF';
|
process.env.BOT_TOKEN = '123456:ABC-DEF';
|
||||||
process.env.STORAGE_CHANNEL_ID = '-1001234567890';
|
process.env.STORAGE_CHANNEL_ID = '-1001234567890';
|
||||||
process.env.BASE_URL = 'http://localhost:3000';
|
process.env.BASE_URL = 'http://localhost:4000';
|
||||||
process.env.DATABASE_URL = 'postgresql://localhost/test';
|
process.env.DATABASE_URL = 'postgresql://localhost/test';
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -2684,7 +2690,7 @@ describe('S3 Bucket Operations', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should return 403 for unauthorized requests', async () => {
|
it('should return 403 for unauthorized requests', async () => {
|
||||||
const req = new Request('http://localhost:3000/', {
|
const req = new Request('http://localhost:4000/', {
|
||||||
method: 'GET',
|
method: 'GET',
|
||||||
headers: { authorization: 'Invalid' },
|
headers: { authorization: 'Invalid' },
|
||||||
});
|
});
|
||||||
@@ -2849,7 +2855,7 @@ describe('Web API v1', () => {
|
|||||||
mockDbExecute.mockClear();
|
mockDbExecute.mockClear();
|
||||||
process.env.BOT_TOKEN = '123456:ABC-DEF';
|
process.env.BOT_TOKEN = '123456:ABC-DEF';
|
||||||
process.env.STORAGE_CHANNEL_ID = '-1001234567890';
|
process.env.STORAGE_CHANNEL_ID = '-1001234567890';
|
||||||
process.env.BASE_URL = 'http://localhost:3000';
|
process.env.BASE_URL = 'http://localhost:4000';
|
||||||
process.env.DATABASE_URL = 'postgresql://localhost/test';
|
process.env.DATABASE_URL = 'postgresql://localhost/test';
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -2858,7 +2864,7 @@ describe('Web API v1', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should list buckets via GET /api/v1/buckets', async () => {
|
it('should list buckets via GET /api/v1/buckets', async () => {
|
||||||
const req = new Request('http://localhost:3000/api/v1/buckets');
|
const req = new Request('http://localhost:4000/api/v1/buckets');
|
||||||
const res = await handleWebApiV1(req);
|
const res = await handleWebApiV1(req);
|
||||||
expect(res.status).toBe(200);
|
expect(res.status).toBe(200);
|
||||||
const data = await res.json();
|
const data = await res.json();
|
||||||
@@ -2867,7 +2873,7 @@ describe('Web API v1', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should return 404 for unknown API path', async () => {
|
it('should return 404 for unknown API path', async () => {
|
||||||
const req = new Request('http://localhost:3000/api/v1/unknown');
|
const req = new Request('http://localhost:4000/api/v1/unknown');
|
||||||
const res = await handleWebApiV1(req);
|
const res = await handleWebApiV1(req);
|
||||||
expect(res.status).toBe(404);
|
expect(res.status).toBe(404);
|
||||||
const data = await res.json();
|
const data = await res.json();
|
||||||
@@ -2875,7 +2881,7 @@ describe('Web API v1', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should return bucket object listing', async () => {
|
it('should return bucket object listing', async () => {
|
||||||
const req = new Request('http://localhost:3000/api/v1/buckets/test-bucket/objects?prefix=');
|
const req = new Request('http://localhost:4000/api/v1/buckets/test-bucket/objects?prefix=');
|
||||||
const res = await handleWebApiV1(req);
|
const res = await handleWebApiV1(req);
|
||||||
// Should return 200 even with empty results
|
// Should return 200 even with empty results
|
||||||
expect(res.status).toBe(200);
|
expect(res.status).toBe(200);
|
||||||
@@ -2885,7 +2891,7 @@ describe('Web API v1', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should reject invalid bucket name on create', async () => {
|
it('should reject invalid bucket name on create', async () => {
|
||||||
const req = new Request('http://localhost:3000/api/v1/buckets', {
|
const req = new Request('http://localhost:4000/api/v1/buckets', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'content-type': 'application/json' },
|
headers: { 'content-type': 'application/json' },
|
||||||
body: JSON.stringify({ name: 'INVALID_NAME!' }),
|
body: JSON.stringify({ name: 'INVALID_NAME!' }),
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
# Gitea CI/CD Migration Implementation Plan
|
# Gitea CI/CD Migration Implementation Plan
|
||||||
|
|
||||||
|
> Catatan (2026-08-02): Produksi sekarang port 4000, deploy Nix+systemd di orangevps, Caddy reverse proxy upload.asepharyana.my.id, DB via pgbouncer pool imrnes 100.121.180.82:6432. Docker/Traefik/Gitea-CI legacy.
|
||||||
|
|
||||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
**Goal:** Move `origin` from GitLab to `git.imrnes.team:MythEclipse/TeleUploader` and add Gitea Actions deployment on push to `main` using the existing VPS deploy path.
|
**Goal:** Move `origin` from GitLab to `git.imrnes.team:MythEclipse/TeleUploader` and add Gitea Actions deployment on push to `main` using the existing VPS deploy path.
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
# S3 Compatibility Completion Implementation Plan
|
# S3 Compatibility Completion Implementation Plan
|
||||||
|
|
||||||
|
> Catatan (2026-08-02): Produksi sekarang port 4000, deploy Nix+systemd di orangevps, Caddy reverse proxy upload.asepharyana.my.id, DB via pgbouncer pool imrnes 100.121.180.82:6432. Docker/Traefik/Gitea-CI legacy.
|
||||||
|
|
||||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
**Goal:** Finish TeleUploader S3 compatibility gaps: strict presigned GET, byte ranges, complete multipart GetObject streaming, strict AWS SDK multipart investigation/fix, and warning-free lint.
|
**Goal:** Finish TeleUploader S3 compatibility gaps: strict presigned GET, byte ranges, complete multipart GetObject streaming, strict AWS SDK multipart investigation/fix, and warning-free lint.
|
||||||
|
|||||||
@@ -0,0 +1,576 @@
|
|||||||
|
# Per-Bot Queue Implementation Plan
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** Replace global PQueue with per-bot queues (concurrency=1 per bot) to eliminate 429 collisions and improve rate-limit safety.
|
||||||
|
|
||||||
|
**Architecture:** Each bot token gets its own PQueue with concurrency=1. Uploads are assigned to the least-loaded available bot via `selectBot()`. On 429, the bot is marked rate-limited and the upload retries on the next available bot. The global `upload-queue.ts` is removed; `uploadConcurrency` config is replaced by `botCount * perBotConcurrency`.
|
||||||
|
|
||||||
|
**Tech Stack:** TypeScript, PQueue, Telegraf
|
||||||
|
|
||||||
|
## Global Constraints
|
||||||
|
|
||||||
|
- Use `Bun` runtime, not Node.js
|
||||||
|
- Follow existing code style (Biome lint)
|
||||||
|
- Each bot queue has concurrency=1 (no two uploads hit same bot simultaneously)
|
||||||
|
- `selectBot()` picks bot with lowest pending queue count, skipping rate-limited bots
|
||||||
|
- Remove `uploadConcurrency` from config; derive effective concurrency from bot count
|
||||||
|
- Remove `upload-queue.ts` entirely
|
||||||
|
- Remove `enqueueUpload` from `ITelegramService` interface
|
||||||
|
|
||||||
|
---
|
||||||
|
## File Structure
|
||||||
|
|
||||||
|
### Files to Modify
|
||||||
|
| File | Change |
|
||||||
|
|------|--------|
|
||||||
|
| `src/infrastructure/telegram/bot-pool.ts` | BotEntry array, selectBot(), per-bot queues, retry logic |
|
||||||
|
| `src/domain/ports/telegram-service.ts` | Remove `enqueueUpload` from interface |
|
||||||
|
| `src/env.ts` | Remove `uploadConcurrency` field |
|
||||||
|
| `src/index.ts` | Remove upload-queue import and usage |
|
||||||
|
| `src/utils/chunked-storage.ts` | Replace `config.uploadConcurrency` with bot count |
|
||||||
|
|
||||||
|
### Files to Delete
|
||||||
|
| File | Reason |
|
||||||
|
|------|--------|
|
||||||
|
| `src/infrastructure/telegram/upload-queue.ts` | Global queue replaced by per-bot queues |
|
||||||
|
| `test/telegramQueue.test.ts` | Tests for deleted module |
|
||||||
|
|
||||||
|
### Files Not Changed
|
||||||
|
| File | Reason |
|
||||||
|
|------|--------|
|
||||||
|
| `test/bot.test.ts` | Only uses ITelegramService interface (via `forwardToStorage`) |
|
||||||
|
| `src/infrastructure/telegram/chunked-storage.ts` | Uses ITelegramService interface, not BotPool directly |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 1: Refresh the spec & plan files after compaction
|
||||||
|
|
||||||
|
Due to context compaction, re-read the current spec and plan files to ensure accuracy before implementing.
|
||||||
|
|
||||||
|
- [ ] **Step 1: Re-read the spec**
|
||||||
|
|
||||||
|
Read: `docs/superpowers/specs/2026-07-29-per-bot-queue-design.md`
|
||||||
|
|
||||||
|
- [ ] **Step 2: Re-read key implementation files**
|
||||||
|
|
||||||
|
Read: `src/infrastructure/telegram/bot-pool.ts`, `src/env.ts`, `src/utils/chunked-storage.ts`
|
||||||
|
|
||||||
|
### Task 2: Refactor ITelegramService interface
|
||||||
|
|
||||||
|
Remove `enqueueUpload` from the interface — BotPool handles queueing internally now.
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `src/domain/ports/telegram-service.ts`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Remove `enqueueUpload` from interface**
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// src/domain/ports/telegram-service.ts — remove entire section:
|
||||||
|
/**
|
||||||
|
* Enqueue a task for sequential upload execution.
|
||||||
|
*
|
||||||
|
* Ensures only one Telegram upload runs at a time to avoid
|
||||||
|
* rate limits and resource contention.
|
||||||
|
*
|
||||||
|
* @param task - An async function performing the upload.
|
||||||
|
* @returns The result of the task.
|
||||||
|
*/
|
||||||
|
enqueueUpload<T>(task: () => Promise<T>): Promise<T>;
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Run lint to verify**
|
||||||
|
|
||||||
|
Run: `bunx biome check src/domain/ports/telegram-service.ts`
|
||||||
|
Expected: No errors.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add src/domain/ports/telegram-service.ts
|
||||||
|
git commit -m "refactor: remove enqueueUpload from ITelegramService
|
||||||
|
|
||||||
|
Per-bot queue handles queueing internally.
|
||||||
|
|
||||||
|
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Task 3: Refactor BotPool with per-bot queues
|
||||||
|
|
||||||
|
The core of the redesign. Replace `claimBotIndex()` round-robin with per-bot PQueue instances and `selectBot()` for least-loaded assignment.
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `src/infrastructure/telegram/bot-pool.ts`
|
||||||
|
|
||||||
|
**Interfaces:**
|
||||||
|
- Consumes: `ITelegramService` (no `enqueueUpload` method)
|
||||||
|
- Produces: `botPool` singleton with per-bot queues, `selectBot()`, per-bot rate-limit tracking
|
||||||
|
|
||||||
|
- [ ] **Step 1: Write test file for per-bot queue behavior**
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// test/bot-pool.test.ts
|
||||||
|
import { beforeEach, describe, expect, it, mock, spyOn } from 'bun:test';
|
||||||
|
|
||||||
|
// We'll test the BotEntry queue behavior and selectBot logic
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Implement BotEntry type and BotPool refactor**
|
||||||
|
|
||||||
|
Replace the class body:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
import PQueue from 'p-queue';
|
||||||
|
import { Telegraf } from 'telegraf';
|
||||||
|
import type {
|
||||||
|
ForwardResult,
|
||||||
|
ITelegramService,
|
||||||
|
TelegramFileInfo,
|
||||||
|
} from '../../domain/ports/telegram-service';
|
||||||
|
import { config } from '../../env';
|
||||||
|
import logger from '../../shared/logger/index';
|
||||||
|
import {
|
||||||
|
buildSendPayload,
|
||||||
|
extractUploadedFile,
|
||||||
|
type SendMethod,
|
||||||
|
sendMethodMap,
|
||||||
|
type TelegramMessageResult,
|
||||||
|
} from './types';
|
||||||
|
|
||||||
|
const sleep = (ms: number): Promise<void> =>
|
||||||
|
new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
|
||||||
|
const isTransientError = (error: unknown): boolean => {
|
||||||
|
const str = error instanceof Error ? error.message : String(error);
|
||||||
|
const transientPatterns = [
|
||||||
|
'timeout', 'Timed out', 'etimedout', 'econnrefused', 'econnreset',
|
||||||
|
'ECONNREFUSED', 'ECONNRESET', 'ETIMEDOUT', '5xx', '502', '503', '504',
|
||||||
|
'Bad Gateway', 'Service Unavailable', 'Gateway Timeout', 'socket hang up',
|
||||||
|
'socket closed', 'fetch failed', 'network error', 'network timeout',
|
||||||
|
'API closed', 'read ECONNRESET', 'write EPIPE',
|
||||||
|
];
|
||||||
|
return transientPatterns.some((p) => str.toLowerCase().includes(p.toLowerCase()));
|
||||||
|
};
|
||||||
|
|
||||||
|
const MAX_TRANSIENT_RETRIES = 3;
|
||||||
|
const TELEGRAM_API_TIMEOUT_MS = 120_000;
|
||||||
|
const PER_BOT_CONCURRENCY = 1;
|
||||||
|
|
||||||
|
interface BotEntry {
|
||||||
|
index: number;
|
||||||
|
token: string;
|
||||||
|
instance: Telegraf;
|
||||||
|
queue: PQueue;
|
||||||
|
rateLimitedUntil: number; // 0 = not rate-limited
|
||||||
|
}
|
||||||
|
|
||||||
|
export class BotPool implements ITelegramService {
|
||||||
|
private readonly bots: BotEntry[] = [];
|
||||||
|
|
||||||
|
constructor() {
|
||||||
|
const tokens = Array.from(new Set([config.botToken, ...config.additionalBotTokens]));
|
||||||
|
this.bots = tokens.map((token, index) => ({
|
||||||
|
index,
|
||||||
|
token,
|
||||||
|
instance: new Telegraf(token),
|
||||||
|
queue: new PQueue({ concurrency: PER_BOT_CONCURRENCY }),
|
||||||
|
rateLimitedUntil: 0,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Number of bots in the pool */
|
||||||
|
get size(): number {
|
||||||
|
return this.bots.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Select the bot with the fewest pending tasks that isn't rate-limited
|
||||||
|
* or in the skip set.
|
||||||
|
*/
|
||||||
|
private selectBot(skipIndexes?: Set<number>): BotEntry | null {
|
||||||
|
let best: BotEntry | null = null;
|
||||||
|
let bestPending = Infinity;
|
||||||
|
|
||||||
|
for (const bot of this.bots) {
|
||||||
|
if (skipIndexes?.has(bot.index)) continue;
|
||||||
|
if (bot.rateLimitedUntil > Date.now()) continue;
|
||||||
|
|
||||||
|
const pending = bot.queue.pending + bot.queue.size;
|
||||||
|
if (pending < bestPending) {
|
||||||
|
bestPending = pending;
|
||||||
|
best = bot;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return best;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute a Telegram API action on a specific bot entry.
|
||||||
|
* Wraps with timeout.
|
||||||
|
*/
|
||||||
|
private async executeBotAction<T>(
|
||||||
|
bot: BotEntry,
|
||||||
|
action: (instance: Telegraf, token: string) => Promise<T>,
|
||||||
|
): Promise<T> {
|
||||||
|
return Promise.race([
|
||||||
|
action(bot.instance, bot.token),
|
||||||
|
new Promise<never>((_, reject) =>
|
||||||
|
setTimeout(
|
||||||
|
() => reject(new Error(`Telegram API timeout after ${TELEGRAM_API_TIMEOUT_MS}ms`)),
|
||||||
|
TELEGRAM_API_TIMEOUT_MS,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Forward a file chunk to the configured Telegram storage chat.
|
||||||
|
*
|
||||||
|
* The upload is submitted to the least-loaded bot's queue. If the bot
|
||||||
|
* returns 429, it is marked rate-limited and the upload retries on the
|
||||||
|
* next available bot. If all bots are rate-limited, sleeps before retrying.
|
||||||
|
*/
|
||||||
|
async forwardToStorage(
|
||||||
|
fileChunk: unknown,
|
||||||
|
fileName: string,
|
||||||
|
fileType: string,
|
||||||
|
): Promise<ForwardResult> {
|
||||||
|
let lastError: unknown;
|
||||||
|
const attemptedIndexes = new Set<number>();
|
||||||
|
let transientAttempts = 0;
|
||||||
|
|
||||||
|
// Outer retry loop — up to 10 attempts across all bots
|
||||||
|
for (let attempt = 0; attempt < 10; attempt++) {
|
||||||
|
const bot = this.selectBot(attemptedIndexes);
|
||||||
|
|
||||||
|
if (!bot) {
|
||||||
|
// No available bots — either all rate-limited or all attempted
|
||||||
|
if (attemptedIndexes.size > 0) {
|
||||||
|
// All non-rate-limited bots were tried and failed — wait & reset
|
||||||
|
logger.warn('All available bots exhausted, sleeping 5s before retry');
|
||||||
|
await sleep(5000 + Math.random() * 1000);
|
||||||
|
attemptedIndexes.clear();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// All bots rate-limited — wait for the shortest cooldown
|
||||||
|
const earliestCooldown = Math.min(
|
||||||
|
...this.bots.map((b) => b.rateLimitedUntil || Infinity),
|
||||||
|
);
|
||||||
|
const waitMs = Math.max(1000, earliestCooldown - Date.now() + 500);
|
||||||
|
logger.warn('All bots rate-limited, waiting', { waitMs });
|
||||||
|
await sleep(waitMs);
|
||||||
|
attemptedIndexes.clear();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
attemptedIndexes.add(bot.index);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await bot.queue.add(async () => {
|
||||||
|
// Inner transient retry loop inside the queue
|
||||||
|
for (let innerRetry = 0; innerRetry <= MAX_TRANSIENT_RETRIES; innerRetry++) {
|
||||||
|
try {
|
||||||
|
const filePayload = { source: fileChunk, filename: fileName };
|
||||||
|
const sendMethodName = sendMethodMap[fileType] || 'sendDocument';
|
||||||
|
const payload = buildSendPayload(fileType, fileName);
|
||||||
|
|
||||||
|
const tgResult = await this.executeBotAction<TelegramMessageResult>(
|
||||||
|
bot,
|
||||||
|
(activeBot) => {
|
||||||
|
const telegram = activeBot.telegram as unknown as Record<string, SendMethod>;
|
||||||
|
return telegram[sendMethodName](config.storageChatId, filePayload, payload);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const uploadedFile = extractUploadedFile(tgResult, fileType);
|
||||||
|
return {
|
||||||
|
telegramFileId: uploadedFile?.file_id || '',
|
||||||
|
telegramFileUniqueId: uploadedFile?.file_unique_id || '',
|
||||||
|
storageMessageId: tgResult.message_id,
|
||||||
|
};
|
||||||
|
} catch (error: unknown) {
|
||||||
|
const errorStr = error instanceof Error ? error.message : String(error);
|
||||||
|
const retryAfterMatch = errorStr.match(/retry after (\d+)/i);
|
||||||
|
|
||||||
|
if (retryAfterMatch) {
|
||||||
|
// 429 — mark bot rate-limited, throw to outer loop for retry on different bot
|
||||||
|
const seconds = parseInt(retryAfterMatch[1], 10);
|
||||||
|
bot.rateLimitedUntil = Date.now() + seconds * 1000;
|
||||||
|
logger.info(`Bot #${bot.index} rate-limited for ${seconds}s`, { fileName, attempt });
|
||||||
|
throw error; // caught by outer retry loop
|
||||||
|
}
|
||||||
|
|
||||||
|
if (innerRetry < MAX_TRANSIENT_RETRIES && isTransientError(error)) {
|
||||||
|
const backoffMs = Math.min(1000 * 2 ** innerRetry, 10_000);
|
||||||
|
logger.warn(
|
||||||
|
`Transient error on bot #${bot.index}, retrying (${innerRetry + 1}/${MAX_TRANSIENT_RETRIES})`,
|
||||||
|
{ fileName, error: errorStr, backoffMs },
|
||||||
|
);
|
||||||
|
await sleep(backoffMs);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw error; // non-transient — propagate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error(`Exhausted transient retries on bot #${bot.index}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
logger.info('File forwarded to storage', { fileName, message: result.storageMessageId });
|
||||||
|
return result;
|
||||||
|
} catch (error: unknown) {
|
||||||
|
lastError = error;
|
||||||
|
const errorStr = error instanceof Error ? error.message : String(error);
|
||||||
|
const retryAfterMatch = errorStr.match(/retry after (\d+)/i);
|
||||||
|
|
||||||
|
if (retryAfterMatch) {
|
||||||
|
// Bot was rate-limited — already marked, try next bot
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Transient error at the queue level (timeout, 5xx)
|
||||||
|
if (transientAttempts < MAX_TRANSIENT_RETRIES && isTransientError(error)) {
|
||||||
|
transientAttempts++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Non-transient — give up
|
||||||
|
logger.error('Failed to forward file to storage', {
|
||||||
|
fileName,
|
||||||
|
error: errorStr,
|
||||||
|
attempt,
|
||||||
|
});
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw lastError || new Error('Failed to forward file after all retries');
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Get total effective concurrency across all bots */
|
||||||
|
getEffectiveConcurrency(): number {
|
||||||
|
return this.bots.length * PER_BOT_CONCURRENCY;
|
||||||
|
}
|
||||||
|
|
||||||
|
async getFileInfo(telegramFileId: string): Promise<TelegramFileInfo> {
|
||||||
|
let lastError: unknown;
|
||||||
|
for (const bot of this.bots) {
|
||||||
|
for (let retry = 0; retry <= MAX_TRANSIENT_RETRIES; retry++) {
|
||||||
|
try {
|
||||||
|
const result = await bot.instance.telegram.getFile(telegramFileId);
|
||||||
|
const fileData = result as unknown as Omit<TelegramFileInfo, 'bot_token'>;
|
||||||
|
return {
|
||||||
|
file_size: fileData.file_size || 0,
|
||||||
|
mime_type: fileData.mime_type || 'application/octet-stream',
|
||||||
|
file_path: fileData.file_path || '',
|
||||||
|
bot_token: bot.token,
|
||||||
|
};
|
||||||
|
} catch (error: unknown) {
|
||||||
|
lastError = error;
|
||||||
|
const errorStr = error instanceof Error ? error.message : String(error);
|
||||||
|
if (
|
||||||
|
errorStr.includes('wrong file_id') ||
|
||||||
|
errorStr.includes('file is temporarily unavailable')
|
||||||
|
) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (retry < MAX_TRANSIENT_RETRIES && isTransientError(error)) {
|
||||||
|
const backoffMs = Math.min(1000 * 2 ** (retry + 1), 5_000);
|
||||||
|
await sleep(backoffMs);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
logger.error('Failed to get file info from any bot', {
|
||||||
|
error: lastError instanceof Error ? lastError.message : String(lastError),
|
||||||
|
});
|
||||||
|
throw lastError;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const botPool = new BotPool();
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: Run lint**
|
||||||
|
|
||||||
|
Run: `bunx biome check src/infrastructure/telegram/bot-pool.ts`
|
||||||
|
Expected: No errors.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Run existing test suite**
|
||||||
|
|
||||||
|
Run: `bun test test/bot.test.ts`
|
||||||
|
Expected: All tests pass.
|
||||||
|
|
||||||
|
- [ ] **Step 5: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add src/infrastructure/telegram/bot-pool.ts
|
||||||
|
git commit -m "refactor: per-bot queue with selectBot() and rate-limit tracking
|
||||||
|
|
||||||
|
Each bot has its own PQueue (concurrency=1). Uploads are assigned to
|
||||||
|
the least-loaded available bot. On 429, the bot is marked rate-limited
|
||||||
|
and the upload retries on the next available bot.
|
||||||
|
|
||||||
|
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Task 4: Remove global upload queue
|
||||||
|
|
||||||
|
Delete the global queue, its test, and all references to it from index.ts.
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Delete: `src/infrastructure/telegram/upload-queue.ts`
|
||||||
|
- Modify: `src/index.ts` (lines 4, 71-81)
|
||||||
|
- Delete: `test/telegramQueue.test.ts`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Delete upload-queue.ts**
|
||||||
|
|
||||||
|
Run: `rm src/infrastructure/telegram/upload-queue.ts`
|
||||||
|
|
||||||
|
- [ ] **Step 2: Delete the test file**
|
||||||
|
|
||||||
|
Run: `rm test/telegramQueue.test.ts`
|
||||||
|
|
||||||
|
- [ ] **Step 3: Update index.ts — remove upload-queue import and shutdown drain logic**
|
||||||
|
|
||||||
|
Remove line:
|
||||||
|
```typescript
|
||||||
|
import { clearQueue, getQueueStats, waitForQueue } from './infrastructure/telegram/upload-queue';
|
||||||
|
```
|
||||||
|
|
||||||
|
Remove the drain block (lines 70-81):
|
||||||
|
```typescript
|
||||||
|
// Drain pending upload queue with a timeout
|
||||||
|
const { pending, size } = getQueueStats();
|
||||||
|
if (pending > 0 || size > 0) {
|
||||||
|
logger.info('Draining upload queue', { pending, size });
|
||||||
|
const drainTimeout = setTimeout(() => {
|
||||||
|
logger.warn('Upload queue drain timeout — clearing remaining tasks');
|
||||||
|
clearQueue();
|
||||||
|
}, 30_000);
|
||||||
|
await waitForQueue();
|
||||||
|
clearTimeout(drainTimeout);
|
||||||
|
logger.info('Upload queue drained');
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: Run lint**
|
||||||
|
|
||||||
|
Run: `bunx biome check src/index.ts`
|
||||||
|
Expected: No errors.
|
||||||
|
|
||||||
|
- [ ] **Step 5: Run tests**
|
||||||
|
|
||||||
|
Run: `bun test`
|
||||||
|
Expected: All tests pass (some may be skipped due to missing queue test).
|
||||||
|
|
||||||
|
- [ ] **Step 6: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add src/infrastructure/telegram/upload-queue.ts test/telegramQueue.test.ts src/index.ts
|
||||||
|
git commit -m "refactor: remove global upload queue
|
||||||
|
|
||||||
|
Per-bot queues now handle concontrol internally.
|
||||||
|
|
||||||
|
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Task 5: Update env.ts and chunked-storage backpressure
|
||||||
|
|
||||||
|
Remove `uploadConcurrency` from config and update chunked-storage to derive effective concurrency from bot pool.
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `src/env.ts`
|
||||||
|
- Modify: `src/utils/chunked-storage.ts`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Remove `uploadConcurrency` from env.ts**
|
||||||
|
|
||||||
|
Remove:
|
||||||
|
```typescript
|
||||||
|
uploadConcurrency: number;
|
||||||
|
```
|
||||||
|
and:
|
||||||
|
```typescript
|
||||||
|
uploadConcurrency: parseNumber(process.env.UPLOAD_CONCURRENCY, 8),
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Update chunked-storage.ts backpressure**
|
||||||
|
|
||||||
|
Replace:
|
||||||
|
```typescript
|
||||||
|
import { config } from '../env';
|
||||||
|
// ...
|
||||||
|
if (inFlight.size >= config.uploadConcurrency * 2) {
|
||||||
|
```
|
||||||
|
With:
|
||||||
|
```typescript
|
||||||
|
import { botPool } from '../infrastructure/telegram/bot-pool';
|
||||||
|
// ...
|
||||||
|
if (inFlight.size >= botPool.getEffectiveConcurrency()) {
|
||||||
|
```
|
||||||
|
(Use effective concurrency * 2 for backpressure, or just use effective concurrency as the limit.)
|
||||||
|
|
||||||
|
Actually let me think about this more carefully. The backpressure in chunked-storage:
|
||||||
|
```
|
||||||
|
if (inFlight.size >= config.uploadConcurrency * 2) {
|
||||||
|
await Promise.race(inFlight);
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
This limits the number of in-flight chunks per file. With `uploadConcurrency: 16`, it was 32. Now with effective concurrency of 6 (6 bots), it would be 12. That's fine as backpressure — it prevents too many chunks from being in memory at once.
|
||||||
|
|
||||||
|
Let me use `botPool.getEffectiveConcurrency() * 2` to keep the same multiplier.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Run lint and tests**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bunx biome check src/env.ts src/utils/chunked-storage.ts
|
||||||
|
bun test test/chunked-storage.test.ts
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: All checks pass.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add src/env.ts src/utils/chunked-storage.ts
|
||||||
|
git commit -m "refactor: remove uploadConcurrency from config
|
||||||
|
|
||||||
|
Effective concurrency derived from bot pool size. Chunked-storage
|
||||||
|
backpressure now uses botPool.getEffectiveConcurrency().
|
||||||
|
|
||||||
|
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Task 6: Full integration test
|
||||||
|
|
||||||
|
- [ ] **Step 1: Run the full test suite**
|
||||||
|
|
||||||
|
Run: `bun test`
|
||||||
|
Expected: All tests pass.
|
||||||
|
|
||||||
|
- [ ] **Step 2: Run lint**
|
||||||
|
|
||||||
|
Run: `bunx biome check src test`
|
||||||
|
Expected: No errors.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Create summary commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add -A
|
||||||
|
git commit -m "refactor: implement per-bot queue architecture
|
||||||
|
|
||||||
|
- Each bot has its own PQueue with concurrency=1
|
||||||
|
- selectBot() assigns uploads to least-loaded available bot
|
||||||
|
- 429 rate limits are tracked per-bot with cooldown timers
|
||||||
|
- Failed uploads retry on next available bot
|
||||||
|
- Removed global upload-queue.ts and uploadConcurrency config
|
||||||
|
- Updated ITelegramService interface
|
||||||
|
|
||||||
|
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>"
|
||||||
|
```
|
||||||
@@ -1,5 +1,10 @@
|
|||||||
# Telegram Bot Uploader Backend Design
|
# Telegram Bot Uploader Backend Design
|
||||||
|
|
||||||
|
> ⚠️ **LEGACY** — Dokumen historis (2026-05-17). Port & infrastruktur sudah berubah:
|
||||||
|
> produksi kini berjalan di port `4000` (Nix + systemd + Caddy, domain `upload.asepharyana.my.id`)
|
||||||
|
> dan database via PgBouncer pool `100.121.180.82:6432` (bukan port 5432, bukan localhost).
|
||||||
|
|
||||||
|
|
||||||
**Date:** 2026-05-17
|
**Date:** 2026-05-17
|
||||||
**Status:** Approved
|
**Status:** Approved
|
||||||
**Stack:** Bun, Telegraf, PostgreSQL, Drizzle ORM, Winston, nanoid
|
**Stack:** Bun, Telegraf, PostgreSQL, Drizzle ORM, Winston, nanoid
|
||||||
@@ -35,7 +40,7 @@ Production-ready backend untuk Telegram file uploader dengan dual upload methods
|
|||||||
4. Bot extracts `telegram_file_id`, `telegram_file_unique_id`, `storage_message_id`
|
4. Bot extracts `telegram_file_id`, `telegram_file_unique_id`, `storage_message_id`
|
||||||
5. Bot generates `public_id` using nanoid
|
5. Bot generates `public_id` using nanoid
|
||||||
6. Bot saves metadata to PostgreSQL
|
6. Bot saves metadata to PostgreSQL
|
||||||
7. Bot replies with download link: `https://tele.asepharyana.my.id/f/{public_id}`
|
7. Bot replies with download link: `https://upload.asepharyana.my.id/f/{public_id}`
|
||||||
|
|
||||||
#### Upload via HTTP API
|
#### Upload via HTTP API
|
||||||
1. Client POSTs to `/api/upload` with file (multipart or base64)
|
1. Client POSTs to `/api/upload` with file (multipart or base64)
|
||||||
@@ -115,7 +120,7 @@ fileName: optional_filename.ext
|
|||||||
"size_bytes": 1024000,
|
"size_bytes": 1024000,
|
||||||
"file_type": "document",
|
"file_type": "document",
|
||||||
"created_at": "2026-05-17T23:42:19Z",
|
"created_at": "2026-05-17T23:42:19Z",
|
||||||
"download_url": "https://tele.asepharyana.my.id/f/abc123xyz"
|
"download_url": "https://upload.asepharyana.my.id/f/abc123xyz"
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -212,9 +217,9 @@ schema.sql
|
|||||||
```
|
```
|
||||||
BOT_TOKEN=<telegram_bot_token>
|
BOT_TOKEN=<telegram_bot_token>
|
||||||
STORAGE_CHANNEL_ID=<private_channel_id>
|
STORAGE_CHANNEL_ID=<private_channel_id>
|
||||||
BASE_URL=https://tele.asepharyana.my.id
|
BASE_URL=https://upload.asepharyana.my.id
|
||||||
DATABASE_URL=postgresql://user:password@localhost:5432/telegram_uploader
|
DATABASE_URL=postgresql://asephs:***@100.121.180.82:6432/uploader
|
||||||
PORT=3000
|
PORT=4000
|
||||||
NODE_ENV=production
|
NODE_ENV=production
|
||||||
LOG_LEVEL=info
|
LOG_LEVEL=info
|
||||||
RATE_LIMIT_WINDOW_MS=60000
|
RATE_LIMIT_WINDOW_MS=60000
|
||||||
|
|||||||
@@ -7,6 +7,12 @@ metadata:
|
|||||||
|
|
||||||
# Design: TeleUploader Deployment & CI/CD Setup
|
# Design: TeleUploader Deployment & CI/CD Setup
|
||||||
|
|
||||||
|
> ⚠️ **LEGACY** — Dokumen historis (2026-05-18) untuk arsitektur Docker + Traefik +
|
||||||
|
> GitHub Actions. Docker & Traefik sudah dihapus dari VPS produksi (2026-08-02):
|
||||||
|
> deploy sekarang Nix + systemd + Caddy di orangevps, port `4000`, domain
|
||||||
|
> `upload.asepharyana.my.id`, database via PgBouncer pool `100.121.180.82:6432`.
|
||||||
|
|
||||||
|
|
||||||
We are setting up production deployment for TeleUploader on VPS `45.127.35.244` behind Traefik utilizing GitHub Actions.
|
We are setting up production deployment for TeleUploader on VPS `45.127.35.244` behind Traefik utilizing GitHub Actions.
|
||||||
|
|
||||||
## 1. System Architecture
|
## 1. System Architecture
|
||||||
@@ -22,7 +28,7 @@ TeleUploader is a Bun-based service.
|
|||||||
### `Dockerfile`
|
### `Dockerfile`
|
||||||
- Multi-stage build.
|
- Multi-stage build.
|
||||||
- **Stage 1 (Build)**: Install dependencies, copy source files, run Biome lint/format checks, compile TS build to `dist/index.js` using `bun build`.
|
- **Stage 1 (Build)**: Install dependencies, copy source files, run Biome lint/format checks, compile TS build to `dist/index.js` using `bun build`.
|
||||||
- **Stage 2 (Run)**: Use minimal `oven/bun:1.1-slim` runtime. Copy `dist/index.js`, `schema.sql`, and `package.json`. Expose port `3000`.
|
- **Stage 2 (Run)**: Use minimal `oven/bun:1.1-slim` runtime. Copy `dist/index.js`, `schema.sql`, and `package.json`. Expose port `4000`.
|
||||||
|
|
||||||
### `docker-compose.yml`
|
### `docker-compose.yml`
|
||||||
```yaml
|
```yaml
|
||||||
@@ -38,7 +44,7 @@ services:
|
|||||||
- STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID}
|
- STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID}
|
||||||
- BASE_URL=${BASE_URL}
|
- BASE_URL=${BASE_URL}
|
||||||
- DATABASE_URL=${DATABASE_URL}
|
- DATABASE_URL=${DATABASE_URL}
|
||||||
- PORT=3000
|
- PORT=4000
|
||||||
- NODE_ENV=production
|
- NODE_ENV=production
|
||||||
- LOG_LEVEL=info
|
- LOG_LEVEL=info
|
||||||
networks:
|
networks:
|
||||||
@@ -49,7 +55,7 @@ services:
|
|||||||
- "traefik.http.routers.teleuploader.entrypoints=websecure"
|
- "traefik.http.routers.teleuploader.entrypoints=websecure"
|
||||||
- "traefik.http.routers.teleuploader.tls=true"
|
- "traefik.http.routers.teleuploader.tls=true"
|
||||||
- "traefik.http.routers.teleuploader.tls.certresolver=letsencrypt"
|
- "traefik.http.routers.teleuploader.tls.certresolver=letsencrypt"
|
||||||
- "traefik.http.services.teleuploader.loadbalancer.server.port=3000"
|
- "traefik.http.services.teleuploader.loadbalancer.server.port=4000"
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
app-shared-net:
|
app-shared-net:
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
# Gitea Remote and CI/CD Migration Design
|
# Gitea Remote and CI/CD Migration Design
|
||||||
|
|
||||||
|
> Catatan (2026-08-02): Produksi sekarang port 4000, deploy Nix+systemd di orangevps, Caddy reverse proxy upload.asepharyana.my.id, DB via pgbouncer pool imrnes 100.121.180.82:6432. Docker/Traefik/Gitea-CI legacy.
|
||||||
|
|
||||||
## Goal
|
## Goal
|
||||||
|
|
||||||
Move the repository origin from GitLab to a new Gitea repository and add a Gitea Actions deployment flow that behaves like a GitHub Actions CI/CD pipeline.
|
Move the repository origin from GitLab to a new Gitea repository and add a Gitea Actions deployment flow that behaves like a GitHub Actions CI/CD pipeline.
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
# S3 Compatibility Completion Design
|
# S3 Compatibility Completion Design
|
||||||
|
|
||||||
|
> Catatan (2026-08-02): Produksi sekarang port 4000, deploy Nix+systemd di orangevps, Caddy reverse proxy upload.asepharyana.my.id, DB via pgbouncer pool imrnes 100.121.180.82:6432. Docker/Traefik/Gitea-CI legacy.
|
||||||
|
|
||||||
Date: 2026-07-07
|
Date: 2026-07-07
|
||||||
|
|
||||||
## Goal
|
## Goal
|
||||||
|
|||||||
@@ -0,0 +1,127 @@
|
|||||||
|
# Per-Bot Queue: Rate-Limit Safe Telegram Upload
|
||||||
|
|
||||||
|
**Date:** 2026-07-29
|
||||||
|
**Status:** Approved Design
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
Telegram Bot API rate-limits each bot to approximately 1-2 concurrent uploads. When multiple upload chunks hit the same bot simultaneously, Telegram returns HTTP 429 (Too Many Requests), causing delays of 30-60 seconds per retry. Under Docker push load, these cumulative delays trigger Gitea client timeouts and `500 Internal Server Error`.
|
||||||
|
|
||||||
|
> Catatan (2026-08-02): Produksi sekarang port 4000, deploy Nix+systemd di orangevps, Caddy reverse proxy upload.asepharyana.my.id, DB via pgbouncer pool imrnes 100.121.180.82:6432. Docker/Traefik/Gitea-CI legacy.
|
||||||
|
|
||||||
|
The current architecture uses a **global PQueue** with `concurrency=N` where each task picks a bot via round-robin (`claimBotIndex()`). This means two concurrent tasks can both land on the same bot index (after wrap-around), causing 429 collisions.
|
||||||
|
|
||||||
|
## Solution: Per-Bot Queue
|
||||||
|
|
||||||
|
Each bot has its own PQueue with `concurrency=1`. Uploads are assigned to the bot with the fewest pending tasks. If a bot rate-limits, the upload moves to the next available bot.
|
||||||
|
|
||||||
|
### Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
┌────────────────────────────────────────────────────────┐
|
||||||
|
│ BotPool │
|
||||||
|
│ │
|
||||||
|
│ ┌──────────────────────────────────────────────────┐ │
|
||||||
|
│ │ BotEntry[0] token=b1 queue=PQueue(conc=1) │ │
|
||||||
|
│ │ rateLimitedUntil=0 │ │
|
||||||
|
│ ├──────────────────────────────────────────────────┤ │
|
||||||
|
│ │ BotEntry[1] token=b2 queue=PQueue(conc=1) │ │
|
||||||
|
│ │ rateLimitedUntil=0 │ │
|
||||||
|
│ ├──────────────────────────────────────────────────┤ │
|
||||||
|
│ │ ... up to N bots │ │
|
||||||
|
│ └──────────────────────────────────────────────────┘ │
|
||||||
|
│ │
|
||||||
|
│ selectBot(skip?): number │
|
||||||
|
│ └─ bot dengan pending queue paling sedikit │
|
||||||
|
│ dan tidak sedang rate-limited │
|
||||||
|
│ │
|
||||||
|
│ forwardToStorage(file): ForwardResult │
|
||||||
|
│ └─ retry loop: selectBot → queue.add → handle 429 │
|
||||||
|
└────────────────────────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
### BotEntry Structure
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
interface BotEntry {
|
||||||
|
index: number;
|
||||||
|
token: string;
|
||||||
|
instance: Telegraf;
|
||||||
|
queue: PQueue; // concurrency: 1
|
||||||
|
rateLimitedUntil: number; // epoch ms, 0 = not limited
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Data Flow: Upload
|
||||||
|
|
||||||
|
```
|
||||||
|
forwardToStorage(fileChunk, fileName, fileType)
|
||||||
|
│
|
||||||
|
├─ MAX_RETRIES loop (attempt up to all bots)
|
||||||
|
│ │
|
||||||
|
│ ├─ selectBot(attemptedIndexes)
|
||||||
|
│ │ ├─ Filter out rate-limited bots (rateLimitedUntil > Date.now())
|
||||||
|
│ │ ├─ Filter out already-attempted bots
|
||||||
|
│ │ ├─ If none available:
|
||||||
|
│ │ │ ├─ Wait MIN_SLEEP_MS (5000ms)
|
||||||
|
│ │ │ ├─ Reset rate-limited timers (clear attemptedIndexes)
|
||||||
|
│ │ │ └─ Retry selectBot
|
||||||
|
│ │ └─ Return bot with smallest queue.pending count
|
||||||
|
│ │
|
||||||
|
│ ├─ attemptedIndexes.add(selectedBot)
|
||||||
|
│ │
|
||||||
|
│ ├─ result = await bots[selectedBot].queue.add(() =>
|
||||||
|
│ │ executeTelegramCall(bot, fileChunk, fileName)
|
||||||
|
│ │ )
|
||||||
|
│ │ │
|
||||||
|
│ │ ├─ ✅ Success → return ForwardResult
|
||||||
|
│ │ │
|
||||||
|
│ │ └─ ❌ Error
|
||||||
|
│ │ ├─ 429 → markRateLimited(bot, retryAfter)
|
||||||
|
│ │ │ → continue to next bot in retry loop
|
||||||
|
│ │ ├─ Transient (timeout, 5xx) → continue
|
||||||
|
│ │ └─ Non-transient → throw (propagate up)
|
||||||
|
│ │
|
||||||
|
│ └─ Attempt counter exhausted → throw lastError
|
||||||
|
│
|
||||||
|
└─ Sorted part tracking (for chunked uploads)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Key Design Decisions
|
||||||
|
|
||||||
|
1. **Concurrency=1 per bot**: Guarantees no two Telegram API calls compete for the same bot token. With 6 bots, effective concurrency = 6.
|
||||||
|
|
||||||
|
2. **Least-loaded assignment**: `selectBot()` picks the bot with the fewest queued + pending tasks. This naturally load-balances even when some bots are slower.
|
||||||
|
|
||||||
|
3. **Rate-limit isolation**: When bot A hits 429, only bot A's queue is paused. Other 5 bots continue serving uploads uninterrupted.
|
||||||
|
|
||||||
|
4. **Per-bot rate-limit timer**: `rateLimitedUntil` prevents re-selecting a recently-429'd bot until its cooldown expires.
|
||||||
|
|
||||||
|
5. **No global PQueue**: The old `upload-queue.ts` is removed. Each bot owns its queue, eliminating the global backpressure problem.
|
||||||
|
|
||||||
|
### Changes by File
|
||||||
|
|
||||||
|
| File | Action |
|
||||||
|
|------|--------|
|
||||||
|
| `src/infrastructure/telegram/bot-pool.ts` | **Major refactor**: BotEntry array, selectBot(), per-bot queues, retry loop |
|
||||||
|
| `src/infrastructure/telegram/upload-queue.ts` | **Delete**: No longer needed |
|
||||||
|
| `src/domain/ports/telegram-service.ts` | **Remove** `enqueueUpload<T>(task: () => Promise<T>): Promise<T>` from interface |
|
||||||
|
| `src/utils/chunked-storage.ts` | **No changes** — only uses `forwardToStorage()` |
|
||||||
|
| `src/env.ts` | **Remove** `uploadConcurrency` config (no longer needed) |
|
||||||
|
|
||||||
|
### Error Handling
|
||||||
|
|
||||||
|
- **429 per bot**: Mark bot rate-limited, move to next. Clear timer after `retryAfter` seconds.
|
||||||
|
- **All bots 429**: Sleep 5 seconds with jitter, then retry from bot 0.
|
||||||
|
- **Transient errors** (timeout, 5xx, connection reset): Retry on same bot (inside its queue), then on next bot.
|
||||||
|
- **Non-transient errors** (4xx other than 429, wrong file_id, auth errors): Propagate immediately.
|
||||||
|
- **MAX_RETRIES**: 10 attempts across all bots before giving up.
|
||||||
|
|
||||||
|
### Testing
|
||||||
|
|
||||||
|
- Unit: `selectBot()` returns bot with fewest pending tasks
|
||||||
|
- Unit: `selectBot()` skips rate-limited bots
|
||||||
|
- Unit: 429 on bot 0 → retries on bot 1 → succeeds
|
||||||
|
- Unit: All bots rate-limited → sleeps → retries → succeeds
|
||||||
|
- Unit: Per-bot queue has concurrency=1 (two tasks to same bot queue sequentially)
|
||||||
|
- Integration: Forward a real file through the per-bot pool
|
||||||
Generated
+61
@@ -0,0 +1,61 @@
|
|||||||
|
{
|
||||||
|
"nodes": {
|
||||||
|
"flake-utils": {
|
||||||
|
"inputs": {
|
||||||
|
"systems": "systems"
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1731533236,
|
||||||
|
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
|
||||||
|
"owner": "numtide",
|
||||||
|
"repo": "flake-utils",
|
||||||
|
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "numtide",
|
||||||
|
"repo": "flake-utils",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nixpkgs": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1785301185,
|
||||||
|
"narHash": "sha256-eoS3KQTO0aPWXZvIaRbRAzSSHW3l5wdMFXtT1ISfoKA=",
|
||||||
|
"owner": "NixOS",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"rev": "9bc02893134c733dd85de46ee4fb2fac696b5529",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "NixOS",
|
||||||
|
"ref": "nixpkgs-unstable",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": {
|
||||||
|
"inputs": {
|
||||||
|
"flake-utils": "flake-utils",
|
||||||
|
"nixpkgs": "nixpkgs"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"systems": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1681028828,
|
||||||
|
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||||
|
"owner": "nix-systems",
|
||||||
|
"repo": "default",
|
||||||
|
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "nix-systems",
|
||||||
|
"repo": "default",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": "root",
|
||||||
|
"version": 7
|
||||||
|
}
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
{
|
||||||
|
description = "TeleUploader — Nix build";
|
||||||
|
|
||||||
|
inputs = {
|
||||||
|
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
|
||||||
|
flake-utils.url = "github:numtide/flake-utils";
|
||||||
|
};
|
||||||
|
|
||||||
|
outputs = { self, nixpkgs, flake-utils }:
|
||||||
|
flake-utils.lib.eachSystem [ "x86_64-linux" ] (system:
|
||||||
|
let
|
||||||
|
pkgs = import nixpkgs { inherit system; };
|
||||||
|
|
||||||
|
# TeleUploader package
|
||||||
|
teleuploader = pkgs.stdenvNoCC.mkDerivation rec {
|
||||||
|
pname = "teleuploader";
|
||||||
|
version = "1.1.0";
|
||||||
|
|
||||||
|
src = ./.;
|
||||||
|
|
||||||
|
nativeBuildInputs = [
|
||||||
|
pkgs.bun
|
||||||
|
pkgs.makeBinaryWrapper
|
||||||
|
];
|
||||||
|
|
||||||
|
# Bun cache di sandbox — prevent online fetch
|
||||||
|
# Karena bun.lock sudah di repo, bun install --frozen-lockfile
|
||||||
|
# akan pake cache, tapi di Nix sandbox gak ada internet.
|
||||||
|
# Solusi: offline flag
|
||||||
|
preBuild = ''
|
||||||
|
export HOME=$TMPDIR/home
|
||||||
|
mkdir -p $HOME
|
||||||
|
export BUN_INSTALL=$HOME/.bun
|
||||||
|
'';
|
||||||
|
|
||||||
|
buildPhase = ''
|
||||||
|
echo "=== Installing dependencies ==="
|
||||||
|
bun install --frozen-lockfile --ignore-scripts 2>&1
|
||||||
|
|
||||||
|
echo "=== Building ==="
|
||||||
|
bun run build 2>&1
|
||||||
|
'';
|
||||||
|
|
||||||
|
installPhase = ''
|
||||||
|
mkdir -p $out/bin $out/share/teleuploader
|
||||||
|
|
||||||
|
# Copy dist files
|
||||||
|
cp -r dist $out/share/teleuploader/dist
|
||||||
|
cp src/home.html $out/share/teleuploader/ 2>/dev/null || true
|
||||||
|
cp schema.sql $out/share/teleuploader/ 2>/dev/null || true
|
||||||
|
|
||||||
|
# Wrap with bun from Nix store (dependency sharing!)
|
||||||
|
# Note: NO --chdir — systemd WorkingDirectory controls this
|
||||||
|
makeBinaryWrapper ${pkgs.bun}/bin/bun $out/bin/teleuploader \
|
||||||
|
--add-flags "$out/share/teleuploader/dist/index.js" \
|
||||||
|
--set-default NODE_ENV production \
|
||||||
|
--prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.bun ]}
|
||||||
|
|
||||||
|
# Also create the migrate wrapper
|
||||||
|
makeBinaryWrapper ${pkgs.bun}/bin/bun $out/bin/teleuploader-migrate \
|
||||||
|
--add-flags "$out/share/teleuploader/dist/migrate.js" \
|
||||||
|
--prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.bun ]}
|
||||||
|
'';
|
||||||
|
|
||||||
|
meta = {
|
||||||
|
description = "Telegram file uploader backend (S3 → Telegram)";
|
||||||
|
license = pkgs.lib.licenses.mit;
|
||||||
|
platforms = pkgs.lib.platforms.linux;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
in {
|
||||||
|
packages = {
|
||||||
|
inherit teleuploader;
|
||||||
|
default = teleuploader;
|
||||||
|
};
|
||||||
|
|
||||||
|
# Dev shell with bun for local development
|
||||||
|
devShells.default = pkgs.mkShell {
|
||||||
|
buildInputs = [
|
||||||
|
pkgs.bun
|
||||||
|
pkgs.nodejs_22
|
||||||
|
];
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
+3
-3
@@ -1,14 +1,14 @@
|
|||||||
{
|
{
|
||||||
"name": "filedrop",
|
"name": "filedrop",
|
||||||
"version": "1.0.0",
|
"version": "1.1.0",
|
||||||
"description": "Telegram file uploader backend",
|
"description": "Telegram file uploader backend",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "bun --hot src/index.ts",
|
"dev": "bun --hot src/index.ts",
|
||||||
"build": "bun build src/index.ts --target=bun --outfile=dist/index.js && bun build src/db/migrate.ts --target=bun --outfile=dist/migrate.js",
|
"build": "bun build src/index.ts --target=bun --outfile=dist/index.js && bun build src/infrastructure/persistence/drizzle/migrate.ts --target=bun --outfile=dist/migrate.js",
|
||||||
"start": "NODE_ENV=production bun dist/index.js",
|
"start": "NODE_ENV=production bun dist/index.js",
|
||||||
"db:migrate": "bun dist/migrate.js",
|
"db:migrate": "bun dist/migrate.js",
|
||||||
"test": "bun test --preload ./test/helpers/setup-env.ts test/rateLimit.test.ts && bun test --preload ./test/helpers/setup-env.ts test/file.test.ts && bun test --preload ./test/helpers/setup-env.ts test/telegram.test.ts && bun test --preload ./test/helpers/setup-env.ts test/upload.test.ts && bun test --preload ./test/helpers/setup-env.ts test/files.test.ts && bun test --preload ./test/helpers/setup-env.ts test/health.test.ts && bun test --preload ./test/helpers/setup-env.ts test/db.test.ts && bun test --preload ./test/helpers/setup-env.ts test/bot.test.ts && bun test --preload ./test/helpers/setup-env.ts test/bootstrap.test.ts && bun test --preload ./test/helpers/setup-env.ts test/swagger.test.ts && bun test --preload ./test/helpers/setup-env.ts test/auth.test.ts && bun test --preload ./test/helpers/setup-env.ts test/auth-routes.test.ts && bun test --preload ./test/helpers/setup-env.ts test/s3-auth.test.ts && bun test --preload ./test/helpers/setup-env.ts test/s3-operations.test.ts && bun test --preload ./test/helpers/setup-env.ts test/s3-bucket-config.test.ts && bun test --preload ./test/helpers/setup-env.ts test/web-api.test.ts && bun test --preload ./test/helpers/setup-env.ts test/env.test.ts && bun test --preload ./test/helpers/setup-env.ts test/telegramQueue.test.ts",
|
"test": "bun test --preload ./test/helpers/setup-env.ts test/rateLimit.test.ts && bun test --preload ./test/helpers/setup-env.ts test/file.test.ts && bun test --preload ./test/helpers/setup-env.ts test/telegram.test.ts && bun test --preload ./test/helpers/setup-env.ts test/upload.test.ts && bun test --preload ./test/helpers/setup-env.ts test/files.test.ts && bun test --preload ./test/helpers/setup-env.ts test/health.test.ts && bun test --preload ./test/helpers/setup-env.ts test/db.test.ts && bun test --preload ./test/helpers/setup-env.ts test/bot.test.ts && bun test --preload ./test/helpers/setup-env.ts test/bootstrap.test.ts && bun test --preload ./test/helpers/setup-env.ts test/swagger.test.ts && bun test --preload ./test/helpers/setup-env.ts test/auth.test.ts && bun test --preload ./test/helpers/setup-env.ts test/auth-routes.test.ts && bun test --preload ./test/helpers/setup-env.ts test/s3-auth.test.ts && bun test --preload ./test/helpers/setup-env.ts test/s3-operations.test.ts && bun test --preload ./test/helpers/setup-env.ts test/s3-bucket-config.test.ts && bun test --preload ./test/helpers/setup-env.ts test/web-api.test.ts && bun test --preload ./test/helpers/setup-env.ts test/env.test.ts && bun test --preload ./test/helpers/setup-env.ts test/bot-pool.test.ts",
|
||||||
"test:s3-auth": "bun test --preload ./test/helpers/setup-env.ts test/s3-auth.test.ts",
|
"test:s3-auth": "bun test --preload ./test/helpers/setup-env.ts test/s3-auth.test.ts",
|
||||||
"test:s3-ops": "bun test --preload ./test/helpers/setup-env.ts test/s3-operations.test.ts",
|
"test:s3-ops": "bun test --preload ./test/helpers/setup-env.ts test/s3-operations.test.ts",
|
||||||
"test:web-api": "bun test --preload ./test/helpers/setup-env.ts test/web-api.test.ts",
|
"test:web-api": "bun test --preload ./test/helpers/setup-env.ts test/web-api.test.ts",
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
/**
|
/**
|
||||||
* Input for the upload file use case.
|
* Input for the upload file use case.
|
||||||
* Carries all metadata needed to persist an uploaded file,
|
* Carries all metadata needed to persist an uploaded file,
|
||||||
* including its temporary location on disk and optional bucket/S3 context.
|
* including its temporary location on disk and optional bucket/S3 context.a
|
||||||
*/
|
*/
|
||||||
export interface UploadInput {
|
export interface UploadInput {
|
||||||
/** Absolute path to the temporary file on disk */
|
/** Absolute path to the temporary file on disk */
|
||||||
|
|||||||
@@ -1,10 +1,11 @@
|
|||||||
import { nanoid } from 'nanoid';
|
import { nanoid } from 'nanoid';
|
||||||
|
import { buildNewFile } from '../../domain/entities/file-factory';
|
||||||
import type { MultipartUpload } from '../../domain/entities/multipart';
|
import type { MultipartUpload } from '../../domain/entities/multipart';
|
||||||
import type { IBucketRepository } from '../../domain/ports/bucket-repository';
|
import type { IBucketRepository } from '../../domain/ports/bucket-repository';
|
||||||
import type { IFileRepository } from '../../domain/ports/file-repository';
|
import type { IFileRepository } from '../../domain/ports/file-repository';
|
||||||
import type { IMultipartRepository } from '../../domain/ports/multipart-repository';
|
import type { IMultipartRepository } from '../../domain/ports/multipart-repository';
|
||||||
import type { ITelegramService } from '../../domain/ports/telegram-service';
|
import type { ITelegramService } from '../../domain/ports/telegram-service';
|
||||||
import { computeHash } from '../../shared/utils/file';
|
import { computeHash, DEFAULT_FILE_TYPE } from '../../shared/utils/file';
|
||||||
|
|
||||||
// ─── Types ──────────────────────────────────────────────────────────
|
// ─── Types ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -270,31 +271,23 @@ export function createCompleteMultipartUploadUseCase(deps: MultipartDeps) {
|
|||||||
|
|
||||||
const publicId = nanoid();
|
const publicId = nanoid();
|
||||||
|
|
||||||
await deps.fileRepo.create({
|
await deps.fileRepo.create(
|
||||||
publicId,
|
buildNewFile({
|
||||||
telegramFileId: firstPart.telegramFileId,
|
publicId,
|
||||||
telegramFileUniqueId: firstPart.telegramFileUniqueId,
|
telegramFileId: firstPart.telegramFileId,
|
||||||
storageChatId: deps.config.storageChatId,
|
telegramFileUniqueId: firstPart.telegramFileUniqueId,
|
||||||
storageMessageId: firstPart.storageMessageId,
|
storageChatId: deps.config.storageChatId,
|
||||||
fileName: input.key.split('/').pop() || 'file',
|
storageMessageId: firstPart.storageMessageId,
|
||||||
mimeType: 'application/octet-stream',
|
fileName: input.key.split('/').pop() || 'file',
|
||||||
sizeBytes: totalSize,
|
mimeType: 'application/octet-stream',
|
||||||
fileType: 'document',
|
sizeBytes: totalSize,
|
||||||
uploaderId: 0,
|
fileType: DEFAULT_FILE_TYPE,
|
||||||
fileHash: null,
|
storageBackend: 'telegram',
|
||||||
archiveTelegramFileId: null,
|
bucketId: multipart.bucketId,
|
||||||
archiveStorageMessageId: null,
|
s3Key: input.key,
|
||||||
archiveFileName: null,
|
multipartUploadId: input.uploadId,
|
||||||
archiveEntryName: null,
|
}),
|
||||||
archiveMimeType: null,
|
);
|
||||||
archiveSizeBytes: null,
|
|
||||||
bucketId: multipart.bucketId,
|
|
||||||
s3Key: input.key,
|
|
||||||
storageBackend: 'telegram',
|
|
||||||
isDeleted: false,
|
|
||||||
multipartUploadId: input.uploadId,
|
|
||||||
partCount: null,
|
|
||||||
});
|
|
||||||
|
|
||||||
await deps.multipartRepo.complete(input.uploadId);
|
await deps.multipartRepo.complete(input.uploadId);
|
||||||
|
|
||||||
|
|||||||
@@ -1,23 +1,13 @@
|
|||||||
import { randomUUID } from 'node:crypto';
|
|
||||||
import { gzipSync } from 'node:zlib';
|
|
||||||
import { nanoid } from 'nanoid';
|
|
||||||
import type { File } from '../../domain/entities/file';
|
import type { File } from '../../domain/entities/file';
|
||||||
import type { NewFilePart } from '../../domain/entities/file-part';
|
|
||||||
import type { MultipartPart } from '../../domain/entities/multipart';
|
|
||||||
import type { IBucketRepository } from '../../domain/ports/bucket-repository';
|
import type { IBucketRepository } from '../../domain/ports/bucket-repository';
|
||||||
import type { IFilePartRepository } from '../../domain/ports/file-part-repository';
|
import type { IFilePartRepository } from '../../domain/ports/file-part-repository';
|
||||||
import type { IFileRepository, S3FileRecord } from '../../domain/ports/file-repository';
|
import type { IFileRepository } from '../../domain/ports/file-repository';
|
||||||
import type { IMultipartRepository } from '../../domain/ports/multipart-repository';
|
import type { IMultipartRepository } from '../../domain/ports/multipart-repository';
|
||||||
import type { ITelegramService, TelegramFileInfo } from '../../domain/ports/telegram-service';
|
import type { ITelegramService, TelegramFileInfo } from '../../domain/ports/telegram-service';
|
||||||
import { computeHash, ensureExtension, formatCreatedAt } from '../../shared/utils/file';
|
import type { CompressionAlgorithm } from '../../shared/utils/compress';
|
||||||
|
|
||||||
// ─── Types ──────────────────────────────────────────────────────────
|
// ─── Types ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/**
|
|
||||||
* Compression algorithm used for chunked object storage.
|
|
||||||
*/
|
|
||||||
type CompressionAlgorithm = 'gzip' | null;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A single part source for building a multi-part streaming response.
|
* A single part source for building a multi-part streaming response.
|
||||||
* Each part corresponds to a Telegram-stored file chunk.
|
* Each part corresponds to a Telegram-stored file chunk.
|
||||||
@@ -173,619 +163,3 @@ export interface S3ObjectDeps {
|
|||||||
/** Application configuration subset. */
|
/** Application configuration subset. */
|
||||||
config: S3ObjectConfig;
|
config: S3ObjectConfig;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── Helpers ────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Validates the configured chunk size and returns it as a safe integer.
|
|
||||||
*
|
|
||||||
* @param chunkSizeBytes - The configured chunk size in bytes.
|
|
||||||
* @returns The same value if it is a positive safe integer.
|
|
||||||
*/
|
|
||||||
const asSafeChunkSize = (chunkSizeBytes: number): number => {
|
|
||||||
if (!Number.isSafeInteger(chunkSizeBytes) || chunkSizeBytes <= 0) {
|
|
||||||
throw new Error('Invalid Telegram chunk size');
|
|
||||||
}
|
|
||||||
return chunkSizeBytes;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Optionally gzip-compresses a chunk if compression is enabled and the chunk
|
|
||||||
* is large enough to benefit from it.
|
|
||||||
*
|
|
||||||
* @param chunk - The raw chunk buffer.
|
|
||||||
* @param compress - Whether compression is enabled.
|
|
||||||
* @param compressionMinSizeBytes - Minimum chunk size to attempt compression.
|
|
||||||
* @returns The (possibly compressed) buffer and the algorithm used.
|
|
||||||
*/
|
|
||||||
const maybeCompressChunk = (
|
|
||||||
chunk: Buffer,
|
|
||||||
compress: boolean,
|
|
||||||
compressionMinSizeBytes: number,
|
|
||||||
): { bytes: Buffer; compressionAlgorithm: CompressionAlgorithm } => {
|
|
||||||
if (!compress || chunk.byteLength < compressionMinSizeBytes) {
|
|
||||||
return { bytes: chunk, compressionAlgorithm: null };
|
|
||||||
}
|
|
||||||
|
|
||||||
const gzipped = gzipSync(chunk);
|
|
||||||
if (gzipped.byteLength >= chunk.byteLength) {
|
|
||||||
return { bytes: chunk, compressionAlgorithm: null };
|
|
||||||
}
|
|
||||||
|
|
||||||
return { bytes: gzipped, compressionAlgorithm: 'gzip' };
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Metadata for a single uploaded chunk/part during S3 put-object.
|
|
||||||
*/
|
|
||||||
interface UploadedPart {
|
|
||||||
/** 1-based part number. */
|
|
||||||
partNumber: number;
|
|
||||||
/** Telegram file identifier for this part. */
|
|
||||||
telegramFileId: string;
|
|
||||||
/** Telegram unique file identifier (stable across bot tokens). */
|
|
||||||
telegramFileUniqueId: string;
|
|
||||||
/** Message ID within the storage chat. */
|
|
||||||
storageMessageId: number;
|
|
||||||
/** Original size of the chunk in bytes before compression. */
|
|
||||||
sizeBytes: number;
|
|
||||||
/** Stored (post-compression) size in bytes. */
|
|
||||||
storedSizeBytes: number;
|
|
||||||
/** Compression algorithm applied, or null if uncompressed. */
|
|
||||||
compressionAlgorithm: CompressionAlgorithm;
|
|
||||||
/** SHA-256 hash of the original chunk content. */
|
|
||||||
etag: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Result of uploading an object in multiple Telegram chunks. */
|
|
||||||
interface ChunkedUploadResult {
|
|
||||||
/** Metadata for each uploaded part. */
|
|
||||||
parts: UploadedPart[];
|
|
||||||
/** SHA-256 hex digest of the complete object content. */
|
|
||||||
fileHash: string;
|
|
||||||
/** Total object size in bytes (sum of all original chunks). */
|
|
||||||
totalSizeBytes: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Uploads a buffer to Telegram in chunks, returning metadata for all parts.
|
|
||||||
*
|
|
||||||
* @param buffer - The full object buffer.
|
|
||||||
* @param partFileNamePrefix - Prefix used for each chunk's file name in Telegram.
|
|
||||||
* @param chunkSizeBytes - Maximum size of each chunk in bytes.
|
|
||||||
* @param compress - Whether gzip compression is enabled.
|
|
||||||
* @param compressionMinSizeBytes - Minimum chunk size to attempt compression.
|
|
||||||
* @param telegramService - The Telegram service to forward each chunk.
|
|
||||||
* @returns The aggregated chunked upload result.
|
|
||||||
*/
|
|
||||||
const uploadInChunks = async (
|
|
||||||
buffer: Buffer,
|
|
||||||
partFileNamePrefix: string,
|
|
||||||
chunkSizeBytes: number,
|
|
||||||
compress: boolean,
|
|
||||||
compressionMinSizeBytes: number,
|
|
||||||
telegramService: ITelegramService,
|
|
||||||
): Promise<ChunkedUploadResult> => {
|
|
||||||
const safeChunkSize = asSafeChunkSize(chunkSizeBytes);
|
|
||||||
const hasher = new Bun.CryptoHasher('sha256');
|
|
||||||
const parts: UploadedPart[] = [];
|
|
||||||
let totalSizeBytes = 0;
|
|
||||||
let partNumber = 0;
|
|
||||||
let offset = 0;
|
|
||||||
|
|
||||||
while (offset < buffer.byteLength) {
|
|
||||||
const chunk = buffer.subarray(offset, offset + safeChunkSize);
|
|
||||||
if (chunk.byteLength === 0) break;
|
|
||||||
|
|
||||||
partNumber += 1;
|
|
||||||
totalSizeBytes += chunk.byteLength;
|
|
||||||
hasher.update(chunk);
|
|
||||||
|
|
||||||
const { bytes, compressionAlgorithm } = maybeCompressChunk(
|
|
||||||
chunk,
|
|
||||||
compress,
|
|
||||||
compressionMinSizeBytes,
|
|
||||||
);
|
|
||||||
|
|
||||||
const forwardResult = await telegramService.forwardToStorage(
|
|
||||||
bytes,
|
|
||||||
`${partFileNamePrefix}.part-${partNumber}`,
|
|
||||||
'document',
|
|
||||||
);
|
|
||||||
|
|
||||||
parts.push({
|
|
||||||
partNumber,
|
|
||||||
telegramFileId: forwardResult.telegramFileId,
|
|
||||||
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
|
|
||||||
storageMessageId: forwardResult.storageMessageId,
|
|
||||||
sizeBytes: chunk.byteLength,
|
|
||||||
storedSizeBytes: bytes.byteLength,
|
|
||||||
compressionAlgorithm,
|
|
||||||
etag: computeHash(chunk),
|
|
||||||
});
|
|
||||||
|
|
||||||
offset += safeChunkSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
parts,
|
|
||||||
fileHash: hasher.digest('hex'),
|
|
||||||
totalSizeBytes,
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Resolves a list of multipart parts to their Telegram CDN URLs.
|
|
||||||
*
|
|
||||||
* @param parts - The stored multipart parts.
|
|
||||||
* @param telegramService - The Telegram service for resolving file metadata.
|
|
||||||
* @returns An array of resolved part sources.
|
|
||||||
*/
|
|
||||||
const resolveMultipartParts = async (
|
|
||||||
parts: MultipartPart[],
|
|
||||||
telegramService: ITelegramService,
|
|
||||||
): Promise<ObjectPartSource[]> => {
|
|
||||||
const sources: ObjectPartSource[] = [];
|
|
||||||
for (const part of parts) {
|
|
||||||
const fileInfo = await telegramService.getFileInfo(part.telegramFileId);
|
|
||||||
sources.push({
|
|
||||||
telegramFileId: part.telegramFileId,
|
|
||||||
telegramUrl: `https://api.telegram.org/file/bot${fileInfo.bot_token}/${fileInfo.file_path}`,
|
|
||||||
sizeBytes: part.sizeBytes,
|
|
||||||
partNumber: part.partNumber,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return sources;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Formats a `createdAt` value into an HTTP Last-Modified header value.
|
|
||||||
*
|
|
||||||
* @param date - The date to format.
|
|
||||||
* @returns The UTC string representation.
|
|
||||||
*/
|
|
||||||
const formatLastModified = (date: Date | string | number): string => {
|
|
||||||
return date instanceof Date ? date.toUTCString() : new Date(date).toUTCString();
|
|
||||||
};
|
|
||||||
|
|
||||||
// ─── Use Case Factories ─────────────────────────────────────────────
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a use case that resolves an S3 object for retrieval (GET).
|
|
||||||
*
|
|
||||||
* Looks up the bucket and file by key, then determines the storage type:
|
|
||||||
* - **direct**: regular Telegram-stored object — resolves the Telegram CDN URL.
|
|
||||||
* - **chunked**: object stored across multiple Telegram file parts.
|
|
||||||
* - **multipart**: object assembled from a completed multipart upload — resolves
|
|
||||||
* the Telegram CDN URLs for each part.
|
|
||||||
*
|
|
||||||
* @param deps - The injected dependencies.
|
|
||||||
* @returns An async function accepting bucket name and object key, returning
|
|
||||||
* a discriminated union of possible results, or `null` when the
|
|
||||||
* bucket or file is not found.
|
|
||||||
*/
|
|
||||||
export function createGetObjectUseCase(deps: S3ObjectDeps) {
|
|
||||||
return async (bucketName: string, key: string): Promise<GetObjectResult | null> => {
|
|
||||||
const bucket = await deps.bucketRepo.findByName(bucketName);
|
|
||||||
if (!bucket) return null;
|
|
||||||
|
|
||||||
const file = await deps.fileRepo.findByBucketAndKey(bucket.id, key);
|
|
||||||
if (!file) return null;
|
|
||||||
|
|
||||||
// Chunked storage — return the entity; the caller resolves parts via
|
|
||||||
// chunked-storage helpers.
|
|
||||||
if (file.storageBackend === 'chunked') {
|
|
||||||
return { type: 'chunked', file };
|
|
||||||
}
|
|
||||||
|
|
||||||
// Multipart upload object — resolve part Telegram URLs
|
|
||||||
if (file.multipartUploadId) {
|
|
||||||
const parts = await deps.multipartRepo.listParts(file.multipartUploadId);
|
|
||||||
const resolvedParts = await resolveMultipartParts(parts, deps.telegramService);
|
|
||||||
return { type: 'multipart', file, parts: resolvedParts };
|
|
||||||
}
|
|
||||||
|
|
||||||
// Regular direct object — resolve Telegram CDN URL
|
|
||||||
const fileInfo = await deps.telegramService.getFileInfo(file.telegramFileId);
|
|
||||||
const telegramUrl = `https://api.telegram.org/file/bot${fileInfo.bot_token}/${fileInfo.file_path}`;
|
|
||||||
|
|
||||||
return { type: 'direct', file, telegramUrl, fileInfo };
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a use case that retrieves S3 object metadata (HEAD).
|
|
||||||
*
|
|
||||||
* @param deps - The injected dependencies.
|
|
||||||
* @returns An async function accepting bucket name and object key, returning
|
|
||||||
* metadata or `null` when the bucket or file is not found.
|
|
||||||
*/
|
|
||||||
export function createHeadObjectUseCase(deps: S3ObjectDeps) {
|
|
||||||
return async (bucketName: string, key: string): Promise<HeadObjectMetadata | null> => {
|
|
||||||
const bucket = await deps.bucketRepo.findByName(bucketName);
|
|
||||||
if (!bucket) return null;
|
|
||||||
|
|
||||||
const file = await deps.fileRepo.findByBucketAndKey(bucket.id, key);
|
|
||||||
if (!file) return null;
|
|
||||||
|
|
||||||
return {
|
|
||||||
contentType: file.mimeType,
|
|
||||||
contentLength: file.sizeBytes,
|
|
||||||
etag: file.fileHash || nanoid(16),
|
|
||||||
lastModified: formatLastModified(file.createdAt),
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a use case that stores an S3 object (PUT).
|
|
||||||
*
|
|
||||||
* Handles both chunked (large) and single-message (small) upload paths,
|
|
||||||
* deduplicates by bucket+key (idempotent PUT), and persists the file
|
|
||||||
* record and (for chunked storage) part records.
|
|
||||||
*
|
|
||||||
* @param deps - The injected dependencies.
|
|
||||||
* @returns An async function accepting bucket name, key, body buffer, and
|
|
||||||
* content type, returning the etag of the stored object. Returns
|
|
||||||
* `null` when the bucket is not found.
|
|
||||||
*/
|
|
||||||
export function createPutObjectUseCase(deps: S3ObjectDeps) {
|
|
||||||
return async (
|
|
||||||
bucketName: string,
|
|
||||||
key: string,
|
|
||||||
body: Buffer,
|
|
||||||
contentType: string,
|
|
||||||
): Promise<PutObjectResult | null> => {
|
|
||||||
const bucket = await deps.bucketRepo.findByName(bucketName);
|
|
||||||
if (!bucket) return null;
|
|
||||||
|
|
||||||
const hash = computeHash(body);
|
|
||||||
|
|
||||||
// Idempotent PUT: if the object already exists, skip upload
|
|
||||||
const existing = await deps.fileRepo.findByBucketAndKey(bucket.id, key);
|
|
||||||
if (existing) {
|
|
||||||
return { etag: `"${hash}"` };
|
|
||||||
}
|
|
||||||
|
|
||||||
const fileName = key.split('/').pop() || 'file';
|
|
||||||
const signatureBuffer = body.subarray(0, 16);
|
|
||||||
const { fileName: finalFileName, mimeType } = ensureExtension(
|
|
||||||
fileName,
|
|
||||||
signatureBuffer,
|
|
||||||
contentType,
|
|
||||||
);
|
|
||||||
|
|
||||||
const partFileNamePrefix = `s3-${bucket.name}-${key.replace(/\//g, '_')}`;
|
|
||||||
const {
|
|
||||||
telegramChunkSizeBytes,
|
|
||||||
compressChunkedUploads,
|
|
||||||
chunkCompressionMinSizeBytes,
|
|
||||||
storageChatId,
|
|
||||||
} = deps.config;
|
|
||||||
|
|
||||||
if (body.byteLength > telegramChunkSizeBytes) {
|
|
||||||
// Chunked upload path
|
|
||||||
const chunkResult = await uploadInChunks(
|
|
||||||
body,
|
|
||||||
partFileNamePrefix,
|
|
||||||
telegramChunkSizeBytes,
|
|
||||||
compressChunkedUploads,
|
|
||||||
chunkCompressionMinSizeBytes,
|
|
||||||
deps.telegramService,
|
|
||||||
);
|
|
||||||
|
|
||||||
const firstPart = chunkResult.parts[0];
|
|
||||||
if (!firstPart) {
|
|
||||||
throw new Error('Chunked upload produced no parts');
|
|
||||||
}
|
|
||||||
|
|
||||||
const fileId = randomUUID();
|
|
||||||
const publicId = nanoid();
|
|
||||||
|
|
||||||
await deps.fileRepo.create({
|
|
||||||
publicId,
|
|
||||||
telegramFileId: firstPart.telegramFileId,
|
|
||||||
telegramFileUniqueId: firstPart.telegramFileUniqueId,
|
|
||||||
storageChatId,
|
|
||||||
storageMessageId: firstPart.storageMessageId,
|
|
||||||
fileName: finalFileName,
|
|
||||||
mimeType,
|
|
||||||
sizeBytes: chunkResult.totalSizeBytes,
|
|
||||||
fileType: 'document',
|
|
||||||
uploaderId: 0,
|
|
||||||
fileHash: chunkResult.fileHash,
|
|
||||||
archiveTelegramFileId: null,
|
|
||||||
archiveStorageMessageId: null,
|
|
||||||
archiveFileName: null,
|
|
||||||
archiveEntryName: null,
|
|
||||||
archiveMimeType: null,
|
|
||||||
archiveSizeBytes: null,
|
|
||||||
bucketId: bucket.id,
|
|
||||||
s3Key: key,
|
|
||||||
storageBackend: 'chunked',
|
|
||||||
isDeleted: false,
|
|
||||||
multipartUploadId: null,
|
|
||||||
partCount: chunkResult.parts.length,
|
|
||||||
});
|
|
||||||
|
|
||||||
const fileParts: NewFilePart[] = chunkResult.parts.map((part) => ({
|
|
||||||
fileId,
|
|
||||||
partNumber: part.partNumber,
|
|
||||||
telegramFileId: part.telegramFileId,
|
|
||||||
telegramFileUniqueId: part.telegramFileUniqueId,
|
|
||||||
storageChatId,
|
|
||||||
storageMessageId: part.storageMessageId,
|
|
||||||
sizeBytes: part.sizeBytes,
|
|
||||||
storedSizeBytes: part.storedSizeBytes,
|
|
||||||
compressionAlgorithm: part.compressionAlgorithm,
|
|
||||||
etag: part.etag,
|
|
||||||
}));
|
|
||||||
|
|
||||||
await deps.filePartRepo.insert(fileParts);
|
|
||||||
|
|
||||||
return { etag: `"${chunkResult.fileHash}"` };
|
|
||||||
}
|
|
||||||
|
|
||||||
// Single-message upload path
|
|
||||||
const forwardResult = await deps.telegramService.forwardToStorage(
|
|
||||||
body,
|
|
||||||
partFileNamePrefix,
|
|
||||||
'document',
|
|
||||||
);
|
|
||||||
|
|
||||||
const publicId = nanoid();
|
|
||||||
|
|
||||||
await deps.fileRepo.create({
|
|
||||||
publicId,
|
|
||||||
telegramFileId: forwardResult.telegramFileId,
|
|
||||||
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
|
|
||||||
storageChatId,
|
|
||||||
storageMessageId: forwardResult.storageMessageId,
|
|
||||||
fileName: finalFileName,
|
|
||||||
mimeType,
|
|
||||||
sizeBytes: body.byteLength,
|
|
||||||
fileType: 'document',
|
|
||||||
uploaderId: 0,
|
|
||||||
fileHash: hash,
|
|
||||||
archiveTelegramFileId: null,
|
|
||||||
archiveStorageMessageId: null,
|
|
||||||
archiveFileName: null,
|
|
||||||
archiveEntryName: null,
|
|
||||||
archiveMimeType: null,
|
|
||||||
archiveSizeBytes: null,
|
|
||||||
bucketId: bucket.id,
|
|
||||||
s3Key: key,
|
|
||||||
storageBackend: 'telegram',
|
|
||||||
isDeleted: false,
|
|
||||||
multipartUploadId: null,
|
|
||||||
partCount: null,
|
|
||||||
});
|
|
||||||
|
|
||||||
return { etag: `"${hash}"` };
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a use case that copies an S3 object to a new key (PUT with
|
|
||||||
* x-amz-copy-source).
|
|
||||||
*
|
|
||||||
* Creates a new file record referencing the same Telegram-stored data.
|
|
||||||
* Chunked source objects are not supported for copy.
|
|
||||||
*
|
|
||||||
* @param deps - The injected dependencies.
|
|
||||||
* @returns An async function accepting source + destination identifiers and
|
|
||||||
* optional precondition headers, returning the copy result or
|
|
||||||
* `null` when a required bucket or file is not found.
|
|
||||||
*/
|
|
||||||
export function createCopyObjectUseCase(deps: S3ObjectDeps) {
|
|
||||||
return async (input: {
|
|
||||||
/** Source bucket name. */
|
|
||||||
sourceBucket: string;
|
|
||||||
/** Source object key. */
|
|
||||||
sourceKey: string;
|
|
||||||
/** Destination bucket UUID (must already exist). */
|
|
||||||
destBucketId: string;
|
|
||||||
/** Destination object key. */
|
|
||||||
destKey: string;
|
|
||||||
/** Optional if-match precondition (raw etag value, without surrounding quotes). */
|
|
||||||
ifMatch?: string | null;
|
|
||||||
/** Optional if-none-match precondition (raw etag value, without surrounding quotes). */
|
|
||||||
ifNoneMatch?: string | null;
|
|
||||||
}): Promise<CopyObjectResult | null> => {
|
|
||||||
const sourceBucket = await deps.bucketRepo.findByName(input.sourceBucket);
|
|
||||||
if (!sourceBucket) return null;
|
|
||||||
|
|
||||||
const sourceFile = await deps.fileRepo.findByBucketAndKey(sourceBucket.id, input.sourceKey);
|
|
||||||
if (!sourceFile) return null;
|
|
||||||
|
|
||||||
if (sourceFile.storageBackend === 'chunked') {
|
|
||||||
throw new ObjectError(
|
|
||||||
'NotImplemented',
|
|
||||||
'Copying chunked objects is not yet implemented.',
|
|
||||||
501,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Conditional copy: if-match / if-none-match checks
|
|
||||||
const sourceEtag = sourceFile.fileHash;
|
|
||||||
if (input.ifMatch && sourceEtag && input.ifMatch !== sourceEtag) {
|
|
||||||
throw new ObjectError(
|
|
||||||
'PreconditionFailed',
|
|
||||||
'The preconditions you specified did not hold.',
|
|
||||||
412,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
if (input.ifNoneMatch && sourceEtag && input.ifNoneMatch === sourceEtag) {
|
|
||||||
throw new ObjectError(
|
|
||||||
'PreconditionFailed',
|
|
||||||
'The preconditions you specified did not hold.',
|
|
||||||
412,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const publicId = nanoid();
|
|
||||||
|
|
||||||
await deps.fileRepo.create({
|
|
||||||
publicId,
|
|
||||||
telegramFileId: sourceFile.telegramFileId,
|
|
||||||
telegramFileUniqueId: sourceFile.telegramFileUniqueId,
|
|
||||||
storageChatId: sourceFile.storageChatId,
|
|
||||||
storageMessageId: sourceFile.storageMessageId,
|
|
||||||
fileName: sourceFile.fileName,
|
|
||||||
mimeType: sourceFile.mimeType,
|
|
||||||
sizeBytes: sourceFile.sizeBytes,
|
|
||||||
fileType: sourceFile.fileType,
|
|
||||||
uploaderId: 0,
|
|
||||||
fileHash: sourceFile.fileHash,
|
|
||||||
archiveTelegramFileId: sourceFile.archiveTelegramFileId,
|
|
||||||
archiveStorageMessageId: sourceFile.archiveStorageMessageId,
|
|
||||||
archiveFileName: sourceFile.archiveFileName,
|
|
||||||
archiveEntryName: sourceFile.archiveEntryName,
|
|
||||||
archiveMimeType: sourceFile.archiveMimeType,
|
|
||||||
archiveSizeBytes: sourceFile.archiveSizeBytes,
|
|
||||||
bucketId: input.destBucketId,
|
|
||||||
s3Key: input.destKey,
|
|
||||||
storageBackend: 'telegram',
|
|
||||||
isDeleted: false,
|
|
||||||
multipartUploadId: null,
|
|
||||||
partCount: null,
|
|
||||||
});
|
|
||||||
|
|
||||||
return {
|
|
||||||
etag: sourceEtag || nanoid(16),
|
|
||||||
lastModified: new Date().toISOString(),
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Error type for S3 object-level application errors.
|
|
||||||
*/
|
|
||||||
export class ObjectError extends Error {
|
|
||||||
/** S3-compatible error code. */
|
|
||||||
readonly code: string;
|
|
||||||
/** Suggested HTTP status code. */
|
|
||||||
readonly status: number;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param code - The S3 error code.
|
|
||||||
* @param message - Human-readable error description.
|
|
||||||
* @param status - Suggested HTTP status.
|
|
||||||
*/
|
|
||||||
constructor(code: string, message: string, status: number) {
|
|
||||||
super(message);
|
|
||||||
this.name = 'ObjectError';
|
|
||||||
this.code = code;
|
|
||||||
this.status = status;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a use case that soft-deletes an S3 object (DELETE).
|
|
||||||
*
|
|
||||||
* @param deps - The injected dependencies.
|
|
||||||
* @returns An async function accepting bucket name and object key, returning
|
|
||||||
* `true` if a row was soft-deleted. Returns `null` when the bucket
|
|
||||||
* is not found.
|
|
||||||
*/
|
|
||||||
export function createDeleteObjectUseCase(deps: S3ObjectDeps) {
|
|
||||||
return async (bucketName: string, key: string): Promise<boolean | null> => {
|
|
||||||
const bucket = await deps.bucketRepo.findByName(bucketName);
|
|
||||||
if (!bucket) return null;
|
|
||||||
return deps.fileRepo.softDelete(bucket.id, key);
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a use case that batch-deletes multiple S3 objects (POST with
|
|
||||||
* ?delete).
|
|
||||||
*
|
|
||||||
* @param deps - The injected dependencies.
|
|
||||||
* @returns An async function accepting bucket name and an array of keys,
|
|
||||||
* returning the array of keys that were actually deleted. Returns
|
|
||||||
* `null` when the bucket is not found.
|
|
||||||
*/
|
|
||||||
export function createDeleteObjectsUseCase(deps: S3ObjectDeps) {
|
|
||||||
return async (bucketName: string, keys: string[]): Promise<string[] | null> => {
|
|
||||||
const bucket = await deps.bucketRepo.findByName(bucketName);
|
|
||||||
if (!bucket) return null;
|
|
||||||
|
|
||||||
const deletedKeys: string[] = [];
|
|
||||||
for (const key of keys) {
|
|
||||||
const ok = await deps.fileRepo.softDelete(bucket.id, key);
|
|
||||||
if (ok) deletedKeys.push(key);
|
|
||||||
}
|
|
||||||
return deletedKeys;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a use case that lists objects within a bucket (ListObjectsV1/V2).
|
|
||||||
*
|
|
||||||
* Supports prefix filtering, delimiter-based pseudo-directory grouping, and
|
|
||||||
* pagination via marker/startAfter.
|
|
||||||
*
|
|
||||||
* @param deps - The injected dependencies.
|
|
||||||
* @returns An async function accepting query parameters and returning the
|
|
||||||
* listing result, or `null` when the bucket is not found.
|
|
||||||
*/
|
|
||||||
export function createListObjectsUseCase(deps: S3ObjectDeps) {
|
|
||||||
return async (input: {
|
|
||||||
/** Bucket name to list from. */
|
|
||||||
bucketName: string;
|
|
||||||
/** Key prefix to filter by (empty string for no filter). */
|
|
||||||
prefix: string;
|
|
||||||
/** Delimiter character (e.g. "/") or null for flat listing. */
|
|
||||||
delimiter: string | null;
|
|
||||||
/** Maximum number of object records to return (clamped to 1000). */
|
|
||||||
maxKeys: number;
|
|
||||||
/** Return only keys strictly greater than this value, or null. */
|
|
||||||
startAfter: string | null;
|
|
||||||
}): Promise<ListObjectsResult | null> => {
|
|
||||||
const bucket = await deps.bucketRepo.findByName(input.bucketName);
|
|
||||||
if (!bucket) return null;
|
|
||||||
|
|
||||||
const clampedMaxKeys = Math.min(input.maxKeys, 1000);
|
|
||||||
|
|
||||||
const { objects, prefixes } = await deps.fileRepo.listByPrefix(
|
|
||||||
bucket.id,
|
|
||||||
input.prefix,
|
|
||||||
input.delimiter,
|
|
||||||
clampedMaxKeys,
|
|
||||||
input.startAfter,
|
|
||||||
);
|
|
||||||
|
|
||||||
const isTruncated = objects.length > clampedMaxKeys;
|
|
||||||
const displayObjects = objects.slice(0, clampedMaxKeys);
|
|
||||||
const nextMarker = isTruncated
|
|
||||||
? (displayObjects[displayObjects.length - 1]?.s3Key ?? null)
|
|
||||||
: null;
|
|
||||||
|
|
||||||
return {
|
|
||||||
objects: displayObjects.map((o: S3FileRecord) => ({
|
|
||||||
key: o.s3Key,
|
|
||||||
sizeBytes: o.sizeBytes,
|
|
||||||
etag: o.fileHash || nanoid(16),
|
|
||||||
lastModified: formatCreatedAt(o.createdAt),
|
|
||||||
mimeType: o.mimeType,
|
|
||||||
})),
|
|
||||||
prefixes,
|
|
||||||
isTruncated,
|
|
||||||
nextMarker,
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a use case that checks whether an object exists and is accessible
|
|
||||||
* within a bucket.
|
|
||||||
*
|
|
||||||
* @param deps - The injected dependencies.
|
|
||||||
* @returns An async function accepting a bucket ID and object key,
|
|
||||||
* returning the file entity or null.
|
|
||||||
*/
|
|
||||||
export function createFindObjectUseCase(deps: Pick<S3ObjectDeps, 'fileRepo'>) {
|
|
||||||
return async (bucketId: string, key: string): Promise<File | null> => {
|
|
||||||
return deps.fileRepo.findByBucketAndKey(bucketId, key);
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,48 +1,13 @@
|
|||||||
import { randomUUID } from 'node:crypto';
|
|
||||||
import { createReadStream } from 'node:fs';
|
import { createReadStream } from 'node:fs';
|
||||||
import { open } from 'node:fs/promises';
|
import { open } from 'node:fs/promises';
|
||||||
import { gzipSync } from 'node:zlib';
|
|
||||||
import { nanoid } from 'nanoid';
|
import { nanoid } from 'nanoid';
|
||||||
import type { NewFilePart } from '../../domain/entities/file-part';
|
import { buildNewFile } from '../../domain/entities/file-factory';
|
||||||
import type { IFilePartRepository } from '../../domain/ports/file-part-repository';
|
|
||||||
import type { IFileRepository } from '../../domain/ports/file-repository';
|
import type { IFileRepository } from '../../domain/ports/file-repository';
|
||||||
import type { ITelegramService } from '../../domain/ports/telegram-service';
|
import type { ITelegramService } from '../../domain/ports/telegram-service';
|
||||||
import { checkFileSize, computeHash, ensureExtension, getFileType } from '../../shared/utils/file';
|
import type { ChunkedStorage } from '../../infrastructure/telegram/chunked-storage';
|
||||||
|
import { checkFileSize, ensureExtension, getFileType } from '../../shared/utils/file';
|
||||||
import type { UploadInput, UploadOutput } from '../dto/upload';
|
import type { UploadInput, UploadOutput } from '../dto/upload';
|
||||||
|
|
||||||
/** Compression algorithm string literal used in chunked storage. */
|
|
||||||
type ChunkCompressionAlgorithm = 'gzip' | null;
|
|
||||||
|
|
||||||
/** Metadata for a single uploaded chunk/part. */
|
|
||||||
interface UploadedPart {
|
|
||||||
/** 1-based part number. */
|
|
||||||
partNumber: number;
|
|
||||||
/** Telegram file identifier for this part. */
|
|
||||||
telegramFileId: string;
|
|
||||||
/** Telegram unique file identifier (stable across bot tokens). */
|
|
||||||
telegramFileUniqueId: string;
|
|
||||||
/** Message ID within the storage chat. */
|
|
||||||
storageMessageId: number;
|
|
||||||
/** Original size of the chunk in bytes before compression. */
|
|
||||||
sizeBytes: number;
|
|
||||||
/** Stored (post-compression) size in bytes. */
|
|
||||||
storedSizeBytes: number;
|
|
||||||
/** Compression algorithm applied, or null if uncompressed. */
|
|
||||||
compressionAlgorithm: ChunkCompressionAlgorithm;
|
|
||||||
/** SHA-256 hash of the original chunk content. */
|
|
||||||
etag: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Result of uploading a file in multiple Telegram chunks. */
|
|
||||||
interface ChunkedUploadResult {
|
|
||||||
/** Metadata for each uploaded part. */
|
|
||||||
parts: UploadedPart[];
|
|
||||||
/** SHA-256 hex digest of the complete file content. */
|
|
||||||
fileHash: string;
|
|
||||||
/** Total file size in bytes (sum of all original chunks). */
|
|
||||||
totalSizeBytes: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Subset of application configuration consumed by the upload-file use case. */
|
/** Subset of application configuration consumed by the upload-file use case. */
|
||||||
export interface UploadFileConfig {
|
export interface UploadFileConfig {
|
||||||
/** Server base URL for constructing download links. */
|
/** Server base URL for constructing download links. */
|
||||||
@@ -61,53 +26,14 @@ export interface UploadFileConfig {
|
|||||||
export interface UploadFileUseCaseDeps {
|
export interface UploadFileUseCaseDeps {
|
||||||
/** File repository for CRUD operations on file records. */
|
/** File repository for CRUD operations on file records. */
|
||||||
fileRepo: IFileRepository;
|
fileRepo: IFileRepository;
|
||||||
/** File-part repository for chunked file metadata. */
|
|
||||||
filePartRepo: IFilePartRepository;
|
|
||||||
/** Telegram service for forwarding file content to storage. */
|
/** Telegram service for forwarding file content to storage. */
|
||||||
telegramService: ITelegramService;
|
telegramService: ITelegramService;
|
||||||
|
/** Chunked storage handler for large file uploads. */
|
||||||
|
chunkedStorage: ChunkedStorage;
|
||||||
/** Application configuration subset. */
|
/** Application configuration subset. */
|
||||||
config: UploadFileConfig;
|
config: UploadFileConfig;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Validates the configured chunk size and returns it as a safe integer.
|
|
||||||
*
|
|
||||||
* @param chunkSizeBytes - The configured chunk size in bytes.
|
|
||||||
* @returns The same value if it is a positive safe integer.
|
|
||||||
*/
|
|
||||||
const asSafeChunkSize = (chunkSizeBytes: number): number => {
|
|
||||||
if (!Number.isSafeInteger(chunkSizeBytes) || chunkSizeBytes <= 0) {
|
|
||||||
throw new Error('Invalid Telegram chunk size');
|
|
||||||
}
|
|
||||||
return chunkSizeBytes;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Optionally gzip-compresses a chunk if compression is enabled and the chunk
|
|
||||||
* is large enough to benefit from it.
|
|
||||||
*
|
|
||||||
* @param chunk - The raw chunk buffer.
|
|
||||||
* @param compress - Whether compression is enabled.
|
|
||||||
* @param compressionMinSizeBytes - Minimum chunk size to attempt compression.
|
|
||||||
* @returns The (possibly compressed) buffer and the algorithm used.
|
|
||||||
*/
|
|
||||||
const maybeCompressChunk = (
|
|
||||||
chunk: Buffer,
|
|
||||||
compress: boolean,
|
|
||||||
compressionMinSizeBytes: number,
|
|
||||||
): { bytes: Buffer; compressionAlgorithm: ChunkCompressionAlgorithm } => {
|
|
||||||
if (!compress || chunk.byteLength < compressionMinSizeBytes) {
|
|
||||||
return { bytes: chunk, compressionAlgorithm: null };
|
|
||||||
}
|
|
||||||
|
|
||||||
const gzipped = gzipSync(chunk);
|
|
||||||
if (gzipped.byteLength >= chunk.byteLength) {
|
|
||||||
return { bytes: chunk, compressionAlgorithm: null };
|
|
||||||
}
|
|
||||||
|
|
||||||
return { bytes: gzipped, compressionAlgorithm: 'gzip' };
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Reads the first 16 bytes from a file on disk for magic-byte detection.
|
* Reads the first 16 bytes from a file on disk for magic-byte detection.
|
||||||
*
|
*
|
||||||
@@ -125,74 +51,6 @@ const readSignatureBuffer = async (tempPath: string): Promise<Buffer> => {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
|
||||||
* Reads a file from disk in chunks, forwards each chunk to Telegram storage,
|
|
||||||
* and returns metadata for all uploaded parts together with the total file
|
|
||||||
* hash.
|
|
||||||
*
|
|
||||||
* @param tempPath - Absolute path to the temporary file on disk.
|
|
||||||
* @param partFileNamePrefix - Prefix used for each chunk's file name in Telegram.
|
|
||||||
* @param chunkSizeBytes - Maximum size of each chunk in bytes.
|
|
||||||
* @param compress - Whether gzip compression is enabled.
|
|
||||||
* @param compressionMinSizeBytes - Minimum chunk size to attempt compression.
|
|
||||||
* @param telegramService - The Telegram service to forward each chunk.
|
|
||||||
* @returns The aggregated chunked upload result.
|
|
||||||
*/
|
|
||||||
const uploadFileInTelegramChunks = async (
|
|
||||||
tempPath: string,
|
|
||||||
partFileNamePrefix: string,
|
|
||||||
chunkSizeBytes: number,
|
|
||||||
compress: boolean,
|
|
||||||
compressionMinSizeBytes: number,
|
|
||||||
telegramService: ITelegramService,
|
|
||||||
): Promise<ChunkedUploadResult> => {
|
|
||||||
const safeChunkSize = asSafeChunkSize(chunkSizeBytes);
|
|
||||||
const hasher = new Bun.CryptoHasher('sha256');
|
|
||||||
const parts: UploadedPart[] = [];
|
|
||||||
let totalSizeBytes = 0;
|
|
||||||
let partNumber = 0;
|
|
||||||
|
|
||||||
const stream = createReadStream(tempPath, { highWaterMark: safeChunkSize });
|
|
||||||
|
|
||||||
for await (const data of stream) {
|
|
||||||
const chunk = Buffer.isBuffer(data) ? data : Buffer.from(data as Uint8Array);
|
|
||||||
if (chunk.byteLength === 0) continue;
|
|
||||||
|
|
||||||
partNumber += 1;
|
|
||||||
totalSizeBytes += chunk.byteLength;
|
|
||||||
hasher.update(chunk);
|
|
||||||
|
|
||||||
const { bytes, compressionAlgorithm } = maybeCompressChunk(
|
|
||||||
chunk,
|
|
||||||
compress,
|
|
||||||
compressionMinSizeBytes,
|
|
||||||
);
|
|
||||||
|
|
||||||
const forwardResult = await telegramService.forwardToStorage(
|
|
||||||
bytes,
|
|
||||||
`${partFileNamePrefix}.part-${partNumber}`,
|
|
||||||
'document',
|
|
||||||
);
|
|
||||||
|
|
||||||
parts.push({
|
|
||||||
partNumber,
|
|
||||||
telegramFileId: forwardResult.telegramFileId,
|
|
||||||
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
|
|
||||||
storageMessageId: forwardResult.storageMessageId,
|
|
||||||
sizeBytes: chunk.byteLength,
|
|
||||||
storedSizeBytes: bytes.byteLength,
|
|
||||||
compressionAlgorithm,
|
|
||||||
etag: computeHash(chunk),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
parts,
|
|
||||||
fileHash: hasher.digest('hex'),
|
|
||||||
totalSizeBytes,
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates a factory function for the upload-file use case.
|
* Creates a factory function for the upload-file use case.
|
||||||
*
|
*
|
||||||
@@ -200,9 +58,9 @@ const uploadFileInTelegramChunks = async (
|
|||||||
* 1. Checks for an existing file with the same SHA-256 hash (deduplication).
|
* 1. Checks for an existing file with the same SHA-256 hash (deduplication).
|
||||||
* 2. Normalises the file name and MIME type based on magic bytes.
|
* 2. Normalises the file name and MIME type based on magic bytes.
|
||||||
* 3. Validates the file size against Telegram type-specific limits.
|
* 3. Validates the file size against Telegram type-specific limits.
|
||||||
* 4. Chooses a storage strategy — chunked (for files exceeding the chunk
|
* 4. Chooses a storage strategy — chunked (delegated to ChunkedStorage) or
|
||||||
* threshold) or single-message upload.
|
* single-message upload.
|
||||||
* 5. Persists the file record (and, for chunked uploads, part records).
|
* 5. Persists the file record.
|
||||||
* 6. Builds and returns the public `UploadOutput` DTO.
|
* 6. Builds and returns the public `UploadOutput` DTO.
|
||||||
*
|
*
|
||||||
* @param deps - The injected dependencies.
|
* @param deps - The injected dependencies.
|
||||||
@@ -241,75 +99,28 @@ export function createUploadFileUseCase(deps: UploadFileUseCaseDeps) {
|
|||||||
throw new Error(`File size exceeds ${fileType} limit`);
|
throw new Error(`File size exceeds ${fileType} limit`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 4. Upload — chunked for files above the threshold, single otherwise
|
// 4. Upload — chunked via ChunkedStorage for files above the threshold
|
||||||
if (input.sizeBytes > deps.config.telegramChunkSizeBytes) {
|
if (input.sizeBytes > deps.config.telegramChunkSizeBytes) {
|
||||||
// Chunked upload path
|
const uploadedFile = await deps.chunkedStorage.storeFileInTelegramChunks({
|
||||||
const chunkResult = await uploadFileInTelegramChunks(
|
tempPath: input.tempPath,
|
||||||
input.tempPath,
|
partFileNamePrefix: `direct-${input.fileHash.slice(0, 16)}`,
|
||||||
`direct-${input.fileHash.slice(0, 16)}`,
|
|
||||||
deps.config.telegramChunkSizeBytes,
|
|
||||||
deps.config.compressChunkedUploads,
|
|
||||||
deps.config.chunkCompressionMinSizeBytes,
|
|
||||||
deps.telegramService,
|
|
||||||
);
|
|
||||||
|
|
||||||
const firstPart = chunkResult.parts[0];
|
|
||||||
if (!firstPart) {
|
|
||||||
throw new Error('Chunked upload produced no parts');
|
|
||||||
}
|
|
||||||
|
|
||||||
const fileId = randomUUID();
|
|
||||||
const publicId = nanoid();
|
|
||||||
|
|
||||||
const newFile = await deps.fileRepo.create({
|
|
||||||
publicId,
|
|
||||||
telegramFileId: firstPart.telegramFileId,
|
|
||||||
telegramFileUniqueId: firstPart.telegramFileUniqueId,
|
|
||||||
storageChatId: deps.config.storageChatId,
|
|
||||||
storageMessageId: firstPart.storageMessageId,
|
|
||||||
fileName: finalFileName,
|
fileName: finalFileName,
|
||||||
mimeType,
|
mimeType,
|
||||||
sizeBytes: chunkResult.totalSizeBytes,
|
sizeBytes: input.sizeBytes,
|
||||||
fileType,
|
fileType,
|
||||||
uploaderId: input.uploaderId ?? 0,
|
uploaderId: input.uploaderId ?? 0,
|
||||||
fileHash: chunkResult.fileHash,
|
bucketId: input.bucketId,
|
||||||
archiveTelegramFileId: null,
|
s3Key: input.s3Key,
|
||||||
archiveStorageMessageId: null,
|
|
||||||
archiveFileName: null,
|
|
||||||
archiveEntryName: null,
|
|
||||||
archiveMimeType: null,
|
|
||||||
archiveSizeBytes: null,
|
|
||||||
bucketId: input.bucketId ?? null,
|
|
||||||
s3Key: input.s3Key ?? null,
|
|
||||||
storageBackend: 'chunked',
|
|
||||||
isDeleted: false,
|
|
||||||
multipartUploadId: null,
|
|
||||||
partCount: chunkResult.parts.length,
|
|
||||||
});
|
});
|
||||||
|
|
||||||
const fileParts: NewFilePart[] = chunkResult.parts.map((part) => ({
|
|
||||||
fileId,
|
|
||||||
partNumber: part.partNumber,
|
|
||||||
telegramFileId: part.telegramFileId,
|
|
||||||
telegramFileUniqueId: part.telegramFileUniqueId,
|
|
||||||
storageChatId: deps.config.storageChatId,
|
|
||||||
storageMessageId: part.storageMessageId,
|
|
||||||
sizeBytes: part.sizeBytes,
|
|
||||||
storedSizeBytes: part.storedSizeBytes,
|
|
||||||
compressionAlgorithm: part.compressionAlgorithm,
|
|
||||||
etag: part.etag,
|
|
||||||
}));
|
|
||||||
|
|
||||||
await deps.filePartRepo.insert(fileParts);
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
publicId: newFile.publicId,
|
publicId: uploadedFile.publicId,
|
||||||
fileName: newFile.fileName,
|
fileName: uploadedFile.fileName,
|
||||||
mimeType: newFile.mimeType,
|
mimeType: uploadedFile.mimeType,
|
||||||
sizeBytes: newFile.sizeBytes,
|
sizeBytes: uploadedFile.sizeBytes,
|
||||||
fileType: newFile.fileType,
|
fileType: uploadedFile.fileType,
|
||||||
createdAt: newFile.createdAt,
|
createdAt: uploadedFile.createdAt,
|
||||||
downloadUrl: `${deps.config.baseUrl}/f/${newFile.publicId}`,
|
downloadUrl: `${deps.config.baseUrl}/f/${uploadedFile.publicId}`,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -322,31 +133,24 @@ export function createUploadFileUseCase(deps: UploadFileUseCaseDeps) {
|
|||||||
|
|
||||||
const singlePublicId = nanoid();
|
const singlePublicId = nanoid();
|
||||||
|
|
||||||
const createdFile = await deps.fileRepo.create({
|
const createdFile = await deps.fileRepo.create(
|
||||||
publicId: singlePublicId,
|
buildNewFile({
|
||||||
telegramFileId: forwardResult.telegramFileId,
|
publicId: singlePublicId,
|
||||||
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
|
telegramFileId: forwardResult.telegramFileId,
|
||||||
storageChatId: deps.config.storageChatId,
|
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
|
||||||
storageMessageId: forwardResult.storageMessageId,
|
storageChatId: deps.config.storageChatId,
|
||||||
fileName: finalFileName,
|
storageMessageId: forwardResult.storageMessageId,
|
||||||
mimeType,
|
fileName: finalFileName,
|
||||||
sizeBytes: input.sizeBytes,
|
mimeType,
|
||||||
fileType,
|
sizeBytes: input.sizeBytes,
|
||||||
uploaderId: input.uploaderId ?? 0,
|
fileType,
|
||||||
fileHash: input.fileHash,
|
storageBackend: 'telegram',
|
||||||
archiveTelegramFileId: null,
|
uploaderId: input.uploaderId,
|
||||||
archiveStorageMessageId: null,
|
fileHash: input.fileHash,
|
||||||
archiveFileName: null,
|
bucketId: input.bucketId,
|
||||||
archiveEntryName: null,
|
s3Key: input.s3Key,
|
||||||
archiveMimeType: null,
|
}),
|
||||||
archiveSizeBytes: null,
|
);
|
||||||
bucketId: input.bucketId ?? null,
|
|
||||||
s3Key: input.s3Key ?? null,
|
|
||||||
storageBackend: 'telegram',
|
|
||||||
isDeleted: false,
|
|
||||||
multipartUploadId: null,
|
|
||||||
partCount: null,
|
|
||||||
});
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
publicId: createdFile.publicId,
|
publicId: createdFile.publicId,
|
||||||
|
|||||||
@@ -1,3 +0,0 @@
|
|||||||
import { config } from '../env';
|
|
||||||
|
|
||||||
export { config };
|
|
||||||
@@ -1,81 +0,0 @@
|
|||||||
import { sql } from 'drizzle-orm';
|
|
||||||
import { db } from './index';
|
|
||||||
|
|
||||||
export interface Bucket {
|
|
||||||
id: string;
|
|
||||||
name: string;
|
|
||||||
createdAt: Date;
|
|
||||||
updatedAt: Date;
|
|
||||||
}
|
|
||||||
|
|
||||||
type QueryRow = Record<string, unknown>;
|
|
||||||
type QueryResult = QueryRow[];
|
|
||||||
|
|
||||||
export const createBucket = async (name: string): Promise<Bucket> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`INSERT INTO buckets (name) VALUES (${name}) RETURNING id, name, created_at, updated_at`,
|
|
||||||
)) as unknown as QueryResult;
|
|
||||||
const row = result[0]!;
|
|
||||||
return {
|
|
||||||
id: row.id as string,
|
|
||||||
name: row.name as string,
|
|
||||||
createdAt: new Date(row.created_at as string),
|
|
||||||
updatedAt: new Date(row.updated_at as string),
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export const findBucketByName = async (name: string): Promise<Bucket | null> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`SELECT id, name, created_at, updated_at FROM buckets WHERE name = ${name}`,
|
|
||||||
)) as unknown as QueryResult;
|
|
||||||
if (result.length === 0) return null;
|
|
||||||
const row = result[0]!;
|
|
||||||
return {
|
|
||||||
id: row.id as string,
|
|
||||||
name: row.name as string,
|
|
||||||
createdAt: new Date(row.created_at as string),
|
|
||||||
updatedAt: new Date(row.updated_at as string),
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export const listBuckets = async (): Promise<Bucket[]> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`SELECT id, name, created_at, updated_at FROM buckets ORDER BY name`,
|
|
||||||
)) as unknown as QueryResult;
|
|
||||||
return result.map((row) => ({
|
|
||||||
id: row.id as string,
|
|
||||||
name: row.name as string,
|
|
||||||
createdAt: new Date(row.created_at as string),
|
|
||||||
updatedAt: new Date(row.updated_at as string),
|
|
||||||
}));
|
|
||||||
};
|
|
||||||
|
|
||||||
export const deleteBucket = async (name: string): Promise<boolean> => {
|
|
||||||
// Cascade-delete rows that hold FK references to the bucket
|
|
||||||
await db
|
|
||||||
.execute(
|
|
||||||
sql`DELETE FROM multipart_parts WHERE upload_id IN (SELECT upload_id FROM multipart_uploads WHERE bucket_id IN (SELECT id FROM buckets WHERE name = ${name}))`,
|
|
||||||
)
|
|
||||||
.catch(() => {});
|
|
||||||
await db
|
|
||||||
.execute(
|
|
||||||
sql`DELETE FROM multipart_uploads WHERE bucket_id IN (SELECT id FROM buckets WHERE name = ${name})`,
|
|
||||||
)
|
|
||||||
.catch(() => {});
|
|
||||||
await db
|
|
||||||
.execute(
|
|
||||||
sql`DELETE FROM files WHERE bucket_id IN (SELECT id FROM buckets WHERE name = ${name})`,
|
|
||||||
)
|
|
||||||
.catch(() => {});
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`DELETE FROM buckets WHERE name = ${name}`,
|
|
||||||
)) as unknown as QueryResult;
|
|
||||||
return result.length > 0;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const bucketExists = async (name: string): Promise<boolean> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`SELECT 1 FROM buckets WHERE name = ${name}`,
|
|
||||||
)) as unknown as QueryResult;
|
|
||||||
return result.length > 0;
|
|
||||||
};
|
|
||||||
@@ -1,97 +0,0 @@
|
|||||||
import { sql } from 'drizzle-orm';
|
|
||||||
import { db } from './index';
|
|
||||||
|
|
||||||
export type CompressionAlgorithm = 'gzip' | null;
|
|
||||||
|
|
||||||
export interface FilePart {
|
|
||||||
id: number;
|
|
||||||
fileId: string;
|
|
||||||
partNumber: number;
|
|
||||||
telegramFileId: string;
|
|
||||||
telegramFileUniqueId: string;
|
|
||||||
storageChatId: number;
|
|
||||||
storageMessageId: number;
|
|
||||||
sizeBytes: number;
|
|
||||||
storedSizeBytes: number;
|
|
||||||
compressionAlgorithm: CompressionAlgorithm;
|
|
||||||
etag: string;
|
|
||||||
createdAt: Date;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type NewFilePartInput = Omit<FilePart, 'id' | 'createdAt'>;
|
|
||||||
|
|
||||||
const toNumber = (value: unknown): number => Number(value ?? 0);
|
|
||||||
|
|
||||||
const mapRowToFilePart = (row: Record<string, unknown>): FilePart => ({
|
|
||||||
id: toNumber(row.id),
|
|
||||||
fileId: row.file_id as string,
|
|
||||||
partNumber: toNumber(row.part_number),
|
|
||||||
telegramFileId: row.telegram_file_id as string,
|
|
||||||
telegramFileUniqueId: row.telegram_file_unique_id as string,
|
|
||||||
storageChatId: toNumber(row.storage_chat_id),
|
|
||||||
storageMessageId: toNumber(row.storage_message_id),
|
|
||||||
sizeBytes: toNumber(row.size_bytes),
|
|
||||||
storedSizeBytes: toNumber(row.stored_size_bytes),
|
|
||||||
compressionAlgorithm: (row.compression_algorithm as CompressionAlgorithm) || null,
|
|
||||||
etag: row.etag as string,
|
|
||||||
createdAt: new Date(row.created_at as string),
|
|
||||||
});
|
|
||||||
|
|
||||||
export const insertFileParts = async (parts: NewFilePartInput[]): Promise<void> => {
|
|
||||||
for (const part of parts) {
|
|
||||||
await db.execute(
|
|
||||||
sql`INSERT INTO file_parts (
|
|
||||||
file_id,
|
|
||||||
part_number,
|
|
||||||
telegram_file_id,
|
|
||||||
telegram_file_unique_id,
|
|
||||||
storage_chat_id,
|
|
||||||
storage_message_id,
|
|
||||||
size_bytes,
|
|
||||||
stored_size_bytes,
|
|
||||||
compression_algorithm,
|
|
||||||
etag
|
|
||||||
) VALUES (
|
|
||||||
${part.fileId}::uuid,
|
|
||||||
${part.partNumber},
|
|
||||||
${part.telegramFileId},
|
|
||||||
${part.telegramFileUniqueId},
|
|
||||||
${part.storageChatId},
|
|
||||||
${part.storageMessageId},
|
|
||||||
${part.sizeBytes},
|
|
||||||
${part.storedSizeBytes},
|
|
||||||
${part.compressionAlgorithm},
|
|
||||||
${part.etag}
|
|
||||||
)`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const listFileParts = async (fileId: string): Promise<FilePart[]> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`SELECT id,
|
|
||||||
file_id,
|
|
||||||
part_number,
|
|
||||||
telegram_file_id,
|
|
||||||
telegram_file_unique_id,
|
|
||||||
storage_chat_id,
|
|
||||||
storage_message_id,
|
|
||||||
size_bytes,
|
|
||||||
stored_size_bytes,
|
|
||||||
compression_algorithm,
|
|
||||||
etag,
|
|
||||||
created_at
|
|
||||||
FROM file_parts
|
|
||||||
WHERE file_id = ${fileId}::uuid
|
|
||||||
ORDER BY part_number`,
|
|
||||||
)) as unknown as Record<string, unknown>[];
|
|
||||||
|
|
||||||
return result.map(mapRowToFilePart);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const countFileParts = async (fileId: string): Promise<number> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`SELECT COUNT(*) AS count FROM file_parts WHERE file_id = ${fileId}::uuid`,
|
|
||||||
)) as unknown as Record<string, unknown>[];
|
|
||||||
return toNumber(result[0]?.count);
|
|
||||||
};
|
|
||||||
@@ -1,143 +0,0 @@
|
|||||||
import { and, eq, sql } from 'drizzle-orm';
|
|
||||||
import { db, files as fileSchema } from './index';
|
|
||||||
import type { File } from './schema';
|
|
||||||
|
|
||||||
export interface S3FileRecord extends File {
|
|
||||||
bucketId: string;
|
|
||||||
s3Key: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const findFileByBucketAndKey = async (
|
|
||||||
bucketId: string,
|
|
||||||
s3Key: string,
|
|
||||||
): Promise<File | null> => {
|
|
||||||
const result = await db
|
|
||||||
.select()
|
|
||||||
.from(fileSchema)
|
|
||||||
.where(
|
|
||||||
and(
|
|
||||||
eq(fileSchema.bucketId, bucketId),
|
|
||||||
eq(fileSchema.s3Key, s3Key),
|
|
||||||
eq(fileSchema.isDeleted, false),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.limit(1);
|
|
||||||
return result[0] || null;
|
|
||||||
};
|
|
||||||
|
|
||||||
const mapDbRowToS3Record = (row: Record<string, unknown>): S3FileRecord => {
|
|
||||||
return {
|
|
||||||
id: row.id as string,
|
|
||||||
publicId: row.public_id as string,
|
|
||||||
telegramFileId: row.telegram_file_id as string,
|
|
||||||
telegramFileUniqueId: row.telegram_file_unique_id as string,
|
|
||||||
storageChatId: toNumber(row.storage_chat_id),
|
|
||||||
storageMessageId: toNumber(row.storage_message_id),
|
|
||||||
fileName: row.file_name as string,
|
|
||||||
mimeType: row.mime_type as string,
|
|
||||||
sizeBytes: toNumber(row.size_bytes),
|
|
||||||
fileType: row.file_type as string,
|
|
||||||
uploaderId: toNumber(row.uploader_id),
|
|
||||||
fileHash: row.file_hash as string | null,
|
|
||||||
archiveTelegramFileId: row.archive_telegram_file_id as string | null,
|
|
||||||
archiveStorageMessageId:
|
|
||||||
row.archive_storage_message_id === null ? null : toNumber(row.archive_storage_message_id),
|
|
||||||
archiveFileName: row.archive_file_name as string | null,
|
|
||||||
archiveEntryName: row.archive_entry_name as string | null,
|
|
||||||
archiveMimeType: row.archive_mime_type as string | null,
|
|
||||||
archiveSizeBytes: row.archive_size_bytes === null ? null : toNumber(row.archive_size_bytes),
|
|
||||||
bucketId: row.bucket_id as string,
|
|
||||||
s3Key: row.s3_key as string,
|
|
||||||
storageBackend: (row.storage_backend as string) || 'telegram',
|
|
||||||
isDeleted: row.is_deleted as boolean,
|
|
||||||
multipartUploadId: row.multipart_upload_id as string | null,
|
|
||||||
partCount:
|
|
||||||
row.part_count === null || row.part_count === undefined ? null : toNumber(row.part_count),
|
|
||||||
createdAt: new Date(row.created_at as string),
|
|
||||||
updatedAt: new Date(row.updated_at as string),
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
const escapeLike = (s: string): string => s.replace(/[%_\\]/g, '\\$&');
|
|
||||||
|
|
||||||
const toNumber = (value: unknown): number => Number(value ?? 0);
|
|
||||||
|
|
||||||
export const listObjectsByPrefix = async (
|
|
||||||
bucketId: string,
|
|
||||||
prefix: string,
|
|
||||||
delimiter: string | null,
|
|
||||||
maxKeys: number,
|
|
||||||
startAfter: string | null,
|
|
||||||
): Promise<{ objects: S3FileRecord[]; prefixes: string[] }> => {
|
|
||||||
let query = prefix
|
|
||||||
? sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false AND s3_key LIKE ${`${escapeLike(prefix)}%`}`
|
|
||||||
: sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false`;
|
|
||||||
|
|
||||||
if (startAfter) {
|
|
||||||
query = sql`${query} AND s3_key > ${startAfter}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
query = sql`${query} ORDER BY s3_key LIMIT ${maxKeys + 1}`;
|
|
||||||
|
|
||||||
const rawResult = (await db.execute(query)) as unknown as Record<string, unknown>[];
|
|
||||||
|
|
||||||
if (delimiter === '/') {
|
|
||||||
const prefixSet = new Set<string>();
|
|
||||||
const objects: S3FileRecord[] = [];
|
|
||||||
|
|
||||||
for (const row of rawResult) {
|
|
||||||
const s3Key = row.s3_key as string;
|
|
||||||
const relativeKey = s3Key.substring(prefix.length);
|
|
||||||
const slashIndex = relativeKey.indexOf('/');
|
|
||||||
if (slashIndex >= 0) {
|
|
||||||
const folderPrefix = prefix + relativeKey.substring(0, slashIndex + 1);
|
|
||||||
if (folderPrefix !== prefix) {
|
|
||||||
prefixSet.add(folderPrefix);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
objects.push(mapDbRowToS3Record(row));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
objects: objects.slice(0, maxKeys),
|
|
||||||
prefixes: Array.from(prefixSet).sort(),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
objects: rawResult.slice(0, maxKeys).map(mapDbRowToS3Record),
|
|
||||||
prefixes: [],
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export const softDeleteFile = async (bucketId: string, s3Key: string): Promise<boolean> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`UPDATE files SET is_deleted = true WHERE bucket_id = ${bucketId}::uuid AND s3_key = ${s3Key} RETURNING id`,
|
|
||||||
)) as unknown as Record<string, unknown>[];
|
|
||||||
return result.length > 0;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const softDeleteFilesBatch = async (bucketId: string, keys: string[]): Promise<number> => {
|
|
||||||
let deleted = 0;
|
|
||||||
for (const key of keys) {
|
|
||||||
const ok = await softDeleteFile(bucketId, key);
|
|
||||||
if (ok) deleted++;
|
|
||||||
}
|
|
||||||
return deleted;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const countBucketObjects = async (bucketId: string): Promise<number> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`SELECT count(*) as count FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false`,
|
|
||||||
)) as unknown as Record<string, unknown>[];
|
|
||||||
return Number(result[0]?.count || 0);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const findOrphanFilesByBucket = async (bucketId: string): Promise<File[]> => {
|
|
||||||
return await db
|
|
||||||
.select()
|
|
||||||
.from(fileSchema)
|
|
||||||
.where(and(eq(fileSchema.bucketId, bucketId), eq(fileSchema.isDeleted, true)))
|
|
||||||
.limit(100);
|
|
||||||
};
|
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
import { eq } from 'drizzle-orm';
|
|
||||||
import { db, files as fileSchema } from './index';
|
|
||||||
import type { File } from './schema';
|
|
||||||
|
|
||||||
export const findFileByHash = async (hash: string): Promise<File | null> => {
|
|
||||||
const result = await db.select().from(fileSchema).where(eq(fileSchema.fileHash, hash)).limit(1);
|
|
||||||
return result[0] || null;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const findFileByPublicId = async (publicId: string): Promise<File | null> => {
|
|
||||||
const result = await db
|
|
||||||
.select()
|
|
||||||
.from(fileSchema)
|
|
||||||
.where(eq(fileSchema.publicId, publicId))
|
|
||||||
.limit(1);
|
|
||||||
return result[0] || null;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const findFileByUniqueId = async (telegramFileUniqueId: string): Promise<File | null> => {
|
|
||||||
const result = await db
|
|
||||||
.select()
|
|
||||||
.from(fileSchema)
|
|
||||||
.where(eq(fileSchema.telegramFileUniqueId, telegramFileUniqueId))
|
|
||||||
.limit(1);
|
|
||||||
return result[0] || null;
|
|
||||||
};
|
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
import { drizzle } from 'drizzle-orm/postgres-js';
|
|
||||||
import postgres from 'postgres';
|
|
||||||
import { fileParts, files } from './schema';
|
|
||||||
|
|
||||||
const client = postgres(process.env.DATABASE_URL!, {
|
|
||||||
max: 10,
|
|
||||||
idle_timeout: 20,
|
|
||||||
connect_timeout: 10,
|
|
||||||
});
|
|
||||||
|
|
||||||
export const db = drizzle(client, { schema: { fileParts, files } });
|
|
||||||
export { fileParts, files };
|
|
||||||
export default db;
|
|
||||||
@@ -1,53 +0,0 @@
|
|||||||
import postgres from 'postgres';
|
|
||||||
import { config } from '../env';
|
|
||||||
import { getErrorMessage } from '../utils/file';
|
|
||||||
import logger from '../utils/logger';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Run raw SQL migration from schema.sql.
|
|
||||||
* Safe to call multiple times — all statements use IF NOT EXISTS.
|
|
||||||
*/
|
|
||||||
export const runMigration = async (): Promise<void> => {
|
|
||||||
// In compiled dist: import.meta.dir = .../dist/
|
|
||||||
// In source via bun --hot: import.meta.dir = .../src/db/
|
|
||||||
const dir = import.meta.dir || '';
|
|
||||||
const candidates = [
|
|
||||||
`${dir}/../../schema.sql`, // from dist/
|
|
||||||
`${dir}/../schema.sql`, // from src/ (bun --hot src/index.ts)
|
|
||||||
`${dir}/../schema.sql`, // from src/db/ (bun --hot src/db/migrate.ts)
|
|
||||||
`${dir}/schema.sql`, // from src/ (bun run db:migrate)
|
|
||||||
];
|
|
||||||
|
|
||||||
let schemaSql: string | null = null;
|
|
||||||
for (const p of candidates) {
|
|
||||||
const file = Bun.file(p);
|
|
||||||
const exists = await file.exists();
|
|
||||||
if (exists) {
|
|
||||||
schemaSql = await file.text();
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!schemaSql) {
|
|
||||||
logger.error(`Migration failed: schema.sql not found (tried ${candidates.join(', ')})`);
|
|
||||||
process.exitCode = 1;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const sql = postgres(config.databaseUrl, { max: 1 });
|
|
||||||
|
|
||||||
try {
|
|
||||||
await sql.unsafe(schemaSql);
|
|
||||||
logger.info('Database migration completed');
|
|
||||||
} catch (error: unknown) {
|
|
||||||
logger.error('Database migration failed', { error: getErrorMessage(error) });
|
|
||||||
process.exitCode = 1;
|
|
||||||
} finally {
|
|
||||||
await sql.end();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// When run directly: `bun src/db/migrate.ts` or `bun dist/migrate.js`
|
|
||||||
if (import.meta.path === Bun.main) {
|
|
||||||
await runMigration();
|
|
||||||
}
|
|
||||||
@@ -1,137 +0,0 @@
|
|||||||
import { sql } from 'drizzle-orm';
|
|
||||||
import { nanoid } from 'nanoid';
|
|
||||||
import { db } from './index';
|
|
||||||
|
|
||||||
export interface MultipartUpload {
|
|
||||||
uploadId: string;
|
|
||||||
bucketId: string;
|
|
||||||
s3Key: string;
|
|
||||||
initiatedAt: Date;
|
|
||||||
status: string;
|
|
||||||
initiatedBy: string;
|
|
||||||
contentType: string | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface MultipartPart {
|
|
||||||
id: number;
|
|
||||||
uploadId: string;
|
|
||||||
partNumber: number;
|
|
||||||
telegramFileId: string;
|
|
||||||
telegramFileUniqueId: string;
|
|
||||||
storageMessageId: number;
|
|
||||||
sizeBytes: number;
|
|
||||||
etag: string;
|
|
||||||
createdAt: Date;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const createMultipartUpload = async (
|
|
||||||
bucketId: string,
|
|
||||||
s3Key: string,
|
|
||||||
initiatedBy: string,
|
|
||||||
contentType?: string | null,
|
|
||||||
): Promise<string> => {
|
|
||||||
const uploadId = nanoid(32);
|
|
||||||
await db.execute(
|
|
||||||
contentType
|
|
||||||
? sql`INSERT INTO multipart_uploads (upload_id, bucket_id, s3_key, initiated_by, content_type) VALUES (${uploadId}, ${bucketId}, ${s3Key}, ${initiatedBy}, ${contentType})`
|
|
||||||
: sql`INSERT INTO multipart_uploads (upload_id, bucket_id, s3_key, initiated_by) VALUES (${uploadId}, ${bucketId}, ${s3Key}, ${initiatedBy})`,
|
|
||||||
);
|
|
||||||
return uploadId;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const findMultipartUpload = async (uploadId: string): Promise<MultipartUpload | null> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, content_type FROM multipart_uploads WHERE upload_id = ${uploadId} AND status = 'in_progress'`,
|
|
||||||
)) as unknown as Record<string, unknown>[];
|
|
||||||
if (result.length === 0) return null;
|
|
||||||
const r = result[0]!;
|
|
||||||
return {
|
|
||||||
uploadId: r.upload_id as string,
|
|
||||||
bucketId: r.bucket_id as string,
|
|
||||||
s3Key: r.s3_key as string,
|
|
||||||
initiatedAt: new Date(r.initiated_at as string),
|
|
||||||
status: r.status as string,
|
|
||||||
initiatedBy: '',
|
|
||||||
contentType: (r.content_type as string | null) || null,
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export const completeMultipartUpload = async (uploadId: string): Promise<void> => {
|
|
||||||
await db.execute(
|
|
||||||
sql`UPDATE multipart_uploads SET status = 'completed' WHERE upload_id = ${uploadId}`,
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const abortMultipartUpload = async (uploadId: string): Promise<void> => {
|
|
||||||
// H7: FK cascade only fires on DELETE, not UPDATE. Delete parts explicitly
|
|
||||||
// before updating the upload status.
|
|
||||||
await db.execute(sql`DELETE FROM multipart_parts WHERE upload_id = ${uploadId}`);
|
|
||||||
await db.execute(
|
|
||||||
sql`UPDATE multipart_uploads SET status = 'aborted' WHERE upload_id = ${uploadId}`,
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const insertMultipartPart = async (
|
|
||||||
part: Omit<MultipartPart, 'id' | 'createdAt'>,
|
|
||||||
): Promise<void> => {
|
|
||||||
await db.execute(
|
|
||||||
sql`INSERT INTO multipart_parts (upload_id, part_number, telegram_file_id, telegram_file_unique_id, storage_message_id, size_bytes, etag)
|
|
||||||
VALUES (${part.uploadId}, ${part.partNumber}, ${part.telegramFileId}, ${part.telegramFileUniqueId}, ${part.storageMessageId}, ${part.sizeBytes}, ${part.etag})`,
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const listMultipartParts = async (uploadId: string): Promise<MultipartPart[]> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`SELECT id, upload_id, part_number, telegram_file_id, telegram_file_unique_id, storage_message_id, size_bytes, etag, created_at
|
|
||||||
FROM multipart_parts WHERE upload_id = ${uploadId} ORDER BY part_number`,
|
|
||||||
)) as unknown as Record<string, unknown>[];
|
|
||||||
return result.map((r) => ({
|
|
||||||
id: r.id as number,
|
|
||||||
uploadId: r.upload_id as string,
|
|
||||||
partNumber: r.part_number as number,
|
|
||||||
telegramFileId: r.telegram_file_id as string,
|
|
||||||
telegramFileUniqueId: r.telegram_file_unique_id as string,
|
|
||||||
storageMessageId: r.storage_message_id as number,
|
|
||||||
sizeBytes: Number(r.size_bytes),
|
|
||||||
etag: r.etag as string,
|
|
||||||
createdAt: new Date(r.created_at as string),
|
|
||||||
}));
|
|
||||||
};
|
|
||||||
|
|
||||||
const mapRowToMultipartUpload = (r: Record<string, unknown>): MultipartUpload => ({
|
|
||||||
uploadId: r.upload_id as string,
|
|
||||||
bucketId: r.bucket_id as string,
|
|
||||||
s3Key: r.s3_key as string,
|
|
||||||
initiatedAt: new Date(r.initiated_at as string),
|
|
||||||
status: r.status as string,
|
|
||||||
initiatedBy: (r.initiated_by as string | null) || '',
|
|
||||||
contentType: (r.content_type as string | null) || null,
|
|
||||||
});
|
|
||||||
|
|
||||||
export const listMultipartUploadsByBucket = async (
|
|
||||||
bucketId: string,
|
|
||||||
maxUploads: number,
|
|
||||||
keyMarker: string | null,
|
|
||||||
): Promise<{ uploads: MultipartUpload[]; isTruncated: boolean; nextKeyMarker: string | null }> => {
|
|
||||||
const limit = Math.min(Math.max(maxUploads || 1000, 1), 1000);
|
|
||||||
const result = (await db.execute(
|
|
||||||
keyMarker
|
|
||||||
? sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, initiated_by
|
|
||||||
FROM multipart_uploads
|
|
||||||
WHERE bucket_id = ${bucketId}::uuid AND status = 'in_progress' AND s3_key > ${keyMarker}
|
|
||||||
ORDER BY s3_key, initiated_at
|
|
||||||
LIMIT ${limit + 1}`
|
|
||||||
: sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, initiated_by
|
|
||||||
FROM multipart_uploads
|
|
||||||
WHERE bucket_id = ${bucketId}::uuid AND status = 'in_progress'
|
|
||||||
ORDER BY s3_key, initiated_at
|
|
||||||
LIMIT ${limit + 1}`,
|
|
||||||
)) as unknown as Record<string, unknown>[];
|
|
||||||
|
|
||||||
const uploads = result.slice(0, limit).map(mapRowToMultipartUpload);
|
|
||||||
return {
|
|
||||||
uploads,
|
|
||||||
isTruncated: result.length > limit,
|
|
||||||
nextKeyMarker: result.length > limit ? uploads.at(-1)?.s3Key || null : null,
|
|
||||||
};
|
|
||||||
};
|
|
||||||
@@ -1,71 +0,0 @@
|
|||||||
import { eq, type InferInsertModel, type InferSelectModel } from 'drizzle-orm';
|
|
||||||
import {
|
|
||||||
bigint,
|
|
||||||
boolean,
|
|
||||||
integer,
|
|
||||||
pgTable,
|
|
||||||
serial,
|
|
||||||
text,
|
|
||||||
timestamp,
|
|
||||||
uniqueIndex,
|
|
||||||
uuid,
|
|
||||||
} from 'drizzle-orm/pg-core';
|
|
||||||
|
|
||||||
export const files = pgTable(
|
|
||||||
'files',
|
|
||||||
{
|
|
||||||
id: uuid('id').primaryKey().defaultRandom(),
|
|
||||||
publicId: text('public_id').unique().notNull(),
|
|
||||||
telegramFileId: text('telegram_file_id').notNull(),
|
|
||||||
telegramFileUniqueId: text('telegram_file_unique_id').notNull(),
|
|
||||||
storageChatId: bigint('storage_chat_id', { mode: 'number' }).notNull(),
|
|
||||||
storageMessageId: bigint('storage_message_id', { mode: 'number' }).notNull(),
|
|
||||||
fileName: text('file_name').notNull(),
|
|
||||||
mimeType: text('mime_type').notNull(),
|
|
||||||
sizeBytes: bigint('size_bytes', { mode: 'number' }).notNull(),
|
|
||||||
fileType: text('file_type').notNull(),
|
|
||||||
uploaderId: bigint('uploader_id', { mode: 'number' }).notNull(),
|
|
||||||
fileHash: text('file_hash'),
|
|
||||||
archiveTelegramFileId: text('archive_telegram_file_id'),
|
|
||||||
archiveStorageMessageId: bigint('archive_storage_message_id', { mode: 'number' }),
|
|
||||||
archiveFileName: text('archive_file_name'),
|
|
||||||
archiveEntryName: text('archive_entry_name'),
|
|
||||||
archiveMimeType: text('archive_mime_type'),
|
|
||||||
archiveSizeBytes: bigint('archive_size_bytes', { mode: 'number' }),
|
|
||||||
bucketId: text('bucket_id'),
|
|
||||||
s3Key: text('s3_key'),
|
|
||||||
storageBackend: text('storage_backend').default('telegram'),
|
|
||||||
isDeleted: boolean('is_deleted').default(false),
|
|
||||||
multipartUploadId: text('multipart_upload_id'),
|
|
||||||
partCount: integer('part_count'),
|
|
||||||
createdAt: timestamp('created_at').defaultNow().notNull(),
|
|
||||||
updatedAt: timestamp('updated_at').defaultNow().notNull(),
|
|
||||||
},
|
|
||||||
(table) => ({
|
|
||||||
// H2: Prevent TOCTOU race on concurrent PUT — only one active (non-deleted)
|
|
||||||
// object per (bucket_id, s3_key) pair.
|
|
||||||
activeObjectIdx: uniqueIndex('active_object_idx')
|
|
||||||
.on(table.bucketId, table.s3Key)
|
|
||||||
.where(eq(table.isDeleted, false)),
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
export const fileParts = pgTable('file_parts', {
|
|
||||||
id: serial('id').primaryKey(),
|
|
||||||
fileId: uuid('file_id').notNull(),
|
|
||||||
partNumber: integer('part_number').notNull(),
|
|
||||||
telegramFileId: text('telegram_file_id').notNull(),
|
|
||||||
telegramFileUniqueId: text('telegram_file_unique_id').notNull(),
|
|
||||||
storageChatId: bigint('storage_chat_id', { mode: 'number' }).notNull(),
|
|
||||||
storageMessageId: bigint('storage_message_id', { mode: 'number' }).notNull(),
|
|
||||||
sizeBytes: bigint('size_bytes', { mode: 'number' }).notNull(),
|
|
||||||
storedSizeBytes: bigint('stored_size_bytes', { mode: 'number' }).notNull(),
|
|
||||||
compressionAlgorithm: text('compression_algorithm'),
|
|
||||||
etag: text('etag').notNull(),
|
|
||||||
createdAt: timestamp('created_at').defaultNow().notNull(),
|
|
||||||
});
|
|
||||||
|
|
||||||
export type File = InferSelectModel<typeof files>;
|
|
||||||
export type NewFile = InferInsertModel<typeof files>;
|
|
||||||
export type FilePart = InferSelectModel<typeof fileParts>;
|
|
||||||
export type NewFilePart = InferInsertModel<typeof fileParts>;
|
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
/**
|
||||||
|
* Factory function for building NewFile records with sensible defaults.
|
||||||
|
*
|
||||||
|
* Most call sites set the same null defaults for archive/S3/soft-delete fields.
|
||||||
|
* This factory eliminates ~20 lines of boilerplate per call site (~27 sites).
|
||||||
|
*/
|
||||||
|
import type { NewFile } from './file';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Partial input for creating a file record.
|
||||||
|
* Only the required unique fields must be provided; optional fields default to null/0/false.
|
||||||
|
*/
|
||||||
|
export interface FileInput {
|
||||||
|
publicId: string;
|
||||||
|
telegramFileId: string;
|
||||||
|
telegramFileUniqueId: string;
|
||||||
|
storageChatId: number;
|
||||||
|
storageMessageId: number;
|
||||||
|
fileName: string;
|
||||||
|
mimeType: string;
|
||||||
|
sizeBytes: number;
|
||||||
|
fileType: string;
|
||||||
|
storageBackend: string | null;
|
||||||
|
/** Optional overrides */
|
||||||
|
uploaderId?: number;
|
||||||
|
fileHash?: string | null;
|
||||||
|
bucketId?: string | null;
|
||||||
|
s3Key?: string | null;
|
||||||
|
partCount?: number | null;
|
||||||
|
multipartUploadId?: string | null;
|
||||||
|
/** Archive fields (for batch/zip archives) */
|
||||||
|
archiveTelegramFileId?: string | null;
|
||||||
|
archiveStorageMessageId?: number | null;
|
||||||
|
archiveFileName?: string | null;
|
||||||
|
archiveEntryName?: string | null;
|
||||||
|
archiveMimeType?: string | null;
|
||||||
|
archiveSizeBytes?: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build a NewFile record, filling in null/zero defaults for omitted fields.
|
||||||
|
*
|
||||||
|
* @example
|
||||||
|
* ```ts
|
||||||
|
* await fileRepo.create(buildNewFile({
|
||||||
|
* publicId,
|
||||||
|
* telegramFileId: result.telegramFileId,
|
||||||
|
* telegramFileUniqueId: result.telegramFileUniqueId,
|
||||||
|
* storageChatId,
|
||||||
|
* storageMessageId: result.storageMessageId,
|
||||||
|
* fileName: input.fileName,
|
||||||
|
* mimeType,
|
||||||
|
* sizeBytes: input.sizeBytes,
|
||||||
|
* fileType,
|
||||||
|
* storageBackend: 'telegram',
|
||||||
|
* uploaderId: input.uploaderId,
|
||||||
|
* fileHash: input.fileHash,
|
||||||
|
* }));
|
||||||
|
* ```
|
||||||
|
*/
|
||||||
|
export const buildNewFile = (input: FileInput): NewFile => ({
|
||||||
|
publicId: input.publicId,
|
||||||
|
telegramFileId: input.telegramFileId,
|
||||||
|
telegramFileUniqueId: input.telegramFileUniqueId,
|
||||||
|
storageChatId: input.storageChatId,
|
||||||
|
storageMessageId: input.storageMessageId,
|
||||||
|
fileName: input.fileName,
|
||||||
|
mimeType: input.mimeType,
|
||||||
|
sizeBytes: input.sizeBytes,
|
||||||
|
fileType: input.fileType,
|
||||||
|
uploaderId: input.uploaderId ?? 0,
|
||||||
|
fileHash: input.fileHash ?? null,
|
||||||
|
archiveTelegramFileId: input.archiveTelegramFileId ?? null,
|
||||||
|
archiveStorageMessageId: input.archiveStorageMessageId ?? null,
|
||||||
|
archiveFileName: input.archiveFileName ?? null,
|
||||||
|
archiveEntryName: input.archiveEntryName ?? null,
|
||||||
|
archiveMimeType: input.archiveMimeType ?? null,
|
||||||
|
archiveSizeBytes: input.archiveSizeBytes ?? null,
|
||||||
|
bucketId: input.bucketId ?? null,
|
||||||
|
s3Key: input.s3Key ?? null,
|
||||||
|
storageBackend: input.storageBackend,
|
||||||
|
isDeleted: false,
|
||||||
|
multipartUploadId: input.multipartUploadId ?? null,
|
||||||
|
partCount: input.partCount ?? null,
|
||||||
|
});
|
||||||
@@ -50,15 +50,4 @@ export interface ITelegramService {
|
|||||||
* @returns Metadata including size, MIME type, download path, and bot token.
|
* @returns Metadata including size, MIME type, download path, and bot token.
|
||||||
*/
|
*/
|
||||||
getFileInfo(telegramFileId: string): Promise<TelegramFileInfo>;
|
getFileInfo(telegramFileId: string): Promise<TelegramFileInfo>;
|
||||||
|
|
||||||
/**
|
|
||||||
* Enqueue a task for sequential upload execution.
|
|
||||||
*
|
|
||||||
* Ensures only one Telegram upload runs at a time to avoid
|
|
||||||
* rate limits and resource contention.
|
|
||||||
*
|
|
||||||
* @param task - An async function performing the upload.
|
|
||||||
* @returns The result of the task.
|
|
||||||
*/
|
|
||||||
enqueueUpload<T>(task: () => Promise<T>): Promise<T>;
|
|
||||||
}
|
}
|
||||||
|
|||||||
+48
-12
@@ -1,8 +1,11 @@
|
|||||||
import logger from './utils/logger';
|
import logger from './shared/logger/index';
|
||||||
|
import { TELEGRAM_CHUNK_SIZE_MAX_BYTES } from './shared/utils/validation';
|
||||||
|
|
||||||
interface AppConfig {
|
interface AppConfig {
|
||||||
botToken: string;
|
/** All bot tokens merged from BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS fallback) */
|
||||||
additionalBotTokens: string[];
|
botTokens: string[];
|
||||||
|
/** Per-bot concurrency for Telegram API calls (default 1). */
|
||||||
|
telegramBotConcurrency: number;
|
||||||
storageChatId: number;
|
storageChatId: number;
|
||||||
baseUrl: string;
|
baseUrl: string;
|
||||||
databaseUrl: string;
|
databaseUrl: string;
|
||||||
@@ -12,7 +15,6 @@ interface AppConfig {
|
|||||||
rateLimitWindowMs: number;
|
rateLimitWindowMs: number;
|
||||||
rateLimitMaxRequests: number;
|
rateLimitMaxRequests: number;
|
||||||
trustProxy: boolean;
|
trustProxy: boolean;
|
||||||
uploadConcurrency: number;
|
|
||||||
batchMaxItems: number;
|
batchMaxItems: number;
|
||||||
batchMaxSizeBytes: number;
|
batchMaxSizeBytes: number;
|
||||||
maxRequestBodyBytes: number;
|
maxRequestBodyBytes: number;
|
||||||
@@ -29,8 +31,21 @@ interface AppConfig {
|
|||||||
s3VhostDomains: string[];
|
s3VhostDomains: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Validate bot tokens: BOT_TOKENS (new) or fallback to BOT_TOKEN + ADDITIONAL_BOT_TOKENS
|
||||||
|
const botTokensRaw =
|
||||||
|
process.env.BOT_TOKENS ||
|
||||||
|
[process.env.BOT_TOKEN, process.env.ADDITIONAL_BOT_TOKENS].filter(Boolean).join(',');
|
||||||
|
|
||||||
|
if (!botTokensRaw) {
|
||||||
|
logger.error(
|
||||||
|
'Missing required environment variables: BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS)',
|
||||||
|
);
|
||||||
|
throw new Error(
|
||||||
|
'Missing environment variables: BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS)',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const requiredEnv = {
|
const requiredEnv = {
|
||||||
BOT_TOKEN: process.env.BOT_TOKEN,
|
|
||||||
STORAGE_CHANNEL_ID: process.env.STORAGE_CHANNEL_ID,
|
STORAGE_CHANNEL_ID: process.env.STORAGE_CHANNEL_ID,
|
||||||
BASE_URL: process.env.BASE_URL,
|
BASE_URL: process.env.BASE_URL,
|
||||||
DATABASE_URL: process.env.DATABASE_URL,
|
DATABASE_URL: process.env.DATABASE_URL,
|
||||||
@@ -93,23 +108,45 @@ const maskSecret = (value: string): string => {
|
|||||||
const maskDatabaseUrl = (value: string): string =>
|
const maskDatabaseUrl = (value: string): string =>
|
||||||
value.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:***@');
|
value.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:***@');
|
||||||
|
|
||||||
|
// Fail-fast guard for TELEGRAM_CHUNK_SIZE_BYTES: chunked uploads store every
|
||||||
|
// part as a Telegram document and later resolve it via getFile, which only
|
||||||
|
// supports files up to 20 MB ("Bad Request: file is too big" above that).
|
||||||
|
// A chunk above the limit makes every part undownloadable — refuse to start
|
||||||
|
// instead of failing on the first large-file download.
|
||||||
|
const telegramChunkSizeBytes = parseNumber(
|
||||||
|
process.env.TELEGRAM_CHUNK_SIZE_BYTES,
|
||||||
|
TELEGRAM_CHUNK_SIZE_MAX_BYTES,
|
||||||
|
);
|
||||||
|
if (telegramChunkSizeBytes > TELEGRAM_CHUNK_SIZE_MAX_BYTES) {
|
||||||
|
logger.error(
|
||||||
|
`TELEGRAM_CHUNK_SIZE_BYTES=${telegramChunkSizeBytes} exceeds the maximum allowed chunk size ` +
|
||||||
|
`${TELEGRAM_CHUNK_SIZE_MAX_BYTES} bytes (${TELEGRAM_CHUNK_SIZE_MAX_BYTES / (1024 * 1024)} MB). ` +
|
||||||
|
'Telegram Bot API getFile cannot download files larger than 20 MB, so every stored part would ' +
|
||||||
|
'be undownloadable ("Bad Request: file is too big"). ' +
|
||||||
|
`Set TELEGRAM_CHUNK_SIZE_BYTES to ${TELEGRAM_CHUNK_SIZE_MAX_BYTES} or lower.`,
|
||||||
|
);
|
||||||
|
throw new Error(
|
||||||
|
`TELEGRAM_CHUNK_SIZE_BYTES=${telegramChunkSizeBytes} exceeds the maximum allowed chunk size ` +
|
||||||
|
`${TELEGRAM_CHUNK_SIZE_MAX_BYTES} bytes (${TELEGRAM_CHUNK_SIZE_MAX_BYTES / (1024 * 1024)} MB)`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export const config: AppConfig = {
|
export const config: AppConfig = {
|
||||||
botToken: process.env.BOT_TOKEN!,
|
botTokens: parseTokens(botTokensRaw),
|
||||||
additionalBotTokens: parseTokens(process.env.ADDITIONAL_BOT_TOKENS),
|
telegramBotConcurrency: parseNumber(process.env.TELEGRAM_BOT_CONCURRENCY, 1),
|
||||||
storageChatId: parseInt(process.env.STORAGE_CHANNEL_ID!, 10),
|
storageChatId: parseInt(process.env.STORAGE_CHANNEL_ID!, 10),
|
||||||
baseUrl: process.env.BASE_URL!,
|
baseUrl: process.env.BASE_URL!,
|
||||||
databaseUrl: process.env.DATABASE_URL!,
|
databaseUrl: process.env.DATABASE_URL!,
|
||||||
port: parseInt(process.env.PORT!, 10) || 3000,
|
port: parseInt(process.env.PORT!, 10) || 4000,
|
||||||
nodeEnv: process.env.NODE_ENV || 'development',
|
nodeEnv: process.env.NODE_ENV || 'development',
|
||||||
logLevel: process.env.LOG_LEVEL || 'info',
|
logLevel: process.env.LOG_LEVEL || 'info',
|
||||||
rateLimitWindowMs: parseNumber(process.env.RATE_LIMIT_WINDOW_MS, 60000),
|
rateLimitWindowMs: parseNumber(process.env.RATE_LIMIT_WINDOW_MS, 60000),
|
||||||
rateLimitMaxRequests: parseNumber(process.env.RATE_LIMIT_MAX_REQUESTS, 150),
|
rateLimitMaxRequests: parseNumber(process.env.RATE_LIMIT_MAX_REQUESTS, 150),
|
||||||
trustProxy: process.env.TRUST_PROXY === 'true',
|
trustProxy: process.env.TRUST_PROXY === 'true',
|
||||||
uploadConcurrency: parseNumber(process.env.UPLOAD_CONCURRENCY, 8),
|
|
||||||
batchMaxItems: parseNumber(process.env.BATCH_MAX_ITEMS, 20),
|
batchMaxItems: parseNumber(process.env.BATCH_MAX_ITEMS, 20),
|
||||||
batchMaxSizeBytes: parseNumber(process.env.BATCH_MAX_SIZE_BYTES, 500 * 1024 * 1024),
|
batchMaxSizeBytes: parseNumber(process.env.BATCH_MAX_SIZE_BYTES, 500 * 1024 * 1024),
|
||||||
maxRequestBodyBytes: parseNumber(process.env.MAX_REQUEST_BODY_BYTES, 2 * 1024 * 1024 * 1024),
|
maxRequestBodyBytes: parseNumber(process.env.MAX_REQUEST_BODY_BYTES, 2 * 1024 * 1024 * 1024),
|
||||||
telegramChunkSizeBytes: parseNumber(process.env.TELEGRAM_CHUNK_SIZE_BYTES, 20 * 1024 * 1024),
|
telegramChunkSizeBytes,
|
||||||
compressChunkedUploads: process.env.COMPRESS_CHUNKED_UPLOADS !== 'false',
|
compressChunkedUploads: process.env.COMPRESS_CHUNKED_UPLOADS !== 'false',
|
||||||
chunkCompressionMinSizeBytes: parseNumber(process.env.CHUNK_COMPRESSION_MIN_SIZE_BYTES, 4096),
|
chunkCompressionMinSizeBytes: parseNumber(process.env.CHUNK_COMPRESSION_MIN_SIZE_BYTES, 4096),
|
||||||
adminApiToken: process.env.ADMIN_API_TOKEN || '',
|
adminApiToken: process.env.ADMIN_API_TOKEN || '',
|
||||||
@@ -128,8 +165,7 @@ export const config: AppConfig = {
|
|||||||
logger.info('Environment variables loaded', {
|
logger.info('Environment variables loaded', {
|
||||||
config: {
|
config: {
|
||||||
...config,
|
...config,
|
||||||
botToken: maskSecret(config.botToken),
|
botTokens: config.botTokens.map(maskSecret),
|
||||||
additionalBotTokens: config.additionalBotTokens.map(maskSecret),
|
|
||||||
databaseUrl: maskDatabaseUrl(config.databaseUrl),
|
databaseUrl: maskDatabaseUrl(config.databaseUrl),
|
||||||
adminApiToken: maskSecret(config.adminApiToken),
|
adminApiToken: maskSecret(config.adminApiToken),
|
||||||
adminApiTokenEnabled: config.adminApiToken.length > 0,
|
adminApiTokenEnabled: config.adminApiToken.length > 0,
|
||||||
|
|||||||
+59
-27
@@ -136,6 +136,10 @@
|
|||||||
}
|
}
|
||||||
.auth-card button:disabled { opacity: 0.7; cursor: wait; }
|
.auth-card button:disabled { opacity: 0.7; cursor: wait; }
|
||||||
.auth-error { color: var(--danger); font-size: 0.85rem; margin-bottom: 12px; }
|
.auth-error { color: var(--danger); font-size: 0.85rem; margin-bottom: 12px; }
|
||||||
|
.readonly-badge {
|
||||||
|
font-size: 0.75rem; color: var(--text2); background: var(--bg2);
|
||||||
|
border: 1px solid var(--border); border-radius: 999px; padding: 2px 10px;
|
||||||
|
}
|
||||||
</style>
|
</style>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
@@ -153,9 +157,11 @@
|
|||||||
<select id="bucketSelect" onchange="window.switchBucket(this.value)">
|
<select id="bucketSelect" onchange="window.switchBucket(this.value)">
|
||||||
<option value="">— Select bucket —</option>
|
<option value="">— Select bucket —</option>
|
||||||
</select>
|
</select>
|
||||||
<button type="button" onclick="window.showCreateBucketModal()">+ New</button>
|
<button id="newBucketBtn" type="button" onclick="window.showCreateBucketModal()">+ New</button>
|
||||||
<button type="button" onclick="window.showCredentialsModal()" title="S3 Credentials">🔑</button>
|
<button id="credsBtn" type="button" onclick="window.showCredentialsModal()" title="S3 Credentials">🔑</button>
|
||||||
|
<button id="loginBtn" type="button" onclick="window.showAuthScreen()" style="display:none">Login</button>
|
||||||
<button id="logoutBtn" type="button" onclick="window.logout()" style="display:none">Logout</button>
|
<button id="logoutBtn" type="button" onclick="window.logout()" style="display:none">Logout</button>
|
||||||
|
<span id="readonlyBadge" class="readonly-badge" style="display:none">👀 read-only</span>
|
||||||
<span class="spacer"></span>
|
<span class="spacer"></span>
|
||||||
<div class="search">
|
<div class="search">
|
||||||
<input id="searchInput" type="text" placeholder="Filter prefix..." oninput="window.debouncedSearch()">
|
<input id="searchInput" type="text" placeholder="Filter prefix..." oninput="window.debouncedSearch()">
|
||||||
@@ -179,6 +185,7 @@
|
|||||||
</div>
|
</div>
|
||||||
<script>
|
<script>
|
||||||
let currentBucket = null, currentPrefix = '', currentObjects = [], currentPrefixes = [], allBuckets = [], searchTimer = null;
|
let currentBucket = null, currentPrefix = '', currentObjects = [], currentPrefixes = [], allBuckets = [], searchTimer = null;
|
||||||
|
let isAdmin = false;
|
||||||
const setAuthError = (message) => {
|
const setAuthError = (message) => {
|
||||||
const errorEl = document.getElementById('authError');
|
const errorEl = document.getElementById('authError');
|
||||||
errorEl.textContent = message;
|
errorEl.textContent = message;
|
||||||
@@ -186,25 +193,36 @@
|
|||||||
};
|
};
|
||||||
const showAuthScreen = () => {
|
const showAuthScreen = () => {
|
||||||
document.getElementById('authScreen').style.display = 'flex';
|
document.getElementById('authScreen').style.display = 'flex';
|
||||||
document.getElementById('logoutBtn').style.display = 'none';
|
|
||||||
setTimeout(() => document.getElementById('authTokenInput')?.focus(), 50);
|
setTimeout(() => document.getElementById('authTokenInput')?.focus(), 50);
|
||||||
};
|
};
|
||||||
const hideAuthScreen = (showLogout) => {
|
const hideAuthScreen = () => {
|
||||||
document.getElementById('authScreen').style.display = 'none';
|
document.getElementById('authScreen').style.display = 'none';
|
||||||
document.getElementById('logoutBtn').style.display = showLogout ? 'inline-block' : 'none';
|
|
||||||
};
|
};
|
||||||
|
// Applies the admin/read-only UI state based on isAdmin.
|
||||||
|
const applyAdminUI = () => {
|
||||||
|
document.getElementById('newBucketBtn').style.display = isAdmin ? 'inline-block' : 'none';
|
||||||
|
document.getElementById('credsBtn').style.display = isAdmin ? 'inline-block' : 'none';
|
||||||
|
document.getElementById('loginBtn').style.display = isAdmin ? 'none' : 'inline-block';
|
||||||
|
document.getElementById('logoutBtn').style.display = isAdmin ? 'inline-block' : 'none';
|
||||||
|
document.getElementById('readonlyBadge').style.display = isAdmin ? 'none' : 'inline-block';
|
||||||
|
// Dropzone (upload) is admin-only.
|
||||||
|
document.getElementById('dropzone').style.display = isAdmin && currentBucket ? 'block' : 'none';
|
||||||
|
if (currentObjects.length || currentPrefixes.length) renderFileList();
|
||||||
|
};
|
||||||
|
// Non-blocking auth check: read-only visitors still get the file browser.
|
||||||
const checkAuth = async () => {
|
const checkAuth = async () => {
|
||||||
try {
|
try {
|
||||||
const res = await fetch('/api/v1/auth/me');
|
const res = await fetch('/api/v1/auth/me');
|
||||||
if (res.ok) { hideAuthScreen(true); return true; }
|
if (res.ok) { isAdmin = true; }
|
||||||
if (res.status === 401) { showAuthScreen(); return false; }
|
else if (res.status === 401) { isAdmin = false; }
|
||||||
if (res.status === 404) { hideAuthScreen(false); return true; }
|
else if (res.status === 404) { isAdmin = true; } // auth disabled — full access
|
||||||
setAuthError('Unable to verify login status. Please try again.');
|
else { isAdmin = false; }
|
||||||
showAuthScreen(); return false;
|
|
||||||
} catch {
|
} catch {
|
||||||
setAuthError('Network error while checking login status.');
|
isAdmin = false;
|
||||||
showAuthScreen(); return false;
|
|
||||||
}
|
}
|
||||||
|
hideAuthScreen();
|
||||||
|
applyAdminUI();
|
||||||
|
return isAdmin;
|
||||||
};
|
};
|
||||||
const handleLogin = async () => {
|
const handleLogin = async () => {
|
||||||
const input = document.getElementById('authTokenInput');
|
const input = document.getElementById('authTokenInput');
|
||||||
@@ -217,7 +235,7 @@
|
|||||||
method: 'POST', headers: { 'content-type': 'application/json' },
|
method: 'POST', headers: { 'content-type': 'application/json' },
|
||||||
body: JSON.stringify({ token }),
|
body: JSON.stringify({ token }),
|
||||||
});
|
});
|
||||||
if (res.ok) { hideAuthScreen(true); input.value = ''; await loadBuckets(); return; }
|
if (res.ok) { isAdmin = true; hideAuthScreen(); input.value = ''; applyAdminUI(); await loadBuckets(); return; }
|
||||||
const body = await res.json().catch(() => ({ error: 'Login failed' }));
|
const body = await res.json().catch(() => ({ error: 'Login failed' }));
|
||||||
setAuthError(body.error || 'Login failed');
|
setAuthError(body.error || 'Login failed');
|
||||||
} catch {
|
} catch {
|
||||||
@@ -228,11 +246,8 @@
|
|||||||
};
|
};
|
||||||
const logout = async () => {
|
const logout = async () => {
|
||||||
await fetch('/api/v1/auth/logout', { method: 'POST' }).catch(() => {});
|
await fetch('/api/v1/auth/logout', { method: 'POST' }).catch(() => {});
|
||||||
currentBucket = null; currentPrefix = ''; currentObjects = []; currentPrefixes = [];
|
isAdmin = false;
|
||||||
document.getElementById('bucketSelect').innerHTML = '<option value="">— Select bucket —</option>';
|
applyAdminUI();
|
||||||
document.getElementById('fileList').innerHTML = '<div class="empty"><h2>Logged out</h2><p>Enter the admin token to continue.</p></div>';
|
|
||||||
document.getElementById('dropzone').style.display = 'none';
|
|
||||||
showAuthScreen();
|
|
||||||
};
|
};
|
||||||
const api = async (path, opts = {}) => {
|
const api = async (path, opts = {}) => {
|
||||||
const res = await fetch(path, opts);
|
const res = await fetch(path, opts);
|
||||||
@@ -249,11 +264,13 @@
|
|||||||
};
|
};
|
||||||
const switchBucket = async (name) => {
|
const switchBucket = async (name) => {
|
||||||
currentBucket = name || null; currentPrefix = '';
|
currentBucket = name || null; currentPrefix = '';
|
||||||
if (name) { await loadObjects(); document.getElementById('dropzone').style.display = 'block'; }
|
if (name) { await loadObjects(); }
|
||||||
else {
|
else {
|
||||||
document.getElementById('fileList').innerHTML = '<div class="empty"><h2>Select a bucket</h2><p>Choose a bucket from the dropdown above.</p></div>';
|
document.getElementById('fileList').innerHTML = '<div class="empty"><h2>Select a bucket</h2><p>Choose a bucket from the dropdown above.</p></div>';
|
||||||
document.getElementById('breadcrumb').style.display = 'none'; document.getElementById('dropzone').style.display = 'none';
|
document.getElementById('breadcrumb').style.display = 'none';
|
||||||
}
|
}
|
||||||
|
// Dropzone (upload) is admin-only.
|
||||||
|
document.getElementById('dropzone').style.display = isAdmin && currentBucket ? 'block' : 'none';
|
||||||
};
|
};
|
||||||
const renderBreadcrumb = () => {
|
const renderBreadcrumb = () => {
|
||||||
const bc = document.getElementById('breadcrumb');
|
const bc = document.getElementById('breadcrumb');
|
||||||
@@ -278,7 +295,12 @@
|
|||||||
};
|
};
|
||||||
const renderFileList = () => {
|
const renderFileList = () => {
|
||||||
const container = document.getElementById('fileList');
|
const container = document.getElementById('fileList');
|
||||||
if (currentPrefixes.length === 0 && currentObjects.length === 0) { container.innerHTML = '<div class="empty"><h2>This bucket is empty</h2><p>Drop files here to upload.</p></div>'; return; }
|
if (currentPrefixes.length === 0 && currentObjects.length === 0) {
|
||||||
|
container.innerHTML = isAdmin
|
||||||
|
? '<div class="empty"><h2>This bucket is empty</h2><p>Drop files here to upload.</p></div>'
|
||||||
|
: '<div class="empty"><h2>This bucket is empty</h2></div>';
|
||||||
|
return;
|
||||||
|
}
|
||||||
let html = '';
|
let html = '';
|
||||||
for (const prefix of currentPrefixes) {
|
for (const prefix of currentPrefixes) {
|
||||||
const displayName = prefix.replace(currentPrefix, '');
|
const displayName = prefix.replace(currentPrefix, '');
|
||||||
@@ -286,7 +308,9 @@
|
|||||||
}
|
}
|
||||||
for (const obj of currentObjects) {
|
for (const obj of currentObjects) {
|
||||||
const displayName = obj.key.replace(currentPrefix, '');
|
const displayName = obj.key.replace(currentPrefix, '');
|
||||||
html += `<div class="file-row"><span class="icon">📄</span><span class="name">${escapeHtml(displayName)}</span><span class="size">${formatSize(obj.sizeBytes)}</span><span class="date">${formatDate(obj.lastModified)}</span><span class="actions"><button onclick="event.stopPropagation();downloadObject('${obj.key}')" title="Download">⬇</button><button onclick="event.stopPropagation();copyLink('${obj.key}')" title="Copy link">🔗</button><button onclick="event.stopPropagation();deleteObject('${obj.key}')" title="Delete">🗑</button></span></div>`;
|
// Delete is admin-only; download + copy link are always available.
|
||||||
|
const deleteBtn = isAdmin ? `<button onclick="event.stopPropagation();deleteObject('${obj.key}')" title="Delete">🗑</button>` : '';
|
||||||
|
html += `<div class="file-row"><span class="icon">📄</span><span class="name">${escapeHtml(displayName)}</span><span class="size">${formatSize(obj.sizeBytes)}</span><span class="date">${formatDate(obj.lastModified)}</span><span class="actions"><button onclick="event.stopPropagation();downloadObject('${obj.key}')" title="Download">⬇</button><button onclick="event.stopPropagation();copyLink('${obj.key}')" title="Copy link">🔗</button>${deleteBtn}</span></div>`;
|
||||||
}
|
}
|
||||||
container.innerHTML = html;
|
container.innerHTML = html;
|
||||||
};
|
};
|
||||||
@@ -297,11 +321,13 @@
|
|||||||
const downloadObject = async (key) => { window.open(`/api/v1/buckets/${encodeURIComponent(currentBucket)}/download/${encodeURIComponent(key)}`,'_blank'); };
|
const downloadObject = async (key) => { window.open(`/api/v1/buckets/${encodeURIComponent(currentBucket)}/download/${encodeURIComponent(key)}`,'_blank'); };
|
||||||
const copyLink = (key) => { navigator.clipboard.writeText(`${window.location.origin}/api/v1/buckets/${encodeURIComponent(currentBucket)}/download/${encodeURIComponent(key)}`).catch(()=>{}); };
|
const copyLink = (key) => { navigator.clipboard.writeText(`${window.location.origin}/api/v1/buckets/${encodeURIComponent(currentBucket)}/download/${encodeURIComponent(key)}`).catch(()=>{}); };
|
||||||
const deleteObject = async (key) => {
|
const deleteObject = async (key) => {
|
||||||
|
if (!isAdmin) { alert('Read-only mode — login as admin to delete.'); return; }
|
||||||
if(!confirm(`Delete "${key}"?`))return;
|
if(!confirm(`Delete "${key}"?`))return;
|
||||||
try{await api(`/api/v1/buckets/${encodeURIComponent(currentBucket)}/${encodeURIComponent(key)}`,{method:'DELETE'});await loadObjects();}
|
try{await api(`/api/v1/buckets/${encodeURIComponent(currentBucket)}/${encodeURIComponent(key)}`,{method:'DELETE'});await loadObjects();}
|
||||||
catch(e){alert(`Delete failed: ${e.message}`);}
|
catch(e){alert(`Delete failed: ${e.message}`);}
|
||||||
};
|
};
|
||||||
const uploadFiles = async (files) => {
|
const uploadFiles = async (files) => {
|
||||||
|
if (!isAdmin) { alert('Read-only mode — login as admin to upload.'); return; }
|
||||||
if(!currentBucket||files.length===0)return;
|
if(!currentBucket||files.length===0)return;
|
||||||
const overlay=document.getElementById('progressOverlay'), fill=document.getElementById('progressFill'), pn=document.getElementById('progressFileName'), pp=document.getElementById('progressPercent');
|
const overlay=document.getElementById('progressOverlay'), fill=document.getElementById('progressFill'), pn=document.getElementById('progressFileName'), pp=document.getElementById('progressPercent');
|
||||||
overlay.style.display='flex';
|
overlay.style.display='flex';
|
||||||
@@ -325,13 +351,19 @@
|
|||||||
dropzone.addEventListener('click',()=>{const i=document.createElement('input');i.type='file';i.multiple=true;i.onchange=()=>{if(i.files.length>0)uploadFiles(i.files);};i.click();});
|
dropzone.addEventListener('click',()=>{const i=document.createElement('input');i.type='file';i.multiple=true;i.onchange=()=>{if(i.files.length>0)uploadFiles(i.files);};i.click();});
|
||||||
const showModal=(html)=>{document.getElementById('modalContent').innerHTML=html;document.getElementById('modalOverlay').style.display='flex';};
|
const showModal=(html)=>{document.getElementById('modalContent').innerHTML=html;document.getElementById('modalOverlay').style.display='flex';};
|
||||||
const closeModal=(e)=>{if(e&&e.target!==e.currentTarget)return;document.getElementById('modalOverlay').style.display='none';};
|
const closeModal=(e)=>{if(e&&e.target!==e.currentTarget)return;document.getElementById('modalOverlay').style.display='none';};
|
||||||
const showCreateBucketModal=()=>{showModal(`<h3>Create Bucket</h3><input id="bucketNameInput" type="text" placeholder="my-bucket-name" pattern="[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]"><p style="font-size:0.8rem;color:var(--text2);margin-bottom:12px">Lowercase, 3-63 chars, no underscores</p><div class="buttons"><button onclick="closeModal()">Cancel</button><button class="primary" onclick="createBucket()">Create</button></div>`);setTimeout(()=>document.getElementById('bucketNameInput')?.focus(),100);};
|
const showCreateBucketModal=()=>{
|
||||||
const createBucket=async()=>{const n=document.getElementById('bucketNameInput').value.trim();if(!n)return;try{await apiJson('/api/v1/buckets',{method:'POST',body:JSON.stringify({name:n})});closeModal();await loadBuckets();document.getElementById('bucketSelect').value=n;await switchBucket(n);}catch(e){alert(`Failed: ${e.message}`);}};
|
if (!isAdmin) { alert('Read-only mode — login as admin to create buckets.'); return; }
|
||||||
const showCredentialsModal=()=>{showModal(`<h3>S3 Credentials</h3><p style="margin-bottom:12px;font-size:0.85rem;color:var(--text2)">Use these in any S3 client (aws-cli, rclone, s3cmd, etc.)</p><label style="font-size:0.85rem;font-weight:600">Endpoint URL</label><input type="text" value="${window.location.origin}" readonly onclick="this.select()"><label style="font-size:0.85rem;font-weight:600">Region</label><input type="text" value="us-east-1" readonly onclick="this.select()"><label style="font-size:0.85rem;font-weight:600">Access Key</label><input id="s3AccessKey" type="text" readonly onclick="this.select()"><label style="font-size:0.85rem;font-weight:600">Secret Key</label><input id="s3SecretKey" type="password" readonly onclick="this.select()"><div class="buttons"><button type="button" onclick="window.closeModal()">Close</button></div>`);};
|
showModal(`<h3>Create Bucket</h3><input id="bucketNameInput" type="text" placeholder="my-bucket-name" pattern="[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]"><p style="font-size:0.8rem;color:var(--text2);margin-bottom:12px">Lowercase, 3-63 chars, no underscores</p><div class="buttons"><button onclick="closeModal()">Cancel</button><button class="primary" onclick="createBucket()">Create</button></div>`);setTimeout(()=>document.getElementById('bucketNameInput')?.focus(),100);};
|
||||||
const init=async()=>{if(await checkAuth())await loadBuckets();};
|
const createBucket=async()=>{
|
||||||
|
if (!isAdmin) { alert('Read-only mode — login as admin to create buckets.'); return; }
|
||||||
|
const n=document.getElementById('bucketNameInput').value.trim();if(!n)return;try{await apiJson('/api/v1/buckets',{method:'POST',body:JSON.stringify({name:n})});closeModal();await loadBuckets();document.getElementById('bucketSelect').value=n;await switchBucket(n);}catch(e){alert(`Failed: ${e.message}`);}};
|
||||||
|
const showCredentialsModal=()=>{
|
||||||
|
if (!isAdmin) { alert('Read-only mode — login as admin to view S3 credentials.'); return; }
|
||||||
|
showModal(`<h3>S3 Credentials</h3><p style="margin-bottom:12px;font-size:0.85rem;color:var(--text2)">Use these in any S3 client (aws-cli, rclone, s3cmd, etc.)</p><label style="font-size:0.85rem;font-weight:600">Endpoint URL</label><input type="text" value="${window.location.origin}" readonly onclick="this.select()"><label style="font-size:0.85rem;font-weight:600">Region</label><input type="text" value="us-east-1" readonly onclick="this.select()"><label style="font-size:0.85rem;font-weight:600">Access Key</label><input id="s3AccessKey" type="text" readonly onclick="this.select()"><label style="font-size:0.85rem;font-weight:600">Secret Key</label><input id="s3SecretKey" type="password" readonly onclick="this.select()"><div class="buttons"><button type="button" onclick="window.closeModal()">Close</button></div>`);};
|
||||||
|
const init=async()=>{await checkAuth();await loadBuckets();};
|
||||||
document.getElementById('authLoginBtn').addEventListener('click',handleLogin);
|
document.getElementById('authLoginBtn').addEventListener('click',handleLogin);
|
||||||
document.getElementById('authTokenInput').addEventListener('keydown',e=>{if(e.key==='Enter')handleLogin();});
|
document.getElementById('authTokenInput').addEventListener('keydown',e=>{if(e.key==='Enter')handleLogin();});
|
||||||
Object.assign(window, { switchBucket, navigateTo, debouncedSearch, downloadObject, copyLink, deleteObject, closeModal, showCreateBucketModal, createBucket, showCredentialsModal, logout });
|
Object.assign(window, { switchBucket, navigateTo, debouncedSearch, downloadObject, copyLink, deleteObject, closeModal, showCreateBucketModal, createBucket, showCredentialsModal, showAuthScreen, logout });
|
||||||
init();
|
init();
|
||||||
</script>
|
</script>
|
||||||
</body>
|
</body>
|
||||||
|
|||||||
+3
-44
@@ -1,54 +1,26 @@
|
|||||||
import { serve } from 'bun';
|
import { serve } from 'bun';
|
||||||
import { config } from './config/index';
|
import { config } from './env';
|
||||||
import { fileInfoCache } from './infrastructure/cache/index';
|
import { fileInfoCache } from './infrastructure/cache/index';
|
||||||
import { clearQueue, getQueueStats, waitForQueue } from './infrastructure/telegram/upload-queue';
|
|
||||||
import { startBot } from './interfaces/bot/handler';
|
import { startBot } from './interfaces/bot/handler';
|
||||||
import { handleS3Request } from './interfaces/http/controllers/s3-controller';
|
import { handleS3Request } from './interfaces/http/controllers/s3-controller';
|
||||||
import { cleanupRateLimitCache } from './interfaces/http/middleware/rate-limit';
|
import { cleanupRateLimitCache } from './interfaces/http/middleware/rate-limit';
|
||||||
import { routes } from './interfaces/http/routes/index';
|
import { routes } from './interfaces/http/routes/index';
|
||||||
import { isS3Request } from './interfaces/s3/auth';
|
|
||||||
import { extractS3BucketFromHost } from './interfaces/s3/virtual-host';
|
|
||||||
import { logger } from './shared/logger/index';
|
import { logger } from './shared/logger/index';
|
||||||
import { metricsCollector } from './shared/metrics/index';
|
import { metricsCollector } from './shared/metrics/index';
|
||||||
|
import { getS3RouteBucket, shouldHandleS3 } from './shared/utils/s3-detection';
|
||||||
|
|
||||||
// ─── Auto-run migration at startup ──────────────────────────────────────────
|
// ─── Auto-run migration at startup ──────────────────────────────────────────
|
||||||
try {
|
try {
|
||||||
const { runMigration } = await import('./db/migrate');
|
const { runMigration } = await import('./infrastructure/persistence/drizzle/migrate');
|
||||||
await runMigration();
|
await runMigration();
|
||||||
} catch {
|
} catch {
|
||||||
logger.warn('Auto-migration skipped (non-fatal)');
|
logger.warn('Auto-migration skipped (non-fatal)');
|
||||||
}
|
}
|
||||||
|
|
||||||
const getS3RouteBucket = (req: Request): string | null => {
|
|
||||||
const host = req.headers.get('host') || '';
|
|
||||||
return extractS3BucketFromHost(host, config.s3VhostDomains);
|
|
||||||
};
|
|
||||||
|
|
||||||
const shouldHandleS3 = (req: Request, headers: Record<string, string>): boolean => {
|
|
||||||
const url = new URL(req.url);
|
|
||||||
return Boolean(
|
|
||||||
getS3RouteBucket(req) || isS3Request(headers) || url.searchParams.has('X-Amz-Signature'),
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
const _handleMaybeS3Root = (req: Request): Response | Promise<Response> => {
|
|
||||||
if (req.method === 'OPTIONS') {
|
|
||||||
return handleS3Request(req, getS3RouteBucket(req));
|
|
||||||
}
|
|
||||||
const headers = Object.fromEntries(req.headers);
|
|
||||||
if (shouldHandleS3(req, headers)) {
|
|
||||||
return handleS3Request(req, getS3RouteBucket(req));
|
|
||||||
}
|
|
||||||
return new Response('Not Allowed', { status: 405 });
|
|
||||||
};
|
|
||||||
|
|
||||||
const server = serve({
|
const server = serve({
|
||||||
port: config.port,
|
port: config.port,
|
||||||
routes,
|
routes,
|
||||||
fetch: async (req: Request) => {
|
fetch: async (req: Request) => {
|
||||||
if (req.method === 'OPTIONS') {
|
|
||||||
return handleS3Request(req, getS3RouteBucket(req));
|
|
||||||
}
|
|
||||||
const headers = Object.fromEntries(req.headers);
|
const headers = Object.fromEntries(req.headers);
|
||||||
if (shouldHandleS3(req, headers)) {
|
if (shouldHandleS3(req, headers)) {
|
||||||
return handleS3Request(req, getS3RouteBucket(req));
|
return handleS3Request(req, getS3RouteBucket(req));
|
||||||
@@ -67,19 +39,6 @@ const gracefulShutdown = async (signal: string): Promise<void> => {
|
|||||||
logger.info('Closing HTTP server — no new requests accepted');
|
logger.info('Closing HTTP server — no new requests accepted');
|
||||||
server.stop();
|
server.stop();
|
||||||
|
|
||||||
// Drain pending upload queue with a timeout
|
|
||||||
const { pending, size } = getQueueStats();
|
|
||||||
if (pending > 0 || size > 0) {
|
|
||||||
logger.info('Draining upload queue', { pending, size });
|
|
||||||
const drainTimeout = setTimeout(() => {
|
|
||||||
logger.warn('Upload queue drain timeout — clearing remaining tasks');
|
|
||||||
clearQueue();
|
|
||||||
}, 30_000);
|
|
||||||
await waitForQueue();
|
|
||||||
clearTimeout(drainTimeout);
|
|
||||||
logger.info('Upload queue drained');
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info('Stopping Telegram bot');
|
logger.info('Stopping Telegram bot');
|
||||||
bot.stop(signal);
|
bot.stop(signal);
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
/**
|
||||||
|
* Dependency Injection container.
|
||||||
|
*
|
||||||
|
* Wires up singleton instances of all repositories and application services,
|
||||||
|
* making them available to controllers and other adapters without requiring
|
||||||
|
* a full DI framework.
|
||||||
|
*
|
||||||
|
* @module infrastructure/di
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type { IBucketRepository } from '../domain/ports/bucket-repository';
|
||||||
|
import type { IFilePartRepository } from '../domain/ports/file-part-repository';
|
||||||
|
import type { IFileRepository } from '../domain/ports/file-repository';
|
||||||
|
import type { IMultipartRepository } from '../domain/ports/multipart-repository';
|
||||||
|
import type { ITelegramService } from '../domain/ports/telegram-service';
|
||||||
|
import { DrizzleBucketRepository } from './persistence/repositories/bucket-repository';
|
||||||
|
import { DrizzleFilePartRepository } from './persistence/repositories/file-part-repository';
|
||||||
|
import { DrizzleFileRepository } from './persistence/repositories/file-repository';
|
||||||
|
import { DrizzleMultipartRepository } from './persistence/repositories/multipart-repository';
|
||||||
|
import { botPool } from './telegram/bot-pool';
|
||||||
|
import { ChunkedStorage } from './telegram/chunked-storage';
|
||||||
|
|
||||||
|
// ─── Repository Singletons ──────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Singleton IFileRepository instance backed by Drizzle ORM. */
|
||||||
|
export const fileRepository: IFileRepository = new DrizzleFileRepository();
|
||||||
|
|
||||||
|
/** Singleton IBucketRepository instance backed by Drizzle ORM. */
|
||||||
|
export const bucketRepository: IBucketRepository = new DrizzleBucketRepository();
|
||||||
|
|
||||||
|
/** Singleton IFilePartRepository instance backed by Drizzle ORM. */
|
||||||
|
export const filePartRepository: IFilePartRepository = new DrizzleFilePartRepository();
|
||||||
|
|
||||||
|
/** Singleton IMultipartRepository instance backed by Drizzle ORM. */
|
||||||
|
export const multipartRepository: IMultipartRepository = new DrizzleMultipartRepository();
|
||||||
|
|
||||||
|
/** Singleton ITelegramService instance backed by the bot pool. */
|
||||||
|
export const telegramService: ITelegramService = botPool;
|
||||||
|
|
||||||
|
// ─── Service Singletons ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Singleton ChunkedStorage for large file chunked uploads. */
|
||||||
|
export const chunkedStorage = new ChunkedStorage(
|
||||||
|
fileRepository,
|
||||||
|
filePartRepository,
|
||||||
|
telegramService,
|
||||||
|
);
|
||||||
@@ -1,4 +1,3 @@
|
|||||||
import type { InferInsertModel, InferSelectModel } from 'drizzle-orm';
|
|
||||||
import {
|
import {
|
||||||
bigint,
|
bigint,
|
||||||
boolean,
|
boolean,
|
||||||
@@ -63,15 +62,3 @@ export const fileParts = pgTable('file_parts', {
|
|||||||
etag: text('etag').notNull(),
|
etag: text('etag').notNull(),
|
||||||
createdAt: timestamp('created_at').defaultNow().notNull(),
|
createdAt: timestamp('created_at').defaultNow().notNull(),
|
||||||
});
|
});
|
||||||
|
|
||||||
/** Type representing a file row selected from the database. */
|
|
||||||
export type File = InferSelectModel<typeof files>;
|
|
||||||
|
|
||||||
/** Type representing a file row being inserted into the database. */
|
|
||||||
export type NewFile = InferInsertModel<typeof files>;
|
|
||||||
|
|
||||||
/** Type representing a file part row selected from the database. */
|
|
||||||
export type FilePart = InferSelectModel<typeof fileParts>;
|
|
||||||
|
|
||||||
/** Type representing a file part row being inserted into the database. */
|
|
||||||
export type NewFilePart = InferInsertModel<typeof fileParts>;
|
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import PQueue from 'p-queue';
|
||||||
import { Telegraf } from 'telegraf';
|
import { Telegraf } from 'telegraf';
|
||||||
import type {
|
import type {
|
||||||
ForwardResult,
|
ForwardResult,
|
||||||
@@ -13,38 +14,12 @@ import {
|
|||||||
sendMethodMap,
|
sendMethodMap,
|
||||||
type TelegramMessageResult,
|
type TelegramMessageResult,
|
||||||
} from './types';
|
} from './types';
|
||||||
import { enqueueUpload } from './upload-queue';
|
|
||||||
|
|
||||||
/**
|
const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
* Sleep for a given number of milliseconds.
|
|
||||||
*
|
|
||||||
* Used as a backoff mechanism when all bots in the pool are rate-limited
|
|
||||||
* or when retrying transient Telegram API errors.
|
|
||||||
*
|
|
||||||
* @param ms - Number of milliseconds to sleep.
|
|
||||||
* @returns A promise that resolves after the specified delay.
|
|
||||||
*/
|
|
||||||
const sleep = (ms: number): Promise<void> => {
|
|
||||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Determines whether an error from the Telegram API is likely transient
|
|
||||||
* and worth retrying.
|
|
||||||
*
|
|
||||||
* Transient telegrams errors include: network timeouts, 5xx server errors,
|
|
||||||
* and "Too Many Requests" (429) which is already handled by bot rotation
|
|
||||||
* but is also transient at the network level.
|
|
||||||
*
|
|
||||||
* @param error - The caught error object.
|
|
||||||
* @returns True if the error is likely transient and worth retrying.
|
|
||||||
*/
|
|
||||||
const isTransientError = (error: unknown): boolean => {
|
const isTransientError = (error: unknown): boolean => {
|
||||||
const str = error instanceof Error ? error.message : String(error);
|
const str = error instanceof Error ? error.message : String(error);
|
||||||
const transientPatterns = [
|
const transientPatterns = [
|
||||||
// 'retry after' is deliberately omitted — 429 is handled by
|
|
||||||
// executeWithBotRetry at a deeper layer. Including it here would
|
|
||||||
// cause double-retry (up to 96 attempts per chunk).
|
|
||||||
'timeout',
|
'timeout',
|
||||||
'Timed out',
|
'Timed out',
|
||||||
'etimedout',
|
'etimedout',
|
||||||
@@ -72,120 +47,86 @@ const isTransientError = (error: unknown): boolean => {
|
|||||||
return transientPatterns.some((p) => str.toLowerCase().includes(p.toLowerCase()));
|
return transientPatterns.some((p) => str.toLowerCase().includes(p.toLowerCase()));
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
|
||||||
* Maximum number of retries for transient Telegram API errors
|
|
||||||
* before giving up and propagating the error to the caller.
|
|
||||||
*/
|
|
||||||
const MAX_TRANSIENT_RETRIES = 3;
|
const MAX_TRANSIENT_RETRIES = 3;
|
||||||
|
const MAX_OUTER_RETRIES = 10;
|
||||||
/**
|
|
||||||
* Timeout in milliseconds for individual Telegram API calls.
|
|
||||||
* 120 seconds to accommodate large document uploads.
|
|
||||||
*/
|
|
||||||
const TELEGRAM_API_TIMEOUT_MS = 120_000;
|
const TELEGRAM_API_TIMEOUT_MS = 120_000;
|
||||||
|
|
||||||
/**
|
interface BotEntry {
|
||||||
* Manages a pool of Telegram bots with automatic rotation and rate-limit handling.
|
index: number;
|
||||||
*
|
token: string;
|
||||||
* Distributes uploads across multiple bot tokens to maximise throughput.
|
instance: Telegraf;
|
||||||
* When a bot receives a 429 (rate-limit) error, the pool instantly rotates
|
queue: PQueue;
|
||||||
* to the next available bot. If all bots are rate-limited, a coordinated
|
rateLimitedUntil: number; // 0 = not rate-limited
|
||||||
* sleep is performed before retrying.
|
}
|
||||||
*
|
|
||||||
* Implements the {@link ITelegramService} contract.
|
|
||||||
*/
|
|
||||||
export class BotPool implements ITelegramService {
|
export class BotPool implements ITelegramService {
|
||||||
private readonly bots: Telegraf[];
|
private readonly bots: BotEntry[] = [];
|
||||||
private readonly botTokens: string[];
|
|
||||||
private nextBotIndex = 0;
|
|
||||||
|
|
||||||
/** Create a new BotPool from the application configuration. */
|
|
||||||
constructor() {
|
constructor() {
|
||||||
this.botTokens = Array.from(new Set([config.botToken, ...config.additionalBotTokens]));
|
const tokens = Array.from(new Set(config.botTokens));
|
||||||
this.bots = this.botTokens.map((token) => new Telegraf(token));
|
this.bots = tokens.map((token, index) => ({
|
||||||
|
index,
|
||||||
|
token,
|
||||||
|
instance: new Telegraf(token),
|
||||||
|
queue: new PQueue({ concurrency: config.telegramBotConcurrency }),
|
||||||
|
rateLimitedUntil: 0,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Number of bots in the pool */
|
||||||
|
get size(): number {
|
||||||
|
return this.bots.length;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Claim the next bot index using round-robin rotation.
|
* Select the bot with the fewest pending tasks that isn't rate-limited
|
||||||
*
|
* or in the skip set.
|
||||||
* @returns The index of the selected bot.
|
|
||||||
*/
|
*/
|
||||||
private claimBotIndex(): number {
|
private selectBot(skipIndexes?: Set<number>): BotEntry | null {
|
||||||
const botIndex = this.nextBotIndex;
|
if (this.bots.length === 0) return null;
|
||||||
this.nextBotIndex = (this.nextBotIndex + 1) % this.bots.length;
|
|
||||||
return botIndex;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
let best: BotEntry | null = null;
|
||||||
* Execute a Telegram API action with automatic retry and bot rotation.
|
let bestPending = Infinity;
|
||||||
*
|
|
||||||
* On 429 errors the pool either:
|
|
||||||
* 1. Rotates to the next bot immediately (if another bot is available), or
|
|
||||||
* 2. Sleeps for the required duration after all bots are exhausted, then retries.
|
|
||||||
*
|
|
||||||
* @param action - The action to execute on a bot instance.
|
|
||||||
* @param retries - Number of full-pool retry cycles remaining.
|
|
||||||
* @param attemptedBots - Number of bots attempted in the current cycle.
|
|
||||||
* @returns The result of the action.
|
|
||||||
*/
|
|
||||||
private async executeWithBotRetry<T>(
|
|
||||||
action: (botInstance: Telegraf, botToken: string) => Promise<T>,
|
|
||||||
retries = 5,
|
|
||||||
attemptedBots = 0,
|
|
||||||
): Promise<T> {
|
|
||||||
const botIndex = this.claimBotIndex();
|
|
||||||
const currentBot = this.bots[botIndex];
|
|
||||||
const currentToken = this.botTokens[botIndex];
|
|
||||||
try {
|
|
||||||
// Add timeout to prevent hung API calls from occupying queue slots
|
|
||||||
const result = await Promise.race([
|
|
||||||
action(currentBot, currentToken),
|
|
||||||
new Promise<never>((_, reject) =>
|
|
||||||
setTimeout(
|
|
||||||
() => reject(new Error(`Telegram API timeout after ${TELEGRAM_API_TIMEOUT_MS}ms`)),
|
|
||||||
TELEGRAM_API_TIMEOUT_MS,
|
|
||||||
),
|
|
||||||
),
|
|
||||||
]);
|
|
||||||
return result;
|
|
||||||
} catch (error: unknown) {
|
|
||||||
const errorStr = error instanceof Error ? error.message : String(error);
|
|
||||||
const match = errorStr.match(/retry after (\d+)/i);
|
|
||||||
|
|
||||||
if (match) {
|
for (const bot of this.bots) {
|
||||||
const nextIndex = this.nextBotIndex;
|
if (skipIndexes?.has(bot.index)) continue;
|
||||||
const nextAttemptedBots = attemptedBots + 1;
|
if (bot.rateLimitedUntil > Date.now()) continue;
|
||||||
|
|
||||||
if (nextAttemptedBots < this.bots.length) {
|
const pending = bot.queue.pending + bot.queue.size;
|
||||||
logger.info(
|
if (pending < bestPending) {
|
||||||
`Bot Index ${botIndex} hit 429. Instantly rotating to Bot Index ${nextIndex}...`,
|
bestPending = pending;
|
||||||
);
|
best = bot;
|
||||||
return this.executeWithBotRetry(action, retries, nextAttemptedBots);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (retries > 0) {
|
|
||||||
const seconds = parseInt(match[1], 10);
|
|
||||||
logger.warn(`All bots in the pool are rate-limited. Sleeping for ${seconds} seconds...`, {
|
|
||||||
error: errorStr,
|
|
||||||
});
|
|
||||||
await sleep(seconds);
|
|
||||||
return this.executeWithBotRetry(action, retries - 1, 0);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
throw error;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return best;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute a Telegram API action on a specific bot entry.
|
||||||
|
* Wraps with timeout.
|
||||||
|
*/
|
||||||
|
private async executeBotAction<T>(
|
||||||
|
bot: BotEntry,
|
||||||
|
action: (instance: Telegraf, token: string) => Promise<T>,
|
||||||
|
): Promise<T> {
|
||||||
|
return Promise.race([
|
||||||
|
action(bot.instance, bot.token),
|
||||||
|
new Promise<never>((_, reject) =>
|
||||||
|
setTimeout(
|
||||||
|
() => reject(new Error(`Telegram API timeout after ${TELEGRAM_API_TIMEOUT_MS}ms`)),
|
||||||
|
TELEGRAM_API_TIMEOUT_MS,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Forward a file chunk to the configured Telegram storage chat.
|
* Forward a file chunk to the configured Telegram storage chat.
|
||||||
*
|
*
|
||||||
* The upload is queued (via {@link enqueueUpload}) and executed with
|
* The upload is submitted to the least-loaded bot's queue. If the bot
|
||||||
* automatic bot rotation on rate-limit errors.
|
* returns 429, it is marked rate-limited and the upload retries on the
|
||||||
*
|
* next available bot. If all bots are rate-limited, sleeps before retrying.
|
||||||
* @param fileChunk - The file data (ReadStream, Buffer, or file path).
|
|
||||||
* @param fileName - The original file name.
|
|
||||||
* @param fileType - The file type classification (e.g. "document", "photo").
|
|
||||||
* @returns The Telegram identifiers of the stored file.
|
|
||||||
*/
|
*/
|
||||||
async forwardToStorage(
|
async forwardToStorage(
|
||||||
fileChunk: unknown,
|
fileChunk: unknown,
|
||||||
@@ -193,48 +134,113 @@ export class BotPool implements ITelegramService {
|
|||||||
fileType: string,
|
fileType: string,
|
||||||
): Promise<ForwardResult> {
|
): Promise<ForwardResult> {
|
||||||
let lastError: unknown;
|
let lastError: unknown;
|
||||||
let attempt = 0;
|
const attemptedIndexes = new Set<number>();
|
||||||
|
let transientAttempts = 0;
|
||||||
|
|
||||||
|
// Outer retry loop — up to MAX_OUTER_RETRIES attempts across all bots
|
||||||
|
for (let attempt = 0; attempt < MAX_OUTER_RETRIES; attempt++) {
|
||||||
|
const bot = this.selectBot(attemptedIndexes);
|
||||||
|
|
||||||
|
if (!bot) {
|
||||||
|
// No available bots — either all rate-limited or all attempted
|
||||||
|
if (attemptedIndexes.size > 0) {
|
||||||
|
// All non-rate-limited bots were tried and failed — wait & reset
|
||||||
|
logger.warn('All available bots exhausted, sleeping 5s before retry');
|
||||||
|
await sleep(5000 + Math.random() * 1000);
|
||||||
|
attemptedIndexes.clear();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// All bots rate-limited — wait for the shortest cooldown
|
||||||
|
const earliestCooldown = Math.min(...this.bots.map((b) => b.rateLimitedUntil || Infinity));
|
||||||
|
const waitMs = Math.max(1000, earliestCooldown - Date.now() + 500);
|
||||||
|
logger.warn('All bots rate-limited, waiting', { waitMs });
|
||||||
|
await sleep(waitMs);
|
||||||
|
attemptedIndexes.clear();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
attemptedIndexes.add(bot.index);
|
||||||
|
|
||||||
while (attempt <= MAX_TRANSIENT_RETRIES) {
|
|
||||||
attempt++;
|
|
||||||
try {
|
try {
|
||||||
const result = await this.enqueueUpload<TelegramMessageResult>(async () => {
|
const result = await bot.queue.add(async () => {
|
||||||
const filePayload = { source: fileChunk, filename: fileName };
|
// Inner transient retry loop inside the queue
|
||||||
const sendMethodName = sendMethodMap[fileType] || 'sendDocument';
|
for (let innerRetry = 0; innerRetry <= MAX_TRANSIENT_RETRIES; innerRetry++) {
|
||||||
const payload = buildSendPayload(fileType, fileName);
|
try {
|
||||||
|
const filePayload = { source: fileChunk, filename: fileName };
|
||||||
|
const sendMethodName = sendMethodMap[fileType] || 'sendDocument';
|
||||||
|
const payload = buildSendPayload(fileType, fileName);
|
||||||
|
|
||||||
return this.executeWithBotRetry<TelegramMessageResult>((activeBot) => {
|
const tgResult = await this.executeBotAction<TelegramMessageResult>(
|
||||||
const telegram = activeBot.telegram as unknown as Record<string, SendMethod>;
|
bot,
|
||||||
return telegram[sendMethodName](config.storageChatId, filePayload, payload);
|
(activeBot) => {
|
||||||
});
|
const telegram = activeBot.telegram as unknown as Record<string, SendMethod>;
|
||||||
|
return telegram[sendMethodName](config.storageChatId, filePayload, payload);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const uploadedFile = extractUploadedFile(tgResult, fileType);
|
||||||
|
return {
|
||||||
|
telegramFileId: uploadedFile?.file_id || '',
|
||||||
|
telegramFileUniqueId: uploadedFile?.file_unique_id || '',
|
||||||
|
storageMessageId: tgResult.message_id,
|
||||||
|
};
|
||||||
|
} catch (error: unknown) {
|
||||||
|
const errorStr = error instanceof Error ? error.message : String(error);
|
||||||
|
const retryAfterMatch = errorStr.match(/retry after (\d+)/i);
|
||||||
|
|
||||||
|
if (retryAfterMatch) {
|
||||||
|
// 429 — mark bot rate-limited, throw to outer loop for retry on different bot
|
||||||
|
const seconds = parseInt(retryAfterMatch[1], 10);
|
||||||
|
bot.rateLimitedUntil = Date.now() + seconds * 1000;
|
||||||
|
logger.info(`Bot #${bot.index} rate-limited for ${seconds}s`, {
|
||||||
|
fileName,
|
||||||
|
attempt,
|
||||||
|
});
|
||||||
|
throw error; // caught by outer retry loop
|
||||||
|
}
|
||||||
|
|
||||||
|
if (innerRetry < MAX_TRANSIENT_RETRIES && isTransientError(error)) {
|
||||||
|
const backoffMs = Math.min(1000 * 2 ** innerRetry, 10_000);
|
||||||
|
logger.warn(
|
||||||
|
`Transient error on bot #${bot.index}, retrying (${innerRetry + 1}/${MAX_TRANSIENT_RETRIES})`,
|
||||||
|
{ fileName, error: errorStr, backoffMs },
|
||||||
|
);
|
||||||
|
await sleep(backoffMs);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw error; // non-transient — propagate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error(`Exhausted transient retries on bot #${bot.index}`);
|
||||||
});
|
});
|
||||||
|
|
||||||
const uploadedFile = extractUploadedFile(result, fileType);
|
logger.info('File forwarded to storage', { fileName, message: result.storageMessageId });
|
||||||
logger.info('File forwarded to storage', { fileName, message: result.message_id });
|
return result;
|
||||||
|
|
||||||
return {
|
|
||||||
telegramFileId: uploadedFile?.file_id || '',
|
|
||||||
telegramFileUniqueId: uploadedFile?.file_unique_id || '',
|
|
||||||
storageMessageId: result.message_id,
|
|
||||||
};
|
|
||||||
} catch (error: unknown) {
|
} catch (error: unknown) {
|
||||||
lastError = error;
|
lastError = error;
|
||||||
const errorStr = error instanceof Error ? error.message : String(error);
|
const errorStr = error instanceof Error ? error.message : String(error);
|
||||||
|
const retryAfterMatch = errorStr.match(/retry after (\d+)/i);
|
||||||
|
|
||||||
if (attempt <= MAX_TRANSIENT_RETRIES && isTransientError(error)) {
|
if (retryAfterMatch) {
|
||||||
const backoffMs = Math.min(1000 * 2 ** attempt, 10_000);
|
// 429 catch in outer block: serves as a safety net for errors that
|
||||||
logger.warn(
|
// contain "retry after N" wording but were rethrown from the inner
|
||||||
`Transient error forwarding file, retrying (${attempt}/${MAX_TRANSIENT_RETRIES})`,
|
// queue task's fallback path (e.g., non-429 errors with similar text).
|
||||||
{
|
logger.warn('Retry-after pattern caught in outer loop (safety net)', {
|
||||||
fileName,
|
fileName,
|
||||||
error: errorStr,
|
error: errorStr,
|
||||||
backoffMs,
|
});
|
||||||
},
|
|
||||||
);
|
|
||||||
await sleep(backoffMs);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Transient error at the queue level — retry on next bot
|
||||||
|
if (transientAttempts < MAX_TRANSIENT_RETRIES && isTransientError(error)) {
|
||||||
|
transientAttempts++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Non-transient — give up
|
||||||
logger.error('Failed to forward file to storage', {
|
logger.error('Failed to forward file to storage', {
|
||||||
fileName,
|
fileName,
|
||||||
error: errorStr,
|
error: errorStr,
|
||||||
@@ -244,80 +250,54 @@ export class BotPool implements ITelegramService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Should not reach here — last iteration throws above
|
throw lastError || new Error('Failed to forward file after all retries');
|
||||||
throw lastError;
|
}
|
||||||
|
|
||||||
|
/** Get total effective concurrency across all bots */
|
||||||
|
getEffectiveConcurrency(): number {
|
||||||
|
return this.bots.length * config.telegramBotConcurrency;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Retrieve file metadata from Telegram by file ID.
|
|
||||||
*
|
|
||||||
* Tries all configured bots sequentially; returns info from the first
|
|
||||||
* bot that can retrieve the file. Errors indicating the file belongs
|
|
||||||
* to a different bot are silently skipped.
|
|
||||||
*
|
|
||||||
* @param telegramFileId - The Telegram file_id to look up.
|
|
||||||
* @returns Metadata including size, MIME type, download path, and bot token.
|
|
||||||
*/
|
|
||||||
async getFileInfo(telegramFileId: string): Promise<TelegramFileInfo> {
|
async getFileInfo(telegramFileId: string): Promise<TelegramFileInfo> {
|
||||||
let lastError: unknown;
|
let lastError: unknown;
|
||||||
for (const activeBot of this.bots) {
|
for (const bot of this.bots) {
|
||||||
for (let retry = 0; retry <= MAX_TRANSIENT_RETRIES; retry++) {
|
for (let retry = 0; retry <= MAX_TRANSIENT_RETRIES; retry++) {
|
||||||
try {
|
try {
|
||||||
const result = await activeBot.telegram.getFile(telegramFileId);
|
const result = await bot.instance.telegram.getFile(telegramFileId);
|
||||||
const fileData = result as unknown as Omit<TelegramFileInfo, 'bot_token'>;
|
const fileData = result as unknown as Omit<TelegramFileInfo, 'bot_token'>;
|
||||||
return {
|
return {
|
||||||
file_size: fileData.file_size || 0,
|
file_size: fileData.file_size || 0,
|
||||||
mime_type: fileData.mime_type || 'application/octet-stream',
|
mime_type: fileData.mime_type || 'application/octet-stream',
|
||||||
file_path: fileData.file_path || '',
|
file_path: fileData.file_path || '',
|
||||||
bot_token: activeBot.telegram.token,
|
bot_token: bot.token,
|
||||||
};
|
};
|
||||||
} catch (error: unknown) {
|
} catch (error: unknown) {
|
||||||
lastError = error;
|
lastError = error;
|
||||||
const errorStr = error instanceof Error ? error.message : String(error);
|
const errorStr = error instanceof Error ? error.message : String(error);
|
||||||
// Belongs to a different bot — skip to next bot immediately
|
|
||||||
if (
|
if (
|
||||||
errorStr.includes('wrong file_id') ||
|
errorStr.includes('wrong file_id') ||
|
||||||
errorStr.includes('file is temporarily unavailable')
|
errorStr.includes('file is temporarily unavailable')
|
||||||
) {
|
) {
|
||||||
break; // skip to next bot
|
break;
|
||||||
}
|
}
|
||||||
// Transient — retry on the same bot
|
|
||||||
if (retry < MAX_TRANSIENT_RETRIES && isTransientError(error)) {
|
if (retry < MAX_TRANSIENT_RETRIES && isTransientError(error)) {
|
||||||
const backoffMs = Math.min(1000 * 2 ** (retry + 1), 5_000);
|
const backoffMs = Math.min(1000 * 2 ** (retry + 1), 5_000);
|
||||||
logger.warn(
|
logger.warn(
|
||||||
`Transient error getting file info, retrying bot ${activeBot.telegram.token.slice(0, 8)}... (${retry + 1}/${MAX_TRANSIENT_RETRIES})`,
|
`Transient error getting file info, retrying bot ${bot.token.slice(0, 8)}... (${retry + 1}/${MAX_TRANSIENT_RETRIES})`,
|
||||||
{ telegramFileId, error: errorStr, backoffMs },
|
{ telegramFileId, error: errorStr, backoffMs },
|
||||||
);
|
);
|
||||||
await sleep(backoffMs);
|
await sleep(backoffMs);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
// Non-transient or exhausted retries — try next bot
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.error('Failed to get file info from any bot', {
|
logger.error('Failed to get file info from any bot', {
|
||||||
error: lastError instanceof Error ? lastError.message : String(lastError),
|
error: lastError instanceof Error ? lastError.message : String(lastError),
|
||||||
});
|
});
|
||||||
throw lastError;
|
throw lastError;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Enqueue a task for sequential upload execution.
|
|
||||||
*
|
|
||||||
* Delegates to the shared upload queue to ensure only a limited number
|
|
||||||
* of Telegram uploads run concurrently.
|
|
||||||
*
|
|
||||||
* @param task - An async function performing the upload.
|
|
||||||
* @returns The result of the task.
|
|
||||||
*/
|
|
||||||
enqueueUpload<T>(task: () => Promise<T>): Promise<T> {
|
|
||||||
return enqueueUpload(task);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Singleton BotPool instance initialised from application configuration.
|
|
||||||
*/
|
|
||||||
export const botPool = new BotPool();
|
export const botPool = new BotPool();
|
||||||
|
|||||||
@@ -1,21 +1,17 @@
|
|||||||
import { createReadStream } from 'node:fs';
|
import { createReadStream } from 'node:fs';
|
||||||
import { gzipSync } from 'node:zlib';
|
|
||||||
import { nanoid } from 'nanoid';
|
import { nanoid } from 'nanoid';
|
||||||
import type { File as FileEntity } from '../../domain/entities/file';
|
import type { File as FileEntity } from '../../domain/entities/file';
|
||||||
import type { CompressionAlgorithm, NewFilePart } from '../../domain/entities/file-part';
|
import { buildNewFile } from '../../domain/entities/file-factory';
|
||||||
|
import type { NewFilePart } from '../../domain/entities/file-part';
|
||||||
import type { IFilePartRepository } from '../../domain/ports/file-part-repository';
|
import type { IFilePartRepository } from '../../domain/ports/file-part-repository';
|
||||||
import type { IFileRepository } from '../../domain/ports/file-repository';
|
import type { IFileRepository } from '../../domain/ports/file-repository';
|
||||||
import type { ITelegramService } from '../../domain/ports/telegram-service';
|
import type { ITelegramService } from '../../domain/ports/telegram-service';
|
||||||
import { config } from '../../env';
|
import { config } from '../../env';
|
||||||
import { createGetObjectResponse, type ObjectPartSource } from '../../interfaces/s3/object-stream';
|
import { createGetObjectResponse, type ObjectPartSource } from '../../interfaces/s3/object-stream';
|
||||||
import type { RangeParseResult } from '../../interfaces/s3/range';
|
import type { RangeParseResult } from '../../interfaces/s3/range';
|
||||||
|
import { type CompressionAlgorithm, maybeCompressChunk } from '../../shared/utils/compress';
|
||||||
import { computeHash } from '../../shared/utils/file';
|
import { computeHash } from '../../shared/utils/file';
|
||||||
|
import { asSafeChunkSize } from '../../shared/utils/validation';
|
||||||
/**
|
|
||||||
* Chunk compression algorithm identifier.
|
|
||||||
* `"gzip"` if gzip compression was applied, `null` for uncompressed.
|
|
||||||
*/
|
|
||||||
export type ChunkCompressionAlgorithm = CompressionAlgorithm;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Metadata about a single uploaded chunk (part) stored in Telegram.
|
* Metadata about a single uploaded chunk (part) stored in Telegram.
|
||||||
@@ -34,7 +30,7 @@ export interface ChunkedUploadPart {
|
|||||||
/** Stored (post-compression) size in bytes */
|
/** Stored (post-compression) size in bytes */
|
||||||
storedSizeBytes: number;
|
storedSizeBytes: number;
|
||||||
/** Compression algorithm applied, or null */
|
/** Compression algorithm applied, or null */
|
||||||
compressionAlgorithm: ChunkCompressionAlgorithm;
|
compressionAlgorithm: CompressionAlgorithm;
|
||||||
/** ETag (SHA-256 hash) of the original chunk */
|
/** ETag (SHA-256 hash) of the original chunk */
|
||||||
etag: string;
|
etag: string;
|
||||||
}
|
}
|
||||||
@@ -75,50 +71,6 @@ export interface ChunkedFileInput {
|
|||||||
s3Key?: string | null;
|
s3Key?: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Validate and sanitise the Telegram chunk size.
|
|
||||||
*
|
|
||||||
* @param chunkSizeBytes - The desired chunk size in bytes.
|
|
||||||
* @returns The validated chunk size.
|
|
||||||
* @throws {Error} If the chunk size is not a safe positive integer.
|
|
||||||
*/
|
|
||||||
const asSafeChunkSize = (chunkSizeBytes: number): number => {
|
|
||||||
if (!Number.isSafeInteger(chunkSizeBytes) || chunkSizeBytes <= 0) {
|
|
||||||
throw new Error('Invalid Telegram chunk size');
|
|
||||||
}
|
|
||||||
return chunkSizeBytes;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Optionally compress a chunk with gzip.
|
|
||||||
*
|
|
||||||
* Compression is skipped if:
|
|
||||||
* - The `compress` flag is false.
|
|
||||||
* - The chunk is smaller than `compressionMinSizeBytes`.
|
|
||||||
* - The compressed result is larger than the original.
|
|
||||||
*
|
|
||||||
* @param chunk - The raw chunk buffer.
|
|
||||||
* @param compress - Whether compression is enabled.
|
|
||||||
* @param compressionMinSizeBytes - Minimum chunk size to attempt compression.
|
|
||||||
* @returns The (possibly compressed) bytes and the algorithm used.
|
|
||||||
*/
|
|
||||||
const maybeCompressChunk = (
|
|
||||||
chunk: Buffer,
|
|
||||||
compress: boolean,
|
|
||||||
compressionMinSizeBytes: number,
|
|
||||||
): { bytes: Buffer; compressionAlgorithm: ChunkCompressionAlgorithm } => {
|
|
||||||
if (!compress || chunk.byteLength < compressionMinSizeBytes) {
|
|
||||||
return { bytes: chunk, compressionAlgorithm: null };
|
|
||||||
}
|
|
||||||
|
|
||||||
const gzipped = gzipSync(chunk);
|
|
||||||
if (gzipped.byteLength >= chunk.byteLength) {
|
|
||||||
return { bytes: chunk, compressionAlgorithm: null };
|
|
||||||
}
|
|
||||||
|
|
||||||
return { bytes: gzipped, compressionAlgorithm: 'gzip' };
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Manages chunked storage of large files in Telegram.
|
* Manages chunked storage of large files in Telegram.
|
||||||
*
|
*
|
||||||
@@ -229,31 +181,25 @@ export class ChunkedStorage {
|
|||||||
|
|
||||||
const publicId = nanoid();
|
const publicId = nanoid();
|
||||||
|
|
||||||
const file = await this.fileRepository.create({
|
const file = await this.fileRepository.create(
|
||||||
publicId,
|
buildNewFile({
|
||||||
telegramFileId: firstPart.telegramFileId,
|
publicId,
|
||||||
telegramFileUniqueId: firstPart.telegramFileUniqueId,
|
telegramFileId: firstPart.telegramFileId,
|
||||||
storageChatId: config.storageChatId,
|
telegramFileUniqueId: firstPart.telegramFileUniqueId,
|
||||||
storageMessageId: firstPart.storageMessageId,
|
storageChatId: config.storageChatId,
|
||||||
fileName: input.fileName,
|
storageMessageId: firstPart.storageMessageId,
|
||||||
mimeType: input.mimeType,
|
fileName: input.fileName,
|
||||||
sizeBytes: upload.totalSizeBytes,
|
mimeType: input.mimeType,
|
||||||
fileType: input.fileType,
|
sizeBytes: upload.totalSizeBytes,
|
||||||
uploaderId: input.uploaderId,
|
fileType: input.fileType,
|
||||||
fileHash: upload.fileHash,
|
storageBackend: 'chunked',
|
||||||
archiveTelegramFileId: null,
|
uploaderId: input.uploaderId,
|
||||||
archiveStorageMessageId: null,
|
fileHash: upload.fileHash,
|
||||||
archiveFileName: null,
|
bucketId: input.bucketId,
|
||||||
archiveEntryName: null,
|
s3Key: input.s3Key,
|
||||||
archiveMimeType: null,
|
partCount: upload.parts.length,
|
||||||
archiveSizeBytes: null,
|
}),
|
||||||
bucketId: input.bucketId ?? null,
|
);
|
||||||
s3Key: input.s3Key ?? null,
|
|
||||||
storageBackend: 'chunked',
|
|
||||||
isDeleted: false,
|
|
||||||
multipartUploadId: null,
|
|
||||||
partCount: upload.parts.length,
|
|
||||||
});
|
|
||||||
|
|
||||||
const fileParts: NewFilePart[] = upload.parts.map((part) => ({
|
const fileParts: NewFilePart[] = upload.parts.map((part) => ({
|
||||||
fileId: file.id,
|
fileId: file.id,
|
||||||
|
|||||||
@@ -1,238 +0,0 @@
|
|||||||
import { createReadStream } from 'node:fs';
|
|
||||||
import { nanoid } from 'nanoid';
|
|
||||||
import type { File as FileEntity, NewFile } from '../../domain/entities/file';
|
|
||||||
import type { IFileRepository } from '../../domain/ports/file-repository';
|
|
||||||
import type { ITelegramService } from '../../domain/ports/telegram-service';
|
|
||||||
import { config } from '../../env';
|
|
||||||
import { cleanupTempFile } from '../../shared/utils/file';
|
|
||||||
import { createZip, type ZipEntry } from '../../shared/utils/zip';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Metadata about a prepared upload before it is submitted to the batcher.
|
|
||||||
*/
|
|
||||||
export type PreparedUpload = {
|
|
||||||
/** Temporary file path on disk */
|
|
||||||
tempPath: string;
|
|
||||||
/** SHA-256 hash of the file contents */
|
|
||||||
fileHash: string;
|
|
||||||
/** File size in bytes */
|
|
||||||
sizeBytes: number;
|
|
||||||
/** First bytes of the file for MIME detection */
|
|
||||||
signatureBuffer: Buffer;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A fully materialised file record returned from the batcher.
|
|
||||||
*/
|
|
||||||
export type UploadedFile = FileEntity;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* An item ready for batched upload to Telegram storage.
|
|
||||||
*/
|
|
||||||
export type BatchUploadItem = {
|
|
||||||
/** Prepared upload metadata */
|
|
||||||
prepared: PreparedUpload;
|
|
||||||
/** Original file name */
|
|
||||||
fileName: string;
|
|
||||||
/** MIME type of the file */
|
|
||||||
mimeType: string;
|
|
||||||
/** File type classification (e.g. "document", "photo") */
|
|
||||||
fileType: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Internal pending upload tracking type, extending BatchUploadItem
|
|
||||||
* with resolve/reject callbacks.
|
|
||||||
*/
|
|
||||||
type PendingUpload = BatchUploadItem & {
|
|
||||||
resolve: (file: FileEntity) => void;
|
|
||||||
reject: (error: unknown) => void;
|
|
||||||
};
|
|
||||||
|
|
||||||
/** Time window in milliseconds during which uploads are batched together. */
|
|
||||||
const BATCH_WINDOW_MS = 2000;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Batches multiple file uploads into a single ZIP archive before forwarding
|
|
||||||
* them to Telegram storage. This reduces the number of Telegram API calls
|
|
||||||
* and improves throughput for small-file workloads.
|
|
||||||
*
|
|
||||||
* Injects dependencies via constructor — can be used with any
|
|
||||||
* {@link IFileRepository} and {@link ITelegramService} implementation.
|
|
||||||
*/
|
|
||||||
export class UploadBatcher {
|
|
||||||
private readonly pendingUploads: PendingUpload[] = [];
|
|
||||||
private flushTimer: ReturnType<typeof setTimeout> | null = null;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param fileRepository - Repository for persisting file records.
|
|
||||||
* @param telegramService - Service for forwarding files to Telegram storage.
|
|
||||||
*/
|
|
||||||
constructor(
|
|
||||||
private readonly fileRepository: IFileRepository,
|
|
||||||
private readonly telegramService: ITelegramService,
|
|
||||||
) {}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Build a NewFile record from a batch item and its archive metadata.
|
|
||||||
*
|
|
||||||
* @param item - The batched upload item.
|
|
||||||
* @param entry - ZIP entry metadata for the individual file.
|
|
||||||
* @param archive - Archive-level Telegram storage metadata.
|
|
||||||
* @returns A NewFile record ready for repository insertion.
|
|
||||||
*/
|
|
||||||
private buildUploadedFile(
|
|
||||||
item: BatchUploadItem,
|
|
||||||
entry: ZipEntry,
|
|
||||||
archive: {
|
|
||||||
telegramFileId: string;
|
|
||||||
telegramFileUniqueId: string;
|
|
||||||
storageMessageId: number;
|
|
||||||
fileName: string;
|
|
||||||
sizeBytes: number;
|
|
||||||
},
|
|
||||||
): NewFile {
|
|
||||||
return {
|
|
||||||
publicId: nanoid(),
|
|
||||||
telegramFileId: archive.telegramFileId,
|
|
||||||
telegramFileUniqueId: archive.telegramFileUniqueId,
|
|
||||||
storageChatId: config.storageChatId,
|
|
||||||
storageMessageId: archive.storageMessageId,
|
|
||||||
fileName: item.fileName,
|
|
||||||
mimeType: item.mimeType || 'application/octet-stream',
|
|
||||||
sizeBytes: item.prepared.sizeBytes,
|
|
||||||
fileType: item.fileType,
|
|
||||||
uploaderId: 0,
|
|
||||||
fileHash: item.prepared.fileHash,
|
|
||||||
archiveTelegramFileId: archive.telegramFileId,
|
|
||||||
archiveStorageMessageId: archive.storageMessageId,
|
|
||||||
archiveFileName: archive.fileName,
|
|
||||||
archiveEntryName: entry.entryName,
|
|
||||||
archiveMimeType: 'application/zip',
|
|
||||||
archiveSizeBytes: archive.sizeBytes,
|
|
||||||
bucketId: null,
|
|
||||||
s3Key: null,
|
|
||||||
storageBackend: null,
|
|
||||||
isDeleted: null,
|
|
||||||
multipartUploadId: null,
|
|
||||||
partCount: null,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Flush all pending uploads by zipping them together and sending
|
|
||||||
* the archive to Telegram storage.
|
|
||||||
*/
|
|
||||||
private async flushUploads(): Promise<void> {
|
|
||||||
if (this.flushTimer) {
|
|
||||||
clearTimeout(this.flushTimer);
|
|
||||||
this.flushTimer = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const batch = this.pendingUploads.splice(0);
|
|
||||||
if (batch.length === 0) return;
|
|
||||||
|
|
||||||
let zipTempPath: string | null = null;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const zip = await createZip(
|
|
||||||
batch.map((item) => ({ tempPath: item.prepared.tempPath, fileName: item.fileName })),
|
|
||||||
);
|
|
||||||
zipTempPath = zip.tempPath;
|
|
||||||
const archiveFileName = `filedrop-${nanoid()}.zip`;
|
|
||||||
const archiveResult = await this.telegramService.forwardToStorage(
|
|
||||||
createReadStream(zip.tempPath),
|
|
||||||
archiveFileName,
|
|
||||||
'document',
|
|
||||||
);
|
|
||||||
|
|
||||||
const newFileInputs = batch.map((item, index) =>
|
|
||||||
this.buildUploadedFile(item, zip.entries[index], {
|
|
||||||
telegramFileId: archiveResult.telegramFileId,
|
|
||||||
telegramFileUniqueId: archiveResult.telegramFileUniqueId,
|
|
||||||
storageMessageId: archiveResult.storageMessageId,
|
|
||||||
fileName: archiveFileName,
|
|
||||||
sizeBytes: zip.sizeBytes,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
// Persist each file record through the repository
|
|
||||||
const createdFiles = await Promise.all(
|
|
||||||
newFileInputs.map((input) => this.fileRepository.create(input)),
|
|
||||||
);
|
|
||||||
|
|
||||||
for (let i = 0; i < batch.length; i++) {
|
|
||||||
batch[i].resolve(createdFiles[i]);
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
for (const item of batch) {
|
|
||||||
item.reject(error);
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
await Promise.all(batch.map((item) => cleanupTempFile(item.prepared.tempPath)));
|
|
||||||
if (zipTempPath) await cleanupTempFile(zipTempPath);
|
|
||||||
// Reschedule timer if new items arrived during async processing
|
|
||||||
if (this.pendingUploads.length > 0 && !this.flushTimer) {
|
|
||||||
this.flushTimer = setTimeout(() => {
|
|
||||||
void this.flushUploads();
|
|
||||||
}, BATCH_WINDOW_MS);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Calculate total size of all pending uploads in bytes.
|
|
||||||
*
|
|
||||||
* @returns The sum of all pending file sizes.
|
|
||||||
*/
|
|
||||||
private getPendingSize(): number {
|
|
||||||
return this.pendingUploads.reduce((total, item) => total + item.prepared.sizeBytes, 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Enqueue a prepared upload for batched processing.
|
|
||||||
*
|
|
||||||
* The upload is held for up to {@link BATCH_WINDOW_MS} milliseconds
|
|
||||||
* (or until the batch size/byte thresholds in config are exceeded)
|
|
||||||
* before being flushed to Telegram storage.
|
|
||||||
*
|
|
||||||
* @param item - The prepared upload item to enqueue.
|
|
||||||
* @returns A promise that resolves with the fully created File record.
|
|
||||||
*/
|
|
||||||
enqueuePreparedUpload(item: BatchUploadItem): Promise<FileEntity> {
|
|
||||||
return new Promise<FileEntity>((resolve, reject) => {
|
|
||||||
this.pendingUploads.push({ ...item, resolve, reject });
|
|
||||||
|
|
||||||
if (!this.flushTimer) {
|
|
||||||
this.flushTimer = setTimeout(() => {
|
|
||||||
void this.flushUploads();
|
|
||||||
}, BATCH_WINDOW_MS);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (
|
|
||||||
this.pendingUploads.length >= config.batchMaxItems ||
|
|
||||||
this.getPendingSize() >= config.batchMaxSizeBytes
|
|
||||||
) {
|
|
||||||
void this.flushUploads();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Immediately flush all pending uploads, regardless of batch size.
|
|
||||||
*
|
|
||||||
* @returns A promise that resolves when the flush is complete.
|
|
||||||
*/
|
|
||||||
async flushPendingUploads(): Promise<void> {
|
|
||||||
await this.flushUploads();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the number of uploads currently waiting in the batch queue.
|
|
||||||
*
|
|
||||||
* @returns The pending upload count.
|
|
||||||
*/
|
|
||||||
getPendingUploadCount(): number {
|
|
||||||
return this.pendingUploads.length;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,100 +0,0 @@
|
|||||||
import PQueue from 'p-queue';
|
|
||||||
import { config } from '../../env';
|
|
||||||
import logger from '../../shared/logger/index';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Maximum number of pending (queued + in-flight) upload tasks before
|
|
||||||
* new submissions are rejected. Prevents unbounded memory growth when
|
|
||||||
* Telegram is slow or unavailable.
|
|
||||||
*/
|
|
||||||
const MAX_QUEUE_PENDING = 1000;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* P-queue instance for serialising Telegram upload tasks.
|
|
||||||
*
|
|
||||||
* Concurrency is governed by {@link config.uploadConcurrency}.
|
|
||||||
* Built-in logging emits warnings when the queue grows beyond 5 pending items.
|
|
||||||
*/
|
|
||||||
const uploadQueue = new PQueue({
|
|
||||||
concurrency: config.uploadConcurrency,
|
|
||||||
});
|
|
||||||
|
|
||||||
/* Monitor queue growth and emit warnings for large backlogs */
|
|
||||||
uploadQueue.on('add', () => {
|
|
||||||
const stats = getQueueStats();
|
|
||||||
if (stats.size > 5) {
|
|
||||||
logger.warn('Upload queue building up', {
|
|
||||||
pending: stats.pending,
|
|
||||||
size: stats.size,
|
|
||||||
max: MAX_QUEUE_PENDING,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
uploadQueue.on('next', () => {
|
|
||||||
const stats = getQueueStats();
|
|
||||||
logger.debug('Processing next upload', { pending: stats.pending, size: stats.size });
|
|
||||||
});
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Enqueue an upload task to be executed by the queue.
|
|
||||||
*
|
|
||||||
* Tasks are executed in FIFO order, subject to the concurrency limit.
|
|
||||||
*
|
|
||||||
* @param task - An async function representing the upload operation.
|
|
||||||
* @returns A promise that resolves with the task's result.
|
|
||||||
*/
|
|
||||||
export const enqueueUpload = <T>(task: () => Promise<T>): Promise<T> => {
|
|
||||||
const stats = getQueueStats();
|
|
||||||
if (stats.pending + stats.size > MAX_QUEUE_PENDING) {
|
|
||||||
return Promise.reject(
|
|
||||||
new Error(
|
|
||||||
`Upload queue full (${stats.pending + stats.size} pending, max ${MAX_QUEUE_PENDING})`,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return uploadQueue.add(task);
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get current queue statistics.
|
|
||||||
*
|
|
||||||
* @returns An object with `pending` (actively executing) and `size` (waiting) counts.
|
|
||||||
*/
|
|
||||||
export const getQueueStats = (): { pending: number; size: number } => ({
|
|
||||||
pending: uploadQueue.pending,
|
|
||||||
size: uploadQueue.size,
|
|
||||||
});
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the number of items waiting in the queue (not yet started).
|
|
||||||
*
|
|
||||||
* @returns The number of queued items.
|
|
||||||
*/
|
|
||||||
export const getQueueSize = (): number => uploadQueue.size;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the number of items currently being processed.
|
|
||||||
*
|
|
||||||
* @returns The number of pending (in-flight) items.
|
|
||||||
*/
|
|
||||||
export const getPendingCount = (): number => uploadQueue.pending;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear all pending items and wait for in-flight ones to finish.
|
|
||||||
*
|
|
||||||
* @returns A promise that resolves when the queue is idle after clearing.
|
|
||||||
*/
|
|
||||||
export const clearQueue = async (): Promise<void> => {
|
|
||||||
uploadQueue.clear();
|
|
||||||
await uploadQueue.onIdle();
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Wait for the queue to become idle (all tasks finished).
|
|
||||||
*
|
|
||||||
* @returns A promise that resolves when no tasks are pending or in-flight.
|
|
||||||
*/
|
|
||||||
export const waitForQueue = async (): Promise<void> => {
|
|
||||||
await uploadQueue.onIdle();
|
|
||||||
};
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
import { nanoid } from 'nanoid';
|
import { nanoid } from 'nanoid';
|
||||||
import { type Context, Telegraf } from 'telegraf';
|
import { type Context, Telegraf } from 'telegraf';
|
||||||
import type { NewFile } from '../../domain/entities/file';
|
import { buildNewFile } from '../../domain/entities/file-factory';
|
||||||
import type { IFileRepository } from '../../domain/ports/file-repository';
|
import type { IFileRepository } from '../../domain/ports/file-repository';
|
||||||
import type { ITelegramService } from '../../domain/ports/telegram-service';
|
import type { ITelegramService } from '../../domain/ports/telegram-service';
|
||||||
import { config } from '../../env';
|
import { config } from '../../env';
|
||||||
@@ -8,6 +8,7 @@ import { DrizzleFileRepository } from '../../infrastructure/persistence/reposito
|
|||||||
import { botPool } from '../../infrastructure/telegram/bot-pool';
|
import { botPool } from '../../infrastructure/telegram/bot-pool';
|
||||||
import logger from '../../shared/logger/index';
|
import logger from '../../shared/logger/index';
|
||||||
import {
|
import {
|
||||||
|
checkFileSize,
|
||||||
detectFileType,
|
detectFileType,
|
||||||
extractFileFromMessage,
|
extractFileFromMessage,
|
||||||
getErrorMessage,
|
getErrorMessage,
|
||||||
@@ -92,7 +93,7 @@ export async function startBot(
|
|||||||
const fileRepo = deps.fileRepo ?? new DrizzleFileRepository();
|
const fileRepo = deps.fileRepo ?? new DrizzleFileRepository();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const bot = new Telegraf(config.botToken);
|
const bot = new Telegraf(config.botTokens[0]);
|
||||||
|
|
||||||
bot.command('start', async (ctx) => {
|
bot.command('start', async (ctx) => {
|
||||||
await ctx.reply(
|
await ctx.reply(
|
||||||
@@ -127,10 +128,10 @@ export async function startBot(
|
|||||||
ctx.message.voice?.file_name ||
|
ctx.message.voice?.file_name ||
|
||||||
'file';
|
'file';
|
||||||
|
|
||||||
const maxSize = getFileSizeLimit(fileType);
|
if (!checkFileSize(fileSize, fileType)) {
|
||||||
|
return ctx.reply(
|
||||||
if (fileSize > maxSize) {
|
`File size exceeds ${getFileSizeLimit(fileType) / (1024 * 1024)}MB limit`,
|
||||||
return ctx.reply(`File size exceeds ${maxSize / (1024 * 1024)}MB limit`);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const existing = await fileRepo.findByUniqueId(fileObj.file_unique_id);
|
const existing = await fileRepo.findByUniqueId(fileObj.file_unique_id);
|
||||||
@@ -149,33 +150,21 @@ export async function startBot(
|
|||||||
const result = await telegramService.forwardToStorage(file_id, fileName, fileType);
|
const result = await telegramService.forwardToStorage(file_id, fileName, fileType);
|
||||||
const publicId = nanoid();
|
const publicId = nanoid();
|
||||||
|
|
||||||
const uploaded: NewFile = {
|
await fileRepo.create(
|
||||||
publicId,
|
buildNewFile({
|
||||||
telegramFileId: result.telegramFileId,
|
publicId,
|
||||||
telegramFileUniqueId: result.telegramFileUniqueId,
|
telegramFileId: result.telegramFileId,
|
||||||
storageChatId: config.storageChatId,
|
telegramFileUniqueId: result.telegramFileUniqueId,
|
||||||
storageMessageId: result.storageMessageId,
|
storageChatId: config.storageChatId,
|
||||||
fileName,
|
storageMessageId: result.storageMessageId,
|
||||||
mimeType: mime_type || 'application/octet-stream',
|
fileName,
|
||||||
sizeBytes: fileSize,
|
mimeType: mime_type || 'application/octet-stream',
|
||||||
fileType,
|
sizeBytes: fileSize,
|
||||||
uploaderId: ctx.from.id,
|
fileType,
|
||||||
fileHash: null,
|
uploaderId: ctx.from.id,
|
||||||
archiveTelegramFileId: null,
|
storageBackend: 'telegram',
|
||||||
archiveStorageMessageId: null,
|
}),
|
||||||
archiveFileName: null,
|
);
|
||||||
archiveEntryName: null,
|
|
||||||
archiveMimeType: null,
|
|
||||||
archiveSizeBytes: null,
|
|
||||||
bucketId: null,
|
|
||||||
s3Key: null,
|
|
||||||
storageBackend: 'telegram',
|
|
||||||
isDeleted: false,
|
|
||||||
multipartUploadId: null,
|
|
||||||
partCount: null,
|
|
||||||
};
|
|
||||||
|
|
||||||
await fileRepo.create(uploaded);
|
|
||||||
|
|
||||||
await replyWithDownloadUrl(ctx, publicId);
|
await replyWithDownloadUrl(ctx, publicId);
|
||||||
|
|
||||||
@@ -197,7 +186,9 @@ export async function startBot(
|
|||||||
|
|
||||||
await bot.launch();
|
await bot.launch();
|
||||||
|
|
||||||
logger.info('Telegram bot started', { botToken: `${config.botToken?.substring(0, 10)}...` });
|
logger.info('Telegram bot started', {
|
||||||
|
botToken: `${config.botTokens[0]?.substring(0, 10)}...`,
|
||||||
|
});
|
||||||
|
|
||||||
return bot;
|
return bot;
|
||||||
} catch (error: unknown) {
|
} catch (error: unknown) {
|
||||||
|
|||||||
@@ -4,14 +4,14 @@ import {
|
|||||||
createLogoutUseCase,
|
createLogoutUseCase,
|
||||||
createMeUseCase,
|
createMeUseCase,
|
||||||
} from '../../../application/use-cases/authenticate';
|
} from '../../../application/use-cases/authenticate';
|
||||||
import { config } from '../../../config/index';
|
import { config } from '../../../env';
|
||||||
import {
|
import {
|
||||||
checkBearerToken,
|
checkBearerToken,
|
||||||
clearSessionCookie,
|
clearSessionCookie,
|
||||||
createSessionCookie,
|
createSessionCookie,
|
||||||
getAuthSession,
|
getAuthSession,
|
||||||
isAuthEnabled,
|
isAuthEnabled,
|
||||||
} from '../../../utils/auth';
|
} from '../middleware/auth';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Helper that builds a JSON Response with optional extra headers.
|
* Helper that builds a JSON Response with optional extra headers.
|
||||||
|
|||||||
@@ -2,11 +2,11 @@ import { createReadStream } from 'node:fs';
|
|||||||
import { nanoid } from 'nanoid';
|
import { nanoid } from 'nanoid';
|
||||||
import type { TelegramFileInfo } from '../../../domain/ports/telegram-service';
|
import type { TelegramFileInfo } from '../../../domain/ports/telegram-service';
|
||||||
import { fileInfoCache } from '../../../infrastructure/cache/index';
|
import { fileInfoCache } from '../../../infrastructure/cache/index';
|
||||||
|
import { chunkedStorage, fileRepository } from '../../../infrastructure/di';
|
||||||
import { botPool } from '../../../infrastructure/telegram/bot-pool';
|
import { botPool } from '../../../infrastructure/telegram/bot-pool';
|
||||||
import logger from '../../../shared/logger/index';
|
import logger from '../../../shared/logger/index';
|
||||||
import { cleanupTempFile, formatCreatedAt, getErrorMessage } from '../../../shared/utils/file';
|
import { cleanupTempFile, formatCreatedAt, getErrorMessage } from '../../../shared/utils/file';
|
||||||
import { createChunkedObjectResponse } from '../../../utils/chunked-storage';
|
import { locateZipEntry } from '../../../shared/utils/zip';
|
||||||
import { locateZipEntry } from '../../../utils/zip';
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Extended Request type that includes route parameter access.
|
* Extended Request type that includes route parameter access.
|
||||||
@@ -19,14 +19,6 @@ type RequestWithParams = Request & {
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
|
||||||
* Maps a string into a `string | string[]` for cookie append operations.
|
|
||||||
*
|
|
||||||
* @param value - The string value to wrap.
|
|
||||||
* @returns The value as a single-element tuple.
|
|
||||||
*/
|
|
||||||
const _asArray = (value: string): string[] => [value];
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resolves Telegram file metadata for a given file ID, using the in-memory
|
* Resolves Telegram file metadata for a given file ID, using the in-memory
|
||||||
* cache to avoid repeated API calls to Telegram.
|
* cache to avoid repeated API calls to Telegram.
|
||||||
@@ -103,8 +95,7 @@ export const handleFileRedirect = async (req: RequestWithParams): Promise<Respon
|
|||||||
return fail(400, 'Missing file id');
|
return fail(400, 'Missing file id');
|
||||||
}
|
}
|
||||||
|
|
||||||
const { findFileByPublicId } = await import('../../../db/files');
|
const file = await fileRepository.findByPublicId(publicId);
|
||||||
const file = await findFileByPublicId(publicId);
|
|
||||||
if (!file) {
|
if (!file) {
|
||||||
logger.warn('File not found', { publicId });
|
logger.warn('File not found', { publicId });
|
||||||
return fail(404, 'File not found');
|
return fail(404, 'File not found');
|
||||||
@@ -115,7 +106,7 @@ export const handleFileRedirect = async (req: RequestWithParams): Promise<Respon
|
|||||||
return fail(501, 'Archive entry extraction is not supported for chunked files');
|
return fail(501, 'Archive entry extraction is not supported for chunked files');
|
||||||
}
|
}
|
||||||
const range = { type: 'none' as const };
|
const range = { type: 'none' as const };
|
||||||
return createChunkedObjectResponse({ file, range, reqId: '' });
|
return chunkedStorage.createChunkedObjectResponse({ file, range, reqId: '' });
|
||||||
}
|
}
|
||||||
|
|
||||||
const archiveEntryName = file.archiveEntryName;
|
const archiveEntryName = file.archiveEntryName;
|
||||||
@@ -193,8 +184,7 @@ export const handleFileInfo = async (req: RequestWithParams): Promise<Response>
|
|||||||
return fail(400, 'Missing file id');
|
return fail(400, 'Missing file id');
|
||||||
}
|
}
|
||||||
|
|
||||||
const { findFileByPublicId } = await import('../../../db/files');
|
const file = await fileRepository.findByPublicId(publicId);
|
||||||
const file = await findFileByPublicId(publicId);
|
|
||||||
if (!file) {
|
if (!file) {
|
||||||
logger.warn('File not found', { publicId });
|
logger.warn('File not found', { publicId });
|
||||||
return fail(404, 'File not found');
|
return fail(404, 'File not found');
|
||||||
|
|||||||
@@ -1,15 +1,60 @@
|
|||||||
|
import { existsSync } from 'node:fs';
|
||||||
|
import { dirname, join } from 'node:path';
|
||||||
import type { BunFile } from 'bun';
|
import type { BunFile } from 'bun';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Maximum number of parent directories to walk up when locating home.html.
|
||||||
|
* Deep enough for the dev layout (controllers/ -> src/ = 4 levels) with margin.
|
||||||
|
*/
|
||||||
|
const MAX_PARENT_WALK = 6;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolves the absolute path to `home.html` by walking up from `startDir`.
|
||||||
|
*
|
||||||
|
* The file lives at different depths depending on how the app is run:
|
||||||
|
* - Dev (`bun --hot src/index.ts`): `import.meta.dir` is
|
||||||
|
* `src/interfaces/http/controllers/`, home.html lives at `src/home.html`
|
||||||
|
* (4 levels up).
|
||||||
|
* - Prod (bundled `dist/index.js`): `import.meta.dir` is
|
||||||
|
* `$out/share/teleuploader/dist/`, home.html lives next to dist/
|
||||||
|
* (1 level up, per flake.nix installPhase).
|
||||||
|
*
|
||||||
|
* Returns the first existing candidate, or `null` if none is found within
|
||||||
|
* the walk bound.
|
||||||
|
*
|
||||||
|
* @param startDir - Directory to start the search from (typically `import.meta.dir`).
|
||||||
|
* @param maxDepth - Maximum number of parent directories to walk (default: 6).
|
||||||
|
* @returns Absolute path to home.html, or `null` if not found.
|
||||||
|
*/
|
||||||
|
export const resolveHomeHtml = (startDir: string, maxDepth = MAX_PARENT_WALK): string | null => {
|
||||||
|
let dir = startDir;
|
||||||
|
for (let depth = 0; depth <= maxDepth; depth++) {
|
||||||
|
const candidate = join(dir, 'home.html');
|
||||||
|
if (existsSync(candidate)) return candidate;
|
||||||
|
const parent = dirname(dir);
|
||||||
|
if (parent === dir) break;
|
||||||
|
dir = parent;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Handles the home/dashboard page request.
|
* Handles the home/dashboard page request.
|
||||||
*
|
*
|
||||||
* Reads the `home.html` file from the adjacent directory and serves it as
|
* Reads the `home.html` file and serves it as an HTML response with UTF-8
|
||||||
* an HTML response with UTF-8 charset.
|
* charset. Fails fast with a clear error when the file cannot be located
|
||||||
|
* instead of letting Bun.serve swallow the ENOENT into a bare 500.
|
||||||
*
|
*
|
||||||
* @returns An HTML response containing the home page content.
|
* @returns An HTML response containing the home page content.
|
||||||
*/
|
*/
|
||||||
export const handleHome = async (): Promise<Response> => {
|
export const handleHome = async (): Promise<Response> => {
|
||||||
const html = await (Bun.file(`${import.meta.dir}/home.html`) as BunFile).text();
|
const homeHtml = resolveHomeHtml(import.meta.dir);
|
||||||
|
if (!homeHtml) {
|
||||||
|
throw new Error(
|
||||||
|
`home.html not found — looked up from ${import.meta.dir} and ${MAX_PARENT_WALK} parent dirs`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const html = await (Bun.file(homeHtml) as BunFile).text();
|
||||||
return new Response(html, {
|
return new Response(html, {
|
||||||
status: 200,
|
status: 200,
|
||||||
headers: {
|
headers: {
|
||||||
|
|||||||
@@ -136,6 +136,10 @@
|
|||||||
}
|
}
|
||||||
.auth-card button:disabled { opacity: 0.7; cursor: wait; }
|
.auth-card button:disabled { opacity: 0.7; cursor: wait; }
|
||||||
.auth-error { color: var(--danger); font-size: 0.85rem; margin-bottom: 12px; }
|
.auth-error { color: var(--danger); font-size: 0.85rem; margin-bottom: 12px; }
|
||||||
|
.readonly-badge {
|
||||||
|
font-size: 0.75rem; color: var(--text2); background: var(--bg2);
|
||||||
|
border: 1px solid var(--border); border-radius: 999px; padding: 2px 10px;
|
||||||
|
}
|
||||||
</style>
|
</style>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
@@ -153,9 +157,11 @@
|
|||||||
<select id="bucketSelect" onchange="window.switchBucket(this.value)">
|
<select id="bucketSelect" onchange="window.switchBucket(this.value)">
|
||||||
<option value="">— Select bucket —</option>
|
<option value="">— Select bucket —</option>
|
||||||
</select>
|
</select>
|
||||||
<button type="button" onclick="window.showCreateBucketModal()">+ New</button>
|
<button id="newBucketBtn" type="button" onclick="window.showCreateBucketModal()">+ New</button>
|
||||||
<button type="button" onclick="window.showCredentialsModal()" title="S3 Credentials">🔑</button>
|
<button id="credsBtn" type="button" onclick="window.showCredentialsModal()" title="S3 Credentials">🔑</button>
|
||||||
|
<button id="loginBtn" type="button" onclick="window.showAuthScreen()" style="display:none">Login</button>
|
||||||
<button id="logoutBtn" type="button" onclick="window.logout()" style="display:none">Logout</button>
|
<button id="logoutBtn" type="button" onclick="window.logout()" style="display:none">Logout</button>
|
||||||
|
<span id="readonlyBadge" class="readonly-badge" style="display:none">👀 read-only</span>
|
||||||
<span class="spacer"></span>
|
<span class="spacer"></span>
|
||||||
<div class="search">
|
<div class="search">
|
||||||
<input id="searchInput" type="text" placeholder="Filter prefix..." oninput="window.debouncedSearch()">
|
<input id="searchInput" type="text" placeholder="Filter prefix..." oninput="window.debouncedSearch()">
|
||||||
@@ -179,6 +185,7 @@
|
|||||||
</div>
|
</div>
|
||||||
<script>
|
<script>
|
||||||
let currentBucket = null, currentPrefix = '', currentObjects = [], currentPrefixes = [], allBuckets = [], searchTimer = null;
|
let currentBucket = null, currentPrefix = '', currentObjects = [], currentPrefixes = [], allBuckets = [], searchTimer = null;
|
||||||
|
let isAdmin = false;
|
||||||
const setAuthError = (message) => {
|
const setAuthError = (message) => {
|
||||||
const errorEl = document.getElementById('authError');
|
const errorEl = document.getElementById('authError');
|
||||||
errorEl.textContent = message;
|
errorEl.textContent = message;
|
||||||
@@ -186,25 +193,36 @@
|
|||||||
};
|
};
|
||||||
const showAuthScreen = () => {
|
const showAuthScreen = () => {
|
||||||
document.getElementById('authScreen').style.display = 'flex';
|
document.getElementById('authScreen').style.display = 'flex';
|
||||||
document.getElementById('logoutBtn').style.display = 'none';
|
|
||||||
setTimeout(() => document.getElementById('authTokenInput')?.focus(), 50);
|
setTimeout(() => document.getElementById('authTokenInput')?.focus(), 50);
|
||||||
};
|
};
|
||||||
const hideAuthScreen = (showLogout) => {
|
const hideAuthScreen = () => {
|
||||||
document.getElementById('authScreen').style.display = 'none';
|
document.getElementById('authScreen').style.display = 'none';
|
||||||
document.getElementById('logoutBtn').style.display = showLogout ? 'inline-block' : 'none';
|
|
||||||
};
|
};
|
||||||
|
// Applies the admin/read-only UI state based on isAdmin.
|
||||||
|
const applyAdminUI = () => {
|
||||||
|
document.getElementById('newBucketBtn').style.display = isAdmin ? 'inline-block' : 'none';
|
||||||
|
document.getElementById('credsBtn').style.display = isAdmin ? 'inline-block' : 'none';
|
||||||
|
document.getElementById('loginBtn').style.display = isAdmin ? 'none' : 'inline-block';
|
||||||
|
document.getElementById('logoutBtn').style.display = isAdmin ? 'inline-block' : 'none';
|
||||||
|
document.getElementById('readonlyBadge').style.display = isAdmin ? 'none' : 'inline-block';
|
||||||
|
// Dropzone (upload) is admin-only.
|
||||||
|
document.getElementById('dropzone').style.display = isAdmin && currentBucket ? 'block' : 'none';
|
||||||
|
if (currentObjects.length || currentPrefixes.length) renderFileList();
|
||||||
|
};
|
||||||
|
// Non-blocking auth check: read-only visitors still get the file browser.
|
||||||
const checkAuth = async () => {
|
const checkAuth = async () => {
|
||||||
try {
|
try {
|
||||||
const res = await fetch('/api/v1/auth/me');
|
const res = await fetch('/api/v1/auth/me');
|
||||||
if (res.ok) { hideAuthScreen(true); return true; }
|
if (res.ok) { isAdmin = true; }
|
||||||
if (res.status === 401) { showAuthScreen(); return false; }
|
else if (res.status === 401) { isAdmin = false; }
|
||||||
if (res.status === 404) { hideAuthScreen(false); return true; }
|
else if (res.status === 404) { isAdmin = true; } // auth disabled — full access
|
||||||
setAuthError('Unable to verify login status. Please try again.');
|
else { isAdmin = false; }
|
||||||
showAuthScreen(); return false;
|
|
||||||
} catch {
|
} catch {
|
||||||
setAuthError('Network error while checking login status.');
|
isAdmin = false;
|
||||||
showAuthScreen(); return false;
|
|
||||||
}
|
}
|
||||||
|
hideAuthScreen();
|
||||||
|
applyAdminUI();
|
||||||
|
return isAdmin;
|
||||||
};
|
};
|
||||||
const handleLogin = async () => {
|
const handleLogin = async () => {
|
||||||
const input = document.getElementById('authTokenInput');
|
const input = document.getElementById('authTokenInput');
|
||||||
@@ -217,7 +235,7 @@
|
|||||||
method: 'POST', headers: { 'content-type': 'application/json' },
|
method: 'POST', headers: { 'content-type': 'application/json' },
|
||||||
body: JSON.stringify({ token }),
|
body: JSON.stringify({ token }),
|
||||||
});
|
});
|
||||||
if (res.ok) { hideAuthScreen(true); input.value = ''; await loadBuckets(); return; }
|
if (res.ok) { isAdmin = true; hideAuthScreen(); input.value = ''; applyAdminUI(); await loadBuckets(); return; }
|
||||||
const body = await res.json().catch(() => ({ error: 'Login failed' }));
|
const body = await res.json().catch(() => ({ error: 'Login failed' }));
|
||||||
setAuthError(body.error || 'Login failed');
|
setAuthError(body.error || 'Login failed');
|
||||||
} catch {
|
} catch {
|
||||||
@@ -228,11 +246,8 @@
|
|||||||
};
|
};
|
||||||
const logout = async () => {
|
const logout = async () => {
|
||||||
await fetch('/api/v1/auth/logout', { method: 'POST' }).catch(() => {});
|
await fetch('/api/v1/auth/logout', { method: 'POST' }).catch(() => {});
|
||||||
currentBucket = null; currentPrefix = ''; currentObjects = []; currentPrefixes = [];
|
isAdmin = false;
|
||||||
document.getElementById('bucketSelect').innerHTML = '<option value="">— Select bucket —</option>';
|
applyAdminUI();
|
||||||
document.getElementById('fileList').innerHTML = '<div class="empty"><h2>Logged out</h2><p>Enter the admin token to continue.</p></div>';
|
|
||||||
document.getElementById('dropzone').style.display = 'none';
|
|
||||||
showAuthScreen();
|
|
||||||
};
|
};
|
||||||
const api = async (path, opts = {}) => {
|
const api = async (path, opts = {}) => {
|
||||||
const res = await fetch(path, opts);
|
const res = await fetch(path, opts);
|
||||||
@@ -249,11 +264,13 @@
|
|||||||
};
|
};
|
||||||
const switchBucket = async (name) => {
|
const switchBucket = async (name) => {
|
||||||
currentBucket = name || null; currentPrefix = '';
|
currentBucket = name || null; currentPrefix = '';
|
||||||
if (name) { await loadObjects(); document.getElementById('dropzone').style.display = 'block'; }
|
if (name) { await loadObjects(); }
|
||||||
else {
|
else {
|
||||||
document.getElementById('fileList').innerHTML = '<div class="empty"><h2>Select a bucket</h2><p>Choose a bucket from the dropdown above.</p></div>';
|
document.getElementById('fileList').innerHTML = '<div class="empty"><h2>Select a bucket</h2><p>Choose a bucket from the dropdown above.</p></div>';
|
||||||
document.getElementById('breadcrumb').style.display = 'none'; document.getElementById('dropzone').style.display = 'none';
|
document.getElementById('breadcrumb').style.display = 'none';
|
||||||
}
|
}
|
||||||
|
// Dropzone (upload) is admin-only.
|
||||||
|
document.getElementById('dropzone').style.display = isAdmin && currentBucket ? 'block' : 'none';
|
||||||
};
|
};
|
||||||
const renderBreadcrumb = () => {
|
const renderBreadcrumb = () => {
|
||||||
const bc = document.getElementById('breadcrumb');
|
const bc = document.getElementById('breadcrumb');
|
||||||
@@ -278,7 +295,12 @@
|
|||||||
};
|
};
|
||||||
const renderFileList = () => {
|
const renderFileList = () => {
|
||||||
const container = document.getElementById('fileList');
|
const container = document.getElementById('fileList');
|
||||||
if (currentPrefixes.length === 0 && currentObjects.length === 0) { container.innerHTML = '<div class="empty"><h2>This bucket is empty</h2><p>Drop files here to upload.</p></div>'; return; }
|
if (currentPrefixes.length === 0 && currentObjects.length === 0) {
|
||||||
|
container.innerHTML = isAdmin
|
||||||
|
? '<div class="empty"><h2>This bucket is empty</h2><p>Drop files here to upload.</p></div>'
|
||||||
|
: '<div class="empty"><h2>This bucket is empty</h2></div>';
|
||||||
|
return;
|
||||||
|
}
|
||||||
let html = '';
|
let html = '';
|
||||||
for (const prefix of currentPrefixes) {
|
for (const prefix of currentPrefixes) {
|
||||||
const displayName = prefix.replace(currentPrefix, '');
|
const displayName = prefix.replace(currentPrefix, '');
|
||||||
@@ -286,7 +308,9 @@
|
|||||||
}
|
}
|
||||||
for (const obj of currentObjects) {
|
for (const obj of currentObjects) {
|
||||||
const displayName = obj.key.replace(currentPrefix, '');
|
const displayName = obj.key.replace(currentPrefix, '');
|
||||||
html += `<div class="file-row"><span class="icon">📄</span><span class="name">${escapeHtml(displayName)}</span><span class="size">${formatSize(obj.sizeBytes)}</span><span class="date">${formatDate(obj.lastModified)}</span><span class="actions"><button onclick="event.stopPropagation();downloadObject('${obj.key}')" title="Download">⬇</button><button onclick="event.stopPropagation();copyLink('${obj.key}')" title="Copy link">🔗</button><button onclick="event.stopPropagation();deleteObject('${obj.key}')" title="Delete">🗑</button></span></div>`;
|
// Delete is admin-only; download + copy link are always available.
|
||||||
|
const deleteBtn = isAdmin ? `<button onclick="event.stopPropagation();deleteObject('${obj.key}')" title="Delete">🗑</button>` : '';
|
||||||
|
html += `<div class="file-row"><span class="icon">📄</span><span class="name">${escapeHtml(displayName)}</span><span class="size">${formatSize(obj.sizeBytes)}</span><span class="date">${formatDate(obj.lastModified)}</span><span class="actions"><button onclick="event.stopPropagation();downloadObject('${obj.key}')" title="Download">⬇</button><button onclick="event.stopPropagation();copyLink('${obj.key}')" title="Copy link">🔗</button>${deleteBtn}</span></div>`;
|
||||||
}
|
}
|
||||||
container.innerHTML = html;
|
container.innerHTML = html;
|
||||||
};
|
};
|
||||||
@@ -297,11 +321,13 @@
|
|||||||
const downloadObject = async (key) => { window.open(`/api/v1/buckets/${encodeURIComponent(currentBucket)}/download/${encodeURIComponent(key)}`,'_blank'); };
|
const downloadObject = async (key) => { window.open(`/api/v1/buckets/${encodeURIComponent(currentBucket)}/download/${encodeURIComponent(key)}`,'_blank'); };
|
||||||
const copyLink = (key) => { navigator.clipboard.writeText(`${window.location.origin}/api/v1/buckets/${encodeURIComponent(currentBucket)}/download/${encodeURIComponent(key)}`).catch(()=>{}); };
|
const copyLink = (key) => { navigator.clipboard.writeText(`${window.location.origin}/api/v1/buckets/${encodeURIComponent(currentBucket)}/download/${encodeURIComponent(key)}`).catch(()=>{}); };
|
||||||
const deleteObject = async (key) => {
|
const deleteObject = async (key) => {
|
||||||
|
if (!isAdmin) { alert('Read-only mode — login as admin to delete.'); return; }
|
||||||
if(!confirm(`Delete "${key}"?`))return;
|
if(!confirm(`Delete "${key}"?`))return;
|
||||||
try{await api(`/api/v1/buckets/${encodeURIComponent(currentBucket)}/${encodeURIComponent(key)}`,{method:'DELETE'});await loadObjects();}
|
try{await api(`/api/v1/buckets/${encodeURIComponent(currentBucket)}/${encodeURIComponent(key)}`,{method:'DELETE'});await loadObjects();}
|
||||||
catch(e){alert(`Delete failed: ${e.message}`);}
|
catch(e){alert(`Delete failed: ${e.message}`);}
|
||||||
};
|
};
|
||||||
const uploadFiles = async (files) => {
|
const uploadFiles = async (files) => {
|
||||||
|
if (!isAdmin) { alert('Read-only mode — login as admin to upload.'); return; }
|
||||||
if(!currentBucket||files.length===0)return;
|
if(!currentBucket||files.length===0)return;
|
||||||
const overlay=document.getElementById('progressOverlay'), fill=document.getElementById('progressFill'), pn=document.getElementById('progressFileName'), pp=document.getElementById('progressPercent');
|
const overlay=document.getElementById('progressOverlay'), fill=document.getElementById('progressFill'), pn=document.getElementById('progressFileName'), pp=document.getElementById('progressPercent');
|
||||||
overlay.style.display='flex';
|
overlay.style.display='flex';
|
||||||
@@ -325,13 +351,19 @@
|
|||||||
dropzone.addEventListener('click',()=>{const i=document.createElement('input');i.type='file';i.multiple=true;i.onchange=()=>{if(i.files.length>0)uploadFiles(i.files);};i.click();});
|
dropzone.addEventListener('click',()=>{const i=document.createElement('input');i.type='file';i.multiple=true;i.onchange=()=>{if(i.files.length>0)uploadFiles(i.files);};i.click();});
|
||||||
const showModal=(html)=>{document.getElementById('modalContent').innerHTML=html;document.getElementById('modalOverlay').style.display='flex';};
|
const showModal=(html)=>{document.getElementById('modalContent').innerHTML=html;document.getElementById('modalOverlay').style.display='flex';};
|
||||||
const closeModal=(e)=>{if(e&&e.target!==e.currentTarget)return;document.getElementById('modalOverlay').style.display='none';};
|
const closeModal=(e)=>{if(e&&e.target!==e.currentTarget)return;document.getElementById('modalOverlay').style.display='none';};
|
||||||
const showCreateBucketModal=()=>{showModal(`<h3>Create Bucket</h3><input id="bucketNameInput" type="text" placeholder="my-bucket-name" pattern="[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]"><p style="font-size:0.8rem;color:var(--text2);margin-bottom:12px">Lowercase, 3-63 chars, no underscores</p><div class="buttons"><button onclick="closeModal()">Cancel</button><button class="primary" onclick="createBucket()">Create</button></div>`);setTimeout(()=>document.getElementById('bucketNameInput')?.focus(),100);};
|
const showCreateBucketModal=()=>{
|
||||||
const createBucket=async()=>{const n=document.getElementById('bucketNameInput').value.trim();if(!n)return;try{await apiJson('/api/v1/buckets',{method:'POST',body:JSON.stringify({name:n})});closeModal();await loadBuckets();document.getElementById('bucketSelect').value=n;await switchBucket(n);}catch(e){alert(`Failed: ${e.message}`);}};
|
if (!isAdmin) { alert('Read-only mode — login as admin to create buckets.'); return; }
|
||||||
const showCredentialsModal=()=>{showModal(`<h3>S3 Credentials</h3><p style="margin-bottom:12px;font-size:0.85rem;color:var(--text2)">Use these in any S3 client (aws-cli, rclone, s3cmd, etc.)</p><label style="font-size:0.85rem;font-weight:600">Endpoint URL</label><input type="text" value="${window.location.origin}" readonly onclick="this.select()"><label style="font-size:0.85rem;font-weight:600">Region</label><input type="text" value="us-east-1" readonly onclick="this.select()"><label style="font-size:0.85rem;font-weight:600">Access Key</label><input id="s3AccessKey" type="text" readonly onclick="this.select()"><label style="font-size:0.85rem;font-weight:600">Secret Key</label><input id="s3SecretKey" type="password" readonly onclick="this.select()"><div class="buttons"><button type="button" onclick="window.closeModal()">Close</button></div>`);};
|
showModal(`<h3>Create Bucket</h3><input id="bucketNameInput" type="text" placeholder="my-bucket-name" pattern="[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]"><p style="font-size:0.8rem;color:var(--text2);margin-bottom:12px">Lowercase, 3-63 chars, no underscores</p><div class="buttons"><button onclick="closeModal()">Cancel</button><button class="primary" onclick="createBucket()">Create</button></div>`);setTimeout(()=>document.getElementById('bucketNameInput')?.focus(),100);};
|
||||||
const init=async()=>{if(await checkAuth())await loadBuckets();};
|
const createBucket=async()=>{
|
||||||
|
if (!isAdmin) { alert('Read-only mode — login as admin to create buckets.'); return; }
|
||||||
|
const n=document.getElementById('bucketNameInput').value.trim();if(!n)return;try{await apiJson('/api/v1/buckets',{method:'POST',body:JSON.stringify({name:n})});closeModal();await loadBuckets();document.getElementById('bucketSelect').value=n;await switchBucket(n);}catch(e){alert(`Failed: ${e.message}`);}};
|
||||||
|
const showCredentialsModal=()=>{
|
||||||
|
if (!isAdmin) { alert('Read-only mode — login as admin to view S3 credentials.'); return; }
|
||||||
|
showModal(`<h3>S3 Credentials</h3><p style="margin-bottom:12px;font-size:0.85rem;color:var(--text2)">Use these in any S3 client (aws-cli, rclone, s3cmd, etc.)</p><label style="font-size:0.85rem;font-weight:600">Endpoint URL</label><input type="text" value="${window.location.origin}" readonly onclick="this.select()"><label style="font-size:0.85rem;font-weight:600">Region</label><input type="text" value="us-east-1" readonly onclick="this.select()"><label style="font-size:0.85rem;font-weight:600">Access Key</label><input id="s3AccessKey" type="text" readonly onclick="this.select()"><label style="font-size:0.85rem;font-weight:600">Secret Key</label><input id="s3SecretKey" type="password" readonly onclick="this.select()"><div class="buttons"><button type="button" onclick="window.closeModal()">Close</button></div>`);};
|
||||||
|
const init=async()=>{await checkAuth();await loadBuckets();};
|
||||||
document.getElementById('authLoginBtn').addEventListener('click',handleLogin);
|
document.getElementById('authLoginBtn').addEventListener('click',handleLogin);
|
||||||
document.getElementById('authTokenInput').addEventListener('keydown',e=>{if(e.key==='Enter')handleLogin();});
|
document.getElementById('authTokenInput').addEventListener('keydown',e=>{if(e.key==='Enter')handleLogin();});
|
||||||
Object.assign(window, { switchBucket, navigateTo, debouncedSearch, downloadObject, copyLink, deleteObject, closeModal, showCreateBucketModal, createBucket, showCredentialsModal, logout });
|
Object.assign(window, { switchBucket, navigateTo, debouncedSearch, downloadObject, copyLink, deleteObject, closeModal, showCreateBucketModal, createBucket, showCredentialsModal, showAuthScreen, logout });
|
||||||
init();
|
init();
|
||||||
</script>
|
</script>
|
||||||
</body>
|
</body>
|
||||||
|
|||||||
@@ -1,35 +1,28 @@
|
|||||||
import { createReadStream } from 'node:fs';
|
import { createReadStream } from 'node:fs';
|
||||||
import { nanoid } from 'nanoid';
|
import { nanoid } from 'nanoid';
|
||||||
import { config } from '../../../config/index';
|
import type { File as FileEntity } from '../../../domain/entities/file';
|
||||||
import { createBucket, deleteBucket, findBucketByName, listBuckets } from '../../../db/buckets';
|
import { buildNewFile } from '../../../domain/entities/file-factory';
|
||||||
import {
|
|
||||||
countBucketObjects,
|
|
||||||
findFileByBucketAndKey,
|
|
||||||
listObjectsByPrefix,
|
|
||||||
softDeleteFile,
|
|
||||||
} from '../../../db/files-ext';
|
|
||||||
import {
|
|
||||||
abortMultipartUpload,
|
|
||||||
completeMultipartUpload,
|
|
||||||
createMultipartUpload,
|
|
||||||
findMultipartUpload,
|
|
||||||
insertMultipartPart,
|
|
||||||
listMultipartParts,
|
|
||||||
listMultipartUploadsByBucket,
|
|
||||||
} from '../../../db/multipart';
|
|
||||||
import type { File } from '../../../db/schema';
|
|
||||||
import type { ForwardResult } from '../../../domain/ports/telegram-service';
|
import type { ForwardResult } from '../../../domain/ports/telegram-service';
|
||||||
|
import { config } from '../../../env';
|
||||||
|
import {
|
||||||
|
bucketRepository,
|
||||||
|
chunkedStorage,
|
||||||
|
fileRepository,
|
||||||
|
multipartRepository,
|
||||||
|
} from '../../../infrastructure/di';
|
||||||
import { botPool } from '../../../infrastructure/telegram/bot-pool';
|
import { botPool } from '../../../infrastructure/telegram/bot-pool';
|
||||||
import logger from '../../../shared/logger/index';
|
import logger from '../../../shared/logger/index';
|
||||||
import { cleanupTempFile, ensureExtension, getErrorMessage } from '../../../shared/utils/file';
|
|
||||||
import {
|
import {
|
||||||
createChunkedObjectResponse,
|
cleanupTempFile,
|
||||||
storeFileInTelegramChunks,
|
DEFAULT_FILE_TYPE,
|
||||||
} from '../../../utils/chunked-storage';
|
ensureExtension,
|
||||||
import { verifyBodyHash, verifyPresignedUrl, verifySignature } from '../../../utils/s3/auth';
|
getErrorMessage,
|
||||||
import { S3_CORS_HEADERS, s3Headers } from '../../../utils/s3/headers';
|
} from '../../../shared/utils/file';
|
||||||
import { createGetObjectResponse, type ObjectPartSource } from '../../../utils/s3/object-stream';
|
import { streamToTemp } from '../../../shared/utils/temp-stream';
|
||||||
import { parseRangeHeader, unsatisfiedContentRange } from '../../../utils/s3/range';
|
import { verifyBodyHash, verifyPresignedUrl, verifySignature } from '../../s3/auth';
|
||||||
|
import { S3_CORS_HEADERS, s3Headers } from '../../s3/headers';
|
||||||
|
import { createGetObjectResponse, type ObjectPartSource } from '../../s3/object-stream';
|
||||||
|
import { parseRangeHeader, unsatisfiedContentRange } from '../../s3/range';
|
||||||
import {
|
import {
|
||||||
bucketVersioningConfigurationXml,
|
bucketVersioningConfigurationXml,
|
||||||
completeMultipartUploadXml,
|
completeMultipartUploadXml,
|
||||||
@@ -44,7 +37,7 @@ import {
|
|||||||
parseCompleteMultipartBody,
|
parseCompleteMultipartBody,
|
||||||
parseDeleteObjectsBody,
|
parseDeleteObjectsBody,
|
||||||
s3ErrorResponse,
|
s3ErrorResponse,
|
||||||
} from '../../../utils/s3/xml';
|
} from '../../s3/xml';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The default S3 region returned when no region is explicitly configured.
|
* The default S3 region returned when no region is explicitly configured.
|
||||||
@@ -308,7 +301,7 @@ export const handleS3Request = async (
|
|||||||
* @returns An S3 XML response with the bucket list.
|
* @returns An S3 XML response with the bucket list.
|
||||||
*/
|
*/
|
||||||
const handleListBuckets = async (reqId: string): Promise<Response> => {
|
const handleListBuckets = async (reqId: string): Promise<Response> => {
|
||||||
const buckets = await listBuckets();
|
const buckets = await bucketRepository.list();
|
||||||
const xml = listBucketsXml(buckets, reqId);
|
const xml = listBucketsXml(buckets, reqId);
|
||||||
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
|
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
|
||||||
};
|
};
|
||||||
@@ -338,7 +331,7 @@ const handleCreateBucket = async (bucketName: string, reqId: string): Promise<Re
|
|||||||
reqId,
|
reqId,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
const existing = await findBucketByName(bucketName);
|
const existing = await bucketRepository.findByName(bucketName);
|
||||||
if (existing) {
|
if (existing) {
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'BucketAlreadyExists',
|
'BucketAlreadyExists',
|
||||||
@@ -348,7 +341,7 @@ const handleCreateBucket = async (bucketName: string, reqId: string): Promise<Re
|
|||||||
reqId,
|
reqId,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
await createBucket(bucketName);
|
await bucketRepository.create(bucketName);
|
||||||
return s3Response(null, 200, reqId);
|
return s3Response(null, 200, reqId);
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -360,7 +353,7 @@ const handleCreateBucket = async (bucketName: string, reqId: string): Promise<Re
|
|||||||
* @returns A 200 response when the bucket exists, or an S3 XML error.
|
* @returns A 200 response when the bucket exists, or an S3 XML error.
|
||||||
*/
|
*/
|
||||||
const handleHeadBucket = async (bucketName: string, reqId: string): Promise<Response> => {
|
const handleHeadBucket = async (bucketName: string, reqId: string): Promise<Response> => {
|
||||||
const bucket = await findBucketByName(bucketName);
|
const bucket = await bucketRepository.findByName(bucketName);
|
||||||
if (!bucket) {
|
if (!bucket) {
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'NoSuchBucket',
|
'NoSuchBucket',
|
||||||
@@ -383,7 +376,7 @@ const handleHeadBucket = async (bucketName: string, reqId: string): Promise<Resp
|
|||||||
* @returns A 204 response on success, or an S3 XML error.
|
* @returns A 204 response on success, or an S3 XML error.
|
||||||
*/
|
*/
|
||||||
const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Response> => {
|
const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Response> => {
|
||||||
const bucket = await findBucketByName(bucketName);
|
const bucket = await bucketRepository.findByName(bucketName);
|
||||||
if (!bucket) {
|
if (!bucket) {
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'NoSuchBucket',
|
'NoSuchBucket',
|
||||||
@@ -393,7 +386,7 @@ const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Re
|
|||||||
reqId,
|
reqId,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
const objCount = await countBucketObjects(bucket.id);
|
const objCount = await fileRepository.countByBucket(bucket.id);
|
||||||
if (objCount > 0) {
|
if (objCount > 0) {
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'BucketNotEmpty',
|
'BucketNotEmpty',
|
||||||
@@ -403,7 +396,7 @@ const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Re
|
|||||||
reqId,
|
reqId,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
await deleteBucket(bucketName);
|
await bucketRepository.delete(bucketName);
|
||||||
return s3Response(null, 204, reqId);
|
return s3Response(null, 204, reqId);
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -416,7 +409,7 @@ const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Re
|
|||||||
* @returns An S3 XML response with the versioning configuration.
|
* @returns An S3 XML response with the versioning configuration.
|
||||||
*/
|
*/
|
||||||
const handleGetBucketVersioning = async (bucketName: string, reqId: string): Promise<Response> => {
|
const handleGetBucketVersioning = async (bucketName: string, reqId: string): Promise<Response> => {
|
||||||
const bucket = await findBucketByName(bucketName);
|
const bucket = await bucketRepository.findByName(bucketName);
|
||||||
if (!bucket) {
|
if (!bucket) {
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'NoSuchBucket',
|
'NoSuchBucket',
|
||||||
@@ -431,6 +424,95 @@ const handleGetBucketVersioning = async (bucketName: string, reqId: string): Pro
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// ─────── Conditional Headers Helper ──────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* S3-compatible response for 304 Not Modified.
|
||||||
|
*/
|
||||||
|
const notModifiedResponse = (
|
||||||
|
reqId: string,
|
||||||
|
etag: string,
|
||||||
|
mimeType: string,
|
||||||
|
sizeBytes: number,
|
||||||
|
lastModified: Date,
|
||||||
|
): Response =>
|
||||||
|
new Response(null, {
|
||||||
|
status: 304,
|
||||||
|
headers: s3Headers(reqId, {
|
||||||
|
etag,
|
||||||
|
'content-type': mimeType,
|
||||||
|
'content-length': String(sizeBytes),
|
||||||
|
'last-modified': lastModified.toUTCString(),
|
||||||
|
'x-amz-version-id': 'null',
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* S3-compatible response for 412 Precondition Failed.
|
||||||
|
*/
|
||||||
|
const preconditionFailedResponse = (path: string, reqId: string): Response =>
|
||||||
|
s3ErrorResponse(
|
||||||
|
'PreconditionFailed',
|
||||||
|
'At least one of the pre-conditions you specified did not hold.',
|
||||||
|
path,
|
||||||
|
412,
|
||||||
|
reqId,
|
||||||
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checks conditional headers (If-Match, If-None-Match, If-Modified-Since,
|
||||||
|
* If-Unmodified-Since) and returns a prepared Response if the condition
|
||||||
|
* is not satisfied, or `null` to let the request proceed.
|
||||||
|
*
|
||||||
|
* @returns A 304 / 412 Response when a condition fails, or `null` to continue.
|
||||||
|
*/
|
||||||
|
const checkConditionalHeaders = (
|
||||||
|
headers: Record<string, string>,
|
||||||
|
file: {
|
||||||
|
mimeType: string;
|
||||||
|
sizeBytes: number;
|
||||||
|
fileHash: string | null;
|
||||||
|
createdAt: Date | string | number;
|
||||||
|
},
|
||||||
|
path: string,
|
||||||
|
reqId: string,
|
||||||
|
): Response | null => {
|
||||||
|
const etag = `"${file.fileHash || nanoid(16)}"`;
|
||||||
|
const lastModified = file.createdAt instanceof Date ? file.createdAt : new Date(file.createdAt);
|
||||||
|
|
||||||
|
// If-Match
|
||||||
|
const ifMatch = headers['if-match'];
|
||||||
|
if (ifMatch && ifMatch !== '*' && ifMatch !== etag) {
|
||||||
|
return preconditionFailedResponse(path, reqId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// If-None-Match
|
||||||
|
const ifNoneMatch = headers['if-none-match'];
|
||||||
|
if (ifNoneMatch && ifNoneMatch === etag) {
|
||||||
|
return notModifiedResponse(reqId, etag, file.mimeType, file.sizeBytes, lastModified);
|
||||||
|
}
|
||||||
|
|
||||||
|
// If-Modified-Since
|
||||||
|
const ifModifiedSince = headers['if-modified-since'];
|
||||||
|
if (ifModifiedSince) {
|
||||||
|
const since = new Date(ifModifiedSince);
|
||||||
|
if (!Number.isNaN(since.getTime()) && lastModified.getTime() <= since.getTime()) {
|
||||||
|
return notModifiedResponse(reqId, etag, file.mimeType, file.sizeBytes, lastModified);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If-Unmodified-Since
|
||||||
|
const ifUnmodifiedSince = headers['if-unmodified-since'];
|
||||||
|
if (ifUnmodifiedSince) {
|
||||||
|
const since = new Date(ifUnmodifiedSince);
|
||||||
|
if (!Number.isNaN(since.getTime()) && lastModified.getTime() > since.getTime()) {
|
||||||
|
return preconditionFailedResponse(path, reqId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
|
||||||
// ─────── Object Operations ───────
|
// ─────── Object Operations ───────
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -455,7 +537,7 @@ const handleGetObject = async (
|
|||||||
headers: Record<string, string>,
|
headers: Record<string, string>,
|
||||||
reqId: string,
|
reqId: string,
|
||||||
): Promise<Response> => {
|
): Promise<Response> => {
|
||||||
const bucketRecord = await findBucketByName(bucket);
|
const bucketRecord = await bucketRepository.findByName(bucket);
|
||||||
if (!bucketRecord)
|
if (!bucketRecord)
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'NoSuchBucket',
|
'NoSuchBucket',
|
||||||
@@ -465,7 +547,7 @@ const handleGetObject = async (
|
|||||||
reqId,
|
reqId,
|
||||||
);
|
);
|
||||||
|
|
||||||
const file = await findFileByBucketAndKey(bucketRecord.id, key);
|
const file = await fileRepository.findByBucketAndKey(bucketRecord.id, key);
|
||||||
if (!file)
|
if (!file)
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'NoSuchKey',
|
'NoSuchKey',
|
||||||
@@ -475,62 +557,10 @@ const handleGetObject = async (
|
|||||||
reqId,
|
reqId,
|
||||||
);
|
);
|
||||||
|
|
||||||
// H3: Conditional headers — If-Match / If-None-Match
|
// H3: Conditional headers — If-Match / If-None-Match / If-Modified-Since / If-Unmodified-Since
|
||||||
const etag = `"${file.fileHash || nanoid(16)}"`;
|
const conditionResult = checkConditionalHeaders(headers, file, `/${bucket}/${key}`, reqId);
|
||||||
const lastModified = file.createdAt instanceof Date ? file.createdAt : new Date(file.createdAt);
|
if (conditionResult) {
|
||||||
const ifMatch = headers['if-match'];
|
return conditionResult;
|
||||||
if (ifMatch && ifMatch !== '*' && ifMatch !== etag) {
|
|
||||||
return s3ErrorResponse(
|
|
||||||
'PreconditionFailed',
|
|
||||||
'At least one of the pre-conditions you specified did not hold.',
|
|
||||||
`/${bucket}/${key}`,
|
|
||||||
412,
|
|
||||||
reqId,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
const ifNoneMatch = headers['if-none-match'];
|
|
||||||
if (ifNoneMatch && ifNoneMatch === etag) {
|
|
||||||
return new Response(null, {
|
|
||||||
status: 304,
|
|
||||||
headers: s3Headers(reqId, {
|
|
||||||
etag,
|
|
||||||
'content-type': file.mimeType,
|
|
||||||
'content-length': String(file.sizeBytes),
|
|
||||||
'last-modified': lastModified.toUTCString(),
|
|
||||||
'x-amz-version-id': 'null',
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// H3: Conditional headers — If-Modified-Since / If-Unmodified-Since
|
|
||||||
const ifModifiedSince = headers['if-modified-since'];
|
|
||||||
if (ifModifiedSince) {
|
|
||||||
const since = new Date(ifModifiedSince);
|
|
||||||
if (!Number.isNaN(since.getTime()) && lastModified.getTime() <= since.getTime()) {
|
|
||||||
return new Response(null, {
|
|
||||||
status: 304,
|
|
||||||
headers: s3Headers(reqId, {
|
|
||||||
etag,
|
|
||||||
'content-type': file.mimeType,
|
|
||||||
'content-length': String(file.sizeBytes),
|
|
||||||
'last-modified': lastModified.toUTCString(),
|
|
||||||
'x-amz-version-id': 'null',
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const ifUnmodifiedSince = headers['if-unmodified-since'];
|
|
||||||
if (ifUnmodifiedSince) {
|
|
||||||
const since = new Date(ifUnmodifiedSince);
|
|
||||||
if (!Number.isNaN(since.getTime()) && lastModified.getTime() > since.getTime()) {
|
|
||||||
return s3ErrorResponse(
|
|
||||||
'PreconditionFailed',
|
|
||||||
'At least one of the pre-conditions you specified did not hold.',
|
|
||||||
`/${bucket}/${key}`,
|
|
||||||
412,
|
|
||||||
reqId,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Chunked storage object
|
// Chunked storage object
|
||||||
@@ -550,7 +580,7 @@ const handleGetObject = async (
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
return await createChunkedObjectResponse({ file, range, reqId });
|
return await chunkedStorage.createChunkedObjectResponse({ file, range, reqId });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.warn('Chunked object content fetch failed', { key, error: getErrorMessage(error) });
|
logger.warn('Chunked object content fetch failed', { key, error: getErrorMessage(error) });
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
@@ -637,14 +667,14 @@ const handleGetObject = async (
|
|||||||
* @returns An S3 response streaming the assembled object content.
|
* @returns An S3 response streaming the assembled object content.
|
||||||
*/
|
*/
|
||||||
const handleGetMultipartObject = async (
|
const handleGetMultipartObject = async (
|
||||||
file: File,
|
file: FileEntity,
|
||||||
bucket: string,
|
bucket: string,
|
||||||
key: string,
|
key: string,
|
||||||
headers: Record<string, string>,
|
headers: Record<string, string>,
|
||||||
reqId: string,
|
reqId: string,
|
||||||
): Promise<Response> => {
|
): Promise<Response> => {
|
||||||
const uploadId = file.multipartUploadId!;
|
const uploadId = file.multipartUploadId!;
|
||||||
const parts = await listMultipartParts(uploadId);
|
const parts = await multipartRepository.listParts(uploadId);
|
||||||
|
|
||||||
if (parts.length === 0) {
|
if (parts.length === 0) {
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
@@ -723,7 +753,7 @@ const handleHeadObject = async (
|
|||||||
headers: Record<string, string>,
|
headers: Record<string, string>,
|
||||||
reqId: string,
|
reqId: string,
|
||||||
): Promise<Response> => {
|
): Promise<Response> => {
|
||||||
const bucketRecord = await findBucketByName(bucket);
|
const bucketRecord = await bucketRepository.findByName(bucket);
|
||||||
if (!bucketRecord)
|
if (!bucketRecord)
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'NoSuchBucket',
|
'NoSuchBucket',
|
||||||
@@ -733,7 +763,7 @@ const handleHeadObject = async (
|
|||||||
reqId,
|
reqId,
|
||||||
);
|
);
|
||||||
|
|
||||||
const file = await findFileByBucketAndKey(bucketRecord.id, key);
|
const file = await fileRepository.findByBucketAndKey(bucketRecord.id, key);
|
||||||
if (!file)
|
if (!file)
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'NoSuchKey',
|
'NoSuchKey',
|
||||||
@@ -743,62 +773,10 @@ const handleHeadObject = async (
|
|||||||
reqId,
|
reqId,
|
||||||
);
|
);
|
||||||
|
|
||||||
// H3: Conditional headers for HEAD — If-Match / If-None-Match
|
// H3: Conditional headers for HEAD — If-Match / If-None-Match / If-Modified-Since / If-Unmodified-Since
|
||||||
const etag = `"${file.fileHash || nanoid(16)}"`;
|
const headConditionResult = checkConditionalHeaders(headers, file, `/${bucket}/${key}`, reqId);
|
||||||
const lastModified = file.createdAt instanceof Date ? file.createdAt : new Date(file.createdAt);
|
if (headConditionResult) {
|
||||||
const ifMatch = headers['if-match'];
|
return headConditionResult;
|
||||||
if (ifMatch && ifMatch !== '*' && ifMatch !== etag) {
|
|
||||||
return s3ErrorResponse(
|
|
||||||
'PreconditionFailed',
|
|
||||||
'At least one of the pre-conditions you specified did not hold.',
|
|
||||||
`/${bucket}/${key}`,
|
|
||||||
412,
|
|
||||||
reqId,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
const ifNoneMatch = headers['if-none-match'];
|
|
||||||
if (ifNoneMatch && ifNoneMatch === etag) {
|
|
||||||
return new Response(null, {
|
|
||||||
status: 304,
|
|
||||||
headers: s3Headers(reqId, {
|
|
||||||
etag,
|
|
||||||
'content-type': file.mimeType,
|
|
||||||
'content-length': String(file.sizeBytes),
|
|
||||||
'last-modified': lastModified.toUTCString(),
|
|
||||||
'x-amz-version-id': 'null',
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// H3: Conditional headers for HEAD — If-Modified-Since / If-Unmodified-Since
|
|
||||||
const ifModifiedSince = headers['if-modified-since'];
|
|
||||||
if (ifModifiedSince) {
|
|
||||||
const since = new Date(ifModifiedSince);
|
|
||||||
if (!Number.isNaN(since.getTime()) && lastModified.getTime() <= since.getTime()) {
|
|
||||||
return new Response(null, {
|
|
||||||
status: 304,
|
|
||||||
headers: s3Headers(reqId, {
|
|
||||||
etag,
|
|
||||||
'content-type': file.mimeType,
|
|
||||||
'content-length': String(file.sizeBytes),
|
|
||||||
'last-modified': lastModified.toUTCString(),
|
|
||||||
'x-amz-version-id': 'null',
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const ifUnmodifiedSince = headers['if-unmodified-since'];
|
|
||||||
if (ifUnmodifiedSince) {
|
|
||||||
const since = new Date(ifUnmodifiedSince);
|
|
||||||
if (!Number.isNaN(since.getTime()) && lastModified.getTime() > since.getTime()) {
|
|
||||||
return s3ErrorResponse(
|
|
||||||
'PreconditionFailed',
|
|
||||||
'At least one of the pre-conditions you specified did not hold.',
|
|
||||||
`/${bucket}/${key}`,
|
|
||||||
412,
|
|
||||||
reqId,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return s3Response(null, 200, reqId, {
|
return s3Response(null, 200, reqId, {
|
||||||
@@ -831,17 +809,10 @@ const streamBodyToTemp = async (
|
|||||||
): Promise<{
|
): Promise<{
|
||||||
tempPath: string;
|
tempPath: string;
|
||||||
fileHash: string;
|
fileHash: string;
|
||||||
md5Hash: string;
|
md5Hash?: string;
|
||||||
sizeBytes: number;
|
sizeBytes: number;
|
||||||
signatureBuffer: Buffer;
|
signatureBuffer: Buffer;
|
||||||
}> => {
|
}> => {
|
||||||
const tempPath = `/tmp/filedrop-s3-${nanoid()}`;
|
|
||||||
const writer = Bun.file(tempPath).writer();
|
|
||||||
const sha256 = new Bun.CryptoHasher('sha256');
|
|
||||||
const md5 = new Bun.CryptoHasher('md5');
|
|
||||||
let writerFailed = false;
|
|
||||||
|
|
||||||
// Handle body being null (GET/HEAD/DELETE or empty PUT)
|
|
||||||
const reader = (
|
const reader = (
|
||||||
body ??
|
body ??
|
||||||
new ReadableStream({
|
new ReadableStream({
|
||||||
@@ -849,56 +820,8 @@ const streamBodyToTemp = async (
|
|||||||
c.close();
|
c.close();
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
).getReader();
|
).getReader() as ReadableStreamDefaultReader<Uint8Array>;
|
||||||
const SIGNATURE_BYTES = 16;
|
return streamToTemp(reader, { computeMd5: true, prefix: '/tmp/filedrop-s3-' });
|
||||||
const signatureChunks: Buffer[] = [];
|
|
||||||
let signatureBytes = 0;
|
|
||||||
let sizeBytes = 0;
|
|
||||||
|
|
||||||
try {
|
|
||||||
while (true) {
|
|
||||||
const { done, value } = await reader.read();
|
|
||||||
if (done) break;
|
|
||||||
const chunk = Buffer.from(value);
|
|
||||||
sizeBytes += chunk.byteLength;
|
|
||||||
sha256.update(chunk);
|
|
||||||
md5.update(chunk);
|
|
||||||
writer.write(chunk);
|
|
||||||
|
|
||||||
if (signatureBytes < SIGNATURE_BYTES) {
|
|
||||||
const remaining = SIGNATURE_BYTES - signatureBytes;
|
|
||||||
const sigChunk = chunk.subarray(0, remaining);
|
|
||||||
signatureChunks.push(sigChunk);
|
|
||||||
signatureBytes += sigChunk.byteLength;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
writer.end();
|
|
||||||
} catch {
|
|
||||||
writerFailed = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
tempPath,
|
|
||||||
fileHash: sha256.digest('hex'),
|
|
||||||
md5Hash: md5.digest('base64'),
|
|
||||||
sizeBytes,
|
|
||||||
signatureBuffer: Buffer.concat(signatureChunks, signatureBytes),
|
|
||||||
};
|
|
||||||
} catch (error) {
|
|
||||||
if (!writerFailed) {
|
|
||||||
try {
|
|
||||||
writer.end();
|
|
||||||
} catch {
|
|
||||||
/* writer may already be errored */
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await cleanupTempFile(tempPath);
|
|
||||||
throw error;
|
|
||||||
} finally {
|
|
||||||
reader.releaseLock();
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -929,7 +852,7 @@ const handlePutObject = async (
|
|||||||
req: Request,
|
req: Request,
|
||||||
reqId: string,
|
reqId: string,
|
||||||
): Promise<Response> => {
|
): Promise<Response> => {
|
||||||
const bucketRecord = await findBucketByName(bucket);
|
const bucketRecord = await bucketRepository.findByName(bucket);
|
||||||
if (!bucketRecord)
|
if (!bucketRecord)
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'NoSuchBucket',
|
'NoSuchBucket',
|
||||||
@@ -1010,7 +933,7 @@ const handlePutObject = async (
|
|||||||
|
|
||||||
// Idempotent PUT: if the object already exists, skip upload
|
// Idempotent PUT: if the object already exists, skip upload
|
||||||
try {
|
try {
|
||||||
const existing = await findFileByBucketAndKey(bucketRecord.id, key);
|
const existing = await fileRepository.findByBucketAndKey(bucketRecord.id, key);
|
||||||
if (existing) {
|
if (existing) {
|
||||||
await cleanupTempFile(streamed.tempPath);
|
await cleanupTempFile(streamed.tempPath);
|
||||||
return s3Response(null, 200, reqId, { etag: `"${streamed.fileHash}"` });
|
return s3Response(null, 200, reqId, { etag: `"${streamed.fileHash}"` });
|
||||||
@@ -1056,13 +979,13 @@ const storeFileFromTemp = async (
|
|||||||
const partFileNamePrefix = `s3-${bucketRecord.name}-${key.replace(/\//g, '_')}`;
|
const partFileNamePrefix = `s3-${bucketRecord.name}-${key.replace(/\//g, '_')}`;
|
||||||
|
|
||||||
if (streamed.sizeBytes > config.telegramChunkSizeBytes) {
|
if (streamed.sizeBytes > config.telegramChunkSizeBytes) {
|
||||||
const file = await storeFileInTelegramChunks({
|
const file = await chunkedStorage.storeFileInTelegramChunks({
|
||||||
tempPath: streamed.tempPath,
|
tempPath: streamed.tempPath,
|
||||||
partFileNamePrefix,
|
partFileNamePrefix,
|
||||||
fileName: finalFileName,
|
fileName: finalFileName,
|
||||||
mimeType,
|
mimeType,
|
||||||
sizeBytes: streamed.sizeBytes,
|
sizeBytes: streamed.sizeBytes,
|
||||||
fileType: 'document',
|
fileType: DEFAULT_FILE_TYPE,
|
||||||
uploaderId: 0,
|
uploaderId: 0,
|
||||||
bucketId,
|
bucketId,
|
||||||
s3Key: key,
|
s3Key: key,
|
||||||
@@ -1082,27 +1005,25 @@ const storeFileFromTemp = async (
|
|||||||
fileStream.destroy();
|
fileStream.destroy();
|
||||||
|
|
||||||
const publicId = nanoid();
|
const publicId = nanoid();
|
||||||
const { db, files: fileSchema } = await import('../../../db/index');
|
|
||||||
|
|
||||||
await db.insert(fileSchema).values({
|
await fileRepository.create(
|
||||||
publicId,
|
buildNewFile({
|
||||||
telegramFileId: forwardResult.telegramFileId,
|
publicId,
|
||||||
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
|
telegramFileId: forwardResult.telegramFileId,
|
||||||
storageChatId: config.storageChatId,
|
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
|
||||||
storageMessageId: forwardResult.storageMessageId,
|
storageChatId: config.storageChatId,
|
||||||
fileName: finalFileName,
|
storageMessageId: forwardResult.storageMessageId,
|
||||||
mimeType,
|
fileName: finalFileName,
|
||||||
sizeBytes: streamed.sizeBytes,
|
mimeType,
|
||||||
fileType: 'document',
|
sizeBytes: streamed.sizeBytes,
|
||||||
uploaderId: 0,
|
fileType: DEFAULT_FILE_TYPE,
|
||||||
fileHash: streamed.fileHash,
|
uploaderId: 0,
|
||||||
bucketId,
|
fileHash: streamed.fileHash,
|
||||||
s3Key: key,
|
bucketId,
|
||||||
storageBackend: 'telegram',
|
s3Key: key,
|
||||||
isDeleted: false,
|
storageBackend: 'telegram',
|
||||||
createdAt: new Date(),
|
}),
|
||||||
updatedAt: new Date(),
|
);
|
||||||
});
|
|
||||||
|
|
||||||
await cleanupTempFile(streamed.tempPath);
|
await cleanupTempFile(streamed.tempPath);
|
||||||
|
|
||||||
@@ -1138,7 +1059,7 @@ const handleCopyObject = async (
|
|||||||
const sourceBucket = parts[0];
|
const sourceBucket = parts[0];
|
||||||
const sourceKey = parts.slice(1).join('/');
|
const sourceKey = parts.slice(1).join('/');
|
||||||
|
|
||||||
const sourceBucketRecord = await findBucketByName(sourceBucket);
|
const sourceBucketRecord = await bucketRepository.findByName(sourceBucket);
|
||||||
if (!sourceBucketRecord)
|
if (!sourceBucketRecord)
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'NoSuchBucket',
|
'NoSuchBucket',
|
||||||
@@ -1148,7 +1069,7 @@ const handleCopyObject = async (
|
|||||||
reqId,
|
reqId,
|
||||||
);
|
);
|
||||||
|
|
||||||
const sourceFile = await findFileByBucketAndKey(sourceBucketRecord.id, sourceKey);
|
const sourceFile = await fileRepository.findByBucketAndKey(sourceBucketRecord.id, sourceKey);
|
||||||
if (!sourceFile)
|
if (!sourceFile)
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'NoSuchKey',
|
'NoSuchKey',
|
||||||
@@ -1194,27 +1115,25 @@ const handleCopyObject = async (
|
|||||||
}
|
}
|
||||||
|
|
||||||
const publicId = nanoid();
|
const publicId = nanoid();
|
||||||
const { db, files: fileSchema } = await import('../../../db/index');
|
|
||||||
|
|
||||||
await db.insert(fileSchema).values({
|
await fileRepository.create(
|
||||||
publicId,
|
buildNewFile({
|
||||||
telegramFileId: sourceFile.telegramFileId,
|
publicId,
|
||||||
telegramFileUniqueId: sourceFile.telegramFileUniqueId,
|
telegramFileId: sourceFile.telegramFileId,
|
||||||
storageChatId: sourceFile.storageChatId,
|
telegramFileUniqueId: sourceFile.telegramFileUniqueId,
|
||||||
storageMessageId: sourceFile.storageMessageId,
|
storageChatId: sourceFile.storageChatId,
|
||||||
fileName: sourceFile.fileName,
|
storageMessageId: sourceFile.storageMessageId,
|
||||||
mimeType: sourceFile.mimeType,
|
fileName: sourceFile.fileName,
|
||||||
sizeBytes: sourceFile.sizeBytes,
|
mimeType: sourceFile.mimeType,
|
||||||
fileType: sourceFile.fileType,
|
sizeBytes: sourceFile.sizeBytes,
|
||||||
uploaderId: 0,
|
fileType: sourceFile.fileType,
|
||||||
fileHash: sourceFile.fileHash,
|
uploaderId: 0,
|
||||||
bucketId: destBucketId,
|
fileHash: sourceFile.fileHash,
|
||||||
s3Key: destKey,
|
bucketId: destBucketId,
|
||||||
storageBackend: 'telegram',
|
s3Key: destKey,
|
||||||
isDeleted: false,
|
storageBackend: 'telegram',
|
||||||
createdAt: new Date(),
|
}),
|
||||||
updatedAt: new Date(),
|
);
|
||||||
});
|
|
||||||
|
|
||||||
const xml = copyObjectResultXml(sourceFile.fileHash || nanoid(16), new Date());
|
const xml = copyObjectResultXml(sourceFile.fileHash || nanoid(16), new Date());
|
||||||
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
|
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
|
||||||
@@ -1233,7 +1152,7 @@ const handleDeleteObject = async (
|
|||||||
key: string,
|
key: string,
|
||||||
reqId: string,
|
reqId: string,
|
||||||
): Promise<Response> => {
|
): Promise<Response> => {
|
||||||
const bucketRecord = await findBucketByName(bucket);
|
const bucketRecord = await bucketRepository.findByName(bucket);
|
||||||
if (!bucketRecord)
|
if (!bucketRecord)
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'NoSuchBucket',
|
'NoSuchBucket',
|
||||||
@@ -1243,7 +1162,7 @@ const handleDeleteObject = async (
|
|||||||
reqId,
|
reqId,
|
||||||
);
|
);
|
||||||
|
|
||||||
await softDeleteFile(bucketRecord.id, key);
|
await fileRepository.softDelete(bucketRecord.id, key);
|
||||||
return s3Response(null, 204, reqId);
|
return s3Response(null, 204, reqId);
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1263,7 +1182,7 @@ const handleDeleteObjects = async (
|
|||||||
body: string,
|
body: string,
|
||||||
reqId: string,
|
reqId: string,
|
||||||
): Promise<Response> => {
|
): Promise<Response> => {
|
||||||
const bucketRecord = await findBucketByName(bucket);
|
const bucketRecord = await bucketRepository.findByName(bucket);
|
||||||
if (!bucketRecord)
|
if (!bucketRecord)
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'NoSuchBucket',
|
'NoSuchBucket',
|
||||||
@@ -1289,7 +1208,7 @@ const handleDeleteObjects = async (
|
|||||||
const deletedKeys: string[] = [];
|
const deletedKeys: string[] = [];
|
||||||
const errors: Array<{ key: string; code: string; message: string }> = [];
|
const errors: Array<{ key: string; code: string; message: string }> = [];
|
||||||
for (const key of keys) {
|
for (const key of keys) {
|
||||||
const ok = await softDeleteFile(bucketRecord.id, key);
|
const ok = await fileRepository.softDelete(bucketRecord.id, key);
|
||||||
if (ok) {
|
if (ok) {
|
||||||
deletedKeys.push(key);
|
deletedKeys.push(key);
|
||||||
} else {
|
} else {
|
||||||
@@ -1317,7 +1236,7 @@ const handleListObjectsV1 = async (
|
|||||||
searchParams: URLSearchParams,
|
searchParams: URLSearchParams,
|
||||||
reqId: string,
|
reqId: string,
|
||||||
): Promise<Response> => {
|
): Promise<Response> => {
|
||||||
const bucketRecord = await findBucketByName(bucket);
|
const bucketRecord = await bucketRepository.findByName(bucket);
|
||||||
if (!bucketRecord)
|
if (!bucketRecord)
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'NoSuchBucket',
|
'NoSuchBucket',
|
||||||
@@ -1336,7 +1255,7 @@ const handleListObjectsV1 = async (
|
|||||||
const marker = searchParams.get('marker') || null;
|
const marker = searchParams.get('marker') || null;
|
||||||
const encodingType = searchParams.get('encoding-type') || null;
|
const encodingType = searchParams.get('encoding-type') || null;
|
||||||
|
|
||||||
const { objects, prefixes: commonPrefixes } = await listObjectsByPrefix(
|
const { objects, prefixes: commonPrefixes } = await fileRepository.listByPrefix(
|
||||||
bucketRecord.id,
|
bucketRecord.id,
|
||||||
prefix,
|
prefix,
|
||||||
delimiter,
|
delimiter,
|
||||||
@@ -1352,13 +1271,7 @@ const handleListObjectsV1 = async (
|
|||||||
|
|
||||||
const xml = listBucketResultXml(
|
const xml = listBucketResultXml(
|
||||||
bucket,
|
bucket,
|
||||||
displayObjects.map((o) => ({
|
displayObjects.map(mapFileToListEntry),
|
||||||
key: o.s3Key ?? '',
|
|
||||||
sizeBytes: o.sizeBytes,
|
|
||||||
etag: o.fileHash || nanoid(16),
|
|
||||||
lastModified: o.createdAt instanceof Date ? o.createdAt : new Date(),
|
|
||||||
mimeType: o.mimeType,
|
|
||||||
})),
|
|
||||||
commonPrefixes,
|
commonPrefixes,
|
||||||
isTruncated,
|
isTruncated,
|
||||||
marker,
|
marker,
|
||||||
@@ -1373,6 +1286,29 @@ const handleListObjectsV1 = async (
|
|||||||
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
|
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/** Shape of an S3 list entry object. */
|
||||||
|
type S3ListEntry = {
|
||||||
|
key: string;
|
||||||
|
sizeBytes: number;
|
||||||
|
etag: string;
|
||||||
|
lastModified: Date;
|
||||||
|
mimeType: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Maps a File entity to an S3 list entry object.
|
||||||
|
*
|
||||||
|
* @param file - The file entity from the repository.
|
||||||
|
* @returns An S3 list entry with key, size, etag, last modified, and MIME type.
|
||||||
|
*/
|
||||||
|
const mapFileToListEntry = (file: FileEntity): S3ListEntry => ({
|
||||||
|
key: file.s3Key ?? '',
|
||||||
|
sizeBytes: file.sizeBytes,
|
||||||
|
etag: file.fileHash || nanoid(16),
|
||||||
|
lastModified: file.createdAt instanceof Date ? file.createdAt : new Date(),
|
||||||
|
mimeType: file.mimeType,
|
||||||
|
});
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Handles GET /{bucket}?list-type=2 (ListObjectsV2).
|
* Handles GET /{bucket}?list-type=2 (ListObjectsV2).
|
||||||
*
|
*
|
||||||
@@ -1387,7 +1323,7 @@ const handleListObjectsV2 = async (
|
|||||||
searchParams: URLSearchParams,
|
searchParams: URLSearchParams,
|
||||||
reqId: string,
|
reqId: string,
|
||||||
): Promise<Response> => {
|
): Promise<Response> => {
|
||||||
const bucketRecord = await findBucketByName(bucket);
|
const bucketRecord = await bucketRepository.findByName(bucket);
|
||||||
if (!bucketRecord)
|
if (!bucketRecord)
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'NoSuchBucket',
|
'NoSuchBucket',
|
||||||
@@ -1404,7 +1340,7 @@ const handleListObjectsV2 = async (
|
|||||||
const startAfter = searchParams.get('start-after') || null;
|
const startAfter = searchParams.get('start-after') || null;
|
||||||
const encodingType = searchParams.get('encoding-type') || null;
|
const encodingType = searchParams.get('encoding-type') || null;
|
||||||
|
|
||||||
const { objects, prefixes: commonPrefixes } = await listObjectsByPrefix(
|
const { objects, prefixes: commonPrefixes } = await fileRepository.listByPrefix(
|
||||||
bucketRecord.id,
|
bucketRecord.id,
|
||||||
prefix,
|
prefix,
|
||||||
delimiter,
|
delimiter,
|
||||||
@@ -1420,13 +1356,7 @@ const handleListObjectsV2 = async (
|
|||||||
|
|
||||||
const xml = listBucketV2ResultXml(
|
const xml = listBucketV2ResultXml(
|
||||||
bucket,
|
bucket,
|
||||||
displayObjects.map((o) => ({
|
displayObjects.map(mapFileToListEntry),
|
||||||
key: o.s3Key ?? '',
|
|
||||||
sizeBytes: o.sizeBytes,
|
|
||||||
etag: o.fileHash || nanoid(16),
|
|
||||||
lastModified: o.createdAt instanceof Date ? o.createdAt : new Date(),
|
|
||||||
mimeType: o.mimeType,
|
|
||||||
})),
|
|
||||||
commonPrefixes,
|
commonPrefixes,
|
||||||
isTruncated,
|
isTruncated,
|
||||||
maxKeys,
|
maxKeys,
|
||||||
@@ -1460,7 +1390,7 @@ const handleCreateMultipartUpload = async (
|
|||||||
headers: Record<string, string>,
|
headers: Record<string, string>,
|
||||||
reqId: string,
|
reqId: string,
|
||||||
): Promise<Response> => {
|
): Promise<Response> => {
|
||||||
const bucketRecord = await findBucketByName(bucket);
|
const bucketRecord = await bucketRepository.findByName(bucket);
|
||||||
if (!bucketRecord)
|
if (!bucketRecord)
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'NoSuchBucket',
|
'NoSuchBucket',
|
||||||
@@ -1471,7 +1401,7 @@ const handleCreateMultipartUpload = async (
|
|||||||
);
|
);
|
||||||
|
|
||||||
const contentType = headers['content-type'] || null;
|
const contentType = headers['content-type'] || null;
|
||||||
const uploadId = await createMultipartUpload(bucketRecord.id, key, 's3', contentType);
|
const uploadId = await multipartRepository.create(bucketRecord.id, key, 's3', contentType);
|
||||||
|
|
||||||
const xml = initiateMultipartUploadXml(bucket, key, uploadId);
|
const xml = initiateMultipartUploadXml(bucket, key, uploadId);
|
||||||
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
|
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
|
||||||
@@ -1515,7 +1445,7 @@ const handleUploadPart = async (
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const multipart = await findMultipartUpload(uploadId);
|
const multipart = await multipartRepository.findById(uploadId);
|
||||||
if (!multipart || multipart.s3Key !== key) {
|
if (!multipart || multipart.s3Key !== key) {
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'NoSuchUpload',
|
'NoSuchUpload',
|
||||||
@@ -1584,7 +1514,7 @@ const handleUploadPart = async (
|
|||||||
await cleanupTempFile(tempPath);
|
await cleanupTempFile(tempPath);
|
||||||
|
|
||||||
const etag = hasher.digest('hex');
|
const etag = hasher.digest('hex');
|
||||||
await insertMultipartPart({
|
await multipartRepository.insertPart({
|
||||||
uploadId,
|
uploadId,
|
||||||
partNumber,
|
partNumber,
|
||||||
telegramFileId: forwardResult.telegramFileId,
|
telegramFileId: forwardResult.telegramFileId,
|
||||||
@@ -1618,7 +1548,7 @@ const handleCompleteMultipartUpload = async (
|
|||||||
reqId: string,
|
reqId: string,
|
||||||
): Promise<Response> => {
|
): Promise<Response> => {
|
||||||
const uploadId = searchParams.get('uploadId')!;
|
const uploadId = searchParams.get('uploadId')!;
|
||||||
const multipart = await findMultipartUpload(uploadId);
|
const multipart = await multipartRepository.findById(uploadId);
|
||||||
// H5: Verify both upload exists AND key matches (consistent with handleUploadPart)
|
// H5: Verify both upload exists AND key matches (consistent with handleUploadPart)
|
||||||
if (!multipart || multipart.s3Key !== key) {
|
if (!multipart || multipart.s3Key !== key) {
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
@@ -1631,7 +1561,7 @@ const handleCompleteMultipartUpload = async (
|
|||||||
}
|
}
|
||||||
|
|
||||||
const parts = parseCompleteMultipartBody(body);
|
const parts = parseCompleteMultipartBody(body);
|
||||||
const storedParts = await listMultipartParts(uploadId);
|
const storedParts = await multipartRepository.listParts(uploadId);
|
||||||
|
|
||||||
// Validate ascending part order
|
// Validate ascending part order
|
||||||
const partNumbers = parts.map((p) => p.partNumber);
|
const partNumbers = parts.map((p) => p.partNumber);
|
||||||
@@ -1679,32 +1609,30 @@ const handleCompleteMultipartUpload = async (
|
|||||||
const combinedEtag = storedParts.map((p) => p.etag).join('-');
|
const combinedEtag = storedParts.map((p) => p.etag).join('-');
|
||||||
|
|
||||||
const publicId = nanoid();
|
const publicId = nanoid();
|
||||||
const { db, files: fileSchema } = await import('../../../db/index');
|
|
||||||
|
|
||||||
// M7: Use stored content-type from the multipart record if available
|
// M7: Use stored content-type from the multipart record if available
|
||||||
const mimeType = multipart.contentType || 'application/octet-stream';
|
const mimeType = multipart.contentType || 'application/octet-stream';
|
||||||
|
|
||||||
await db.insert(fileSchema).values({
|
await fileRepository.create(
|
||||||
publicId,
|
buildNewFile({
|
||||||
telegramFileId: storedParts[0]!.telegramFileId,
|
publicId,
|
||||||
telegramFileUniqueId: storedParts[0]!.telegramFileUniqueId,
|
telegramFileId: storedParts[0]!.telegramFileId,
|
||||||
storageChatId: config.storageChatId,
|
telegramFileUniqueId: storedParts[0]!.telegramFileUniqueId,
|
||||||
storageMessageId: storedParts[0]!.storageMessageId,
|
storageChatId: config.storageChatId,
|
||||||
fileName: key.split('/').pop() || 'file',
|
storageMessageId: storedParts[0]!.storageMessageId,
|
||||||
mimeType,
|
fileName: key.split('/').pop() || 'file',
|
||||||
sizeBytes: totalSize,
|
mimeType,
|
||||||
fileType: 'document',
|
sizeBytes: totalSize,
|
||||||
uploaderId: 0,
|
fileType: DEFAULT_FILE_TYPE,
|
||||||
bucketId: multipart.bucketId,
|
uploaderId: 0,
|
||||||
s3Key: key,
|
bucketId: multipart.bucketId,
|
||||||
storageBackend: 'telegram',
|
s3Key: key,
|
||||||
isDeleted: false,
|
storageBackend: 'telegram',
|
||||||
multipartUploadId: uploadId,
|
multipartUploadId: uploadId,
|
||||||
createdAt: new Date(),
|
}),
|
||||||
updatedAt: new Date(),
|
);
|
||||||
});
|
|
||||||
|
|
||||||
await completeMultipartUpload(uploadId);
|
await multipartRepository.complete(uploadId);
|
||||||
|
|
||||||
const location = `${config.baseUrl}/${bucket}/${key}`;
|
const location = `${config.baseUrl}/${bucket}/${key}`;
|
||||||
const xml = completeMultipartUploadXml(bucket, key, combinedEtag, location);
|
const xml = completeMultipartUploadXml(bucket, key, combinedEtag, location);
|
||||||
@@ -1725,7 +1653,7 @@ const handleListMultipartUploads = async (
|
|||||||
searchParams: URLSearchParams,
|
searchParams: URLSearchParams,
|
||||||
reqId: string,
|
reqId: string,
|
||||||
): Promise<Response> => {
|
): Promise<Response> => {
|
||||||
const bucketRecord = await findBucketByName(bucket);
|
const bucketRecord = await bucketRepository.findByName(bucket);
|
||||||
if (!bucketRecord)
|
if (!bucketRecord)
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'NoSuchBucket',
|
'NoSuchBucket',
|
||||||
@@ -1737,7 +1665,7 @@ const handleListMultipartUploads = async (
|
|||||||
|
|
||||||
const maxUploads = Math.min(Number.parseInt(searchParams.get('max-uploads') || '1000', 10), 1000);
|
const maxUploads = Math.min(Number.parseInt(searchParams.get('max-uploads') || '1000', 10), 1000);
|
||||||
const keyMarker = searchParams.get('key-marker') || null;
|
const keyMarker = searchParams.get('key-marker') || null;
|
||||||
const { uploads, isTruncated, nextKeyMarker } = await listMultipartUploadsByBucket(
|
const { uploads, isTruncated, nextKeyMarker } = await multipartRepository.listByBucket(
|
||||||
bucketRecord.id,
|
bucketRecord.id,
|
||||||
maxUploads,
|
maxUploads,
|
||||||
keyMarker,
|
keyMarker,
|
||||||
@@ -1776,7 +1704,7 @@ const handleAbortMultipartUpload = async (
|
|||||||
reqId: string,
|
reqId: string,
|
||||||
): Promise<Response> => {
|
): Promise<Response> => {
|
||||||
const uploadId = searchParams.get('uploadId')!;
|
const uploadId = searchParams.get('uploadId')!;
|
||||||
const multipart = await findMultipartUpload(uploadId);
|
const multipart = await multipartRepository.findById(uploadId);
|
||||||
if (!multipart) {
|
if (!multipart) {
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'NoSuchUpload',
|
'NoSuchUpload',
|
||||||
@@ -1787,7 +1715,7 @@ const handleAbortMultipartUpload = async (
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
await abortMultipartUpload(uploadId);
|
await multipartRepository.abort(uploadId);
|
||||||
return s3Response(null, 204, reqId);
|
return s3Response(null, 204, reqId);
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1809,7 +1737,7 @@ const handleListParts = async (
|
|||||||
reqId: string,
|
reqId: string,
|
||||||
): Promise<Response> => {
|
): Promise<Response> => {
|
||||||
const uploadId = searchParams.get('uploadId')!;
|
const uploadId = searchParams.get('uploadId')!;
|
||||||
const multipart = await findMultipartUpload(uploadId);
|
const multipart = await multipartRepository.findById(uploadId);
|
||||||
if (!multipart) {
|
if (!multipart) {
|
||||||
return s3ErrorResponse(
|
return s3ErrorResponse(
|
||||||
'NoSuchUpload',
|
'NoSuchUpload',
|
||||||
@@ -1820,7 +1748,7 @@ const handleListParts = async (
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const parts = await listMultipartParts(uploadId);
|
const parts = await multipartRepository.listParts(uploadId);
|
||||||
const maxParts = Math.min(Number.parseInt(searchParams.get('max-parts') || '1000', 10), 1000);
|
const maxParts = Math.min(Number.parseInt(searchParams.get('max-parts') || '1000', 10), 1000);
|
||||||
|
|
||||||
const xml = listPartsXml(
|
const xml = listPartsXml(
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
import { createWriteStream } from 'node:fs';
|
import { Readable } from 'node:stream';
|
||||||
import { nanoid } from 'nanoid';
|
import { nanoid } from 'nanoid';
|
||||||
import { config } from '../../../config/index';
|
import { buildNewFile } from '../../../domain/entities/file-factory';
|
||||||
import { findFileByHash } from '../../../db/files';
|
import { config } from '../../../env';
|
||||||
|
import { chunkedStorage, fileRepository, telegramService } from '../../../infrastructure/di';
|
||||||
import logger from '../../../shared/logger/index';
|
import logger from '../../../shared/logger/index';
|
||||||
import { metricsCollector } from '../../../shared/metrics/index';
|
import { metricsCollector } from '../../../shared/metrics/index';
|
||||||
import {
|
import {
|
||||||
@@ -14,8 +15,15 @@ import {
|
|||||||
getErrorMessage,
|
getErrorMessage,
|
||||||
getFileType,
|
getFileType,
|
||||||
} from '../../../shared/utils/file';
|
} from '../../../shared/utils/file';
|
||||||
import { storeFileInTelegramChunks } from '../../../utils/chunked-storage';
|
import { streamToTemp } from '../../../shared/utils/temp-stream';
|
||||||
import { enqueuePreparedUpload, type PreparedUpload } from '../../../utils/uploadBatcher';
|
|
||||||
|
/** Prepared upload metadata before submission to storage. */
|
||||||
|
interface PreparedUpload {
|
||||||
|
tempPath: string;
|
||||||
|
fileHash: string;
|
||||||
|
sizeBytes: number;
|
||||||
|
signatureBuffer: Buffer;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Maximum allowed size (in bytes) for a base64 JSON upload.
|
* Maximum allowed size (in bytes) for a base64 JSON upload.
|
||||||
@@ -94,7 +102,7 @@ const rejectOversizedRequest = (req: Request): Response | null => {
|
|||||||
* Streams a multipart `File` to a temporary file on disk while computing
|
* Streams a multipart `File` to a temporary file on disk while computing
|
||||||
* its SHA-256 hash and extracting the signature (first 16 bytes).
|
* its SHA-256 hash and extracting the signature (first 16 bytes).
|
||||||
*
|
*
|
||||||
* Backpressure from the write stream is respected via the drain event.
|
* Delegates to the shared {@link streamToTemp} utility.
|
||||||
*
|
*
|
||||||
* @param file - The multipart `File` object.
|
* @param file - The multipart `File` object.
|
||||||
* @param maxSizeBytes - Maximum allowed file size; an error is thrown if
|
* @param maxSizeBytes - Maximum allowed file size; an error is thrown if
|
||||||
@@ -103,67 +111,8 @@ const rejectOversizedRequest = (req: Request): Response | null => {
|
|||||||
* @throws {Error} When the file size exceeds `maxSizeBytes`.
|
* @throws {Error} When the file size exceeds `maxSizeBytes`.
|
||||||
*/
|
*/
|
||||||
const streamFileToTemp = async (file: File, maxSizeBytes: number): Promise<PreparedUpload> => {
|
const streamFileToTemp = async (file: File, maxSizeBytes: number): Promise<PreparedUpload> => {
|
||||||
const tempPath = `/tmp/filedrop-${nanoid()}`;
|
const result = await streamToTemp(file.stream().getReader(), { maxSizeBytes });
|
||||||
const writer = createWriteStream(tempPath);
|
return result;
|
||||||
const hasher = new Bun.CryptoHasher('sha256');
|
|
||||||
const reader = file.stream().getReader();
|
|
||||||
const signatureChunks: Buffer[] = [];
|
|
||||||
let signatureBytes = 0;
|
|
||||||
let sizeBytes = 0;
|
|
||||||
|
|
||||||
const writeChunk = async (chunk: Buffer): Promise<void> => {
|
|
||||||
if (!writer.write(chunk)) {
|
|
||||||
await new Promise<void>((resolve, reject) => {
|
|
||||||
writer.once('drain', resolve);
|
|
||||||
writer.once('error', reject);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const finishWriter = async (): Promise<void> => {
|
|
||||||
await new Promise<void>((resolve, reject) => {
|
|
||||||
writer.end(() => resolve());
|
|
||||||
writer.once('error', reject);
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
while (true) {
|
|
||||||
const { done, value } = await reader.read();
|
|
||||||
if (done) break;
|
|
||||||
|
|
||||||
const chunk = Buffer.from(value);
|
|
||||||
sizeBytes += chunk.byteLength;
|
|
||||||
if (sizeBytes > maxSizeBytes) {
|
|
||||||
throw new Error('File size exceeds upload limit');
|
|
||||||
}
|
|
||||||
|
|
||||||
hasher.update(chunk);
|
|
||||||
await writeChunk(chunk);
|
|
||||||
|
|
||||||
if (signatureBytes < SIGNATURE_BYTES) {
|
|
||||||
const remaining = SIGNATURE_BYTES - signatureBytes;
|
|
||||||
const signatureChunk = chunk.subarray(0, remaining);
|
|
||||||
signatureChunks.push(signatureChunk);
|
|
||||||
signatureBytes += signatureChunk.byteLength;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
await finishWriter();
|
|
||||||
|
|
||||||
return {
|
|
||||||
tempPath,
|
|
||||||
fileHash: hasher.digest('hex'),
|
|
||||||
sizeBytes,
|
|
||||||
signatureBuffer: Buffer.concat(signatureChunks, signatureBytes),
|
|
||||||
};
|
|
||||||
} catch (error) {
|
|
||||||
writer.destroy();
|
|
||||||
await cleanupTempFile(tempPath);
|
|
||||||
throw error;
|
|
||||||
} finally {
|
|
||||||
reader.releaseLock();
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -223,7 +172,7 @@ const handleMultipartUpload = async (req: Request): Promise<Response> => {
|
|||||||
|
|
||||||
const prepared = await streamFileToTemp(file, config.maxRequestBodyBytes);
|
const prepared = await streamFileToTemp(file, config.maxRequestBodyBytes);
|
||||||
|
|
||||||
const existingFile = await findFileByHash(prepared.fileHash);
|
const existingFile = await fileRepository.findByHash(prepared.fileHash);
|
||||||
if (existingFile) {
|
if (existingFile) {
|
||||||
await cleanupTempFile(prepared.tempPath);
|
await cleanupTempFile(prepared.tempPath);
|
||||||
return Response.json(buildUploadResponse(existingFile, config.baseUrl), { status: 200 });
|
return Response.json(buildUploadResponse(existingFile, config.baseUrl), { status: 200 });
|
||||||
@@ -243,7 +192,7 @@ const handleMultipartUpload = async (req: Request): Promise<Response> => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (prepared.sizeBytes > config.telegramChunkSizeBytes) {
|
if (prepared.sizeBytes > config.telegramChunkSizeBytes) {
|
||||||
const uploadedFile = await storeFileInTelegramChunks({
|
const uploadedFile = await chunkedStorage.storeFileInTelegramChunks({
|
||||||
tempPath: prepared.tempPath,
|
tempPath: prepared.tempPath,
|
||||||
partFileNamePrefix: `direct-${prepared.fileHash?.slice(0, 16) || 'upload'}`,
|
partFileNamePrefix: `direct-${prepared.fileHash?.slice(0, 16) || 'upload'}`,
|
||||||
fileName: finalFileName,
|
fileName: finalFileName,
|
||||||
@@ -256,14 +205,35 @@ const handleMultipartUpload = async (req: Request): Promise<Response> => {
|
|||||||
return Response.json(buildUploadResponse(uploadedFile, config.baseUrl), { status: 200 });
|
return Response.json(buildUploadResponse(uploadedFile, config.baseUrl), { status: 200 });
|
||||||
}
|
}
|
||||||
|
|
||||||
const uploaded = await enqueuePreparedUpload({
|
// Single-message — direct to Telegram storage
|
||||||
prepared,
|
const forwardResult = await telegramService.forwardToStorage(
|
||||||
fileName: finalFileName,
|
Readable.from(Bun.file(prepared.tempPath).stream()),
|
||||||
mimeType,
|
finalFileName,
|
||||||
fileType,
|
fileType,
|
||||||
});
|
);
|
||||||
|
|
||||||
return Response.json(buildUploadResponse(uploaded, config.baseUrl), { status: 200 });
|
const publicId = nanoid();
|
||||||
|
|
||||||
|
const createdFile = await fileRepository.create(
|
||||||
|
buildNewFile({
|
||||||
|
publicId,
|
||||||
|
telegramFileId: forwardResult.telegramFileId,
|
||||||
|
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
|
||||||
|
storageChatId: config.storageChatId,
|
||||||
|
storageMessageId: forwardResult.storageMessageId,
|
||||||
|
fileName: finalFileName,
|
||||||
|
mimeType,
|
||||||
|
sizeBytes: prepared.sizeBytes,
|
||||||
|
fileType,
|
||||||
|
storageBackend: 'telegram',
|
||||||
|
uploaderId: 0,
|
||||||
|
fileHash: prepared.fileHash,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
await cleanupTempFile(prepared.tempPath);
|
||||||
|
|
||||||
|
return Response.json(buildUploadResponse(createdFile, config.baseUrl), { status: 200 });
|
||||||
} catch (error: unknown) {
|
} catch (error: unknown) {
|
||||||
const message = getErrorMessage(error);
|
const message = getErrorMessage(error);
|
||||||
logger.error('Multipart upload error', { error: message });
|
logger.error('Multipart upload error', { error: message });
|
||||||
@@ -317,7 +287,7 @@ const handleJSONUpload = async (req: Request): Promise<Response> => {
|
|||||||
const fileBytes = Buffer.from(base64Data, 'base64');
|
const fileBytes = Buffer.from(base64Data, 'base64');
|
||||||
const hash = computeHash(fileBytes);
|
const hash = computeHash(fileBytes);
|
||||||
|
|
||||||
const existingFile = await findFileByHash(hash);
|
const existingFile = await fileRepository.findByHash(hash);
|
||||||
if (existingFile) {
|
if (existingFile) {
|
||||||
return Response.json(buildUploadResponse(existingFile, config.baseUrl), { status: 200 });
|
return Response.json(buildUploadResponse(existingFile, config.baseUrl), { status: 200 });
|
||||||
}
|
}
|
||||||
@@ -334,7 +304,7 @@ const handleJSONUpload = async (req: Request): Promise<Response> => {
|
|||||||
const prepared = await writeBufferToTemp(fileBytes, hash);
|
const prepared = await writeBufferToTemp(fileBytes, hash);
|
||||||
|
|
||||||
if (prepared.sizeBytes > config.telegramChunkSizeBytes) {
|
if (prepared.sizeBytes > config.telegramChunkSizeBytes) {
|
||||||
const uploadedFile = await storeFileInTelegramChunks({
|
const uploadedFile = await chunkedStorage.storeFileInTelegramChunks({
|
||||||
tempPath: prepared.tempPath,
|
tempPath: prepared.tempPath,
|
||||||
partFileNamePrefix: `direct-${prepared.fileHash?.slice(0, 16) || 'json'}`,
|
partFileNamePrefix: `direct-${prepared.fileHash?.slice(0, 16) || 'json'}`,
|
||||||
fileName: finalFileName,
|
fileName: finalFileName,
|
||||||
@@ -347,14 +317,35 @@ const handleJSONUpload = async (req: Request): Promise<Response> => {
|
|||||||
return Response.json(buildUploadResponse(uploadedFile, config.baseUrl), { status: 200 });
|
return Response.json(buildUploadResponse(uploadedFile, config.baseUrl), { status: 200 });
|
||||||
}
|
}
|
||||||
|
|
||||||
const uploaded = await enqueuePreparedUpload({
|
// Single-message — direct to Telegram storage
|
||||||
prepared,
|
const forwardResult = await telegramService.forwardToStorage(
|
||||||
fileName: finalFileName,
|
Readable.from(Bun.file(prepared.tempPath).stream()),
|
||||||
mimeType,
|
finalFileName,
|
||||||
fileType,
|
fileType,
|
||||||
});
|
);
|
||||||
|
|
||||||
return Response.json(buildUploadResponse(uploaded, config.baseUrl), { status: 200 });
|
const publicId = nanoid();
|
||||||
|
|
||||||
|
const createdFile = await fileRepository.create(
|
||||||
|
buildNewFile({
|
||||||
|
publicId,
|
||||||
|
telegramFileId: forwardResult.telegramFileId,
|
||||||
|
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
|
||||||
|
storageChatId: config.storageChatId,
|
||||||
|
storageMessageId: forwardResult.storageMessageId,
|
||||||
|
fileName: finalFileName,
|
||||||
|
mimeType,
|
||||||
|
sizeBytes: prepared.sizeBytes,
|
||||||
|
fileType,
|
||||||
|
storageBackend: 'telegram',
|
||||||
|
uploaderId: 0,
|
||||||
|
fileHash: prepared.fileHash,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
await cleanupTempFile(prepared.tempPath);
|
||||||
|
|
||||||
|
return Response.json(buildUploadResponse(createdFile, config.baseUrl), { status: 200 });
|
||||||
} catch (error: unknown) {
|
} catch (error: unknown) {
|
||||||
const message = getErrorMessage(error);
|
const message = getErrorMessage(error);
|
||||||
logger.error('JSON upload error', { error: message });
|
logger.error('JSON upload error', { error: message });
|
||||||
|
|||||||
@@ -1,20 +1,17 @@
|
|||||||
import { createReadStream } from 'node:fs';
|
import { createReadStream } from 'node:fs';
|
||||||
import { nanoid } from 'nanoid';
|
import { nanoid } from 'nanoid';
|
||||||
import { config } from '../../../config/index';
|
import { buildNewFile } from '../../../domain/entities/file-factory';
|
||||||
import { createBucket, deleteBucket, findBucketByName, listBuckets } from '../../../db/buckets';
|
import { config } from '../../../env';
|
||||||
import {
|
import { bucketRepository, chunkedStorage, fileRepository } from '../../../infrastructure/di';
|
||||||
countBucketObjects,
|
|
||||||
findFileByBucketAndKey,
|
|
||||||
listObjectsByPrefix,
|
|
||||||
softDeleteFile,
|
|
||||||
} from '../../../db/files-ext';
|
|
||||||
import { botPool } from '../../../infrastructure/telegram/bot-pool';
|
import { botPool } from '../../../infrastructure/telegram/bot-pool';
|
||||||
import logger from '../../../shared/logger/index';
|
import logger from '../../../shared/logger/index';
|
||||||
import { cleanupTempFile, ensureExtension, getErrorMessage } from '../../../shared/utils/file';
|
|
||||||
import {
|
import {
|
||||||
createChunkedObjectResponse,
|
cleanupTempFile,
|
||||||
storeFileInTelegramChunks,
|
DEFAULT_FILE_TYPE,
|
||||||
} from '../../../utils/chunked-storage';
|
ensureExtension,
|
||||||
|
getErrorMessage,
|
||||||
|
} from '../../../shared/utils/file';
|
||||||
|
import { streamToTemp } from '../../../shared/utils/temp-stream';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Route parameters extracted from the URL path.
|
* Route parameters extracted from the URL path.
|
||||||
@@ -47,13 +44,13 @@ const jsonError = (error: string, status: number): Response => Response.json({ e
|
|||||||
* @returns A JSON response with the bucket list.
|
* @returns A JSON response with the bucket list.
|
||||||
*/
|
*/
|
||||||
export const handleListBucketsV1 = async (): Promise<Response> => {
|
export const handleListBucketsV1 = async (): Promise<Response> => {
|
||||||
const buckets = await listBuckets();
|
const buckets = await bucketRepository.list();
|
||||||
const result = await Promise.all(
|
const result = await Promise.all(
|
||||||
buckets.map(async (b) => ({
|
buckets.map(async (b) => ({
|
||||||
id: b.id,
|
id: b.id,
|
||||||
name: b.name,
|
name: b.name,
|
||||||
createdAt: b.createdAt.toISOString(),
|
createdAt: b.createdAt.toISOString(),
|
||||||
objectCount: await countBucketObjects(b.id),
|
objectCount: await fileRepository.countByBucket(b.id),
|
||||||
})),
|
})),
|
||||||
);
|
);
|
||||||
return json({ buckets: result });
|
return json({ buckets: result });
|
||||||
@@ -72,9 +69,9 @@ export const handleCreateBucketV1 = async (req: Request): Promise<Response> => {
|
|||||||
if (!body.name || !/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(body.name)) {
|
if (!body.name || !/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(body.name)) {
|
||||||
return jsonError('Invalid bucket name. Use lowercase, 3-63 chars, no underscore', 400);
|
return jsonError('Invalid bucket name. Use lowercase, 3-63 chars, no underscore', 400);
|
||||||
}
|
}
|
||||||
const existing = await findBucketByName(body.name);
|
const existing = await bucketRepository.findByName(body.name);
|
||||||
if (existing) return jsonError('Bucket already exists', 409);
|
if (existing) return jsonError('Bucket already exists', 409);
|
||||||
const bucket = await createBucket(body.name);
|
const bucket = await bucketRepository.create(body.name);
|
||||||
return json({ id: bucket.id, name: bucket.name }, 201);
|
return json({ id: bucket.id, name: bucket.name }, 201);
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -91,11 +88,11 @@ export const handleDeleteBucketV1 = async (
|
|||||||
_req: Request,
|
_req: Request,
|
||||||
params: RouteParams,
|
params: RouteParams,
|
||||||
): Promise<Response> => {
|
): Promise<Response> => {
|
||||||
const bucket = await findBucketByName(params.bucket!);
|
const bucket = await bucketRepository.findByName(params.bucket!);
|
||||||
if (!bucket) return jsonError('Bucket not found', 404);
|
if (!bucket) return jsonError('Bucket not found', 404);
|
||||||
const count = await countBucketObjects(bucket.id);
|
const count = await fileRepository.countByBucket(bucket.id);
|
||||||
if (count > 0) return jsonError('Bucket is not empty', 409);
|
if (count > 0) return jsonError('Bucket is not empty', 409);
|
||||||
await deleteBucket(params.bucket!);
|
await bucketRepository.delete(params.bucket!);
|
||||||
return json({ success: true });
|
return json({ success: true });
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -109,7 +106,7 @@ export const handleDeleteBucketV1 = async (
|
|||||||
* @returns A JSON response with the object list.
|
* @returns A JSON response with the object list.
|
||||||
*/
|
*/
|
||||||
export const handleListObjectsV1 = async (req: Request, params: RouteParams): Promise<Response> => {
|
export const handleListObjectsV1 = async (req: Request, params: RouteParams): Promise<Response> => {
|
||||||
const bucket = await findBucketByName(params.bucket!);
|
const bucket = await bucketRepository.findByName(params.bucket!);
|
||||||
if (!bucket) return jsonError('Bucket not found', 404);
|
if (!bucket) return jsonError('Bucket not found', 404);
|
||||||
|
|
||||||
const url = new URL(req.url);
|
const url = new URL(req.url);
|
||||||
@@ -118,7 +115,7 @@ export const handleListObjectsV1 = async (req: Request, params: RouteParams): Pr
|
|||||||
const maxKeys = Number.parseInt(url.searchParams.get('max-keys') || '1000', 10);
|
const maxKeys = Number.parseInt(url.searchParams.get('max-keys') || '1000', 10);
|
||||||
const continuationToken = url.searchParams.get('continuation-token') || null;
|
const continuationToken = url.searchParams.get('continuation-token') || null;
|
||||||
|
|
||||||
const { objects, prefixes } = await listObjectsByPrefix(
|
const { objects, prefixes } = await fileRepository.listByPrefix(
|
||||||
bucket.id,
|
bucket.id,
|
||||||
prefix,
|
prefix,
|
||||||
delimiter,
|
delimiter,
|
||||||
@@ -161,7 +158,7 @@ export const handleUploadObjectV1 = async (
|
|||||||
req: Request,
|
req: Request,
|
||||||
params: RouteParams,
|
params: RouteParams,
|
||||||
): Promise<Response> => {
|
): Promise<Response> => {
|
||||||
const bucket = await findBucketByName(params.bucket!);
|
const bucket = await bucketRepository.findByName(params.bucket!);
|
||||||
if (!bucket) return jsonError('Bucket not found', 404);
|
if (!bucket) return jsonError('Bucket not found', 404);
|
||||||
|
|
||||||
const formData = await req.formData();
|
const formData = await req.formData();
|
||||||
@@ -172,67 +169,33 @@ export const handleUploadObjectV1 = async (
|
|||||||
}
|
}
|
||||||
|
|
||||||
const key = (formData.get('key') as string) || file.name;
|
const key = (formData.get('key') as string) || file.name;
|
||||||
const tempPath = `/tmp/filedrop-web-${nanoid()}`;
|
const streamed = await streamToTemp(file.stream().getReader(), { prefix: '/tmp/filedrop-web-' });
|
||||||
const writer = Bun.file(tempPath).writer();
|
|
||||||
const reader = file.stream().getReader();
|
|
||||||
const hasher = new Bun.CryptoHasher('sha256');
|
|
||||||
const SIGNATURE_BYTES = 16;
|
|
||||||
const signatureChunks: Buffer[] = [];
|
|
||||||
let signatureBytes = 0;
|
|
||||||
let sizeBytes = 0;
|
|
||||||
|
|
||||||
try {
|
|
||||||
while (true) {
|
|
||||||
const { done, value } = await reader.read();
|
|
||||||
if (done) break;
|
|
||||||
const chunk = Buffer.from(value);
|
|
||||||
sizeBytes += chunk.byteLength;
|
|
||||||
hasher.update(chunk);
|
|
||||||
writer.write(chunk);
|
|
||||||
if (signatureBytes < SIGNATURE_BYTES) {
|
|
||||||
const remaining = SIGNATURE_BYTES - signatureBytes;
|
|
||||||
const sigChunk = chunk.subarray(0, remaining);
|
|
||||||
signatureChunks.push(sigChunk);
|
|
||||||
signatureBytes += sigChunk.byteLength;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
writer.end();
|
|
||||||
} catch (error) {
|
|
||||||
writer.end();
|
|
||||||
await cleanupTempFile(tempPath);
|
|
||||||
throw error;
|
|
||||||
} finally {
|
|
||||||
reader.releaseLock();
|
|
||||||
}
|
|
||||||
|
|
||||||
const hash = hasher.digest('hex');
|
|
||||||
const signatureBuffer = Buffer.concat(signatureChunks, signatureBytes);
|
|
||||||
const { fileName: finalFileName, mimeType } = ensureExtension(
|
const { fileName: finalFileName, mimeType } = ensureExtension(
|
||||||
key.split('/').pop() || 'file',
|
key.split('/').pop() || 'file',
|
||||||
signatureBuffer,
|
streamed.signatureBuffer,
|
||||||
file.type || 'application/octet-stream',
|
file.type || 'application/octet-stream',
|
||||||
);
|
);
|
||||||
|
|
||||||
const partFileNamePrefix = `s3-${bucket.name}-${key.replace(/\//g, '_')}`;
|
const partFileNamePrefix = `s3-${bucket.name}-${key.replace(/\//g, '_')}`;
|
||||||
|
|
||||||
if (sizeBytes > config.telegramChunkSizeBytes) {
|
if (streamed.sizeBytes > config.telegramChunkSizeBytes) {
|
||||||
const uploadedFile = await storeFileInTelegramChunks({
|
const uploadedFile = await chunkedStorage.storeFileInTelegramChunks({
|
||||||
tempPath,
|
tempPath: streamed.tempPath,
|
||||||
partFileNamePrefix,
|
partFileNamePrefix,
|
||||||
fileName: finalFileName,
|
fileName: finalFileName,
|
||||||
mimeType,
|
mimeType,
|
||||||
sizeBytes,
|
sizeBytes: streamed.sizeBytes,
|
||||||
fileType: 'document',
|
fileType: DEFAULT_FILE_TYPE,
|
||||||
uploaderId: 0,
|
uploaderId: 0,
|
||||||
bucketId: bucket.id,
|
bucketId: bucket.id,
|
||||||
s3Key: key,
|
s3Key: key,
|
||||||
});
|
});
|
||||||
await cleanupTempFile(tempPath);
|
await cleanupTempFile(streamed.tempPath);
|
||||||
return json(
|
return json(
|
||||||
{
|
{
|
||||||
key,
|
key,
|
||||||
size: sizeBytes,
|
size: streamed.sizeBytes,
|
||||||
etag: hash,
|
etag: streamed.fileHash,
|
||||||
downloadUrl: `${config.baseUrl}/f/${uploadedFile.publicId}`,
|
downloadUrl: `${config.baseUrl}/f/${uploadedFile.publicId}`,
|
||||||
},
|
},
|
||||||
201,
|
201,
|
||||||
@@ -240,38 +203,41 @@ export const handleUploadObjectV1 = async (
|
|||||||
}
|
}
|
||||||
|
|
||||||
const forwardResult = await botPool.forwardToStorage(
|
const forwardResult = await botPool.forwardToStorage(
|
||||||
createReadStream(tempPath),
|
createReadStream(streamed.tempPath),
|
||||||
partFileNamePrefix,
|
partFileNamePrefix,
|
||||||
'document',
|
'document',
|
||||||
);
|
);
|
||||||
|
|
||||||
const publicId = nanoid();
|
const publicId = nanoid();
|
||||||
const { db, files: fileSchema } = await import('../../../db/index');
|
|
||||||
|
|
||||||
await db.insert(fileSchema).values({
|
await fileRepository.create(
|
||||||
publicId,
|
buildNewFile({
|
||||||
telegramFileId: forwardResult.telegramFileId,
|
publicId,
|
||||||
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
|
telegramFileId: forwardResult.telegramFileId,
|
||||||
storageChatId: config.storageChatId,
|
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
|
||||||
storageMessageId: forwardResult.storageMessageId,
|
storageChatId: config.storageChatId,
|
||||||
fileName: finalFileName,
|
storageMessageId: forwardResult.storageMessageId,
|
||||||
mimeType,
|
fileName: finalFileName,
|
||||||
sizeBytes,
|
mimeType,
|
||||||
fileType: 'document',
|
sizeBytes: streamed.sizeBytes,
|
||||||
uploaderId: 0,
|
fileType: DEFAULT_FILE_TYPE,
|
||||||
fileHash: hash,
|
uploaderId: 0,
|
||||||
bucketId: bucket.id,
|
fileHash: streamed.fileHash,
|
||||||
s3Key: key,
|
bucketId: bucket.id,
|
||||||
storageBackend: 'telegram',
|
s3Key: key,
|
||||||
isDeleted: false,
|
storageBackend: 'telegram',
|
||||||
createdAt: new Date(),
|
}),
|
||||||
updatedAt: new Date(),
|
);
|
||||||
});
|
|
||||||
|
|
||||||
await cleanupTempFile(tempPath);
|
await cleanupTempFile(streamed.tempPath);
|
||||||
|
|
||||||
return json(
|
return json(
|
||||||
{ key, size: sizeBytes, etag: hash, downloadUrl: `${config.baseUrl}/f/${publicId}` },
|
{
|
||||||
|
key,
|
||||||
|
size: streamed.sizeBytes,
|
||||||
|
etag: streamed.fileHash,
|
||||||
|
downloadUrl: `${config.baseUrl}/f/${publicId}`,
|
||||||
|
},
|
||||||
201,
|
201,
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
@@ -287,9 +253,9 @@ export const handleDeleteObjectV1 = async (
|
|||||||
_req: Request,
|
_req: Request,
|
||||||
params: RouteParams,
|
params: RouteParams,
|
||||||
): Promise<Response> => {
|
): Promise<Response> => {
|
||||||
const bucket = await findBucketByName(params.bucket!);
|
const bucket = await bucketRepository.findByName(params.bucket!);
|
||||||
if (!bucket) return jsonError('Bucket not found', 404);
|
if (!bucket) return jsonError('Bucket not found', 404);
|
||||||
await softDeleteFile(bucket.id, params.key!);
|
await fileRepository.softDelete(bucket.id, params.key!);
|
||||||
return json({ success: true });
|
return json({ success: true });
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -307,15 +273,15 @@ export const handleDownloadObjectV1 = async (
|
|||||||
_req: Request,
|
_req: Request,
|
||||||
params: RouteParams,
|
params: RouteParams,
|
||||||
): Promise<Response> => {
|
): Promise<Response> => {
|
||||||
const bucket = await findBucketByName(params.bucket!);
|
const bucket = await bucketRepository.findByName(params.bucket!);
|
||||||
if (!bucket) return jsonError('Bucket not found', 404);
|
if (!bucket) return jsonError('Bucket not found', 404);
|
||||||
|
|
||||||
const file = await findFileByBucketAndKey(bucket.id, params.key!);
|
const file = await fileRepository.findByBucketAndKey(bucket.id, params.key!);
|
||||||
if (!file) return jsonError('Object not found', 404);
|
if (!file) return jsonError('Object not found', 404);
|
||||||
|
|
||||||
if (file.storageBackend === 'chunked') {
|
if (file.storageBackend === 'chunked') {
|
||||||
const range = { type: 'none' as const };
|
const range = { type: 'none' as const };
|
||||||
return createChunkedObjectResponse({ file, range, reqId: '' });
|
return chunkedStorage.createChunkedObjectResponse({ file, range, reqId: '' });
|
||||||
}
|
}
|
||||||
|
|
||||||
const fileInfo = await botPool.getFileInfo(file.telegramFileId);
|
const fileInfo = await botPool.getFileInfo(file.telegramFileId);
|
||||||
@@ -348,12 +314,12 @@ export const handleCopyObjectV1 = async (req: Request, params: RouteParams): Pro
|
|||||||
}
|
}
|
||||||
|
|
||||||
const destBucketName = body.destBucket || params.bucket!;
|
const destBucketName = body.destBucket || params.bucket!;
|
||||||
const sourceBucket = await findBucketByName(params.bucket!);
|
const sourceBucket = await bucketRepository.findByName(params.bucket!);
|
||||||
const destBucket = await findBucketByName(destBucketName);
|
const destBucket = await bucketRepository.findByName(destBucketName);
|
||||||
|
|
||||||
if (!sourceBucket || !destBucket) return jsonError('Bucket not found', 404);
|
if (!sourceBucket || !destBucket) return jsonError('Bucket not found', 404);
|
||||||
|
|
||||||
const sourceFile = await findFileByBucketAndKey(sourceBucket.id, body.sourceKey);
|
const sourceFile = await fileRepository.findByBucketAndKey(sourceBucket.id, body.sourceKey);
|
||||||
if (!sourceFile) return jsonError('Source object not found', 404);
|
if (!sourceFile) return jsonError('Source object not found', 404);
|
||||||
|
|
||||||
if (sourceFile.storageBackend === 'chunked') {
|
if (sourceFile.storageBackend === 'chunked') {
|
||||||
@@ -361,27 +327,25 @@ export const handleCopyObjectV1 = async (req: Request, params: RouteParams): Pro
|
|||||||
}
|
}
|
||||||
|
|
||||||
const publicId = nanoid();
|
const publicId = nanoid();
|
||||||
const { db, files: fileSchema } = await import('../../../db/index');
|
|
||||||
|
|
||||||
await db.insert(fileSchema).values({
|
await fileRepository.create(
|
||||||
publicId,
|
buildNewFile({
|
||||||
telegramFileId: sourceFile.telegramFileId,
|
publicId,
|
||||||
telegramFileUniqueId: sourceFile.telegramFileUniqueId,
|
telegramFileId: sourceFile.telegramFileId,
|
||||||
storageChatId: sourceFile.storageChatId,
|
telegramFileUniqueId: sourceFile.telegramFileUniqueId,
|
||||||
storageMessageId: sourceFile.storageMessageId,
|
storageChatId: sourceFile.storageChatId,
|
||||||
fileName: sourceFile.fileName,
|
storageMessageId: sourceFile.storageMessageId,
|
||||||
mimeType: sourceFile.mimeType,
|
fileName: sourceFile.fileName,
|
||||||
sizeBytes: sourceFile.sizeBytes,
|
mimeType: sourceFile.mimeType,
|
||||||
fileType: sourceFile.fileType,
|
sizeBytes: Number(sourceFile.sizeBytes),
|
||||||
uploaderId: 0,
|
fileType: sourceFile.fileType,
|
||||||
fileHash: sourceFile.fileHash,
|
uploaderId: 0,
|
||||||
bucketId: destBucket.id,
|
fileHash: sourceFile.fileHash,
|
||||||
s3Key: body.destKey,
|
bucketId: destBucket.id,
|
||||||
storageBackend: 'telegram',
|
s3Key: body.destKey,
|
||||||
isDeleted: false,
|
storageBackend: 'telegram',
|
||||||
createdAt: new Date(),
|
}),
|
||||||
updatedAt: new Date(),
|
);
|
||||||
});
|
|
||||||
|
|
||||||
return json({ sourceKey: body.sourceKey, destKey: body.destKey, destBucket: destBucketName });
|
return json({ sourceKey: body.sourceKey, destKey: body.destKey, destBucket: destBucketName });
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { createHmac, timingSafeEqual } from 'node:crypto';
|
import { createHmac, timingSafeEqual } from 'node:crypto';
|
||||||
import { config } from '../../../config/index';
|
import { config } from '../../../env';
|
||||||
|
|
||||||
const ADMIN_USERNAME = 'admin';
|
const ADMIN_USERNAME = 'admin';
|
||||||
const SIGNATURE_SEPARATOR = '.';
|
const SIGNATURE_SEPARATOR = '.';
|
||||||
|
|||||||
@@ -1,7 +1,89 @@
|
|||||||
export {
|
import { config } from '../../../env';
|
||||||
checkRateLimit,
|
import logger from '../../../shared/logger/index';
|
||||||
cleanupRateLimitCache,
|
import { extractClientIp } from '../../../shared/utils/ip';
|
||||||
clearRateLimitCache,
|
|
||||||
getRateLimitStats,
|
interface RateLimitEntry {
|
||||||
withRateLimit,
|
count: number;
|
||||||
} from '../../../utils/rateLimit';
|
resetTime: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const rateLimitStore = new Map<string, RateLimitEntry>();
|
||||||
|
const MAX_STORE_ENTRIES = 50000;
|
||||||
|
|
||||||
|
const evictExpiredEntries = (now = Date.now()): number => {
|
||||||
|
let cleaned = 0;
|
||||||
|
|
||||||
|
for (const [key, entry] of rateLimitStore.entries()) {
|
||||||
|
if (now > entry.resetTime) {
|
||||||
|
rateLimitStore.delete(key);
|
||||||
|
cleaned++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return cleaned;
|
||||||
|
};
|
||||||
|
|
||||||
|
const ensureStoreCapacity = (now: number): void => {
|
||||||
|
if (rateLimitStore.size < MAX_STORE_ENTRIES) return;
|
||||||
|
|
||||||
|
evictExpiredEntries(now);
|
||||||
|
while (rateLimitStore.size >= MAX_STORE_ENTRIES) {
|
||||||
|
const oldestKey = rateLimitStore.keys().next().value;
|
||||||
|
if (!oldestKey) break;
|
||||||
|
rateLimitStore.delete(oldestKey);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const checkRateLimit = (key: string): boolean => {
|
||||||
|
const now = Date.now();
|
||||||
|
const entry = rateLimitStore.get(key);
|
||||||
|
|
||||||
|
if (!entry || now > entry.resetTime) {
|
||||||
|
ensureStoreCapacity(now);
|
||||||
|
rateLimitStore.set(key, {
|
||||||
|
count: 1,
|
||||||
|
resetTime: now + config.rateLimitWindowMs,
|
||||||
|
});
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (entry.count >= config.rateLimitMaxRequests) {
|
||||||
|
logger.warn('Rate limit exceeded', { key, count: entry.count });
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
entry.count++;
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const withRateLimit = <T extends Request>(
|
||||||
|
handler: (req: T) => Promise<Response>,
|
||||||
|
): ((req: T) => Promise<Response>) => {
|
||||||
|
return async (req: T): Promise<Response> => {
|
||||||
|
const ip = extractClientIp(req);
|
||||||
|
if (!checkRateLimit(ip)) {
|
||||||
|
return Response.json({ error: 'Rate limit exceeded' }, { status: 429 });
|
||||||
|
}
|
||||||
|
|
||||||
|
return handler(req);
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export const cleanupRateLimitCache = (): void => {
|
||||||
|
const cleaned = evictExpiredEntries();
|
||||||
|
|
||||||
|
if (cleaned > 0) {
|
||||||
|
logger.debug('Rate limit cache cleanup', { cleaned, remaining: rateLimitStore.size });
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getRateLimitStats = () => ({
|
||||||
|
trackedIPs: rateLimitStore.size,
|
||||||
|
windowSize: config.rateLimitWindowMs,
|
||||||
|
maxRequests: config.rateLimitMaxRequests,
|
||||||
|
maxTrackedIPs: MAX_STORE_ENTRIES,
|
||||||
|
});
|
||||||
|
|
||||||
|
export const clearRateLimitCache = (): void => {
|
||||||
|
rateLimitStore.clear();
|
||||||
|
};
|
||||||
|
|||||||
@@ -1,7 +1,5 @@
|
|||||||
import { config } from '../../../config/index';
|
|
||||||
import { handleSwaggerHtml, handleSwaggerJson } from '../../../routes/swagger';
|
import { handleSwaggerHtml, handleSwaggerJson } from '../../../routes/swagger';
|
||||||
import { extractS3BucketFromHost } from '../../../utils/s3/virtual-host';
|
import { getS3RouteBucket, shouldHandleS3 } from '../../../shared/utils/s3-detection';
|
||||||
import { isS3Request } from '../../s3/auth';
|
|
||||||
import { handleLogin, handleLogout, handleMe } from '../controllers/auth-controller';
|
import { handleLogin, handleLogout, handleMe } from '../controllers/auth-controller';
|
||||||
import { handleFileInfo, handleFileRedirect } from '../controllers/file-controller';
|
import { handleFileInfo, handleFileRedirect } from '../controllers/file-controller';
|
||||||
import { handleHealth } from '../controllers/health-controller';
|
import { handleHealth } from '../controllers/health-controller';
|
||||||
@@ -12,52 +10,6 @@ import { handleWebApiV1 } from '../controllers/web-api-controller';
|
|||||||
import { requireAuth } from '../middleware/auth';
|
import { requireAuth } from '../middleware/auth';
|
||||||
import { withRateLimit } from '../middleware/rate-limit';
|
import { withRateLimit } from '../middleware/rate-limit';
|
||||||
|
|
||||||
/**
|
|
||||||
* Extracts the S3 bucket name from the request host
|
|
||||||
* if it matches a virtual-hosted-style domain.
|
|
||||||
*
|
|
||||||
* @param req - The incoming HTTP request.
|
|
||||||
* @returns The bucket name if found, or null.
|
|
||||||
*/
|
|
||||||
const getS3RouteBucket = (req: Request): string | null => {
|
|
||||||
const host = req.headers.get('host') || '';
|
|
||||||
return extractS3BucketFromHost(host, config.s3VhostDomains);
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Determines whether the incoming request appears to be an S3 API request
|
|
||||||
* based on host headers, authorization headers, or query parameters.
|
|
||||||
*
|
|
||||||
* @param req - The incoming HTTP request.
|
|
||||||
* @param headers - A record of parsed request headers.
|
|
||||||
* @returns True if the request should be handled by the S3 handler.
|
|
||||||
*/
|
|
||||||
const shouldHandleS3 = (req: Request, headers: Record<string, string>): boolean => {
|
|
||||||
const url = new URL(req.url);
|
|
||||||
return Boolean(
|
|
||||||
getS3RouteBucket(req) || isS3Request(headers) || url.searchParams.has('X-Amz-Signature'),
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handles non-GET requests to the root path by dispatching to the S3 handler
|
|
||||||
* if the request matches S3 patterns (virtual-hosted bucket, S3 auth headers,
|
|
||||||
* or presigned URL signature), or returning a 405 Method Not Allowed otherwise.
|
|
||||||
*
|
|
||||||
* @param req - The incoming HTTP request.
|
|
||||||
* @returns A Response from the S3 handler or a 405 response.
|
|
||||||
*/
|
|
||||||
const _handleMaybeS3Root = (req: Request): Response | Promise<Response> => {
|
|
||||||
if (req.method === 'OPTIONS') {
|
|
||||||
return handleS3Request(req, getS3RouteBucket(req));
|
|
||||||
}
|
|
||||||
const headers = Object.fromEntries(req.headers);
|
|
||||||
if (shouldHandleS3(req, headers)) {
|
|
||||||
return handleS3Request(req, getS3RouteBucket(req));
|
|
||||||
}
|
|
||||||
return new Response('Not Allowed', { status: 405 });
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Dispatches an S3 request directly, bypassing rate limiting.
|
* Dispatches an S3 request directly, bypassing rate limiting.
|
||||||
*
|
*
|
||||||
@@ -106,20 +58,12 @@ export const routes = {
|
|||||||
},
|
},
|
||||||
'/': {
|
'/': {
|
||||||
GET: (req: Request): Promise<Response> => {
|
GET: (req: Request): Promise<Response> => {
|
||||||
const headers = Object.fromEntries(req.headers);
|
if (shouldHandleS3(req)) return handleS3Direct(req);
|
||||||
if (shouldHandleS3(req, headers)) {
|
|
||||||
return handleS3Direct(req);
|
|
||||||
}
|
|
||||||
return handleHome();
|
return handleHome();
|
||||||
},
|
},
|
||||||
PUT: (req: Request): Promise<Response> => {
|
PUT: (req: Request): Promise<Response> => {
|
||||||
if (req.method === 'OPTIONS') {
|
|
||||||
return handleS3Request(req, getS3RouteBucket(req));
|
|
||||||
}
|
|
||||||
const headers = Object.fromEntries(req.headers);
|
const headers = Object.fromEntries(req.headers);
|
||||||
if (shouldHandleS3(req, headers)) {
|
if (shouldHandleS3(req, headers)) return handleS3Direct(req);
|
||||||
return handleS3Direct(req);
|
|
||||||
}
|
|
||||||
return Promise.resolve(new Response('Not Allowed', { status: 405 }));
|
return Promise.resolve(new Response('Not Allowed', { status: 405 }));
|
||||||
},
|
},
|
||||||
HEAD: handleS3Direct,
|
HEAD: handleS3Direct,
|
||||||
@@ -165,8 +109,11 @@ export const routes = {
|
|||||||
'/api/v1/auth/me': {
|
'/api/v1/auth/me': {
|
||||||
GET: handleMe,
|
GET: handleMe,
|
||||||
},
|
},
|
||||||
|
// Read endpoints (GET) are public — anyone can list buckets/objects and
|
||||||
|
// download files. Write endpoints (POST/DELETE/PUT) require admin auth so
|
||||||
|
// visitors cannot upload, edit, copy, or delete.
|
||||||
'/api/v1/*': {
|
'/api/v1/*': {
|
||||||
GET: requireAuth(handleWebApiV1),
|
GET: handleWebApiV1,
|
||||||
POST: requireAuth(handleWebApiV1),
|
POST: requireAuth(handleWebApiV1),
|
||||||
DELETE: requireAuth(handleWebApiV1),
|
DELETE: requireAuth(handleWebApiV1),
|
||||||
PUT: requireAuth(handleWebApiV1),
|
PUT: requireAuth(handleWebApiV1),
|
||||||
|
|||||||
+463
-7
@@ -1,7 +1,463 @@
|
|||||||
export type { SigV4Result, VerifyPresignedUrlInput } from '../../utils/s3/auth';
|
import { timingSafeEqual } from 'node:crypto';
|
||||||
export {
|
|
||||||
buildCanonicalQueryString,
|
/**
|
||||||
isS3Request,
|
* Timing-safe string comparison that prevents timing attacks.
|
||||||
verifyPresignedUrl,
|
*
|
||||||
verifySignature,
|
* Uses `crypto.timingSafeEqual` which runs in constant time regardless of
|
||||||
} from '../../utils/s3/auth';
|
* where the strings differ. Returns false for mismatched-length inputs
|
||||||
|
* to avoid leaking length information via early return.
|
||||||
|
*
|
||||||
|
* @param left - The first string to compare.
|
||||||
|
* @param right - The second string to compare.
|
||||||
|
* @returns True if both strings are equal.
|
||||||
|
*/
|
||||||
|
const timingSafeCompare = (left: string, right: string): boolean => {
|
||||||
|
const leftBuffer = Buffer.from(left);
|
||||||
|
const rightBuffer = Buffer.from(right);
|
||||||
|
|
||||||
|
if (leftBuffer.length !== rightBuffer.length) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return timingSafeEqual(leftBuffer, rightBuffer);
|
||||||
|
};
|
||||||
|
|
||||||
|
export interface SigV4Result {
|
||||||
|
isValid: boolean;
|
||||||
|
credential: {
|
||||||
|
accessKey: string;
|
||||||
|
date: string;
|
||||||
|
region: string;
|
||||||
|
service: string;
|
||||||
|
} | null;
|
||||||
|
errorCode?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface VerifyPresignedUrlInput {
|
||||||
|
url: string;
|
||||||
|
method: string;
|
||||||
|
headers: Record<string, string>;
|
||||||
|
s3AccessKey: string;
|
||||||
|
s3SecretKey: string;
|
||||||
|
region: string;
|
||||||
|
now?: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
const SERVICE = 's3';
|
||||||
|
const TERMINATION = 'aws4_request';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Maximum acceptable clock skew between client and server for header-based
|
||||||
|
* SigV4 authentication. AWS allows 15 minutes.
|
||||||
|
*/
|
||||||
|
const MAX_CLOCK_SKEW_MS = 15 * 60 * 1000;
|
||||||
|
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
|
const buf = (data: string | ArrayBuffer | Uint8Array): Uint8Array => {
|
||||||
|
if (data instanceof Uint8Array) return data;
|
||||||
|
if (data instanceof ArrayBuffer) return new Uint8Array(data);
|
||||||
|
return new TextEncoder().encode(data);
|
||||||
|
};
|
||||||
|
|
||||||
|
const sha256Hex = async (data: string | Uint8Array | ArrayBuffer): Promise<string> => {
|
||||||
|
const hashBuffer = await crypto.subtle.digest('SHA-256', buf(data) as never);
|
||||||
|
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
||||||
|
return hashArray.map((b) => b.toString(16).padStart(2, '0')).join('');
|
||||||
|
};
|
||||||
|
|
||||||
|
const hmacSha256 = async (key: Uint8Array, message: string): Promise<Uint8Array> => {
|
||||||
|
const cryptoKey = await crypto.subtle.importKey(
|
||||||
|
'raw',
|
||||||
|
key as never,
|
||||||
|
{ name: 'HMAC', hash: 'SHA-256' },
|
||||||
|
false,
|
||||||
|
['sign'],
|
||||||
|
);
|
||||||
|
const result = await crypto.subtle.sign('HMAC', cryptoKey, buf(message) as never);
|
||||||
|
return new Uint8Array(result);
|
||||||
|
};
|
||||||
|
|
||||||
|
const getSigningKey = async (
|
||||||
|
secretKey: string,
|
||||||
|
dateStamp: string,
|
||||||
|
region: string,
|
||||||
|
): Promise<Uint8Array> => {
|
||||||
|
let key = await hmacSha256(buf(`AWS4${secretKey}`), dateStamp);
|
||||||
|
key = await hmacSha256(key, region);
|
||||||
|
key = await hmacSha256(key, SERVICE);
|
||||||
|
return await hmacSha256(key, TERMINATION);
|
||||||
|
};
|
||||||
|
|
||||||
|
const hmacHex = async (key: Uint8Array, message: string): Promise<string> => {
|
||||||
|
const result = await hmacSha256(key, message);
|
||||||
|
return Array.from(result)
|
||||||
|
.map((b) => b.toString(16).padStart(2, '0'))
|
||||||
|
.join('');
|
||||||
|
};
|
||||||
|
|
||||||
|
const parseAuthorizationHeader = (authHeader: string) => {
|
||||||
|
const credentialMatch = authHeader.match(/Credential=([^,]+)/);
|
||||||
|
const signedHeadersMatch = authHeader.match(/SignedHeaders=([^,]+)/);
|
||||||
|
const signatureMatch = authHeader.match(/Signature=([^,]+)/);
|
||||||
|
|
||||||
|
if (!credentialMatch || !signedHeadersMatch || !signatureMatch) return null;
|
||||||
|
|
||||||
|
const credentialParts = credentialMatch[1].split('/');
|
||||||
|
if (credentialParts.length !== 5) return null;
|
||||||
|
|
||||||
|
return {
|
||||||
|
accessKey: credentialParts[0],
|
||||||
|
date: credentialParts[1],
|
||||||
|
region: credentialParts[2],
|
||||||
|
service: credentialParts[3],
|
||||||
|
termination: credentialParts[4],
|
||||||
|
signedHeaders: signedHeadersMatch[1],
|
||||||
|
signature: signatureMatch[1],
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const buildCanonicalRequest = (
|
||||||
|
method: string,
|
||||||
|
canonicalUri: string,
|
||||||
|
canonicalQueryString: string,
|
||||||
|
signedHeaders: string,
|
||||||
|
headers: Record<string, string>,
|
||||||
|
hashedPayload: string,
|
||||||
|
): string => {
|
||||||
|
const canonicalHeaders = signedHeaders
|
||||||
|
.split(';')
|
||||||
|
.map((h) => {
|
||||||
|
const value = headers[h.toLowerCase()] || '';
|
||||||
|
return `${h.toLowerCase()}:${value.trim()}\n`;
|
||||||
|
})
|
||||||
|
.join('');
|
||||||
|
|
||||||
|
return `${method}\n${canonicalUri}\n${canonicalQueryString}\n${canonicalHeaders}\n${signedHeaders}\n${hashedPayload}`;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normalizes a URI per AWS SigV4 requirements plus RFC 3986:
|
||||||
|
*
|
||||||
|
* 1. Decode percent-encoded characters
|
||||||
|
* 2. Remove dot-segments (`.` and `..`) per RFC 3986 section 5.2.4
|
||||||
|
*
|
||||||
|
* @param uri - The raw URI path to normalize.
|
||||||
|
* @returns The normalized URI path.
|
||||||
|
*/
|
||||||
|
const normalizeUri = (uri: string): string => {
|
||||||
|
if (!uri || uri === '') return '/';
|
||||||
|
|
||||||
|
// AWS SigV4 requires URI-decoded paths in the canonical request
|
||||||
|
// Only `.` and `..` segments are removed per RFC 3986 section 5.2.4
|
||||||
|
// Empty segments (from `//` or trailing `/`) are preserved — they are
|
||||||
|
// part of the URI and the SDK signs them.
|
||||||
|
const decoded = decodeURIComponent(uri);
|
||||||
|
const segments = decoded.split('/');
|
||||||
|
const result: string[] = [];
|
||||||
|
|
||||||
|
for (const segment of segments) {
|
||||||
|
if (segment === '.') continue;
|
||||||
|
if (segment === '..') {
|
||||||
|
result.pop();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
result.push(segment);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Join preserves empty first segment (from leading /) automatically
|
||||||
|
return result.join('/') || '/';
|
||||||
|
};
|
||||||
|
|
||||||
|
const awsEncode = (value: string): string =>
|
||||||
|
encodeURIComponent(value).replace(
|
||||||
|
/[!'()*]/g,
|
||||||
|
(ch) => `%${ch.charCodeAt(0).toString(16).toUpperCase()}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
export const buildCanonicalQueryString = (
|
||||||
|
searchParams: URLSearchParams,
|
||||||
|
excludeKeys: Set<string> = new Set(),
|
||||||
|
): string => {
|
||||||
|
const pairs: Array<[string, string]> = [];
|
||||||
|
for (const [key, value] of searchParams.entries()) {
|
||||||
|
if (!excludeKeys.has(key)) pairs.push([key, value]);
|
||||||
|
}
|
||||||
|
// AWS SigV4 requires UTF-8 byte-order (code point) comparison, NOT localeCompare
|
||||||
|
pairs.sort(([ak, av], [bk, bv]) => {
|
||||||
|
const a = `${awsEncode(ak)}=${awsEncode(av)}`;
|
||||||
|
const b = `${awsEncode(bk)}=${awsEncode(bv)}`;
|
||||||
|
if (a < b) return -1;
|
||||||
|
if (a > b) return 1;
|
||||||
|
return 0;
|
||||||
|
});
|
||||||
|
return pairs.map(([key, value]) => `${awsEncode(key)}=${awsEncode(value)}`).join('&');
|
||||||
|
};
|
||||||
|
|
||||||
|
const getHashedPayload = async (body: string | null): Promise<string> => {
|
||||||
|
if (!body || body.length === 0) return await sha256Hex('');
|
||||||
|
return await sha256Hex(body);
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parses an AWS SigV4 `x-amz-date` value (e.g. `20260707T120000Z`) into a Date.
|
||||||
|
*
|
||||||
|
* @param amzDate - The date string in `YYYYMMDDTHHmmssZ` format.
|
||||||
|
* @returns The parsed Date, or null if the format is invalid.
|
||||||
|
*/
|
||||||
|
const parseAmzDateUtc = (amzDate: string): Date | null => {
|
||||||
|
const match = amzDate.match(/^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/);
|
||||||
|
if (!match) return null;
|
||||||
|
const [, year, month, day, hour, minute, second] = match;
|
||||||
|
return new Date(
|
||||||
|
Date.UTC(
|
||||||
|
Number.parseInt(year, 10),
|
||||||
|
Number.parseInt(month, 10) - 1,
|
||||||
|
Number.parseInt(day, 10),
|
||||||
|
Number.parseInt(hour, 10),
|
||||||
|
Number.parseInt(minute, 10),
|
||||||
|
Number.parseInt(second, 10),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validates that `host` is included in the signed headers list.
|
||||||
|
*
|
||||||
|
* AWS SigV4 mandates that `host` is always signed. Reject requests that
|
||||||
|
* omit it to prevent header injection / replay variants.
|
||||||
|
*
|
||||||
|
* @param signedHeaders - The semicolon-separated signed headers string.
|
||||||
|
* @returns True if `host` is present.
|
||||||
|
*/
|
||||||
|
const validateSignedHeaders = (signedHeaders: string): boolean => {
|
||||||
|
return signedHeaders.split(';').some((h) => h.toLowerCase() === 'host');
|
||||||
|
};
|
||||||
|
|
||||||
|
export const verifySignature = async (
|
||||||
|
method: string,
|
||||||
|
url: string,
|
||||||
|
headers: Record<string, string>,
|
||||||
|
body: string | null,
|
||||||
|
s3AccessKey: string,
|
||||||
|
s3SecretKey: string,
|
||||||
|
region: string,
|
||||||
|
): Promise<SigV4Result> => {
|
||||||
|
const authHeader = headers.authorization;
|
||||||
|
if (!authHeader?.startsWith('AWS4-HMAC-SHA256')) {
|
||||||
|
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||||
|
}
|
||||||
|
|
||||||
|
const parsed = parseAuthorizationHeader(authHeader);
|
||||||
|
if (!parsed) {
|
||||||
|
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!timingSafeCompare(parsed.accessKey, s3AccessKey)) {
|
||||||
|
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!timingSafeCompare(parsed.region, region)) {
|
||||||
|
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate service and termination in credential scope (M2)
|
||||||
|
if (parsed.service !== SERVICE || parsed.termination !== TERMINATION) {
|
||||||
|
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate host is in signed headers (LOW/host)
|
||||||
|
if (!validateSignedHeaders(parsed.signedHeaders)) {
|
||||||
|
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||||
|
}
|
||||||
|
|
||||||
|
const parsedUrl = new URL(url, 'http://localhost');
|
||||||
|
const canonicalUri = normalizeUri(parsedUrl.pathname);
|
||||||
|
const canonicalQueryString = buildCanonicalQueryString(parsedUrl.searchParams);
|
||||||
|
|
||||||
|
const contentSha256 = headers['x-amz-content-sha256'] || null;
|
||||||
|
if (contentSha256?.startsWith('STREAMING-')) {
|
||||||
|
return { isValid: false, credential: null, errorCode: 'NotImplemented' };
|
||||||
|
}
|
||||||
|
|
||||||
|
// CRITICAL: Use the x-amz-content-sha256 header value in the canonical
|
||||||
|
// request because that's what the client signed. The actual body hash is
|
||||||
|
// verified by verifyBodyHash() after streaming, ensuring integrity without
|
||||||
|
// breaking SigV4.
|
||||||
|
const hashedPayload = contentSha256 || (await getHashedPayload(body));
|
||||||
|
|
||||||
|
const canonicalRequest = buildCanonicalRequest(
|
||||||
|
method,
|
||||||
|
canonicalUri,
|
||||||
|
canonicalQueryString,
|
||||||
|
parsed.signedHeaders,
|
||||||
|
headers,
|
||||||
|
hashedPayload,
|
||||||
|
);
|
||||||
|
|
||||||
|
const hashedCanonicalRequest = await sha256Hex(canonicalRequest);
|
||||||
|
|
||||||
|
// M1: Fall back to Date header if x-amz-date is missing
|
||||||
|
const amzDate = headers['x-amz-date'] || headers.date || '';
|
||||||
|
|
||||||
|
// H5: Validate request freshness (clock skew / replay protection)
|
||||||
|
if (amzDate) {
|
||||||
|
const requestDate = parseAmzDateUtc(amzDate);
|
||||||
|
if (requestDate) {
|
||||||
|
const now = Date.now();
|
||||||
|
const skew = Math.abs(now - requestDate.getTime());
|
||||||
|
if (skew > MAX_CLOCK_SKEW_MS) {
|
||||||
|
return { isValid: false, credential: null, errorCode: 'RequestExpired' };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const dateStamp = parsed.date;
|
||||||
|
|
||||||
|
// M3: Ensure date in credential scope matches x-amz-date
|
||||||
|
if (amzDate) {
|
||||||
|
const amzDateStamp = amzDate.slice(0, 8); // "YYYYMMDD"
|
||||||
|
if (amzDateStamp !== dateStamp) {
|
||||||
|
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const credentialScope = `${dateStamp}/${region}/${parsed.service}/${parsed.termination}`;
|
||||||
|
|
||||||
|
const stringToSign = `AWS4-HMAC-SHA256\n${amzDate}\n${credentialScope}\n${hashedCanonicalRequest}`;
|
||||||
|
|
||||||
|
const signingKey = await getSigningKey(s3SecretKey, dateStamp, region);
|
||||||
|
const expectedSignature = await hmacHex(signingKey, stringToSign);
|
||||||
|
|
||||||
|
if (!timingSafeCompare(expectedSignature, parsed.signature)) {
|
||||||
|
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
isValid: true,
|
||||||
|
credential: {
|
||||||
|
accessKey: parsed.accessKey,
|
||||||
|
date: parsed.date,
|
||||||
|
region: parsed.region,
|
||||||
|
service: parsed.service,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export const verifyPresignedUrl = async ({
|
||||||
|
url,
|
||||||
|
method,
|
||||||
|
headers,
|
||||||
|
s3AccessKey,
|
||||||
|
s3SecretKey,
|
||||||
|
region,
|
||||||
|
now = new Date(),
|
||||||
|
}: VerifyPresignedUrlInput): Promise<SigV4Result> => {
|
||||||
|
const parsedUrl = new URL(url);
|
||||||
|
const searchParams = parsedUrl.searchParams;
|
||||||
|
|
||||||
|
const algorithm = searchParams.get('X-Amz-Algorithm');
|
||||||
|
const credential = searchParams.get('X-Amz-Credential');
|
||||||
|
const signedHeaders = searchParams.get('X-Amz-SignedHeaders');
|
||||||
|
const signature = searchParams.get('X-Amz-Signature');
|
||||||
|
const expiresText = searchParams.get('X-Amz-Expires');
|
||||||
|
const amzDate = searchParams.get('X-Amz-Date');
|
||||||
|
|
||||||
|
if (
|
||||||
|
algorithm !== 'AWS4-HMAC-SHA256' ||
|
||||||
|
!credential ||
|
||||||
|
!signedHeaders ||
|
||||||
|
!signature ||
|
||||||
|
!expiresText ||
|
||||||
|
!amzDate
|
||||||
|
) {
|
||||||
|
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||||
|
}
|
||||||
|
|
||||||
|
const expires = Number.parseInt(expiresText, 10);
|
||||||
|
const signedAt = parseAmzDateUtc(amzDate);
|
||||||
|
if (!Number.isFinite(expires) || expires <= 0 || !signedAt) {
|
||||||
|
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||||
|
}
|
||||||
|
// AWS S3 spec limits presigned URLs to 7 days (604800 seconds)
|
||||||
|
const MAX_PRESIGNED_EXPIRY_SECONDS = 604800;
|
||||||
|
if (
|
||||||
|
now.getTime() > signedAt.getTime() + expires * 1000 ||
|
||||||
|
expires > MAX_PRESIGNED_EXPIRY_SECONDS
|
||||||
|
) {
|
||||||
|
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||||
|
}
|
||||||
|
|
||||||
|
const credParts = credential.split('/');
|
||||||
|
if (credParts.length !== 5) {
|
||||||
|
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||||
|
}
|
||||||
|
const [accessKey, dateStamp, credentialRegion, service, termination] = credParts;
|
||||||
|
if (
|
||||||
|
!timingSafeCompare(accessKey, s3AccessKey) ||
|
||||||
|
!timingSafeCompare(credentialRegion, region) ||
|
||||||
|
service !== SERVICE ||
|
||||||
|
termination !== TERMINATION
|
||||||
|
) {
|
||||||
|
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate host is in signed headers for presigned URLs too
|
||||||
|
if (!validateSignedHeaders(signedHeaders)) {
|
||||||
|
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||||
|
}
|
||||||
|
|
||||||
|
const signedHeaderList = signedHeaders.split(';').filter(Boolean);
|
||||||
|
const canonicalHeaders = signedHeaderList
|
||||||
|
.map((headerName) => {
|
||||||
|
const lower = headerName.toLowerCase();
|
||||||
|
const value = lower === 'host' ? headers.host || parsedUrl.host : headers[lower] || '';
|
||||||
|
return `${lower}:${value.trim()}\n`;
|
||||||
|
})
|
||||||
|
.join('');
|
||||||
|
|
||||||
|
const canonicalRequest = `${method}\n${normalizeUri(parsedUrl.pathname)}\n${buildCanonicalQueryString(searchParams, new Set(['X-Amz-Signature']))}\n${canonicalHeaders}\n${signedHeaders}\nUNSIGNED-PAYLOAD`;
|
||||||
|
const hashedCanonicalRequest = await sha256Hex(canonicalRequest);
|
||||||
|
const credentialScope = `${dateStamp}/${region}/${SERVICE}/${TERMINATION}`;
|
||||||
|
const stringToSign = `AWS4-HMAC-SHA256\n${amzDate}\n${credentialScope}\n${hashedCanonicalRequest}`;
|
||||||
|
const expectedSignature = await hmacHex(
|
||||||
|
await getSigningKey(s3SecretKey, dateStamp, region),
|
||||||
|
stringToSign,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!timingSafeCompare(expectedSignature, signature)) {
|
||||||
|
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||||
|
}
|
||||||
|
return { isValid: true, credential: { accessKey, date: dateStamp, region, service } };
|
||||||
|
};
|
||||||
|
|
||||||
|
export const isS3Request = (headers: Record<string, string>): boolean => {
|
||||||
|
const auth = headers.authorization || '';
|
||||||
|
return auth.startsWith('AWS4-HMAC-SHA256');
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that the actual body SHA-256 matches the `x-amz-content-sha256`
|
||||||
|
* header from the original request.
|
||||||
|
*
|
||||||
|
* This MUST be called AFTER the body has been fully streamed and hashed,
|
||||||
|
* as a second pass after `verifySignature` (which cannot hash a streaming
|
||||||
|
* body without consuming it).
|
||||||
|
*
|
||||||
|
* @param bodySha256 - The SHA-256 hex digest of the actual body content.
|
||||||
|
* @param headers - The original request headers.
|
||||||
|
* @returns An error result on mismatch, or null if the check passes.
|
||||||
|
*/
|
||||||
|
export const verifyBodyHash = (
|
||||||
|
bodySha256: string,
|
||||||
|
headers: Record<string, string>,
|
||||||
|
): SigV4Result | null => {
|
||||||
|
const claimedHash = headers['x-amz-content-sha256'];
|
||||||
|
// If the client sent UNSIGNED-PAYLOAD, skip verification
|
||||||
|
if (!claimedHash || claimedHash === 'UNSIGNED-PAYLOAD' || claimedHash.startsWith('STREAMING-')) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (claimedHash !== bodySha256) {
|
||||||
|
return { isValid: false, credential: null, errorCode: 'BadDigest' };
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
|||||||
@@ -1,10 +1,52 @@
|
|||||||
/**
|
import { nanoid } from 'nanoid';
|
||||||
* Re-export from the canonical headers implementation.
|
|
||||||
*
|
export const S3_CORS_HEADERS: Record<string, string> = {
|
||||||
* @module
|
'access-control-allow-origin': '*',
|
||||||
*/
|
'access-control-allow-methods': 'GET, PUT, HEAD, DELETE, POST, OPTIONS',
|
||||||
export {
|
'access-control-allow-headers': [
|
||||||
applyS3Headers,
|
'Authorization',
|
||||||
S3_CORS_HEADERS,
|
'Content-Type',
|
||||||
s3Headers,
|
'Content-MD5',
|
||||||
} from '../../utils/s3/headers';
|
'Range',
|
||||||
|
'If-Match',
|
||||||
|
'If-None-Match',
|
||||||
|
'If-Modified-Since',
|
||||||
|
'If-Unmodified-Since',
|
||||||
|
'X-Amz-*',
|
||||||
|
'x-amz-*',
|
||||||
|
].join(', '),
|
||||||
|
'access-control-expose-headers': [
|
||||||
|
'Accept-Ranges',
|
||||||
|
'Content-Length',
|
||||||
|
'Content-Range',
|
||||||
|
'Content-Type',
|
||||||
|
'ETag',
|
||||||
|
'Last-Modified',
|
||||||
|
'x-amz-id-2',
|
||||||
|
'x-amz-request-id',
|
||||||
|
].join(', '),
|
||||||
|
'access-control-max-age': '86400',
|
||||||
|
};
|
||||||
|
|
||||||
|
export const s3Headers = (
|
||||||
|
requestId: string,
|
||||||
|
extraHeaders: Record<string, string> = {},
|
||||||
|
): Record<string, string> => ({
|
||||||
|
...S3_CORS_HEADERS,
|
||||||
|
server: 'AmazonS3',
|
||||||
|
...(requestId
|
||||||
|
? {
|
||||||
|
'x-amz-request-id': requestId,
|
||||||
|
'x-amz-id-2': `${requestId}+${nanoid(16)}`,
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
|
...extraHeaders,
|
||||||
|
});
|
||||||
|
|
||||||
|
export const applyS3Headers = (headers: Headers, requestId: string): Headers => {
|
||||||
|
const result = new Headers(headers);
|
||||||
|
for (const [key, value] of Object.entries(s3Headers(requestId))) {
|
||||||
|
result.set(key, value);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
|||||||
@@ -1 +1,27 @@
|
|||||||
export { extractS3BucketFromHost } from '../../utils/s3/virtual-host';
|
const stripPort = (host: string): string => {
|
||||||
|
// Handle IPv6: [::1]:4321 -> [::1]
|
||||||
|
if (host.startsWith('[')) {
|
||||||
|
const closeBracket = host.indexOf(']');
|
||||||
|
return host.slice(0, closeBracket + 1).toLowerCase();
|
||||||
|
}
|
||||||
|
return host.split(':')[0].toLowerCase().replace(/\.$/, '');
|
||||||
|
};
|
||||||
|
|
||||||
|
const isValidBucketLabel = (bucket: string): boolean =>
|
||||||
|
/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket) &&
|
||||||
|
!bucket.includes('..') &&
|
||||||
|
!bucket.includes('.-') &&
|
||||||
|
!bucket.includes('-.');
|
||||||
|
|
||||||
|
export const extractS3BucketFromHost = (host: string, domains: string[]): string | null => {
|
||||||
|
const normalizedHost = stripPort(host);
|
||||||
|
for (const domain of domains) {
|
||||||
|
const normalizedDomain = stripPort(domain);
|
||||||
|
if (!normalizedDomain || normalizedHost === normalizedDomain) continue;
|
||||||
|
if (!normalizedHost.endsWith(`.${normalizedDomain}`)) continue;
|
||||||
|
|
||||||
|
const bucket = normalizedHost.slice(0, -(normalizedDomain.length + 1));
|
||||||
|
return isValidBucketLabel(bucket) ? bucket : null;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
|||||||
+311
-22
@@ -1,22 +1,311 @@
|
|||||||
/**
|
import { s3Headers } from './headers';
|
||||||
* Re-export from the canonical XML implementation.
|
|
||||||
*
|
const escapeXml = (str: string): string =>
|
||||||
* @module
|
str
|
||||||
*/
|
.replace(/&/g, '&')
|
||||||
export {
|
.replace(/</g, '<')
|
||||||
bucketVersioningConfigurationXml,
|
.replace(/>/g, '>')
|
||||||
type CompletePart,
|
.replace(/"/g, '"')
|
||||||
completeMultipartUploadXml,
|
.replace(/'/g, ''');
|
||||||
copyObjectResultXml,
|
|
||||||
deleteResultXml,
|
const isoDate = (d: Date): string => d.toISOString().replace(/\.\d{3}Z$/, 'Z');
|
||||||
initiateMultipartUploadXml,
|
|
||||||
listBucketResultXml,
|
const encodeKey = (value: string, encodingType: string | null = null): string =>
|
||||||
listBucketsXml,
|
encodingType === 'url' ? encodeURIComponent(value) : escapeXml(value);
|
||||||
listBucketV2ResultXml,
|
|
||||||
listMultipartUploadsXml,
|
// ─────── Bucket operations ───────
|
||||||
listPartsXml,
|
|
||||||
parseCompleteMultipartBody,
|
export const listBucketsXml = (
|
||||||
parseDeleteObjectsBody,
|
buckets: { name: string; createdAt: Date }[],
|
||||||
s3ErrorResponse,
|
_requestId: string,
|
||||||
s3ErrorXml,
|
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||||
} from '../../utils/s3/xml';
|
<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||||
|
<Buckets>
|
||||||
|
${buckets
|
||||||
|
.map(
|
||||||
|
(b) => `<Bucket>
|
||||||
|
<Name>${escapeXml(b.name)}</Name>
|
||||||
|
<CreationDate>${isoDate(b.createdAt)}</CreationDate>
|
||||||
|
</Bucket>`,
|
||||||
|
)
|
||||||
|
.join('')}
|
||||||
|
</Buckets>
|
||||||
|
</ListAllMyBucketsResult>`;
|
||||||
|
|
||||||
|
export const bucketVersioningConfigurationXml =
|
||||||
|
(): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`;
|
||||||
|
|
||||||
|
// ─────── Object listing ───────
|
||||||
|
|
||||||
|
export const listBucketResultXml = (
|
||||||
|
bucketName: string,
|
||||||
|
objects: { key: string; sizeBytes: number; etag: string; lastModified: Date; mimeType: string }[],
|
||||||
|
prefixes: string[],
|
||||||
|
isTruncated: boolean,
|
||||||
|
marker: string | null,
|
||||||
|
maxKeys: number,
|
||||||
|
prefix: string,
|
||||||
|
delimiter: string | null,
|
||||||
|
nextMarker: string | null,
|
||||||
|
_requestId: string,
|
||||||
|
encodingType: string | null = null,
|
||||||
|
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||||
|
<Name>${escapeXml(bucketName)}</Name>
|
||||||
|
<Prefix>${encodeKey(prefix, encodingType)}</Prefix>
|
||||||
|
<Marker>${encodeKey(marker || '', encodingType)}</Marker>
|
||||||
|
<MaxKeys>${maxKeys}</MaxKeys>
|
||||||
|
<Delimiter>${encodeKey(delimiter || '', encodingType)}</Delimiter>
|
||||||
|
${encodingType ? `<EncodingType>${escapeXml(encodingType)}</EncodingType>` : ''}
|
||||||
|
<IsTruncated>${isTruncated}</IsTruncated>
|
||||||
|
${objects
|
||||||
|
.map(
|
||||||
|
(o) => `<Contents>
|
||||||
|
<Key>${encodeKey(o.key, encodingType)}</Key>
|
||||||
|
<LastModified>${isoDate(o.lastModified)}</LastModified>
|
||||||
|
<ETag>"${o.etag}"</ETag>
|
||||||
|
<Size>${o.sizeBytes}</Size>
|
||||||
|
<StorageClass>STANDARD</StorageClass>
|
||||||
|
</Contents>`,
|
||||||
|
)
|
||||||
|
.join('')}
|
||||||
|
${prefixes
|
||||||
|
.map(
|
||||||
|
(p) => `<CommonPrefixes>
|
||||||
|
<Prefix>${encodeKey(p, encodingType)}</Prefix>
|
||||||
|
</CommonPrefixes>`,
|
||||||
|
)
|
||||||
|
.join('')}
|
||||||
|
${nextMarker ? `<NextMarker>${encodeKey(nextMarker, encodingType)}</NextMarker>` : ''}
|
||||||
|
</ListBucketResult>`;
|
||||||
|
|
||||||
|
export const listBucketV2ResultXml = (
|
||||||
|
bucketName: string,
|
||||||
|
objects: { key: string; sizeBytes: number; etag: string; lastModified: Date; mimeType: string }[],
|
||||||
|
prefixes: string[],
|
||||||
|
isTruncated: boolean,
|
||||||
|
maxKeys: number,
|
||||||
|
prefix: string,
|
||||||
|
delimiter: string | null,
|
||||||
|
continuationToken: string | null,
|
||||||
|
nextContinuationToken: string | null,
|
||||||
|
keyCount: number,
|
||||||
|
_requestId: string,
|
||||||
|
encodingType: string | null = null,
|
||||||
|
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<ListBucketResultV2 xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||||
|
<Name>${escapeXml(bucketName)}</Name>
|
||||||
|
<Prefix>${encodeKey(prefix, encodingType)}</Prefix>
|
||||||
|
<MaxKeys>${maxKeys}</MaxKeys>
|
||||||
|
<KeyCount>${keyCount}</KeyCount>
|
||||||
|
${delimiter ? `<Delimiter>${encodeKey(delimiter, encodingType)}</Delimiter>` : ''}
|
||||||
|
${encodingType ? `<EncodingType>${escapeXml(encodingType)}</EncodingType>` : ''}
|
||||||
|
${continuationToken ? `<ContinuationToken>${encodeKey(continuationToken, encodingType)}</ContinuationToken>` : ''}
|
||||||
|
<IsTruncated>${isTruncated}</IsTruncated>
|
||||||
|
${objects
|
||||||
|
.map(
|
||||||
|
(o) => `<Contents>
|
||||||
|
<Key>${encodeKey(o.key, encodingType)}</Key>
|
||||||
|
<LastModified>${isoDate(o.lastModified)}</LastModified>
|
||||||
|
<ETag>"${o.etag}"</ETag>
|
||||||
|
<Size>${o.sizeBytes}</Size>
|
||||||
|
<StorageClass>STANDARD</StorageClass>
|
||||||
|
</Contents>`,
|
||||||
|
)
|
||||||
|
.join('')}
|
||||||
|
${prefixes
|
||||||
|
.map(
|
||||||
|
(p) => `<CommonPrefixes>
|
||||||
|
<Prefix>${encodeKey(p, encodingType)}</Prefix>
|
||||||
|
</CommonPrefixes>`,
|
||||||
|
)
|
||||||
|
.join('')}
|
||||||
|
${nextContinuationToken ? `<NextContinuationToken>${encodeKey(nextContinuationToken, encodingType)}</NextContinuationToken>` : ''}
|
||||||
|
</ListBucketResultV2>`;
|
||||||
|
|
||||||
|
// ─────── Multipart ───────
|
||||||
|
|
||||||
|
export const initiateMultipartUploadXml = (
|
||||||
|
bucketName: string,
|
||||||
|
key: string,
|
||||||
|
uploadId: string,
|
||||||
|
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||||
|
<Bucket>${escapeXml(bucketName)}</Bucket>
|
||||||
|
<Key>${escapeXml(key)}</Key>
|
||||||
|
<UploadId>${uploadId}</UploadId>
|
||||||
|
</InitiateMultipartUploadResult>`;
|
||||||
|
|
||||||
|
export const listPartsXml = (
|
||||||
|
bucketName: string,
|
||||||
|
key: string,
|
||||||
|
uploadId: string,
|
||||||
|
parts: { partNumber: number; etag: string; sizeBytes: number; createdAt: Date }[],
|
||||||
|
maxParts: number,
|
||||||
|
isTruncated: boolean,
|
||||||
|
_requestId: string,
|
||||||
|
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<ListPartsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||||
|
<Bucket>${escapeXml(bucketName)}</Bucket>
|
||||||
|
<Key>${escapeXml(key)}</Key>
|
||||||
|
<UploadId>${uploadId}</UploadId>
|
||||||
|
<MaxParts>${maxParts}</MaxParts>
|
||||||
|
<IsTruncated>${isTruncated}</IsTruncated>
|
||||||
|
${parts
|
||||||
|
.map(
|
||||||
|
(p) => `<Part>
|
||||||
|
<PartNumber>${p.partNumber}</PartNumber>
|
||||||
|
<LastModified>${isoDate(p.createdAt)}</LastModified>
|
||||||
|
<ETag>"${p.etag}"</ETag>
|
||||||
|
<Size>${p.sizeBytes}</Size>
|
||||||
|
</Part>`,
|
||||||
|
)
|
||||||
|
.join('')}
|
||||||
|
</ListPartsResult>`;
|
||||||
|
|
||||||
|
export const listMultipartUploadsXml = (
|
||||||
|
bucketName: string,
|
||||||
|
uploads: { key: string; uploadId: string; initiatedAt: Date; initiatedBy: string }[],
|
||||||
|
maxUploads: number,
|
||||||
|
isTruncated: boolean,
|
||||||
|
nextKeyMarker: string | null,
|
||||||
|
_requestId: string,
|
||||||
|
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<ListMultipartUploadsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||||
|
<Bucket>${escapeXml(bucketName)}</Bucket>
|
||||||
|
<KeyMarker></KeyMarker>
|
||||||
|
<UploadIdMarker></UploadIdMarker>
|
||||||
|
${nextKeyMarker ? `<NextKeyMarker>${escapeXml(nextKeyMarker)}</NextKeyMarker>` : ''}
|
||||||
|
<MaxUploads>${maxUploads}</MaxUploads>
|
||||||
|
<IsTruncated>${isTruncated}</IsTruncated>
|
||||||
|
${uploads
|
||||||
|
.map(
|
||||||
|
(u) => `<Upload>
|
||||||
|
<Key>${escapeXml(u.key)}</Key>
|
||||||
|
<UploadId>${u.uploadId}</UploadId>
|
||||||
|
<Initiator><ID>${escapeXml(u.initiatedBy || 's3')}</ID><DisplayName>${escapeXml(u.initiatedBy || 's3')}</DisplayName></Initiator>
|
||||||
|
<Owner><ID>${escapeXml(u.initiatedBy || 's3')}</ID><DisplayName>${escapeXml(u.initiatedBy || 's3')}</DisplayName></Owner>
|
||||||
|
<StorageClass>STANDARD</StorageClass>
|
||||||
|
<Initiated>${isoDate(u.initiatedAt)}</Initiated>
|
||||||
|
</Upload>`,
|
||||||
|
)
|
||||||
|
.join('')}
|
||||||
|
</ListMultipartUploadsResult>`;
|
||||||
|
|
||||||
|
export const completeMultipartUploadXml = (
|
||||||
|
bucketName: string,
|
||||||
|
key: string,
|
||||||
|
etag: string,
|
||||||
|
location: string,
|
||||||
|
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<CompleteMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||||
|
<Location>${escapeXml(location)}</Location>
|
||||||
|
<Bucket>${escapeXml(bucketName)}</Bucket>
|
||||||
|
<Key>${escapeXml(key)}</Key>
|
||||||
|
<ETag>"${etag}"</ETag>
|
||||||
|
</CompleteMultipartUploadResult>`;
|
||||||
|
|
||||||
|
// ─────── Delete result ───────
|
||||||
|
|
||||||
|
export const deleteResultXml = (
|
||||||
|
deleted: string[],
|
||||||
|
errors: { key: string; code: string; message: string }[],
|
||||||
|
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<DeleteResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||||
|
${deleted
|
||||||
|
.map(
|
||||||
|
(key) => `<Deleted>
|
||||||
|
<Key>${escapeXml(key)}</Key>
|
||||||
|
</Deleted>`,
|
||||||
|
)
|
||||||
|
.join('')}
|
||||||
|
${errors
|
||||||
|
.map(
|
||||||
|
(e) => `<Error>
|
||||||
|
<Key>${escapeXml(e.key)}</Key>
|
||||||
|
<Code>${e.code}</Code>
|
||||||
|
<Message>${escapeXml(e.message)}</Message>
|
||||||
|
</Error>`,
|
||||||
|
)
|
||||||
|
.join('')}
|
||||||
|
</DeleteResult>`;
|
||||||
|
|
||||||
|
// ─────── Copy ───────
|
||||||
|
|
||||||
|
export const copyObjectResultXml = (
|
||||||
|
etag: string,
|
||||||
|
lastModified: Date,
|
||||||
|
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<CopyObjectResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||||
|
<ETag>"${etag}"</ETag>
|
||||||
|
<LastModified>${isoDate(lastModified)}</LastModified>
|
||||||
|
</CopyObjectResult>`;
|
||||||
|
|
||||||
|
// ─────── Error ───────
|
||||||
|
|
||||||
|
export const s3ErrorXml = (
|
||||||
|
code: string,
|
||||||
|
message: string,
|
||||||
|
resource: string,
|
||||||
|
requestId: string,
|
||||||
|
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<Error>
|
||||||
|
<Code>${code}</Code>
|
||||||
|
<Message>${escapeXml(message)}</Message>
|
||||||
|
<Resource>${escapeXml(resource)}</Resource>
|
||||||
|
<RequestId>${requestId}</RequestId>
|
||||||
|
<HostId>${requestId}</HostId>
|
||||||
|
</Error>`;
|
||||||
|
|
||||||
|
export const s3ErrorResponse = (
|
||||||
|
code: string,
|
||||||
|
message: string,
|
||||||
|
resource: string,
|
||||||
|
status: number,
|
||||||
|
requestId: string = '',
|
||||||
|
extraHeaders: Record<string, string> = {},
|
||||||
|
): Response =>
|
||||||
|
new Response(s3ErrorXml(code, message, resource, requestId), {
|
||||||
|
status,
|
||||||
|
headers: s3Headers(requestId, {
|
||||||
|
'content-type': 'application/xml',
|
||||||
|
...extraHeaders,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
// ─────── DeleteObjects XML parser ───────
|
||||||
|
|
||||||
|
export const parseDeleteObjectsBody = (body: string): { keys: string[]; quiet: boolean } => {
|
||||||
|
// H9: Use non-greedy match to handle keys containing < character
|
||||||
|
const keys = Array.from(body.matchAll(/<Key>([\s\S]*?)<\/Key>/g), (match) => match[1]);
|
||||||
|
// Handle whitespace inside <Quiet> element + namespace prefix support
|
||||||
|
const quiet = /<\w*:?Quiet\w*>\s*true\s*<\/\w*:?Quiet\w*>/i.test(body);
|
||||||
|
return { keys, quiet };
|
||||||
|
};
|
||||||
|
|
||||||
|
// ─────── CompleteMultipartUpload XML parser ───────
|
||||||
|
|
||||||
|
export interface CompletePart {
|
||||||
|
partNumber: number;
|
||||||
|
etag: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const parseCompleteMultipartBody = (body: string): CompletePart[] => {
|
||||||
|
const parts: CompletePart[] = [];
|
||||||
|
const partRegex = /<Part>[\s\S]*?<\/Part>/g;
|
||||||
|
const partMatch = body.match(partRegex) || [];
|
||||||
|
|
||||||
|
for (const partXml of partMatch) {
|
||||||
|
const numMatch = partXml.match(/<PartNumber>(\d+)<\/PartNumber>/);
|
||||||
|
const etagMatch = partXml.match(/<ETag>"?([^"<\s]+)"?<\/ETag>/);
|
||||||
|
if (numMatch && etagMatch) {
|
||||||
|
parts.push({
|
||||||
|
partNumber: Number.parseInt(numMatch[1], 10),
|
||||||
|
etag: etagMatch[1].replace(/^"/, '').replace(/"$/, ''),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return parts;
|
||||||
|
};
|
||||||
|
|||||||
@@ -1,73 +0,0 @@
|
|||||||
/**
|
|
||||||
* Base domain error class for all application-specific errors.
|
|
||||||
* Extends the built-in Error with a fixed name property for reliable
|
|
||||||
* instance checking across layers.
|
|
||||||
*/
|
|
||||||
export class DomainError extends Error {
|
|
||||||
constructor(msg: string) {
|
|
||||||
super(msg);
|
|
||||||
this.name = 'DomainError';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Thrown when a requested file cannot be found in storage.
|
|
||||||
*/
|
|
||||||
export class FileNotFoundError extends DomainError {
|
|
||||||
constructor(msg: string) {
|
|
||||||
super(msg);
|
|
||||||
this.name = 'FileNotFoundError';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Thrown when a requested bucket does not exist.
|
|
||||||
*/
|
|
||||||
export class BucketNotFoundError extends DomainError {
|
|
||||||
constructor(msg: string) {
|
|
||||||
super(msg);
|
|
||||||
this.name = 'BucketNotFoundError';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Thrown when a file exceeds the maximum allowed size for upload.
|
|
||||||
*/
|
|
||||||
export class FileTooLargeError extends DomainError {
|
|
||||||
constructor(msg: string) {
|
|
||||||
super(msg);
|
|
||||||
this.name = 'FileTooLargeError';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Thrown when an attempt is made to upload a file that already exists
|
|
||||||
* (detected by content hash deduplication).
|
|
||||||
*/
|
|
||||||
export class DuplicateFileError extends DomainError {
|
|
||||||
constructor(msg: string) {
|
|
||||||
super(msg);
|
|
||||||
this.name = 'DuplicateFileError';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Thrown when authentication fails or a valid session is not present.
|
|
||||||
*/
|
|
||||||
export class AuthenticationError extends DomainError {
|
|
||||||
constructor(msg: string) {
|
|
||||||
super(msg);
|
|
||||||
this.name = 'AuthenticationError';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Thrown when input validation fails (e.g. missing required fields,
|
|
||||||
* invalid format, or constraint violations).
|
|
||||||
*/
|
|
||||||
export class ValidationError extends DomainError {
|
|
||||||
constructor(msg: string) {
|
|
||||||
super(msg);
|
|
||||||
this.name = 'ValidationError';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,4 +1,26 @@
|
|||||||
import _logger from '../../utils/logger';
|
import winston from 'winston';
|
||||||
export default _logger;
|
|
||||||
export type { Logger } from 'winston';
|
const logger = winston.createLogger({
|
||||||
export { _logger as logger };
|
level: process.env.LOG_LEVEL || 'info',
|
||||||
|
format: winston.format.combine(
|
||||||
|
winston.format.timestamp(),
|
||||||
|
winston.format.errors({ stack: true }),
|
||||||
|
winston.format.json(),
|
||||||
|
),
|
||||||
|
defaultMeta: { service: 'filedrop' },
|
||||||
|
transports: [
|
||||||
|
// Write all logs including error logs to file
|
||||||
|
new winston.transports.File({ filename: 'logs/error.log', level: 'error' }),
|
||||||
|
new winston.transports.File({ filename: 'logs/combined.log' }),
|
||||||
|
// Console transport for docker logs / CLI visibility
|
||||||
|
new winston.transports.Console({
|
||||||
|
format:
|
||||||
|
process.env.NODE_ENV !== 'production'
|
||||||
|
? winston.format.combine(winston.format.colorize(), winston.format.simple())
|
||||||
|
: winston.format.json(),
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
export default logger;
|
||||||
|
export { logger };
|
||||||
|
|||||||
+111
-1
@@ -1 +1,111 @@
|
|||||||
export { MetricsCollector, metricsCollector } from '../../utils/metrics';
|
// No imports needed — logger used only by setInterval which moved to index.ts
|
||||||
|
|
||||||
|
interface Metric {
|
||||||
|
name: string;
|
||||||
|
value: number;
|
||||||
|
timestamp: number;
|
||||||
|
tags?: Record<string, string>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface MetricsSnapshot {
|
||||||
|
uploadLatency: { p50: number; p95: number; p99: number };
|
||||||
|
uploadThroughput: number;
|
||||||
|
queueSize: number;
|
||||||
|
errorRate: number;
|
||||||
|
cacheHitRate: number;
|
||||||
|
botUtilization: number;
|
||||||
|
timestamp: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
class MetricsCollector {
|
||||||
|
private metrics: Metric[] = [];
|
||||||
|
private uploadTimes: number[] = [];
|
||||||
|
private errorCount = 0;
|
||||||
|
private totalRequests = 0;
|
||||||
|
private cacheHits = 0;
|
||||||
|
private cacheMisses = 0;
|
||||||
|
private maxMetricsSize = 10000;
|
||||||
|
|
||||||
|
recordUploadTime(durationMs: number): void {
|
||||||
|
this.uploadTimes.push(durationMs);
|
||||||
|
this.totalRequests++;
|
||||||
|
|
||||||
|
// Keep only last 1000 measurements
|
||||||
|
if (this.uploadTimes.length > 1000) {
|
||||||
|
this.uploadTimes.shift();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
recordError(): void {
|
||||||
|
this.errorCount++;
|
||||||
|
}
|
||||||
|
|
||||||
|
recordCacheHit(): void {
|
||||||
|
this.cacheHits++;
|
||||||
|
}
|
||||||
|
|
||||||
|
recordCacheMiss(): void {
|
||||||
|
this.cacheMisses++;
|
||||||
|
}
|
||||||
|
|
||||||
|
recordMetric(name: string, value: number, tags?: Record<string, string>): void {
|
||||||
|
this.metrics.push({
|
||||||
|
name,
|
||||||
|
value,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
tags,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Keep metrics bounded
|
||||||
|
if (this.metrics.length > this.maxMetricsSize) {
|
||||||
|
this.metrics = this.metrics.slice(-this.maxMetricsSize);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private calculatePercentile(arr: number[], percentile: number): number {
|
||||||
|
if (arr.length === 0) return 0;
|
||||||
|
const sorted = [...arr].sort((a, b) => a - b);
|
||||||
|
const index = Math.ceil((percentile / 100) * sorted.length) - 1;
|
||||||
|
return sorted[Math.max(0, index)];
|
||||||
|
}
|
||||||
|
|
||||||
|
getSnapshot(): MetricsSnapshot {
|
||||||
|
const errorRate = this.totalRequests > 0 ? (this.errorCount / this.totalRequests) * 100 : 0;
|
||||||
|
const cacheHitRate =
|
||||||
|
this.cacheHits + this.cacheMisses > 0
|
||||||
|
? (this.cacheHits / (this.cacheHits + this.cacheMisses)) * 100
|
||||||
|
: 0;
|
||||||
|
|
||||||
|
return {
|
||||||
|
uploadLatency: {
|
||||||
|
p50: this.calculatePercentile(this.uploadTimes, 50),
|
||||||
|
p95: this.calculatePercentile(this.uploadTimes, 95),
|
||||||
|
p99: this.calculatePercentile(this.uploadTimes, 99),
|
||||||
|
},
|
||||||
|
uploadThroughput: this.totalRequests > 0 ? this.totalRequests / 60 : 0,
|
||||||
|
queueSize: 0, // Will be updated by queue
|
||||||
|
errorRate,
|
||||||
|
cacheHitRate,
|
||||||
|
botUtilization: 0, // Will be updated by bot tracker
|
||||||
|
timestamp: Date.now(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
reset(): void {
|
||||||
|
this.uploadTimes = [];
|
||||||
|
this.errorCount = 0;
|
||||||
|
this.totalRequests = 0;
|
||||||
|
this.cacheHits = 0;
|
||||||
|
this.cacheMisses = 0;
|
||||||
|
this.metrics = [];
|
||||||
|
}
|
||||||
|
|
||||||
|
getMetrics(name?: string): Metric[] {
|
||||||
|
if (!name) return this.metrics;
|
||||||
|
return this.metrics.filter((m) => m.name === name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const metricsCollector = new MetricsCollector();
|
||||||
|
|
||||||
|
export { MetricsCollector };
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
/** Compression algorithm for chunked file storage. */
|
||||||
|
export type CompressionAlgorithm = 'gzip' | null;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Optionally compress a chunk with gzip.
|
||||||
|
*
|
||||||
|
* Compression is skipped if:
|
||||||
|
* - The `compress` flag is false.
|
||||||
|
* - The chunk is smaller than `compressionMinSizeBytes`.
|
||||||
|
* - The compressed result is larger than the original.
|
||||||
|
*
|
||||||
|
* @param chunk - The raw chunk buffer.
|
||||||
|
* @param compress - Whether compression is enabled.
|
||||||
|
* @param compressionMinSizeBytes - Minimum chunk size to attempt compression.
|
||||||
|
* @returns The (possibly compressed) bytes and the algorithm used.
|
||||||
|
*/
|
||||||
|
export const maybeCompressChunk = (
|
||||||
|
chunk: Buffer,
|
||||||
|
compress: boolean,
|
||||||
|
compressionMinSizeBytes: number,
|
||||||
|
): { bytes: Buffer; compressionAlgorithm: CompressionAlgorithm } => {
|
||||||
|
if (!compress || chunk.byteLength < compressionMinSizeBytes) {
|
||||||
|
return { bytes: chunk, compressionAlgorithm: null };
|
||||||
|
}
|
||||||
|
|
||||||
|
const gzipped = Bun.gzipSync(chunk);
|
||||||
|
if (gzipped.byteLength >= chunk.byteLength) {
|
||||||
|
return { bytes: chunk, compressionAlgorithm: null };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { bytes: gzipped, compressionAlgorithm: 'gzip' };
|
||||||
|
};
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
import { unlink } from 'node:fs/promises';
|
import { unlink } from 'node:fs/promises';
|
||||||
import logger from '../../utils/logger';
|
import logger from '../logger/index';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Safely extracts an error message from an unknown value.
|
* Safely extracts an error message from an unknown value.
|
||||||
@@ -51,6 +51,9 @@ interface FileMetadata {
|
|||||||
createdAt: Date | string | number;
|
createdAt: Date | string | number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Default file type used when no specific type can be determined. */
|
||||||
|
export const DEFAULT_FILE_TYPE = 'document';
|
||||||
|
|
||||||
/** Per-file-type size limits in bytes. */
|
/** Per-file-type size limits in bytes. */
|
||||||
const FILE_TYPES: Record<string, number> = {
|
const FILE_TYPES: Record<string, number> = {
|
||||||
document: 2 * 1024 * 1024 * 1024, // 2GB
|
document: 2 * 1024 * 1024 * 1024, // 2GB
|
||||||
@@ -82,7 +85,12 @@ export const getFileType = (mime: string | null, caption?: string): string => {
|
|||||||
if (mimeUpper === 'video') return 'video';
|
if (mimeUpper === 'video') return 'video';
|
||||||
if (mimeUpper === 'audio') return 'audio';
|
if (mimeUpper === 'audio') return 'audio';
|
||||||
if (mimeUpper === 'document') return 'document';
|
if (mimeUpper === 'document') return 'document';
|
||||||
if (mimeUpper === 'image') return captionLower?.includes('gif') ? 'animation' : 'photo';
|
// ── Photo (JPEG only; Telegram Bot API rejects non-JPEG for sendPhoto) ──
|
||||||
|
if (mimeUpper === 'image') {
|
||||||
|
if (captionLower?.includes('gif')) return 'animation';
|
||||||
|
if (mime?.toLowerCase() === 'image/jpeg' || mime?.toLowerCase() === 'image/jpg') return 'photo';
|
||||||
|
return 'document';
|
||||||
|
}
|
||||||
if (captionLower?.includes('voice')) return 'voice';
|
if (captionLower?.includes('voice')) return 'voice';
|
||||||
if (captionLower?.includes('animation')) return 'animation';
|
if (captionLower?.includes('animation')) return 'animation';
|
||||||
|
|
||||||
|
|||||||
@@ -1,135 +0,0 @@
|
|||||||
import logger from '../../utils/logger';
|
|
||||||
|
|
||||||
/** Configuration options for retry behaviour. */
|
|
||||||
interface RetryOptions {
|
|
||||||
/** Maximum number of retry attempts (default: 3). */
|
|
||||||
maxRetries?: number;
|
|
||||||
/** Delay before the first retry in milliseconds (default: 100). */
|
|
||||||
initialDelayMs?: number;
|
|
||||||
/** Maximum delay between retries in milliseconds (default: 5000). */
|
|
||||||
maxDelayMs?: number;
|
|
||||||
/** Multiplier for exponential backoff (default: 2). */
|
|
||||||
backoffMultiplier?: number;
|
|
||||||
/**
|
|
||||||
* Predicate that determines whether a given error should trigger a retry.
|
|
||||||
* When omitted, transient network / timeout errors are retried.
|
|
||||||
*/
|
|
||||||
shouldRetry?: (error: unknown) => boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
const DEFAULT_OPTIONS: Required<RetryOptions> = {
|
|
||||||
maxRetries: 3,
|
|
||||||
initialDelayMs: 100,
|
|
||||||
maxDelayMs: 5000,
|
|
||||||
backoffMultiplier: 2,
|
|
||||||
shouldRetry: (error: unknown) => {
|
|
||||||
const errorStr = error instanceof Error ? error.message : String(error);
|
|
||||||
// Retry on transient errors
|
|
||||||
return (
|
|
||||||
errorStr.includes('ECONNREFUSED') ||
|
|
||||||
errorStr.includes('ETIMEDOUT') ||
|
|
||||||
errorStr.includes('ENOTFOUND') ||
|
|
||||||
errorStr.includes('429') ||
|
|
||||||
errorStr.includes('timeout')
|
|
||||||
);
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Executes an async function with exponential backoff retry logic.
|
|
||||||
*
|
|
||||||
* The function is retried up to `maxRetries` times. Between attempts the
|
|
||||||
* delay grows by `backoffMultiplier` (capped at `maxDelayMs`). Only errors
|
|
||||||
* for which `shouldRetry` returns `true` trigger a retry; all others are
|
|
||||||
* thrown immediately. When all retries are exhausted the last error is
|
|
||||||
* thrown.
|
|
||||||
*
|
|
||||||
* @param fn - The async function to execute.
|
|
||||||
* @param options - Optional retry configuration overrides.
|
|
||||||
* @returns The resolved value of `fn`.
|
|
||||||
*/
|
|
||||||
export const withRetry = async <T>(
|
|
||||||
fn: () => Promise<T>,
|
|
||||||
options: RetryOptions = {},
|
|
||||||
): Promise<T> => {
|
|
||||||
const opts = { ...DEFAULT_OPTIONS, ...options };
|
|
||||||
let lastError: unknown;
|
|
||||||
let delay = opts.initialDelayMs;
|
|
||||||
|
|
||||||
for (let attempt = 0; attempt <= opts.maxRetries; attempt++) {
|
|
||||||
try {
|
|
||||||
return await fn();
|
|
||||||
} catch (error: unknown) {
|
|
||||||
lastError = error;
|
|
||||||
const errorStr = error instanceof Error ? error.message : String(error);
|
|
||||||
|
|
||||||
if (attempt === opts.maxRetries || !opts.shouldRetry(error)) {
|
|
||||||
logger.error('Retry exhausted', {
|
|
||||||
attempt,
|
|
||||||
maxRetries: opts.maxRetries,
|
|
||||||
error: errorStr,
|
|
||||||
});
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.warn('Retrying after error', {
|
|
||||||
attempt,
|
|
||||||
delay,
|
|
||||||
error: errorStr,
|
|
||||||
});
|
|
||||||
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
||||||
delay = Math.min(delay * opts.backoffMultiplier, opts.maxDelayMs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw lastError;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Wraps an async function with a configurable timeout.
|
|
||||||
*
|
|
||||||
* If `fn` does not settle within `timeoutMs` milliseconds the returned
|
|
||||||
* promise rejects with a timeout error. The underlying `fn` continues
|
|
||||||
* executing but its result is ignored.
|
|
||||||
*
|
|
||||||
* @param fn - The async function to execute.
|
|
||||||
* @param timeoutMs - Timeout in milliseconds (default: 30000).
|
|
||||||
* @returns The resolved value of `fn`.
|
|
||||||
*/
|
|
||||||
export const withTimeout = async <T>(
|
|
||||||
fn: () => Promise<T>,
|
|
||||||
timeoutMs: number = 30000,
|
|
||||||
): Promise<T> => {
|
|
||||||
return Promise.race([
|
|
||||||
fn(),
|
|
||||||
new Promise<T>((_, reject) =>
|
|
||||||
setTimeout(() => reject(new Error(`Operation timeout after ${timeoutMs}ms`)), timeoutMs),
|
|
||||||
),
|
|
||||||
]);
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Executes a primary async function and falls back to a secondary function
|
|
||||||
* if the primary throws.
|
|
||||||
*
|
|
||||||
* The fallback function is called only when the primary rejects. If the
|
|
||||||
* fallback also throws the error propagates to the caller.
|
|
||||||
*
|
|
||||||
* @param primary - The primary async function to attempt first.
|
|
||||||
* @param fallback - The fallback async function invoked on failure.
|
|
||||||
* @returns The resolved value of `primary` or, on failure, of `fallback`.
|
|
||||||
*/
|
|
||||||
export const withFallback = async <T>(
|
|
||||||
primary: () => Promise<T>,
|
|
||||||
fallback: () => Promise<T>,
|
|
||||||
): Promise<T> => {
|
|
||||||
try {
|
|
||||||
return await primary();
|
|
||||||
} catch (error: unknown) {
|
|
||||||
logger.warn('Primary operation failed, using fallback', {
|
|
||||||
error: error instanceof Error ? error.message : String(error),
|
|
||||||
});
|
|
||||||
return fallback();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
import { config } from '../../env';
|
||||||
|
import { isS3Request } from '../../interfaces/s3/auth';
|
||||||
|
import { extractS3BucketFromHost } from '../../interfaces/s3/virtual-host';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extracts the S3 bucket name from the request host
|
||||||
|
* if it matches a virtual-hosted-style domain.
|
||||||
|
*
|
||||||
|
* @param req - The incoming HTTP request.
|
||||||
|
* @returns The bucket name if found, or null.
|
||||||
|
*/
|
||||||
|
export const getS3RouteBucket = (req: Request): string | null => {
|
||||||
|
const host = req.headers.get('host') || '';
|
||||||
|
return extractS3BucketFromHost(host, config.s3VhostDomains);
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determines whether the incoming request appears to be an S3 API request
|
||||||
|
* based on host headers, authorization headers, or query parameters.
|
||||||
|
*
|
||||||
|
* @param req - The incoming HTTP request.
|
||||||
|
* @param headers - A record of parsed request headers (optional — derived from req if omitted).
|
||||||
|
* @returns True if the request should be handled by the S3 handler.
|
||||||
|
*/
|
||||||
|
export const shouldHandleS3 = (req: Request, headers?: Record<string, string>): boolean => {
|
||||||
|
const resolvedHeaders = headers ?? Object.fromEntries(req.headers);
|
||||||
|
const url = new URL(req.url);
|
||||||
|
return Boolean(
|
||||||
|
getS3RouteBucket(req) ||
|
||||||
|
isS3Request(resolvedHeaders) ||
|
||||||
|
url.searchParams.has('X-Amz-Signature'),
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -0,0 +1,132 @@
|
|||||||
|
/**
|
||||||
|
* Shared utility for streaming data into a temporary file on disk while
|
||||||
|
* computing its SHA-256 hash (and optionally MD5) and extracting the
|
||||||
|
* signature (first 16 bytes) for magic-byte detection.
|
||||||
|
*
|
||||||
|
* Consolidates the duplicated streaming-to-temp pattern found across
|
||||||
|
* multiple HTTP controllers (s3-controller, upload-controller,
|
||||||
|
* web-api-controller) into a single, reusable function.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { unlink } from 'node:fs/promises';
|
||||||
|
import { nanoid } from 'nanoid';
|
||||||
|
|
||||||
|
/** Options for the {@link streamToTemp} function. */
|
||||||
|
export interface StreamToTempOptions {
|
||||||
|
/** Temporary file path prefix (default: `'/tmp/filedrop-'`). */
|
||||||
|
prefix?: string;
|
||||||
|
/** When true, also compute the MD5 hash (default: false). */
|
||||||
|
computeMd5?: boolean;
|
||||||
|
/** Maximum allowed bytes; throws if the stream exceeds this size. */
|
||||||
|
maxSizeBytes?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Result of a successful {@link streamToTemp} call. */
|
||||||
|
export interface StreamToTempResult {
|
||||||
|
/** Absolute path to the written temp file. */
|
||||||
|
tempPath: string;
|
||||||
|
/** SHA-256 hex digest of the entire stream. */
|
||||||
|
fileHash: string;
|
||||||
|
/** MD5 base-64 digest — only present when `computeMd5` was true. */
|
||||||
|
md5Hash?: string;
|
||||||
|
/** Total number of bytes written. */
|
||||||
|
sizeBytes: number;
|
||||||
|
/** First 16 bytes of the stream (padded with zeros if shorter). */
|
||||||
|
signatureBuffer: Buffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Streams data from a `ReadableStreamDefaultReader` into a temporary file
|
||||||
|
* while computing hashes and extracting the first 16 bytes as a signature
|
||||||
|
* buffer.
|
||||||
|
*
|
||||||
|
* The temp file is cleaned up automatically on error.
|
||||||
|
*
|
||||||
|
* @param reader - A reader obtained from a `ReadableStream`.
|
||||||
|
* @param options - Optional behaviour flags.
|
||||||
|
* @returns A promise resolving with the temp-file metadata.
|
||||||
|
* @throws {Error} If `maxSizeBytes` is exceeded.
|
||||||
|
*/
|
||||||
|
export const streamToTemp = async (
|
||||||
|
reader: ReadableStreamDefaultReader<Uint8Array>,
|
||||||
|
options?: StreamToTempOptions,
|
||||||
|
): Promise<StreamToTempResult> => {
|
||||||
|
const prefix = options?.prefix ?? '/tmp/filedrop-';
|
||||||
|
const computeMd5 = options?.computeMd5 ?? false;
|
||||||
|
const maxSizeBytes = options?.maxSizeBytes;
|
||||||
|
|
||||||
|
const tempPath = `${prefix}${nanoid()}`;
|
||||||
|
const writer = Bun.file(tempPath).writer();
|
||||||
|
const sha256 = new Bun.CryptoHasher('sha256');
|
||||||
|
const md5 = computeMd5 ? new Bun.CryptoHasher('md5') : null;
|
||||||
|
|
||||||
|
const SIGNATURE_BYTES = 16;
|
||||||
|
const signatureChunks: Buffer[] = [];
|
||||||
|
let signatureBytes = 0;
|
||||||
|
let sizeBytes = 0;
|
||||||
|
|
||||||
|
try {
|
||||||
|
while (true) {
|
||||||
|
const { done, value } = await reader.read();
|
||||||
|
if (done) break;
|
||||||
|
|
||||||
|
const chunk = Buffer.from(value);
|
||||||
|
sizeBytes += chunk.byteLength;
|
||||||
|
|
||||||
|
if (maxSizeBytes !== undefined && sizeBytes > maxSizeBytes) {
|
||||||
|
reader.cancel();
|
||||||
|
throw new Error('File size exceeds upload limit');
|
||||||
|
}
|
||||||
|
|
||||||
|
sha256.update(chunk);
|
||||||
|
md5?.update(chunk);
|
||||||
|
writer.write(chunk);
|
||||||
|
|
||||||
|
if (signatureBytes < SIGNATURE_BYTES) {
|
||||||
|
const remaining = SIGNATURE_BYTES - signatureBytes;
|
||||||
|
const sigChunk = chunk.subarray(0, remaining);
|
||||||
|
signatureChunks.push(sigChunk);
|
||||||
|
signatureBytes += sigChunk.byteLength;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
writer.end();
|
||||||
|
} catch {
|
||||||
|
// Writer may have already errored — ignore on success path
|
||||||
|
}
|
||||||
|
|
||||||
|
const result: StreamToTempResult = {
|
||||||
|
tempPath,
|
||||||
|
fileHash: sha256.digest('hex'),
|
||||||
|
sizeBytes,
|
||||||
|
signatureBuffer: Buffer.concat(signatureChunks, signatureBytes),
|
||||||
|
};
|
||||||
|
|
||||||
|
if (md5) {
|
||||||
|
result.md5Hash = md5.digest('base64');
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
try {
|
||||||
|
writer.end();
|
||||||
|
} catch {
|
||||||
|
// ignore writer end failure during error path
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await unlink(tempPath);
|
||||||
|
} catch {
|
||||||
|
// ignore unlink failure
|
||||||
|
}
|
||||||
|
|
||||||
|
throw error;
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
reader.releaseLock();
|
||||||
|
} catch {
|
||||||
|
// ignore release lock failure
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
/**
|
||||||
|
* Maximum allowed chunk/part size in bytes for Telegram storage.
|
||||||
|
*
|
||||||
|
* Telegram Bot API `getFile` can only resolve files up to 20 MB; anything
|
||||||
|
* larger fails with "Bad Request: file is too big". Chunked uploads store
|
||||||
|
* each part as a Telegram document and later resolve it via `getFile`, so a
|
||||||
|
* part must never reach that limit. 19 MB (19922944 bytes) leaves a safety
|
||||||
|
* margin and is the value used in production (/etc/teleuploader/env).
|
||||||
|
*/
|
||||||
|
export const TELEGRAM_CHUNK_SIZE_MAX_BYTES = 19 * 1024 * 1024;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validates a chunk size value and returns it as a safe integer.
|
||||||
|
*
|
||||||
|
* @param chunkSizeBytes - The desired chunk size in bytes.
|
||||||
|
* @returns The same value if it is a positive safe integer at or below
|
||||||
|
* {@link TELEGRAM_CHUNK_SIZE_MAX_BYTES}.
|
||||||
|
* @throws {Error} If the chunk size is not a safe positive integer or exceeds
|
||||||
|
* the Telegram `getFile` limit (with margin).
|
||||||
|
*/
|
||||||
|
export const asSafeChunkSize = (chunkSizeBytes: number): number => {
|
||||||
|
if (!Number.isSafeInteger(chunkSizeBytes) || chunkSizeBytes <= 0) {
|
||||||
|
throw new Error('Invalid Telegram chunk size');
|
||||||
|
}
|
||||||
|
if (chunkSizeBytes > TELEGRAM_CHUNK_SIZE_MAX_BYTES) {
|
||||||
|
throw new Error(
|
||||||
|
`Telegram chunk size ${chunkSizeBytes} exceeds the maximum allowed part size ` +
|
||||||
|
`${TELEGRAM_CHUNK_SIZE_MAX_BYTES} bytes (${TELEGRAM_CHUNK_SIZE_MAX_BYTES / (1024 * 1024)} MB). ` +
|
||||||
|
'Telegram getFile cannot download files larger than 20 MB, so such parts would be undownloadable.',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return chunkSizeBytes;
|
||||||
|
};
|
||||||
@@ -1,205 +0,0 @@
|
|||||||
import { createHmac, timingSafeEqual } from 'node:crypto';
|
|
||||||
import { config } from '../env';
|
|
||||||
|
|
||||||
const ADMIN_USERNAME = 'admin';
|
|
||||||
const SIGNATURE_SEPARATOR = '.';
|
|
||||||
|
|
||||||
type Handler = (req: Request) => Response | Promise<Response>;
|
|
||||||
|
|
||||||
export interface AuthSession {
|
|
||||||
username: string;
|
|
||||||
expiresAt: Date | null;
|
|
||||||
method: 'cookie' | 'bearer';
|
|
||||||
}
|
|
||||||
|
|
||||||
interface CookieOptions {
|
|
||||||
secret?: string;
|
|
||||||
cookieName?: string;
|
|
||||||
maxAgeMs?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface SessionPayload {
|
|
||||||
u: string;
|
|
||||||
e: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
const getSecret = (secret?: string): string => secret ?? config.adminApiToken;
|
|
||||||
const getCookieName = (cookieName?: string): string => cookieName ?? config.sessionCookieName;
|
|
||||||
const getMaxAgeMs = (maxAgeMs?: number): number => maxAgeMs ?? config.sessionMaxAgeMs;
|
|
||||||
|
|
||||||
const encodePayload = (value: string): string => Buffer.from(value, 'utf8').toString('base64url');
|
|
||||||
|
|
||||||
const decodePayload = (value: string): string | null => {
|
|
||||||
try {
|
|
||||||
return Buffer.from(value, 'base64url').toString('utf8');
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const isAuthEnabled = (secret = config.adminApiToken): boolean => secret.length > 0;
|
|
||||||
|
|
||||||
export const timingSafeCompare = (left: string, right: string): boolean => {
|
|
||||||
const leftBuffer = Buffer.from(left);
|
|
||||||
const rightBuffer = Buffer.from(right);
|
|
||||||
|
|
||||||
if (leftBuffer.length !== rightBuffer.length) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return timingSafeEqual(leftBuffer, rightBuffer);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const signCookiePayload = (payload: string, secret: string): string =>
|
|
||||||
createHmac('sha256', secret).update(payload).digest('base64url');
|
|
||||||
|
|
||||||
export const verifyCookieSignature = (cookieValue: string, secret: string): string | null => {
|
|
||||||
const separatorIndex = cookieValue.lastIndexOf(SIGNATURE_SEPARATOR);
|
|
||||||
if (separatorIndex <= 0 || separatorIndex === cookieValue.length - 1) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const payload = cookieValue.slice(0, separatorIndex);
|
|
||||||
const signature = cookieValue.slice(separatorIndex + 1);
|
|
||||||
const expectedSignature = signCookiePayload(payload, secret);
|
|
||||||
|
|
||||||
if (!timingSafeCompare(signature, expectedSignature)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return payload;
|
|
||||||
};
|
|
||||||
|
|
||||||
const cookieAttributes = (maxAgeSeconds: number): string =>
|
|
||||||
[`Max-Age=${maxAgeSeconds}`, 'Path=/', 'HttpOnly', 'SameSite=Lax', 'Secure'].join('; ');
|
|
||||||
|
|
||||||
export const createSessionCookie = (
|
|
||||||
username = ADMIN_USERNAME,
|
|
||||||
options: CookieOptions = {},
|
|
||||||
): string => {
|
|
||||||
const secret = getSecret(options.secret);
|
|
||||||
const cookieName = getCookieName(options.cookieName);
|
|
||||||
const maxAgeMs = getMaxAgeMs(options.maxAgeMs);
|
|
||||||
const expiresAt = Date.now() + maxAgeMs;
|
|
||||||
const payload = encodePayload(
|
|
||||||
JSON.stringify({ u: username, e: expiresAt } satisfies SessionPayload),
|
|
||||||
);
|
|
||||||
const signature = signCookiePayload(payload, secret);
|
|
||||||
const maxAgeSeconds = Math.max(1, Math.floor(maxAgeMs / 1000));
|
|
||||||
|
|
||||||
return `${cookieName}=${payload}${SIGNATURE_SEPARATOR}${signature}; ${cookieAttributes(maxAgeSeconds)}`;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const clearSessionCookie = (cookieName = config.sessionCookieName): string =>
|
|
||||||
`${cookieName}=; ${cookieAttributes(0)}`;
|
|
||||||
|
|
||||||
const findCookieValue = (cookieHeader: string | null, cookieName: string): string | null => {
|
|
||||||
if (!cookieHeader) return null;
|
|
||||||
|
|
||||||
for (const rawCookie of cookieHeader.split(';')) {
|
|
||||||
const cookie = rawCookie.trim();
|
|
||||||
const equalsIndex = cookie.indexOf('=');
|
|
||||||
if (equalsIndex <= 0) continue;
|
|
||||||
|
|
||||||
const name = cookie.slice(0, equalsIndex);
|
|
||||||
if (name === cookieName) {
|
|
||||||
return cookie.slice(equalsIndex + 1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const parseSessionFromCookie = (
|
|
||||||
cookieHeader: string | null,
|
|
||||||
options: Pick<CookieOptions, 'secret' | 'cookieName'> = {},
|
|
||||||
): AuthSession | null => {
|
|
||||||
const secret = getSecret(options.secret);
|
|
||||||
const cookieName = getCookieName(options.cookieName);
|
|
||||||
if (!isAuthEnabled(secret)) return null;
|
|
||||||
|
|
||||||
const cookieValue = findCookieValue(cookieHeader, cookieName);
|
|
||||||
if (!cookieValue) return null;
|
|
||||||
|
|
||||||
const encodedPayload = verifyCookieSignature(cookieValue, secret);
|
|
||||||
if (!encodedPayload) return null;
|
|
||||||
|
|
||||||
const rawPayload = decodePayload(encodedPayload);
|
|
||||||
if (!rawPayload) return null;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const payload = JSON.parse(rawPayload) as Partial<SessionPayload>;
|
|
||||||
if (payload.u !== ADMIN_USERNAME || typeof payload.e !== 'number') return null;
|
|
||||||
if (!Number.isFinite(payload.e) || payload.e <= Date.now()) return null;
|
|
||||||
|
|
||||||
return {
|
|
||||||
username: payload.u,
|
|
||||||
expiresAt: new Date(payload.e),
|
|
||||||
method: 'cookie',
|
|
||||||
};
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const checkBearerToken = (
|
|
||||||
authorizationHeader: string | null,
|
|
||||||
secret = config.adminApiToken,
|
|
||||||
): boolean => {
|
|
||||||
if (!isAuthEnabled(secret) || !authorizationHeader) return false;
|
|
||||||
|
|
||||||
const [scheme, ...rest] = authorizationHeader.split(' ');
|
|
||||||
if (scheme !== 'Bearer' || rest.length === 0) return false;
|
|
||||||
|
|
||||||
const token = rest.join(' ').trim();
|
|
||||||
return token.length > 0 && timingSafeCompare(token, secret);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const getAuthSession = (
|
|
||||||
req: Request,
|
|
||||||
options: Pick<CookieOptions, 'secret' | 'cookieName'> = {},
|
|
||||||
): AuthSession | null => {
|
|
||||||
const secret = getSecret(options.secret);
|
|
||||||
if (!isAuthEnabled(secret)) {
|
|
||||||
return {
|
|
||||||
username: ADMIN_USERNAME,
|
|
||||||
expiresAt: null,
|
|
||||||
method: 'bearer',
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const cookieSession = parseSessionFromCookie(req.headers.get('cookie'), options);
|
|
||||||
if (cookieSession) return cookieSession;
|
|
||||||
|
|
||||||
if (checkBearerToken(req.headers.get('authorization'), secret)) {
|
|
||||||
return {
|
|
||||||
username: ADMIN_USERNAME,
|
|
||||||
expiresAt: null,
|
|
||||||
method: 'bearer',
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const unauthorizedResponse = (): Response =>
|
|
||||||
Response.json({ error: 'Unauthorized' }, { status: 401 });
|
|
||||||
|
|
||||||
export const requireAuth = (
|
|
||||||
handler: Handler,
|
|
||||||
options: Pick<CookieOptions, 'secret' | 'cookieName'> = {},
|
|
||||||
): ((req: Request) => Promise<Response>) => {
|
|
||||||
return async (req: Request): Promise<Response> => {
|
|
||||||
const secret = getSecret(options.secret);
|
|
||||||
if (!isAuthEnabled(secret)) {
|
|
||||||
return handler(req);
|
|
||||||
}
|
|
||||||
|
|
||||||
const session = getAuthSession(req, options);
|
|
||||||
if (!session) {
|
|
||||||
return unauthorizedResponse();
|
|
||||||
}
|
|
||||||
|
|
||||||
return handler(req);
|
|
||||||
};
|
|
||||||
};
|
|
||||||
@@ -1,74 +0,0 @@
|
|||||||
// Simple in-memory cache with TTL support
|
|
||||||
interface CacheEntry<T> {
|
|
||||||
value: T;
|
|
||||||
expiresAt: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
class Cache<T> {
|
|
||||||
private store = new Map<string, CacheEntry<T>>();
|
|
||||||
private ttlMs: number;
|
|
||||||
|
|
||||||
constructor(ttlSeconds: number = 3600) {
|
|
||||||
this.ttlMs = ttlSeconds * 1000;
|
|
||||||
}
|
|
||||||
|
|
||||||
set(key: string, value: T): void {
|
|
||||||
this.store.set(key, {
|
|
||||||
value,
|
|
||||||
expiresAt: Date.now() + this.ttlMs,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
get(key: string): T | null {
|
|
||||||
const entry = this.store.get(key);
|
|
||||||
if (!entry) return null;
|
|
||||||
|
|
||||||
if (Date.now() > entry.expiresAt) {
|
|
||||||
this.store.delete(key);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return entry.value;
|
|
||||||
}
|
|
||||||
|
|
||||||
has(key: string): boolean {
|
|
||||||
return this.get(key) !== null;
|
|
||||||
}
|
|
||||||
|
|
||||||
delete(key: string): void {
|
|
||||||
this.store.delete(key);
|
|
||||||
}
|
|
||||||
|
|
||||||
clear(): void {
|
|
||||||
this.store.clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
size(): number {
|
|
||||||
return this.store.size;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cleanup expired entries
|
|
||||||
cleanup(): number {
|
|
||||||
let removed = 0;
|
|
||||||
const now = Date.now();
|
|
||||||
|
|
||||||
for (const [key, entry] of this.store.entries()) {
|
|
||||||
if (now > entry.expiresAt) {
|
|
||||||
this.store.delete(key);
|
|
||||||
removed++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return removed;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// File info cache (1 hour TTL)
|
|
||||||
export const fileInfoCache = new Cache<{
|
|
||||||
file_size: number;
|
|
||||||
mime_type: string;
|
|
||||||
file_path: string;
|
|
||||||
bot_token: string;
|
|
||||||
}>(3600);
|
|
||||||
|
|
||||||
export { Cache };
|
|
||||||
@@ -1,249 +0,0 @@
|
|||||||
import { randomUUID } from 'node:crypto';
|
|
||||||
import { createReadStream } from 'node:fs';
|
|
||||||
import { gzipSync } from 'node:zlib';
|
|
||||||
import { nanoid } from 'nanoid';
|
|
||||||
import { db, files as fileSchema } from '../db';
|
|
||||||
import { insertFileParts, listFileParts, type NewFilePartInput } from '../db/file-parts';
|
|
||||||
import type { File } from '../db/schema';
|
|
||||||
import { config } from '../env';
|
|
||||||
import { botPool } from '../infrastructure/telegram/bot-pool';
|
|
||||||
import { computeHash } from './file';
|
|
||||||
import { createGetObjectResponse, type ObjectPartSource } from './s3/object-stream';
|
|
||||||
import type { RangeParseResult } from './s3/range';
|
|
||||||
|
|
||||||
export type ChunkCompressionAlgorithm = 'gzip' | null;
|
|
||||||
|
|
||||||
export interface ChunkedUploadPart {
|
|
||||||
partNumber: number;
|
|
||||||
telegramFileId: string;
|
|
||||||
telegramFileUniqueId: string;
|
|
||||||
storageMessageId: number;
|
|
||||||
sizeBytes: number;
|
|
||||||
storedSizeBytes: number;
|
|
||||||
compressionAlgorithm: ChunkCompressionAlgorithm;
|
|
||||||
etag: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ChunkedUploadResult {
|
|
||||||
parts: ChunkedUploadPart[];
|
|
||||||
fileHash: string;
|
|
||||||
totalSizeBytes: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ChunkedFileInput {
|
|
||||||
tempPath: string;
|
|
||||||
partFileNamePrefix: string;
|
|
||||||
fileName: string;
|
|
||||||
mimeType: string;
|
|
||||||
sizeBytes: number;
|
|
||||||
fileType: string;
|
|
||||||
uploaderId: number;
|
|
||||||
bucketId?: string | null;
|
|
||||||
s3Key?: string | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const asSafeChunkSize = (chunkSizeBytes: number): number => {
|
|
||||||
if (!Number.isSafeInteger(chunkSizeBytes) || chunkSizeBytes <= 0) {
|
|
||||||
throw new Error('Invalid Telegram chunk size');
|
|
||||||
}
|
|
||||||
return chunkSizeBytes;
|
|
||||||
};
|
|
||||||
|
|
||||||
const maybeCompressChunk = (
|
|
||||||
chunk: Buffer,
|
|
||||||
compress: boolean,
|
|
||||||
compressionMinSizeBytes: number,
|
|
||||||
): { bytes: Buffer; compressionAlgorithm: ChunkCompressionAlgorithm } => {
|
|
||||||
if (!compress || chunk.byteLength < compressionMinSizeBytes) {
|
|
||||||
return { bytes: chunk, compressionAlgorithm: null };
|
|
||||||
}
|
|
||||||
|
|
||||||
const gzipped = gzipSync(chunk);
|
|
||||||
if (gzipped.byteLength >= chunk.byteLength) {
|
|
||||||
return { bytes: chunk, compressionAlgorithm: null };
|
|
||||||
}
|
|
||||||
|
|
||||||
return { bytes: gzipped, compressionAlgorithm: 'gzip' };
|
|
||||||
};
|
|
||||||
|
|
||||||
export const uploadFileInTelegramChunks = async (input: {
|
|
||||||
tempPath: string;
|
|
||||||
partFileNamePrefix: string;
|
|
||||||
chunkSizeBytes: number;
|
|
||||||
compress: boolean;
|
|
||||||
compressionMinSizeBytes: number;
|
|
||||||
}): Promise<ChunkedUploadResult> => {
|
|
||||||
const chunkSizeBytes = asSafeChunkSize(input.chunkSizeBytes);
|
|
||||||
const hasher = new Bun.CryptoHasher('sha256');
|
|
||||||
const parts: ChunkedUploadPart[] = [];
|
|
||||||
let totalSizeBytes = 0;
|
|
||||||
let partNumber = 0;
|
|
||||||
|
|
||||||
const stream = createReadStream(input.tempPath, { highWaterMark: chunkSizeBytes });
|
|
||||||
|
|
||||||
// Concurrent upload set: tracks in-flight uploads and limits how many
|
|
||||||
// chunks are being uploaded at once from this single file.
|
|
||||||
const inFlight = new Set<Promise<void>>();
|
|
||||||
|
|
||||||
for await (const data of stream) {
|
|
||||||
const chunk = Buffer.isBuffer(data) ? data : Buffer.from(data as Uint8Array);
|
|
||||||
if (chunk.byteLength === 0) continue;
|
|
||||||
|
|
||||||
partNumber += 1;
|
|
||||||
totalSizeBytes += chunk.byteLength;
|
|
||||||
hasher.update(chunk);
|
|
||||||
|
|
||||||
const { bytes, compressionAlgorithm } = maybeCompressChunk(
|
|
||||||
chunk,
|
|
||||||
input.compress,
|
|
||||||
input.compressionMinSizeBytes,
|
|
||||||
);
|
|
||||||
const currentPart = partNumber;
|
|
||||||
|
|
||||||
// Fire upload concurrently — don't await inside the read loop
|
|
||||||
const uploadPromise = botPool
|
|
||||||
.forwardToStorage(bytes, `${input.partFileNamePrefix}.part-${currentPart}`, 'document')
|
|
||||||
.then((forwardResult) => {
|
|
||||||
parts.push({
|
|
||||||
partNumber: currentPart,
|
|
||||||
telegramFileId: forwardResult.telegramFileId,
|
|
||||||
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
|
|
||||||
storageMessageId: forwardResult.storageMessageId,
|
|
||||||
sizeBytes: chunk.byteLength,
|
|
||||||
storedSizeBytes: bytes.byteLength,
|
|
||||||
compressionAlgorithm,
|
|
||||||
etag: computeHash(chunk),
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
// Clean up from in-flight set when done (regardless of success/failure)
|
|
||||||
const trackPromise = uploadPromise.finally(() => {
|
|
||||||
inFlight.delete(trackPromise);
|
|
||||||
});
|
|
||||||
|
|
||||||
inFlight.add(trackPromise);
|
|
||||||
|
|
||||||
// Backpressure: if too many chunks are in-flight, wait for one to
|
|
||||||
// finish before reading more — prevents unbounded memory growth.
|
|
||||||
if (inFlight.size >= config.uploadConcurrency * 2) {
|
|
||||||
await Promise.race(inFlight);
|
|
||||||
// Yield microtask to let .finally() run and remove from inFlight
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wait for all remaining uploads to finish
|
|
||||||
await Promise.all(inFlight);
|
|
||||||
|
|
||||||
return {
|
|
||||||
parts,
|
|
||||||
fileHash: hasher.digest('hex'),
|
|
||||||
totalSizeBytes,
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export const storeFileInTelegramChunks = async (input: ChunkedFileInput): Promise<File> => {
|
|
||||||
const upload = await uploadFileInTelegramChunks({
|
|
||||||
tempPath: input.tempPath,
|
|
||||||
partFileNamePrefix: input.partFileNamePrefix,
|
|
||||||
chunkSizeBytes: config.telegramChunkSizeBytes,
|
|
||||||
compress: config.compressChunkedUploads,
|
|
||||||
compressionMinSizeBytes: config.chunkCompressionMinSizeBytes,
|
|
||||||
});
|
|
||||||
|
|
||||||
const firstPart = upload.parts[0];
|
|
||||||
if (!firstPart) {
|
|
||||||
throw new Error('Chunked upload produced no parts');
|
|
||||||
}
|
|
||||||
|
|
||||||
const now = new Date();
|
|
||||||
const fileId = randomUUID();
|
|
||||||
const publicId = nanoid();
|
|
||||||
const file: File = {
|
|
||||||
id: fileId,
|
|
||||||
publicId,
|
|
||||||
telegramFileId: firstPart.telegramFileId,
|
|
||||||
telegramFileUniqueId: firstPart.telegramFileUniqueId,
|
|
||||||
storageChatId: config.storageChatId,
|
|
||||||
storageMessageId: firstPart.storageMessageId,
|
|
||||||
fileName: input.fileName,
|
|
||||||
mimeType: input.mimeType,
|
|
||||||
sizeBytes: upload.totalSizeBytes,
|
|
||||||
fileType: input.fileType,
|
|
||||||
uploaderId: input.uploaderId,
|
|
||||||
fileHash: upload.fileHash,
|
|
||||||
archiveTelegramFileId: null,
|
|
||||||
archiveStorageMessageId: null,
|
|
||||||
archiveFileName: null,
|
|
||||||
archiveEntryName: null,
|
|
||||||
archiveMimeType: null,
|
|
||||||
archiveSizeBytes: null,
|
|
||||||
bucketId: input.bucketId ?? null,
|
|
||||||
s3Key: input.s3Key ?? null,
|
|
||||||
storageBackend: 'chunked',
|
|
||||||
isDeleted: false,
|
|
||||||
multipartUploadId: null,
|
|
||||||
partCount: upload.parts.length,
|
|
||||||
createdAt: now,
|
|
||||||
updatedAt: now,
|
|
||||||
};
|
|
||||||
|
|
||||||
await db.insert(fileSchema).values(file);
|
|
||||||
|
|
||||||
const fileParts: NewFilePartInput[] = upload.parts.map((part) => ({
|
|
||||||
fileId,
|
|
||||||
partNumber: part.partNumber,
|
|
||||||
telegramFileId: part.telegramFileId,
|
|
||||||
telegramFileUniqueId: part.telegramFileUniqueId,
|
|
||||||
storageChatId: config.storageChatId,
|
|
||||||
storageMessageId: part.storageMessageId,
|
|
||||||
sizeBytes: part.sizeBytes,
|
|
||||||
storedSizeBytes: part.storedSizeBytes,
|
|
||||||
compressionAlgorithm: part.compressionAlgorithm,
|
|
||||||
etag: part.etag,
|
|
||||||
}));
|
|
||||||
|
|
||||||
await insertFileParts(fileParts);
|
|
||||||
return file;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const buildChunkedObjectSources = async (file: File): Promise<ObjectPartSource[]> => {
|
|
||||||
const parts = await listFileParts(file.id);
|
|
||||||
const sources: ObjectPartSource[] = [];
|
|
||||||
|
|
||||||
for (const part of parts) {
|
|
||||||
const fileInfo = await botPool.getFileInfo(part.telegramFileId);
|
|
||||||
sources.push({
|
|
||||||
telegramFileId: part.telegramFileId,
|
|
||||||
telegramUrl: `https://api.telegram.org/file/bot${fileInfo.bot_token}/${fileInfo.file_path}`,
|
|
||||||
sizeBytes: part.sizeBytes,
|
|
||||||
storedSizeBytes: part.storedSizeBytes,
|
|
||||||
compressionAlgorithm: part.compressionAlgorithm,
|
|
||||||
partNumber: part.partNumber,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return sources;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const createChunkedObjectResponse = async (input: {
|
|
||||||
file: File;
|
|
||||||
range: RangeParseResult;
|
|
||||||
reqId: string;
|
|
||||||
}): Promise<Response> => {
|
|
||||||
const parts = await buildChunkedObjectSources(input.file);
|
|
||||||
if (parts.length === 0) {
|
|
||||||
throw new Error('Chunked object has no parts');
|
|
||||||
}
|
|
||||||
|
|
||||||
return createGetObjectResponse({
|
|
||||||
reqId: input.reqId,
|
|
||||||
contentType: input.file.mimeType,
|
|
||||||
etag: input.file.fileHash || parts.map((p) => p.telegramFileId).join('-'),
|
|
||||||
lastModified:
|
|
||||||
input.file.createdAt instanceof Date ? input.file.createdAt : new Date(input.file.createdAt),
|
|
||||||
totalSize: Number(input.file.sizeBytes),
|
|
||||||
parts,
|
|
||||||
range: input.range,
|
|
||||||
});
|
|
||||||
};
|
|
||||||
@@ -1,237 +0,0 @@
|
|||||||
import { unlink } from 'node:fs/promises';
|
|
||||||
import logger from './logger';
|
|
||||||
|
|
||||||
export const getErrorMessage = (error: unknown): string => {
|
|
||||||
return error instanceof Error ? error.message : String(error);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const cleanupTempFile = async (tempPath: string): Promise<void> => {
|
|
||||||
try {
|
|
||||||
await unlink(tempPath);
|
|
||||||
} catch (err) {
|
|
||||||
logger.warn('Failed to cleanup temp file', { tempPath, error: getErrorMessage(err) });
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
interface FileMetadata {
|
|
||||||
publicId: string;
|
|
||||||
telegramFileId: string;
|
|
||||||
telegramFileUniqueId: string;
|
|
||||||
storageChatId: number;
|
|
||||||
storageMessageId: number;
|
|
||||||
fileName: string;
|
|
||||||
mimeType: string;
|
|
||||||
sizeBytes: number;
|
|
||||||
fileType: string;
|
|
||||||
uploaderId: number;
|
|
||||||
createdAt: Date | string | number;
|
|
||||||
}
|
|
||||||
|
|
||||||
const FILE_TYPES: Record<string, number> = {
|
|
||||||
document: 2 * 1024 * 1024 * 1024, // 2GB
|
|
||||||
photo: 10 * 1024 * 1024, // 10MB
|
|
||||||
video: 2 * 1024 * 1024 * 1024, // 2GB
|
|
||||||
audio: 200 * 1024 * 1024, // 200MB
|
|
||||||
voice: 200 * 1024 * 1024, // 200MB
|
|
||||||
animation: 2 * 1024 * 1024 * 1024, // 2GB
|
|
||||||
sticker: 10 * 1024 * 1024, // 10MB
|
|
||||||
video_note: 2 * 1024 * 1024 * 1024, // 2GB
|
|
||||||
};
|
|
||||||
|
|
||||||
export const getFileType = (mime: string | null, caption?: string): string => {
|
|
||||||
const mimeUpper = mime?.split('/')[0]?.toLowerCase();
|
|
||||||
const captionLower = caption?.toLowerCase();
|
|
||||||
|
|
||||||
if (mime?.toLowerCase() === 'image/webp' || captionLower?.includes('sticker')) return 'sticker';
|
|
||||||
if (captionLower?.includes('video_note')) return 'video_note';
|
|
||||||
if (mimeUpper === 'video') return 'video';
|
|
||||||
if (mimeUpper === 'audio') return 'audio';
|
|
||||||
if (mimeUpper === 'document') return 'document';
|
|
||||||
if (mimeUpper === 'image') return captionLower?.includes('gif') ? 'animation' : 'photo';
|
|
||||||
if (captionLower?.includes('voice')) return 'voice';
|
|
||||||
if (captionLower?.includes('animation')) return 'animation';
|
|
||||||
|
|
||||||
return mimeUpper === 'application' ? 'application' : 'document';
|
|
||||||
};
|
|
||||||
|
|
||||||
export const checkFileSize = (sizeBytes: number, fileType: string): boolean => {
|
|
||||||
const limit = FILE_TYPES[fileType] || FILE_TYPES.document;
|
|
||||||
return sizeBytes <= limit;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const ensureExtension = (
|
|
||||||
fileName: string,
|
|
||||||
buffer: Buffer,
|
|
||||||
detectedMime?: string,
|
|
||||||
): { fileName: string; mimeType: string } => {
|
|
||||||
const mimeMap: Record<string, string> = {
|
|
||||||
'application/pdf': 'pdf',
|
|
||||||
'image/png': 'png',
|
|
||||||
'image/jpeg': 'jpg',
|
|
||||||
'image/gif': 'gif',
|
|
||||||
'text/plain': 'txt',
|
|
||||||
'application/zip': 'zip',
|
|
||||||
};
|
|
||||||
|
|
||||||
let ext: string | null = null;
|
|
||||||
if (buffer.subarray(0, 4).toString() === '%PDF') {
|
|
||||||
ext = 'pdf';
|
|
||||||
} else if (buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47) {
|
|
||||||
ext = 'png';
|
|
||||||
} else if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) {
|
|
||||||
ext = 'jpg';
|
|
||||||
} else if (buffer.subarray(0, 4).toString() === 'GIF8') {
|
|
||||||
ext = 'gif';
|
|
||||||
} else if (detectedMime) {
|
|
||||||
ext = mimeMap[detectedMime.toLowerCase()] || null;
|
|
||||||
}
|
|
||||||
|
|
||||||
let finalFileName = fileName;
|
|
||||||
const hasExtension = fileName.includes('.') && fileName.split('.').pop()!.length >= 2;
|
|
||||||
if (!hasExtension && ext) {
|
|
||||||
finalFileName = `${fileName}.${ext}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
const mimeType = ext
|
|
||||||
? Object.keys(mimeMap).find((k) => mimeMap[k] === ext) ||
|
|
||||||
detectedMime ||
|
|
||||||
'application/octet-stream'
|
|
||||||
: detectedMime || 'application/octet-stream';
|
|
||||||
|
|
||||||
return { fileName: finalFileName, mimeType };
|
|
||||||
};
|
|
||||||
|
|
||||||
type HeaderMapRequest = {
|
|
||||||
headers?:
|
|
||||||
| {
|
|
||||||
get?: (name: string) => string | null;
|
|
||||||
}
|
|
||||||
| Record<string, string>;
|
|
||||||
};
|
|
||||||
|
|
||||||
type FileLike = {
|
|
||||||
fileName?: string;
|
|
||||||
mimeType?: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
type MessageLike = {
|
|
||||||
document?: FileLike;
|
|
||||||
photo?: FileLike[];
|
|
||||||
audio?: FileLike;
|
|
||||||
voice?: FileLike;
|
|
||||||
animation?: FileLike;
|
|
||||||
};
|
|
||||||
|
|
||||||
const getHeader = (request: HeaderMapRequest | null, name: string): string | undefined => {
|
|
||||||
const headers = request?.headers;
|
|
||||||
if (!headers) return undefined;
|
|
||||||
|
|
||||||
const get = 'get' in headers ? headers.get : undefined;
|
|
||||||
if (typeof get === 'function') return get(name) || undefined;
|
|
||||||
|
|
||||||
return (headers as Record<string, string>)[name];
|
|
||||||
};
|
|
||||||
|
|
||||||
export const extractFileName = (msg: MessageLike, request: HeaderMapRequest | null): string => {
|
|
||||||
const headerFileName = getHeader(request, 'x-file-name');
|
|
||||||
if (headerFileName) return headerFileName;
|
|
||||||
|
|
||||||
return (
|
|
||||||
msg.document?.fileName ||
|
|
||||||
msg.photo?.slice(-1)[0]?.fileName ||
|
|
||||||
msg.audio?.fileName ||
|
|
||||||
msg.voice?.fileName ||
|
|
||||||
msg.animation?.fileName ||
|
|
||||||
'file'
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const extractMimeType = (msg: MessageLike, request: HeaderMapRequest | null): string => {
|
|
||||||
const headerMimeType = getHeader(request, 'x-mime-type');
|
|
||||||
if (headerMimeType) return headerMimeType;
|
|
||||||
|
|
||||||
return (
|
|
||||||
msg.document?.mimeType ||
|
|
||||||
msg.photo?.slice(-1)[0]?.mimeType ||
|
|
||||||
msg.audio?.mimeType ||
|
|
||||||
msg.voice?.mimeType ||
|
|
||||||
msg.animation?.mimeType ||
|
|
||||||
'application/octet-stream'
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const computeHash = (buffer: Buffer): string => {
|
|
||||||
const hasher = new Bun.CryptoHasher('sha256');
|
|
||||||
hasher.update(buffer);
|
|
||||||
return hasher.digest('hex');
|
|
||||||
};
|
|
||||||
|
|
||||||
export interface TelegramMessageFile {
|
|
||||||
file_id: string;
|
|
||||||
file_unique_id: string;
|
|
||||||
file_size?: number;
|
|
||||||
mime_type?: string;
|
|
||||||
file_name?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface TelegramMediaMessage {
|
|
||||||
message_id: number;
|
|
||||||
document?: TelegramMessageFile;
|
|
||||||
photo?: TelegramMessageFile[];
|
|
||||||
video?: TelegramMessageFile;
|
|
||||||
audio?: TelegramMessageFile;
|
|
||||||
voice?: TelegramMessageFile;
|
|
||||||
animation?: TelegramMessageFile;
|
|
||||||
sticker?: TelegramMessageFile;
|
|
||||||
video_note?: TelegramMessageFile;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const extractFileFromMessage = (
|
|
||||||
msg: TelegramMediaMessage,
|
|
||||||
fileType: string,
|
|
||||||
): TelegramMessageFile => {
|
|
||||||
if (fileType === 'photo') return msg.photo?.slice(-1)[0] as TelegramMessageFile;
|
|
||||||
if (fileType === 'sticker') return msg.sticker as TelegramMessageFile;
|
|
||||||
return msg[fileType as keyof TelegramMediaMessage] as TelegramMessageFile;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const detectFileType = (msg: TelegramMediaMessage): string => {
|
|
||||||
if (msg.document) return 'document';
|
|
||||||
if (msg.photo) return 'photo';
|
|
||||||
if (msg.video) return 'video';
|
|
||||||
if (msg.audio) return 'audio';
|
|
||||||
if (msg.voice) return 'voice';
|
|
||||||
if (msg.animation) return 'animation';
|
|
||||||
if (msg.sticker) return 'sticker';
|
|
||||||
if (msg.video_note) return 'video_note';
|
|
||||||
return 'document';
|
|
||||||
};
|
|
||||||
|
|
||||||
export const getFileSizeLimit = (fileType: string): number =>
|
|
||||||
FILE_TYPES[fileType] || FILE_TYPES.document;
|
|
||||||
|
|
||||||
export const formatCreatedAt = (createdAt: Date | string | number): string => {
|
|
||||||
return createdAt instanceof Date ? createdAt.toISOString() : new Date(createdAt).toISOString();
|
|
||||||
};
|
|
||||||
|
|
||||||
export interface UploadResponse {
|
|
||||||
public_id: string;
|
|
||||||
file_name: string;
|
|
||||||
mime_type: string;
|
|
||||||
size_bytes: number;
|
|
||||||
file_type: string;
|
|
||||||
created_at: string;
|
|
||||||
download_url: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const buildUploadResponse = (file: FileMetadata, baseUrl: string): UploadResponse => {
|
|
||||||
return {
|
|
||||||
public_id: file.publicId,
|
|
||||||
file_name: file.fileName,
|
|
||||||
mime_type: file.mimeType,
|
|
||||||
size_bytes: file.sizeBytes,
|
|
||||||
file_type: file.fileType,
|
|
||||||
created_at: formatCreatedAt(file.createdAt),
|
|
||||||
download_url: `${baseUrl}/f/${file.publicId}`,
|
|
||||||
};
|
|
||||||
};
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
import { config } from '../env';
|
|
||||||
|
|
||||||
export const extractClientIp = (req: Request): string => {
|
|
||||||
if (!config.trustProxy) return '127.0.0.1';
|
|
||||||
|
|
||||||
const forwardedFor = req.headers.get('x-forwarded-for');
|
|
||||||
if (forwardedFor) {
|
|
||||||
const firstIp = forwardedFor.split(',')[0]?.trim();
|
|
||||||
if (firstIp) return firstIp;
|
|
||||||
}
|
|
||||||
|
|
||||||
const realIp = req.headers.get('x-real-ip')?.trim();
|
|
||||||
if (realIp) return realIp;
|
|
||||||
|
|
||||||
return '127.0.0.1';
|
|
||||||
};
|
|
||||||
@@ -1,25 +0,0 @@
|
|||||||
import winston from 'winston';
|
|
||||||
|
|
||||||
const logger = winston.createLogger({
|
|
||||||
level: process.env.LOG_LEVEL || 'info',
|
|
||||||
format: winston.format.combine(
|
|
||||||
winston.format.timestamp(),
|
|
||||||
winston.format.errors({ stack: true }),
|
|
||||||
winston.format.json(),
|
|
||||||
),
|
|
||||||
defaultMeta: { service: 'filedrop' },
|
|
||||||
transports: [
|
|
||||||
// Write all logs including error logs to file
|
|
||||||
new winston.transports.File({ filename: 'logs/error.log', level: 'error' }),
|
|
||||||
new winston.transports.File({ filename: 'logs/combined.log' }),
|
|
||||||
// Console transport for docker logs / CLI visibility
|
|
||||||
new winston.transports.Console({
|
|
||||||
format:
|
|
||||||
process.env.NODE_ENV !== 'production'
|
|
||||||
? winston.format.combine(winston.format.colorize(), winston.format.simple())
|
|
||||||
: winston.format.json(),
|
|
||||||
}),
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
export default logger;
|
|
||||||
@@ -1,111 +0,0 @@
|
|||||||
// No imports needed — logger used only by setInterval which moved to index.ts
|
|
||||||
|
|
||||||
interface Metric {
|
|
||||||
name: string;
|
|
||||||
value: number;
|
|
||||||
timestamp: number;
|
|
||||||
tags?: Record<string, string>;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface MetricsSnapshot {
|
|
||||||
uploadLatency: { p50: number; p95: number; p99: number };
|
|
||||||
uploadThroughput: number;
|
|
||||||
queueSize: number;
|
|
||||||
errorRate: number;
|
|
||||||
cacheHitRate: number;
|
|
||||||
botUtilization: number;
|
|
||||||
timestamp: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
class MetricsCollector {
|
|
||||||
private metrics: Metric[] = [];
|
|
||||||
private uploadTimes: number[] = [];
|
|
||||||
private errorCount = 0;
|
|
||||||
private totalRequests = 0;
|
|
||||||
private cacheHits = 0;
|
|
||||||
private cacheMisses = 0;
|
|
||||||
private maxMetricsSize = 10000;
|
|
||||||
|
|
||||||
recordUploadTime(durationMs: number): void {
|
|
||||||
this.uploadTimes.push(durationMs);
|
|
||||||
this.totalRequests++;
|
|
||||||
|
|
||||||
// Keep only last 1000 measurements
|
|
||||||
if (this.uploadTimes.length > 1000) {
|
|
||||||
this.uploadTimes.shift();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
recordError(): void {
|
|
||||||
this.errorCount++;
|
|
||||||
}
|
|
||||||
|
|
||||||
recordCacheHit(): void {
|
|
||||||
this.cacheHits++;
|
|
||||||
}
|
|
||||||
|
|
||||||
recordCacheMiss(): void {
|
|
||||||
this.cacheMisses++;
|
|
||||||
}
|
|
||||||
|
|
||||||
recordMetric(name: string, value: number, tags?: Record<string, string>): void {
|
|
||||||
this.metrics.push({
|
|
||||||
name,
|
|
||||||
value,
|
|
||||||
timestamp: Date.now(),
|
|
||||||
tags,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Keep metrics bounded
|
|
||||||
if (this.metrics.length > this.maxMetricsSize) {
|
|
||||||
this.metrics = this.metrics.slice(-this.maxMetricsSize);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private calculatePercentile(arr: number[], percentile: number): number {
|
|
||||||
if (arr.length === 0) return 0;
|
|
||||||
const sorted = [...arr].sort((a, b) => a - b);
|
|
||||||
const index = Math.ceil((percentile / 100) * sorted.length) - 1;
|
|
||||||
return sorted[Math.max(0, index)];
|
|
||||||
}
|
|
||||||
|
|
||||||
getSnapshot(): MetricsSnapshot {
|
|
||||||
const errorRate = this.totalRequests > 0 ? (this.errorCount / this.totalRequests) * 100 : 0;
|
|
||||||
const cacheHitRate =
|
|
||||||
this.cacheHits + this.cacheMisses > 0
|
|
||||||
? (this.cacheHits / (this.cacheHits + this.cacheMisses)) * 100
|
|
||||||
: 0;
|
|
||||||
|
|
||||||
return {
|
|
||||||
uploadLatency: {
|
|
||||||
p50: this.calculatePercentile(this.uploadTimes, 50),
|
|
||||||
p95: this.calculatePercentile(this.uploadTimes, 95),
|
|
||||||
p99: this.calculatePercentile(this.uploadTimes, 99),
|
|
||||||
},
|
|
||||||
uploadThroughput: this.totalRequests > 0 ? this.totalRequests / 60 : 0,
|
|
||||||
queueSize: 0, // Will be updated by queue
|
|
||||||
errorRate,
|
|
||||||
cacheHitRate,
|
|
||||||
botUtilization: 0, // Will be updated by bot tracker
|
|
||||||
timestamp: Date.now(),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
reset(): void {
|
|
||||||
this.uploadTimes = [];
|
|
||||||
this.errorCount = 0;
|
|
||||||
this.totalRequests = 0;
|
|
||||||
this.cacheHits = 0;
|
|
||||||
this.cacheMisses = 0;
|
|
||||||
this.metrics = [];
|
|
||||||
}
|
|
||||||
|
|
||||||
getMetrics(name?: string): Metric[] {
|
|
||||||
if (!name) return this.metrics;
|
|
||||||
return this.metrics.filter((m) => m.name === name);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export const metricsCollector = new MetricsCollector();
|
|
||||||
|
|
||||||
export { MetricsCollector };
|
|
||||||
@@ -1,89 +0,0 @@
|
|||||||
import { config } from '../env';
|
|
||||||
import { extractClientIp } from './ip';
|
|
||||||
import logger from './logger';
|
|
||||||
|
|
||||||
interface RateLimitEntry {
|
|
||||||
count: number;
|
|
||||||
resetTime: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
const rateLimitStore = new Map<string, RateLimitEntry>();
|
|
||||||
const MAX_STORE_ENTRIES = 50000;
|
|
||||||
|
|
||||||
const evictExpiredEntries = (now = Date.now()): number => {
|
|
||||||
let cleaned = 0;
|
|
||||||
|
|
||||||
for (const [key, entry] of rateLimitStore.entries()) {
|
|
||||||
if (now > entry.resetTime) {
|
|
||||||
rateLimitStore.delete(key);
|
|
||||||
cleaned++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return cleaned;
|
|
||||||
};
|
|
||||||
|
|
||||||
const ensureStoreCapacity = (now: number): void => {
|
|
||||||
if (rateLimitStore.size < MAX_STORE_ENTRIES) return;
|
|
||||||
|
|
||||||
evictExpiredEntries(now);
|
|
||||||
while (rateLimitStore.size >= MAX_STORE_ENTRIES) {
|
|
||||||
const oldestKey = rateLimitStore.keys().next().value;
|
|
||||||
if (!oldestKey) break;
|
|
||||||
rateLimitStore.delete(oldestKey);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const checkRateLimit = (key: string): boolean => {
|
|
||||||
const now = Date.now();
|
|
||||||
const entry = rateLimitStore.get(key);
|
|
||||||
|
|
||||||
if (!entry || now > entry.resetTime) {
|
|
||||||
ensureStoreCapacity(now);
|
|
||||||
rateLimitStore.set(key, {
|
|
||||||
count: 1,
|
|
||||||
resetTime: now + config.rateLimitWindowMs,
|
|
||||||
});
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (entry.count >= config.rateLimitMaxRequests) {
|
|
||||||
logger.warn('Rate limit exceeded', { key, count: entry.count });
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
entry.count++;
|
|
||||||
return true;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const withRateLimit = <T extends Request>(
|
|
||||||
handler: (req: T) => Promise<Response>,
|
|
||||||
): ((req: T) => Promise<Response>) => {
|
|
||||||
return async (req: T): Promise<Response> => {
|
|
||||||
const ip = extractClientIp(req);
|
|
||||||
if (!checkRateLimit(ip)) {
|
|
||||||
return Response.json({ error: 'Rate limit exceeded' }, { status: 429 });
|
|
||||||
}
|
|
||||||
|
|
||||||
return handler(req);
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export const cleanupRateLimitCache = (): void => {
|
|
||||||
const cleaned = evictExpiredEntries();
|
|
||||||
|
|
||||||
if (cleaned > 0) {
|
|
||||||
logger.debug('Rate limit cache cleanup', { cleaned, remaining: rateLimitStore.size });
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const getRateLimitStats = () => ({
|
|
||||||
trackedIPs: rateLimitStore.size,
|
|
||||||
windowSize: config.rateLimitWindowMs,
|
|
||||||
maxRequests: config.rateLimitMaxRequests,
|
|
||||||
maxTrackedIPs: MAX_STORE_ENTRIES,
|
|
||||||
});
|
|
||||||
|
|
||||||
export const clearRateLimitCache = (): void => {
|
|
||||||
rateLimitStore.clear();
|
|
||||||
};
|
|
||||||
@@ -1,91 +0,0 @@
|
|||||||
import logger from './logger';
|
|
||||||
|
|
||||||
interface RetryOptions {
|
|
||||||
maxRetries?: number;
|
|
||||||
initialDelayMs?: number;
|
|
||||||
maxDelayMs?: number;
|
|
||||||
backoffMultiplier?: number;
|
|
||||||
shouldRetry?: (error: unknown) => boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
const DEFAULT_OPTIONS: Required<RetryOptions> = {
|
|
||||||
maxRetries: 3,
|
|
||||||
initialDelayMs: 100,
|
|
||||||
maxDelayMs: 5000,
|
|
||||||
backoffMultiplier: 2,
|
|
||||||
shouldRetry: (error: unknown) => {
|
|
||||||
const errorStr = error instanceof Error ? error.message : String(error);
|
|
||||||
// Retry on transient errors
|
|
||||||
return (
|
|
||||||
errorStr.includes('ECONNREFUSED') ||
|
|
||||||
errorStr.includes('ETIMEDOUT') ||
|
|
||||||
errorStr.includes('ENOTFOUND') ||
|
|
||||||
errorStr.includes('429') ||
|
|
||||||
errorStr.includes('timeout')
|
|
||||||
);
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
export const withRetry = async <T>(
|
|
||||||
fn: () => Promise<T>,
|
|
||||||
options: RetryOptions = {},
|
|
||||||
): Promise<T> => {
|
|
||||||
const opts = { ...DEFAULT_OPTIONS, ...options };
|
|
||||||
let lastError: unknown;
|
|
||||||
let delay = opts.initialDelayMs;
|
|
||||||
|
|
||||||
for (let attempt = 0; attempt <= opts.maxRetries; attempt++) {
|
|
||||||
try {
|
|
||||||
return await fn();
|
|
||||||
} catch (error: unknown) {
|
|
||||||
lastError = error;
|
|
||||||
const errorStr = error instanceof Error ? error.message : String(error);
|
|
||||||
|
|
||||||
if (attempt === opts.maxRetries || !opts.shouldRetry(error)) {
|
|
||||||
logger.error('Retry exhausted', {
|
|
||||||
attempt,
|
|
||||||
maxRetries: opts.maxRetries,
|
|
||||||
error: errorStr,
|
|
||||||
});
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.warn('Retrying after error', {
|
|
||||||
attempt,
|
|
||||||
delay,
|
|
||||||
error: errorStr,
|
|
||||||
});
|
|
||||||
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
||||||
delay = Math.min(delay * opts.backoffMultiplier, opts.maxDelayMs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw lastError;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const withTimeout = async <T>(
|
|
||||||
fn: () => Promise<T>,
|
|
||||||
timeoutMs: number = 30000,
|
|
||||||
): Promise<T> => {
|
|
||||||
return Promise.race([
|
|
||||||
fn(),
|
|
||||||
new Promise<T>((_, reject) =>
|
|
||||||
setTimeout(() => reject(new Error(`Operation timeout after ${timeoutMs}ms`)), timeoutMs),
|
|
||||||
),
|
|
||||||
]);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const withFallback = async <T>(
|
|
||||||
primary: () => Promise<T>,
|
|
||||||
fallback: () => Promise<T>,
|
|
||||||
): Promise<T> => {
|
|
||||||
try {
|
|
||||||
return await primary();
|
|
||||||
} catch (error: unknown) {
|
|
||||||
logger.warn('Primary operation failed, using fallback', {
|
|
||||||
error: error instanceof Error ? error.message : String(error),
|
|
||||||
});
|
|
||||||
return fallback();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
@@ -1,463 +0,0 @@
|
|||||||
import { timingSafeEqual } from 'node:crypto';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Timing-safe string comparison that prevents timing attacks.
|
|
||||||
*
|
|
||||||
* Uses `crypto.timingSafeEqual` which runs in constant time regardless of
|
|
||||||
* where the strings differ. Returns false for mismatched-length inputs
|
|
||||||
* to avoid leaking length information via early return.
|
|
||||||
*
|
|
||||||
* @param left - The first string to compare.
|
|
||||||
* @param right - The second string to compare.
|
|
||||||
* @returns True if both strings are equal.
|
|
||||||
*/
|
|
||||||
const timingSafeCompare = (left: string, right: string): boolean => {
|
|
||||||
const leftBuffer = Buffer.from(left);
|
|
||||||
const rightBuffer = Buffer.from(right);
|
|
||||||
|
|
||||||
if (leftBuffer.length !== rightBuffer.length) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return timingSafeEqual(leftBuffer, rightBuffer);
|
|
||||||
};
|
|
||||||
|
|
||||||
export interface SigV4Result {
|
|
||||||
isValid: boolean;
|
|
||||||
credential: {
|
|
||||||
accessKey: string;
|
|
||||||
date: string;
|
|
||||||
region: string;
|
|
||||||
service: string;
|
|
||||||
} | null;
|
|
||||||
errorCode?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface VerifyPresignedUrlInput {
|
|
||||||
url: string;
|
|
||||||
method: string;
|
|
||||||
headers: Record<string, string>;
|
|
||||||
s3AccessKey: string;
|
|
||||||
s3SecretKey: string;
|
|
||||||
region: string;
|
|
||||||
now?: Date;
|
|
||||||
}
|
|
||||||
|
|
||||||
const SERVICE = 's3';
|
|
||||||
const TERMINATION = 'aws4_request';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Maximum acceptable clock skew between client and server for header-based
|
|
||||||
* SigV4 authentication. AWS allows 15 minutes.
|
|
||||||
*/
|
|
||||||
const MAX_CLOCK_SKEW_MS = 15 * 60 * 1000;
|
|
||||||
|
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
||||||
const buf = (data: string | ArrayBuffer | Uint8Array): Uint8Array => {
|
|
||||||
if (data instanceof Uint8Array) return data;
|
|
||||||
if (data instanceof ArrayBuffer) return new Uint8Array(data);
|
|
||||||
return new TextEncoder().encode(data);
|
|
||||||
};
|
|
||||||
|
|
||||||
const sha256Hex = async (data: string | Uint8Array | ArrayBuffer): Promise<string> => {
|
|
||||||
const hashBuffer = await crypto.subtle.digest('SHA-256', buf(data) as never);
|
|
||||||
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
|
||||||
return hashArray.map((b) => b.toString(16).padStart(2, '0')).join('');
|
|
||||||
};
|
|
||||||
|
|
||||||
const hmacSha256 = async (key: Uint8Array, message: string): Promise<Uint8Array> => {
|
|
||||||
const cryptoKey = await crypto.subtle.importKey(
|
|
||||||
'raw',
|
|
||||||
key as never,
|
|
||||||
{ name: 'HMAC', hash: 'SHA-256' },
|
|
||||||
false,
|
|
||||||
['sign'],
|
|
||||||
);
|
|
||||||
const result = await crypto.subtle.sign('HMAC', cryptoKey, buf(message) as never);
|
|
||||||
return new Uint8Array(result);
|
|
||||||
};
|
|
||||||
|
|
||||||
const getSigningKey = async (
|
|
||||||
secretKey: string,
|
|
||||||
dateStamp: string,
|
|
||||||
region: string,
|
|
||||||
): Promise<Uint8Array> => {
|
|
||||||
let key = await hmacSha256(buf(`AWS4${secretKey}`), dateStamp);
|
|
||||||
key = await hmacSha256(key, region);
|
|
||||||
key = await hmacSha256(key, SERVICE);
|
|
||||||
return await hmacSha256(key, TERMINATION);
|
|
||||||
};
|
|
||||||
|
|
||||||
const hmacHex = async (key: Uint8Array, message: string): Promise<string> => {
|
|
||||||
const result = await hmacSha256(key, message);
|
|
||||||
return Array.from(result)
|
|
||||||
.map((b) => b.toString(16).padStart(2, '0'))
|
|
||||||
.join('');
|
|
||||||
};
|
|
||||||
|
|
||||||
const parseAuthorizationHeader = (authHeader: string) => {
|
|
||||||
const credentialMatch = authHeader.match(/Credential=([^,]+)/);
|
|
||||||
const signedHeadersMatch = authHeader.match(/SignedHeaders=([^,]+)/);
|
|
||||||
const signatureMatch = authHeader.match(/Signature=([^,]+)/);
|
|
||||||
|
|
||||||
if (!credentialMatch || !signedHeadersMatch || !signatureMatch) return null;
|
|
||||||
|
|
||||||
const credentialParts = credentialMatch[1].split('/');
|
|
||||||
if (credentialParts.length !== 5) return null;
|
|
||||||
|
|
||||||
return {
|
|
||||||
accessKey: credentialParts[0],
|
|
||||||
date: credentialParts[1],
|
|
||||||
region: credentialParts[2],
|
|
||||||
service: credentialParts[3],
|
|
||||||
termination: credentialParts[4],
|
|
||||||
signedHeaders: signedHeadersMatch[1],
|
|
||||||
signature: signatureMatch[1],
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
const buildCanonicalRequest = (
|
|
||||||
method: string,
|
|
||||||
canonicalUri: string,
|
|
||||||
canonicalQueryString: string,
|
|
||||||
signedHeaders: string,
|
|
||||||
headers: Record<string, string>,
|
|
||||||
hashedPayload: string,
|
|
||||||
): string => {
|
|
||||||
const canonicalHeaders = signedHeaders
|
|
||||||
.split(';')
|
|
||||||
.map((h) => {
|
|
||||||
const value = headers[h.toLowerCase()] || '';
|
|
||||||
return `${h.toLowerCase()}:${value.trim()}\n`;
|
|
||||||
})
|
|
||||||
.join('');
|
|
||||||
|
|
||||||
return `${method}\n${canonicalUri}\n${canonicalQueryString}\n${canonicalHeaders}\n${signedHeaders}\n${hashedPayload}`;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Normalizes a URI per AWS SigV4 requirements plus RFC 3986:
|
|
||||||
*
|
|
||||||
* 1. Decode percent-encoded characters
|
|
||||||
* 2. Remove dot-segments (`.` and `..`) per RFC 3986 section 5.2.4
|
|
||||||
*
|
|
||||||
* @param uri - The raw URI path to normalize.
|
|
||||||
* @returns The normalized URI path.
|
|
||||||
*/
|
|
||||||
const normalizeUri = (uri: string): string => {
|
|
||||||
if (!uri || uri === '') return '/';
|
|
||||||
|
|
||||||
// AWS SigV4 requires URI-decoded paths in the canonical request
|
|
||||||
// Only `.` and `..` segments are removed per RFC 3986 section 5.2.4
|
|
||||||
// Empty segments (from `//` or trailing `/`) are preserved — they are
|
|
||||||
// part of the URI and the SDK signs them.
|
|
||||||
const decoded = decodeURIComponent(uri);
|
|
||||||
const segments = decoded.split('/');
|
|
||||||
const result: string[] = [];
|
|
||||||
|
|
||||||
for (const segment of segments) {
|
|
||||||
if (segment === '.') continue;
|
|
||||||
if (segment === '..') {
|
|
||||||
result.pop();
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
result.push(segment);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Join preserves empty first segment (from leading /) automatically
|
|
||||||
return result.join('/') || '/';
|
|
||||||
};
|
|
||||||
|
|
||||||
const awsEncode = (value: string): string =>
|
|
||||||
encodeURIComponent(value).replace(
|
|
||||||
/[!'()*]/g,
|
|
||||||
(ch) => `%${ch.charCodeAt(0).toString(16).toUpperCase()}`,
|
|
||||||
);
|
|
||||||
|
|
||||||
export const buildCanonicalQueryString = (
|
|
||||||
searchParams: URLSearchParams,
|
|
||||||
excludeKeys: Set<string> = new Set(),
|
|
||||||
): string => {
|
|
||||||
const pairs: Array<[string, string]> = [];
|
|
||||||
for (const [key, value] of searchParams.entries()) {
|
|
||||||
if (!excludeKeys.has(key)) pairs.push([key, value]);
|
|
||||||
}
|
|
||||||
// AWS SigV4 requires UTF-8 byte-order (code point) comparison, NOT localeCompare
|
|
||||||
pairs.sort(([ak, av], [bk, bv]) => {
|
|
||||||
const a = `${awsEncode(ak)}=${awsEncode(av)}`;
|
|
||||||
const b = `${awsEncode(bk)}=${awsEncode(bv)}`;
|
|
||||||
if (a < b) return -1;
|
|
||||||
if (a > b) return 1;
|
|
||||||
return 0;
|
|
||||||
});
|
|
||||||
return pairs.map(([key, value]) => `${awsEncode(key)}=${awsEncode(value)}`).join('&');
|
|
||||||
};
|
|
||||||
|
|
||||||
const getHashedPayload = async (body: string | null): Promise<string> => {
|
|
||||||
if (!body || body.length === 0) return await sha256Hex('');
|
|
||||||
return await sha256Hex(body);
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parses an AWS SigV4 `x-amz-date` value (e.g. `20260707T120000Z`) into a Date.
|
|
||||||
*
|
|
||||||
* @param amzDate - The date string in `YYYYMMDDTHHmmssZ` format.
|
|
||||||
* @returns The parsed Date, or null if the format is invalid.
|
|
||||||
*/
|
|
||||||
const parseAmzDateUtc = (amzDate: string): Date | null => {
|
|
||||||
const match = amzDate.match(/^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/);
|
|
||||||
if (!match) return null;
|
|
||||||
const [, year, month, day, hour, minute, second] = match;
|
|
||||||
return new Date(
|
|
||||||
Date.UTC(
|
|
||||||
Number.parseInt(year, 10),
|
|
||||||
Number.parseInt(month, 10) - 1,
|
|
||||||
Number.parseInt(day, 10),
|
|
||||||
Number.parseInt(hour, 10),
|
|
||||||
Number.parseInt(minute, 10),
|
|
||||||
Number.parseInt(second, 10),
|
|
||||||
),
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Validates that `host` is included in the signed headers list.
|
|
||||||
*
|
|
||||||
* AWS SigV4 mandates that `host` is always signed. Reject requests that
|
|
||||||
* omit it to prevent header injection / replay variants.
|
|
||||||
*
|
|
||||||
* @param signedHeaders - The semicolon-separated signed headers string.
|
|
||||||
* @returns True if `host` is present.
|
|
||||||
*/
|
|
||||||
const validateSignedHeaders = (signedHeaders: string): boolean => {
|
|
||||||
return signedHeaders.split(';').some((h) => h.toLowerCase() === 'host');
|
|
||||||
};
|
|
||||||
|
|
||||||
export const verifySignature = async (
|
|
||||||
method: string,
|
|
||||||
url: string,
|
|
||||||
headers: Record<string, string>,
|
|
||||||
body: string | null,
|
|
||||||
s3AccessKey: string,
|
|
||||||
s3SecretKey: string,
|
|
||||||
region: string,
|
|
||||||
): Promise<SigV4Result> => {
|
|
||||||
const authHeader = headers.authorization;
|
|
||||||
if (!authHeader?.startsWith('AWS4-HMAC-SHA256')) {
|
|
||||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
|
||||||
}
|
|
||||||
|
|
||||||
const parsed = parseAuthorizationHeader(authHeader);
|
|
||||||
if (!parsed) {
|
|
||||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!timingSafeCompare(parsed.accessKey, s3AccessKey)) {
|
|
||||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!timingSafeCompare(parsed.region, region)) {
|
|
||||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate service and termination in credential scope (M2)
|
|
||||||
if (parsed.service !== SERVICE || parsed.termination !== TERMINATION) {
|
|
||||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate host is in signed headers (LOW/host)
|
|
||||||
if (!validateSignedHeaders(parsed.signedHeaders)) {
|
|
||||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
|
||||||
}
|
|
||||||
|
|
||||||
const parsedUrl = new URL(url, 'http://localhost');
|
|
||||||
const canonicalUri = normalizeUri(parsedUrl.pathname);
|
|
||||||
const canonicalQueryString = buildCanonicalQueryString(parsedUrl.searchParams);
|
|
||||||
|
|
||||||
const contentSha256 = headers['x-amz-content-sha256'] || null;
|
|
||||||
if (contentSha256?.startsWith('STREAMING-')) {
|
|
||||||
return { isValid: false, credential: null, errorCode: 'NotImplemented' };
|
|
||||||
}
|
|
||||||
|
|
||||||
// CRITICAL: Use the x-amz-content-sha256 header value in the canonical
|
|
||||||
// request because that's what the client signed. The actual body hash is
|
|
||||||
// verified by verifyBodyHash() after streaming, ensuring integrity without
|
|
||||||
// breaking SigV4.
|
|
||||||
const hashedPayload = contentSha256 || (await getHashedPayload(body));
|
|
||||||
|
|
||||||
const canonicalRequest = buildCanonicalRequest(
|
|
||||||
method,
|
|
||||||
canonicalUri,
|
|
||||||
canonicalQueryString,
|
|
||||||
parsed.signedHeaders,
|
|
||||||
headers,
|
|
||||||
hashedPayload,
|
|
||||||
);
|
|
||||||
|
|
||||||
const hashedCanonicalRequest = await sha256Hex(canonicalRequest);
|
|
||||||
|
|
||||||
// M1: Fall back to Date header if x-amz-date is missing
|
|
||||||
const amzDate = headers['x-amz-date'] || headers.date || '';
|
|
||||||
|
|
||||||
// H5: Validate request freshness (clock skew / replay protection)
|
|
||||||
if (amzDate) {
|
|
||||||
const requestDate = parseAmzDateUtc(amzDate);
|
|
||||||
if (requestDate) {
|
|
||||||
const now = Date.now();
|
|
||||||
const skew = Math.abs(now - requestDate.getTime());
|
|
||||||
if (skew > MAX_CLOCK_SKEW_MS) {
|
|
||||||
return { isValid: false, credential: null, errorCode: 'RequestExpired' };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const dateStamp = parsed.date;
|
|
||||||
|
|
||||||
// M3: Ensure date in credential scope matches x-amz-date
|
|
||||||
if (amzDate) {
|
|
||||||
const amzDateStamp = amzDate.slice(0, 8); // "YYYYMMDD"
|
|
||||||
if (amzDateStamp !== dateStamp) {
|
|
||||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const credentialScope = `${dateStamp}/${region}/${parsed.service}/${parsed.termination}`;
|
|
||||||
|
|
||||||
const stringToSign = `AWS4-HMAC-SHA256\n${amzDate}\n${credentialScope}\n${hashedCanonicalRequest}`;
|
|
||||||
|
|
||||||
const signingKey = await getSigningKey(s3SecretKey, dateStamp, region);
|
|
||||||
const expectedSignature = await hmacHex(signingKey, stringToSign);
|
|
||||||
|
|
||||||
if (!timingSafeCompare(expectedSignature, parsed.signature)) {
|
|
||||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
isValid: true,
|
|
||||||
credential: {
|
|
||||||
accessKey: parsed.accessKey,
|
|
||||||
date: parsed.date,
|
|
||||||
region: parsed.region,
|
|
||||||
service: parsed.service,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export const verifyPresignedUrl = async ({
|
|
||||||
url,
|
|
||||||
method,
|
|
||||||
headers,
|
|
||||||
s3AccessKey,
|
|
||||||
s3SecretKey,
|
|
||||||
region,
|
|
||||||
now = new Date(),
|
|
||||||
}: VerifyPresignedUrlInput): Promise<SigV4Result> => {
|
|
||||||
const parsedUrl = new URL(url);
|
|
||||||
const searchParams = parsedUrl.searchParams;
|
|
||||||
|
|
||||||
const algorithm = searchParams.get('X-Amz-Algorithm');
|
|
||||||
const credential = searchParams.get('X-Amz-Credential');
|
|
||||||
const signedHeaders = searchParams.get('X-Amz-SignedHeaders');
|
|
||||||
const signature = searchParams.get('X-Amz-Signature');
|
|
||||||
const expiresText = searchParams.get('X-Amz-Expires');
|
|
||||||
const amzDate = searchParams.get('X-Amz-Date');
|
|
||||||
|
|
||||||
if (
|
|
||||||
algorithm !== 'AWS4-HMAC-SHA256' ||
|
|
||||||
!credential ||
|
|
||||||
!signedHeaders ||
|
|
||||||
!signature ||
|
|
||||||
!expiresText ||
|
|
||||||
!amzDate
|
|
||||||
) {
|
|
||||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
|
||||||
}
|
|
||||||
|
|
||||||
const expires = Number.parseInt(expiresText, 10);
|
|
||||||
const signedAt = parseAmzDateUtc(amzDate);
|
|
||||||
if (!Number.isFinite(expires) || expires <= 0 || !signedAt) {
|
|
||||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
|
||||||
}
|
|
||||||
// AWS S3 spec limits presigned URLs to 7 days (604800 seconds)
|
|
||||||
const MAX_PRESIGNED_EXPIRY_SECONDS = 604800;
|
|
||||||
if (
|
|
||||||
now.getTime() > signedAt.getTime() + expires * 1000 ||
|
|
||||||
expires > MAX_PRESIGNED_EXPIRY_SECONDS
|
|
||||||
) {
|
|
||||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
|
||||||
}
|
|
||||||
|
|
||||||
const credParts = credential.split('/');
|
|
||||||
if (credParts.length !== 5) {
|
|
||||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
|
||||||
}
|
|
||||||
const [accessKey, dateStamp, credentialRegion, service, termination] = credParts;
|
|
||||||
if (
|
|
||||||
!timingSafeCompare(accessKey, s3AccessKey) ||
|
|
||||||
!timingSafeCompare(credentialRegion, region) ||
|
|
||||||
service !== SERVICE ||
|
|
||||||
termination !== TERMINATION
|
|
||||||
) {
|
|
||||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate host is in signed headers for presigned URLs too
|
|
||||||
if (!validateSignedHeaders(signedHeaders)) {
|
|
||||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
|
||||||
}
|
|
||||||
|
|
||||||
const signedHeaderList = signedHeaders.split(';').filter(Boolean);
|
|
||||||
const canonicalHeaders = signedHeaderList
|
|
||||||
.map((headerName) => {
|
|
||||||
const lower = headerName.toLowerCase();
|
|
||||||
const value = lower === 'host' ? headers.host || parsedUrl.host : headers[lower] || '';
|
|
||||||
return `${lower}:${value.trim()}\n`;
|
|
||||||
})
|
|
||||||
.join('');
|
|
||||||
|
|
||||||
const canonicalRequest = `${method}\n${normalizeUri(parsedUrl.pathname)}\n${buildCanonicalQueryString(searchParams, new Set(['X-Amz-Signature']))}\n${canonicalHeaders}\n${signedHeaders}\nUNSIGNED-PAYLOAD`;
|
|
||||||
const hashedCanonicalRequest = await sha256Hex(canonicalRequest);
|
|
||||||
const credentialScope = `${dateStamp}/${region}/${SERVICE}/${TERMINATION}`;
|
|
||||||
const stringToSign = `AWS4-HMAC-SHA256\n${amzDate}\n${credentialScope}\n${hashedCanonicalRequest}`;
|
|
||||||
const expectedSignature = await hmacHex(
|
|
||||||
await getSigningKey(s3SecretKey, dateStamp, region),
|
|
||||||
stringToSign,
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!timingSafeCompare(expectedSignature, signature)) {
|
|
||||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
|
||||||
}
|
|
||||||
return { isValid: true, credential: { accessKey, date: dateStamp, region, service } };
|
|
||||||
};
|
|
||||||
|
|
||||||
export const isS3Request = (headers: Record<string, string>): boolean => {
|
|
||||||
const auth = headers.authorization || '';
|
|
||||||
return auth.startsWith('AWS4-HMAC-SHA256');
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Verifies that the actual body SHA-256 matches the `x-amz-content-sha256`
|
|
||||||
* header from the original request.
|
|
||||||
*
|
|
||||||
* This MUST be called AFTER the body has been fully streamed and hashed,
|
|
||||||
* as a second pass after `verifySignature` (which cannot hash a streaming
|
|
||||||
* body without consuming it).
|
|
||||||
*
|
|
||||||
* @param bodySha256 - The SHA-256 hex digest of the actual body content.
|
|
||||||
* @param headers - The original request headers.
|
|
||||||
* @returns An error result on mismatch, or null if the check passes.
|
|
||||||
*/
|
|
||||||
export const verifyBodyHash = (
|
|
||||||
bodySha256: string,
|
|
||||||
headers: Record<string, string>,
|
|
||||||
): SigV4Result | null => {
|
|
||||||
const claimedHash = headers['x-amz-content-sha256'];
|
|
||||||
// If the client sent UNSIGNED-PAYLOAD, skip verification
|
|
||||||
if (!claimedHash || claimedHash === 'UNSIGNED-PAYLOAD' || claimedHash.startsWith('STREAMING-')) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
if (claimedHash !== bodySha256) {
|
|
||||||
return { isValid: false, credential: null, errorCode: 'BadDigest' };
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
};
|
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
import { nanoid } from 'nanoid';
|
|
||||||
|
|
||||||
export const S3_CORS_HEADERS: Record<string, string> = {
|
|
||||||
'access-control-allow-origin': '*',
|
|
||||||
'access-control-allow-methods': 'GET, PUT, HEAD, DELETE, POST, OPTIONS',
|
|
||||||
'access-control-allow-headers': [
|
|
||||||
'Authorization',
|
|
||||||
'Content-Type',
|
|
||||||
'Content-MD5',
|
|
||||||
'Range',
|
|
||||||
'If-Match',
|
|
||||||
'If-None-Match',
|
|
||||||
'If-Modified-Since',
|
|
||||||
'If-Unmodified-Since',
|
|
||||||
'X-Amz-*',
|
|
||||||
'x-amz-*',
|
|
||||||
].join(', '),
|
|
||||||
'access-control-expose-headers': [
|
|
||||||
'Accept-Ranges',
|
|
||||||
'Content-Length',
|
|
||||||
'Content-Range',
|
|
||||||
'Content-Type',
|
|
||||||
'ETag',
|
|
||||||
'Last-Modified',
|
|
||||||
'x-amz-id-2',
|
|
||||||
'x-amz-request-id',
|
|
||||||
].join(', '),
|
|
||||||
'access-control-max-age': '86400',
|
|
||||||
};
|
|
||||||
|
|
||||||
export const s3Headers = (
|
|
||||||
requestId: string,
|
|
||||||
extraHeaders: Record<string, string> = {},
|
|
||||||
): Record<string, string> => ({
|
|
||||||
...S3_CORS_HEADERS,
|
|
||||||
server: 'AmazonS3',
|
|
||||||
...(requestId
|
|
||||||
? {
|
|
||||||
'x-amz-request-id': requestId,
|
|
||||||
'x-amz-id-2': `${requestId}+${nanoid(16)}`,
|
|
||||||
}
|
|
||||||
: {}),
|
|
||||||
...extraHeaders,
|
|
||||||
});
|
|
||||||
|
|
||||||
export const applyS3Headers = (headers: Headers, requestId: string): Headers => {
|
|
||||||
const result = new Headers(headers);
|
|
||||||
for (const [key, value] of Object.entries(s3Headers(requestId))) {
|
|
||||||
result.set(key, value);
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
};
|
|
||||||
@@ -1,133 +0,0 @@
|
|||||||
import { gunzipSync } from 'node:zlib';
|
|
||||||
import { applyS3Headers } from './headers';
|
|
||||||
import { contentRange, type RangeParseResult } from './range';
|
|
||||||
|
|
||||||
export interface ObjectPartSource {
|
|
||||||
telegramFileId: string;
|
|
||||||
telegramUrl: string;
|
|
||||||
sizeBytes: number;
|
|
||||||
partNumber: number;
|
|
||||||
storedSizeBytes?: number;
|
|
||||||
compressionAlgorithm?: 'gzip' | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ObjectResponseInput {
|
|
||||||
reqId: string;
|
|
||||||
contentType: string;
|
|
||||||
etag: string;
|
|
||||||
lastModified: Date;
|
|
||||||
totalSize: number;
|
|
||||||
parts: ObjectPartSource[];
|
|
||||||
range: RangeParseResult;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface PlannedPart {
|
|
||||||
part: ObjectPartSource;
|
|
||||||
relativeStart: number;
|
|
||||||
relativeEnd: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
const baseHeaders = (input: ObjectResponseInput, contentLength: number): Headers => {
|
|
||||||
const headers = new Headers({
|
|
||||||
'content-type': input.contentType,
|
|
||||||
'content-length': String(contentLength),
|
|
||||||
etag: `"${input.etag}"`,
|
|
||||||
'last-modified': input.lastModified.toUTCString(),
|
|
||||||
'x-amz-request-id': input.reqId,
|
|
||||||
'accept-ranges': 'bytes',
|
|
||||||
'cache-control': 'public, max-age=31536000',
|
|
||||||
});
|
|
||||||
return headers;
|
|
||||||
};
|
|
||||||
|
|
||||||
const planParts = (parts: ObjectPartSource[], start: number, end: number): PlannedPart[] => {
|
|
||||||
const planned: PlannedPart[] = [];
|
|
||||||
let offset = 0;
|
|
||||||
for (const part of parts) {
|
|
||||||
const partStart = offset;
|
|
||||||
const partEnd = offset + part.sizeBytes - 1;
|
|
||||||
offset += part.sizeBytes;
|
|
||||||
if (end < partStart || start > partEnd) continue;
|
|
||||||
planned.push({
|
|
||||||
part,
|
|
||||||
relativeStart: Math.max(start, partStart) - partStart,
|
|
||||||
relativeEnd: Math.min(end, partEnd) - partStart,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return planned;
|
|
||||||
};
|
|
||||||
|
|
||||||
const streamFromBytes = (bytes: Uint8Array): ReadableStream<Uint8Array> =>
|
|
||||||
new Response(bytes).body!;
|
|
||||||
|
|
||||||
const TELEGRAM_FETCH_TIMEOUT_MS = 30_000;
|
|
||||||
|
|
||||||
const fetchWholePartBytes = async (telegramUrl: string): Promise<Uint8Array> => {
|
|
||||||
const res = await fetch(telegramUrl, { signal: AbortSignal.timeout(TELEGRAM_FETCH_TIMEOUT_MS) });
|
|
||||||
if (!res.ok) throw new Error(`Telegram fetch failed: ${res.status}`);
|
|
||||||
return new Uint8Array(await res.arrayBuffer());
|
|
||||||
};
|
|
||||||
|
|
||||||
const fetchPartBody = async (planned: PlannedPart): Promise<ReadableStream<Uint8Array>> => {
|
|
||||||
const wantsWholePart =
|
|
||||||
planned.relativeStart === 0 && planned.relativeEnd === planned.part.sizeBytes - 1;
|
|
||||||
|
|
||||||
if (planned.part.compressionAlgorithm === 'gzip') {
|
|
||||||
const storedBytes = await fetchWholePartBytes(planned.part.telegramUrl);
|
|
||||||
const bytes = gunzipSync(storedBytes);
|
|
||||||
return streamFromBytes(bytes.subarray(planned.relativeStart, planned.relativeEnd + 1));
|
|
||||||
}
|
|
||||||
|
|
||||||
const rangeHeader = `bytes=${planned.relativeStart}-${planned.relativeEnd}`;
|
|
||||||
const fetchOpts: RequestInit = { signal: AbortSignal.timeout(TELEGRAM_FETCH_TIMEOUT_MS) };
|
|
||||||
if (!wantsWholePart) {
|
|
||||||
fetchOpts.headers = { range: rangeHeader };
|
|
||||||
}
|
|
||||||
const res = await fetch(planned.part.telegramUrl, fetchOpts);
|
|
||||||
if (!res.ok) throw new Error(`Telegram fetch failed: ${res.status}`);
|
|
||||||
if (wantsWholePart || res.status === 206) return res.body!;
|
|
||||||
|
|
||||||
const bytes = new Uint8Array(await res.arrayBuffer());
|
|
||||||
return streamFromBytes(bytes.slice(planned.relativeStart, planned.relativeEnd + 1));
|
|
||||||
};
|
|
||||||
|
|
||||||
const concatPartStreams = (plannedParts: PlannedPart[]): ReadableStream<Uint8Array> =>
|
|
||||||
new ReadableStream<Uint8Array>({
|
|
||||||
async start(controller) {
|
|
||||||
try {
|
|
||||||
for (const planned of plannedParts) {
|
|
||||||
const stream = await fetchPartBody(planned);
|
|
||||||
const reader = stream.getReader();
|
|
||||||
while (true) {
|
|
||||||
const { value, done } = await reader.read();
|
|
||||||
if (done) break;
|
|
||||||
if (value) controller.enqueue(value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
controller.close();
|
|
||||||
} catch (error) {
|
|
||||||
controller.error(error);
|
|
||||||
}
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
export const createGetObjectResponse = async (input: ObjectResponseInput): Promise<Response> => {
|
|
||||||
if (input.range.type === 'invalid') {
|
|
||||||
throw new Error('createGetObjectResponse received invalid range');
|
|
||||||
}
|
|
||||||
|
|
||||||
const start = input.range.type === 'valid' ? input.range.start : 0;
|
|
||||||
const end = input.range.type === 'valid' ? input.range.end : input.totalSize - 1;
|
|
||||||
const plannedParts = planParts(input.parts, start, end);
|
|
||||||
const contentLength = end >= start ? end - start + 1 : 0;
|
|
||||||
const headers = applyS3Headers(baseHeaders(input, contentLength), input.reqId);
|
|
||||||
|
|
||||||
if (input.range.type === 'valid') {
|
|
||||||
headers.set('content-range', contentRange(start, end, input.totalSize));
|
|
||||||
}
|
|
||||||
|
|
||||||
return new Response(concatPartStreams(plannedParts), {
|
|
||||||
status: input.range.type === 'valid' ? 206 : 200,
|
|
||||||
headers,
|
|
||||||
});
|
|
||||||
};
|
|
||||||
@@ -1,46 +0,0 @@
|
|||||||
export type RangeParseResult =
|
|
||||||
| { type: 'none' }
|
|
||||||
| { type: 'valid'; start: number; end: number }
|
|
||||||
| { type: 'invalid' };
|
|
||||||
|
|
||||||
const DECIMAL = /^\d+$/;
|
|
||||||
|
|
||||||
export const parseRangeHeader = (rangeHeader: string | null, size: number): RangeParseResult => {
|
|
||||||
if (!rangeHeader) return { type: 'none' };
|
|
||||||
if (!Number.isSafeInteger(size) || size < 0) return { type: 'invalid' };
|
|
||||||
if (!rangeHeader.startsWith('bytes=')) return { type: 'invalid' };
|
|
||||||
|
|
||||||
const spec = rangeHeader.slice('bytes='.length).trim();
|
|
||||||
if (spec.includes(',')) return { type: 'invalid' };
|
|
||||||
|
|
||||||
const dash = spec.indexOf('-');
|
|
||||||
if (dash === -1) return { type: 'invalid' };
|
|
||||||
|
|
||||||
const startText = spec.slice(0, dash).trim();
|
|
||||||
const endText = spec.slice(dash + 1).trim();
|
|
||||||
if (!startText && !endText) return { type: 'invalid' };
|
|
||||||
if (size === 0) return { type: 'invalid' };
|
|
||||||
|
|
||||||
if (!startText) {
|
|
||||||
if (!DECIMAL.test(endText)) return { type: 'invalid' };
|
|
||||||
const suffixLength = Number.parseInt(endText, 10);
|
|
||||||
if (suffixLength <= 0) return { type: 'invalid' };
|
|
||||||
return { type: 'valid', start: Math.max(size - suffixLength, 0), end: size - 1 };
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!DECIMAL.test(startText)) return { type: 'invalid' };
|
|
||||||
const start = Number.parseInt(startText, 10);
|
|
||||||
if (start >= size) return { type: 'invalid' };
|
|
||||||
|
|
||||||
if (!endText) return { type: 'valid', start, end: size - 1 };
|
|
||||||
if (!DECIMAL.test(endText)) return { type: 'invalid' };
|
|
||||||
|
|
||||||
const requestedEnd = Number.parseInt(endText, 10);
|
|
||||||
if (requestedEnd < start) return { type: 'invalid' };
|
|
||||||
return { type: 'valid', start, end: Math.min(requestedEnd, size - 1) };
|
|
||||||
};
|
|
||||||
|
|
||||||
export const contentRange = (start: number, end: number, size: number): string =>
|
|
||||||
`bytes ${start}-${end}/${size}`;
|
|
||||||
|
|
||||||
export const unsatisfiedContentRange = (size: number): string => `bytes */${size}`;
|
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
const stripPort = (host: string): string => {
|
|
||||||
// Handle IPv6: [::1]:8080 -> [::1]
|
|
||||||
if (host.startsWith('[')) {
|
|
||||||
const closeBracket = host.indexOf(']');
|
|
||||||
return host.slice(0, closeBracket + 1).toLowerCase();
|
|
||||||
}
|
|
||||||
return host.split(':')[0].toLowerCase().replace(/\.$/, '');
|
|
||||||
};
|
|
||||||
|
|
||||||
const isValidBucketLabel = (bucket: string): boolean =>
|
|
||||||
/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket) &&
|
|
||||||
!bucket.includes('..') &&
|
|
||||||
!bucket.includes('.-') &&
|
|
||||||
!bucket.includes('-.');
|
|
||||||
|
|
||||||
export const extractS3BucketFromHost = (host: string, domains: string[]): string | null => {
|
|
||||||
const normalizedHost = stripPort(host);
|
|
||||||
for (const domain of domains) {
|
|
||||||
const normalizedDomain = stripPort(domain);
|
|
||||||
if (!normalizedDomain || normalizedHost === normalizedDomain) continue;
|
|
||||||
if (!normalizedHost.endsWith(`.${normalizedDomain}`)) continue;
|
|
||||||
|
|
||||||
const bucket = normalizedHost.slice(0, -(normalizedDomain.length + 1));
|
|
||||||
return isValidBucketLabel(bucket) ? bucket : null;
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
};
|
|
||||||
@@ -1,311 +0,0 @@
|
|||||||
import { s3Headers } from './headers';
|
|
||||||
|
|
||||||
const escapeXml = (str: string): string =>
|
|
||||||
str
|
|
||||||
.replace(/&/g, '&')
|
|
||||||
.replace(/</g, '<')
|
|
||||||
.replace(/>/g, '>')
|
|
||||||
.replace(/"/g, '"')
|
|
||||||
.replace(/'/g, ''');
|
|
||||||
|
|
||||||
const isoDate = (d: Date): string => d.toISOString().replace(/\.\d{3}Z$/, 'Z');
|
|
||||||
|
|
||||||
const encodeKey = (value: string, encodingType: string | null = null): string =>
|
|
||||||
encodingType === 'url' ? encodeURIComponent(value) : escapeXml(value);
|
|
||||||
|
|
||||||
// ─────── Bucket operations ───────
|
|
||||||
|
|
||||||
export const listBucketsXml = (
|
|
||||||
buckets: { name: string; createdAt: Date }[],
|
|
||||||
_requestId: string,
|
|
||||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
|
||||||
<Buckets>
|
|
||||||
${buckets
|
|
||||||
.map(
|
|
||||||
(b) => `<Bucket>
|
|
||||||
<Name>${escapeXml(b.name)}</Name>
|
|
||||||
<CreationDate>${isoDate(b.createdAt)}</CreationDate>
|
|
||||||
</Bucket>`,
|
|
||||||
)
|
|
||||||
.join('')}
|
|
||||||
</Buckets>
|
|
||||||
</ListAllMyBucketsResult>`;
|
|
||||||
|
|
||||||
export const bucketVersioningConfigurationXml =
|
|
||||||
(): string => `<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`;
|
|
||||||
|
|
||||||
// ─────── Object listing ───────
|
|
||||||
|
|
||||||
export const listBucketResultXml = (
|
|
||||||
bucketName: string,
|
|
||||||
objects: { key: string; sizeBytes: number; etag: string; lastModified: Date; mimeType: string }[],
|
|
||||||
prefixes: string[],
|
|
||||||
isTruncated: boolean,
|
|
||||||
marker: string | null,
|
|
||||||
maxKeys: number,
|
|
||||||
prefix: string,
|
|
||||||
delimiter: string | null,
|
|
||||||
nextMarker: string | null,
|
|
||||||
_requestId: string,
|
|
||||||
encodingType: string | null = null,
|
|
||||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
|
||||||
<Name>${escapeXml(bucketName)}</Name>
|
|
||||||
<Prefix>${encodeKey(prefix, encodingType)}</Prefix>
|
|
||||||
<Marker>${encodeKey(marker || '', encodingType)}</Marker>
|
|
||||||
<MaxKeys>${maxKeys}</MaxKeys>
|
|
||||||
<Delimiter>${encodeKey(delimiter || '', encodingType)}</Delimiter>
|
|
||||||
${encodingType ? `<EncodingType>${escapeXml(encodingType)}</EncodingType>` : ''}
|
|
||||||
<IsTruncated>${isTruncated}</IsTruncated>
|
|
||||||
${objects
|
|
||||||
.map(
|
|
||||||
(o) => `<Contents>
|
|
||||||
<Key>${encodeKey(o.key, encodingType)}</Key>
|
|
||||||
<LastModified>${isoDate(o.lastModified)}</LastModified>
|
|
||||||
<ETag>"${o.etag}"</ETag>
|
|
||||||
<Size>${o.sizeBytes}</Size>
|
|
||||||
<StorageClass>STANDARD</StorageClass>
|
|
||||||
</Contents>`,
|
|
||||||
)
|
|
||||||
.join('')}
|
|
||||||
${prefixes
|
|
||||||
.map(
|
|
||||||
(p) => `<CommonPrefixes>
|
|
||||||
<Prefix>${encodeKey(p, encodingType)}</Prefix>
|
|
||||||
</CommonPrefixes>`,
|
|
||||||
)
|
|
||||||
.join('')}
|
|
||||||
${nextMarker ? `<NextMarker>${encodeKey(nextMarker, encodingType)}</NextMarker>` : ''}
|
|
||||||
</ListBucketResult>`;
|
|
||||||
|
|
||||||
export const listBucketV2ResultXml = (
|
|
||||||
bucketName: string,
|
|
||||||
objects: { key: string; sizeBytes: number; etag: string; lastModified: Date; mimeType: string }[],
|
|
||||||
prefixes: string[],
|
|
||||||
isTruncated: boolean,
|
|
||||||
maxKeys: number,
|
|
||||||
prefix: string,
|
|
||||||
delimiter: string | null,
|
|
||||||
continuationToken: string | null,
|
|
||||||
nextContinuationToken: string | null,
|
|
||||||
keyCount: number,
|
|
||||||
_requestId: string,
|
|
||||||
encodingType: string | null = null,
|
|
||||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<ListBucketResultV2 xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
|
||||||
<Name>${escapeXml(bucketName)}</Name>
|
|
||||||
<Prefix>${encodeKey(prefix, encodingType)}</Prefix>
|
|
||||||
<MaxKeys>${maxKeys}</MaxKeys>
|
|
||||||
<KeyCount>${keyCount}</KeyCount>
|
|
||||||
${delimiter ? `<Delimiter>${encodeKey(delimiter, encodingType)}</Delimiter>` : ''}
|
|
||||||
${encodingType ? `<EncodingType>${escapeXml(encodingType)}</EncodingType>` : ''}
|
|
||||||
${continuationToken ? `<ContinuationToken>${encodeKey(continuationToken, encodingType)}</ContinuationToken>` : ''}
|
|
||||||
<IsTruncated>${isTruncated}</IsTruncated>
|
|
||||||
${objects
|
|
||||||
.map(
|
|
||||||
(o) => `<Contents>
|
|
||||||
<Key>${encodeKey(o.key, encodingType)}</Key>
|
|
||||||
<LastModified>${isoDate(o.lastModified)}</LastModified>
|
|
||||||
<ETag>"${o.etag}"</ETag>
|
|
||||||
<Size>${o.sizeBytes}</Size>
|
|
||||||
<StorageClass>STANDARD</StorageClass>
|
|
||||||
</Contents>`,
|
|
||||||
)
|
|
||||||
.join('')}
|
|
||||||
${prefixes
|
|
||||||
.map(
|
|
||||||
(p) => `<CommonPrefixes>
|
|
||||||
<Prefix>${encodeKey(p, encodingType)}</Prefix>
|
|
||||||
</CommonPrefixes>`,
|
|
||||||
)
|
|
||||||
.join('')}
|
|
||||||
${nextContinuationToken ? `<NextContinuationToken>${encodeKey(nextContinuationToken, encodingType)}</NextContinuationToken>` : ''}
|
|
||||||
</ListBucketResultV2>`;
|
|
||||||
|
|
||||||
// ─────── Multipart ───────
|
|
||||||
|
|
||||||
export const initiateMultipartUploadXml = (
|
|
||||||
bucketName: string,
|
|
||||||
key: string,
|
|
||||||
uploadId: string,
|
|
||||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
|
||||||
<Bucket>${escapeXml(bucketName)}</Bucket>
|
|
||||||
<Key>${escapeXml(key)}</Key>
|
|
||||||
<UploadId>${uploadId}</UploadId>
|
|
||||||
</InitiateMultipartUploadResult>`;
|
|
||||||
|
|
||||||
export const listPartsXml = (
|
|
||||||
bucketName: string,
|
|
||||||
key: string,
|
|
||||||
uploadId: string,
|
|
||||||
parts: { partNumber: number; etag: string; sizeBytes: number; createdAt: Date }[],
|
|
||||||
maxParts: number,
|
|
||||||
isTruncated: boolean,
|
|
||||||
_requestId: string,
|
|
||||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<ListPartsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
|
||||||
<Bucket>${escapeXml(bucketName)}</Bucket>
|
|
||||||
<Key>${escapeXml(key)}</Key>
|
|
||||||
<UploadId>${uploadId}</UploadId>
|
|
||||||
<MaxParts>${maxParts}</MaxParts>
|
|
||||||
<IsTruncated>${isTruncated}</IsTruncated>
|
|
||||||
${parts
|
|
||||||
.map(
|
|
||||||
(p) => `<Part>
|
|
||||||
<PartNumber>${p.partNumber}</PartNumber>
|
|
||||||
<LastModified>${isoDate(p.createdAt)}</LastModified>
|
|
||||||
<ETag>"${p.etag}"</ETag>
|
|
||||||
<Size>${p.sizeBytes}</Size>
|
|
||||||
</Part>`,
|
|
||||||
)
|
|
||||||
.join('')}
|
|
||||||
</ListPartsResult>`;
|
|
||||||
|
|
||||||
export const listMultipartUploadsXml = (
|
|
||||||
bucketName: string,
|
|
||||||
uploads: { key: string; uploadId: string; initiatedAt: Date; initiatedBy: string }[],
|
|
||||||
maxUploads: number,
|
|
||||||
isTruncated: boolean,
|
|
||||||
nextKeyMarker: string | null,
|
|
||||||
_requestId: string,
|
|
||||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<ListMultipartUploadsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
|
||||||
<Bucket>${escapeXml(bucketName)}</Bucket>
|
|
||||||
<KeyMarker></KeyMarker>
|
|
||||||
<UploadIdMarker></UploadIdMarker>
|
|
||||||
${nextKeyMarker ? `<NextKeyMarker>${escapeXml(nextKeyMarker)}</NextKeyMarker>` : ''}
|
|
||||||
<MaxUploads>${maxUploads}</MaxUploads>
|
|
||||||
<IsTruncated>${isTruncated}</IsTruncated>
|
|
||||||
${uploads
|
|
||||||
.map(
|
|
||||||
(u) => `<Upload>
|
|
||||||
<Key>${escapeXml(u.key)}</Key>
|
|
||||||
<UploadId>${u.uploadId}</UploadId>
|
|
||||||
<Initiator><ID>${escapeXml(u.initiatedBy || 's3')}</ID><DisplayName>${escapeXml(u.initiatedBy || 's3')}</DisplayName></Initiator>
|
|
||||||
<Owner><ID>${escapeXml(u.initiatedBy || 's3')}</ID><DisplayName>${escapeXml(u.initiatedBy || 's3')}</DisplayName></Owner>
|
|
||||||
<StorageClass>STANDARD</StorageClass>
|
|
||||||
<Initiated>${isoDate(u.initiatedAt)}</Initiated>
|
|
||||||
</Upload>`,
|
|
||||||
)
|
|
||||||
.join('')}
|
|
||||||
</ListMultipartUploadsResult>`;
|
|
||||||
|
|
||||||
export const completeMultipartUploadXml = (
|
|
||||||
bucketName: string,
|
|
||||||
key: string,
|
|
||||||
etag: string,
|
|
||||||
location: string,
|
|
||||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<CompleteMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
|
||||||
<Location>${escapeXml(location)}</Location>
|
|
||||||
<Bucket>${escapeXml(bucketName)}</Bucket>
|
|
||||||
<Key>${escapeXml(key)}</Key>
|
|
||||||
<ETag>"${etag}"</ETag>
|
|
||||||
</CompleteMultipartUploadResult>`;
|
|
||||||
|
|
||||||
// ─────── Delete result ───────
|
|
||||||
|
|
||||||
export const deleteResultXml = (
|
|
||||||
deleted: string[],
|
|
||||||
errors: { key: string; code: string; message: string }[],
|
|
||||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<DeleteResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
|
||||||
${deleted
|
|
||||||
.map(
|
|
||||||
(key) => `<Deleted>
|
|
||||||
<Key>${escapeXml(key)}</Key>
|
|
||||||
</Deleted>`,
|
|
||||||
)
|
|
||||||
.join('')}
|
|
||||||
${errors
|
|
||||||
.map(
|
|
||||||
(e) => `<Error>
|
|
||||||
<Key>${escapeXml(e.key)}</Key>
|
|
||||||
<Code>${e.code}</Code>
|
|
||||||
<Message>${escapeXml(e.message)}</Message>
|
|
||||||
</Error>`,
|
|
||||||
)
|
|
||||||
.join('')}
|
|
||||||
</DeleteResult>`;
|
|
||||||
|
|
||||||
// ─────── Copy ───────
|
|
||||||
|
|
||||||
export const copyObjectResultXml = (
|
|
||||||
etag: string,
|
|
||||||
lastModified: Date,
|
|
||||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<CopyObjectResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
|
||||||
<ETag>"${etag}"</ETag>
|
|
||||||
<LastModified>${isoDate(lastModified)}</LastModified>
|
|
||||||
</CopyObjectResult>`;
|
|
||||||
|
|
||||||
// ─────── Error ───────
|
|
||||||
|
|
||||||
export const s3ErrorXml = (
|
|
||||||
code: string,
|
|
||||||
message: string,
|
|
||||||
resource: string,
|
|
||||||
requestId: string,
|
|
||||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<Error>
|
|
||||||
<Code>${code}</Code>
|
|
||||||
<Message>${escapeXml(message)}</Message>
|
|
||||||
<Resource>${escapeXml(resource)}</Resource>
|
|
||||||
<RequestId>${requestId}</RequestId>
|
|
||||||
<HostId>${requestId}</HostId>
|
|
||||||
</Error>`;
|
|
||||||
|
|
||||||
export const s3ErrorResponse = (
|
|
||||||
code: string,
|
|
||||||
message: string,
|
|
||||||
resource: string,
|
|
||||||
status: number,
|
|
||||||
requestId: string = '',
|
|
||||||
extraHeaders: Record<string, string> = {},
|
|
||||||
): Response =>
|
|
||||||
new Response(s3ErrorXml(code, message, resource, requestId), {
|
|
||||||
status,
|
|
||||||
headers: s3Headers(requestId, {
|
|
||||||
'content-type': 'application/xml',
|
|
||||||
...extraHeaders,
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
|
|
||||||
// ─────── DeleteObjects XML parser ───────
|
|
||||||
|
|
||||||
export const parseDeleteObjectsBody = (body: string): { keys: string[]; quiet: boolean } => {
|
|
||||||
// H9: Use non-greedy match to handle keys containing < character
|
|
||||||
const keys = Array.from(body.matchAll(/<Key>([\s\S]*?)<\/Key>/g), (match) => match[1]);
|
|
||||||
// Handle whitespace inside <Quiet> element + namespace prefix support
|
|
||||||
const quiet = /<\w*:?Quiet\w*>\s*true\s*<\/\w*:?Quiet\w*>/i.test(body);
|
|
||||||
return { keys, quiet };
|
|
||||||
};
|
|
||||||
|
|
||||||
// ─────── CompleteMultipartUpload XML parser ───────
|
|
||||||
|
|
||||||
export interface CompletePart {
|
|
||||||
partNumber: number;
|
|
||||||
etag: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const parseCompleteMultipartBody = (body: string): CompletePart[] => {
|
|
||||||
const parts: CompletePart[] = [];
|
|
||||||
const partRegex = /<Part>[\s\S]*?<\/Part>/g;
|
|
||||||
const partMatch = body.match(partRegex) || [];
|
|
||||||
|
|
||||||
for (const partXml of partMatch) {
|
|
||||||
const numMatch = partXml.match(/<PartNumber>(\d+)<\/PartNumber>/);
|
|
||||||
const etagMatch = partXml.match(/<ETag>"?([^"<\s]+)"?<\/ETag>/);
|
|
||||||
if (numMatch && etagMatch) {
|
|
||||||
parts.push({
|
|
||||||
partNumber: Number.parseInt(numMatch[1], 10),
|
|
||||||
etag: etagMatch[1].replace(/^"/, '').replace(/"$/, ''),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return parts;
|
|
||||||
};
|
|
||||||
@@ -1,152 +0,0 @@
|
|||||||
import { createReadStream } from 'node:fs';
|
|
||||||
import { nanoid } from 'nanoid';
|
|
||||||
import { db, files as fileSchema } from '../db';
|
|
||||||
import type { NewFile } from '../db/schema';
|
|
||||||
import { config } from '../env';
|
|
||||||
import { botPool } from '../infrastructure/telegram/bot-pool';
|
|
||||||
import { cleanupTempFile } from './file';
|
|
||||||
import { createZip, type ZipEntry } from './zip';
|
|
||||||
|
|
||||||
export type PreparedUpload = {
|
|
||||||
tempPath: string;
|
|
||||||
fileHash: string;
|
|
||||||
sizeBytes: number;
|
|
||||||
signatureBuffer: Buffer;
|
|
||||||
};
|
|
||||||
|
|
||||||
export type UploadedFile = NewFile & {
|
|
||||||
createdAt: Date;
|
|
||||||
updatedAt: Date;
|
|
||||||
};
|
|
||||||
|
|
||||||
export type BatchUploadItem = {
|
|
||||||
prepared: PreparedUpload;
|
|
||||||
fileName: string;
|
|
||||||
mimeType: string;
|
|
||||||
fileType: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
type PendingUpload = BatchUploadItem & {
|
|
||||||
resolve: (file: UploadedFile) => void;
|
|
||||||
reject: (error: unknown) => void;
|
|
||||||
};
|
|
||||||
|
|
||||||
const BATCH_WINDOW_MS = 2000;
|
|
||||||
|
|
||||||
let pendingUploads: PendingUpload[] = [];
|
|
||||||
let flushTimer: ReturnType<typeof setTimeout> | null = null;
|
|
||||||
|
|
||||||
const buildUploadedFile = (
|
|
||||||
item: BatchUploadItem,
|
|
||||||
entry: ZipEntry,
|
|
||||||
archive: {
|
|
||||||
telegramFileId: string;
|
|
||||||
telegramFileUniqueId: string;
|
|
||||||
storageMessageId: number;
|
|
||||||
fileName: string;
|
|
||||||
sizeBytes: number;
|
|
||||||
},
|
|
||||||
): UploadedFile => ({
|
|
||||||
publicId: nanoid(),
|
|
||||||
telegramFileId: archive.telegramFileId,
|
|
||||||
telegramFileUniqueId: archive.telegramFileUniqueId,
|
|
||||||
storageChatId: config.storageChatId,
|
|
||||||
storageMessageId: archive.storageMessageId,
|
|
||||||
fileName: item.fileName,
|
|
||||||
mimeType: item.mimeType || 'application/octet-stream',
|
|
||||||
sizeBytes: item.prepared.sizeBytes,
|
|
||||||
fileType: item.fileType,
|
|
||||||
uploaderId: 0,
|
|
||||||
fileHash: item.prepared.fileHash,
|
|
||||||
archiveTelegramFileId: archive.telegramFileId,
|
|
||||||
archiveStorageMessageId: archive.storageMessageId,
|
|
||||||
archiveFileName: archive.fileName,
|
|
||||||
archiveEntryName: entry.entryName,
|
|
||||||
archiveMimeType: 'application/zip',
|
|
||||||
archiveSizeBytes: archive.sizeBytes,
|
|
||||||
createdAt: new Date(),
|
|
||||||
updatedAt: new Date(),
|
|
||||||
});
|
|
||||||
|
|
||||||
const flushUploads = async (): Promise<void> => {
|
|
||||||
if (flushTimer) {
|
|
||||||
clearTimeout(flushTimer);
|
|
||||||
flushTimer = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const batch = pendingUploads;
|
|
||||||
pendingUploads = [];
|
|
||||||
if (batch.length === 0) return;
|
|
||||||
|
|
||||||
let zipTempPath: string | null = null;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const zip = await createZip(
|
|
||||||
batch.map((item) => ({ tempPath: item.prepared.tempPath, fileName: item.fileName })),
|
|
||||||
);
|
|
||||||
zipTempPath = zip.tempPath;
|
|
||||||
const archiveFileName = `filedrop-${nanoid()}.zip`;
|
|
||||||
const archiveResult = await botPool.forwardToStorage(
|
|
||||||
createReadStream(zip.tempPath),
|
|
||||||
archiveFileName,
|
|
||||||
'document',
|
|
||||||
);
|
|
||||||
|
|
||||||
const uploadedFiles = batch.map((item, index) =>
|
|
||||||
buildUploadedFile(item, zip.entries[index], {
|
|
||||||
telegramFileId: archiveResult.telegramFileId,
|
|
||||||
telegramFileUniqueId: archiveResult.telegramFileUniqueId,
|
|
||||||
storageMessageId: archiveResult.storageMessageId,
|
|
||||||
fileName: archiveFileName,
|
|
||||||
sizeBytes: zip.sizeBytes,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
await db.insert(fileSchema).values(uploadedFiles);
|
|
||||||
|
|
||||||
for (let i = 0; i < batch.length; i++) {
|
|
||||||
batch[i].resolve(uploadedFiles[i]);
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
for (const item of batch) {
|
|
||||||
item.reject(error);
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
await Promise.all(batch.map((item) => cleanupTempFile(item.prepared.tempPath)));
|
|
||||||
if (zipTempPath) await cleanupTempFile(zipTempPath);
|
|
||||||
// Reschedule timer if new items arrived during async processing
|
|
||||||
if (pendingUploads.length > 0 && !flushTimer) {
|
|
||||||
flushTimer = setTimeout(() => {
|
|
||||||
void flushUploads();
|
|
||||||
}, BATCH_WINDOW_MS);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const getPendingSize = (): number =>
|
|
||||||
pendingUploads.reduce((total, item) => total + item.prepared.sizeBytes, 0);
|
|
||||||
|
|
||||||
export const enqueuePreparedUpload = (item: BatchUploadItem): Promise<UploadedFile> => {
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
pendingUploads.push({ ...item, resolve, reject });
|
|
||||||
|
|
||||||
if (!flushTimer) {
|
|
||||||
flushTimer = setTimeout(() => {
|
|
||||||
void flushUploads();
|
|
||||||
}, BATCH_WINDOW_MS);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (
|
|
||||||
pendingUploads.length >= config.batchMaxItems ||
|
|
||||||
getPendingSize() >= config.batchMaxSizeBytes
|
|
||||||
) {
|
|
||||||
void flushUploads();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
export const flushPendingUploads = async (): Promise<void> => {
|
|
||||||
await flushUploads();
|
|
||||||
};
|
|
||||||
|
|
||||||
export const getPendingUploadCount = (): number => pendingUploads.length;
|
|
||||||
@@ -1,292 +0,0 @@
|
|||||||
import { once } from 'node:events';
|
|
||||||
import { createReadStream, createWriteStream } from 'node:fs';
|
|
||||||
import { open, stat, unlink } from 'node:fs/promises';
|
|
||||||
import { basename } from 'node:path';
|
|
||||||
import { finished } from 'node:stream/promises';
|
|
||||||
import { nanoid } from 'nanoid';
|
|
||||||
|
|
||||||
export type ZipInputFile = {
|
|
||||||
tempPath: string;
|
|
||||||
fileName: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
export type ZipEntry = {
|
|
||||||
fileName: string;
|
|
||||||
entryName: string;
|
|
||||||
crc32: number;
|
|
||||||
compressedSize: number;
|
|
||||||
uncompressedSize: number;
|
|
||||||
localHeaderOffset: number;
|
|
||||||
};
|
|
||||||
|
|
||||||
export type CreatedZip = {
|
|
||||||
tempPath: string;
|
|
||||||
sizeBytes: number;
|
|
||||||
fileHash: string;
|
|
||||||
entries: ZipEntry[];
|
|
||||||
};
|
|
||||||
|
|
||||||
const CRC32_TABLE = new Uint32Array(256).map((_, index) => {
|
|
||||||
let value = index;
|
|
||||||
for (let bit = 0; bit < 8; bit++) {
|
|
||||||
value = value & 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1;
|
|
||||||
}
|
|
||||||
return value >>> 0;
|
|
||||||
});
|
|
||||||
|
|
||||||
const updateCrc32 = (crc: number, chunk: Buffer): number => {
|
|
||||||
let value = crc;
|
|
||||||
for (const byte of chunk) {
|
|
||||||
value = CRC32_TABLE[(value ^ byte) & 0xff] ^ (value >>> 8);
|
|
||||||
}
|
|
||||||
return value >>> 0;
|
|
||||||
};
|
|
||||||
|
|
||||||
const dosDateTime = (date = new Date()): { date: number; time: number } => {
|
|
||||||
const year = Math.max(date.getFullYear(), 1980);
|
|
||||||
return {
|
|
||||||
time: (date.getHours() << 11) | (date.getMinutes() << 5) | Math.floor(date.getSeconds() / 2),
|
|
||||||
date: ((year - 1980) << 9) | ((date.getMonth() + 1) << 5) | date.getDate(),
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
const writeUInt16 = (value: number): Buffer<ArrayBuffer> => {
|
|
||||||
const buffer = Buffer.allocUnsafe(2);
|
|
||||||
buffer.writeUInt16LE(value & 0xffff, 0);
|
|
||||||
return buffer;
|
|
||||||
};
|
|
||||||
|
|
||||||
const writeUInt32 = (value: number): Buffer<ArrayBuffer> => {
|
|
||||||
const buffer = Buffer.allocUnsafe(4);
|
|
||||||
buffer.writeUInt32LE(value >>> 0, 0);
|
|
||||||
return buffer;
|
|
||||||
};
|
|
||||||
|
|
||||||
const writeChunk = async (
|
|
||||||
writer: ReturnType<typeof createWriteStream>,
|
|
||||||
chunk: Buffer,
|
|
||||||
): Promise<void> => {
|
|
||||||
if (!writer.write(chunk)) {
|
|
||||||
await once(writer, 'drain');
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const finishWriter = async (writer: ReturnType<typeof createWriteStream>): Promise<void> => {
|
|
||||||
writer.end();
|
|
||||||
await finished(writer);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const sanitizeZipEntryName = (fileName: string, usedNames = new Set<string>()): string => {
|
|
||||||
const cleaned = basename(fileName)
|
|
||||||
.replace(/[\\/]+/g, '_')
|
|
||||||
.replace(/\.\.+/g, '.')
|
|
||||||
.trim();
|
|
||||||
const fallback = cleaned && cleaned !== '.' && cleaned !== '..' ? cleaned : 'file';
|
|
||||||
const dotIndex = fallback.lastIndexOf('.');
|
|
||||||
const baseName = dotIndex > 0 ? fallback.slice(0, dotIndex) : fallback;
|
|
||||||
const extension = dotIndex > 0 ? fallback.slice(dotIndex) : '';
|
|
||||||
let candidate = fallback;
|
|
||||||
let counter = 1;
|
|
||||||
|
|
||||||
while (usedNames.has(candidate)) {
|
|
||||||
candidate = `${baseName}-${counter}${extension}`;
|
|
||||||
counter++;
|
|
||||||
}
|
|
||||||
|
|
||||||
usedNames.add(candidate);
|
|
||||||
return candidate;
|
|
||||||
};
|
|
||||||
|
|
||||||
const calculateFileCrc32 = async (tempPath: string): Promise<number> => {
|
|
||||||
let crc = 0xffffffff;
|
|
||||||
const reader = createReadStream(tempPath);
|
|
||||||
for await (const chunk of reader) {
|
|
||||||
crc = updateCrc32(crc, chunk as Buffer);
|
|
||||||
}
|
|
||||||
return (crc ^ 0xffffffff) >>> 0;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const createZip = async (files: ZipInputFile[]): Promise<CreatedZip> => {
|
|
||||||
const tempPath = `/tmp/filedrop-${nanoid()}.zip`;
|
|
||||||
const writer = createWriteStream(tempPath);
|
|
||||||
const hasher = new Bun.CryptoHasher('sha256');
|
|
||||||
const entries: ZipEntry[] = [];
|
|
||||||
const usedNames = new Set<string>();
|
|
||||||
let offset = 0;
|
|
||||||
|
|
||||||
const writeHashed = async (chunk: Buffer): Promise<void> => {
|
|
||||||
hasher.update(chunk);
|
|
||||||
await writeChunk(writer, chunk);
|
|
||||||
offset += chunk.byteLength;
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
for (const file of files) {
|
|
||||||
const entryName = sanitizeZipEntryName(file.fileName, usedNames);
|
|
||||||
const nameBuffer = Buffer.from(entryName);
|
|
||||||
const fileStats = await stat(file.tempPath);
|
|
||||||
const { date, time } = dosDateTime();
|
|
||||||
const localHeaderOffset = offset;
|
|
||||||
const crc32 = await calculateFileCrc32(file.tempPath);
|
|
||||||
|
|
||||||
const localHeader = Buffer.concat([
|
|
||||||
writeUInt32(0x04034b50),
|
|
||||||
writeUInt16(20),
|
|
||||||
writeUInt16(0),
|
|
||||||
writeUInt16(0),
|
|
||||||
writeUInt16(time),
|
|
||||||
writeUInt16(date),
|
|
||||||
writeUInt32(crc32),
|
|
||||||
writeUInt32(fileStats.size),
|
|
||||||
writeUInt32(fileStats.size),
|
|
||||||
writeUInt16(nameBuffer.byteLength),
|
|
||||||
writeUInt16(0),
|
|
||||||
nameBuffer,
|
|
||||||
]);
|
|
||||||
|
|
||||||
await writeHashed(localHeader);
|
|
||||||
const reader = createReadStream(file.tempPath);
|
|
||||||
for await (const chunk of reader) {
|
|
||||||
await writeHashed(chunk as Buffer);
|
|
||||||
}
|
|
||||||
|
|
||||||
entries.push({
|
|
||||||
fileName: file.fileName,
|
|
||||||
entryName,
|
|
||||||
crc32,
|
|
||||||
compressedSize: fileStats.size,
|
|
||||||
uncompressedSize: fileStats.size,
|
|
||||||
localHeaderOffset,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const centralDirectoryOffset = offset;
|
|
||||||
for (const entry of entries) {
|
|
||||||
const nameBuffer = Buffer.from(entry.entryName);
|
|
||||||
const { date, time } = dosDateTime();
|
|
||||||
await writeHashed(
|
|
||||||
Buffer.concat([
|
|
||||||
writeUInt32(0x02014b50),
|
|
||||||
writeUInt16(20),
|
|
||||||
writeUInt16(20),
|
|
||||||
writeUInt16(0),
|
|
||||||
writeUInt16(0),
|
|
||||||
writeUInt16(time),
|
|
||||||
writeUInt16(date),
|
|
||||||
writeUInt32(entry.crc32),
|
|
||||||
writeUInt32(entry.compressedSize),
|
|
||||||
writeUInt32(entry.uncompressedSize),
|
|
||||||
writeUInt16(nameBuffer.byteLength),
|
|
||||||
writeUInt16(0),
|
|
||||||
writeUInt16(0),
|
|
||||||
writeUInt16(0),
|
|
||||||
writeUInt16(0),
|
|
||||||
writeUInt32(0),
|
|
||||||
writeUInt32(entry.localHeaderOffset),
|
|
||||||
nameBuffer,
|
|
||||||
]),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const centralDirectorySize = offset - centralDirectoryOffset;
|
|
||||||
await writeHashed(
|
|
||||||
Buffer.concat([
|
|
||||||
writeUInt32(0x06054b50),
|
|
||||||
writeUInt16(0),
|
|
||||||
writeUInt16(0),
|
|
||||||
writeUInt16(entries.length),
|
|
||||||
writeUInt16(entries.length),
|
|
||||||
writeUInt32(centralDirectorySize),
|
|
||||||
writeUInt32(centralDirectoryOffset),
|
|
||||||
writeUInt16(0),
|
|
||||||
]),
|
|
||||||
);
|
|
||||||
|
|
||||||
await finishWriter(writer);
|
|
||||||
|
|
||||||
return {
|
|
||||||
tempPath,
|
|
||||||
sizeBytes: offset,
|
|
||||||
fileHash: hasher.digest('hex'),
|
|
||||||
entries,
|
|
||||||
};
|
|
||||||
} catch (error) {
|
|
||||||
writer.destroy();
|
|
||||||
await unlink(tempPath).catch(() => {});
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const extractZipEntry = async (
|
|
||||||
zipBuffer: Buffer,
|
|
||||||
entryName: string,
|
|
||||||
): Promise<Buffer | null> => {
|
|
||||||
let offset = 0;
|
|
||||||
|
|
||||||
while (offset + 30 <= zipBuffer.byteLength) {
|
|
||||||
const signature = zipBuffer.readUInt32LE(offset);
|
|
||||||
if (signature !== 0x04034b50) break;
|
|
||||||
|
|
||||||
const compressionMethod = zipBuffer.readUInt16LE(offset + 8);
|
|
||||||
const compressedSize = zipBuffer.readUInt32LE(offset + 18);
|
|
||||||
const fileNameLength = zipBuffer.readUInt16LE(offset + 26);
|
|
||||||
const extraLength = zipBuffer.readUInt16LE(offset + 28);
|
|
||||||
const nameStart = offset + 30;
|
|
||||||
const nameEnd = nameStart + fileNameLength;
|
|
||||||
const dataStart = nameEnd + extraLength;
|
|
||||||
const dataEnd = dataStart + compressedSize;
|
|
||||||
const currentName = zipBuffer.subarray(nameStart, nameEnd).toString();
|
|
||||||
|
|
||||||
if (currentName === entryName) {
|
|
||||||
if (compressionMethod !== 0) return null;
|
|
||||||
return zipBuffer.subarray(dataStart, dataEnd);
|
|
||||||
}
|
|
||||||
|
|
||||||
offset = dataEnd;
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
};
|
|
||||||
|
|
||||||
export type LocatedZipEntry = {
|
|
||||||
start: number;
|
|
||||||
length: number;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const locateZipEntry = async (
|
|
||||||
zipPath: string,
|
|
||||||
entryName: string,
|
|
||||||
): Promise<LocatedZipEntry | null> => {
|
|
||||||
const handle = await open(zipPath, 'r');
|
|
||||||
let offset = 0;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const header = Buffer.alloc(30);
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
const { bytesRead } = await handle.read(header, 0, header.byteLength, offset);
|
|
||||||
if (bytesRead < header.byteLength) return null;
|
|
||||||
|
|
||||||
const signature = header.readUInt32LE(0);
|
|
||||||
if (signature !== 0x04034b50) return null;
|
|
||||||
|
|
||||||
const compressionMethod = header.readUInt16LE(8);
|
|
||||||
const compressedSize = header.readUInt32LE(18);
|
|
||||||
const fileNameLength = header.readUInt16LE(26);
|
|
||||||
const extraLength = header.readUInt16LE(28);
|
|
||||||
const nameBuffer = Buffer.alloc(fileNameLength);
|
|
||||||
const nameOffset = offset + 30;
|
|
||||||
await handle.read(nameBuffer, 0, fileNameLength, nameOffset);
|
|
||||||
|
|
||||||
const dataStart = nameOffset + fileNameLength + extraLength;
|
|
||||||
if (nameBuffer.toString() === entryName) {
|
|
||||||
if (compressionMethod !== 0) return null;
|
|
||||||
return { start: dataStart, length: compressedSize };
|
|
||||||
}
|
|
||||||
|
|
||||||
offset = dataStart + compressedSize;
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
await handle.close();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
@@ -10,14 +10,14 @@ const setEnv = (key: string, value: string) => {
|
|||||||
defaultEnv('BOT_TOKEN', '123456:ABC-DEF');
|
defaultEnv('BOT_TOKEN', '123456:ABC-DEF');
|
||||||
defaultEnv('STORAGE_CHANNEL_ID', '-1001234567890');
|
defaultEnv('STORAGE_CHANNEL_ID', '-1001234567890');
|
||||||
defaultEnv('BASE_URL', 'https://example.com');
|
defaultEnv('BASE_URL', 'https://example.com');
|
||||||
defaultEnv('DATABASE_URL', 'postgresql://user:pass@localhost:5432/test');
|
defaultEnv('DATABASE_URL', 'postgresql://asephs:***@100.121.180.82:6432/test');
|
||||||
defaultEnv('PORT', '3000');
|
defaultEnv('PORT', '4000');
|
||||||
defaultEnv('NODE_ENV', 'test');
|
defaultEnv('NODE_ENV', 'test');
|
||||||
setEnv('ADMIN_API_TOKEN', 'route-secret-token');
|
setEnv('ADMIN_API_TOKEN', 'route-secret-token');
|
||||||
setEnv('SESSION_COOKIE_NAME', 'route_session');
|
setEnv('SESSION_COOKIE_NAME', 'route_session');
|
||||||
setEnv('SESSION_COOKIE_MAX_AGE_SECONDS', '3600');
|
setEnv('SESSION_COOKIE_MAX_AGE_SECONDS', '3600');
|
||||||
|
|
||||||
const { createSessionCookie } = await import('../src/utils/auth');
|
const { createSessionCookie } = await import('../src/interfaces/http/middleware/auth');
|
||||||
const { handleLogin, handleLogout, handleMe } = await import(
|
const { handleLogin, handleLogout, handleMe } = await import(
|
||||||
'../src/interfaces/http/controllers/auth-controller'
|
'../src/interfaces/http/controllers/auth-controller'
|
||||||
);
|
);
|
||||||
|
|||||||
+3
-3
@@ -10,14 +10,14 @@ const setEnv = (key: string, value: string) => {
|
|||||||
defaultEnv('BOT_TOKEN', '123456:ABC-DEF');
|
defaultEnv('BOT_TOKEN', '123456:ABC-DEF');
|
||||||
defaultEnv('STORAGE_CHANNEL_ID', '-1001234567890');
|
defaultEnv('STORAGE_CHANNEL_ID', '-1001234567890');
|
||||||
defaultEnv('BASE_URL', 'https://example.com');
|
defaultEnv('BASE_URL', 'https://example.com');
|
||||||
defaultEnv('DATABASE_URL', 'postgresql://user:pass@localhost:5432/test');
|
defaultEnv('DATABASE_URL', 'postgresql://asephs:***@100.121.180.82:6432/test');
|
||||||
defaultEnv('PORT', '3000');
|
defaultEnv('PORT', '4000');
|
||||||
defaultEnv('NODE_ENV', 'test');
|
defaultEnv('NODE_ENV', 'test');
|
||||||
setEnv('ADMIN_API_TOKEN', 'route-secret-token');
|
setEnv('ADMIN_API_TOKEN', 'route-secret-token');
|
||||||
setEnv('SESSION_COOKIE_NAME', 'route_session');
|
setEnv('SESSION_COOKIE_NAME', 'route_session');
|
||||||
setEnv('SESSION_COOKIE_MAX_AGE_SECONDS', '3600');
|
setEnv('SESSION_COOKIE_MAX_AGE_SECONDS', '3600');
|
||||||
|
|
||||||
const auth = await import('../src/utils/auth');
|
const auth = await import('../src/interfaces/http/middleware/auth');
|
||||||
|
|
||||||
describe('auth utilities', () => {
|
describe('auth utilities', () => {
|
||||||
const secret = 'admin-secret-token';
|
const secret = 'admin-secret-token';
|
||||||
|
|||||||
@@ -0,0 +1,130 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test';
|
||||||
|
|
||||||
|
process.env.BOT_TOKENS = 'bot1:token,bot2:token,bot3:token';
|
||||||
|
process.env.STORAGE_CHANNEL_ID = '-1001234567890';
|
||||||
|
process.env.BASE_URL = 'https://example.com';
|
||||||
|
process.env.DATABASE_URL = 'sqlite://test.db';
|
||||||
|
process.env.PORT = '4000';
|
||||||
|
|
||||||
|
// Track mock queue instances for per-bot assertions
|
||||||
|
const queueInstances: Array<{
|
||||||
|
concurrency: number;
|
||||||
|
add: ReturnType<typeof mock>;
|
||||||
|
pending: number;
|
||||||
|
size: number;
|
||||||
|
}> = [];
|
||||||
|
|
||||||
|
// Mock PQueue so we can verify concurrency
|
||||||
|
const mockAdd = mock(function addFn(this: any, fn: () => Promise<any>) {
|
||||||
|
return Promise.resolve().then(() => fn());
|
||||||
|
});
|
||||||
|
|
||||||
|
mock.module('p-queue', () => {
|
||||||
|
return {
|
||||||
|
default: mock(function MockQueue(this: any, opts?: { concurrency?: number }) {
|
||||||
|
const instance = {
|
||||||
|
concurrency: opts?.concurrency ?? 1,
|
||||||
|
add: mockAdd,
|
||||||
|
pending: 0,
|
||||||
|
size: 0,
|
||||||
|
};
|
||||||
|
queueInstances.push(instance);
|
||||||
|
return instance;
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
// Mock Telegraf — use a class so `new Telegraf(token)` works correctly
|
||||||
|
const mockTelegramInstances: Record<
|
||||||
|
string,
|
||||||
|
{
|
||||||
|
token: string;
|
||||||
|
sendDocument: ReturnType<typeof mock>;
|
||||||
|
sendPhoto: ReturnType<typeof mock>;
|
||||||
|
getFile: ReturnType<typeof mock>;
|
||||||
|
}
|
||||||
|
> = {};
|
||||||
|
|
||||||
|
class MockTelegraf {
|
||||||
|
token: string;
|
||||||
|
telegram: {
|
||||||
|
token: string;
|
||||||
|
sendDocument: ReturnType<typeof mock>;
|
||||||
|
sendPhoto: ReturnType<typeof mock>;
|
||||||
|
getFile: ReturnType<typeof mock>;
|
||||||
|
};
|
||||||
|
|
||||||
|
constructor(token: string) {
|
||||||
|
this.token = token;
|
||||||
|
this.telegram = {
|
||||||
|
token,
|
||||||
|
sendDocument: mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
message_id: 1,
|
||||||
|
document: { file_id: `file_${token}`, file_unique_id: `uniq_${token}` },
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
sendPhoto: mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
message_id: 1,
|
||||||
|
photo: [{ file_id: `photo_${token}`, file_unique_id: `photo_uniq_${token}` }],
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
getFile: mock(() =>
|
||||||
|
Promise.resolve({ file_size: 100, mime_type: 'text/plain', file_path: 'path' }),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
mockTelegramInstances[token] = this.telegram;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
mock.module('telegraf', () => ({
|
||||||
|
Telegraf: MockTelegraf,
|
||||||
|
}));
|
||||||
|
|
||||||
|
describe('BotPool', () => {
|
||||||
|
let BotPool: typeof import('../src/infrastructure/telegram/bot-pool').BotPool;
|
||||||
|
let botPool: import('../src/infrastructure/telegram/bot-pool').BotPool;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
mockAdd.mockClear();
|
||||||
|
queueInstances.length = 0;
|
||||||
|
for (const token of Object.keys(mockTelegramInstances)) {
|
||||||
|
const tg = mockTelegramInstances[token];
|
||||||
|
if (tg) {
|
||||||
|
tg.sendDocument?.mockClear();
|
||||||
|
tg.getFile?.mockClear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const mod = await import('../src/infrastructure/telegram/bot-pool');
|
||||||
|
BotPool = mod.BotPool;
|
||||||
|
botPool = new BotPool();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
// No module cache cleanup needed — Bun handles import caching correctly
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should have correct bot count', () => {
|
||||||
|
expect(botPool.size).toBe(3);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should have correct effective concurrency', () => {
|
||||||
|
// 3 bots * 1 concurrency per bot
|
||||||
|
expect(botPool.getEffectiveConcurrency()).toBe(3);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should forward files through the queue', async () => {
|
||||||
|
const result = await botPool.forwardToStorage(Buffer.from('test data'), 'test.txt', 'document');
|
||||||
|
expect(result.telegramFileId).toBeDefined();
|
||||||
|
expect(result.storageMessageId).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should use per-bot queues with concurrency=1', () => {
|
||||||
|
// Each bot gets its own PQueue instance with concurrency=1
|
||||||
|
expect(queueInstances.length).toBe(3);
|
||||||
|
for (const qi of queueInstances) {
|
||||||
|
expect(qi.concurrency).toBe(1);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
+2
-2
@@ -1,11 +1,11 @@
|
|||||||
import { afterAll, beforeEach, describe, expect, it, mock, spyOn } from 'bun:test';
|
import { afterAll, beforeEach, describe, expect, it, mock, spyOn } from 'bun:test';
|
||||||
|
import logger from '../src/shared/logger/index';
|
||||||
import type { TelegramMediaMessage } from '../src/shared/utils/file';
|
import type { TelegramMediaMessage } from '../src/shared/utils/file';
|
||||||
import logger from '../src/utils/logger';
|
|
||||||
|
|
||||||
// Mock environment
|
// Mock environment
|
||||||
process.env.BOT_TOKEN = process.env.BOT_TOKEN || '123456789:ABCdefGhIJKlmNoPQRsTUVwxyZ';
|
process.env.BOT_TOKEN = process.env.BOT_TOKEN || '123456789:ABCdefGhIJKlmNoPQRsTUVwxyZ';
|
||||||
process.env.STORAGE_CHANNEL_ID = process.env.STORAGE_CHANNEL_ID || '-1001234567890';
|
process.env.STORAGE_CHANNEL_ID = process.env.STORAGE_CHANNEL_ID || '-1001234567890';
|
||||||
process.env.BASE_URL = process.env.BASE_URL || 'https://tele.asepharyana.my.id';
|
process.env.BASE_URL = process.env.BASE_URL || 'https://upload.asepharyana.my.id';
|
||||||
|
|
||||||
type BotTestContext = {
|
type BotTestContext = {
|
||||||
message: TelegramMediaMessage;
|
message: TelegramMediaMessage;
|
||||||
|
|||||||
+2
-2
@@ -1,6 +1,6 @@
|
|||||||
import { describe, expect, it } from 'bun:test';
|
import { describe, expect, it } from 'bun:test';
|
||||||
import { db, files } from '../src/db/index';
|
import { db, files } from '../src/infrastructure/persistence/drizzle/index';
|
||||||
import { files as schemaFiles } from '../src/db/schema';
|
import { files as schemaFiles } from '../src/infrastructure/persistence/drizzle/schema';
|
||||||
|
|
||||||
describe('Database Layer', () => {
|
describe('Database Layer', () => {
|
||||||
it('should export db instance', () => {
|
it('should export db instance', () => {
|
||||||
|
|||||||
+84
-8
@@ -1,9 +1,10 @@
|
|||||||
import { describe, expect, it } from 'bun:test';
|
import { describe, expect, it } from 'bun:test';
|
||||||
import { config } from '../src/env';
|
import { config } from '../src/env';
|
||||||
|
import { asSafeChunkSize, TELEGRAM_CHUNK_SIZE_MAX_BYTES } from '../src/shared/utils/validation';
|
||||||
|
|
||||||
describe('Environment Variables Validation', () => {
|
describe('Environment Variables Validation', () => {
|
||||||
it('config should have all required fields', () => {
|
it('config should have all required fields', () => {
|
||||||
expect(config).toHaveProperty('botToken');
|
expect(config).toHaveProperty('botTokens');
|
||||||
expect(config).toHaveProperty('storageChatId');
|
expect(config).toHaveProperty('storageChatId');
|
||||||
expect(config).toHaveProperty('baseUrl');
|
expect(config).toHaveProperty('baseUrl');
|
||||||
expect(config).toHaveProperty('databaseUrl');
|
expect(config).toHaveProperty('databaseUrl');
|
||||||
@@ -17,8 +18,9 @@ describe('Environment Variables Validation', () => {
|
|||||||
expect(config).toHaveProperty('sessionMaxAgeMs');
|
expect(config).toHaveProperty('sessionMaxAgeMs');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('config.botToken should return BOT_TOKEN from process.env', () => {
|
it('config.botTokens should return array from BOT_TOKENS env', () => {
|
||||||
expect(config.botToken).toBe(process.env.BOT_TOKEN || '');
|
expect(Array.isArray(config.botTokens)).toBe(true);
|
||||||
|
expect(config.botTokens.length).toBeGreaterThanOrEqual(3);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('config.storageChatId should be parsed as integer from STORAGE_CHANNEL_ID', () => {
|
it('config.storageChatId should be parsed as integer from STORAGE_CHANNEL_ID', () => {
|
||||||
@@ -26,7 +28,7 @@ describe('Environment Variables Validation', () => {
|
|||||||
expect(config.storageChatId).toBe(parseInt(process.env.STORAGE_CHANNEL_ID || '0', 10));
|
expect(config.storageChatId).toBe(parseInt(process.env.STORAGE_CHANNEL_ID || '0', 10));
|
||||||
});
|
});
|
||||||
|
|
||||||
it('config.port should default to 3000 when not specified', () => {
|
it('config.port should default to 4000 when not specified', () => {
|
||||||
expect(typeof config.port).toBe('number');
|
expect(typeof config.port).toBe('number');
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -52,10 +54,10 @@ describe('Environment Variables Validation', () => {
|
|||||||
expect(config.sessionMaxAgeMs).toBe(86400 * 1000);
|
expect(config.sessionMaxAgeMs).toBe(86400 * 1000);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('additionalBotTokens should be populated in test environment', () => {
|
it('botTokens should be populated in test environment', () => {
|
||||||
expect(Array.isArray(config.additionalBotTokens)).toBe(true);
|
expect(Array.isArray(config.botTokens)).toBe(true);
|
||||||
// With mock tokens from setup-env.ts there should be 2 additional tokens
|
// With mock tokens from setup-env.ts there should be at least 3 tokens
|
||||||
expect(config.additionalBotTokens.length).toBeGreaterThanOrEqual(2);
|
expect(config.botTokens.length).toBeGreaterThanOrEqual(3);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('S3 validation should not throw — env already loaded without error at import time', () => {
|
it('S3 validation should not throw — env already loaded without error at import time', () => {
|
||||||
@@ -66,3 +68,77 @@ describe('Environment Variables Validation', () => {
|
|||||||
expect(config.s3SecretKey).toBeDefined();
|
expect(config.s3SecretKey).toBeDefined();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('Telegram chunk size validation', () => {
|
||||||
|
it('config.telegramChunkSizeBytes should default to the safe 19 MB value when unset', () => {
|
||||||
|
// setup-env.ts does not set TELEGRAM_CHUNK_SIZE_BYTES, so the default must
|
||||||
|
// be the safe 19 MB value — never the raw 20 MB getFile limit.
|
||||||
|
expect(config.telegramChunkSizeBytes).toBe(TELEGRAM_CHUNK_SIZE_MAX_BYTES);
|
||||||
|
expect(config.telegramChunkSizeBytes).toBeLessThan(20 * 1024 * 1024);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('asSafeChunkSize accepts sizes at or below the maximum (19 MB)', () => {
|
||||||
|
expect(asSafeChunkSize(TELEGRAM_CHUNK_SIZE_MAX_BYTES)).toBe(TELEGRAM_CHUNK_SIZE_MAX_BYTES);
|
||||||
|
expect(asSafeChunkSize(1024)).toBe(1024);
|
||||||
|
expect(asSafeChunkSize(19 * 1024 * 1024)).toBe(19 * 1024 * 1024);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('asSafeChunkSize rejects sizes above the Telegram getFile limit (incl. the 48 MB production bug)', () => {
|
||||||
|
// The production bug value (48 MB / 50331648) must be rejected.
|
||||||
|
expect(() => asSafeChunkSize(48 * 1024 * 1024)).toThrow(/exceeds/);
|
||||||
|
// Even exactly 20 MB is at the raw Telegram limit — rejected by the margin.
|
||||||
|
expect(() => asSafeChunkSize(20 * 1024 * 1024)).toThrow(/exceeds/);
|
||||||
|
expect(() => asSafeChunkSize(TELEGRAM_CHUNK_SIZE_MAX_BYTES + 1)).toThrow(/exceeds/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('asSafeChunkSize rejects non-positive or non-integer sizes', () => {
|
||||||
|
expect(() => asSafeChunkSize(0)).toThrow('Invalid Telegram chunk size');
|
||||||
|
expect(() => asSafeChunkSize(-1)).toThrow('Invalid Telegram chunk size');
|
||||||
|
expect(() => asSafeChunkSize(1.5)).toThrow('Invalid Telegram chunk size');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('startup fails fast when TELEGRAM_CHUNK_SIZE_BYTES exceeds the limit', async () => {
|
||||||
|
// Spawn a real process that imports src/env with an oversized chunk size;
|
||||||
|
// it must exit non-zero with a clear error instead of starting silently.
|
||||||
|
const proc = Bun.spawn({
|
||||||
|
cmd: ['bun', '-e', "import('./src/env')"],
|
||||||
|
cwd: `${import.meta.dir}/..`,
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
BOT_TOKENS: '123456:ABC-DEF',
|
||||||
|
STORAGE_CHANNEL_ID: '-1001234567890',
|
||||||
|
BASE_URL: 'https://example.com',
|
||||||
|
DATABASE_URL: 'postgresql://asephs:***@100.121.180.82:6432/test',
|
||||||
|
PORT: '4000',
|
||||||
|
TELEGRAM_CHUNK_SIZE_BYTES: String(48 * 1024 * 1024),
|
||||||
|
},
|
||||||
|
stdout: 'pipe',
|
||||||
|
stderr: 'pipe',
|
||||||
|
});
|
||||||
|
const exitCode = await proc.exited;
|
||||||
|
const stderr = await new Response(proc.stderr).text();
|
||||||
|
expect(exitCode).not.toBe(0);
|
||||||
|
expect(stderr).toContain('TELEGRAM_CHUNK_SIZE_BYTES');
|
||||||
|
expect(stderr).toContain('exceeds');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('startup succeeds when TELEGRAM_CHUNK_SIZE_BYTES is at the safe limit', async () => {
|
||||||
|
const proc = Bun.spawn({
|
||||||
|
cmd: ['bun', '-e', "import('./src/env')"],
|
||||||
|
cwd: `${import.meta.dir}/..`,
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
BOT_TOKENS: '123456:ABC-DEF',
|
||||||
|
STORAGE_CHANNEL_ID: '-1001234567890',
|
||||||
|
BASE_URL: 'https://example.com',
|
||||||
|
DATABASE_URL: 'postgresql://asephs:***@100.121.180.82:6432/test',
|
||||||
|
PORT: '4000',
|
||||||
|
TELEGRAM_CHUNK_SIZE_BYTES: String(TELEGRAM_CHUNK_SIZE_MAX_BYTES),
|
||||||
|
},
|
||||||
|
stdout: 'pipe',
|
||||||
|
stderr: 'pipe',
|
||||||
|
});
|
||||||
|
const exitCode = await proc.exited;
|
||||||
|
expect(exitCode).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
+2
-2
@@ -5,7 +5,7 @@ import {
|
|||||||
extractFileName,
|
extractFileName,
|
||||||
extractMimeType,
|
extractMimeType,
|
||||||
getFileType,
|
getFileType,
|
||||||
} from '../src/utils/file';
|
} from '../src/shared/utils/file';
|
||||||
|
|
||||||
describe('File Utilities', () => {
|
describe('File Utilities', () => {
|
||||||
describe('getFileType', () => {
|
describe('getFileType', () => {
|
||||||
@@ -21,7 +21,7 @@ describe('File Utilities', () => {
|
|||||||
|
|
||||||
it('should classify image mime types based on caption', () => {
|
it('should classify image mime types based on caption', () => {
|
||||||
expect(getFileType('image/jpeg', 'my photo')).toBe('photo');
|
expect(getFileType('image/jpeg', 'my photo')).toBe('photo');
|
||||||
expect(getFileType('image/png', 'cool image.png')).toBe('photo');
|
expect(getFileType('image/png', 'cool image.png')).toBe('document');
|
||||||
expect(getFileType('image/gif', 'funny.gif')).toBe('animation');
|
expect(getFileType('image/gif', 'funny.gif')).toBe('animation');
|
||||||
expect(getFileType('image/png', 'funny gif')).toBe('animation');
|
expect(getFileType('image/png', 'funny gif')).toBe('animation');
|
||||||
});
|
});
|
||||||
|
|||||||
+143
-82
@@ -21,14 +21,19 @@ type FileInfoBody = {
|
|||||||
|
|
||||||
type JsonBody = ErrorBody | FileInfoBody | Record<string, unknown>;
|
type JsonBody = ErrorBody | FileInfoBody | Record<string, unknown>;
|
||||||
|
|
||||||
type MockFileRecord = Record<string, unknown>;
|
type MockFileRecord = {
|
||||||
|
publicId: string;
|
||||||
type MockSelectChain = {
|
fileName: string;
|
||||||
from: () => {
|
mimeType: string;
|
||||||
where: () => {
|
sizeBytes: number;
|
||||||
limit: () => Promise<MockFileRecord[]>;
|
fileType: string;
|
||||||
};
|
uploaderId?: number;
|
||||||
};
|
createdAt?: Date;
|
||||||
|
telegramFileId?: string;
|
||||||
|
storageBackend?: string | null;
|
||||||
|
archiveEntryName?: string | null;
|
||||||
|
fileHash?: string | null;
|
||||||
|
archiveTelegramFileId?: string | null;
|
||||||
};
|
};
|
||||||
|
|
||||||
const requestWithPublicId = (url: string, publicId: string): RequestWithParams => {
|
const requestWithPublicId = (url: string, publicId: string): RequestWithParams => {
|
||||||
@@ -41,25 +46,11 @@ const responseJson = async <T extends JsonBody>(res: Response): Promise<T> => {
|
|||||||
return (await res.json()) as T;
|
return (await res.json()) as T;
|
||||||
};
|
};
|
||||||
|
|
||||||
// Mock database layer
|
// Mock the DI module — file-controller imports fileRepository + chunkedStorage from here
|
||||||
const emptySelectChain = (): MockSelectChain => ({
|
const mockFindByPublicId = mock(
|
||||||
from: () => ({
|
(_publicId: string): Promise<MockFileRecord | null> => Promise.resolve(null),
|
||||||
where: () => ({
|
);
|
||||||
limit: () => Promise.resolve([]),
|
|
||||||
}),
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
|
|
||||||
const mockSelect = mock(() => emptySelectChain());
|
|
||||||
|
|
||||||
mock.module('../src/db/files', () => ({
|
|
||||||
findFileByPublicId: async () => {
|
|
||||||
const chain = mockSelect();
|
|
||||||
return (await chain.from().where().limit())[0] || null;
|
|
||||||
},
|
|
||||||
}));
|
|
||||||
|
|
||||||
// Mock telegram utils
|
|
||||||
const mockGetFileInfo = mock(async (_telegramFileId: string) => ({
|
const mockGetFileInfo = mock(async (_telegramFileId: string) => ({
|
||||||
file_size: 98765,
|
file_size: 98765,
|
||||||
mime_type: 'image/jpeg',
|
mime_type: 'image/jpeg',
|
||||||
@@ -67,27 +58,87 @@ const mockGetFileInfo = mock(async (_telegramFileId: string) => ({
|
|||||||
bot_token: '123456:ABC-DEF',
|
bot_token: '123456:ABC-DEF',
|
||||||
}));
|
}));
|
||||||
|
|
||||||
mock.module('../src/utils/telegram', () => ({
|
const mockCreateChunkedObjectResponse = mock(async () => new Response(null, { status: 200 }));
|
||||||
forwardToStorage: async () => ({
|
|
||||||
telegramFileId: 'mock-tg-id',
|
mock.module('../src/infrastructure/di', () => ({
|
||||||
telegramFileUniqueId: 'mock-tg-unique',
|
fileRepository: {
|
||||||
storageMessageId: 12345,
|
findByPublicId: mockFindByPublicId,
|
||||||
}),
|
findByHash: async () => null,
|
||||||
getFileInfo: mockGetFileInfo,
|
findByUniqueId: async () => null,
|
||||||
|
findByBucketAndKey: async () => null,
|
||||||
|
create: async (data: Record<string, unknown>) => ({
|
||||||
|
...data,
|
||||||
|
id: 'mock-id',
|
||||||
|
createdAt: new Date(),
|
||||||
|
}),
|
||||||
|
softDelete: async () => true,
|
||||||
|
softDeleteBatch: async () => 1,
|
||||||
|
countByBucket: async () => 0,
|
||||||
|
listByPrefix: async () => ({ objects: [], prefixes: [] }),
|
||||||
|
findOrphansByBucket: async () => [],
|
||||||
|
},
|
||||||
|
chunkedStorage: {
|
||||||
|
createChunkedObjectResponse: mockCreateChunkedObjectResponse,
|
||||||
|
buildChunkedObjectSources: async () => [],
|
||||||
|
uploadFileInTelegramChunks: async () => ({ parts: [], fileHash: '', totalSizeBytes: 0 }),
|
||||||
|
storeFileInTelegramChunks: async () => ({
|
||||||
|
id: 'mock-id',
|
||||||
|
publicId: 'mock-public',
|
||||||
|
telegramFileId: 'mock-tg',
|
||||||
|
telegramFileUniqueId: 'mock-tg-unique',
|
||||||
|
storageChatId: 0,
|
||||||
|
storageMessageId: 0,
|
||||||
|
fileName: 'mock',
|
||||||
|
mimeType: 'application/octet-stream',
|
||||||
|
sizeBytes: 0,
|
||||||
|
fileType: 'document',
|
||||||
|
uploaderId: 0,
|
||||||
|
fileHash: null,
|
||||||
|
archiveTelegramFileId: null,
|
||||||
|
archiveStorageMessageId: null,
|
||||||
|
archiveFileName: null,
|
||||||
|
archiveEntryName: null,
|
||||||
|
archiveMimeType: null,
|
||||||
|
archiveSizeBytes: null,
|
||||||
|
bucketId: null,
|
||||||
|
s3Key: null,
|
||||||
|
storageBackend: 'telegram',
|
||||||
|
isDeleted: false,
|
||||||
|
multipartUploadId: null,
|
||||||
|
partCount: null,
|
||||||
|
createdAt: new Date(),
|
||||||
|
updatedAt: new Date(),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Mock botPool.getFileInfo used in file redirect
|
||||||
|
mock.module('../src/infrastructure/telegram/bot-pool', () => ({
|
||||||
|
botPool: {
|
||||||
|
getFileInfo: mockGetFileInfo,
|
||||||
|
forwardToStorage: async () => ({
|
||||||
|
telegramFileId: 'mock-tg-id',
|
||||||
|
telegramFileUniqueId: 'mock-tg-unique',
|
||||||
|
storageMessageId: 12345,
|
||||||
|
}),
|
||||||
|
size: 1,
|
||||||
|
getEffectiveConcurrency: () => 1,
|
||||||
|
},
|
||||||
}));
|
}));
|
||||||
|
|
||||||
describe('File Route Handlers', () => {
|
describe('File Route Handlers', () => {
|
||||||
let handleFileRedirect: typeof import('../src/routes/files').handleFileRedirect;
|
let handleFileRedirect: typeof import('../src/interfaces/http/controllers/file-controller').handleFileRedirect;
|
||||||
let handleFileInfo: typeof import('../src/routes/files').handleFileInfo;
|
let handleFileInfo: typeof import('../src/interfaces/http/controllers/file-controller').handleFileInfo;
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
mockSelect.mockClear();
|
mockFindByPublicId.mockClear();
|
||||||
mockGetFileInfo.mockClear();
|
mockGetFileInfo.mockClear();
|
||||||
|
mockCreateChunkedObjectResponse.mockClear();
|
||||||
|
|
||||||
// Set up mock token
|
// Set up mock token
|
||||||
process.env.BOT_TOKEN = '123456:ABC-DEF';
|
process.env.BOT_TOKEN = '123456:ABC-DEF';
|
||||||
|
|
||||||
const filesRoute = await import('../src/routes/files');
|
const filesRoute = await import('../src/interfaces/http/controllers/file-controller');
|
||||||
handleFileRedirect = filesRoute.handleFileRedirect;
|
handleFileRedirect = filesRoute.handleFileRedirect;
|
||||||
handleFileInfo = filesRoute.handleFileInfo;
|
handleFileInfo = filesRoute.handleFileInfo;
|
||||||
});
|
});
|
||||||
@@ -98,15 +149,9 @@ describe('File Route Handlers', () => {
|
|||||||
|
|
||||||
describe('handleFileRedirect', () => {
|
describe('handleFileRedirect', () => {
|
||||||
it('should return 404 if file is not found in database', async () => {
|
it('should return 404 if file is not found in database', async () => {
|
||||||
mockSelect.mockImplementationOnce(() => ({
|
mockFindByPublicId.mockImplementationOnce(async () => null);
|
||||||
from: () => ({
|
|
||||||
where: () => ({
|
|
||||||
limit: () => Promise.resolve([]),
|
|
||||||
}),
|
|
||||||
}),
|
|
||||||
}));
|
|
||||||
|
|
||||||
const req = requestWithPublicId('http://localhost:3000/f/missing-id', 'missing-id');
|
const req = requestWithPublicId('http://localhost:4000/f/missing-id', 'missing-id');
|
||||||
const res = await handleFileRedirect(req);
|
const res = await handleFileRedirect(req);
|
||||||
expect(res.status).toBe(404);
|
expect(res.status).toBe(404);
|
||||||
const body = await responseJson<ErrorBody>(res);
|
const body = await responseJson<ErrorBody>(res);
|
||||||
@@ -114,25 +159,35 @@ describe('File Route Handlers', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should redirect to telegram file url with 302', async () => {
|
it('should redirect to telegram file url with 302', async () => {
|
||||||
mockSelect.mockImplementationOnce(() => ({
|
mockFindByPublicId.mockImplementationOnce(async () => ({
|
||||||
from: () => ({
|
publicId: 'test-id',
|
||||||
where: () => ({
|
telegramFileId: 'tg-file-id',
|
||||||
limit: () =>
|
telegramFileUniqueId: 'tg-unique',
|
||||||
Promise.resolve([
|
storageChatId: -100123,
|
||||||
{
|
storageMessageId: 42,
|
||||||
id: 'uuid-123',
|
fileName: 'test.jpg',
|
||||||
publicId: 'test-id',
|
mimeType: 'image/jpeg',
|
||||||
telegramFileId: 'tg-file-id',
|
sizeBytes: 100,
|
||||||
fileName: 'test.jpg',
|
fileType: 'photo',
|
||||||
mimeType: 'image/jpeg',
|
uploaderId: 0,
|
||||||
sizeBytes: 100,
|
fileHash: 'abc123',
|
||||||
},
|
archiveTelegramFileId: null,
|
||||||
]),
|
archiveStorageMessageId: null,
|
||||||
}),
|
archiveFileName: null,
|
||||||
}),
|
archiveEntryName: null,
|
||||||
|
archiveMimeType: null,
|
||||||
|
archiveSizeBytes: null,
|
||||||
|
bucketId: null,
|
||||||
|
s3Key: null,
|
||||||
|
storageBackend: 'telegram',
|
||||||
|
isDeleted: false,
|
||||||
|
multipartUploadId: null,
|
||||||
|
partCount: null,
|
||||||
|
createdAt: new Date('2026-05-18T00:00:00.000Z'),
|
||||||
|
updatedAt: new Date('2026-05-18T00:00:00.000Z'),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const req = requestWithPublicId('http://localhost:3000/f/test-id', 'test-id');
|
const req = requestWithPublicId('http://localhost:4000/f/test-id', 'test-id');
|
||||||
const res = await handleFileRedirect(req);
|
const res = await handleFileRedirect(req);
|
||||||
|
|
||||||
expect(res.status).toBe(302);
|
expect(res.status).toBe(302);
|
||||||
@@ -142,11 +197,11 @@ describe('File Route Handlers', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should return 500 on database or external errors', async () => {
|
it('should return 500 on database or external errors', async () => {
|
||||||
mockSelect.mockImplementationOnce(() => {
|
mockFindByPublicId.mockImplementationOnce(async () => {
|
||||||
throw new Error('DB Connection Error');
|
throw new Error('DB Connection Error');
|
||||||
});
|
});
|
||||||
|
|
||||||
const req = requestWithPublicId('http://localhost:3000/f/test-id', 'test-id');
|
const req = requestWithPublicId('http://localhost:4000/f/test-id', 'test-id');
|
||||||
const res = await handleFileRedirect(req);
|
const res = await handleFileRedirect(req);
|
||||||
expect(res.status).toBe(500);
|
expect(res.status).toBe(500);
|
||||||
const body = await responseJson<ErrorBody>(res);
|
const body = await responseJson<ErrorBody>(res);
|
||||||
@@ -156,15 +211,9 @@ describe('File Route Handlers', () => {
|
|||||||
|
|
||||||
describe('handleFileInfo', () => {
|
describe('handleFileInfo', () => {
|
||||||
it('should return 404 if file is not found in database', async () => {
|
it('should return 404 if file is not found in database', async () => {
|
||||||
mockSelect.mockImplementationOnce(() => ({
|
mockFindByPublicId.mockImplementationOnce(async () => null);
|
||||||
from: () => ({
|
|
||||||
where: () => ({
|
|
||||||
limit: () => Promise.resolve([]),
|
|
||||||
}),
|
|
||||||
}),
|
|
||||||
}));
|
|
||||||
|
|
||||||
const req = requestWithPublicId('http://localhost:3000/file/missing-id/info', 'missing-id');
|
const req = requestWithPublicId('http://localhost:4000/file/missing-id/info', 'missing-id');
|
||||||
const res = await handleFileInfo(req);
|
const res = await handleFileInfo(req);
|
||||||
expect(res.status).toBe(404);
|
expect(res.status).toBe(404);
|
||||||
const body = await responseJson<ErrorBody>(res);
|
const body = await responseJson<ErrorBody>(res);
|
||||||
@@ -174,23 +223,35 @@ describe('File Route Handlers', () => {
|
|||||||
it('should return file info JSON without internal fields', async () => {
|
it('should return file info JSON without internal fields', async () => {
|
||||||
const dbFile = {
|
const dbFile = {
|
||||||
publicId: 'test-id',
|
publicId: 'test-id',
|
||||||
|
telegramFileId: 'tg-file-id',
|
||||||
|
telegramFileUniqueId: 'tg-unique',
|
||||||
|
storageChatId: -100123,
|
||||||
|
storageMessageId: 42,
|
||||||
fileName: 'image.png',
|
fileName: 'image.png',
|
||||||
mimeType: 'image/png',
|
mimeType: 'image/png',
|
||||||
sizeBytes: 2048,
|
sizeBytes: 2048,
|
||||||
fileType: 'photo',
|
fileType: 'photo',
|
||||||
uploaderId: 99999,
|
uploaderId: 99999,
|
||||||
|
fileHash: null,
|
||||||
|
archiveTelegramFileId: null,
|
||||||
|
archiveStorageMessageId: null,
|
||||||
|
archiveFileName: null,
|
||||||
|
archiveEntryName: null,
|
||||||
|
archiveMimeType: null,
|
||||||
|
archiveSizeBytes: null,
|
||||||
|
bucketId: null,
|
||||||
|
s3Key: null,
|
||||||
|
storageBackend: 'telegram',
|
||||||
|
isDeleted: false,
|
||||||
|
multipartUploadId: null,
|
||||||
|
partCount: null,
|
||||||
createdAt: new Date('2026-05-18T00:00:00.000Z'),
|
createdAt: new Date('2026-05-18T00:00:00.000Z'),
|
||||||
|
updatedAt: new Date('2026-05-18T00:00:00.000Z'),
|
||||||
};
|
};
|
||||||
|
|
||||||
mockSelect.mockImplementationOnce(() => ({
|
mockFindByPublicId.mockImplementationOnce(async () => dbFile);
|
||||||
from: () => ({
|
|
||||||
where: () => ({
|
|
||||||
limit: () => Promise.resolve([dbFile]),
|
|
||||||
}),
|
|
||||||
}),
|
|
||||||
}));
|
|
||||||
|
|
||||||
const req = requestWithPublicId('http://localhost:3000/file/test-id/info', 'test-id');
|
const req = requestWithPublicId('http://localhost:4000/file/test-id/info', 'test-id');
|
||||||
const res = await handleFileInfo(req);
|
const res = await handleFileInfo(req);
|
||||||
expect(res.status).toBe(200);
|
expect(res.status).toBe(200);
|
||||||
const body = await responseJson<FileInfoBody>(res);
|
const body = await responseJson<FileInfoBody>(res);
|
||||||
@@ -208,11 +269,11 @@ describe('File Route Handlers', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should return 500 on database or external errors', async () => {
|
it('should return 500 on database or external errors', async () => {
|
||||||
mockSelect.mockImplementationOnce(() => {
|
mockFindByPublicId.mockImplementationOnce(async () => {
|
||||||
throw new Error('DB Connection Error');
|
throw new Error('DB Connection Error');
|
||||||
});
|
});
|
||||||
|
|
||||||
const req = requestWithPublicId('http://localhost:3000/file/test-id/info', 'test-id');
|
const req = requestWithPublicId('http://localhost:4000/file/test-id/info', 'test-id');
|
||||||
const res = await handleFileInfo(req);
|
const res = await handleFileInfo(req);
|
||||||
expect(res.status).toBe(500);
|
expect(res.status).toBe(500);
|
||||||
const body = await responseJson<ErrorBody>(res);
|
const body = await responseJson<ErrorBody>(res);
|
||||||
|
|||||||
+2
-2
@@ -19,7 +19,7 @@ describe('Health Route Handler', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should return status 200 and ok when DB is healthy', async () => {
|
it('should return status 200 and ok when DB is healthy', async () => {
|
||||||
const req = new Request('http://localhost:3000/health');
|
const req = new Request('http://localhost:4000/health');
|
||||||
const res = await handleHealth(req);
|
const res = await handleHealth(req);
|
||||||
|
|
||||||
expect(res.status).toBe(200);
|
expect(res.status).toBe(200);
|
||||||
@@ -30,7 +30,7 @@ describe('Health Route Handler', () => {
|
|||||||
|
|
||||||
it('should return status 500 and error details when DB health check fails', async () => {
|
it('should return status 500 and error details when DB health check fails', async () => {
|
||||||
mockExecute.mockImplementationOnce(() => Promise.reject(new Error('DB Connection Failed')));
|
mockExecute.mockImplementationOnce(() => Promise.reject(new Error('DB Connection Failed')));
|
||||||
const req = new Request('http://localhost:3000/health');
|
const req = new Request('http://localhost:4000/health');
|
||||||
const res = await handleHealth(req);
|
const res = await handleHealth(req);
|
||||||
|
|
||||||
expect(res.status).toBe(500);
|
expect(res.status).toBe(500);
|
||||||
|
|||||||
@@ -10,12 +10,17 @@
|
|||||||
* defaults in `src/env.ts` and are not touched.
|
* defaults in `src/env.ts` and are not touched.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
process.env.BOT_TOKEN ||= '123456:ABC-DEF';
|
process.env.BOT_TOKENS ||= '123456:ABC-DEF,789012:GHI-JKL,345678:MNO-PQR';
|
||||||
process.env.STORAGE_CHANNEL_ID ||= '-1001234567890';
|
process.env.STORAGE_CHANNEL_ID ||= '-1001234567890';
|
||||||
process.env.BASE_URL ||= 'https://example.com';
|
process.env.BASE_URL ||= 'https://example.com';
|
||||||
process.env.DATABASE_URL ||= 'postgresql://user:pass@localhost:5432/test';
|
process.env.DATABASE_URL ||= 'postgresql://asephs:***@100.121.180.82:6432/test';
|
||||||
process.env.PORT ||= '3000';
|
process.env.PORT ||= '4000';
|
||||||
process.env.NODE_ENV = 'test';
|
process.env.NODE_ENV = 'test';
|
||||||
|
|
||||||
// Add mock additional bot tokens so multi-bot rotation logic is tested too
|
// Pin the chunk size to the safe 19 MB value UNCONDITIONALLY. Bun auto-loads
|
||||||
process.env.ADDITIONAL_BOT_TOKENS ||= '789012:GHI-JKL,345678:MNO-PQR';
|
// the repo .env before preloads run, and a stale oversized value there would
|
||||||
|
// trip the fail-fast guard in src/env.ts and break every test file's import.
|
||||||
|
// Tests that need a different value set it explicitly in their own process.
|
||||||
|
process.env.TELEGRAM_CHUNK_SIZE_BYTES = String(19 * 1024 * 1024);
|
||||||
|
|
||||||
|
// Keep old env names for backward compat with tests that reference them directly
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user