Compare commits
57
Commits
2cf6aa3275
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d2a306677c | ||
|
|
fbe09ec08a | ||
|
|
9bba0d3c6c | ||
|
|
643c5add70 | ||
|
|
aa7f7058f2 | ||
|
|
55bf04cb65 | ||
|
|
8abd334746 | ||
|
|
aff7e53a2f | ||
|
|
51107e2ccf | ||
|
|
5417cde649 | ||
|
|
2fee3861f0 | ||
|
|
88e2a3faad | ||
|
|
75b6d0a527 | ||
|
|
0c2ee3b3cf | ||
|
|
12b6a133a7 | ||
|
|
d2e2b450f6 | ||
|
|
17d877e2c6 | ||
|
|
62021397e0 | ||
|
|
ad7747bfd4 | ||
|
|
66fb92e5af | ||
|
|
673c934f37 | ||
|
|
90d6c7dd6f | ||
|
|
2653e23283 | ||
|
|
91ec588a88 | ||
|
|
864d41d8fc | ||
|
|
c14be68ff8 | ||
|
|
811a68821d | ||
|
|
bf616f6790 | ||
|
|
24dfb1c6b1 | ||
|
|
b9a3fcd828 | ||
|
|
f00b944e56 | ||
|
|
73b0630cbd | ||
|
|
d1771720bd | ||
|
|
7dd2a4f0c7 | ||
|
|
fbc1f1822a | ||
|
|
c768b29e9d | ||
|
|
59535d66b8 | ||
|
|
319d2980c5 | ||
|
|
4acfa3d2e9 | ||
|
|
5bd3d286ce | ||
|
|
bfafcc0a20 | ||
|
|
c26ca806d0 | ||
|
|
bc3d1f2e3f | ||
|
|
f95022142d | ||
|
|
bd766792b1 | ||
|
|
d22f788e3e | ||
|
|
1ecab987a6 | ||
|
|
9a4853a484 | ||
|
|
ad917f6675 | ||
|
|
fab91ad69c | ||
|
|
3501d547c0 | ||
|
|
332853f398 | ||
|
|
4c216b1d9f | ||
|
|
3580001b8a | ||
|
|
6d3696d261 | ||
|
|
1484d5265d | ||
|
|
245ea169ad |
+7
-8
@@ -1,14 +1,13 @@
|
|||||||
BOT_TOKEN=isi_token_bot_telegram
|
BOT_TOKENS=isi_token_bot_1,isi_token_bot_2,isi_token_bot_3
|
||||||
ADDITIONAL_BOT_TOKENS=token_cadangan_1,token_cadangan_2,token_cadangan_3
|
|
||||||
STORAGE_CHANNEL_ID=-1001234567890
|
STORAGE_CHANNEL_ID=-1001234567890
|
||||||
BASE_URL=https://tele.asepharyana.my.id
|
BASE_URL=https://upload.asepharyana.my.id
|
||||||
DATABASE_URL=postgresql://user:password@localhost:5432/telegram_uploader
|
DATABASE_URL=postgresql://asephs:***@100.121.180.82:6432/uploader
|
||||||
PORT=3000
|
PORT=4000
|
||||||
NODE_ENV=production
|
NODE_ENV=production
|
||||||
LOG_LEVEL=info
|
LOG_LEVEL=info
|
||||||
RATE_LIMIT_WINDOW_MS=60000
|
RATE_LIMIT_WINDOW_MS=60000
|
||||||
RATE_LIMIT_MAX_REQUESTS=30
|
RATE_LIMIT_MAX_REQUESTS=30
|
||||||
# TRUST_PROXY=true # Uncomment when behind reverse proxy (Traefik, Nginx) for correct client IP detection
|
# TRUST_PROXY=true # Uncomment when behind reverse proxy (Caddy, Nginx) for correct client IP detection
|
||||||
|
|
||||||
# S3-compatible API credentials
|
# S3-compatible API credentials
|
||||||
# S3_ACCESS_KEY=filedrop-admin
|
# S3_ACCESS_KEY=filedrop-admin
|
||||||
@@ -17,8 +16,8 @@ RATE_LIMIT_MAX_REQUESTS=30
|
|||||||
# S3_VHOST_DOMAINS=upload.asepharyana.my.id,upload.asepharyana.web.id
|
# S3_VHOST_DOMAINS=upload.asepharyana.my.id,upload.asepharyana.web.id
|
||||||
|
|
||||||
# Telegram-safe internal chunking for large stored files
|
# Telegram-safe internal chunking for large stored files
|
||||||
# Telegram max upload ~49MB, using 48MB for safety
|
# Telegram getFile download limit is 20 MB; guard rejects > 19922944 (19 MB)
|
||||||
# TELEGRAM_CHUNK_SIZE_BYTES=50331648
|
# TELEGRAM_CHUNK_SIZE_BYTES=19922944
|
||||||
# COMPRESS_CHUNKED_UPLOADS=true
|
# COMPRESS_CHUNKED_UPLOADS=true
|
||||||
# CHUNK_COMPRESSION_MIN_SIZE_BYTES=4096
|
# CHUNK_COMPRESSION_MIN_SIZE_BYTES=4096
|
||||||
|
|
||||||
|
|||||||
@@ -1,48 +0,0 @@
|
|||||||
name: Deploy FileDrop
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
deploy:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Check out repository
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Set up Bun
|
|
||||||
uses: oven-sh/setup-bun@v2
|
|
||||||
with:
|
|
||||||
bun-version: latest
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
run: bun install --frozen-lockfile
|
|
||||||
|
|
||||||
- name: Lint
|
|
||||||
run: bun run lint
|
|
||||||
|
|
||||||
- name: Build
|
|
||||||
run: bun run build
|
|
||||||
|
|
||||||
- name: Deploy to VPS
|
|
||||||
shell: bash
|
|
||||||
env:
|
|
||||||
VPS_HOST: ${{ secrets.VPS_HOST }}
|
|
||||||
VPS_USER: ${{ secrets.VPS_USER }}
|
|
||||||
VPS_SSH_KEY_VALUE: ${{ secrets.VPS_SSH_KEY }}
|
|
||||||
PRODUCTION_ENV: ${{ secrets.PRODUCTION_ENV }}
|
|
||||||
ADMIN_PASSWORD: ${{ secrets.ADMIN_PASSWORD }}
|
|
||||||
run: |
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
key_file="${RUNNER_TEMP:-/tmp}/filedrop_deploy_key"
|
|
||||||
printf '%s\n' "$VPS_SSH_KEY_VALUE" > "$key_file"
|
|
||||||
chmod 600 "$key_file"
|
|
||||||
|
|
||||||
printf '%s\n' "$PRODUCTION_ENV" > .env
|
|
||||||
chmod 600 .env
|
|
||||||
|
|
||||||
VPS_SSH_KEY="$key_file" ./deploy.sh --no-build
|
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
name: dependabot-auto-merge
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
- master
|
||||||
|
|
||||||
|
# Auto-merge Dependabot PRs with passing CI
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
pull-requests: write
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
dependabot:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
# Only for Dependabot PRs
|
||||||
|
if: github.actor == 'dependabot[bot]'
|
||||||
|
steps:
|
||||||
|
- name: Auto-merge
|
||||||
|
run: gh pr merge --auto --merge "$PR_NUMBER"
|
||||||
|
env:
|
||||||
|
PR_NUMBER: ${{ github.event.number }}
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
version: 2
|
||||||
|
updates:
|
||||||
|
- package-ecosystem: "npm"
|
||||||
|
directory: "/"
|
||||||
|
schedule:
|
||||||
|
interval: "daily"
|
||||||
|
day: "monday"
|
||||||
|
time: "05:00"
|
||||||
|
timezone: "Asia/Makassar"
|
||||||
|
open-pull-requests-limit: 10
|
||||||
|
assignees:
|
||||||
|
- "asepharyana"
|
||||||
|
commit-message:
|
||||||
|
prefix: "chore"
|
||||||
|
prefix-development: "chore"
|
||||||
|
include: "scope"
|
||||||
|
reviewers:
|
||||||
|
- "asepharyana"
|
||||||
|
labels:
|
||||||
|
- "dependencies"
|
||||||
|
groups:
|
||||||
|
production:
|
||||||
|
dependency-type: "production"
|
||||||
|
update-types:
|
||||||
|
- "minor"
|
||||||
|
- "patch"
|
||||||
|
development:
|
||||||
|
dependency-type: "development"
|
||||||
|
update-types:
|
||||||
|
- "minor"
|
||||||
|
- "patch"
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
name: Build & Deploy (Nix)
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: deploy
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
id-token: write
|
||||||
|
|
||||||
|
env:
|
||||||
|
VPS_HOST: ${{ secrets.VPS_HOST }}
|
||||||
|
VPS_USER: ${{ secrets.VPS_USER }}
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
lint:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
submodules: false
|
||||||
|
|
||||||
|
- uses: oven-sh/setup-bun@v2
|
||||||
|
with:
|
||||||
|
bun-version: latest
|
||||||
|
|
||||||
|
- name: Install deps
|
||||||
|
run: bun install --frozen-lockfile
|
||||||
|
|
||||||
|
- name: Lint (biome)
|
||||||
|
run: bunx biome check src test
|
||||||
|
|
||||||
|
build-and-deploy:
|
||||||
|
needs: lint
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
submodules: false
|
||||||
|
|
||||||
|
- name: Install Nix
|
||||||
|
uses: DeterminateSystems/nix-installer-action@v22
|
||||||
|
with:
|
||||||
|
determinate: false
|
||||||
|
extra-conf: |
|
||||||
|
sandbox = false
|
||||||
|
accept-flake-config = true
|
||||||
|
|
||||||
|
- name: Cache Nix
|
||||||
|
uses: DeterminateSystems/magic-nix-cache-action@v14
|
||||||
|
with:
|
||||||
|
use-flakehub: false
|
||||||
|
|
||||||
|
- name: Build teleuploader
|
||||||
|
id: build
|
||||||
|
run: |
|
||||||
|
nix build .#teleuploader --impure --option sandbox false --print-build-logs
|
||||||
|
STORE_PATH=$(readlink result)
|
||||||
|
echo "store-path=$STORE_PATH" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "Build OK: $STORE_PATH"
|
||||||
|
|
||||||
|
- name: Setup SSH key
|
||||||
|
env:
|
||||||
|
SSH_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||||
|
run: |
|
||||||
|
mkdir -p ~/.ssh
|
||||||
|
echo "$SSH_KEY" > ~/.ssh/id_ed25519
|
||||||
|
chmod 600 ~/.ssh/id_ed25519
|
||||||
|
sed -i 's/\r$//' ~/.ssh/id_ed25519
|
||||||
|
ssh-keygen -y -f ~/.ssh/id_ed25519 >/dev/null 2>&1 || { echo "SSH key invalid"; exit 1; }
|
||||||
|
ssh-keyscan -H "$VPS_HOST" >> ~/.ssh/known_hosts 2>/dev/null
|
||||||
|
|
||||||
|
- name: Deploy teleuploader to VPS
|
||||||
|
run: |
|
||||||
|
STORE_PATH="${{ steps.build.outputs.store-path }}"
|
||||||
|
echo "=== Copying teleuploader: $STORE_PATH ==="
|
||||||
|
nix copy --to "ssh://$VPS_USER@$VPS_HOST" "$STORE_PATH"
|
||||||
|
|
||||||
|
echo "=== Updating profile ==="
|
||||||
|
ssh "$VPS_USER@$VPS_HOST" "sudo /nix/var/nix/profiles/default/bin/nix-env --profile /nix/var/nix/profiles/teleuploader --set '$STORE_PATH'"
|
||||||
|
|
||||||
|
echo "=== Restarting service ==="
|
||||||
|
ssh "$VPS_USER@$VPS_HOST" "sudo systemctl daemon-reload && sudo systemctl restart teleuploader && sleep 3 && sudo systemctl is-active teleuploader"
|
||||||
|
echo "✅ teleuploader deployed"
|
||||||
|
|
||||||
|
cleanup:
|
||||||
|
# Bersihkan sampah Nix di VPS SETELAH deploy: hapus generasi profile lama
|
||||||
|
# + nix store gc. Profil yang sedang dipakai tidak disentuh.
|
||||||
|
needs: build-and-deploy
|
||||||
|
if: always()
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Nix GC on VPS
|
||||||
|
env:
|
||||||
|
VPS_HOST: ${{ secrets.VPS_HOST }}
|
||||||
|
VPS_USER: ${{ secrets.VPS_USER }}
|
||||||
|
SSH_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||||
|
run: |
|
||||||
|
mkdir -p ~/.ssh
|
||||||
|
echo "$SSH_KEY" > ~/.ssh/id_ed25519
|
||||||
|
chmod 600 ~/.ssh/id_ed25519
|
||||||
|
ssh-keyscan -H "$VPS_HOST" >> ~/.ssh/known_hosts 2>/dev/null
|
||||||
|
ssh "$VPS_USER@$VPS_HOST" "sudo /usr/local/bin/nix-gc-vps.sh" || echo "⚠️ Nix GC gagal (non-fatal)"
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
name: Publish to FlakeHub
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main, master]
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
flakehub-publish:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
id-token: write
|
||||||
|
contents: read
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
- uses: DeterminateSystems/determinate-nix-action@main
|
||||||
|
- uses: DeterminateSystems/flakehub-push@main
|
||||||
|
with:
|
||||||
|
visibility: public
|
||||||
|
rolling: true
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
name: Mirror to Gitea
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main, master]
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
mirror:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- name: Mirror to Gitea
|
||||||
|
env:
|
||||||
|
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||||
|
run: |
|
||||||
|
git remote add gitea "https://oauth2:${GITEA_TOKEN}@git.imrnes.team/MythEclipse/TeleUploader.git"
|
||||||
|
git push --mirror gitea
|
||||||
|
echo "✅ Mirrored to Gitea (MythEclipse/TeleUploader)"
|
||||||
@@ -36,3 +36,4 @@ S3_GUIDE.md
|
|||||||
|
|
||||||
# Finder (MacOS) folder config
|
# Finder (MacOS) folder config
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
result
|
||||||
|
|||||||
@@ -20,11 +20,17 @@ Default to using Bun instead of Node.js.
|
|||||||
- Bun.$`ls` instead of execa.
|
- Bun.$`ls` instead of execa.
|
||||||
- Rate limiter lokal dinonaktifkan (`checkRateLimit` di `src/utils/rateLimit.ts` selalu mengembalikan `true`).
|
- Rate limiter lokal dinonaktifkan (`checkRateLimit` di `src/utils/rateLimit.ts` selalu mengembalikan `true`).
|
||||||
- Telegram API memiliki auto-retry otomatis jika mengembalikan error 429 (Too Many Requests) menggunakan pool Telegraf multi-bot di `src/utils/telegram.ts`.
|
- Telegram API memiliki auto-retry otomatis jika mengembalikan error 429 (Too Many Requests) menggunakan pool Telegraf multi-bot di `src/utils/telegram.ts`.
|
||||||
- Multi-bot dikonfigurasi melalui `ADDITIONAL_BOT_TOKENS` (koma terpisah) di `.env` yang digabung dengan `BOT_TOKEN` utama (total 4 bot).
|
|- Multi-bot dikonfigurasi melalui `BOT_TOKENS` (koma terpisah) di `.env` — semua token bot digabung dalam satu variabel.
|
||||||
- Menggunakan mekanisme rotasi instan jika ada bot yang terkena rate limit 429 sebelum memutuskan untuk sleep.
|
- Menggunakan mekanisme rotasi instan jika ada bot yang terkena rate limit 429 sebelum memutuskan untuk sleep.
|
||||||
- Pengiriman berkas ke Telegram dieksekusi secara responsif dan paralel penuh tanpa batas konkurensi/antrian.
|
- Pengiriman berkas ke Telegram dieksekusi secara responsif dan paralel penuh tanpa batas konkurensi/antrian.
|
||||||
- Berkas API upload ditulis secara sementara ke disk `/tmp/teleuploader-*` dan di-stream ke Telegram menggunakan `fs.createReadStream` (RAM-optimized) lalu dihapus otomatis setelah 50ms (timeout aman).
|
- Berkas API upload ditulis secara sementara ke disk `/tmp/teleuploader-*` dan di-stream ke Telegram menggunakan `fs.createReadStream` (RAM-optimized) lalu dihapus otomatis setelah 50ms (timeout aman).
|
||||||
|
|
||||||
|
## Chunk size (TELEGRAM_CHUNK_SIZE_BYTES)
|
||||||
|
|
||||||
|
- Batas keras: Telegram Bot API `getFile` hanya bisa resolve file ≤ 20 MB — di atas itu error `Bad Request: file is too big` dan part tidak bisa di-download.
|
||||||
|
- Guard fail-fast di `src/env.ts`: service MENOLAK start (exit non-zero) jika `TELEGRAM_CHUNK_SIZE_BYTES` > 19922944 (19 MB, margin aman dari limit 20 MB). Konstanta: `TELEGRAM_CHUNK_SIZE_MAX_BYTES` di `src/shared/utils/validation.ts`, juga dipakai `asSafeChunkSize()` di runtime.
|
||||||
|
- Default 19 MB; berlaku untuk chunked storage DAN S3 multipart parts (sama-sama disimpan ke Telegram lalu di-resolve via getFile).
|
||||||
|
|
||||||
## Testing
|
## Testing
|
||||||
|
|
||||||
Use `bun test` to run tests. Jalankan tes secara spesifik (misal `bun test test/rateLimit.test.ts`) untuk menghindari polusi mock antar berkas tes ketika dijalankan bersamaan.
|
Use `bun test` to run tests. Jalankan tes secara spesifik (misal `bun test test/rateLimit.test.ts`) untuk menghindari polusi mock antar berkas tes ketika dijalankan bersamaan.
|
||||||
|
|||||||
+6
-2
@@ -1,3 +1,7 @@
|
|||||||
|
# ⚠️ LEGACY — pembangunan & deploy sekarang DISARANKAN memakai Nix + systemd
|
||||||
|
# (lihat flake.nix + .github/workflows/deploy.yml + Caddy reverse proxy di orangevps).
|
||||||
|
# Dockerfile ini hanya dipertahankan untuk konteks historis / fallback, bukan deploy produksi.
|
||||||
|
|
||||||
# Stage 1: Builder
|
# Stage 1: Builder
|
||||||
FROM oven/bun:alpine AS builder
|
FROM oven/bun:alpine AS builder
|
||||||
|
|
||||||
@@ -21,12 +25,12 @@ WORKDIR /usr/src/app
|
|||||||
# Copy built files, schema, and package.json
|
# Copy built files, schema, and package.json
|
||||||
COPY --from=builder /usr/src/app/dist/index.js ./dist/index.js
|
COPY --from=builder /usr/src/app/dist/index.js ./dist/index.js
|
||||||
COPY --from=builder /usr/src/app/dist/migrate.js ./dist/migrate.js
|
COPY --from=builder /usr/src/app/dist/migrate.js ./dist/migrate.js
|
||||||
COPY --from=builder /usr/src/app/src/home.html ./home.html
|
COPY --from=builder /usr/src/app/src/home.html ./dist/home.html
|
||||||
COPY schema.sql ./
|
COPY schema.sql ./
|
||||||
COPY package.json ./
|
COPY package.json ./
|
||||||
|
|
||||||
# Expose port
|
# Expose port
|
||||||
EXPOSE 3000
|
EXPOSE 4000
|
||||||
|
|
||||||
# Start server
|
# Start server
|
||||||
CMD ["bun", "dist/index.js"]
|
CMD ["bun", "dist/index.js"]
|
||||||
@@ -4,12 +4,11 @@ Backend production-ready untuk upload file ke Telegram yang tersimpan di private
|
|||||||
|
|
||||||
## Setup
|
## Setup
|
||||||
|
|
||||||
1. Install PostgreSQL database
|
1. Siapkan PostgreSQL database (produksi: database `uploader` via PgBouncer pool di `100.121.180.82:6432`)
|
||||||
2. Buat database: `createdb telegram_uploader`
|
2. Setup environment: `cp .env.example .env`
|
||||||
3. Setup environment: `cp .env.example .env`
|
3. Edit `.env` dengan nilai yang sesuai (lihat `DATABASE_URL`, `PORT=4000`)
|
||||||
4. Edit `.env` dengan nilai yang sesuai
|
4. Create table: `bun run db:migrate`
|
||||||
5. Create table: `bun run db:migrate`
|
5. Install dependencies: `bun install`
|
||||||
6. Install dependencies: `bun install`
|
|
||||||
|
|
||||||
## Telegram Private Channel Setup
|
## Telegram Private Channel Setup
|
||||||
|
|
||||||
@@ -24,6 +23,20 @@ bun run dev # Development mode
|
|||||||
bun run start # Production mode
|
bun run start # Production mode
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Deployment (Produksi — Nix + systemd)
|
||||||
|
|
||||||
|
> Infra lama berbasis Docker + Traefik sudah dihapus dari orangevps (2026-08-02).
|
||||||
|
|
||||||
|
- **Host**: orangevps
|
||||||
|
- **Service**: systemd unit `teleuploader` (env via `/etc/teleuploader/env` / BWS secrets)
|
||||||
|
- **Build**: Nix flake (`flake.nix`) — `nix build .#teleuploader` → `nix copy` → `systemctl restart teleuploader`
|
||||||
|
- **CI**: `.github/workflows/deploy.yml` (Gitea Actions / GitHub Actions)
|
||||||
|
- **Port**: `4000` (`PORT` env)
|
||||||
|
- **Domain**: `https://upload.asepharyana.my.id`
|
||||||
|
- **Reverse proxy**: Caddy (bukan Traefik/Docker)
|
||||||
|
- **Database**: `postgresql://asephs:***@100.121.180.82:6432/uploader` (PgBouncer pool di imrnes, **bukan** 5432/localhost)
|
||||||
|
- `deploy.sh` & `Dockerfile` & `docker-compose.yml` bersifat **legacy** — jangan dipakai untuk deploy produksi.
|
||||||
|
|
||||||
## API Endpoints
|
## API Endpoints
|
||||||
|
|
||||||
- `POST /api/upload` - Upload file
|
- `POST /api/upload` - Upload file
|
||||||
@@ -34,7 +47,7 @@ bun run start # Production mode
|
|||||||
## FAQ
|
## FAQ
|
||||||
|
|
||||||
**URL permanen maksudnya apa?**
|
**URL permanen maksudnya apa?**
|
||||||
URL backend tetap permanen: `https://tele.asepharyana.my.id/f/{public_id}`
|
URL backend tetap permanen: `https://upload.asepharyana.my.id/f/{public_id}`
|
||||||
Ini berarti URL service Anda fix, bukan jaminan file Telegram abadi.
|
Ini berarti URL service Anda fix, bukan jaminan file Telegram abadi.
|
||||||
|
|
||||||
## Testing
|
## Testing
|
||||||
|
|||||||
@@ -1,4 +1,9 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
# ⚠️ LEGACY — Script deploy lama berbasis Docker. Sejak 2026-08-02 Docker dihapus
|
||||||
|
# dari orangevps; produksi kini memakai Nix + systemd (lihat flake.nix, dan CI
|
||||||
|
# .github/workflows/deploy.yml yang menjalankan `nix build` → `nix copy` →
|
||||||
|
# `systemctl restart teleuploader`). Berkas ini hanya dipertahankan sebagai
|
||||||
|
# referensi historis — JANGAN dipakai untuk deploy produksi.
|
||||||
# ─── FileDrop Deploy Script ──────────────────────────────────────────────────
|
# ─── FileDrop Deploy Script ──────────────────────────────────────────────────
|
||||||
# Builds the Bun app locally and deploys to the VPS via Docker.
|
# Builds the Bun app locally and deploys to the VPS via Docker.
|
||||||
#
|
#
|
||||||
|
|||||||
+7
-4
@@ -1,19 +1,22 @@
|
|||||||
|
# ⚠️ LEGACY — Docker/Traefik sudah DIHAPUS dari VPS produksi (orangevps).
|
||||||
|
# Deploy sekarang Nix + systemd (flake.nix + .github/workflows/deploy.yml) dengan
|
||||||
|
# Caddy reverse proxy. Berkas ini hanya dipertahankan sebagai referensi historis.
|
||||||
services:
|
services:
|
||||||
app:
|
app:
|
||||||
build: .
|
build: .
|
||||||
container_name: filedrop-app
|
container_name: filedrop-app
|
||||||
restart: always
|
restart: always
|
||||||
environment:
|
environment:
|
||||||
|
- BOT_TOKENS=${BOT_TOKENS}
|
||||||
- BOT_TOKEN=${BOT_TOKEN}
|
- BOT_TOKEN=${BOT_TOKEN}
|
||||||
- ADDITIONAL_BOT_TOKENS=${ADDITIONAL_BOT_TOKENS:-}
|
- ADDITIONAL_BOT_TOKENS=${ADDITIONAL_BOT_TOKENS:-}
|
||||||
- STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID}
|
- STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID}
|
||||||
- BASE_URL=${BASE_URL}
|
- BASE_URL=${BASE_URL}
|
||||||
- DATABASE_URL=${DATABASE_URL}
|
- DATABASE_URL=${DATABASE_URL}
|
||||||
- PORT=3000
|
- PORT=4000
|
||||||
- NODE_ENV=production
|
- NODE_ENV=production
|
||||||
- LOG_LEVEL=info
|
- LOG_LEVEL=info
|
||||||
- TRUST_PROXY=true
|
- TRUST_PROXY=true
|
||||||
- UPLOAD_CONCURRENCY=${UPLOAD_CONCURRENCY:-8}
|
|
||||||
- BATCH_MAX_ITEMS=${BATCH_MAX_ITEMS:-20}
|
- BATCH_MAX_ITEMS=${BATCH_MAX_ITEMS:-20}
|
||||||
- BATCH_MAX_SIZE_BYTES=${BATCH_MAX_SIZE_BYTES:-524288000}
|
- BATCH_MAX_SIZE_BYTES=${BATCH_MAX_SIZE_BYTES:-524288000}
|
||||||
- MAX_REQUEST_BODY_BYTES=${MAX_REQUEST_BODY_BYTES:-2147483648}
|
- MAX_REQUEST_BODY_BYTES=${MAX_REQUEST_BODY_BYTES:-2147483648}
|
||||||
@@ -46,7 +49,7 @@ services:
|
|||||||
healthcheck:
|
healthcheck:
|
||||||
test:
|
test:
|
||||||
- CMD-SHELL
|
- CMD-SHELL
|
||||||
- "bun -e \"fetch('http://localhost:3000/health').then(r => r.status === 200 ? process.exit(0) : process.exit(1))\""
|
- "bun -e \"fetch('http://localhost:4000/health').then(r => r.status === 200 ? process.exit(0) : process.exit(1))\""
|
||||||
interval: 30s
|
interval: 30s
|
||||||
timeout: 10s
|
timeout: 10s
|
||||||
retries: 3
|
retries: 3
|
||||||
@@ -59,7 +62,7 @@ services:
|
|||||||
- "traefik.http.routers.filedrop.entrypoints=websecure"
|
- "traefik.http.routers.filedrop.entrypoints=websecure"
|
||||||
- "traefik.http.routers.filedrop.tls=true"
|
- "traefik.http.routers.filedrop.tls=true"
|
||||||
- "traefik.http.routers.filedrop.tls.certresolver=cloudflare"
|
- "traefik.http.routers.filedrop.tls.certresolver=cloudflare"
|
||||||
- "traefik.http.services.filedrop.loadbalancer.server.port=3000"
|
- "traefik.http.services.filedrop.loadbalancer.server.port=4000"
|
||||||
- "traefik.http.middlewares.filedrop-rl.ratelimit.average=300"
|
- "traefik.http.middlewares.filedrop-rl.ratelimit.average=300"
|
||||||
- "traefik.http.middlewares.filedrop-rl.ratelimit.burst=100"
|
- "traefik.http.middlewares.filedrop-rl.ratelimit.burst=100"
|
||||||
- "traefik.http.middlewares.filedrop-rl.ratelimit.period=1m"
|
- "traefik.http.middlewares.filedrop-rl.ratelimit.period=1m"
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# Telegram Bot Uploader Backend Implementation Plan
|
# Telegram Bot Uploader Backend Implementation Plan
|
||||||
|
|
||||||
|
> ⚠️ **LEGACY** — Dokumen historis (2026-05-17). Port & infrastruktur sudah berubah:
|
||||||
|
> produksi kini berjalan di port `4000` (Nix + systemd + Caddy, domain `upload.asepharyana.my.id`)
|
||||||
|
> dan database via PgBouncer pool `100.121.180.82:6432` (bukan port 5432, bukan localhost).
|
||||||
|
|
||||||
|
|
||||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
**Goal:** Production-ready backend for Telegram file uploader with dual upload methods (bot + HTTP API), PostgreSQL storage, and redirect-based downloads.
|
**Goal:** Production-ready backend for Telegram file uploader with dual upload methods (bot + HTTP API), PostgreSQL storage, and redirect-based downloads.
|
||||||
@@ -75,9 +80,9 @@ schema.sql
|
|||||||
```bash
|
```bash
|
||||||
BOT_TOKEN=isi_token_bot_telegram
|
BOT_TOKEN=isi_token_bot_telegram
|
||||||
STORAGE_CHANNEL_ID=-1001234567890
|
STORAGE_CHANNEL_ID=-1001234567890
|
||||||
BASE_URL=https://tele.asepharyana.my.id
|
BASE_URL=https://upload.asepharyana.my.id
|
||||||
DATABASE_URL=postgresql://user:password@localhost:5432/telegram_uploader
|
DATABASE_URL=postgresql://asephs:***@100.121.180.82:6432/uploader
|
||||||
PORT=3000
|
PORT=4000
|
||||||
NODE_ENV=production
|
NODE_ENV=production
|
||||||
LOG_LEVEL=info
|
LOG_LEVEL=info
|
||||||
RATE_LIMIT_WINDOW_MS=60000
|
RATE_LIMIT_WINDOW_MS=60000
|
||||||
@@ -148,7 +153,7 @@ bun run start # Production mode
|
|||||||
## FAQ
|
## FAQ
|
||||||
|
|
||||||
**URL permanen maksudnya apa?**
|
**URL permanen maksudnya apa?**
|
||||||
URL backend tetap permanen: `https://tele.asepharyana.my.id/f/{public_id}`
|
URL backend tetap permanen: `https://upload.asepharyana.my.id/f/{public_id}`
|
||||||
Ini berarti URL service Anda fix, bukan jaminan file Telegram abadi.
|
Ini berarti URL service Anda fix, bukan jaminan file Telegram abadi.
|
||||||
|
|
||||||
## Testing
|
## Testing
|
||||||
@@ -250,7 +255,7 @@ export const config = {
|
|||||||
storageChatId: parseInt(process.env.STORAGE_CHANNEL_ID, 10),
|
storageChatId: parseInt(process.env.STORAGE_CHANNEL_ID, 10),
|
||||||
baseUrl: process.env.BASE_URL,
|
baseUrl: process.env.BASE_URL,
|
||||||
databaseUrl: process.env.DATABASE_URL,
|
databaseUrl: process.env.DATABASE_URL,
|
||||||
port: parseInt(process.env.PORT, 10) || 3000,
|
port: parseInt(process.env.PORT, 10) || 4000,
|
||||||
nodeEnv: process.env.NODE_ENV || 'development',
|
nodeEnv: process.env.NODE_ENV || 'development',
|
||||||
logLevel: process.env.LOG_LEVEL || 'info',
|
logLevel: process.env.LOG_LEVEL || 'info',
|
||||||
rateLimitWindowMs: parseInt(process.env.RATE_LIMIT_WINDOW_MS, 10) || 60000,
|
rateLimitWindowMs: parseInt(process.env.RATE_LIMIT_WINDOW_MS, 10) || 60000,
|
||||||
@@ -1009,14 +1014,14 @@ bun run dev
|
|||||||
- [ ] **Step 2: Test health endpoint**
|
- [ ] **Step 2: Test health endpoint**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl http://localhost:3000/health
|
curl http://localhost:4000/health
|
||||||
# Expected: {"status":"ok"}
|
# Expected: {"status":"ok"}
|
||||||
```
|
```
|
||||||
|
|
||||||
- [ ] **Step 3: Upload test file via HTTP API (multipart)**
|
- [ ] **Step 3: Upload test file via HTTP API (multipart)**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -X POST http://localhost:3000/api/upload \
|
curl -X POST http://localhost:4000/api/upload \
|
||||||
-F "file=@/path/to/testfile.txt" \
|
-F "file=@/path/to/testfile.txt" \
|
||||||
-F "fileName=test.txt"
|
-F "fileName=test.txt"
|
||||||
```
|
```
|
||||||
@@ -1024,14 +1029,14 @@ curl -X POST http://localhost:3000/api/upload \
|
|||||||
- [ ] **Step 4: Check file info endpoint**
|
- [ ] **Step 4: Check file info endpoint**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl http://localhost:3000/file/{public_id}/info
|
curl http://localhost:4000/file/{public_id}/info
|
||||||
# Expected: JSON with file metadata
|
# Expected: JSON with file metadata
|
||||||
```
|
```
|
||||||
|
|
||||||
- [ ] **Step 5: Download redirect**
|
- [ ] **Step 5: Download redirect**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -I http://localhost:3000/f/{public_id}
|
curl -I http://localhost:4000/f/{public_id}
|
||||||
# Expected: HTTP 302 with Location header to Telegram CDN
|
# Expected: HTTP 302 with Location header to Telegram CDN
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -1047,7 +1052,7 @@ curl -I http://localhost:3000/f/{public_id}
|
|||||||
- [ ] **Step 7: Test error handling (file too large)**
|
- [ ] **Step 7: Test error handling (file too large)**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -X POST http://localhost:3000/api/upload \
|
curl -X POST http://localhost:4000/api/upload \
|
||||||
-F "file=@/dev/null" \
|
-F "file=@/dev/null" \
|
||||||
-H "Content-Length: 10000000000"
|
-H "Content-Length: 10000000000"
|
||||||
# Expected: HTTP 400 with error message
|
# Expected: HTTP 400 with error message
|
||||||
@@ -1057,7 +1062,7 @@ curl -X POST http://localhost:3000/api/upload \
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Send 31 requests within 1 minute
|
# Send 31 requests within 1 minute
|
||||||
for i in {1..31}; do curl http://localhost:3000/f/{public_id} & done
|
for i in {1..31}; do curl http://localhost:4000/f/{public_id} & done
|
||||||
wait
|
wait
|
||||||
# Expected: First 30 succeed, last one returns 429
|
# Expected: First 30 succeed, last one returns 429
|
||||||
```
|
```
|
||||||
@@ -1066,7 +1071,7 @@ wait
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# In terminal 1: bun run dev
|
# In terminal 1: bun run dev
|
||||||
# In terminal 2: curl http://localhost:3000/health && sleep 0.1 && curl http://localhost:3000/health
|
# In terminal 2: curl http://localhost:4000/health && sleep 0.1 && curl http://localhost:4000/health
|
||||||
# Send SIGINT to server (Ctrl+C in terminal 1)
|
# Send SIGINT to server (Ctrl+C in terminal 1)
|
||||||
# Check if server stops cleanly, logs show shutdown sequence
|
# Check if server stops cleanly, logs show shutdown sequence
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -1,5 +1,11 @@
|
|||||||
# Docker, Traefik, and GitHub Actions Deployment Plan
|
# Docker, Traefik, and GitHub Actions Deployment Plan
|
||||||
|
|
||||||
|
> ⚠️ **LEGACY** — Dokumen historis (2026-05-18). Arsitektur Docker + Traefik +
|
||||||
|
> GitHub Actions sudah digantikan (2026-08-02) oleh Nix + systemd + Caddy di
|
||||||
|
> orangevps: port `4000`, domain `upload.asepharyana.my.id`, database via
|
||||||
|
> PgBouncer pool `100.121.180.82:6432` (bukan 5432/localhost).
|
||||||
|
|
||||||
|
|
||||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
**Goal:** Containerize TeleUploader using Bun, configure Traefik labels for routing `upload.asepharyana.my.id`, and set up full GitHub Actions CI/CD to VPS `45.127.35.244`.
|
**Goal:** Containerize TeleUploader using Bun, configure Traefik labels for routing `upload.asepharyana.my.id`, and set up full GitHub Actions CI/CD to VPS `45.127.35.244`.
|
||||||
@@ -57,7 +63,7 @@ WORKDIR /usr/src/app
|
|||||||
|
|
||||||
# Set production environment variables
|
# Set production environment variables
|
||||||
ENV NODE_ENV=production
|
ENV NODE_ENV=production
|
||||||
ENV PORT=3000
|
ENV PORT=4000
|
||||||
|
|
||||||
# Copy necessary files from builder and repo
|
# Copy necessary files from builder and repo
|
||||||
COPY --from=builder /usr/src/app/dist/index.js ./dist/index.js
|
COPY --from=builder /usr/src/app/dist/index.js ./dist/index.js
|
||||||
@@ -65,7 +71,7 @@ COPY --from=builder /usr/src/app/package.json ./package.json
|
|||||||
COPY schema.sql ./schema.sql
|
COPY schema.sql ./schema.sql
|
||||||
|
|
||||||
# Expose server port
|
# Expose server port
|
||||||
EXPOSE 3000
|
EXPOSE 4000
|
||||||
|
|
||||||
# Start server
|
# Start server
|
||||||
CMD ["bun", "dist/index.js"]
|
CMD ["bun", "dist/index.js"]
|
||||||
@@ -104,7 +110,7 @@ services:
|
|||||||
- STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID}
|
- STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID}
|
||||||
- BASE_URL=${BASE_URL}
|
- BASE_URL=${BASE_URL}
|
||||||
- DATABASE_URL=${DATABASE_URL}
|
- DATABASE_URL=${DATABASE_URL}
|
||||||
- PORT=3000
|
- PORT=4000
|
||||||
- NODE_ENV=production
|
- NODE_ENV=production
|
||||||
- LOG_LEVEL=info
|
- LOG_LEVEL=info
|
||||||
networks:
|
networks:
|
||||||
@@ -115,7 +121,7 @@ services:
|
|||||||
- "traefik.http.routers.teleuploader.entrypoints=websecure"
|
- "traefik.http.routers.teleuploader.entrypoints=websecure"
|
||||||
- "traefik.http.routers.teleuploader.tls=true"
|
- "traefik.http.routers.teleuploader.tls=true"
|
||||||
- "traefik.http.routers.teleuploader.tls.certresolver=letsencrypt"
|
- "traefik.http.routers.teleuploader.tls.certresolver=letsencrypt"
|
||||||
- "traefik.http.services.teleuploader.loadbalancer.server.port=3000"
|
- "traefik.http.services.teleuploader.loadbalancer.server.port=4000"
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
app-shared-net:
|
app-shared-net:
|
||||||
@@ -166,9 +172,9 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
BOT_TOKEN: "mock_token"
|
BOT_TOKEN: "mock_token"
|
||||||
STORAGE_CHANNEL_ID: "123456"
|
STORAGE_CHANNEL_ID: "123456"
|
||||||
BASE_URL: "http://localhost:3000"
|
BASE_URL: "http://localhost:4000"
|
||||||
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/postgres"
|
DATABASE_URL: "postgresql://asephs:***@100.121.180.82:6432/postgres"
|
||||||
PORT: "3000"
|
PORT: "4000"
|
||||||
run: bun run test
|
run: bun run test
|
||||||
|
|
||||||
build-and-push:
|
build-and-push:
|
||||||
@@ -223,7 +229,7 @@ jobs:
|
|||||||
- STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID}
|
- STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID}
|
||||||
- BASE_URL=${BASE_URL}
|
- BASE_URL=${BASE_URL}
|
||||||
- DATABASE_URL=${DATABASE_URL}
|
- DATABASE_URL=${DATABASE_URL}
|
||||||
- PORT=3000
|
- PORT=4000
|
||||||
- NODE_ENV=production
|
- NODE_ENV=production
|
||||||
- LOG_LEVEL=info
|
- LOG_LEVEL=info
|
||||||
networks:
|
networks:
|
||||||
@@ -234,7 +240,7 @@ jobs:
|
|||||||
- "traefik.http.routers.teleuploader.entrypoints=websecure"
|
- "traefik.http.routers.teleuploader.entrypoints=websecure"
|
||||||
- "traefik.http.routers.teleuploader.tls=true"
|
- "traefik.http.routers.teleuploader.tls=true"
|
||||||
- "traefik.http.routers.teleuploader.tls.certresolver=letsencrypt"
|
- "traefik.http.routers.teleuploader.tls.certresolver=letsencrypt"
|
||||||
- "traefik.http.services.teleuploader.loadbalancer.server.port=3000"
|
- "traefik.http.services.teleuploader.loadbalancer.server.port=4000"
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
app-shared-net:
|
app-shared-net:
|
||||||
@@ -248,7 +254,7 @@ jobs:
|
|||||||
STORAGE_CHANNEL_ID=${{ secrets.STORAGE_CHANNEL_ID }}
|
STORAGE_CHANNEL_ID=${{ secrets.STORAGE_CHANNEL_ID }}
|
||||||
BASE_URL=${{ secrets.BASE_URL }}
|
BASE_URL=${{ secrets.BASE_URL }}
|
||||||
DATABASE_URL=${{ secrets.DATABASE_URL }}
|
DATABASE_URL=${{ secrets.DATABASE_URL }}
|
||||||
PORT=3000
|
PORT=4000
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
# Pull latest docker image
|
# Pull latest docker image
|
||||||
|
|||||||
@@ -1,5 +1,11 @@
|
|||||||
# S3-Compatible TeleUploader Implementation Plan
|
# S3-Compatible TeleUploader Implementation Plan
|
||||||
|
|
||||||
|
> ⚠️ **LEGACY** — Dokumen historis (2026-07-06). Port & infrastruktur sudah berubah:
|
||||||
|
> produksi kini berjalan di port `4000` (Nix + systemd + Caddy, domain
|
||||||
|
> `upload.asepharyana.my.id`), database via PgBouncer pool `100.121.180.82:6432`
|
||||||
|
> (bukan 5432/localhost). Contoh kode di bawah memakai `localhost:4000` untuk dev.
|
||||||
|
|
||||||
|
|
||||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
**Goal:** Transform TeleUploader into an S3-compatible storage server (Telegram-backed) with a web file manager UI.
|
**Goal:** Transform TeleUploader into an S3-compatible storage server (Telegram-backed) with a web file manager UI.
|
||||||
@@ -2675,7 +2681,7 @@ describe('S3 Bucket Operations', () => {
|
|||||||
process.env.S3_DEFAULT_REGION = 'us-east-1';
|
process.env.S3_DEFAULT_REGION = 'us-east-1';
|
||||||
process.env.BOT_TOKEN = '123456:ABC-DEF';
|
process.env.BOT_TOKEN = '123456:ABC-DEF';
|
||||||
process.env.STORAGE_CHANNEL_ID = '-1001234567890';
|
process.env.STORAGE_CHANNEL_ID = '-1001234567890';
|
||||||
process.env.BASE_URL = 'http://localhost:3000';
|
process.env.BASE_URL = 'http://localhost:4000';
|
||||||
process.env.DATABASE_URL = 'postgresql://localhost/test';
|
process.env.DATABASE_URL = 'postgresql://localhost/test';
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -2684,7 +2690,7 @@ describe('S3 Bucket Operations', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should return 403 for unauthorized requests', async () => {
|
it('should return 403 for unauthorized requests', async () => {
|
||||||
const req = new Request('http://localhost:3000/', {
|
const req = new Request('http://localhost:4000/', {
|
||||||
method: 'GET',
|
method: 'GET',
|
||||||
headers: { authorization: 'Invalid' },
|
headers: { authorization: 'Invalid' },
|
||||||
});
|
});
|
||||||
@@ -2849,7 +2855,7 @@ describe('Web API v1', () => {
|
|||||||
mockDbExecute.mockClear();
|
mockDbExecute.mockClear();
|
||||||
process.env.BOT_TOKEN = '123456:ABC-DEF';
|
process.env.BOT_TOKEN = '123456:ABC-DEF';
|
||||||
process.env.STORAGE_CHANNEL_ID = '-1001234567890';
|
process.env.STORAGE_CHANNEL_ID = '-1001234567890';
|
||||||
process.env.BASE_URL = 'http://localhost:3000';
|
process.env.BASE_URL = 'http://localhost:4000';
|
||||||
process.env.DATABASE_URL = 'postgresql://localhost/test';
|
process.env.DATABASE_URL = 'postgresql://localhost/test';
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -2858,7 +2864,7 @@ describe('Web API v1', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should list buckets via GET /api/v1/buckets', async () => {
|
it('should list buckets via GET /api/v1/buckets', async () => {
|
||||||
const req = new Request('http://localhost:3000/api/v1/buckets');
|
const req = new Request('http://localhost:4000/api/v1/buckets');
|
||||||
const res = await handleWebApiV1(req);
|
const res = await handleWebApiV1(req);
|
||||||
expect(res.status).toBe(200);
|
expect(res.status).toBe(200);
|
||||||
const data = await res.json();
|
const data = await res.json();
|
||||||
@@ -2867,7 +2873,7 @@ describe('Web API v1', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should return 404 for unknown API path', async () => {
|
it('should return 404 for unknown API path', async () => {
|
||||||
const req = new Request('http://localhost:3000/api/v1/unknown');
|
const req = new Request('http://localhost:4000/api/v1/unknown');
|
||||||
const res = await handleWebApiV1(req);
|
const res = await handleWebApiV1(req);
|
||||||
expect(res.status).toBe(404);
|
expect(res.status).toBe(404);
|
||||||
const data = await res.json();
|
const data = await res.json();
|
||||||
@@ -2875,7 +2881,7 @@ describe('Web API v1', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should return bucket object listing', async () => {
|
it('should return bucket object listing', async () => {
|
||||||
const req = new Request('http://localhost:3000/api/v1/buckets/test-bucket/objects?prefix=');
|
const req = new Request('http://localhost:4000/api/v1/buckets/test-bucket/objects?prefix=');
|
||||||
const res = await handleWebApiV1(req);
|
const res = await handleWebApiV1(req);
|
||||||
// Should return 200 even with empty results
|
// Should return 200 even with empty results
|
||||||
expect(res.status).toBe(200);
|
expect(res.status).toBe(200);
|
||||||
@@ -2885,7 +2891,7 @@ describe('Web API v1', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should reject invalid bucket name on create', async () => {
|
it('should reject invalid bucket name on create', async () => {
|
||||||
const req = new Request('http://localhost:3000/api/v1/buckets', {
|
const req = new Request('http://localhost:4000/api/v1/buckets', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'content-type': 'application/json' },
|
headers: { 'content-type': 'application/json' },
|
||||||
body: JSON.stringify({ name: 'INVALID_NAME!' }),
|
body: JSON.stringify({ name: 'INVALID_NAME!' }),
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
# Gitea CI/CD Migration Implementation Plan
|
# Gitea CI/CD Migration Implementation Plan
|
||||||
|
|
||||||
|
> Catatan (2026-08-02): Produksi sekarang port 4000, deploy Nix+systemd di orangevps, Caddy reverse proxy upload.asepharyana.my.id, DB via pgbouncer pool imrnes 100.121.180.82:6432. Docker/Traefik/Gitea-CI legacy.
|
||||||
|
|
||||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
**Goal:** Move `origin` from GitLab to `git.imrnes.team:MythEclipse/TeleUploader` and add Gitea Actions deployment on push to `main` using the existing VPS deploy path.
|
**Goal:** Move `origin` from GitLab to `git.imrnes.team:MythEclipse/TeleUploader` and add Gitea Actions deployment on push to `main` using the existing VPS deploy path.
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
# S3 Compatibility Completion Implementation Plan
|
# S3 Compatibility Completion Implementation Plan
|
||||||
|
|
||||||
|
> Catatan (2026-08-02): Produksi sekarang port 4000, deploy Nix+systemd di orangevps, Caddy reverse proxy upload.asepharyana.my.id, DB via pgbouncer pool imrnes 100.121.180.82:6432. Docker/Traefik/Gitea-CI legacy.
|
||||||
|
|
||||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
**Goal:** Finish TeleUploader S3 compatibility gaps: strict presigned GET, byte ranges, complete multipart GetObject streaming, strict AWS SDK multipart investigation/fix, and warning-free lint.
|
**Goal:** Finish TeleUploader S3 compatibility gaps: strict presigned GET, byte ranges, complete multipart GetObject streaming, strict AWS SDK multipart investigation/fix, and warning-free lint.
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# Telegram Bot Uploader Backend Design
|
# Telegram Bot Uploader Backend Design
|
||||||
|
|
||||||
|
> ⚠️ **LEGACY** — Dokumen historis (2026-05-17). Port & infrastruktur sudah berubah:
|
||||||
|
> produksi kini berjalan di port `4000` (Nix + systemd + Caddy, domain `upload.asepharyana.my.id`)
|
||||||
|
> dan database via PgBouncer pool `100.121.180.82:6432` (bukan port 5432, bukan localhost).
|
||||||
|
|
||||||
|
|
||||||
**Date:** 2026-05-17
|
**Date:** 2026-05-17
|
||||||
**Status:** Approved
|
**Status:** Approved
|
||||||
**Stack:** Bun, Telegraf, PostgreSQL, Drizzle ORM, Winston, nanoid
|
**Stack:** Bun, Telegraf, PostgreSQL, Drizzle ORM, Winston, nanoid
|
||||||
@@ -35,7 +40,7 @@ Production-ready backend untuk Telegram file uploader dengan dual upload methods
|
|||||||
4. Bot extracts `telegram_file_id`, `telegram_file_unique_id`, `storage_message_id`
|
4. Bot extracts `telegram_file_id`, `telegram_file_unique_id`, `storage_message_id`
|
||||||
5. Bot generates `public_id` using nanoid
|
5. Bot generates `public_id` using nanoid
|
||||||
6. Bot saves metadata to PostgreSQL
|
6. Bot saves metadata to PostgreSQL
|
||||||
7. Bot replies with download link: `https://tele.asepharyana.my.id/f/{public_id}`
|
7. Bot replies with download link: `https://upload.asepharyana.my.id/f/{public_id}`
|
||||||
|
|
||||||
#### Upload via HTTP API
|
#### Upload via HTTP API
|
||||||
1. Client POSTs to `/api/upload` with file (multipart or base64)
|
1. Client POSTs to `/api/upload` with file (multipart or base64)
|
||||||
@@ -115,7 +120,7 @@ fileName: optional_filename.ext
|
|||||||
"size_bytes": 1024000,
|
"size_bytes": 1024000,
|
||||||
"file_type": "document",
|
"file_type": "document",
|
||||||
"created_at": "2026-05-17T23:42:19Z",
|
"created_at": "2026-05-17T23:42:19Z",
|
||||||
"download_url": "https://tele.asepharyana.my.id/f/abc123xyz"
|
"download_url": "https://upload.asepharyana.my.id/f/abc123xyz"
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -212,9 +217,9 @@ schema.sql
|
|||||||
```
|
```
|
||||||
BOT_TOKEN=<telegram_bot_token>
|
BOT_TOKEN=<telegram_bot_token>
|
||||||
STORAGE_CHANNEL_ID=<private_channel_id>
|
STORAGE_CHANNEL_ID=<private_channel_id>
|
||||||
BASE_URL=https://tele.asepharyana.my.id
|
BASE_URL=https://upload.asepharyana.my.id
|
||||||
DATABASE_URL=postgresql://user:password@localhost:5432/telegram_uploader
|
DATABASE_URL=postgresql://asephs:***@100.121.180.82:6432/uploader
|
||||||
PORT=3000
|
PORT=4000
|
||||||
NODE_ENV=production
|
NODE_ENV=production
|
||||||
LOG_LEVEL=info
|
LOG_LEVEL=info
|
||||||
RATE_LIMIT_WINDOW_MS=60000
|
RATE_LIMIT_WINDOW_MS=60000
|
||||||
|
|||||||
@@ -7,6 +7,12 @@ metadata:
|
|||||||
|
|
||||||
# Design: TeleUploader Deployment & CI/CD Setup
|
# Design: TeleUploader Deployment & CI/CD Setup
|
||||||
|
|
||||||
|
> ⚠️ **LEGACY** — Dokumen historis (2026-05-18) untuk arsitektur Docker + Traefik +
|
||||||
|
> GitHub Actions. Docker & Traefik sudah dihapus dari VPS produksi (2026-08-02):
|
||||||
|
> deploy sekarang Nix + systemd + Caddy di orangevps, port `4000`, domain
|
||||||
|
> `upload.asepharyana.my.id`, database via PgBouncer pool `100.121.180.82:6432`.
|
||||||
|
|
||||||
|
|
||||||
We are setting up production deployment for TeleUploader on VPS `45.127.35.244` behind Traefik utilizing GitHub Actions.
|
We are setting up production deployment for TeleUploader on VPS `45.127.35.244` behind Traefik utilizing GitHub Actions.
|
||||||
|
|
||||||
## 1. System Architecture
|
## 1. System Architecture
|
||||||
@@ -22,7 +28,7 @@ TeleUploader is a Bun-based service.
|
|||||||
### `Dockerfile`
|
### `Dockerfile`
|
||||||
- Multi-stage build.
|
- Multi-stage build.
|
||||||
- **Stage 1 (Build)**: Install dependencies, copy source files, run Biome lint/format checks, compile TS build to `dist/index.js` using `bun build`.
|
- **Stage 1 (Build)**: Install dependencies, copy source files, run Biome lint/format checks, compile TS build to `dist/index.js` using `bun build`.
|
||||||
- **Stage 2 (Run)**: Use minimal `oven/bun:1.1-slim` runtime. Copy `dist/index.js`, `schema.sql`, and `package.json`. Expose port `3000`.
|
- **Stage 2 (Run)**: Use minimal `oven/bun:1.1-slim` runtime. Copy `dist/index.js`, `schema.sql`, and `package.json`. Expose port `4000`.
|
||||||
|
|
||||||
### `docker-compose.yml`
|
### `docker-compose.yml`
|
||||||
```yaml
|
```yaml
|
||||||
@@ -38,7 +44,7 @@ services:
|
|||||||
- STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID}
|
- STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID}
|
||||||
- BASE_URL=${BASE_URL}
|
- BASE_URL=${BASE_URL}
|
||||||
- DATABASE_URL=${DATABASE_URL}
|
- DATABASE_URL=${DATABASE_URL}
|
||||||
- PORT=3000
|
- PORT=4000
|
||||||
- NODE_ENV=production
|
- NODE_ENV=production
|
||||||
- LOG_LEVEL=info
|
- LOG_LEVEL=info
|
||||||
networks:
|
networks:
|
||||||
@@ -49,7 +55,7 @@ services:
|
|||||||
- "traefik.http.routers.teleuploader.entrypoints=websecure"
|
- "traefik.http.routers.teleuploader.entrypoints=websecure"
|
||||||
- "traefik.http.routers.teleuploader.tls=true"
|
- "traefik.http.routers.teleuploader.tls=true"
|
||||||
- "traefik.http.routers.teleuploader.tls.certresolver=letsencrypt"
|
- "traefik.http.routers.teleuploader.tls.certresolver=letsencrypt"
|
||||||
- "traefik.http.services.teleuploader.loadbalancer.server.port=3000"
|
- "traefik.http.services.teleuploader.loadbalancer.server.port=4000"
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
app-shared-net:
|
app-shared-net:
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
# Gitea Remote and CI/CD Migration Design
|
# Gitea Remote and CI/CD Migration Design
|
||||||
|
|
||||||
|
> Catatan (2026-08-02): Produksi sekarang port 4000, deploy Nix+systemd di orangevps, Caddy reverse proxy upload.asepharyana.my.id, DB via pgbouncer pool imrnes 100.121.180.82:6432. Docker/Traefik/Gitea-CI legacy.
|
||||||
|
|
||||||
## Goal
|
## Goal
|
||||||
|
|
||||||
Move the repository origin from GitLab to a new Gitea repository and add a Gitea Actions deployment flow that behaves like a GitHub Actions CI/CD pipeline.
|
Move the repository origin from GitLab to a new Gitea repository and add a Gitea Actions deployment flow that behaves like a GitHub Actions CI/CD pipeline.
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
# S3 Compatibility Completion Design
|
# S3 Compatibility Completion Design
|
||||||
|
|
||||||
|
> Catatan (2026-08-02): Produksi sekarang port 4000, deploy Nix+systemd di orangevps, Caddy reverse proxy upload.asepharyana.my.id, DB via pgbouncer pool imrnes 100.121.180.82:6432. Docker/Traefik/Gitea-CI legacy.
|
||||||
|
|
||||||
Date: 2026-07-07
|
Date: 2026-07-07
|
||||||
|
|
||||||
## Goal
|
## Goal
|
||||||
|
|||||||
@@ -7,6 +7,8 @@
|
|||||||
|
|
||||||
Telegram Bot API rate-limits each bot to approximately 1-2 concurrent uploads. When multiple upload chunks hit the same bot simultaneously, Telegram returns HTTP 429 (Too Many Requests), causing delays of 30-60 seconds per retry. Under Docker push load, these cumulative delays trigger Gitea client timeouts and `500 Internal Server Error`.
|
Telegram Bot API rate-limits each bot to approximately 1-2 concurrent uploads. When multiple upload chunks hit the same bot simultaneously, Telegram returns HTTP 429 (Too Many Requests), causing delays of 30-60 seconds per retry. Under Docker push load, these cumulative delays trigger Gitea client timeouts and `500 Internal Server Error`.
|
||||||
|
|
||||||
|
> Catatan (2026-08-02): Produksi sekarang port 4000, deploy Nix+systemd di orangevps, Caddy reverse proxy upload.asepharyana.my.id, DB via pgbouncer pool imrnes 100.121.180.82:6432. Docker/Traefik/Gitea-CI legacy.
|
||||||
|
|
||||||
The current architecture uses a **global PQueue** with `concurrency=N` where each task picks a bot via round-robin (`claimBotIndex()`). This means two concurrent tasks can both land on the same bot index (after wrap-around), causing 429 collisions.
|
The current architecture uses a **global PQueue** with `concurrency=N` where each task picks a bot via round-robin (`claimBotIndex()`). This means two concurrent tasks can both land on the same bot index (after wrap-around), causing 429 collisions.
|
||||||
|
|
||||||
## Solution: Per-Bot Queue
|
## Solution: Per-Bot Queue
|
||||||
|
|||||||
Generated
+61
@@ -0,0 +1,61 @@
|
|||||||
|
{
|
||||||
|
"nodes": {
|
||||||
|
"flake-utils": {
|
||||||
|
"inputs": {
|
||||||
|
"systems": "systems"
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1731533236,
|
||||||
|
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
|
||||||
|
"owner": "numtide",
|
||||||
|
"repo": "flake-utils",
|
||||||
|
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "numtide",
|
||||||
|
"repo": "flake-utils",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nixpkgs": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1785301185,
|
||||||
|
"narHash": "sha256-eoS3KQTO0aPWXZvIaRbRAzSSHW3l5wdMFXtT1ISfoKA=",
|
||||||
|
"owner": "NixOS",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"rev": "9bc02893134c733dd85de46ee4fb2fac696b5529",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "NixOS",
|
||||||
|
"ref": "nixpkgs-unstable",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": {
|
||||||
|
"inputs": {
|
||||||
|
"flake-utils": "flake-utils",
|
||||||
|
"nixpkgs": "nixpkgs"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"systems": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1681028828,
|
||||||
|
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||||
|
"owner": "nix-systems",
|
||||||
|
"repo": "default",
|
||||||
|
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "nix-systems",
|
||||||
|
"repo": "default",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": "root",
|
||||||
|
"version": 7
|
||||||
|
}
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
{
|
||||||
|
description = "TeleUploader — Nix build";
|
||||||
|
|
||||||
|
inputs = {
|
||||||
|
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
|
||||||
|
flake-utils.url = "github:numtide/flake-utils";
|
||||||
|
};
|
||||||
|
|
||||||
|
outputs = { self, nixpkgs, flake-utils }:
|
||||||
|
flake-utils.lib.eachSystem [ "x86_64-linux" ] (system:
|
||||||
|
let
|
||||||
|
pkgs = import nixpkgs { inherit system; };
|
||||||
|
|
||||||
|
# TeleUploader package
|
||||||
|
teleuploader = pkgs.stdenvNoCC.mkDerivation rec {
|
||||||
|
pname = "teleuploader";
|
||||||
|
version = "1.1.0";
|
||||||
|
|
||||||
|
src = ./.;
|
||||||
|
|
||||||
|
nativeBuildInputs = [
|
||||||
|
pkgs.bun
|
||||||
|
pkgs.makeBinaryWrapper
|
||||||
|
];
|
||||||
|
|
||||||
|
# Bun cache di sandbox — prevent online fetch
|
||||||
|
# Karena bun.lock sudah di repo, bun install --frozen-lockfile
|
||||||
|
# akan pake cache, tapi di Nix sandbox gak ada internet.
|
||||||
|
# Solusi: offline flag
|
||||||
|
preBuild = ''
|
||||||
|
export HOME=$TMPDIR/home
|
||||||
|
mkdir -p $HOME
|
||||||
|
export BUN_INSTALL=$HOME/.bun
|
||||||
|
'';
|
||||||
|
|
||||||
|
buildPhase = ''
|
||||||
|
echo "=== Installing dependencies ==="
|
||||||
|
bun install --frozen-lockfile --ignore-scripts 2>&1
|
||||||
|
|
||||||
|
echo "=== Building ==="
|
||||||
|
bun run build 2>&1
|
||||||
|
'';
|
||||||
|
|
||||||
|
installPhase = ''
|
||||||
|
mkdir -p $out/bin $out/share/teleuploader
|
||||||
|
|
||||||
|
# Copy dist files
|
||||||
|
cp -r dist $out/share/teleuploader/dist
|
||||||
|
cp src/home.html $out/share/teleuploader/ 2>/dev/null || true
|
||||||
|
cp schema.sql $out/share/teleuploader/ 2>/dev/null || true
|
||||||
|
|
||||||
|
# Wrap with bun from Nix store (dependency sharing!)
|
||||||
|
# Note: NO --chdir — systemd WorkingDirectory controls this
|
||||||
|
makeBinaryWrapper ${pkgs.bun}/bin/bun $out/bin/teleuploader \
|
||||||
|
--add-flags "$out/share/teleuploader/dist/index.js" \
|
||||||
|
--set-default NODE_ENV production \
|
||||||
|
--prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.bun ]}
|
||||||
|
|
||||||
|
# Also create the migrate wrapper
|
||||||
|
makeBinaryWrapper ${pkgs.bun}/bin/bun $out/bin/teleuploader-migrate \
|
||||||
|
--add-flags "$out/share/teleuploader/dist/migrate.js" \
|
||||||
|
--prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.bun ]}
|
||||||
|
'';
|
||||||
|
|
||||||
|
meta = {
|
||||||
|
description = "Telegram file uploader backend (S3 → Telegram)";
|
||||||
|
license = pkgs.lib.licenses.mit;
|
||||||
|
platforms = pkgs.lib.platforms.linux;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
in {
|
||||||
|
packages = {
|
||||||
|
inherit teleuploader;
|
||||||
|
default = teleuploader;
|
||||||
|
};
|
||||||
|
|
||||||
|
# Dev shell with bun for local development
|
||||||
|
devShells.default = pkgs.mkShell {
|
||||||
|
buildInputs = [
|
||||||
|
pkgs.bun
|
||||||
|
pkgs.nodejs_22
|
||||||
|
];
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
+6
-6
File diff suppressed because one or more lines are too long
@@ -1,7 +1,7 @@
|
|||||||
/**
|
/**
|
||||||
* Input for the upload file use case.
|
* Input for the upload file use case.
|
||||||
* Carries all metadata needed to persist an uploaded file,
|
* Carries all metadata needed to persist an uploaded file,
|
||||||
* including its temporary location on disk and optional bucket/S3 context.
|
* including its temporary location on disk and optional bucket/S3 context.a
|
||||||
*/
|
*/
|
||||||
export interface UploadInput {
|
export interface UploadInput {
|
||||||
/** Absolute path to the temporary file on disk */
|
/** Absolute path to the temporary file on disk */
|
||||||
|
|||||||
@@ -1,10 +1,11 @@
|
|||||||
import { nanoid } from 'nanoid';
|
import { nanoid } from 'nanoid';
|
||||||
|
import { buildNewFile } from '../../domain/entities/file-factory';
|
||||||
import type { MultipartUpload } from '../../domain/entities/multipart';
|
import type { MultipartUpload } from '../../domain/entities/multipart';
|
||||||
import type { IBucketRepository } from '../../domain/ports/bucket-repository';
|
import type { IBucketRepository } from '../../domain/ports/bucket-repository';
|
||||||
import type { IFileRepository } from '../../domain/ports/file-repository';
|
import type { IFileRepository } from '../../domain/ports/file-repository';
|
||||||
import type { IMultipartRepository } from '../../domain/ports/multipart-repository';
|
import type { IMultipartRepository } from '../../domain/ports/multipart-repository';
|
||||||
import type { ITelegramService } from '../../domain/ports/telegram-service';
|
import type { ITelegramService } from '../../domain/ports/telegram-service';
|
||||||
import { computeHash } from '../../shared/utils/file';
|
import { computeHash, DEFAULT_FILE_TYPE } from '../../shared/utils/file';
|
||||||
|
|
||||||
// ─── Types ──────────────────────────────────────────────────────────
|
// ─── Types ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -270,7 +271,8 @@ export function createCompleteMultipartUploadUseCase(deps: MultipartDeps) {
|
|||||||
|
|
||||||
const publicId = nanoid();
|
const publicId = nanoid();
|
||||||
|
|
||||||
await deps.fileRepo.create({
|
await deps.fileRepo.create(
|
||||||
|
buildNewFile({
|
||||||
publicId,
|
publicId,
|
||||||
telegramFileId: firstPart.telegramFileId,
|
telegramFileId: firstPart.telegramFileId,
|
||||||
telegramFileUniqueId: firstPart.telegramFileUniqueId,
|
telegramFileUniqueId: firstPart.telegramFileUniqueId,
|
||||||
@@ -279,22 +281,13 @@ export function createCompleteMultipartUploadUseCase(deps: MultipartDeps) {
|
|||||||
fileName: input.key.split('/').pop() || 'file',
|
fileName: input.key.split('/').pop() || 'file',
|
||||||
mimeType: 'application/octet-stream',
|
mimeType: 'application/octet-stream',
|
||||||
sizeBytes: totalSize,
|
sizeBytes: totalSize,
|
||||||
fileType: 'document',
|
fileType: DEFAULT_FILE_TYPE,
|
||||||
uploaderId: 0,
|
storageBackend: 'telegram',
|
||||||
fileHash: null,
|
|
||||||
archiveTelegramFileId: null,
|
|
||||||
archiveStorageMessageId: null,
|
|
||||||
archiveFileName: null,
|
|
||||||
archiveEntryName: null,
|
|
||||||
archiveMimeType: null,
|
|
||||||
archiveSizeBytes: null,
|
|
||||||
bucketId: multipart.bucketId,
|
bucketId: multipart.bucketId,
|
||||||
s3Key: input.key,
|
s3Key: input.key,
|
||||||
storageBackend: 'telegram',
|
|
||||||
isDeleted: false,
|
|
||||||
multipartUploadId: input.uploadId,
|
multipartUploadId: input.uploadId,
|
||||||
partCount: null,
|
}),
|
||||||
});
|
);
|
||||||
|
|
||||||
await deps.multipartRepo.complete(input.uploadId);
|
await deps.multipartRepo.complete(input.uploadId);
|
||||||
|
|
||||||
|
|||||||
@@ -1,23 +1,13 @@
|
|||||||
import { randomUUID } from 'node:crypto';
|
|
||||||
import { gzipSync } from 'node:zlib';
|
|
||||||
import { nanoid } from 'nanoid';
|
|
||||||
import type { File } from '../../domain/entities/file';
|
import type { File } from '../../domain/entities/file';
|
||||||
import type { NewFilePart } from '../../domain/entities/file-part';
|
|
||||||
import type { MultipartPart } from '../../domain/entities/multipart';
|
|
||||||
import type { IBucketRepository } from '../../domain/ports/bucket-repository';
|
import type { IBucketRepository } from '../../domain/ports/bucket-repository';
|
||||||
import type { IFilePartRepository } from '../../domain/ports/file-part-repository';
|
import type { IFilePartRepository } from '../../domain/ports/file-part-repository';
|
||||||
import type { IFileRepository, S3FileRecord } from '../../domain/ports/file-repository';
|
import type { IFileRepository } from '../../domain/ports/file-repository';
|
||||||
import type { IMultipartRepository } from '../../domain/ports/multipart-repository';
|
import type { IMultipartRepository } from '../../domain/ports/multipart-repository';
|
||||||
import type { ITelegramService, TelegramFileInfo } from '../../domain/ports/telegram-service';
|
import type { ITelegramService, TelegramFileInfo } from '../../domain/ports/telegram-service';
|
||||||
import { computeHash, ensureExtension, formatCreatedAt } from '../../shared/utils/file';
|
import type { CompressionAlgorithm } from '../../shared/utils/compress';
|
||||||
|
|
||||||
// ─── Types ──────────────────────────────────────────────────────────
|
// ─── Types ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/**
|
|
||||||
* Compression algorithm used for chunked object storage.
|
|
||||||
*/
|
|
||||||
type CompressionAlgorithm = 'gzip' | null;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A single part source for building a multi-part streaming response.
|
* A single part source for building a multi-part streaming response.
|
||||||
* Each part corresponds to a Telegram-stored file chunk.
|
* Each part corresponds to a Telegram-stored file chunk.
|
||||||
@@ -173,619 +163,3 @@ export interface S3ObjectDeps {
|
|||||||
/** Application configuration subset. */
|
/** Application configuration subset. */
|
||||||
config: S3ObjectConfig;
|
config: S3ObjectConfig;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── Helpers ────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Validates the configured chunk size and returns it as a safe integer.
|
|
||||||
*
|
|
||||||
* @param chunkSizeBytes - The configured chunk size in bytes.
|
|
||||||
* @returns The same value if it is a positive safe integer.
|
|
||||||
*/
|
|
||||||
const asSafeChunkSize = (chunkSizeBytes: number): number => {
|
|
||||||
if (!Number.isSafeInteger(chunkSizeBytes) || chunkSizeBytes <= 0) {
|
|
||||||
throw new Error('Invalid Telegram chunk size');
|
|
||||||
}
|
|
||||||
return chunkSizeBytes;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Optionally gzip-compresses a chunk if compression is enabled and the chunk
|
|
||||||
* is large enough to benefit from it.
|
|
||||||
*
|
|
||||||
* @param chunk - The raw chunk buffer.
|
|
||||||
* @param compress - Whether compression is enabled.
|
|
||||||
* @param compressionMinSizeBytes - Minimum chunk size to attempt compression.
|
|
||||||
* @returns The (possibly compressed) buffer and the algorithm used.
|
|
||||||
*/
|
|
||||||
const maybeCompressChunk = (
|
|
||||||
chunk: Buffer,
|
|
||||||
compress: boolean,
|
|
||||||
compressionMinSizeBytes: number,
|
|
||||||
): { bytes: Buffer; compressionAlgorithm: CompressionAlgorithm } => {
|
|
||||||
if (!compress || chunk.byteLength < compressionMinSizeBytes) {
|
|
||||||
return { bytes: chunk, compressionAlgorithm: null };
|
|
||||||
}
|
|
||||||
|
|
||||||
const gzipped = gzipSync(chunk);
|
|
||||||
if (gzipped.byteLength >= chunk.byteLength) {
|
|
||||||
return { bytes: chunk, compressionAlgorithm: null };
|
|
||||||
}
|
|
||||||
|
|
||||||
return { bytes: gzipped, compressionAlgorithm: 'gzip' };
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Metadata for a single uploaded chunk/part during S3 put-object.
|
|
||||||
*/
|
|
||||||
interface UploadedPart {
|
|
||||||
/** 1-based part number. */
|
|
||||||
partNumber: number;
|
|
||||||
/** Telegram file identifier for this part. */
|
|
||||||
telegramFileId: string;
|
|
||||||
/** Telegram unique file identifier (stable across bot tokens). */
|
|
||||||
telegramFileUniqueId: string;
|
|
||||||
/** Message ID within the storage chat. */
|
|
||||||
storageMessageId: number;
|
|
||||||
/** Original size of the chunk in bytes before compression. */
|
|
||||||
sizeBytes: number;
|
|
||||||
/** Stored (post-compression) size in bytes. */
|
|
||||||
storedSizeBytes: number;
|
|
||||||
/** Compression algorithm applied, or null if uncompressed. */
|
|
||||||
compressionAlgorithm: CompressionAlgorithm;
|
|
||||||
/** SHA-256 hash of the original chunk content. */
|
|
||||||
etag: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Result of uploading an object in multiple Telegram chunks. */
|
|
||||||
interface ChunkedUploadResult {
|
|
||||||
/** Metadata for each uploaded part. */
|
|
||||||
parts: UploadedPart[];
|
|
||||||
/** SHA-256 hex digest of the complete object content. */
|
|
||||||
fileHash: string;
|
|
||||||
/** Total object size in bytes (sum of all original chunks). */
|
|
||||||
totalSizeBytes: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Uploads a buffer to Telegram in chunks, returning metadata for all parts.
|
|
||||||
*
|
|
||||||
* @param buffer - The full object buffer.
|
|
||||||
* @param partFileNamePrefix - Prefix used for each chunk's file name in Telegram.
|
|
||||||
* @param chunkSizeBytes - Maximum size of each chunk in bytes.
|
|
||||||
* @param compress - Whether gzip compression is enabled.
|
|
||||||
* @param compressionMinSizeBytes - Minimum chunk size to attempt compression.
|
|
||||||
* @param telegramService - The Telegram service to forward each chunk.
|
|
||||||
* @returns The aggregated chunked upload result.
|
|
||||||
*/
|
|
||||||
const uploadInChunks = async (
|
|
||||||
buffer: Buffer,
|
|
||||||
partFileNamePrefix: string,
|
|
||||||
chunkSizeBytes: number,
|
|
||||||
compress: boolean,
|
|
||||||
compressionMinSizeBytes: number,
|
|
||||||
telegramService: ITelegramService,
|
|
||||||
): Promise<ChunkedUploadResult> => {
|
|
||||||
const safeChunkSize = asSafeChunkSize(chunkSizeBytes);
|
|
||||||
const hasher = new Bun.CryptoHasher('sha256');
|
|
||||||
const parts: UploadedPart[] = [];
|
|
||||||
let totalSizeBytes = 0;
|
|
||||||
let partNumber = 0;
|
|
||||||
let offset = 0;
|
|
||||||
|
|
||||||
while (offset < buffer.byteLength) {
|
|
||||||
const chunk = buffer.subarray(offset, offset + safeChunkSize);
|
|
||||||
if (chunk.byteLength === 0) break;
|
|
||||||
|
|
||||||
partNumber += 1;
|
|
||||||
totalSizeBytes += chunk.byteLength;
|
|
||||||
hasher.update(chunk);
|
|
||||||
|
|
||||||
const { bytes, compressionAlgorithm } = maybeCompressChunk(
|
|
||||||
chunk,
|
|
||||||
compress,
|
|
||||||
compressionMinSizeBytes,
|
|
||||||
);
|
|
||||||
|
|
||||||
const forwardResult = await telegramService.forwardToStorage(
|
|
||||||
bytes,
|
|
||||||
`${partFileNamePrefix}.part-${partNumber}`,
|
|
||||||
'document',
|
|
||||||
);
|
|
||||||
|
|
||||||
parts.push({
|
|
||||||
partNumber,
|
|
||||||
telegramFileId: forwardResult.telegramFileId,
|
|
||||||
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
|
|
||||||
storageMessageId: forwardResult.storageMessageId,
|
|
||||||
sizeBytes: chunk.byteLength,
|
|
||||||
storedSizeBytes: bytes.byteLength,
|
|
||||||
compressionAlgorithm,
|
|
||||||
etag: computeHash(chunk),
|
|
||||||
});
|
|
||||||
|
|
||||||
offset += safeChunkSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
parts,
|
|
||||||
fileHash: hasher.digest('hex'),
|
|
||||||
totalSizeBytes,
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Resolves a list of multipart parts to their Telegram CDN URLs.
|
|
||||||
*
|
|
||||||
* @param parts - The stored multipart parts.
|
|
||||||
* @param telegramService - The Telegram service for resolving file metadata.
|
|
||||||
* @returns An array of resolved part sources.
|
|
||||||
*/
|
|
||||||
const resolveMultipartParts = async (
|
|
||||||
parts: MultipartPart[],
|
|
||||||
telegramService: ITelegramService,
|
|
||||||
): Promise<ObjectPartSource[]> => {
|
|
||||||
const sources: ObjectPartSource[] = [];
|
|
||||||
for (const part of parts) {
|
|
||||||
const fileInfo = await telegramService.getFileInfo(part.telegramFileId);
|
|
||||||
sources.push({
|
|
||||||
telegramFileId: part.telegramFileId,
|
|
||||||
telegramUrl: `https://api.telegram.org/file/bot${fileInfo.bot_token}/${fileInfo.file_path}`,
|
|
||||||
sizeBytes: part.sizeBytes,
|
|
||||||
partNumber: part.partNumber,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return sources;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Formats a `createdAt` value into an HTTP Last-Modified header value.
|
|
||||||
*
|
|
||||||
* @param date - The date to format.
|
|
||||||
* @returns The UTC string representation.
|
|
||||||
*/
|
|
||||||
const formatLastModified = (date: Date | string | number): string => {
|
|
||||||
return date instanceof Date ? date.toUTCString() : new Date(date).toUTCString();
|
|
||||||
};
|
|
||||||
|
|
||||||
// ─── Use Case Factories ─────────────────────────────────────────────
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a use case that resolves an S3 object for retrieval (GET).
|
|
||||||
*
|
|
||||||
* Looks up the bucket and file by key, then determines the storage type:
|
|
||||||
* - **direct**: regular Telegram-stored object — resolves the Telegram CDN URL.
|
|
||||||
* - **chunked**: object stored across multiple Telegram file parts.
|
|
||||||
* - **multipart**: object assembled from a completed multipart upload — resolves
|
|
||||||
* the Telegram CDN URLs for each part.
|
|
||||||
*
|
|
||||||
* @param deps - The injected dependencies.
|
|
||||||
* @returns An async function accepting bucket name and object key, returning
|
|
||||||
* a discriminated union of possible results, or `null` when the
|
|
||||||
* bucket or file is not found.
|
|
||||||
*/
|
|
||||||
export function createGetObjectUseCase(deps: S3ObjectDeps) {
|
|
||||||
return async (bucketName: string, key: string): Promise<GetObjectResult | null> => {
|
|
||||||
const bucket = await deps.bucketRepo.findByName(bucketName);
|
|
||||||
if (!bucket) return null;
|
|
||||||
|
|
||||||
const file = await deps.fileRepo.findByBucketAndKey(bucket.id, key);
|
|
||||||
if (!file) return null;
|
|
||||||
|
|
||||||
// Chunked storage — return the entity; the caller resolves parts via
|
|
||||||
// chunked-storage helpers.
|
|
||||||
if (file.storageBackend === 'chunked') {
|
|
||||||
return { type: 'chunked', file };
|
|
||||||
}
|
|
||||||
|
|
||||||
// Multipart upload object — resolve part Telegram URLs
|
|
||||||
if (file.multipartUploadId) {
|
|
||||||
const parts = await deps.multipartRepo.listParts(file.multipartUploadId);
|
|
||||||
const resolvedParts = await resolveMultipartParts(parts, deps.telegramService);
|
|
||||||
return { type: 'multipart', file, parts: resolvedParts };
|
|
||||||
}
|
|
||||||
|
|
||||||
// Regular direct object — resolve Telegram CDN URL
|
|
||||||
const fileInfo = await deps.telegramService.getFileInfo(file.telegramFileId);
|
|
||||||
const telegramUrl = `https://api.telegram.org/file/bot${fileInfo.bot_token}/${fileInfo.file_path}`;
|
|
||||||
|
|
||||||
return { type: 'direct', file, telegramUrl, fileInfo };
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a use case that retrieves S3 object metadata (HEAD).
|
|
||||||
*
|
|
||||||
* @param deps - The injected dependencies.
|
|
||||||
* @returns An async function accepting bucket name and object key, returning
|
|
||||||
* metadata or `null` when the bucket or file is not found.
|
|
||||||
*/
|
|
||||||
export function createHeadObjectUseCase(deps: S3ObjectDeps) {
|
|
||||||
return async (bucketName: string, key: string): Promise<HeadObjectMetadata | null> => {
|
|
||||||
const bucket = await deps.bucketRepo.findByName(bucketName);
|
|
||||||
if (!bucket) return null;
|
|
||||||
|
|
||||||
const file = await deps.fileRepo.findByBucketAndKey(bucket.id, key);
|
|
||||||
if (!file) return null;
|
|
||||||
|
|
||||||
return {
|
|
||||||
contentType: file.mimeType,
|
|
||||||
contentLength: file.sizeBytes,
|
|
||||||
etag: file.fileHash || nanoid(16),
|
|
||||||
lastModified: formatLastModified(file.createdAt),
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a use case that stores an S3 object (PUT).
|
|
||||||
*
|
|
||||||
* Handles both chunked (large) and single-message (small) upload paths,
|
|
||||||
* deduplicates by bucket+key (idempotent PUT), and persists the file
|
|
||||||
* record and (for chunked storage) part records.
|
|
||||||
*
|
|
||||||
* @param deps - The injected dependencies.
|
|
||||||
* @returns An async function accepting bucket name, key, body buffer, and
|
|
||||||
* content type, returning the etag of the stored object. Returns
|
|
||||||
* `null` when the bucket is not found.
|
|
||||||
*/
|
|
||||||
export function createPutObjectUseCase(deps: S3ObjectDeps) {
|
|
||||||
return async (
|
|
||||||
bucketName: string,
|
|
||||||
key: string,
|
|
||||||
body: Buffer,
|
|
||||||
contentType: string,
|
|
||||||
): Promise<PutObjectResult | null> => {
|
|
||||||
const bucket = await deps.bucketRepo.findByName(bucketName);
|
|
||||||
if (!bucket) return null;
|
|
||||||
|
|
||||||
const hash = computeHash(body);
|
|
||||||
|
|
||||||
// Idempotent PUT: if the object already exists, skip upload
|
|
||||||
const existing = await deps.fileRepo.findByBucketAndKey(bucket.id, key);
|
|
||||||
if (existing) {
|
|
||||||
return { etag: `"${hash}"` };
|
|
||||||
}
|
|
||||||
|
|
||||||
const fileName = key.split('/').pop() || 'file';
|
|
||||||
const signatureBuffer = body.subarray(0, 16);
|
|
||||||
const { fileName: finalFileName, mimeType } = ensureExtension(
|
|
||||||
fileName,
|
|
||||||
signatureBuffer,
|
|
||||||
contentType,
|
|
||||||
);
|
|
||||||
|
|
||||||
const partFileNamePrefix = `s3-${bucket.name}-${key.replace(/\//g, '_')}`;
|
|
||||||
const {
|
|
||||||
telegramChunkSizeBytes,
|
|
||||||
compressChunkedUploads,
|
|
||||||
chunkCompressionMinSizeBytes,
|
|
||||||
storageChatId,
|
|
||||||
} = deps.config;
|
|
||||||
|
|
||||||
if (body.byteLength > telegramChunkSizeBytes) {
|
|
||||||
// Chunked upload path
|
|
||||||
const chunkResult = await uploadInChunks(
|
|
||||||
body,
|
|
||||||
partFileNamePrefix,
|
|
||||||
telegramChunkSizeBytes,
|
|
||||||
compressChunkedUploads,
|
|
||||||
chunkCompressionMinSizeBytes,
|
|
||||||
deps.telegramService,
|
|
||||||
);
|
|
||||||
|
|
||||||
const firstPart = chunkResult.parts[0];
|
|
||||||
if (!firstPart) {
|
|
||||||
throw new Error('Chunked upload produced no parts');
|
|
||||||
}
|
|
||||||
|
|
||||||
const fileId = randomUUID();
|
|
||||||
const publicId = nanoid();
|
|
||||||
|
|
||||||
await deps.fileRepo.create({
|
|
||||||
publicId,
|
|
||||||
telegramFileId: firstPart.telegramFileId,
|
|
||||||
telegramFileUniqueId: firstPart.telegramFileUniqueId,
|
|
||||||
storageChatId,
|
|
||||||
storageMessageId: firstPart.storageMessageId,
|
|
||||||
fileName: finalFileName,
|
|
||||||
mimeType,
|
|
||||||
sizeBytes: chunkResult.totalSizeBytes,
|
|
||||||
fileType: 'document',
|
|
||||||
uploaderId: 0,
|
|
||||||
fileHash: chunkResult.fileHash,
|
|
||||||
archiveTelegramFileId: null,
|
|
||||||
archiveStorageMessageId: null,
|
|
||||||
archiveFileName: null,
|
|
||||||
archiveEntryName: null,
|
|
||||||
archiveMimeType: null,
|
|
||||||
archiveSizeBytes: null,
|
|
||||||
bucketId: bucket.id,
|
|
||||||
s3Key: key,
|
|
||||||
storageBackend: 'chunked',
|
|
||||||
isDeleted: false,
|
|
||||||
multipartUploadId: null,
|
|
||||||
partCount: chunkResult.parts.length,
|
|
||||||
});
|
|
||||||
|
|
||||||
const fileParts: NewFilePart[] = chunkResult.parts.map((part) => ({
|
|
||||||
fileId,
|
|
||||||
partNumber: part.partNumber,
|
|
||||||
telegramFileId: part.telegramFileId,
|
|
||||||
telegramFileUniqueId: part.telegramFileUniqueId,
|
|
||||||
storageChatId,
|
|
||||||
storageMessageId: part.storageMessageId,
|
|
||||||
sizeBytes: part.sizeBytes,
|
|
||||||
storedSizeBytes: part.storedSizeBytes,
|
|
||||||
compressionAlgorithm: part.compressionAlgorithm,
|
|
||||||
etag: part.etag,
|
|
||||||
}));
|
|
||||||
|
|
||||||
await deps.filePartRepo.insert(fileParts);
|
|
||||||
|
|
||||||
return { etag: `"${chunkResult.fileHash}"` };
|
|
||||||
}
|
|
||||||
|
|
||||||
// Single-message upload path
|
|
||||||
const forwardResult = await deps.telegramService.forwardToStorage(
|
|
||||||
body,
|
|
||||||
partFileNamePrefix,
|
|
||||||
'document',
|
|
||||||
);
|
|
||||||
|
|
||||||
const publicId = nanoid();
|
|
||||||
|
|
||||||
await deps.fileRepo.create({
|
|
||||||
publicId,
|
|
||||||
telegramFileId: forwardResult.telegramFileId,
|
|
||||||
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
|
|
||||||
storageChatId,
|
|
||||||
storageMessageId: forwardResult.storageMessageId,
|
|
||||||
fileName: finalFileName,
|
|
||||||
mimeType,
|
|
||||||
sizeBytes: body.byteLength,
|
|
||||||
fileType: 'document',
|
|
||||||
uploaderId: 0,
|
|
||||||
fileHash: hash,
|
|
||||||
archiveTelegramFileId: null,
|
|
||||||
archiveStorageMessageId: null,
|
|
||||||
archiveFileName: null,
|
|
||||||
archiveEntryName: null,
|
|
||||||
archiveMimeType: null,
|
|
||||||
archiveSizeBytes: null,
|
|
||||||
bucketId: bucket.id,
|
|
||||||
s3Key: key,
|
|
||||||
storageBackend: 'telegram',
|
|
||||||
isDeleted: false,
|
|
||||||
multipartUploadId: null,
|
|
||||||
partCount: null,
|
|
||||||
});
|
|
||||||
|
|
||||||
return { etag: `"${hash}"` };
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a use case that copies an S3 object to a new key (PUT with
|
|
||||||
* x-amz-copy-source).
|
|
||||||
*
|
|
||||||
* Creates a new file record referencing the same Telegram-stored data.
|
|
||||||
* Chunked source objects are not supported for copy.
|
|
||||||
*
|
|
||||||
* @param deps - The injected dependencies.
|
|
||||||
* @returns An async function accepting source + destination identifiers and
|
|
||||||
* optional precondition headers, returning the copy result or
|
|
||||||
* `null` when a required bucket or file is not found.
|
|
||||||
*/
|
|
||||||
export function createCopyObjectUseCase(deps: S3ObjectDeps) {
|
|
||||||
return async (input: {
|
|
||||||
/** Source bucket name. */
|
|
||||||
sourceBucket: string;
|
|
||||||
/** Source object key. */
|
|
||||||
sourceKey: string;
|
|
||||||
/** Destination bucket UUID (must already exist). */
|
|
||||||
destBucketId: string;
|
|
||||||
/** Destination object key. */
|
|
||||||
destKey: string;
|
|
||||||
/** Optional if-match precondition (raw etag value, without surrounding quotes). */
|
|
||||||
ifMatch?: string | null;
|
|
||||||
/** Optional if-none-match precondition (raw etag value, without surrounding quotes). */
|
|
||||||
ifNoneMatch?: string | null;
|
|
||||||
}): Promise<CopyObjectResult | null> => {
|
|
||||||
const sourceBucket = await deps.bucketRepo.findByName(input.sourceBucket);
|
|
||||||
if (!sourceBucket) return null;
|
|
||||||
|
|
||||||
const sourceFile = await deps.fileRepo.findByBucketAndKey(sourceBucket.id, input.sourceKey);
|
|
||||||
if (!sourceFile) return null;
|
|
||||||
|
|
||||||
if (sourceFile.storageBackend === 'chunked') {
|
|
||||||
throw new ObjectError(
|
|
||||||
'NotImplemented',
|
|
||||||
'Copying chunked objects is not yet implemented.',
|
|
||||||
501,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Conditional copy: if-match / if-none-match checks
|
|
||||||
const sourceEtag = sourceFile.fileHash;
|
|
||||||
if (input.ifMatch && sourceEtag && input.ifMatch !== sourceEtag) {
|
|
||||||
throw new ObjectError(
|
|
||||||
'PreconditionFailed',
|
|
||||||
'The preconditions you specified did not hold.',
|
|
||||||
412,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
if (input.ifNoneMatch && sourceEtag && input.ifNoneMatch === sourceEtag) {
|
|
||||||
throw new ObjectError(
|
|
||||||
'PreconditionFailed',
|
|
||||||
'The preconditions you specified did not hold.',
|
|
||||||
412,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const publicId = nanoid();
|
|
||||||
|
|
||||||
await deps.fileRepo.create({
|
|
||||||
publicId,
|
|
||||||
telegramFileId: sourceFile.telegramFileId,
|
|
||||||
telegramFileUniqueId: sourceFile.telegramFileUniqueId,
|
|
||||||
storageChatId: sourceFile.storageChatId,
|
|
||||||
storageMessageId: sourceFile.storageMessageId,
|
|
||||||
fileName: sourceFile.fileName,
|
|
||||||
mimeType: sourceFile.mimeType,
|
|
||||||
sizeBytes: sourceFile.sizeBytes,
|
|
||||||
fileType: sourceFile.fileType,
|
|
||||||
uploaderId: 0,
|
|
||||||
fileHash: sourceFile.fileHash,
|
|
||||||
archiveTelegramFileId: sourceFile.archiveTelegramFileId,
|
|
||||||
archiveStorageMessageId: sourceFile.archiveStorageMessageId,
|
|
||||||
archiveFileName: sourceFile.archiveFileName,
|
|
||||||
archiveEntryName: sourceFile.archiveEntryName,
|
|
||||||
archiveMimeType: sourceFile.archiveMimeType,
|
|
||||||
archiveSizeBytes: sourceFile.archiveSizeBytes,
|
|
||||||
bucketId: input.destBucketId,
|
|
||||||
s3Key: input.destKey,
|
|
||||||
storageBackend: 'telegram',
|
|
||||||
isDeleted: false,
|
|
||||||
multipartUploadId: null,
|
|
||||||
partCount: null,
|
|
||||||
});
|
|
||||||
|
|
||||||
return {
|
|
||||||
etag: sourceEtag || nanoid(16),
|
|
||||||
lastModified: new Date().toISOString(),
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Error type for S3 object-level application errors.
|
|
||||||
*/
|
|
||||||
export class ObjectError extends Error {
|
|
||||||
/** S3-compatible error code. */
|
|
||||||
readonly code: string;
|
|
||||||
/** Suggested HTTP status code. */
|
|
||||||
readonly status: number;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param code - The S3 error code.
|
|
||||||
* @param message - Human-readable error description.
|
|
||||||
* @param status - Suggested HTTP status.
|
|
||||||
*/
|
|
||||||
constructor(code: string, message: string, status: number) {
|
|
||||||
super(message);
|
|
||||||
this.name = 'ObjectError';
|
|
||||||
this.code = code;
|
|
||||||
this.status = status;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a use case that soft-deletes an S3 object (DELETE).
|
|
||||||
*
|
|
||||||
* @param deps - The injected dependencies.
|
|
||||||
* @returns An async function accepting bucket name and object key, returning
|
|
||||||
* `true` if a row was soft-deleted. Returns `null` when the bucket
|
|
||||||
* is not found.
|
|
||||||
*/
|
|
||||||
export function createDeleteObjectUseCase(deps: S3ObjectDeps) {
|
|
||||||
return async (bucketName: string, key: string): Promise<boolean | null> => {
|
|
||||||
const bucket = await deps.bucketRepo.findByName(bucketName);
|
|
||||||
if (!bucket) return null;
|
|
||||||
return deps.fileRepo.softDelete(bucket.id, key);
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a use case that batch-deletes multiple S3 objects (POST with
|
|
||||||
* ?delete).
|
|
||||||
*
|
|
||||||
* @param deps - The injected dependencies.
|
|
||||||
* @returns An async function accepting bucket name and an array of keys,
|
|
||||||
* returning the array of keys that were actually deleted. Returns
|
|
||||||
* `null` when the bucket is not found.
|
|
||||||
*/
|
|
||||||
export function createDeleteObjectsUseCase(deps: S3ObjectDeps) {
|
|
||||||
return async (bucketName: string, keys: string[]): Promise<string[] | null> => {
|
|
||||||
const bucket = await deps.bucketRepo.findByName(bucketName);
|
|
||||||
if (!bucket) return null;
|
|
||||||
|
|
||||||
const deletedKeys: string[] = [];
|
|
||||||
for (const key of keys) {
|
|
||||||
const ok = await deps.fileRepo.softDelete(bucket.id, key);
|
|
||||||
if (ok) deletedKeys.push(key);
|
|
||||||
}
|
|
||||||
return deletedKeys;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a use case that lists objects within a bucket (ListObjectsV1/V2).
|
|
||||||
*
|
|
||||||
* Supports prefix filtering, delimiter-based pseudo-directory grouping, and
|
|
||||||
* pagination via marker/startAfter.
|
|
||||||
*
|
|
||||||
* @param deps - The injected dependencies.
|
|
||||||
* @returns An async function accepting query parameters and returning the
|
|
||||||
* listing result, or `null` when the bucket is not found.
|
|
||||||
*/
|
|
||||||
export function createListObjectsUseCase(deps: S3ObjectDeps) {
|
|
||||||
return async (input: {
|
|
||||||
/** Bucket name to list from. */
|
|
||||||
bucketName: string;
|
|
||||||
/** Key prefix to filter by (empty string for no filter). */
|
|
||||||
prefix: string;
|
|
||||||
/** Delimiter character (e.g. "/") or null for flat listing. */
|
|
||||||
delimiter: string | null;
|
|
||||||
/** Maximum number of object records to return (clamped to 1000). */
|
|
||||||
maxKeys: number;
|
|
||||||
/** Return only keys strictly greater than this value, or null. */
|
|
||||||
startAfter: string | null;
|
|
||||||
}): Promise<ListObjectsResult | null> => {
|
|
||||||
const bucket = await deps.bucketRepo.findByName(input.bucketName);
|
|
||||||
if (!bucket) return null;
|
|
||||||
|
|
||||||
const clampedMaxKeys = Math.min(input.maxKeys, 1000);
|
|
||||||
|
|
||||||
const { objects, prefixes } = await deps.fileRepo.listByPrefix(
|
|
||||||
bucket.id,
|
|
||||||
input.prefix,
|
|
||||||
input.delimiter,
|
|
||||||
clampedMaxKeys,
|
|
||||||
input.startAfter,
|
|
||||||
);
|
|
||||||
|
|
||||||
const isTruncated = objects.length > clampedMaxKeys;
|
|
||||||
const displayObjects = objects.slice(0, clampedMaxKeys);
|
|
||||||
const nextMarker = isTruncated
|
|
||||||
? (displayObjects[displayObjects.length - 1]?.s3Key ?? null)
|
|
||||||
: null;
|
|
||||||
|
|
||||||
return {
|
|
||||||
objects: displayObjects.map((o: S3FileRecord) => ({
|
|
||||||
key: o.s3Key,
|
|
||||||
sizeBytes: o.sizeBytes,
|
|
||||||
etag: o.fileHash || nanoid(16),
|
|
||||||
lastModified: formatCreatedAt(o.createdAt),
|
|
||||||
mimeType: o.mimeType,
|
|
||||||
})),
|
|
||||||
prefixes,
|
|
||||||
isTruncated,
|
|
||||||
nextMarker,
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a use case that checks whether an object exists and is accessible
|
|
||||||
* within a bucket.
|
|
||||||
*
|
|
||||||
* @param deps - The injected dependencies.
|
|
||||||
* @returns An async function accepting a bucket ID and object key,
|
|
||||||
* returning the file entity or null.
|
|
||||||
*/
|
|
||||||
export function createFindObjectUseCase(deps: Pick<S3ObjectDeps, 'fileRepo'>) {
|
|
||||||
return async (bucketId: string, key: string): Promise<File | null> => {
|
|
||||||
return deps.fileRepo.findByBucketAndKey(bucketId, key);
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,48 +1,13 @@
|
|||||||
import { randomUUID } from 'node:crypto';
|
|
||||||
import { createReadStream } from 'node:fs';
|
import { createReadStream } from 'node:fs';
|
||||||
import { open } from 'node:fs/promises';
|
import { open } from 'node:fs/promises';
|
||||||
import { gzipSync } from 'node:zlib';
|
|
||||||
import { nanoid } from 'nanoid';
|
import { nanoid } from 'nanoid';
|
||||||
import type { NewFilePart } from '../../domain/entities/file-part';
|
import { buildNewFile } from '../../domain/entities/file-factory';
|
||||||
import type { IFilePartRepository } from '../../domain/ports/file-part-repository';
|
|
||||||
import type { IFileRepository } from '../../domain/ports/file-repository';
|
import type { IFileRepository } from '../../domain/ports/file-repository';
|
||||||
import type { ITelegramService } from '../../domain/ports/telegram-service';
|
import type { ITelegramService } from '../../domain/ports/telegram-service';
|
||||||
import { checkFileSize, computeHash, ensureExtension, getFileType } from '../../shared/utils/file';
|
import type { ChunkedStorage } from '../../infrastructure/telegram/chunked-storage';
|
||||||
|
import { checkFileSize, ensureExtension, getFileType } from '../../shared/utils/file';
|
||||||
import type { UploadInput, UploadOutput } from '../dto/upload';
|
import type { UploadInput, UploadOutput } from '../dto/upload';
|
||||||
|
|
||||||
/** Compression algorithm string literal used in chunked storage. */
|
|
||||||
type ChunkCompressionAlgorithm = 'gzip' | null;
|
|
||||||
|
|
||||||
/** Metadata for a single uploaded chunk/part. */
|
|
||||||
interface UploadedPart {
|
|
||||||
/** 1-based part number. */
|
|
||||||
partNumber: number;
|
|
||||||
/** Telegram file identifier for this part. */
|
|
||||||
telegramFileId: string;
|
|
||||||
/** Telegram unique file identifier (stable across bot tokens). */
|
|
||||||
telegramFileUniqueId: string;
|
|
||||||
/** Message ID within the storage chat. */
|
|
||||||
storageMessageId: number;
|
|
||||||
/** Original size of the chunk in bytes before compression. */
|
|
||||||
sizeBytes: number;
|
|
||||||
/** Stored (post-compression) size in bytes. */
|
|
||||||
storedSizeBytes: number;
|
|
||||||
/** Compression algorithm applied, or null if uncompressed. */
|
|
||||||
compressionAlgorithm: ChunkCompressionAlgorithm;
|
|
||||||
/** SHA-256 hash of the original chunk content. */
|
|
||||||
etag: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Result of uploading a file in multiple Telegram chunks. */
|
|
||||||
interface ChunkedUploadResult {
|
|
||||||
/** Metadata for each uploaded part. */
|
|
||||||
parts: UploadedPart[];
|
|
||||||
/** SHA-256 hex digest of the complete file content. */
|
|
||||||
fileHash: string;
|
|
||||||
/** Total file size in bytes (sum of all original chunks). */
|
|
||||||
totalSizeBytes: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Subset of application configuration consumed by the upload-file use case. */
|
/** Subset of application configuration consumed by the upload-file use case. */
|
||||||
export interface UploadFileConfig {
|
export interface UploadFileConfig {
|
||||||
/** Server base URL for constructing download links. */
|
/** Server base URL for constructing download links. */
|
||||||
@@ -61,53 +26,14 @@ export interface UploadFileConfig {
|
|||||||
export interface UploadFileUseCaseDeps {
|
export interface UploadFileUseCaseDeps {
|
||||||
/** File repository for CRUD operations on file records. */
|
/** File repository for CRUD operations on file records. */
|
||||||
fileRepo: IFileRepository;
|
fileRepo: IFileRepository;
|
||||||
/** File-part repository for chunked file metadata. */
|
|
||||||
filePartRepo: IFilePartRepository;
|
|
||||||
/** Telegram service for forwarding file content to storage. */
|
/** Telegram service for forwarding file content to storage. */
|
||||||
telegramService: ITelegramService;
|
telegramService: ITelegramService;
|
||||||
|
/** Chunked storage handler for large file uploads. */
|
||||||
|
chunkedStorage: ChunkedStorage;
|
||||||
/** Application configuration subset. */
|
/** Application configuration subset. */
|
||||||
config: UploadFileConfig;
|
config: UploadFileConfig;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Validates the configured chunk size and returns it as a safe integer.
|
|
||||||
*
|
|
||||||
* @param chunkSizeBytes - The configured chunk size in bytes.
|
|
||||||
* @returns The same value if it is a positive safe integer.
|
|
||||||
*/
|
|
||||||
const asSafeChunkSize = (chunkSizeBytes: number): number => {
|
|
||||||
if (!Number.isSafeInteger(chunkSizeBytes) || chunkSizeBytes <= 0) {
|
|
||||||
throw new Error('Invalid Telegram chunk size');
|
|
||||||
}
|
|
||||||
return chunkSizeBytes;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Optionally gzip-compresses a chunk if compression is enabled and the chunk
|
|
||||||
* is large enough to benefit from it.
|
|
||||||
*
|
|
||||||
* @param chunk - The raw chunk buffer.
|
|
||||||
* @param compress - Whether compression is enabled.
|
|
||||||
* @param compressionMinSizeBytes - Minimum chunk size to attempt compression.
|
|
||||||
* @returns The (possibly compressed) buffer and the algorithm used.
|
|
||||||
*/
|
|
||||||
const maybeCompressChunk = (
|
|
||||||
chunk: Buffer,
|
|
||||||
compress: boolean,
|
|
||||||
compressionMinSizeBytes: number,
|
|
||||||
): { bytes: Buffer; compressionAlgorithm: ChunkCompressionAlgorithm } => {
|
|
||||||
if (!compress || chunk.byteLength < compressionMinSizeBytes) {
|
|
||||||
return { bytes: chunk, compressionAlgorithm: null };
|
|
||||||
}
|
|
||||||
|
|
||||||
const gzipped = gzipSync(chunk);
|
|
||||||
if (gzipped.byteLength >= chunk.byteLength) {
|
|
||||||
return { bytes: chunk, compressionAlgorithm: null };
|
|
||||||
}
|
|
||||||
|
|
||||||
return { bytes: gzipped, compressionAlgorithm: 'gzip' };
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Reads the first 16 bytes from a file on disk for magic-byte detection.
|
* Reads the first 16 bytes from a file on disk for magic-byte detection.
|
||||||
*
|
*
|
||||||
@@ -125,74 +51,6 @@ const readSignatureBuffer = async (tempPath: string): Promise<Buffer> => {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
|
||||||
* Reads a file from disk in chunks, forwards each chunk to Telegram storage,
|
|
||||||
* and returns metadata for all uploaded parts together with the total file
|
|
||||||
* hash.
|
|
||||||
*
|
|
||||||
* @param tempPath - Absolute path to the temporary file on disk.
|
|
||||||
* @param partFileNamePrefix - Prefix used for each chunk's file name in Telegram.
|
|
||||||
* @param chunkSizeBytes - Maximum size of each chunk in bytes.
|
|
||||||
* @param compress - Whether gzip compression is enabled.
|
|
||||||
* @param compressionMinSizeBytes - Minimum chunk size to attempt compression.
|
|
||||||
* @param telegramService - The Telegram service to forward each chunk.
|
|
||||||
* @returns The aggregated chunked upload result.
|
|
||||||
*/
|
|
||||||
const uploadFileInTelegramChunks = async (
|
|
||||||
tempPath: string,
|
|
||||||
partFileNamePrefix: string,
|
|
||||||
chunkSizeBytes: number,
|
|
||||||
compress: boolean,
|
|
||||||
compressionMinSizeBytes: number,
|
|
||||||
telegramService: ITelegramService,
|
|
||||||
): Promise<ChunkedUploadResult> => {
|
|
||||||
const safeChunkSize = asSafeChunkSize(chunkSizeBytes);
|
|
||||||
const hasher = new Bun.CryptoHasher('sha256');
|
|
||||||
const parts: UploadedPart[] = [];
|
|
||||||
let totalSizeBytes = 0;
|
|
||||||
let partNumber = 0;
|
|
||||||
|
|
||||||
const stream = createReadStream(tempPath, { highWaterMark: safeChunkSize });
|
|
||||||
|
|
||||||
for await (const data of stream) {
|
|
||||||
const chunk = Buffer.isBuffer(data) ? data : Buffer.from(data as Uint8Array);
|
|
||||||
if (chunk.byteLength === 0) continue;
|
|
||||||
|
|
||||||
partNumber += 1;
|
|
||||||
totalSizeBytes += chunk.byteLength;
|
|
||||||
hasher.update(chunk);
|
|
||||||
|
|
||||||
const { bytes, compressionAlgorithm } = maybeCompressChunk(
|
|
||||||
chunk,
|
|
||||||
compress,
|
|
||||||
compressionMinSizeBytes,
|
|
||||||
);
|
|
||||||
|
|
||||||
const forwardResult = await telegramService.forwardToStorage(
|
|
||||||
bytes,
|
|
||||||
`${partFileNamePrefix}.part-${partNumber}`,
|
|
||||||
'document',
|
|
||||||
);
|
|
||||||
|
|
||||||
parts.push({
|
|
||||||
partNumber,
|
|
||||||
telegramFileId: forwardResult.telegramFileId,
|
|
||||||
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
|
|
||||||
storageMessageId: forwardResult.storageMessageId,
|
|
||||||
sizeBytes: chunk.byteLength,
|
|
||||||
storedSizeBytes: bytes.byteLength,
|
|
||||||
compressionAlgorithm,
|
|
||||||
etag: computeHash(chunk),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
parts,
|
|
||||||
fileHash: hasher.digest('hex'),
|
|
||||||
totalSizeBytes,
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates a factory function for the upload-file use case.
|
* Creates a factory function for the upload-file use case.
|
||||||
*
|
*
|
||||||
@@ -200,9 +58,9 @@ const uploadFileInTelegramChunks = async (
|
|||||||
* 1. Checks for an existing file with the same SHA-256 hash (deduplication).
|
* 1. Checks for an existing file with the same SHA-256 hash (deduplication).
|
||||||
* 2. Normalises the file name and MIME type based on magic bytes.
|
* 2. Normalises the file name and MIME type based on magic bytes.
|
||||||
* 3. Validates the file size against Telegram type-specific limits.
|
* 3. Validates the file size against Telegram type-specific limits.
|
||||||
* 4. Chooses a storage strategy — chunked (for files exceeding the chunk
|
* 4. Chooses a storage strategy — chunked (delegated to ChunkedStorage) or
|
||||||
* threshold) or single-message upload.
|
* single-message upload.
|
||||||
* 5. Persists the file record (and, for chunked uploads, part records).
|
* 5. Persists the file record.
|
||||||
* 6. Builds and returns the public `UploadOutput` DTO.
|
* 6. Builds and returns the public `UploadOutput` DTO.
|
||||||
*
|
*
|
||||||
* @param deps - The injected dependencies.
|
* @param deps - The injected dependencies.
|
||||||
@@ -241,75 +99,28 @@ export function createUploadFileUseCase(deps: UploadFileUseCaseDeps) {
|
|||||||
throw new Error(`File size exceeds ${fileType} limit`);
|
throw new Error(`File size exceeds ${fileType} limit`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 4. Upload — chunked for files above the threshold, single otherwise
|
// 4. Upload — chunked via ChunkedStorage for files above the threshold
|
||||||
if (input.sizeBytes > deps.config.telegramChunkSizeBytes) {
|
if (input.sizeBytes > deps.config.telegramChunkSizeBytes) {
|
||||||
// Chunked upload path
|
const uploadedFile = await deps.chunkedStorage.storeFileInTelegramChunks({
|
||||||
const chunkResult = await uploadFileInTelegramChunks(
|
tempPath: input.tempPath,
|
||||||
input.tempPath,
|
partFileNamePrefix: `direct-${input.fileHash.slice(0, 16)}`,
|
||||||
`direct-${input.fileHash.slice(0, 16)}`,
|
|
||||||
deps.config.telegramChunkSizeBytes,
|
|
||||||
deps.config.compressChunkedUploads,
|
|
||||||
deps.config.chunkCompressionMinSizeBytes,
|
|
||||||
deps.telegramService,
|
|
||||||
);
|
|
||||||
|
|
||||||
const firstPart = chunkResult.parts[0];
|
|
||||||
if (!firstPart) {
|
|
||||||
throw new Error('Chunked upload produced no parts');
|
|
||||||
}
|
|
||||||
|
|
||||||
const fileId = randomUUID();
|
|
||||||
const publicId = nanoid();
|
|
||||||
|
|
||||||
const newFile = await deps.fileRepo.create({
|
|
||||||
publicId,
|
|
||||||
telegramFileId: firstPart.telegramFileId,
|
|
||||||
telegramFileUniqueId: firstPart.telegramFileUniqueId,
|
|
||||||
storageChatId: deps.config.storageChatId,
|
|
||||||
storageMessageId: firstPart.storageMessageId,
|
|
||||||
fileName: finalFileName,
|
fileName: finalFileName,
|
||||||
mimeType,
|
mimeType,
|
||||||
sizeBytes: chunkResult.totalSizeBytes,
|
sizeBytes: input.sizeBytes,
|
||||||
fileType,
|
fileType,
|
||||||
uploaderId: input.uploaderId ?? 0,
|
uploaderId: input.uploaderId ?? 0,
|
||||||
fileHash: chunkResult.fileHash,
|
bucketId: input.bucketId,
|
||||||
archiveTelegramFileId: null,
|
s3Key: input.s3Key,
|
||||||
archiveStorageMessageId: null,
|
|
||||||
archiveFileName: null,
|
|
||||||
archiveEntryName: null,
|
|
||||||
archiveMimeType: null,
|
|
||||||
archiveSizeBytes: null,
|
|
||||||
bucketId: input.bucketId ?? null,
|
|
||||||
s3Key: input.s3Key ?? null,
|
|
||||||
storageBackend: 'chunked',
|
|
||||||
isDeleted: false,
|
|
||||||
multipartUploadId: null,
|
|
||||||
partCount: chunkResult.parts.length,
|
|
||||||
});
|
});
|
||||||
|
|
||||||
const fileParts: NewFilePart[] = chunkResult.parts.map((part) => ({
|
|
||||||
fileId,
|
|
||||||
partNumber: part.partNumber,
|
|
||||||
telegramFileId: part.telegramFileId,
|
|
||||||
telegramFileUniqueId: part.telegramFileUniqueId,
|
|
||||||
storageChatId: deps.config.storageChatId,
|
|
||||||
storageMessageId: part.storageMessageId,
|
|
||||||
sizeBytes: part.sizeBytes,
|
|
||||||
storedSizeBytes: part.storedSizeBytes,
|
|
||||||
compressionAlgorithm: part.compressionAlgorithm,
|
|
||||||
etag: part.etag,
|
|
||||||
}));
|
|
||||||
|
|
||||||
await deps.filePartRepo.insert(fileParts);
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
publicId: newFile.publicId,
|
publicId: uploadedFile.publicId,
|
||||||
fileName: newFile.fileName,
|
fileName: uploadedFile.fileName,
|
||||||
mimeType: newFile.mimeType,
|
mimeType: uploadedFile.mimeType,
|
||||||
sizeBytes: newFile.sizeBytes,
|
sizeBytes: uploadedFile.sizeBytes,
|
||||||
fileType: newFile.fileType,
|
fileType: uploadedFile.fileType,
|
||||||
createdAt: newFile.createdAt,
|
createdAt: uploadedFile.createdAt,
|
||||||
downloadUrl: `${deps.config.baseUrl}/f/${newFile.publicId}`,
|
downloadUrl: `${deps.config.baseUrl}/f/${uploadedFile.publicId}`,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -322,7 +133,8 @@ export function createUploadFileUseCase(deps: UploadFileUseCaseDeps) {
|
|||||||
|
|
||||||
const singlePublicId = nanoid();
|
const singlePublicId = nanoid();
|
||||||
|
|
||||||
const createdFile = await deps.fileRepo.create({
|
const createdFile = await deps.fileRepo.create(
|
||||||
|
buildNewFile({
|
||||||
publicId: singlePublicId,
|
publicId: singlePublicId,
|
||||||
telegramFileId: forwardResult.telegramFileId,
|
telegramFileId: forwardResult.telegramFileId,
|
||||||
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
|
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
|
||||||
@@ -332,21 +144,13 @@ export function createUploadFileUseCase(deps: UploadFileUseCaseDeps) {
|
|||||||
mimeType,
|
mimeType,
|
||||||
sizeBytes: input.sizeBytes,
|
sizeBytes: input.sizeBytes,
|
||||||
fileType,
|
fileType,
|
||||||
uploaderId: input.uploaderId ?? 0,
|
|
||||||
fileHash: input.fileHash,
|
|
||||||
archiveTelegramFileId: null,
|
|
||||||
archiveStorageMessageId: null,
|
|
||||||
archiveFileName: null,
|
|
||||||
archiveEntryName: null,
|
|
||||||
archiveMimeType: null,
|
|
||||||
archiveSizeBytes: null,
|
|
||||||
bucketId: input.bucketId ?? null,
|
|
||||||
s3Key: input.s3Key ?? null,
|
|
||||||
storageBackend: 'telegram',
|
storageBackend: 'telegram',
|
||||||
isDeleted: false,
|
uploaderId: input.uploaderId,
|
||||||
multipartUploadId: null,
|
fileHash: input.fileHash,
|
||||||
partCount: null,
|
bucketId: input.bucketId,
|
||||||
});
|
s3Key: input.s3Key,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
publicId: createdFile.publicId,
|
publicId: createdFile.publicId,
|
||||||
|
|||||||
@@ -1,3 +0,0 @@
|
|||||||
import { config } from '../env';
|
|
||||||
|
|
||||||
export { config };
|
|
||||||
@@ -1,81 +0,0 @@
|
|||||||
import { sql } from 'drizzle-orm';
|
|
||||||
import { db } from './index';
|
|
||||||
|
|
||||||
export interface Bucket {
|
|
||||||
id: string;
|
|
||||||
name: string;
|
|
||||||
createdAt: Date;
|
|
||||||
updatedAt: Date;
|
|
||||||
}
|
|
||||||
|
|
||||||
type QueryRow = Record<string, unknown>;
|
|
||||||
type QueryResult = QueryRow[];
|
|
||||||
|
|
||||||
export const createBucket = async (name: string): Promise<Bucket> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`INSERT INTO buckets (name) VALUES (${name}) RETURNING id, name, created_at, updated_at`,
|
|
||||||
)) as unknown as QueryResult;
|
|
||||||
const row = result[0]!;
|
|
||||||
return {
|
|
||||||
id: row.id as string,
|
|
||||||
name: row.name as string,
|
|
||||||
createdAt: new Date(row.created_at as string),
|
|
||||||
updatedAt: new Date(row.updated_at as string),
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export const findBucketByName = async (name: string): Promise<Bucket | null> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`SELECT id, name, created_at, updated_at FROM buckets WHERE name = ${name}`,
|
|
||||||
)) as unknown as QueryResult;
|
|
||||||
if (result.length === 0) return null;
|
|
||||||
const row = result[0]!;
|
|
||||||
return {
|
|
||||||
id: row.id as string,
|
|
||||||
name: row.name as string,
|
|
||||||
createdAt: new Date(row.created_at as string),
|
|
||||||
updatedAt: new Date(row.updated_at as string),
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export const listBuckets = async (): Promise<Bucket[]> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`SELECT id, name, created_at, updated_at FROM buckets ORDER BY name`,
|
|
||||||
)) as unknown as QueryResult;
|
|
||||||
return result.map((row) => ({
|
|
||||||
id: row.id as string,
|
|
||||||
name: row.name as string,
|
|
||||||
createdAt: new Date(row.created_at as string),
|
|
||||||
updatedAt: new Date(row.updated_at as string),
|
|
||||||
}));
|
|
||||||
};
|
|
||||||
|
|
||||||
export const deleteBucket = async (name: string): Promise<boolean> => {
|
|
||||||
// Cascade-delete rows that hold FK references to the bucket
|
|
||||||
await db
|
|
||||||
.execute(
|
|
||||||
sql`DELETE FROM multipart_parts WHERE upload_id IN (SELECT upload_id FROM multipart_uploads WHERE bucket_id IN (SELECT id FROM buckets WHERE name = ${name}))`,
|
|
||||||
)
|
|
||||||
.catch(() => {});
|
|
||||||
await db
|
|
||||||
.execute(
|
|
||||||
sql`DELETE FROM multipart_uploads WHERE bucket_id IN (SELECT id FROM buckets WHERE name = ${name})`,
|
|
||||||
)
|
|
||||||
.catch(() => {});
|
|
||||||
await db
|
|
||||||
.execute(
|
|
||||||
sql`DELETE FROM files WHERE bucket_id IN (SELECT id FROM buckets WHERE name = ${name})`,
|
|
||||||
)
|
|
||||||
.catch(() => {});
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`DELETE FROM buckets WHERE name = ${name}`,
|
|
||||||
)) as unknown as QueryResult;
|
|
||||||
return result.length > 0;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const bucketExists = async (name: string): Promise<boolean> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`SELECT 1 FROM buckets WHERE name = ${name}`,
|
|
||||||
)) as unknown as QueryResult;
|
|
||||||
return result.length > 0;
|
|
||||||
};
|
|
||||||
@@ -1,97 +0,0 @@
|
|||||||
import { sql } from 'drizzle-orm';
|
|
||||||
import { db } from './index';
|
|
||||||
|
|
||||||
export type CompressionAlgorithm = 'gzip' | null;
|
|
||||||
|
|
||||||
export interface FilePart {
|
|
||||||
id: number;
|
|
||||||
fileId: string;
|
|
||||||
partNumber: number;
|
|
||||||
telegramFileId: string;
|
|
||||||
telegramFileUniqueId: string;
|
|
||||||
storageChatId: number;
|
|
||||||
storageMessageId: number;
|
|
||||||
sizeBytes: number;
|
|
||||||
storedSizeBytes: number;
|
|
||||||
compressionAlgorithm: CompressionAlgorithm;
|
|
||||||
etag: string;
|
|
||||||
createdAt: Date;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type NewFilePartInput = Omit<FilePart, 'id' | 'createdAt'>;
|
|
||||||
|
|
||||||
const toNumber = (value: unknown): number => Number(value ?? 0);
|
|
||||||
|
|
||||||
const mapRowToFilePart = (row: Record<string, unknown>): FilePart => ({
|
|
||||||
id: toNumber(row.id),
|
|
||||||
fileId: row.file_id as string,
|
|
||||||
partNumber: toNumber(row.part_number),
|
|
||||||
telegramFileId: row.telegram_file_id as string,
|
|
||||||
telegramFileUniqueId: row.telegram_file_unique_id as string,
|
|
||||||
storageChatId: toNumber(row.storage_chat_id),
|
|
||||||
storageMessageId: toNumber(row.storage_message_id),
|
|
||||||
sizeBytes: toNumber(row.size_bytes),
|
|
||||||
storedSizeBytes: toNumber(row.stored_size_bytes),
|
|
||||||
compressionAlgorithm: (row.compression_algorithm as CompressionAlgorithm) || null,
|
|
||||||
etag: row.etag as string,
|
|
||||||
createdAt: new Date(row.created_at as string),
|
|
||||||
});
|
|
||||||
|
|
||||||
export const insertFileParts = async (parts: NewFilePartInput[]): Promise<void> => {
|
|
||||||
for (const part of parts) {
|
|
||||||
await db.execute(
|
|
||||||
sql`INSERT INTO file_parts (
|
|
||||||
file_id,
|
|
||||||
part_number,
|
|
||||||
telegram_file_id,
|
|
||||||
telegram_file_unique_id,
|
|
||||||
storage_chat_id,
|
|
||||||
storage_message_id,
|
|
||||||
size_bytes,
|
|
||||||
stored_size_bytes,
|
|
||||||
compression_algorithm,
|
|
||||||
etag
|
|
||||||
) VALUES (
|
|
||||||
${part.fileId}::uuid,
|
|
||||||
${part.partNumber},
|
|
||||||
${part.telegramFileId},
|
|
||||||
${part.telegramFileUniqueId},
|
|
||||||
${part.storageChatId},
|
|
||||||
${part.storageMessageId},
|
|
||||||
${part.sizeBytes},
|
|
||||||
${part.storedSizeBytes},
|
|
||||||
${part.compressionAlgorithm},
|
|
||||||
${part.etag}
|
|
||||||
)`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const listFileParts = async (fileId: string): Promise<FilePart[]> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`SELECT id,
|
|
||||||
file_id,
|
|
||||||
part_number,
|
|
||||||
telegram_file_id,
|
|
||||||
telegram_file_unique_id,
|
|
||||||
storage_chat_id,
|
|
||||||
storage_message_id,
|
|
||||||
size_bytes,
|
|
||||||
stored_size_bytes,
|
|
||||||
compression_algorithm,
|
|
||||||
etag,
|
|
||||||
created_at
|
|
||||||
FROM file_parts
|
|
||||||
WHERE file_id = ${fileId}::uuid
|
|
||||||
ORDER BY part_number`,
|
|
||||||
)) as unknown as Record<string, unknown>[];
|
|
||||||
|
|
||||||
return result.map(mapRowToFilePart);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const countFileParts = async (fileId: string): Promise<number> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`SELECT COUNT(*) AS count FROM file_parts WHERE file_id = ${fileId}::uuid`,
|
|
||||||
)) as unknown as Record<string, unknown>[];
|
|
||||||
return toNumber(result[0]?.count);
|
|
||||||
};
|
|
||||||
@@ -1,143 +0,0 @@
|
|||||||
import { and, eq, sql } from 'drizzle-orm';
|
|
||||||
import { db, files as fileSchema } from './index';
|
|
||||||
import type { File } from './schema';
|
|
||||||
|
|
||||||
export interface S3FileRecord extends File {
|
|
||||||
bucketId: string;
|
|
||||||
s3Key: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const findFileByBucketAndKey = async (
|
|
||||||
bucketId: string,
|
|
||||||
s3Key: string,
|
|
||||||
): Promise<File | null> => {
|
|
||||||
const result = await db
|
|
||||||
.select()
|
|
||||||
.from(fileSchema)
|
|
||||||
.where(
|
|
||||||
and(
|
|
||||||
eq(fileSchema.bucketId, bucketId),
|
|
||||||
eq(fileSchema.s3Key, s3Key),
|
|
||||||
eq(fileSchema.isDeleted, false),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.limit(1);
|
|
||||||
return result[0] || null;
|
|
||||||
};
|
|
||||||
|
|
||||||
const mapDbRowToS3Record = (row: Record<string, unknown>): S3FileRecord => {
|
|
||||||
return {
|
|
||||||
id: row.id as string,
|
|
||||||
publicId: row.public_id as string,
|
|
||||||
telegramFileId: row.telegram_file_id as string,
|
|
||||||
telegramFileUniqueId: row.telegram_file_unique_id as string,
|
|
||||||
storageChatId: toNumber(row.storage_chat_id),
|
|
||||||
storageMessageId: toNumber(row.storage_message_id),
|
|
||||||
fileName: row.file_name as string,
|
|
||||||
mimeType: row.mime_type as string,
|
|
||||||
sizeBytes: toNumber(row.size_bytes),
|
|
||||||
fileType: row.file_type as string,
|
|
||||||
uploaderId: toNumber(row.uploader_id),
|
|
||||||
fileHash: row.file_hash as string | null,
|
|
||||||
archiveTelegramFileId: row.archive_telegram_file_id as string | null,
|
|
||||||
archiveStorageMessageId:
|
|
||||||
row.archive_storage_message_id === null ? null : toNumber(row.archive_storage_message_id),
|
|
||||||
archiveFileName: row.archive_file_name as string | null,
|
|
||||||
archiveEntryName: row.archive_entry_name as string | null,
|
|
||||||
archiveMimeType: row.archive_mime_type as string | null,
|
|
||||||
archiveSizeBytes: row.archive_size_bytes === null ? null : toNumber(row.archive_size_bytes),
|
|
||||||
bucketId: row.bucket_id as string,
|
|
||||||
s3Key: row.s3_key as string,
|
|
||||||
storageBackend: (row.storage_backend as string) || 'telegram',
|
|
||||||
isDeleted: row.is_deleted as boolean,
|
|
||||||
multipartUploadId: row.multipart_upload_id as string | null,
|
|
||||||
partCount:
|
|
||||||
row.part_count === null || row.part_count === undefined ? null : toNumber(row.part_count),
|
|
||||||
createdAt: new Date(row.created_at as string),
|
|
||||||
updatedAt: new Date(row.updated_at as string),
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
const escapeLike = (s: string): string => s.replace(/[%_\\]/g, '\\$&');
|
|
||||||
|
|
||||||
const toNumber = (value: unknown): number => Number(value ?? 0);
|
|
||||||
|
|
||||||
export const listObjectsByPrefix = async (
|
|
||||||
bucketId: string,
|
|
||||||
prefix: string,
|
|
||||||
delimiter: string | null,
|
|
||||||
maxKeys: number,
|
|
||||||
startAfter: string | null,
|
|
||||||
): Promise<{ objects: S3FileRecord[]; prefixes: string[] }> => {
|
|
||||||
let query = prefix
|
|
||||||
? sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false AND s3_key LIKE ${`${escapeLike(prefix)}%`}`
|
|
||||||
: sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false`;
|
|
||||||
|
|
||||||
if (startAfter) {
|
|
||||||
query = sql`${query} AND s3_key > ${startAfter}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
query = sql`${query} ORDER BY s3_key LIMIT ${maxKeys + 1}`;
|
|
||||||
|
|
||||||
const rawResult = (await db.execute(query)) as unknown as Record<string, unknown>[];
|
|
||||||
|
|
||||||
if (delimiter === '/') {
|
|
||||||
const prefixSet = new Set<string>();
|
|
||||||
const objects: S3FileRecord[] = [];
|
|
||||||
|
|
||||||
for (const row of rawResult) {
|
|
||||||
const s3Key = row.s3_key as string;
|
|
||||||
const relativeKey = s3Key.substring(prefix.length);
|
|
||||||
const slashIndex = relativeKey.indexOf('/');
|
|
||||||
if (slashIndex >= 0) {
|
|
||||||
const folderPrefix = prefix + relativeKey.substring(0, slashIndex + 1);
|
|
||||||
if (folderPrefix !== prefix) {
|
|
||||||
prefixSet.add(folderPrefix);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
objects.push(mapDbRowToS3Record(row));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
objects: objects.slice(0, maxKeys),
|
|
||||||
prefixes: Array.from(prefixSet).sort(),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
objects: rawResult.slice(0, maxKeys).map(mapDbRowToS3Record),
|
|
||||||
prefixes: [],
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export const softDeleteFile = async (bucketId: string, s3Key: string): Promise<boolean> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`UPDATE files SET is_deleted = true WHERE bucket_id = ${bucketId}::uuid AND s3_key = ${s3Key} RETURNING id`,
|
|
||||||
)) as unknown as Record<string, unknown>[];
|
|
||||||
return result.length > 0;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const softDeleteFilesBatch = async (bucketId: string, keys: string[]): Promise<number> => {
|
|
||||||
let deleted = 0;
|
|
||||||
for (const key of keys) {
|
|
||||||
const ok = await softDeleteFile(bucketId, key);
|
|
||||||
if (ok) deleted++;
|
|
||||||
}
|
|
||||||
return deleted;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const countBucketObjects = async (bucketId: string): Promise<number> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`SELECT count(*) as count FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false`,
|
|
||||||
)) as unknown as Record<string, unknown>[];
|
|
||||||
return Number(result[0]?.count || 0);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const findOrphanFilesByBucket = async (bucketId: string): Promise<File[]> => {
|
|
||||||
return await db
|
|
||||||
.select()
|
|
||||||
.from(fileSchema)
|
|
||||||
.where(and(eq(fileSchema.bucketId, bucketId), eq(fileSchema.isDeleted, true)))
|
|
||||||
.limit(100);
|
|
||||||
};
|
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
import { eq } from 'drizzle-orm';
|
|
||||||
import { db, files as fileSchema } from './index';
|
|
||||||
import type { File } from './schema';
|
|
||||||
|
|
||||||
export const findFileByHash = async (hash: string): Promise<File | null> => {
|
|
||||||
const result = await db.select().from(fileSchema).where(eq(fileSchema.fileHash, hash)).limit(1);
|
|
||||||
return result[0] || null;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const findFileByPublicId = async (publicId: string): Promise<File | null> => {
|
|
||||||
const result = await db
|
|
||||||
.select()
|
|
||||||
.from(fileSchema)
|
|
||||||
.where(eq(fileSchema.publicId, publicId))
|
|
||||||
.limit(1);
|
|
||||||
return result[0] || null;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const findFileByUniqueId = async (telegramFileUniqueId: string): Promise<File | null> => {
|
|
||||||
const result = await db
|
|
||||||
.select()
|
|
||||||
.from(fileSchema)
|
|
||||||
.where(eq(fileSchema.telegramFileUniqueId, telegramFileUniqueId))
|
|
||||||
.limit(1);
|
|
||||||
return result[0] || null;
|
|
||||||
};
|
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
import { drizzle } from 'drizzle-orm/postgres-js';
|
|
||||||
import postgres from 'postgres';
|
|
||||||
import { fileParts, files } from './schema';
|
|
||||||
|
|
||||||
const client = postgres(process.env.DATABASE_URL!, {
|
|
||||||
max: 10,
|
|
||||||
idle_timeout: 20,
|
|
||||||
connect_timeout: 10,
|
|
||||||
});
|
|
||||||
|
|
||||||
export const db = drizzle(client, { schema: { fileParts, files } });
|
|
||||||
export { fileParts, files };
|
|
||||||
export default db;
|
|
||||||
@@ -1,53 +0,0 @@
|
|||||||
import postgres from 'postgres';
|
|
||||||
import { config } from '../env';
|
|
||||||
import { getErrorMessage } from '../utils/file';
|
|
||||||
import logger from '../utils/logger';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Run raw SQL migration from schema.sql.
|
|
||||||
* Safe to call multiple times — all statements use IF NOT EXISTS.
|
|
||||||
*/
|
|
||||||
export const runMigration = async (): Promise<void> => {
|
|
||||||
// In compiled dist: import.meta.dir = .../dist/
|
|
||||||
// In source via bun --hot: import.meta.dir = .../src/db/
|
|
||||||
const dir = import.meta.dir || '';
|
|
||||||
const candidates = [
|
|
||||||
`${dir}/../../schema.sql`, // from dist/
|
|
||||||
`${dir}/../schema.sql`, // from src/ (bun --hot src/index.ts)
|
|
||||||
`${dir}/../schema.sql`, // from src/db/ (bun --hot src/db/migrate.ts)
|
|
||||||
`${dir}/schema.sql`, // from src/ (bun run db:migrate)
|
|
||||||
];
|
|
||||||
|
|
||||||
let schemaSql: string | null = null;
|
|
||||||
for (const p of candidates) {
|
|
||||||
const file = Bun.file(p);
|
|
||||||
const exists = await file.exists();
|
|
||||||
if (exists) {
|
|
||||||
schemaSql = await file.text();
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!schemaSql) {
|
|
||||||
logger.error(`Migration failed: schema.sql not found (tried ${candidates.join(', ')})`);
|
|
||||||
process.exitCode = 1;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const sql = postgres(config.databaseUrl, { max: 1 });
|
|
||||||
|
|
||||||
try {
|
|
||||||
await sql.unsafe(schemaSql);
|
|
||||||
logger.info('Database migration completed');
|
|
||||||
} catch (error: unknown) {
|
|
||||||
logger.error('Database migration failed', { error: getErrorMessage(error) });
|
|
||||||
process.exitCode = 1;
|
|
||||||
} finally {
|
|
||||||
await sql.end();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// When run directly: `bun src/db/migrate.ts` or `bun dist/migrate.js`
|
|
||||||
if (import.meta.path === Bun.main) {
|
|
||||||
await runMigration();
|
|
||||||
}
|
|
||||||
@@ -1,137 +0,0 @@
|
|||||||
import { sql } from 'drizzle-orm';
|
|
||||||
import { nanoid } from 'nanoid';
|
|
||||||
import { db } from './index';
|
|
||||||
|
|
||||||
export interface MultipartUpload {
|
|
||||||
uploadId: string;
|
|
||||||
bucketId: string;
|
|
||||||
s3Key: string;
|
|
||||||
initiatedAt: Date;
|
|
||||||
status: string;
|
|
||||||
initiatedBy: string;
|
|
||||||
contentType: string | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface MultipartPart {
|
|
||||||
id: number;
|
|
||||||
uploadId: string;
|
|
||||||
partNumber: number;
|
|
||||||
telegramFileId: string;
|
|
||||||
telegramFileUniqueId: string;
|
|
||||||
storageMessageId: number;
|
|
||||||
sizeBytes: number;
|
|
||||||
etag: string;
|
|
||||||
createdAt: Date;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const createMultipartUpload = async (
|
|
||||||
bucketId: string,
|
|
||||||
s3Key: string,
|
|
||||||
initiatedBy: string,
|
|
||||||
contentType?: string | null,
|
|
||||||
): Promise<string> => {
|
|
||||||
const uploadId = nanoid(32);
|
|
||||||
await db.execute(
|
|
||||||
contentType
|
|
||||||
? sql`INSERT INTO multipart_uploads (upload_id, bucket_id, s3_key, initiated_by, content_type) VALUES (${uploadId}, ${bucketId}, ${s3Key}, ${initiatedBy}, ${contentType})`
|
|
||||||
: sql`INSERT INTO multipart_uploads (upload_id, bucket_id, s3_key, initiated_by) VALUES (${uploadId}, ${bucketId}, ${s3Key}, ${initiatedBy})`,
|
|
||||||
);
|
|
||||||
return uploadId;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const findMultipartUpload = async (uploadId: string): Promise<MultipartUpload | null> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, content_type FROM multipart_uploads WHERE upload_id = ${uploadId} AND status = 'in_progress'`,
|
|
||||||
)) as unknown as Record<string, unknown>[];
|
|
||||||
if (result.length === 0) return null;
|
|
||||||
const r = result[0]!;
|
|
||||||
return {
|
|
||||||
uploadId: r.upload_id as string,
|
|
||||||
bucketId: r.bucket_id as string,
|
|
||||||
s3Key: r.s3_key as string,
|
|
||||||
initiatedAt: new Date(r.initiated_at as string),
|
|
||||||
status: r.status as string,
|
|
||||||
initiatedBy: '',
|
|
||||||
contentType: (r.content_type as string | null) || null,
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export const completeMultipartUpload = async (uploadId: string): Promise<void> => {
|
|
||||||
await db.execute(
|
|
||||||
sql`UPDATE multipart_uploads SET status = 'completed' WHERE upload_id = ${uploadId}`,
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const abortMultipartUpload = async (uploadId: string): Promise<void> => {
|
|
||||||
// H7: FK cascade only fires on DELETE, not UPDATE. Delete parts explicitly
|
|
||||||
// before updating the upload status.
|
|
||||||
await db.execute(sql`DELETE FROM multipart_parts WHERE upload_id = ${uploadId}`);
|
|
||||||
await db.execute(
|
|
||||||
sql`UPDATE multipart_uploads SET status = 'aborted' WHERE upload_id = ${uploadId}`,
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const insertMultipartPart = async (
|
|
||||||
part: Omit<MultipartPart, 'id' | 'createdAt'>,
|
|
||||||
): Promise<void> => {
|
|
||||||
await db.execute(
|
|
||||||
sql`INSERT INTO multipart_parts (upload_id, part_number, telegram_file_id, telegram_file_unique_id, storage_message_id, size_bytes, etag)
|
|
||||||
VALUES (${part.uploadId}, ${part.partNumber}, ${part.telegramFileId}, ${part.telegramFileUniqueId}, ${part.storageMessageId}, ${part.sizeBytes}, ${part.etag})`,
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const listMultipartParts = async (uploadId: string): Promise<MultipartPart[]> => {
|
|
||||||
const result = (await db.execute(
|
|
||||||
sql`SELECT id, upload_id, part_number, telegram_file_id, telegram_file_unique_id, storage_message_id, size_bytes, etag, created_at
|
|
||||||
FROM multipart_parts WHERE upload_id = ${uploadId} ORDER BY part_number`,
|
|
||||||
)) as unknown as Record<string, unknown>[];
|
|
||||||
return result.map((r) => ({
|
|
||||||
id: r.id as number,
|
|
||||||
uploadId: r.upload_id as string,
|
|
||||||
partNumber: r.part_number as number,
|
|
||||||
telegramFileId: r.telegram_file_id as string,
|
|
||||||
telegramFileUniqueId: r.telegram_file_unique_id as string,
|
|
||||||
storageMessageId: r.storage_message_id as number,
|
|
||||||
sizeBytes: Number(r.size_bytes),
|
|
||||||
etag: r.etag as string,
|
|
||||||
createdAt: new Date(r.created_at as string),
|
|
||||||
}));
|
|
||||||
};
|
|
||||||
|
|
||||||
const mapRowToMultipartUpload = (r: Record<string, unknown>): MultipartUpload => ({
|
|
||||||
uploadId: r.upload_id as string,
|
|
||||||
bucketId: r.bucket_id as string,
|
|
||||||
s3Key: r.s3_key as string,
|
|
||||||
initiatedAt: new Date(r.initiated_at as string),
|
|
||||||
status: r.status as string,
|
|
||||||
initiatedBy: (r.initiated_by as string | null) || '',
|
|
||||||
contentType: (r.content_type as string | null) || null,
|
|
||||||
});
|
|
||||||
|
|
||||||
export const listMultipartUploadsByBucket = async (
|
|
||||||
bucketId: string,
|
|
||||||
maxUploads: number,
|
|
||||||
keyMarker: string | null,
|
|
||||||
): Promise<{ uploads: MultipartUpload[]; isTruncated: boolean; nextKeyMarker: string | null }> => {
|
|
||||||
const limit = Math.min(Math.max(maxUploads || 1000, 1), 1000);
|
|
||||||
const result = (await db.execute(
|
|
||||||
keyMarker
|
|
||||||
? sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, initiated_by
|
|
||||||
FROM multipart_uploads
|
|
||||||
WHERE bucket_id = ${bucketId}::uuid AND status = 'in_progress' AND s3_key > ${keyMarker}
|
|
||||||
ORDER BY s3_key, initiated_at
|
|
||||||
LIMIT ${limit + 1}`
|
|
||||||
: sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, initiated_by
|
|
||||||
FROM multipart_uploads
|
|
||||||
WHERE bucket_id = ${bucketId}::uuid AND status = 'in_progress'
|
|
||||||
ORDER BY s3_key, initiated_at
|
|
||||||
LIMIT ${limit + 1}`,
|
|
||||||
)) as unknown as Record<string, unknown>[];
|
|
||||||
|
|
||||||
const uploads = result.slice(0, limit).map(mapRowToMultipartUpload);
|
|
||||||
return {
|
|
||||||
uploads,
|
|
||||||
isTruncated: result.length > limit,
|
|
||||||
nextKeyMarker: result.length > limit ? uploads.at(-1)?.s3Key || null : null,
|
|
||||||
};
|
|
||||||
};
|
|
||||||
@@ -1,71 +0,0 @@
|
|||||||
import { eq, type InferInsertModel, type InferSelectModel } from 'drizzle-orm';
|
|
||||||
import {
|
|
||||||
bigint,
|
|
||||||
boolean,
|
|
||||||
integer,
|
|
||||||
pgTable,
|
|
||||||
serial,
|
|
||||||
text,
|
|
||||||
timestamp,
|
|
||||||
uniqueIndex,
|
|
||||||
uuid,
|
|
||||||
} from 'drizzle-orm/pg-core';
|
|
||||||
|
|
||||||
export const files = pgTable(
|
|
||||||
'files',
|
|
||||||
{
|
|
||||||
id: uuid('id').primaryKey().defaultRandom(),
|
|
||||||
publicId: text('public_id').unique().notNull(),
|
|
||||||
telegramFileId: text('telegram_file_id').notNull(),
|
|
||||||
telegramFileUniqueId: text('telegram_file_unique_id').notNull(),
|
|
||||||
storageChatId: bigint('storage_chat_id', { mode: 'number' }).notNull(),
|
|
||||||
storageMessageId: bigint('storage_message_id', { mode: 'number' }).notNull(),
|
|
||||||
fileName: text('file_name').notNull(),
|
|
||||||
mimeType: text('mime_type').notNull(),
|
|
||||||
sizeBytes: bigint('size_bytes', { mode: 'number' }).notNull(),
|
|
||||||
fileType: text('file_type').notNull(),
|
|
||||||
uploaderId: bigint('uploader_id', { mode: 'number' }).notNull(),
|
|
||||||
fileHash: text('file_hash'),
|
|
||||||
archiveTelegramFileId: text('archive_telegram_file_id'),
|
|
||||||
archiveStorageMessageId: bigint('archive_storage_message_id', { mode: 'number' }),
|
|
||||||
archiveFileName: text('archive_file_name'),
|
|
||||||
archiveEntryName: text('archive_entry_name'),
|
|
||||||
archiveMimeType: text('archive_mime_type'),
|
|
||||||
archiveSizeBytes: bigint('archive_size_bytes', { mode: 'number' }),
|
|
||||||
bucketId: text('bucket_id'),
|
|
||||||
s3Key: text('s3_key'),
|
|
||||||
storageBackend: text('storage_backend').default('telegram'),
|
|
||||||
isDeleted: boolean('is_deleted').default(false),
|
|
||||||
multipartUploadId: text('multipart_upload_id'),
|
|
||||||
partCount: integer('part_count'),
|
|
||||||
createdAt: timestamp('created_at').defaultNow().notNull(),
|
|
||||||
updatedAt: timestamp('updated_at').defaultNow().notNull(),
|
|
||||||
},
|
|
||||||
(table) => ({
|
|
||||||
// H2: Prevent TOCTOU race on concurrent PUT — only one active (non-deleted)
|
|
||||||
// object per (bucket_id, s3_key) pair.
|
|
||||||
activeObjectIdx: uniqueIndex('active_object_idx')
|
|
||||||
.on(table.bucketId, table.s3Key)
|
|
||||||
.where(eq(table.isDeleted, false)),
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
export const fileParts = pgTable('file_parts', {
|
|
||||||
id: serial('id').primaryKey(),
|
|
||||||
fileId: uuid('file_id').notNull(),
|
|
||||||
partNumber: integer('part_number').notNull(),
|
|
||||||
telegramFileId: text('telegram_file_id').notNull(),
|
|
||||||
telegramFileUniqueId: text('telegram_file_unique_id').notNull(),
|
|
||||||
storageChatId: bigint('storage_chat_id', { mode: 'number' }).notNull(),
|
|
||||||
storageMessageId: bigint('storage_message_id', { mode: 'number' }).notNull(),
|
|
||||||
sizeBytes: bigint('size_bytes', { mode: 'number' }).notNull(),
|
|
||||||
storedSizeBytes: bigint('stored_size_bytes', { mode: 'number' }).notNull(),
|
|
||||||
compressionAlgorithm: text('compression_algorithm'),
|
|
||||||
etag: text('etag').notNull(),
|
|
||||||
createdAt: timestamp('created_at').defaultNow().notNull(),
|
|
||||||
});
|
|
||||||
|
|
||||||
export type File = InferSelectModel<typeof files>;
|
|
||||||
export type NewFile = InferInsertModel<typeof files>;
|
|
||||||
export type FilePart = InferSelectModel<typeof fileParts>;
|
|
||||||
export type NewFilePart = InferInsertModel<typeof fileParts>;
|
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
/**
|
||||||
|
* Factory function for building NewFile records with sensible defaults.
|
||||||
|
*
|
||||||
|
* Most call sites set the same null defaults for archive/S3/soft-delete fields.
|
||||||
|
* This factory eliminates ~20 lines of boilerplate per call site (~27 sites).
|
||||||
|
*/
|
||||||
|
import type { NewFile } from './file';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Partial input for creating a file record.
|
||||||
|
* Only the required unique fields must be provided; optional fields default to null/0/false.
|
||||||
|
*/
|
||||||
|
export interface FileInput {
|
||||||
|
publicId: string;
|
||||||
|
telegramFileId: string;
|
||||||
|
telegramFileUniqueId: string;
|
||||||
|
storageChatId: number;
|
||||||
|
storageMessageId: number;
|
||||||
|
fileName: string;
|
||||||
|
mimeType: string;
|
||||||
|
sizeBytes: number;
|
||||||
|
fileType: string;
|
||||||
|
storageBackend: string | null;
|
||||||
|
/** Optional overrides */
|
||||||
|
uploaderId?: number;
|
||||||
|
fileHash?: string | null;
|
||||||
|
bucketId?: string | null;
|
||||||
|
s3Key?: string | null;
|
||||||
|
partCount?: number | null;
|
||||||
|
multipartUploadId?: string | null;
|
||||||
|
/** Archive fields (for batch/zip archives) */
|
||||||
|
archiveTelegramFileId?: string | null;
|
||||||
|
archiveStorageMessageId?: number | null;
|
||||||
|
archiveFileName?: string | null;
|
||||||
|
archiveEntryName?: string | null;
|
||||||
|
archiveMimeType?: string | null;
|
||||||
|
archiveSizeBytes?: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build a NewFile record, filling in null/zero defaults for omitted fields.
|
||||||
|
*
|
||||||
|
* @example
|
||||||
|
* ```ts
|
||||||
|
* await fileRepo.create(buildNewFile({
|
||||||
|
* publicId,
|
||||||
|
* telegramFileId: result.telegramFileId,
|
||||||
|
* telegramFileUniqueId: result.telegramFileUniqueId,
|
||||||
|
* storageChatId,
|
||||||
|
* storageMessageId: result.storageMessageId,
|
||||||
|
* fileName: input.fileName,
|
||||||
|
* mimeType,
|
||||||
|
* sizeBytes: input.sizeBytes,
|
||||||
|
* fileType,
|
||||||
|
* storageBackend: 'telegram',
|
||||||
|
* uploaderId: input.uploaderId,
|
||||||
|
* fileHash: input.fileHash,
|
||||||
|
* }));
|
||||||
|
* ```
|
||||||
|
*/
|
||||||
|
export const buildNewFile = (input: FileInput): NewFile => ({
|
||||||
|
publicId: input.publicId,
|
||||||
|
telegramFileId: input.telegramFileId,
|
||||||
|
telegramFileUniqueId: input.telegramFileUniqueId,
|
||||||
|
storageChatId: input.storageChatId,
|
||||||
|
storageMessageId: input.storageMessageId,
|
||||||
|
fileName: input.fileName,
|
||||||
|
mimeType: input.mimeType,
|
||||||
|
sizeBytes: input.sizeBytes,
|
||||||
|
fileType: input.fileType,
|
||||||
|
uploaderId: input.uploaderId ?? 0,
|
||||||
|
fileHash: input.fileHash ?? null,
|
||||||
|
archiveTelegramFileId: input.archiveTelegramFileId ?? null,
|
||||||
|
archiveStorageMessageId: input.archiveStorageMessageId ?? null,
|
||||||
|
archiveFileName: input.archiveFileName ?? null,
|
||||||
|
archiveEntryName: input.archiveEntryName ?? null,
|
||||||
|
archiveMimeType: input.archiveMimeType ?? null,
|
||||||
|
archiveSizeBytes: input.archiveSizeBytes ?? null,
|
||||||
|
bucketId: input.bucketId ?? null,
|
||||||
|
s3Key: input.s3Key ?? null,
|
||||||
|
storageBackend: input.storageBackend,
|
||||||
|
isDeleted: false,
|
||||||
|
multipartUploadId: input.multipartUploadId ?? null,
|
||||||
|
partCount: input.partCount ?? null,
|
||||||
|
});
|
||||||
+48
-10
@@ -1,8 +1,11 @@
|
|||||||
import logger from './utils/logger';
|
import logger from './shared/logger/index';
|
||||||
|
import { TELEGRAM_CHUNK_SIZE_MAX_BYTES } from './shared/utils/validation';
|
||||||
|
|
||||||
interface AppConfig {
|
interface AppConfig {
|
||||||
botToken: string;
|
/** All bot tokens merged from BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS fallback) */
|
||||||
additionalBotTokens: string[];
|
botTokens: string[];
|
||||||
|
/** Per-bot concurrency for Telegram API calls (default 1). */
|
||||||
|
telegramBotConcurrency: number;
|
||||||
storageChatId: number;
|
storageChatId: number;
|
||||||
baseUrl: string;
|
baseUrl: string;
|
||||||
databaseUrl: string;
|
databaseUrl: string;
|
||||||
@@ -28,8 +31,21 @@ interface AppConfig {
|
|||||||
s3VhostDomains: string[];
|
s3VhostDomains: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Validate bot tokens: BOT_TOKENS (new) or fallback to BOT_TOKEN + ADDITIONAL_BOT_TOKENS
|
||||||
|
const botTokensRaw =
|
||||||
|
process.env.BOT_TOKENS ||
|
||||||
|
[process.env.BOT_TOKEN, process.env.ADDITIONAL_BOT_TOKENS].filter(Boolean).join(',');
|
||||||
|
|
||||||
|
if (!botTokensRaw) {
|
||||||
|
logger.error(
|
||||||
|
'Missing required environment variables: BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS)',
|
||||||
|
);
|
||||||
|
throw new Error(
|
||||||
|
'Missing environment variables: BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS)',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const requiredEnv = {
|
const requiredEnv = {
|
||||||
BOT_TOKEN: process.env.BOT_TOKEN,
|
|
||||||
STORAGE_CHANNEL_ID: process.env.STORAGE_CHANNEL_ID,
|
STORAGE_CHANNEL_ID: process.env.STORAGE_CHANNEL_ID,
|
||||||
BASE_URL: process.env.BASE_URL,
|
BASE_URL: process.env.BASE_URL,
|
||||||
DATABASE_URL: process.env.DATABASE_URL,
|
DATABASE_URL: process.env.DATABASE_URL,
|
||||||
@@ -92,13 +108,36 @@ const maskSecret = (value: string): string => {
|
|||||||
const maskDatabaseUrl = (value: string): string =>
|
const maskDatabaseUrl = (value: string): string =>
|
||||||
value.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:***@');
|
value.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:***@');
|
||||||
|
|
||||||
|
// Fail-fast guard for TELEGRAM_CHUNK_SIZE_BYTES: chunked uploads store every
|
||||||
|
// part as a Telegram document and later resolve it via getFile, which only
|
||||||
|
// supports files up to 20 MB ("Bad Request: file is too big" above that).
|
||||||
|
// A chunk above the limit makes every part undownloadable — refuse to start
|
||||||
|
// instead of failing on the first large-file download.
|
||||||
|
const telegramChunkSizeBytes = parseNumber(
|
||||||
|
process.env.TELEGRAM_CHUNK_SIZE_BYTES,
|
||||||
|
TELEGRAM_CHUNK_SIZE_MAX_BYTES,
|
||||||
|
);
|
||||||
|
if (telegramChunkSizeBytes > TELEGRAM_CHUNK_SIZE_MAX_BYTES) {
|
||||||
|
logger.error(
|
||||||
|
`TELEGRAM_CHUNK_SIZE_BYTES=${telegramChunkSizeBytes} exceeds the maximum allowed chunk size ` +
|
||||||
|
`${TELEGRAM_CHUNK_SIZE_MAX_BYTES} bytes (${TELEGRAM_CHUNK_SIZE_MAX_BYTES / (1024 * 1024)} MB). ` +
|
||||||
|
'Telegram Bot API getFile cannot download files larger than 20 MB, so every stored part would ' +
|
||||||
|
'be undownloadable ("Bad Request: file is too big"). ' +
|
||||||
|
`Set TELEGRAM_CHUNK_SIZE_BYTES to ${TELEGRAM_CHUNK_SIZE_MAX_BYTES} or lower.`,
|
||||||
|
);
|
||||||
|
throw new Error(
|
||||||
|
`TELEGRAM_CHUNK_SIZE_BYTES=${telegramChunkSizeBytes} exceeds the maximum allowed chunk size ` +
|
||||||
|
`${TELEGRAM_CHUNK_SIZE_MAX_BYTES} bytes (${TELEGRAM_CHUNK_SIZE_MAX_BYTES / (1024 * 1024)} MB)`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export const config: AppConfig = {
|
export const config: AppConfig = {
|
||||||
botToken: process.env.BOT_TOKEN!,
|
botTokens: parseTokens(botTokensRaw),
|
||||||
additionalBotTokens: parseTokens(process.env.ADDITIONAL_BOT_TOKENS),
|
telegramBotConcurrency: parseNumber(process.env.TELEGRAM_BOT_CONCURRENCY, 1),
|
||||||
storageChatId: parseInt(process.env.STORAGE_CHANNEL_ID!, 10),
|
storageChatId: parseInt(process.env.STORAGE_CHANNEL_ID!, 10),
|
||||||
baseUrl: process.env.BASE_URL!,
|
baseUrl: process.env.BASE_URL!,
|
||||||
databaseUrl: process.env.DATABASE_URL!,
|
databaseUrl: process.env.DATABASE_URL!,
|
||||||
port: parseInt(process.env.PORT!, 10) || 3000,
|
port: parseInt(process.env.PORT!, 10) || 4000,
|
||||||
nodeEnv: process.env.NODE_ENV || 'development',
|
nodeEnv: process.env.NODE_ENV || 'development',
|
||||||
logLevel: process.env.LOG_LEVEL || 'info',
|
logLevel: process.env.LOG_LEVEL || 'info',
|
||||||
rateLimitWindowMs: parseNumber(process.env.RATE_LIMIT_WINDOW_MS, 60000),
|
rateLimitWindowMs: parseNumber(process.env.RATE_LIMIT_WINDOW_MS, 60000),
|
||||||
@@ -107,7 +146,7 @@ export const config: AppConfig = {
|
|||||||
batchMaxItems: parseNumber(process.env.BATCH_MAX_ITEMS, 20),
|
batchMaxItems: parseNumber(process.env.BATCH_MAX_ITEMS, 20),
|
||||||
batchMaxSizeBytes: parseNumber(process.env.BATCH_MAX_SIZE_BYTES, 500 * 1024 * 1024),
|
batchMaxSizeBytes: parseNumber(process.env.BATCH_MAX_SIZE_BYTES, 500 * 1024 * 1024),
|
||||||
maxRequestBodyBytes: parseNumber(process.env.MAX_REQUEST_BODY_BYTES, 2 * 1024 * 1024 * 1024),
|
maxRequestBodyBytes: parseNumber(process.env.MAX_REQUEST_BODY_BYTES, 2 * 1024 * 1024 * 1024),
|
||||||
telegramChunkSizeBytes: parseNumber(process.env.TELEGRAM_CHUNK_SIZE_BYTES, 20 * 1024 * 1024),
|
telegramChunkSizeBytes,
|
||||||
compressChunkedUploads: process.env.COMPRESS_CHUNKED_UPLOADS !== 'false',
|
compressChunkedUploads: process.env.COMPRESS_CHUNKED_UPLOADS !== 'false',
|
||||||
chunkCompressionMinSizeBytes: parseNumber(process.env.CHUNK_COMPRESSION_MIN_SIZE_BYTES, 4096),
|
chunkCompressionMinSizeBytes: parseNumber(process.env.CHUNK_COMPRESSION_MIN_SIZE_BYTES, 4096),
|
||||||
adminApiToken: process.env.ADMIN_API_TOKEN || '',
|
adminApiToken: process.env.ADMIN_API_TOKEN || '',
|
||||||
@@ -126,8 +165,7 @@ export const config: AppConfig = {
|
|||||||
logger.info('Environment variables loaded', {
|
logger.info('Environment variables loaded', {
|
||||||
config: {
|
config: {
|
||||||
...config,
|
...config,
|
||||||
botToken: maskSecret(config.botToken),
|
botTokens: config.botTokens.map(maskSecret),
|
||||||
additionalBotTokens: config.additionalBotTokens.map(maskSecret),
|
|
||||||
databaseUrl: maskDatabaseUrl(config.databaseUrl),
|
databaseUrl: maskDatabaseUrl(config.databaseUrl),
|
||||||
adminApiToken: maskSecret(config.adminApiToken),
|
adminApiToken: maskSecret(config.adminApiToken),
|
||||||
adminApiTokenEnabled: config.adminApiToken.length > 0,
|
adminApiTokenEnabled: config.adminApiToken.length > 0,
|
||||||
|
|||||||
+59
-27
File diff suppressed because one or more lines are too long
+3
-30
@@ -1,53 +1,26 @@
|
|||||||
import { serve } from 'bun';
|
import { serve } from 'bun';
|
||||||
import { config } from './config/index';
|
import { config } from './env';
|
||||||
import { fileInfoCache } from './infrastructure/cache/index';
|
import { fileInfoCache } from './infrastructure/cache/index';
|
||||||
import { startBot } from './interfaces/bot/handler';
|
import { startBot } from './interfaces/bot/handler';
|
||||||
import { handleS3Request } from './interfaces/http/controllers/s3-controller';
|
import { handleS3Request } from './interfaces/http/controllers/s3-controller';
|
||||||
import { cleanupRateLimitCache } from './interfaces/http/middleware/rate-limit';
|
import { cleanupRateLimitCache } from './interfaces/http/middleware/rate-limit';
|
||||||
import { routes } from './interfaces/http/routes/index';
|
import { routes } from './interfaces/http/routes/index';
|
||||||
import { isS3Request } from './interfaces/s3/auth';
|
|
||||||
import { extractS3BucketFromHost } from './interfaces/s3/virtual-host';
|
|
||||||
import { logger } from './shared/logger/index';
|
import { logger } from './shared/logger/index';
|
||||||
import { metricsCollector } from './shared/metrics/index';
|
import { metricsCollector } from './shared/metrics/index';
|
||||||
|
import { getS3RouteBucket, shouldHandleS3 } from './shared/utils/s3-detection';
|
||||||
|
|
||||||
// ─── Auto-run migration at startup ──────────────────────────────────────────
|
// ─── Auto-run migration at startup ──────────────────────────────────────────
|
||||||
try {
|
try {
|
||||||
const { runMigration } = await import('./db/migrate');
|
const { runMigration } = await import('./infrastructure/persistence/drizzle/migrate');
|
||||||
await runMigration();
|
await runMigration();
|
||||||
} catch {
|
} catch {
|
||||||
logger.warn('Auto-migration skipped (non-fatal)');
|
logger.warn('Auto-migration skipped (non-fatal)');
|
||||||
}
|
}
|
||||||
|
|
||||||
const getS3RouteBucket = (req: Request): string | null => {
|
|
||||||
const host = req.headers.get('host') || '';
|
|
||||||
return extractS3BucketFromHost(host, config.s3VhostDomains);
|
|
||||||
};
|
|
||||||
|
|
||||||
const shouldHandleS3 = (req: Request, headers: Record<string, string>): boolean => {
|
|
||||||
const url = new URL(req.url);
|
|
||||||
return Boolean(
|
|
||||||
getS3RouteBucket(req) || isS3Request(headers) || url.searchParams.has('X-Amz-Signature'),
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
const _handleMaybeS3Root = (req: Request): Response | Promise<Response> => {
|
|
||||||
if (req.method === 'OPTIONS') {
|
|
||||||
return handleS3Request(req, getS3RouteBucket(req));
|
|
||||||
}
|
|
||||||
const headers = Object.fromEntries(req.headers);
|
|
||||||
if (shouldHandleS3(req, headers)) {
|
|
||||||
return handleS3Request(req, getS3RouteBucket(req));
|
|
||||||
}
|
|
||||||
return new Response('Not Allowed', { status: 405 });
|
|
||||||
};
|
|
||||||
|
|
||||||
const server = serve({
|
const server = serve({
|
||||||
port: config.port,
|
port: config.port,
|
||||||
routes,
|
routes,
|
||||||
fetch: async (req: Request) => {
|
fetch: async (req: Request) => {
|
||||||
if (req.method === 'OPTIONS') {
|
|
||||||
return handleS3Request(req, getS3RouteBucket(req));
|
|
||||||
}
|
|
||||||
const headers = Object.fromEntries(req.headers);
|
const headers = Object.fromEntries(req.headers);
|
||||||
if (shouldHandleS3(req, headers)) {
|
if (shouldHandleS3(req, headers)) {
|
||||||
return handleS3Request(req, getS3RouteBucket(req));
|
return handleS3Request(req, getS3RouteBucket(req));
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
/**
|
||||||
|
* Dependency Injection container.
|
||||||
|
*
|
||||||
|
* Wires up singleton instances of all repositories and application services,
|
||||||
|
* making them available to controllers and other adapters without requiring
|
||||||
|
* a full DI framework.
|
||||||
|
*
|
||||||
|
* @module infrastructure/di
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type { IBucketRepository } from '../domain/ports/bucket-repository';
|
||||||
|
import type { IFilePartRepository } from '../domain/ports/file-part-repository';
|
||||||
|
import type { IFileRepository } from '../domain/ports/file-repository';
|
||||||
|
import type { IMultipartRepository } from '../domain/ports/multipart-repository';
|
||||||
|
import type { ITelegramService } from '../domain/ports/telegram-service';
|
||||||
|
import { DrizzleBucketRepository } from './persistence/repositories/bucket-repository';
|
||||||
|
import { DrizzleFilePartRepository } from './persistence/repositories/file-part-repository';
|
||||||
|
import { DrizzleFileRepository } from './persistence/repositories/file-repository';
|
||||||
|
import { DrizzleMultipartRepository } from './persistence/repositories/multipart-repository';
|
||||||
|
import { botPool } from './telegram/bot-pool';
|
||||||
|
import { ChunkedStorage } from './telegram/chunked-storage';
|
||||||
|
|
||||||
|
// ─── Repository Singletons ──────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Singleton IFileRepository instance backed by Drizzle ORM. */
|
||||||
|
export const fileRepository: IFileRepository = new DrizzleFileRepository();
|
||||||
|
|
||||||
|
/** Singleton IBucketRepository instance backed by Drizzle ORM. */
|
||||||
|
export const bucketRepository: IBucketRepository = new DrizzleBucketRepository();
|
||||||
|
|
||||||
|
/** Singleton IFilePartRepository instance backed by Drizzle ORM. */
|
||||||
|
export const filePartRepository: IFilePartRepository = new DrizzleFilePartRepository();
|
||||||
|
|
||||||
|
/** Singleton IMultipartRepository instance backed by Drizzle ORM. */
|
||||||
|
export const multipartRepository: IMultipartRepository = new DrizzleMultipartRepository();
|
||||||
|
|
||||||
|
/** Singleton ITelegramService instance backed by the bot pool. */
|
||||||
|
export const telegramService: ITelegramService = botPool;
|
||||||
|
|
||||||
|
// ─── Service Singletons ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Singleton ChunkedStorage for large file chunked uploads. */
|
||||||
|
export const chunkedStorage = new ChunkedStorage(
|
||||||
|
fileRepository,
|
||||||
|
filePartRepository,
|
||||||
|
telegramService,
|
||||||
|
);
|
||||||
@@ -1,4 +1,3 @@
|
|||||||
import type { InferInsertModel, InferSelectModel } from 'drizzle-orm';
|
|
||||||
import {
|
import {
|
||||||
bigint,
|
bigint,
|
||||||
boolean,
|
boolean,
|
||||||
@@ -63,15 +62,3 @@ export const fileParts = pgTable('file_parts', {
|
|||||||
etag: text('etag').notNull(),
|
etag: text('etag').notNull(),
|
||||||
createdAt: timestamp('created_at').defaultNow().notNull(),
|
createdAt: timestamp('created_at').defaultNow().notNull(),
|
||||||
});
|
});
|
||||||
|
|
||||||
/** Type representing a file row selected from the database. */
|
|
||||||
export type File = InferSelectModel<typeof files>;
|
|
||||||
|
|
||||||
/** Type representing a file row being inserted into the database. */
|
|
||||||
export type NewFile = InferInsertModel<typeof files>;
|
|
||||||
|
|
||||||
/** Type representing a file part row selected from the database. */
|
|
||||||
export type FilePart = InferSelectModel<typeof fileParts>;
|
|
||||||
|
|
||||||
/** Type representing a file part row being inserted into the database. */
|
|
||||||
export type NewFilePart = InferInsertModel<typeof fileParts>;
|
|
||||||
|
|||||||
@@ -50,7 +50,6 @@ const isTransientError = (error: unknown): boolean => {
|
|||||||
const MAX_TRANSIENT_RETRIES = 3;
|
const MAX_TRANSIENT_RETRIES = 3;
|
||||||
const MAX_OUTER_RETRIES = 10;
|
const MAX_OUTER_RETRIES = 10;
|
||||||
const TELEGRAM_API_TIMEOUT_MS = 120_000;
|
const TELEGRAM_API_TIMEOUT_MS = 120_000;
|
||||||
const PER_BOT_CONCURRENCY = 1;
|
|
||||||
|
|
||||||
interface BotEntry {
|
interface BotEntry {
|
||||||
index: number;
|
index: number;
|
||||||
@@ -64,12 +63,12 @@ export class BotPool implements ITelegramService {
|
|||||||
private readonly bots: BotEntry[] = [];
|
private readonly bots: BotEntry[] = [];
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
const tokens = Array.from(new Set([config.botToken, ...config.additionalBotTokens]));
|
const tokens = Array.from(new Set(config.botTokens));
|
||||||
this.bots = tokens.map((token, index) => ({
|
this.bots = tokens.map((token, index) => ({
|
||||||
index,
|
index,
|
||||||
token,
|
token,
|
||||||
instance: new Telegraf(token),
|
instance: new Telegraf(token),
|
||||||
queue: new PQueue({ concurrency: PER_BOT_CONCURRENCY }),
|
queue: new PQueue({ concurrency: config.telegramBotConcurrency }),
|
||||||
rateLimitedUntil: 0,
|
rateLimitedUntil: 0,
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
@@ -256,7 +255,7 @@ export class BotPool implements ITelegramService {
|
|||||||
|
|
||||||
/** Get total effective concurrency across all bots */
|
/** Get total effective concurrency across all bots */
|
||||||
getEffectiveConcurrency(): number {
|
getEffectiveConcurrency(): number {
|
||||||
return this.bots.length * PER_BOT_CONCURRENCY;
|
return this.bots.length * config.telegramBotConcurrency;
|
||||||
}
|
}
|
||||||
|
|
||||||
async getFileInfo(telegramFileId: string): Promise<TelegramFileInfo> {
|
async getFileInfo(telegramFileId: string): Promise<TelegramFileInfo> {
|
||||||
|
|||||||
@@ -1,21 +1,17 @@
|
|||||||
import { createReadStream } from 'node:fs';
|
import { createReadStream } from 'node:fs';
|
||||||
import { gzipSync } from 'node:zlib';
|
|
||||||
import { nanoid } from 'nanoid';
|
import { nanoid } from 'nanoid';
|
||||||
import type { File as FileEntity } from '../../domain/entities/file';
|
import type { File as FileEntity } from '../../domain/entities/file';
|
||||||
import type { CompressionAlgorithm, NewFilePart } from '../../domain/entities/file-part';
|
import { buildNewFile } from '../../domain/entities/file-factory';
|
||||||
|
import type { NewFilePart } from '../../domain/entities/file-part';
|
||||||
import type { IFilePartRepository } from '../../domain/ports/file-part-repository';
|
import type { IFilePartRepository } from '../../domain/ports/file-part-repository';
|
||||||
import type { IFileRepository } from '../../domain/ports/file-repository';
|
import type { IFileRepository } from '../../domain/ports/file-repository';
|
||||||
import type { ITelegramService } from '../../domain/ports/telegram-service';
|
import type { ITelegramService } from '../../domain/ports/telegram-service';
|
||||||
import { config } from '../../env';
|
import { config } from '../../env';
|
||||||
import { createGetObjectResponse, type ObjectPartSource } from '../../interfaces/s3/object-stream';
|
import { createGetObjectResponse, type ObjectPartSource } from '../../interfaces/s3/object-stream';
|
||||||
import type { RangeParseResult } from '../../interfaces/s3/range';
|
import type { RangeParseResult } from '../../interfaces/s3/range';
|
||||||
|
import { type CompressionAlgorithm, maybeCompressChunk } from '../../shared/utils/compress';
|
||||||
import { computeHash } from '../../shared/utils/file';
|
import { computeHash } from '../../shared/utils/file';
|
||||||
|
import { asSafeChunkSize } from '../../shared/utils/validation';
|
||||||
/**
|
|
||||||
* Chunk compression algorithm identifier.
|
|
||||||
* `"gzip"` if gzip compression was applied, `null` for uncompressed.
|
|
||||||
*/
|
|
||||||
export type ChunkCompressionAlgorithm = CompressionAlgorithm;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Metadata about a single uploaded chunk (part) stored in Telegram.
|
* Metadata about a single uploaded chunk (part) stored in Telegram.
|
||||||
@@ -34,7 +30,7 @@ export interface ChunkedUploadPart {
|
|||||||
/** Stored (post-compression) size in bytes */
|
/** Stored (post-compression) size in bytes */
|
||||||
storedSizeBytes: number;
|
storedSizeBytes: number;
|
||||||
/** Compression algorithm applied, or null */
|
/** Compression algorithm applied, or null */
|
||||||
compressionAlgorithm: ChunkCompressionAlgorithm;
|
compressionAlgorithm: CompressionAlgorithm;
|
||||||
/** ETag (SHA-256 hash) of the original chunk */
|
/** ETag (SHA-256 hash) of the original chunk */
|
||||||
etag: string;
|
etag: string;
|
||||||
}
|
}
|
||||||
@@ -75,50 +71,6 @@ export interface ChunkedFileInput {
|
|||||||
s3Key?: string | null;
|
s3Key?: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Validate and sanitise the Telegram chunk size.
|
|
||||||
*
|
|
||||||
* @param chunkSizeBytes - The desired chunk size in bytes.
|
|
||||||
* @returns The validated chunk size.
|
|
||||||
* @throws {Error} If the chunk size is not a safe positive integer.
|
|
||||||
*/
|
|
||||||
const asSafeChunkSize = (chunkSizeBytes: number): number => {
|
|
||||||
if (!Number.isSafeInteger(chunkSizeBytes) || chunkSizeBytes <= 0) {
|
|
||||||
throw new Error('Invalid Telegram chunk size');
|
|
||||||
}
|
|
||||||
return chunkSizeBytes;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Optionally compress a chunk with gzip.
|
|
||||||
*
|
|
||||||
* Compression is skipped if:
|
|
||||||
* - The `compress` flag is false.
|
|
||||||
* - The chunk is smaller than `compressionMinSizeBytes`.
|
|
||||||
* - The compressed result is larger than the original.
|
|
||||||
*
|
|
||||||
* @param chunk - The raw chunk buffer.
|
|
||||||
* @param compress - Whether compression is enabled.
|
|
||||||
* @param compressionMinSizeBytes - Minimum chunk size to attempt compression.
|
|
||||||
* @returns The (possibly compressed) bytes and the algorithm used.
|
|
||||||
*/
|
|
||||||
const maybeCompressChunk = (
|
|
||||||
chunk: Buffer,
|
|
||||||
compress: boolean,
|
|
||||||
compressionMinSizeBytes: number,
|
|
||||||
): { bytes: Buffer; compressionAlgorithm: ChunkCompressionAlgorithm } => {
|
|
||||||
if (!compress || chunk.byteLength < compressionMinSizeBytes) {
|
|
||||||
return { bytes: chunk, compressionAlgorithm: null };
|
|
||||||
}
|
|
||||||
|
|
||||||
const gzipped = gzipSync(chunk);
|
|
||||||
if (gzipped.byteLength >= chunk.byteLength) {
|
|
||||||
return { bytes: chunk, compressionAlgorithm: null };
|
|
||||||
}
|
|
||||||
|
|
||||||
return { bytes: gzipped, compressionAlgorithm: 'gzip' };
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Manages chunked storage of large files in Telegram.
|
* Manages chunked storage of large files in Telegram.
|
||||||
*
|
*
|
||||||
@@ -229,7 +181,8 @@ export class ChunkedStorage {
|
|||||||
|
|
||||||
const publicId = nanoid();
|
const publicId = nanoid();
|
||||||
|
|
||||||
const file = await this.fileRepository.create({
|
const file = await this.fileRepository.create(
|
||||||
|
buildNewFile({
|
||||||
publicId,
|
publicId,
|
||||||
telegramFileId: firstPart.telegramFileId,
|
telegramFileId: firstPart.telegramFileId,
|
||||||
telegramFileUniqueId: firstPart.telegramFileUniqueId,
|
telegramFileUniqueId: firstPart.telegramFileUniqueId,
|
||||||
@@ -239,21 +192,14 @@ export class ChunkedStorage {
|
|||||||
mimeType: input.mimeType,
|
mimeType: input.mimeType,
|
||||||
sizeBytes: upload.totalSizeBytes,
|
sizeBytes: upload.totalSizeBytes,
|
||||||
fileType: input.fileType,
|
fileType: input.fileType,
|
||||||
|
storageBackend: 'chunked',
|
||||||
uploaderId: input.uploaderId,
|
uploaderId: input.uploaderId,
|
||||||
fileHash: upload.fileHash,
|
fileHash: upload.fileHash,
|
||||||
archiveTelegramFileId: null,
|
bucketId: input.bucketId,
|
||||||
archiveStorageMessageId: null,
|
s3Key: input.s3Key,
|
||||||
archiveFileName: null,
|
|
||||||
archiveEntryName: null,
|
|
||||||
archiveMimeType: null,
|
|
||||||
archiveSizeBytes: null,
|
|
||||||
bucketId: input.bucketId ?? null,
|
|
||||||
s3Key: input.s3Key ?? null,
|
|
||||||
storageBackend: 'chunked',
|
|
||||||
isDeleted: false,
|
|
||||||
multipartUploadId: null,
|
|
||||||
partCount: upload.parts.length,
|
partCount: upload.parts.length,
|
||||||
});
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
const fileParts: NewFilePart[] = upload.parts.map((part) => ({
|
const fileParts: NewFilePart[] = upload.parts.map((part) => ({
|
||||||
fileId: file.id,
|
fileId: file.id,
|
||||||
|
|||||||
@@ -1,238 +0,0 @@
|
|||||||
import { createReadStream } from 'node:fs';
|
|
||||||
import { nanoid } from 'nanoid';
|
|
||||||
import type { File as FileEntity, NewFile } from '../../domain/entities/file';
|
|
||||||
import type { IFileRepository } from '../../domain/ports/file-repository';
|
|
||||||
import type { ITelegramService } from '../../domain/ports/telegram-service';
|
|
||||||
import { config } from '../../env';
|
|
||||||
import { cleanupTempFile } from '../../shared/utils/file';
|
|
||||||
import { createZip, type ZipEntry } from '../../shared/utils/zip';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Metadata about a prepared upload before it is submitted to the batcher.
|
|
||||||
*/
|
|
||||||
export type PreparedUpload = {
|
|
||||||
/** Temporary file path on disk */
|
|
||||||
tempPath: string;
|
|
||||||
/** SHA-256 hash of the file contents */
|
|
||||||
fileHash: string;
|
|
||||||
/** File size in bytes */
|
|
||||||
sizeBytes: number;
|
|
||||||
/** First bytes of the file for MIME detection */
|
|
||||||
signatureBuffer: Buffer;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A fully materialised file record returned from the batcher.
|
|
||||||
*/
|
|
||||||
export type UploadedFile = FileEntity;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* An item ready for batched upload to Telegram storage.
|
|
||||||
*/
|
|
||||||
export type BatchUploadItem = {
|
|
||||||
/** Prepared upload metadata */
|
|
||||||
prepared: PreparedUpload;
|
|
||||||
/** Original file name */
|
|
||||||
fileName: string;
|
|
||||||
/** MIME type of the file */
|
|
||||||
mimeType: string;
|
|
||||||
/** File type classification (e.g. "document", "photo") */
|
|
||||||
fileType: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Internal pending upload tracking type, extending BatchUploadItem
|
|
||||||
* with resolve/reject callbacks.
|
|
||||||
*/
|
|
||||||
type PendingUpload = BatchUploadItem & {
|
|
||||||
resolve: (file: FileEntity) => void;
|
|
||||||
reject: (error: unknown) => void;
|
|
||||||
};
|
|
||||||
|
|
||||||
/** Time window in milliseconds during which uploads are batched together. */
|
|
||||||
const BATCH_WINDOW_MS = 2000;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Batches multiple file uploads into a single ZIP archive before forwarding
|
|
||||||
* them to Telegram storage. This reduces the number of Telegram API calls
|
|
||||||
* and improves throughput for small-file workloads.
|
|
||||||
*
|
|
||||||
* Injects dependencies via constructor — can be used with any
|
|
||||||
* {@link IFileRepository} and {@link ITelegramService} implementation.
|
|
||||||
*/
|
|
||||||
export class UploadBatcher {
|
|
||||||
private readonly pendingUploads: PendingUpload[] = [];
|
|
||||||
private flushTimer: ReturnType<typeof setTimeout> | null = null;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param fileRepository - Repository for persisting file records.
|
|
||||||
* @param telegramService - Service for forwarding files to Telegram storage.
|
|
||||||
*/
|
|
||||||
constructor(
|
|
||||||
private readonly fileRepository: IFileRepository,
|
|
||||||
private readonly telegramService: ITelegramService,
|
|
||||||
) {}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Build a NewFile record from a batch item and its archive metadata.
|
|
||||||
*
|
|
||||||
* @param item - The batched upload item.
|
|
||||||
* @param entry - ZIP entry metadata for the individual file.
|
|
||||||
* @param archive - Archive-level Telegram storage metadata.
|
|
||||||
* @returns A NewFile record ready for repository insertion.
|
|
||||||
*/
|
|
||||||
private buildUploadedFile(
|
|
||||||
item: BatchUploadItem,
|
|
||||||
entry: ZipEntry,
|
|
||||||
archive: {
|
|
||||||
telegramFileId: string;
|
|
||||||
telegramFileUniqueId: string;
|
|
||||||
storageMessageId: number;
|
|
||||||
fileName: string;
|
|
||||||
sizeBytes: number;
|
|
||||||
},
|
|
||||||
): NewFile {
|
|
||||||
return {
|
|
||||||
publicId: nanoid(),
|
|
||||||
telegramFileId: archive.telegramFileId,
|
|
||||||
telegramFileUniqueId: archive.telegramFileUniqueId,
|
|
||||||
storageChatId: config.storageChatId,
|
|
||||||
storageMessageId: archive.storageMessageId,
|
|
||||||
fileName: item.fileName,
|
|
||||||
mimeType: item.mimeType || 'application/octet-stream',
|
|
||||||
sizeBytes: item.prepared.sizeBytes,
|
|
||||||
fileType: item.fileType,
|
|
||||||
uploaderId: 0,
|
|
||||||
fileHash: item.prepared.fileHash,
|
|
||||||
archiveTelegramFileId: archive.telegramFileId,
|
|
||||||
archiveStorageMessageId: archive.storageMessageId,
|
|
||||||
archiveFileName: archive.fileName,
|
|
||||||
archiveEntryName: entry.entryName,
|
|
||||||
archiveMimeType: 'application/zip',
|
|
||||||
archiveSizeBytes: archive.sizeBytes,
|
|
||||||
bucketId: null,
|
|
||||||
s3Key: null,
|
|
||||||
storageBackend: null,
|
|
||||||
isDeleted: null,
|
|
||||||
multipartUploadId: null,
|
|
||||||
partCount: null,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Flush all pending uploads by zipping them together and sending
|
|
||||||
* the archive to Telegram storage.
|
|
||||||
*/
|
|
||||||
private async flushUploads(): Promise<void> {
|
|
||||||
if (this.flushTimer) {
|
|
||||||
clearTimeout(this.flushTimer);
|
|
||||||
this.flushTimer = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const batch = this.pendingUploads.splice(0);
|
|
||||||
if (batch.length === 0) return;
|
|
||||||
|
|
||||||
let zipTempPath: string | null = null;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const zip = await createZip(
|
|
||||||
batch.map((item) => ({ tempPath: item.prepared.tempPath, fileName: item.fileName })),
|
|
||||||
);
|
|
||||||
zipTempPath = zip.tempPath;
|
|
||||||
const archiveFileName = `filedrop-${nanoid()}.zip`;
|
|
||||||
const archiveResult = await this.telegramService.forwardToStorage(
|
|
||||||
createReadStream(zip.tempPath),
|
|
||||||
archiveFileName,
|
|
||||||
'document',
|
|
||||||
);
|
|
||||||
|
|
||||||
const newFileInputs = batch.map((item, index) =>
|
|
||||||
this.buildUploadedFile(item, zip.entries[index], {
|
|
||||||
telegramFileId: archiveResult.telegramFileId,
|
|
||||||
telegramFileUniqueId: archiveResult.telegramFileUniqueId,
|
|
||||||
storageMessageId: archiveResult.storageMessageId,
|
|
||||||
fileName: archiveFileName,
|
|
||||||
sizeBytes: zip.sizeBytes,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
// Persist each file record through the repository
|
|
||||||
const createdFiles = await Promise.all(
|
|
||||||
newFileInputs.map((input) => this.fileRepository.create(input)),
|
|
||||||
);
|
|
||||||
|
|
||||||
for (let i = 0; i < batch.length; i++) {
|
|
||||||
batch[i].resolve(createdFiles[i]);
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
for (const item of batch) {
|
|
||||||
item.reject(error);
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
await Promise.all(batch.map((item) => cleanupTempFile(item.prepared.tempPath)));
|
|
||||||
if (zipTempPath) await cleanupTempFile(zipTempPath);
|
|
||||||
// Reschedule timer if new items arrived during async processing
|
|
||||||
if (this.pendingUploads.length > 0 && !this.flushTimer) {
|
|
||||||
this.flushTimer = setTimeout(() => {
|
|
||||||
void this.flushUploads();
|
|
||||||
}, BATCH_WINDOW_MS);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Calculate total size of all pending uploads in bytes.
|
|
||||||
*
|
|
||||||
* @returns The sum of all pending file sizes.
|
|
||||||
*/
|
|
||||||
private getPendingSize(): number {
|
|
||||||
return this.pendingUploads.reduce((total, item) => total + item.prepared.sizeBytes, 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Enqueue a prepared upload for batched processing.
|
|
||||||
*
|
|
||||||
* The upload is held for up to {@link BATCH_WINDOW_MS} milliseconds
|
|
||||||
* (or until the batch size/byte thresholds in config are exceeded)
|
|
||||||
* before being flushed to Telegram storage.
|
|
||||||
*
|
|
||||||
* @param item - The prepared upload item to enqueue.
|
|
||||||
* @returns A promise that resolves with the fully created File record.
|
|
||||||
*/
|
|
||||||
enqueuePreparedUpload(item: BatchUploadItem): Promise<FileEntity> {
|
|
||||||
return new Promise<FileEntity>((resolve, reject) => {
|
|
||||||
this.pendingUploads.push({ ...item, resolve, reject });
|
|
||||||
|
|
||||||
if (!this.flushTimer) {
|
|
||||||
this.flushTimer = setTimeout(() => {
|
|
||||||
void this.flushUploads();
|
|
||||||
}, BATCH_WINDOW_MS);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (
|
|
||||||
this.pendingUploads.length >= config.batchMaxItems ||
|
|
||||||
this.getPendingSize() >= config.batchMaxSizeBytes
|
|
||||||
) {
|
|
||||||
void this.flushUploads();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Immediately flush all pending uploads, regardless of batch size.
|
|
||||||
*
|
|
||||||
* @returns A promise that resolves when the flush is complete.
|
|
||||||
*/
|
|
||||||
async flushPendingUploads(): Promise<void> {
|
|
||||||
await this.flushUploads();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the number of uploads currently waiting in the batch queue.
|
|
||||||
*
|
|
||||||
* @returns The pending upload count.
|
|
||||||
*/
|
|
||||||
getPendingUploadCount(): number {
|
|
||||||
return this.pendingUploads.length;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
import { nanoid } from 'nanoid';
|
import { nanoid } from 'nanoid';
|
||||||
import { type Context, Telegraf } from 'telegraf';
|
import { type Context, Telegraf } from 'telegraf';
|
||||||
import type { NewFile } from '../../domain/entities/file';
|
import { buildNewFile } from '../../domain/entities/file-factory';
|
||||||
import type { IFileRepository } from '../../domain/ports/file-repository';
|
import type { IFileRepository } from '../../domain/ports/file-repository';
|
||||||
import type { ITelegramService } from '../../domain/ports/telegram-service';
|
import type { ITelegramService } from '../../domain/ports/telegram-service';
|
||||||
import { config } from '../../env';
|
import { config } from '../../env';
|
||||||
@@ -8,6 +8,7 @@ import { DrizzleFileRepository } from '../../infrastructure/persistence/reposito
|
|||||||
import { botPool } from '../../infrastructure/telegram/bot-pool';
|
import { botPool } from '../../infrastructure/telegram/bot-pool';
|
||||||
import logger from '../../shared/logger/index';
|
import logger from '../../shared/logger/index';
|
||||||
import {
|
import {
|
||||||
|
checkFileSize,
|
||||||
detectFileType,
|
detectFileType,
|
||||||
extractFileFromMessage,
|
extractFileFromMessage,
|
||||||
getErrorMessage,
|
getErrorMessage,
|
||||||
@@ -92,7 +93,7 @@ export async function startBot(
|
|||||||
const fileRepo = deps.fileRepo ?? new DrizzleFileRepository();
|
const fileRepo = deps.fileRepo ?? new DrizzleFileRepository();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const bot = new Telegraf(config.botToken);
|
const bot = new Telegraf(config.botTokens[0]);
|
||||||
|
|
||||||
bot.command('start', async (ctx) => {
|
bot.command('start', async (ctx) => {
|
||||||
await ctx.reply(
|
await ctx.reply(
|
||||||
@@ -127,10 +128,10 @@ export async function startBot(
|
|||||||
ctx.message.voice?.file_name ||
|
ctx.message.voice?.file_name ||
|
||||||
'file';
|
'file';
|
||||||
|
|
||||||
const maxSize = getFileSizeLimit(fileType);
|
if (!checkFileSize(fileSize, fileType)) {
|
||||||
|
return ctx.reply(
|
||||||
if (fileSize > maxSize) {
|
`File size exceeds ${getFileSizeLimit(fileType) / (1024 * 1024)}MB limit`,
|
||||||
return ctx.reply(`File size exceeds ${maxSize / (1024 * 1024)}MB limit`);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const existing = await fileRepo.findByUniqueId(fileObj.file_unique_id);
|
const existing = await fileRepo.findByUniqueId(fileObj.file_unique_id);
|
||||||
@@ -149,7 +150,8 @@ export async function startBot(
|
|||||||
const result = await telegramService.forwardToStorage(file_id, fileName, fileType);
|
const result = await telegramService.forwardToStorage(file_id, fileName, fileType);
|
||||||
const publicId = nanoid();
|
const publicId = nanoid();
|
||||||
|
|
||||||
const uploaded: NewFile = {
|
await fileRepo.create(
|
||||||
|
buildNewFile({
|
||||||
publicId,
|
publicId,
|
||||||
telegramFileId: result.telegramFileId,
|
telegramFileId: result.telegramFileId,
|
||||||
telegramFileUniqueId: result.telegramFileUniqueId,
|
telegramFileUniqueId: result.telegramFileUniqueId,
|
||||||
@@ -160,22 +162,9 @@ export async function startBot(
|
|||||||
sizeBytes: fileSize,
|
sizeBytes: fileSize,
|
||||||
fileType,
|
fileType,
|
||||||
uploaderId: ctx.from.id,
|
uploaderId: ctx.from.id,
|
||||||
fileHash: null,
|
|
||||||
archiveTelegramFileId: null,
|
|
||||||
archiveStorageMessageId: null,
|
|
||||||
archiveFileName: null,
|
|
||||||
archiveEntryName: null,
|
|
||||||
archiveMimeType: null,
|
|
||||||
archiveSizeBytes: null,
|
|
||||||
bucketId: null,
|
|
||||||
s3Key: null,
|
|
||||||
storageBackend: 'telegram',
|
storageBackend: 'telegram',
|
||||||
isDeleted: false,
|
}),
|
||||||
multipartUploadId: null,
|
);
|
||||||
partCount: null,
|
|
||||||
};
|
|
||||||
|
|
||||||
await fileRepo.create(uploaded);
|
|
||||||
|
|
||||||
await replyWithDownloadUrl(ctx, publicId);
|
await replyWithDownloadUrl(ctx, publicId);
|
||||||
|
|
||||||
@@ -197,7 +186,9 @@ export async function startBot(
|
|||||||
|
|
||||||
await bot.launch();
|
await bot.launch();
|
||||||
|
|
||||||
logger.info('Telegram bot started', { botToken: `${config.botToken?.substring(0, 10)}...` });
|
logger.info('Telegram bot started', {
|
||||||
|
botToken: `${config.botTokens[0]?.substring(0, 10)}...`,
|
||||||
|
});
|
||||||
|
|
||||||
return bot;
|
return bot;
|
||||||
} catch (error: unknown) {
|
} catch (error: unknown) {
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user