diff --git a/.env.example b/.env.example index 525dba9..c8f0474 100644 --- a/.env.example +++ b/.env.example @@ -8,4 +8,9 @@ NODE_ENV=production LOG_LEVEL=info RATE_LIMIT_WINDOW_MS=60000 RATE_LIMIT_MAX_REQUESTS=30 -# TRUST_PROXY=true # Uncomment when behind reverse proxy (Traefik, Nginx) for correct client IP detection \ No newline at end of file +# TRUST_PROXY=true # Uncomment when behind reverse proxy (Traefik, Nginx) for correct client IP detection + +# S3-compatible API credentials +# S3_ACCESS_KEY=teleuploader-admin +# S3_SECRET_KEY=your-secret-key-here +# S3_DEFAULT_REGION=us-east-1 \ No newline at end of file diff --git a/schema.sql b/schema.sql index 22941be..52ba0a3 100644 --- a/schema.sql +++ b/schema.sql @@ -26,4 +26,50 @@ CREATE INDEX IF NOT EXISTS idx_files_telegram_file_id ON files(telegram_file_id) CREATE INDEX IF NOT EXISTS idx_files_file_hash ON files(file_hash); CREATE INDEX IF NOT EXISTS idx_files_archive_telegram_file_id ON files(archive_telegram_file_id); CREATE INDEX IF NOT EXISTS idx_files_uploader_id ON files(uploader_id); -CREATE INDEX IF NOT EXISTS idx_files_created_at ON files(created_at DESC); \ No newline at end of file +CREATE INDEX IF NOT EXISTS idx_files_created_at ON files(created_at DESC); + +-- S3-compatible buckets +CREATE TABLE IF NOT EXISTS buckets ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name VARCHAR(63) UNIQUE NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +-- Extend files table for S3 +ALTER TABLE files ADD COLUMN IF NOT EXISTS bucket_id UUID REFERENCES buckets(id); +ALTER TABLE files ADD COLUMN IF NOT EXISTS s3_key TEXT; +ALTER TABLE files ADD COLUMN IF NOT EXISTS storage_backend VARCHAR DEFAULT 'telegram'; +ALTER TABLE files ADD COLUMN IF NOT EXISTS is_deleted BOOLEAN DEFAULT false; +ALTER TABLE files ADD COLUMN IF NOT EXISTS multipart_upload_id TEXT; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_files_bucket_key ON files(bucket_id, s3_key) WHERE is_deleted = false; +CREATE INDEX IF NOT EXISTS idx_files_bucket_prefix ON files(bucket_id, s3_key text_pattern_ops); +CREATE INDEX IF NOT EXISTS idx_files_s3_key ON files(s3_key); +CREATE INDEX IF NOT EXISTS idx_files_bucket_id ON files(bucket_id); + +-- Multipart upload tracking +CREATE TABLE IF NOT EXISTS multipart_uploads ( + upload_id VARCHAR PRIMARY KEY, + bucket_id UUID NOT NULL REFERENCES buckets(id), + s3_key TEXT NOT NULL, + initiated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + status VARCHAR DEFAULT 'in_progress', + initiated_by TEXT +); + +CREATE TABLE IF NOT EXISTS multipart_parts ( + id SERIAL PRIMARY KEY, + upload_id VARCHAR NOT NULL REFERENCES multipart_uploads(upload_id) ON DELETE CASCADE, + part_number INT NOT NULL, + telegram_file_id VARCHAR NOT NULL, + telegram_file_unique_id VARCHAR NOT NULL, + storage_message_id BIGINT NOT NULL, + size_bytes BIGINT NOT NULL, + etag VARCHAR NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(upload_id, part_number) +); + +CREATE INDEX IF NOT EXISTS idx_multipart_parts_upload ON multipart_parts(upload_id, part_number); +CREATE INDEX IF NOT EXISTS idx_multipart_uploads_status ON multipart_uploads(status); \ No newline at end of file diff --git a/src/env.ts b/src/env.ts index 5fb686d..67e9d41 100644 --- a/src/env.ts +++ b/src/env.ts @@ -16,6 +16,9 @@ interface AppConfig { batchMaxItems: number; batchMaxSizeBytes: number; maxRequestBodyBytes: number; + s3AccessKey: string; + s3SecretKey: string; + s3DefaultRegion: string; } const requiredEnv = { @@ -72,6 +75,9 @@ export const config: AppConfig = { batchMaxItems: parseNumber(process.env.BATCH_MAX_ITEMS, 20), batchMaxSizeBytes: parseNumber(process.env.BATCH_MAX_SIZE_BYTES, 500 * 1024 * 1024), maxRequestBodyBytes: parseNumber(process.env.MAX_REQUEST_BODY_BYTES, 2 * 1024 * 1024 * 1024), + s3AccessKey: process.env.S3_ACCESS_KEY || 'teleuploader-admin', + s3SecretKey: process.env.S3_SECRET_KEY || '', + s3DefaultRegion: process.env.S3_DEFAULT_REGION || 'us-east-1', }; logger.info('Environment variables loaded', { @@ -80,5 +86,7 @@ logger.info('Environment variables loaded', { botToken: maskSecret(config.botToken), additionalBotTokens: config.additionalBotTokens.map(maskSecret), databaseUrl: maskDatabaseUrl(config.databaseUrl), + s3AccessKey: maskSecret(config.s3AccessKey), + s3SecretKey: maskSecret(config.s3SecretKey), }, });