fix(infra): nginx upstream DNS staleness — 502 after backend restart

- Replace upstream blocks with variable-based proxy_pass + Docker DNS
  resolver (127.0.0.11) so nginx resolves hostnames dynamically on
  each request instead of caching at startup only.
- Add explicit 'docker compose restart proxy' in CI deploy step
  (belt-and-suspenders: also forces nginx restart after deploy).
- Remove no-op sed commands from CI (docker-compose.yml already uses
  GitLab registry, no ghcr.io replacements needed).

Root cause: docker compose up -d only recreates containers whose image
changed. When backend container is recreated (new Docker IP), nginx
still caches the old IP → 502 Bad Gateway on /api/*
This commit is contained in:
asepharyana
2026-07-02 04:22:16 +07:00
parent fa82593a38
commit c5a9371e71
2 changed files with 21 additions and 14 deletions
+8 -4
View File
@@ -115,15 +115,19 @@ deploy-vps:
echo '$CI_JOB_TOKEN' | docker login $CI_REGISTRY -u '$CI_REGISTRY_USER' --password-stdin
echo '→ Updating docker-compose image references...'
sed -i 's|ghcr.io/\${OWNER:-mytheclipse}|$REGISTRY_PROJECT|g' infra/docker/docker-compose.yml
sed -i 's|\${OWNER:-mytheclipse}|$CI_PROJECT_NAMESPACE|g' infra/docker/docker-compose.yml
echo '→ Pulling latest images...'
docker compose -f infra/docker/docker-compose.yml pull
echo '→ Restarting containers...'
echo '→ Restarting containers (recreates if image changed)...'
docker compose -f infra/docker/docker-compose.yml up -d --remove-orphans
# Force restart proxy to pick up new upstream DNS IPs.
# Docker's DNS changes when backend/frontend containers are recreated,
# but nginx only resolves upstream hostnames at startup. Without this,
# nginx keeps pointing to stale container IPs → 502 Bad Gateway.
echo '→ Ensuring proxy container is restarted (nginx upstream DNS refresh)...'
docker compose -f infra/docker/docker-compose.yml restart proxy
echo '→ Cleaning up...'
docker image prune -f
echo '✓ Deploy complete'
+13 -10
View File
@@ -1,10 +1,10 @@
upstream backend {
server backend:3000;
}
upstream frontend {
server frontend:3000;
}
# Docker DNS resolver (127.0.0.11 = Docker's embedded DNS).
# Required for variable-based proxy_pass below to resolve upstream
# hostnames on each request instead of caching them at startup.
# Without this, when backend/frontend containers are recreated (new IP),
# nginx keeps pointing to stale IPs → 502 Bad Gateway.
# Valid=10s re-resolves at most every 10 seconds to avoid excessive DNS queries.
resolver 127.0.0.11 ipv6=off valid=10s;
map $http_upgrade $connection_upgrade {
default upgrade;
@@ -17,7 +17,8 @@ server {
# API proxy — matches /api and /api/*
location ^~ /api {
proxy_pass http://backend;
set $backend_url "http://backend:3000";
proxy_pass $backend_url$uri$is_args$args;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -26,7 +27,8 @@ server {
# WebSocket proxy — matches /ws and /ws/*
location ^~ /ws {
proxy_pass http://backend;
set $backend_url "http://backend:3000";
proxy_pass $backend_url$uri$is_args$args;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
@@ -43,7 +45,8 @@ server {
# Frontend SPA fallback
location / {
proxy_pass http://frontend;
set $frontend_url "http://frontend:3000";
proxy_pass $frontend_url$uri$is_args$args;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;