From e069dcaa40fd61ea2623f277b54b5204f4a2ee20 Mon Sep 17 00:00:00 2001 From: FragginWagon Date: Thu, 10 Sep 2026 02:09:25 -0400 Subject: [PATCH] fix: wait for healthy deployment services --- docs/DEPLOYMENT.md | 11 ++++++----- scripts/deploy-production.sh | 4 ++-- scripts/deployment-health-wait.test.sh | 11 +++++++++++ scripts/rollback-production.sh | 2 +- 4 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 scripts/deployment-health-wait.test.sh diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index c4b7583..effba1a 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -67,10 +67,10 @@ uses `IMAGE_TAG` to select matching API, worker, and web images. Deploy with: RELEASE_TAG=2026-09-09-01 ./scripts/deploy-production.sh ``` -The deploy script builds the three tagged images, starts healthy data services, -applies migrations with the tagged API image, and then switches the application -services. Keep the previous release tag and migration state with the deployment -record. +The deploy script builds the three tagged images, waits for healthy data +services, applies migrations with the tagged API image, and then switches the +application services after they are healthy. Keep the previous release tag and +migration state with the deployment record. Rollback application images with: @@ -78,7 +78,8 @@ Rollback application images with: RELEASE_TAG=2026-09-08-02 ./scripts/rollback-production.sh ``` -Image rollback is safe only when the previous image understands the current +The rollback script waits for the restored application services to become +healthy before running the smoke check. Image rollback is safe only when the previous image understands the current database schema. If the release included an incompatible schema change, restore the matching PostgreSQL backup before starting the previous images. Always verify the API and web health checks after a rollback. diff --git a/scripts/deploy-production.sh b/scripts/deploy-production.sh index 4c594c7..79d7cbf 100755 --- a/scripts/deploy-production.sh +++ b/scripts/deploy-production.sh @@ -7,8 +7,8 @@ compose_args='-f compose.production.yaml' export IMAGE_TAG="$RELEASE_TAG" docker compose $compose_args build api worker web -docker compose $compose_args up -d postgres redis +docker compose $compose_args up -d --wait postgres redis docker compose $compose_args run --rm api npm run db:migrate -docker compose $compose_args up -d api worker web +docker compose $compose_args up -d --wait api worker web docker compose $compose_args ps ./scripts/smoke-production.sh \ No newline at end of file diff --git a/scripts/deployment-health-wait.test.sh b/scripts/deployment-health-wait.test.sh new file mode 100644 index 0000000..5c81837 --- /dev/null +++ b/scripts/deployment-health-wait.test.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu + +deploy_script=$(cat scripts/deploy-production.sh) +rollback_script=$(cat scripts/rollback-production.sh) + +printf '%s\n' "$deploy_script" | grep -F 'up -d --wait postgres redis' >/dev/null +printf '%s\n' "$deploy_script" | grep -F 'up -d --wait api worker web' >/dev/null +printf '%s\n' "$rollback_script" | grep -F 'up -d --wait api worker web' >/dev/null + +printf '%s\n' 'deployment health wait tests passed' \ No newline at end of file diff --git a/scripts/rollback-production.sh b/scripts/rollback-production.sh index ceb6ba0..3dcfc02 100755 --- a/scripts/rollback-production.sh +++ b/scripts/rollback-production.sh @@ -4,6 +4,6 @@ set -eu : "${RELEASE_TAG:?Set RELEASE_TAG to the previously deployed immutable image tag}" export IMAGE_TAG="$RELEASE_TAG" -docker compose -f compose.production.yaml up -d api worker web +docker compose -f compose.production.yaml up -d --wait api worker web docker compose -f compose.production.yaml ps ./scripts/smoke-production.sh \ No newline at end of file