Restore and restart origin container when swap deploy fails
Release rolling / release (push) Successful in 8m45s

On duplicate/swap failure after parking, rename the origin back and start
it if it was running so a bad deploy does not leave the source offline.
This commit is contained in:
Raven Scott
2026-07-16 22:26:22 -04:00
parent bd512df950
commit abc34b39d8
2 changed files with 148 additions and 16 deletions
+147 -15
View File
@@ -241,6 +241,121 @@ async function removeParkedSource(parked) {
}
}
/**
* Undo a park: rename origin back and start it if it was running before the swap.
* Best-effort; never throws.
*
* @param {{ id: string, originalName: string, parkName: string, wasRunning: boolean }} parked
* @returns {Promise<{ restored: boolean, renamed: boolean, started: boolean, error: string|null }>}
*/
async function restoreParkedSource(parked) {
/** @type {{ restored: boolean, renamed: boolean, started: boolean, error: string|null }} */
const result = { restored: false, renamed: false, started: false, error: null }
if (!parked?.id) {
result.error = 'missing parked source id'
return result
}
const container = docker.getContainer(parked.id)
const targetName = String(parked.originalName || '').replace(/^\//, '') || null
// Rename parkName → originalName (skip if already correct)
try {
const info = await container.inspect()
const current = String(info?.Name || '').replace(/^\//, '')
if (targetName && current !== targetName) {
await container.rename({ name: targetName })
}
result.renamed = true
} catch (renameErr) {
// Name may still be held by a failed new container — report clearly
result.error = `could not rename "${parked.parkName}" back to "${parked.originalName}": ${
renameErr?.message || renameErr
}`
logger.warn('Failed to restore parked source name', {
id: String(parked.id).slice(0, 12),
parkName: parked.parkName,
originalName: parked.originalName,
error: result.error,
})
return result
}
if (parked.wasRunning) {
try {
await startContainerNoBody(parked.id)
result.started = true
} catch (startErr) {
try {
const info = await container.inspect()
if (info?.State?.Running) {
result.started = true
} else {
result.error = `renamed to "${parked.originalName}" but failed to start: ${
startErr?.message || startErr
}`
logger.warn('Failed to start restored origin after swap rollback', {
id: String(parked.id).slice(0, 12),
originalName: parked.originalName,
error: result.error,
})
return result
}
} catch (inspectErr) {
result.error = `renamed but could not confirm start: ${
inspectErr?.message || startErr?.message || startErr
}`
return result
}
}
}
result.restored = true
logger.info('Restored parked source after failed swap deploy', {
id: String(parked.id).slice(0, 12),
originalName: parked.originalName,
parkName: parked.parkName,
wasRunning: parked.wasRunning,
started: result.started,
})
return result
}
/**
* Human-readable suffix describing origin restore outcome.
* @param {{ restored: boolean, renamed?: boolean, started?: boolean, error: string|null }} restoreResult
* @param {{ originalName: string, parkName: string, wasRunning: boolean }} parked
*/
function originRestoreMessage(restoreResult, parked) {
if (restoreResult?.restored) {
if (parked.wasRunning && restoreResult.started) {
return ` Origin "${parked.originalName}" was restored and started.`
}
if (parked.wasRunning && !restoreResult.started) {
return ` Origin "${parked.originalName}" was renamed back but may not be running.`
}
return ` Origin "${parked.originalName}" was restored (it was stopped before the swap).`
}
const detail = restoreResult?.error || 'unknown error'
return (
` Could not automatically restore origin (${detail}). ` +
`It may still be parked as "${parked.parkName}" — rename it to "${parked.originalName}"` +
(parked.wasRunning ? ' and start it' : '') +
'.'
)
}
/**
* Restore parked origin then throw a single error including restore outcome.
* @param {{ id: string, originalName: string, parkName: string, wasRunning: boolean }} parked
* @param {string} message
* @returns {Promise<never>}
*/
async function abortSwapAndThrow(parked, message) {
const restoreResult = await restoreParkedSource(parked)
throw new Error(String(message || 'Deploy failed') + originRestoreMessage(restoreResult, parked))
}
export function registerDeployHandlers(session) {
session.respond('deployContainer', async (args) => {
const containerName = validation.sanitizeString(args.containerName, 63)
@@ -324,11 +439,15 @@ export function registerDeployHandlers(session) {
try {
await old.remove({ force: true })
} catch (rmErr) {
throw formatDeployError(rmErr, {
const formatted = formatDeployError(rmErr, {
stage: 'replace',
containerName: args.containerName,
image: args.image,
})
if (parked) {
await abortSwapAndThrow(parked, formatted.message)
}
throw formatted
}
} else {
const err = new Error(
@@ -344,6 +463,11 @@ export function registerDeployHandlers(session) {
state: existing.State || 'unknown',
image: existing.Image || '',
}
// We may have parked for port reuse before discovering a different name holder
if (parked) {
const restoreResult = await restoreParkedSource(parked)
err.message += originRestoreMessage(restoreResult, parked)
}
throw err
}
}
@@ -369,11 +493,15 @@ export function registerDeployHandlers(session) {
docker.modem.followProgress(pullStream, (err) => (err ? reject(err) : resolve()))
})
} catch (pullErr) {
throw formatDeployError(pullErr, {
const formatted = formatDeployError(pullErr, {
stage: 'pull',
containerName: args.containerName,
image: args.image,
})
if (parked) {
await abortSwapAndThrow(parked, formatted.message)
}
throw formatted
}
}
}
@@ -566,10 +694,7 @@ export function registerDeployHandlers(session) {
image: args.image,
})
if (parked) {
throw new Error(
`${formatted.message} Origin was parked as "${parked.parkName}" (not removed). ` +
`How to fix: rename it back to "${parked.originalName}" and start it, or remove it after fixing the deploy error.`
)
await abortSwapAndThrow(parked, formatted.message)
}
throw formatted
}
@@ -596,9 +721,9 @@ export function registerDeployHandlers(session) {
// ignore
}
if (parked) {
throw new Error(
`Deploy rolled back: could not attach network "${args.customNetwork}". ${formatted.message} ` +
`Origin was parked as "${parked.parkName}" (not removed).`
await abortSwapAndThrow(
parked,
`Deploy rolled back: could not attach network "${args.customNetwork}". ${formatted.message}`
)
}
throw new Error(
@@ -627,10 +752,9 @@ export function registerDeployHandlers(session) {
// ignore
}
if (parked) {
throw new Error(
`Deploy rolled back after start failed for ${args.containerName}. ${formatted.message} ` +
`Origin was parked as "${parked.parkName}" (not removed). ` +
`How to fix: rename it back to "${parked.originalName}" and start it, or remove it after fixing the port/config issue.`
await abortSwapAndThrow(
parked,
`Deploy rolled back after start failed for ${args.containerName}. ${formatted.message}`
)
}
throw new Error(
@@ -651,11 +775,19 @@ export function registerDeployHandlers(session) {
} catch {
// ignore
}
await abortSwapAndThrow(
parked,
`New container failed verification: ${verifyErr?.message || verifyErr}`
)
}
// rollback disabled: leave new container + parked origin; still try restore is unsafe
// if new still holds name/ports — only report parked state
throw new Error(
`New container failed verification: ${verifyErr?.message || verifyErr}. ` +
`Origin was parked as "${parked.parkName}" (not removed). ` +
`How to fix: rename it back to "${parked.originalName}" and start it, or inspect logs for "${args.containerName}".`
`Origin was parked as "${parked.parkName}" (not removed; rollback disabled). ` +
`How to fix: remove the new container, rename origin to "${parked.originalName}"` +
(parked.wasRunning ? ' and start it' : '') +
'.'
)
}
await removeParkedSource(parked)