Restore and restart origin container when swap deploy fails
Release rolling / release (push) Successful in 8m45s
Release rolling / release (push) Successful in 8m45s
On duplicate/swap failure after parking, rename the origin back and start it if it was running so a bad deploy does not leave the source offline.
This commit is contained in:
+147
-15
@@ -241,6 +241,121 @@ async function removeParkedSource(parked) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undo a park: rename origin back and start it if it was running before the swap.
|
||||
* Best-effort; never throws.
|
||||
*
|
||||
* @param {{ id: string, originalName: string, parkName: string, wasRunning: boolean }} parked
|
||||
* @returns {Promise<{ restored: boolean, renamed: boolean, started: boolean, error: string|null }>}
|
||||
*/
|
||||
async function restoreParkedSource(parked) {
|
||||
/** @type {{ restored: boolean, renamed: boolean, started: boolean, error: string|null }} */
|
||||
const result = { restored: false, renamed: false, started: false, error: null }
|
||||
if (!parked?.id) {
|
||||
result.error = 'missing parked source id'
|
||||
return result
|
||||
}
|
||||
|
||||
const container = docker.getContainer(parked.id)
|
||||
const targetName = String(parked.originalName || '').replace(/^\//, '') || null
|
||||
|
||||
// Rename parkName → originalName (skip if already correct)
|
||||
try {
|
||||
const info = await container.inspect()
|
||||
const current = String(info?.Name || '').replace(/^\//, '')
|
||||
if (targetName && current !== targetName) {
|
||||
await container.rename({ name: targetName })
|
||||
}
|
||||
result.renamed = true
|
||||
} catch (renameErr) {
|
||||
// Name may still be held by a failed new container — report clearly
|
||||
result.error = `could not rename "${parked.parkName}" back to "${parked.originalName}": ${
|
||||
renameErr?.message || renameErr
|
||||
}`
|
||||
logger.warn('Failed to restore parked source name', {
|
||||
id: String(parked.id).slice(0, 12),
|
||||
parkName: parked.parkName,
|
||||
originalName: parked.originalName,
|
||||
error: result.error,
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
if (parked.wasRunning) {
|
||||
try {
|
||||
await startContainerNoBody(parked.id)
|
||||
result.started = true
|
||||
} catch (startErr) {
|
||||
try {
|
||||
const info = await container.inspect()
|
||||
if (info?.State?.Running) {
|
||||
result.started = true
|
||||
} else {
|
||||
result.error = `renamed to "${parked.originalName}" but failed to start: ${
|
||||
startErr?.message || startErr
|
||||
}`
|
||||
logger.warn('Failed to start restored origin after swap rollback', {
|
||||
id: String(parked.id).slice(0, 12),
|
||||
originalName: parked.originalName,
|
||||
error: result.error,
|
||||
})
|
||||
return result
|
||||
}
|
||||
} catch (inspectErr) {
|
||||
result.error = `renamed but could not confirm start: ${
|
||||
inspectErr?.message || startErr?.message || startErr
|
||||
}`
|
||||
return result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.restored = true
|
||||
logger.info('Restored parked source after failed swap deploy', {
|
||||
id: String(parked.id).slice(0, 12),
|
||||
originalName: parked.originalName,
|
||||
parkName: parked.parkName,
|
||||
wasRunning: parked.wasRunning,
|
||||
started: result.started,
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Human-readable suffix describing origin restore outcome.
|
||||
* @param {{ restored: boolean, renamed?: boolean, started?: boolean, error: string|null }} restoreResult
|
||||
* @param {{ originalName: string, parkName: string, wasRunning: boolean }} parked
|
||||
*/
|
||||
function originRestoreMessage(restoreResult, parked) {
|
||||
if (restoreResult?.restored) {
|
||||
if (parked.wasRunning && restoreResult.started) {
|
||||
return ` Origin "${parked.originalName}" was restored and started.`
|
||||
}
|
||||
if (parked.wasRunning && !restoreResult.started) {
|
||||
return ` Origin "${parked.originalName}" was renamed back but may not be running.`
|
||||
}
|
||||
return ` Origin "${parked.originalName}" was restored (it was stopped before the swap).`
|
||||
}
|
||||
const detail = restoreResult?.error || 'unknown error'
|
||||
return (
|
||||
` Could not automatically restore origin (${detail}). ` +
|
||||
`It may still be parked as "${parked.parkName}" — rename it to "${parked.originalName}"` +
|
||||
(parked.wasRunning ? ' and start it' : '') +
|
||||
'.'
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore parked origin then throw a single error including restore outcome.
|
||||
* @param {{ id: string, originalName: string, parkName: string, wasRunning: boolean }} parked
|
||||
* @param {string} message
|
||||
* @returns {Promise<never>}
|
||||
*/
|
||||
async function abortSwapAndThrow(parked, message) {
|
||||
const restoreResult = await restoreParkedSource(parked)
|
||||
throw new Error(String(message || 'Deploy failed') + originRestoreMessage(restoreResult, parked))
|
||||
}
|
||||
|
||||
export function registerDeployHandlers(session) {
|
||||
session.respond('deployContainer', async (args) => {
|
||||
const containerName = validation.sanitizeString(args.containerName, 63)
|
||||
@@ -324,11 +439,15 @@ export function registerDeployHandlers(session) {
|
||||
try {
|
||||
await old.remove({ force: true })
|
||||
} catch (rmErr) {
|
||||
throw formatDeployError(rmErr, {
|
||||
const formatted = formatDeployError(rmErr, {
|
||||
stage: 'replace',
|
||||
containerName: args.containerName,
|
||||
image: args.image,
|
||||
})
|
||||
if (parked) {
|
||||
await abortSwapAndThrow(parked, formatted.message)
|
||||
}
|
||||
throw formatted
|
||||
}
|
||||
} else {
|
||||
const err = new Error(
|
||||
@@ -344,6 +463,11 @@ export function registerDeployHandlers(session) {
|
||||
state: existing.State || 'unknown',
|
||||
image: existing.Image || '',
|
||||
}
|
||||
// We may have parked for port reuse before discovering a different name holder
|
||||
if (parked) {
|
||||
const restoreResult = await restoreParkedSource(parked)
|
||||
err.message += originRestoreMessage(restoreResult, parked)
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
@@ -369,11 +493,15 @@ export function registerDeployHandlers(session) {
|
||||
docker.modem.followProgress(pullStream, (err) => (err ? reject(err) : resolve()))
|
||||
})
|
||||
} catch (pullErr) {
|
||||
throw formatDeployError(pullErr, {
|
||||
const formatted = formatDeployError(pullErr, {
|
||||
stage: 'pull',
|
||||
containerName: args.containerName,
|
||||
image: args.image,
|
||||
})
|
||||
if (parked) {
|
||||
await abortSwapAndThrow(parked, formatted.message)
|
||||
}
|
||||
throw formatted
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -566,10 +694,7 @@ export function registerDeployHandlers(session) {
|
||||
image: args.image,
|
||||
})
|
||||
if (parked) {
|
||||
throw new Error(
|
||||
`${formatted.message} Origin was parked as "${parked.parkName}" (not removed). ` +
|
||||
`How to fix: rename it back to "${parked.originalName}" and start it, or remove it after fixing the deploy error.`
|
||||
)
|
||||
await abortSwapAndThrow(parked, formatted.message)
|
||||
}
|
||||
throw formatted
|
||||
}
|
||||
@@ -596,9 +721,9 @@ export function registerDeployHandlers(session) {
|
||||
// ignore
|
||||
}
|
||||
if (parked) {
|
||||
throw new Error(
|
||||
`Deploy rolled back: could not attach network "${args.customNetwork}". ${formatted.message} ` +
|
||||
`Origin was parked as "${parked.parkName}" (not removed).`
|
||||
await abortSwapAndThrow(
|
||||
parked,
|
||||
`Deploy rolled back: could not attach network "${args.customNetwork}". ${formatted.message}`
|
||||
)
|
||||
}
|
||||
throw new Error(
|
||||
@@ -627,10 +752,9 @@ export function registerDeployHandlers(session) {
|
||||
// ignore
|
||||
}
|
||||
if (parked) {
|
||||
throw new Error(
|
||||
`Deploy rolled back after start failed for ${args.containerName}. ${formatted.message} ` +
|
||||
`Origin was parked as "${parked.parkName}" (not removed). ` +
|
||||
`How to fix: rename it back to "${parked.originalName}" and start it, or remove it after fixing the port/config issue.`
|
||||
await abortSwapAndThrow(
|
||||
parked,
|
||||
`Deploy rolled back after start failed for ${args.containerName}. ${formatted.message}`
|
||||
)
|
||||
}
|
||||
throw new Error(
|
||||
@@ -651,11 +775,19 @@ export function registerDeployHandlers(session) {
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
await abortSwapAndThrow(
|
||||
parked,
|
||||
`New container failed verification: ${verifyErr?.message || verifyErr}`
|
||||
)
|
||||
}
|
||||
// rollback disabled: leave new container + parked origin; still try restore is unsafe
|
||||
// if new still holds name/ports — only report parked state
|
||||
throw new Error(
|
||||
`New container failed verification: ${verifyErr?.message || verifyErr}. ` +
|
||||
`Origin was parked as "${parked.parkName}" (not removed). ` +
|
||||
`How to fix: rename it back to "${parked.originalName}" and start it, or inspect logs for "${args.containerName}".`
|
||||
`Origin was parked as "${parked.parkName}" (not removed; rollback disabled). ` +
|
||||
`How to fix: remove the new container, rename origin to "${parked.originalName}"` +
|
||||
(parked.wasRunning ? ' and start it' : '') +
|
||||
'.'
|
||||
)
|
||||
}
|
||||
await removeParkedSource(parked)
|
||||
|
||||
Reference in New Issue
Block a user