Fix container remove timeouts and make force-remove reliable.
Release rolling / release (push) Successful in 9m13s

Lifecycle RPCs now use a 120s operation timeout, remove cleans up
stats/terminal/log streams then SIGKILLs before force-remove, and the
UI awaits the request instead of a fragile 30s wait race.
This commit is contained in:
Raven Scott
2026-07-15 14:12:13 -04:00
parent 1cb85e2cee
commit bbf0607aaf
9 changed files with 235 additions and 41 deletions
+25 -2
View File
@@ -95,8 +95,15 @@ const CODE_MAP = {
severity: 'warning',
},
TIMEOUT_EXCEEDED: {
title: 'Timed out',
recovery: 'The peer did not respond in time. Check the network and try again.',
title: 'Operation timed out',
recovery:
'Docker may still be finishing in the background — refresh the list. If the resource remains, retry the action or check the Docker daemon and peer connectivity.',
severity: 'warning',
},
OPERATION_TIMEOUT: {
title: 'Operation timed out',
recovery:
'Docker may still be finishing in the background — refresh the list. If the resource remains, retry the action or check the Docker daemon.',
severity: 'warning',
},
CHANNEL_CLOSED: {
@@ -194,6 +201,22 @@ export function explainError(err, method) {
}
}
// Bare "timeout of Nms exceeded" without a code (protomux-rpc) — not a lost peer
if (/timeout of \d+ms exceeded/i.test(message)) {
return {
code: code || 'TIMEOUT_EXCEEDED',
method: method || err?.method || null,
title: 'Operation timed out',
message: isGenericRequestFailed(message)
? 'The server took too long to finish this Docker operation'
: message,
recovery:
'Docker may still be finishing in the background — refresh the list. If the resource remains, retry the action or check the Docker daemon and peer connectivity.',
severity: 'warning',
silent: false,
}
}
if (
/not connected|timeout|ECONN|disconnect|CHANNEL_|PEER_NOT_FOUND|PEER_CONNECTION_FAILED|peer not found/i.test(
message + (code || '')
+45 -4
View File
@@ -26,6 +26,44 @@ const MAX_RECONNECT_ATTEMPTS = (() => {
return n
})()
/** Default dial/request timeout (connect + short reads) */
const DEFAULT_TIMEOUT_MS = CONFIG.CONNECTION.TIMEOUT_MS || 30000
/** Lifecycle / Docker mutations that often exceed dial timeout */
const OP_TIMEOUT_MS = CONFIG.CONTAINER?.OPERATION_TIMEOUT_MS || 120000
/**
* Per-method RPC timeouts. Remove/stop can exceed 30s when Docker is busy or
* force-killing a stuck process — the old global 30s caused false "Connection problem".
* @type {Record<string, number>}
*/
const METHOD_TIMEOUT_MS = {
removeContainer: OP_TIMEOUT_MS,
stopContainer: OP_TIMEOUT_MS,
restartContainer: OP_TIMEOUT_MS,
killContainer: OP_TIMEOUT_MS,
startContainer: OP_TIMEOUT_MS,
pauseContainer: DEFAULT_TIMEOUT_MS,
unpauseContainer: DEFAULT_TIMEOUT_MS,
recreateContainer: Math.max(OP_TIMEOUT_MS, 180000),
bulkContainerOperation: Math.max(OP_TIMEOUT_MS * 2, 300000),
deployContainer: Math.max(OP_TIMEOUT_MS, 300000),
pullImage: 600000,
buildImage: 600000,
deployStack: 600000,
removeStack: OP_TIMEOUT_MS,
pruneContainers: OP_TIMEOUT_MS,
systemPrune: OP_TIMEOUT_MS,
}
/**
* @param {string} method
* @param {{ timeout?: number }} [opts]
*/
function resolveRequestTimeout(method, opts = {}) {
if (opts.timeout != null && Number.isFinite(opts.timeout)) return opts.timeout
return METHOD_TIMEOUT_MS[method] ?? DEFAULT_TIMEOUT_MS
}
export class ConnectionManager extends EventEmitter {
constructor() {
super()
@@ -317,12 +355,14 @@ export class ConnectionManager extends EventEmitter {
* RPC on the active connection.
* @param {string} method
* @param {object} [args]
* @param {{ timeout?: number }} [opts]
*/
async request(method, args = {}) {
async request(method, args = {}, opts = {}) {
if (!this.active?.connected) {
throw new Error('No active connection')
}
return this.active.request(method, args)
const timeout = resolveRequestTimeout(method, opts)
return this.active.request(method, args, { ...opts, timeout })
}
/**
@@ -339,14 +379,15 @@ export class ConnectionManager extends EventEmitter {
* UI historically called sendCommand without awaiting.
* @param {string} method
* @param {object} [args]
* @param {{ silent?: boolean }} [opts]
* @param {{ silent?: boolean, timeout?: number }} [opts]
*/
send(method, args = {}, opts = {}) {
if (!this.active?.connected) {
console.debug('[DEBUG] No active connection for', method)
return Promise.resolve(null)
}
return this.active.request(method, args).catch((err) => {
const timeout = resolveRequestTimeout(method, opts)
return this.active.request(method, args, { timeout }).catch((err) => {
const normalized = normalizeRpcError(err, method)
// Emit structured error so handleRpcMessage can quiet background noise.
// Only force silent when the caller opts in (auto-refresh) — user actions still surface real errors.