Raven Scott
bbf0607aaf
Release rolling / release (push) Successful in 9m13s
Lifecycle RPCs now use a 120s operation timeout, remove cleans up stats/terminal/log streams then SIGKILLs before force-remove, and the UI awaits the request instead of a fragile 30s wait race.
612 lines
17 KiB
JavaScript
612 lines
17 KiB
JavaScript
/**
|
|
* Interactive container terminal over protomux-rpc.
|
|
* TTY sessions stream raw PTY bytes (no demux). Supports multi-session via sessionId.
|
|
*
|
|
* Shell selection: try an ordered list of shells (bash, sh, ash, …) until one
|
|
* actually stays running. Docker's exec create often succeeds even when the
|
|
* binary is missing — failure shows up only after start (exit 126/127).
|
|
*/
|
|
import { PassThrough } from 'stream'
|
|
import { docker } from '../services/docker.js'
|
|
import { Pushes } from '../../shared/protocol.js'
|
|
import logger from '../utils/logger.js'
|
|
|
|
const SESSIONS_KEY = 'terminals'
|
|
|
|
/** How long to wait for a candidate shell to prove it is alive. */
|
|
const SHELL_PROBE_MS = 450
|
|
const SHELL_PROBE_STEP_MS = 40
|
|
|
|
/**
|
|
* Ordered shell candidates. Prefer interactive shells, then POSIX sh variants,
|
|
* busybox, then PATH-relative names. First match that stays Running wins.
|
|
*/
|
|
export const DEFAULT_SHELL_CANDIDATES = Object.freeze([
|
|
['/bin/bash'],
|
|
['/usr/bin/bash'],
|
|
['bash'],
|
|
['/bin/zsh'],
|
|
['/usr/bin/zsh'],
|
|
['zsh'],
|
|
['/bin/sh'],
|
|
['/usr/bin/sh'],
|
|
['sh'],
|
|
['/bin/ash'],
|
|
['/usr/bin/ash'],
|
|
['ash'],
|
|
['/bin/dash'],
|
|
['/usr/bin/dash'],
|
|
['dash'],
|
|
['/bin/ksh'],
|
|
['/usr/bin/ksh'],
|
|
['ksh'],
|
|
['/bin/mksh'],
|
|
['/usr/bin/mksh'],
|
|
['mksh'],
|
|
['/bin/fish'],
|
|
['/usr/bin/fish'],
|
|
['fish'],
|
|
['/bin/busybox', 'sh'],
|
|
['/usr/bin/busybox', 'sh'],
|
|
['busybox', 'sh'],
|
|
['/busybox', 'sh'],
|
|
['/bin/busybox', 'ash'],
|
|
['/system/bin/sh'], // Android-ish images
|
|
])
|
|
|
|
/**
|
|
* One-shot script: from a working sh, pick the best available interactive shell.
|
|
* Used when /bin/sh (or equivalent) exists but we still want bash if present.
|
|
*
|
|
* Must be valid when passed to `sh -c`. Join with newlines (not spaces):
|
|
* space-joining turns `done\\nfor` into `done for`, which dash/sh reject
|
|
* as `Syntax error: "if" unexpected (expecting "done")`.
|
|
*/
|
|
const SHELL_PROBE_SCRIPT = [
|
|
'for s in /bin/bash /usr/bin/bash bash /bin/zsh /usr/bin/zsh zsh /bin/ash /usr/bin/ash ash /bin/dash /usr/bin/dash dash /bin/ksh /usr/bin/ksh ksh /bin/mksh /usr/bin/mksh mksh /bin/fish /usr/bin/fish fish; do',
|
|
' if [ -x "$s" ]; then exec "$s"; fi',
|
|
' if command -v "$s" >/dev/null 2>&1; then exec "$s"; fi',
|
|
'done',
|
|
'for b in /bin/busybox /usr/bin/busybox busybox /busybox; do',
|
|
' if [ -x "$b" ]; then exec "$b" sh; fi',
|
|
' if command -v "$b" >/dev/null 2>&1; then exec "$b" sh; fi',
|
|
'done',
|
|
'if [ -x /bin/sh ]; then exec /bin/sh; fi',
|
|
'if [ -x /usr/bin/sh ]; then exec /usr/bin/sh; fi',
|
|
'exit 127',
|
|
].join('\n')
|
|
|
|
/** Bootstrap wrappers that run SHELL_PROBE_SCRIPT. */
|
|
const PROBE_WRAPPERS = Object.freeze([
|
|
['/bin/sh', '-c', SHELL_PROBE_SCRIPT],
|
|
['/usr/bin/sh', '-c', SHELL_PROBE_SCRIPT],
|
|
['sh', '-c', SHELL_PROBE_SCRIPT],
|
|
['/bin/bash', '-c', SHELL_PROBE_SCRIPT],
|
|
['bash', '-c', SHELL_PROBE_SCRIPT],
|
|
['/bin/ash', '-c', SHELL_PROBE_SCRIPT],
|
|
['ash', '-c', SHELL_PROBE_SCRIPT],
|
|
['/bin/busybox', 'sh', '-c', SHELL_PROBE_SCRIPT],
|
|
['busybox', 'sh', '-c', SHELL_PROBE_SCRIPT],
|
|
])
|
|
|
|
function sleep(ms) {
|
|
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
}
|
|
|
|
function getSessions(session) {
|
|
let map = session.state.get(SESSIONS_KEY)
|
|
if (!map) {
|
|
map = new Map()
|
|
session.state.set(SESSIONS_KEY, map)
|
|
}
|
|
return map
|
|
}
|
|
|
|
function resolveSessionId(args) {
|
|
return args.sessionId || args.terminalId || args.containerId || 'default'
|
|
}
|
|
|
|
/**
|
|
* Human-readable shell label for logs / client banner.
|
|
* Never include multi-line probe scripts (those dump into the PTY UI).
|
|
* @param {string[]} cmd
|
|
* @returns {string}
|
|
*/
|
|
export function describeShellCmd(cmd) {
|
|
if (!Array.isArray(cmd) || cmd.length === 0) return 'shell'
|
|
const cIdx = cmd.indexOf('-c')
|
|
if (cIdx >= 0) {
|
|
const script = String(cmd[cIdx + 1] || '')
|
|
const isProbe =
|
|
script.includes('for s in') ||
|
|
script.includes('\n') ||
|
|
script.length > 80
|
|
if (isProbe) {
|
|
const launcher = cmd.slice(0, cIdx).join(' ') || cmd[0]
|
|
return `${launcher} (auto)`
|
|
}
|
|
}
|
|
// busybox sh, etc.
|
|
if (cmd.length <= 3) return cmd.join(' ')
|
|
return cmd[0]
|
|
}
|
|
|
|
/**
|
|
* @param {Buffer|Uint8Array} chunk
|
|
*/
|
|
function toBase64(chunk) {
|
|
return Buffer.isBuffer(chunk) ? chunk.toString('base64') : Buffer.from(chunk).toString('base64')
|
|
}
|
|
|
|
/**
|
|
* Build ordered unique shell command lists for this startTerminal call.
|
|
* @param {object} args
|
|
* @returns {string[][]}
|
|
*/
|
|
export function buildShellCandidates(args = {}) {
|
|
/** @type {string[][]} */
|
|
const out = []
|
|
const seen = new Set()
|
|
|
|
const push = (cmd) => {
|
|
if (!Array.isArray(cmd) || !cmd.length) return
|
|
const key = cmd.join('\0')
|
|
if (seen.has(key)) return
|
|
seen.add(key)
|
|
out.push(cmd)
|
|
}
|
|
|
|
// Explicit user override first (still fall back if it fails)
|
|
if (Array.isArray(args.cmd) && args.cmd.length) {
|
|
push(args.cmd.map(String))
|
|
} else if (typeof args.cmd === 'string' && args.cmd.trim()) {
|
|
push(args.cmd.trim().split(/\s+/))
|
|
} else if (args.shell) {
|
|
push([String(args.shell)])
|
|
}
|
|
|
|
// Prefer adaptive probe wrappers, then each concrete shell
|
|
for (const c of PROBE_WRAPPERS) push(c)
|
|
for (const c of DEFAULT_SHELL_CANDIDATES) push([...c])
|
|
|
|
return out
|
|
}
|
|
|
|
/**
|
|
* Safely tear down a failed probe stream / exec.
|
|
* @param {import('stream').Duplex|null|undefined} stream
|
|
*/
|
|
function destroyStream(stream) {
|
|
if (!stream) return
|
|
try {
|
|
stream.removeAllListeners?.()
|
|
} catch {
|
|
// ignore
|
|
}
|
|
try {
|
|
stream.end?.()
|
|
} catch {
|
|
// ignore
|
|
}
|
|
try {
|
|
stream.destroy?.()
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|
|
|
|
/**
|
|
* After exec.start(), decide whether the process is a live shell.
|
|
* Docker often creates exec configs for missing binaries; they exit 126/127 immediately.
|
|
*
|
|
* @param {object} exec dockerode Exec
|
|
* @param {import('stream').Duplex} stream
|
|
* @param {number} [timeoutMs]
|
|
* @returns {Promise<{ ok: boolean, exitCode: number|null|undefined, buffered: Buffer[] }>}
|
|
*/
|
|
async function probeShellAlive(exec, stream, timeoutMs = SHELL_PROBE_MS) {
|
|
/** @type {Buffer[]} */
|
|
const buffered = []
|
|
let ended = false
|
|
let streamError = null
|
|
|
|
const onData = (chunk) => {
|
|
buffered.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
|
|
}
|
|
const onEnd = () => {
|
|
ended = true
|
|
}
|
|
const onError = (err) => {
|
|
streamError = err
|
|
ended = true
|
|
}
|
|
|
|
stream.on('data', onData)
|
|
stream.on('end', onEnd)
|
|
stream.on('error', onError)
|
|
stream.on('close', onEnd)
|
|
|
|
const deadline = Date.now() + timeoutMs
|
|
/** @type {number|null|undefined} */
|
|
let exitCode
|
|
let running = false
|
|
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
const info = await exec.inspect()
|
|
running = Boolean(info.Running)
|
|
exitCode = info.ExitCode
|
|
if (running) {
|
|
// Live process — interactive shell waiting for input
|
|
detachProbeListeners(stream, onData, onEnd, onError)
|
|
return { ok: true, exitCode, buffered }
|
|
}
|
|
// Not running: if it already exited, treat as failure (try next shell)
|
|
if (info.Pid === 0 || exitCode != null || ended) {
|
|
detachProbeListeners(stream, onData, onEnd, onError)
|
|
return { ok: false, exitCode: exitCode ?? (streamError ? -1 : 127), buffered }
|
|
}
|
|
} catch {
|
|
// inspect can race; keep polling
|
|
}
|
|
if (ended && !running) {
|
|
detachProbeListeners(stream, onData, onEnd, onError)
|
|
return { ok: false, exitCode: exitCode ?? 127, buffered }
|
|
}
|
|
await sleep(SHELL_PROBE_STEP_MS)
|
|
}
|
|
|
|
// Timeout: accept if still running or we got output (prompt)
|
|
try {
|
|
const info = await exec.inspect()
|
|
running = Boolean(info.Running)
|
|
exitCode = info.ExitCode
|
|
} catch {
|
|
// ignore
|
|
}
|
|
detachProbeListeners(stream, onData, onEnd, onError)
|
|
if (running || buffered.length > 0) {
|
|
return { ok: true, exitCode, buffered }
|
|
}
|
|
return { ok: false, exitCode: exitCode ?? 127, buffered }
|
|
}
|
|
|
|
function detachProbeListeners(stream, onData, onEnd, onError) {
|
|
try {
|
|
stream.off?.('data', onData) || stream.removeListener?.('data', onData)
|
|
stream.off?.('end', onEnd) || stream.removeListener?.('end', onEnd)
|
|
stream.off?.('close', onEnd) || stream.removeListener?.('close', onEnd)
|
|
stream.off?.('error', onError) || stream.removeListener?.('error', onError)
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Try shell candidates until one stays running.
|
|
* @param {import('dockerode').Container} container
|
|
* @param {string[][]} candidates
|
|
* @param {boolean} useTty
|
|
* @returns {Promise<{ exec: object, stream: import('stream').Duplex, cmd: string[], buffered: Buffer[] }>}
|
|
*/
|
|
export async function openShellExec(container, candidates, useTty) {
|
|
/** @type {Error|null} */
|
|
let lastErr = null
|
|
const tried = []
|
|
|
|
for (const Cmd of candidates) {
|
|
const label = Cmd.join(' ')
|
|
tried.push(label)
|
|
let stream = null
|
|
try {
|
|
const exec = await container.exec({
|
|
Cmd,
|
|
AttachStdin: true,
|
|
AttachStdout: true,
|
|
AttachStderr: true,
|
|
Tty: useTty,
|
|
Env: ['TERM=xterm-256color', 'COLORTERM=truecolor'],
|
|
})
|
|
stream = await exec.start({
|
|
hijack: true,
|
|
stdin: true,
|
|
Tty: useTty,
|
|
})
|
|
|
|
const probe = await probeShellAlive(exec, stream)
|
|
if (!probe.ok) {
|
|
logger.debug('terminal shell candidate rejected', {
|
|
cmd: label,
|
|
exitCode: probe.exitCode,
|
|
})
|
|
destroyStream(stream)
|
|
lastErr = new Error(
|
|
`Shell not available: ${label}` +
|
|
(probe.exitCode != null ? ` (exit ${probe.exitCode})` : '')
|
|
)
|
|
continue
|
|
}
|
|
|
|
// Pause until live push handlers are attached so no bytes are dropped
|
|
try {
|
|
stream.pause?.()
|
|
} catch {
|
|
// ignore
|
|
}
|
|
|
|
logger.info('terminal shell selected', { cmd: label })
|
|
return { exec, stream, cmd: Cmd, buffered: probe.buffered }
|
|
} catch (err) {
|
|
lastErr = err instanceof Error ? err : new Error(String(err))
|
|
destroyStream(stream)
|
|
logger.debug('terminal shell candidate error', {
|
|
cmd: label,
|
|
error: lastErr.message,
|
|
})
|
|
}
|
|
}
|
|
|
|
const detail = tried.length ? ` Tried: ${tried.slice(0, 12).join(' · ')}${tried.length > 12 ? ' …' : ''}` : ''
|
|
throw (
|
|
lastErr ||
|
|
new Error(
|
|
`No working shell found in container.${detail} Install bash/sh or set startTerminal cmd.`
|
|
)
|
|
)
|
|
}
|
|
|
|
export function registerTerminalHandlers(session) {
|
|
session.respond('startTerminal', async (args) => {
|
|
const containerId = args.containerId
|
|
if (!containerId) throw new Error('containerId required')
|
|
|
|
const sessions = getSessions(session)
|
|
const sessionId = resolveSessionId(args)
|
|
|
|
if (sessions.has(sessionId)) {
|
|
endOne(sessions, sessionId)
|
|
}
|
|
|
|
const useTty = args.tty !== false
|
|
const shellCandidates = buildShellCandidates(args)
|
|
const container = docker.getContainer(containerId)
|
|
|
|
const { exec, stream, cmd, buffered } = await openShellExec(
|
|
container,
|
|
shellCandidates,
|
|
useTty
|
|
)
|
|
|
|
const entry = {
|
|
containerId,
|
|
exec,
|
|
stream,
|
|
sessionId,
|
|
tty: useTty,
|
|
cmd,
|
|
}
|
|
sessions.set(sessionId, entry)
|
|
session.state.set('terminal', entry)
|
|
|
|
const pushOut = (chunk, isErr = false) => {
|
|
const channel = isErr ? Pushes.terminalErrorOutput : Pushes.terminalOutput
|
|
const type = isErr ? 'terminalErrorOutput' : 'terminalOutput'
|
|
try {
|
|
session.push(channel, {
|
|
type,
|
|
containerId,
|
|
sessionId,
|
|
data: toBase64(chunk),
|
|
encoding: 'base64',
|
|
})
|
|
} catch (e) {
|
|
logger.debug('terminal push failed', { error: e.message })
|
|
}
|
|
}
|
|
|
|
// Flush any prompt bytes captured during the probe window, then live stream
|
|
for (const chunk of buffered) {
|
|
pushOut(chunk, false)
|
|
}
|
|
|
|
if (useTty) {
|
|
// Raw PTY — do NOT demux (would corrupt ANSI / binary)
|
|
stream.on('data', (chunk) => pushOut(chunk, false))
|
|
} else {
|
|
const stdout = new PassThrough()
|
|
const stderr = new PassThrough()
|
|
container.modem.demuxStream(stream, stdout, stderr)
|
|
stdout.on('data', (chunk) => pushOut(chunk, false))
|
|
stderr.on('data', (chunk) => pushOut(chunk, true))
|
|
}
|
|
try {
|
|
stream.resume?.()
|
|
} catch {
|
|
// ignore
|
|
}
|
|
|
|
stream.on('end', () => {
|
|
sessions.delete(sessionId)
|
|
if (session.state.get('terminal') === entry) session.state.delete('terminal')
|
|
})
|
|
stream.on('error', (err) => {
|
|
logger.error('Terminal stream error', { containerId, error: err.message })
|
|
sessions.delete(sessionId)
|
|
})
|
|
|
|
const cols = Number(args.cols)
|
|
const rows = Number(args.rows)
|
|
if (cols > 0 && rows > 0) {
|
|
try {
|
|
await exec.resize({ h: rows, w: cols })
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|
|
|
|
const shellLabel = describeShellCmd(cmd)
|
|
logger.info('Terminal session started', {
|
|
containerId,
|
|
sessionId,
|
|
tty: useTty,
|
|
shell: shellLabel,
|
|
peer: session.id.slice(0, 12),
|
|
})
|
|
return {
|
|
success: true,
|
|
message: `Terminal started for ${containerId}`,
|
|
sessionId,
|
|
containerId,
|
|
tty: useTty,
|
|
shell: shellLabel,
|
|
cmd: Array.isArray(cmd) ? cmd.map((p) => (String(p).length > 80 ? `${String(p).slice(0, 40)}…` : p)) : cmd,
|
|
}
|
|
})
|
|
|
|
// Hot path: events (id=0) — keep handler sync/cheap; no audit/metrics in session layer
|
|
session.respond(
|
|
'terminalInput',
|
|
(args) => {
|
|
const sessions = getSessions(session)
|
|
const sessionId = resolveSessionId(args)
|
|
const entry = sessions.get(sessionId) || session.state.get('terminal')
|
|
if (!entry) return null
|
|
if (args.containerId && args.containerId !== entry.containerId) return null
|
|
|
|
let inputData
|
|
if (args.encoding === 'base64') {
|
|
inputData = Buffer.from(args.data || '', 'base64')
|
|
} else {
|
|
// Default utf8 string in JSON (optimal for keystrokes)
|
|
inputData = Buffer.from(args.data || '', 'utf8')
|
|
}
|
|
if (inputData.length && entry.stream && !entry.stream.writableEnded) {
|
|
entry.stream.write(inputData)
|
|
}
|
|
return null
|
|
},
|
|
{ hot: true }
|
|
)
|
|
|
|
session.respond(
|
|
'terminalResize',
|
|
async (args) => {
|
|
const sessions = getSessions(session)
|
|
const sessionId = resolveSessionId(args)
|
|
const entry = sessions.get(sessionId) || session.state.get('terminal')
|
|
if (!entry) return null
|
|
if (args.containerId && args.containerId !== entry.containerId) return null
|
|
const cols = Number(args.cols)
|
|
const rows = Number(args.rows)
|
|
if (cols > 1 && rows > 0) {
|
|
try {
|
|
await entry.exec.resize({ h: rows, w: cols })
|
|
} catch {
|
|
// ignore transient resize errors
|
|
}
|
|
}
|
|
return null
|
|
},
|
|
{ hot: true }
|
|
)
|
|
|
|
session.respond('killTerminal', async (args) => {
|
|
const sessions = getSessions(session)
|
|
const sessionId = args.sessionId || args.terminalId
|
|
if (sessionId && sessions.has(sessionId)) {
|
|
const entry = sessions.get(sessionId)
|
|
endOne(sessions, sessionId)
|
|
return {
|
|
success: true,
|
|
message: `Terminal session ${sessionId} killed`,
|
|
sessionId,
|
|
containerId: entry.containerId,
|
|
}
|
|
}
|
|
if (args.containerId) {
|
|
const killed = cleanupTerminalsForContainer(session, args.containerId)
|
|
if (killed) {
|
|
return { success: true, message: `Killed ${killed} terminal(s) for ${args.containerId}` }
|
|
}
|
|
}
|
|
const entry = session.state.get('terminal')
|
|
if (entry) {
|
|
endOne(sessions, entry.sessionId || 'default')
|
|
session.state.delete('terminal')
|
|
return { success: true, message: 'Terminal killed' }
|
|
}
|
|
return { success: false, message: 'No terminal session found' }
|
|
})
|
|
|
|
/** Used by removeContainer / recreate before Docker delete. */
|
|
session._cleanupTerminalsForContainer = (containerId) => {
|
|
cleanupTerminalsForContainer(session, containerId)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param {import('../rpc/session.js').PeerSession} session
|
|
* @param {string} containerId
|
|
* @returns {number} sessions ended
|
|
*/
|
|
function cleanupTerminalsForContainer(session, containerId) {
|
|
if (!containerId) return 0
|
|
const sessions = getSessions(session)
|
|
const needle = String(containerId)
|
|
let killed = 0
|
|
for (const [id, entry] of [...sessions.entries()]) {
|
|
const cid = String(entry?.containerId || '')
|
|
if (
|
|
cid === needle ||
|
|
(needle.length >= 12 && cid.startsWith(needle)) ||
|
|
(cid.length >= 12 && needle.startsWith(cid.slice(0, 12)))
|
|
) {
|
|
endOne(sessions, id)
|
|
killed += 1
|
|
}
|
|
}
|
|
const legacy = session.state.get('terminal')
|
|
if (legacy) {
|
|
const cid = String(legacy.containerId || '')
|
|
if (
|
|
cid === needle ||
|
|
(needle.length >= 12 && cid.startsWith(needle)) ||
|
|
(cid.length >= 12 && needle.startsWith(cid.slice(0, 12)))
|
|
) {
|
|
endOne(sessions, legacy.sessionId || 'default')
|
|
session.state.delete('terminal')
|
|
killed += 1
|
|
}
|
|
}
|
|
return killed
|
|
}
|
|
|
|
function endOne(sessions, sessionId) {
|
|
const entry = sessions.get(sessionId)
|
|
if (!entry) return
|
|
try {
|
|
entry.stream.end()
|
|
} catch {
|
|
// ignore
|
|
}
|
|
try {
|
|
entry.stream.destroy?.()
|
|
} catch {
|
|
// ignore
|
|
}
|
|
sessions.delete(sessionId)
|
|
}
|
|
|
|
export function endTerminal(session) {
|
|
const sessions = session.state.get(SESSIONS_KEY)
|
|
if (sessions) {
|
|
for (const id of [...sessions.keys()]) endOne(sessions, id)
|
|
}
|
|
session.state.delete('terminal')
|
|
session.state.delete(SESSIONS_KEY)
|
|
}
|
|
|
|
export function cleanupTerminalOnClose(session) {
|
|
endTerminal(session)
|
|
}
|