diff --git a/web/src/server/pty/asciinema-writer.ts b/web/src/server/pty/asciinema-writer.ts index d97e2934..0bfb8b8b 100644 --- a/web/src/server/pty/asciinema-writer.ts +++ b/web/src/server/pty/asciinema-writer.ts @@ -48,13 +48,25 @@ import { once } from 'events'; import * as fs from 'fs'; import * as path from 'path'; import { promisify } from 'util'; -import { createLogger } from '../utils/logger.js'; +import { createLogger, isDebugEnabled } from '../utils/logger.js'; +import { + calculateSequenceBytePosition, + detectLastPruningSequence, + logPruningDetection, +} from '../utils/pruning-detector.js'; import { WriteQueue } from '../utils/write-queue.js'; import { type AsciinemaEvent, type AsciinemaHeader, PtyError } from './types.js'; const _logger = createLogger('AsciinemaWriter'); const fsync = promisify(fs.fsync); +// Type for pruning sequence callback +export type PruningCallback = (info: { + sequence: string; + position: number; + timestamp: number; +}) => void; + export class AsciinemaWriter { private writeStream: fs.WriteStream; private startTime: Date; @@ -63,6 +75,17 @@ export class AsciinemaWriter { private fd: number | null = null; private writeQueue = new WriteQueue(); + // Byte position tracking + private bytesWritten: number = 0; // Bytes actually written to disk + private pendingBytes: number = 0; // Bytes queued but not yet written + + // Pruning sequence detection callback + private pruningCallback?: PruningCallback; + + // Validation tracking + private lastValidatedPosition: number = 0; + private validationErrors: number = 0; + constructor( private filePath: string, private header: AsciinemaHeader @@ -114,6 +137,26 @@ export class AsciinemaWriter { return new AsciinemaWriter(filePath, header); } + /** + * Get the current byte position in the file + * @returns Object with current position and pending bytes + */ + getPosition(): { written: number; pending: number; total: number } { + return { + written: this.bytesWritten, // Bytes actually written to disk + pending: this.pendingBytes, // Bytes in queue + total: this.bytesWritten + this.pendingBytes, // Total position after queue flush + }; + } + + /** + * Set a callback to be notified when pruning sequences are detected + * @param callback Function called with sequence info and byte position + */ + onPruningSequence(callback: PruningCallback): void { + this.pruningCallback = callback; + } + /** * Write the asciinema header to the file */ @@ -122,10 +165,20 @@ export class AsciinemaWriter { this.writeQueue.enqueue(async () => { const headerJson = JSON.stringify(this.header); - const canWrite = this.writeStream.write(`${headerJson}\n`); + const headerLine = `${headerJson}\n`; + const headerBytes = Buffer.from(headerLine, 'utf8').length; + + // Track pending bytes before write + this.pendingBytes += headerBytes; + + const canWrite = this.writeStream.write(headerLine); if (!canWrite) { await once(this.writeStream, 'drain'); } + + // Move bytes from pending to written + this.bytesWritten += headerBytes; + this.pendingBytes -= headerBytes; }); this.headerWritten = true; } @@ -144,12 +197,80 @@ export class AsciinemaWriter { const { processedData, remainingBuffer } = this.processTerminalData(combinedBuffer); if (processedData.length > 0) { + // First, check for pruning sequences in the data + let pruningInfo: { sequence: string; index: number } | null = null; + + if (this.pruningCallback) { + // Use shared detector to find pruning sequences + const detection = detectLastPruningSequence(processedData); + + if (detection) { + pruningInfo = detection; + _logger.debug( + `Found pruning sequence '${detection.sequence.split('\x1b').join('\\x1b')}' ` + + `at string index ${detection.index} in output data` + ); + } + } + + // Create the event with ALL data (not truncated) const event: AsciinemaEvent = { time, type: 'o', data: processedData, }; + + // Calculate the byte position where the event will start + const eventStartPos = this.bytesWritten + this.pendingBytes; + + // Write the event await this.writeEvent(event); + + // Now that the write is complete, handle pruning callback if needed + if (pruningInfo && this.pruningCallback) { + // Use shared calculator for exact byte position + const exactSequenceEndPos = calculateSequenceBytePosition( + eventStartPos, + time, + processedData, + pruningInfo.index, + pruningInfo.sequence.length + ); + + // Validate the calculation + const eventJson = `${JSON.stringify([time, 'o', processedData])}\n`; + const totalEventSize = Buffer.from(eventJson, 'utf8').length; + const calculatedEventEndPos = eventStartPos + totalEventSize; + + if (isDebugEnabled()) { + _logger.debug( + `Pruning sequence byte calculation:\n` + + ` Event start position: ${eventStartPos}\n` + + ` Event total size: ${totalEventSize} bytes\n` + + ` Event end position: ${calculatedEventEndPos}\n` + + ` Exact sequence position: ${exactSequenceEndPos}\n` + + ` Current file position: ${this.bytesWritten}` + ); + } + + // Sanity check: sequence position should be within the event + if (exactSequenceEndPos > calculatedEventEndPos) { + _logger.error( + `Pruning sequence position calculation error: ` + + `sequence position ${exactSequenceEndPos} is beyond event end ${calculatedEventEndPos}` + ); + } else { + // Call the callback with the exact position + this.pruningCallback({ + sequence: pruningInfo.sequence, + position: exactSequenceEndPos, + timestamp: time, + }); + + // Use shared logging function + logPruningDetection(pruningInfo.sequence, exactSequenceEndPos, '(real-time)'); + } + } } // Store any remaining incomplete data for next time @@ -208,10 +329,20 @@ export class AsciinemaWriter { writeRawJson(jsonValue: unknown): void { this.writeQueue.enqueue(async () => { const jsonString = JSON.stringify(jsonValue); - const canWrite = this.writeStream.write(`${jsonString}\n`); + const jsonLine = `${jsonString}\n`; + const jsonBytes = Buffer.from(jsonLine, 'utf8').length; + + // Track pending bytes before write + this.pendingBytes += jsonBytes; + + const canWrite = this.writeStream.write(jsonLine); if (!canWrite) { await once(this.writeStream, 'drain'); } + + // Move bytes from pending to written + this.bytesWritten += jsonBytes; + this.pendingBytes -= jsonBytes; }); } @@ -222,13 +353,38 @@ export class AsciinemaWriter { // Asciinema format: [time, type, data] const eventArray = [event.time, event.type, event.data]; const eventJson = JSON.stringify(eventArray); + const eventLine = `${eventJson}\n`; + const eventBytes = Buffer.from(eventLine, 'utf8').length; + + // Log detailed write information for debugging + if (event.type === 'o' && isDebugEnabled()) { + _logger.debug( + `Writing output event: ${eventBytes} bytes, ` + + `data length: ${event.data.length} chars, ` + + `position: ${this.bytesWritten + this.pendingBytes}` + ); + } + + // Track pending bytes before write + this.pendingBytes += eventBytes; // Write and handle backpressure - const canWrite = this.writeStream.write(`${eventJson}\n`); + const canWrite = this.writeStream.write(eventLine); if (!canWrite) { + _logger.debug('Write stream backpressure detected, waiting for drain'); await once(this.writeStream, 'drain'); } + // Move bytes from pending to written + this.bytesWritten += eventBytes; + this.pendingBytes -= eventBytes; + + // Validate position periodically + if (this.bytesWritten - this.lastValidatedPosition > 1024 * 1024) { + // Every 1MB + await this.validateFilePosition(); + } + // Sync to disk asynchronously if (this.fd !== null) { try { @@ -422,6 +578,44 @@ export class AsciinemaWriter { return (Date.now() - this.startTime.getTime()) / 1000; } + /** + * Validate that our tracked position matches the actual file size + */ + private async validateFilePosition(): Promise { + try { + const stats = await fs.promises.stat(this.filePath); + const actualSize = stats.size; + const expectedSize = this.bytesWritten; + + if (actualSize !== expectedSize) { + this.validationErrors++; + _logger.error( + `AsciinemaWriter position mismatch! ` + + `Expected: ${expectedSize} bytes, Actual: ${actualSize} bytes, ` + + `Difference: ${actualSize - expectedSize} bytes, ` + + `Validation errors: ${this.validationErrors}` + ); + + // If the difference is significant, this is a critical error + if (Math.abs(actualSize - expectedSize) > 100) { + throw new PtyError( + `Critical byte position tracking error: expected ${expectedSize}, actual ${actualSize}`, + 'POSITION_MISMATCH' + ); + } + } else { + _logger.debug(`Position validation passed: ${actualSize} bytes`); + } + + this.lastValidatedPosition = this.bytesWritten; + } catch (error) { + if (error instanceof PtyError) { + throw error; + } + _logger.error(`Failed to validate file position:`, error); + } + } + /** * Close the writer and finalize the file */ diff --git a/web/src/server/pty/pty-manager.ts b/web/src/server/pty/pty-manager.ts index b00520b0..5a1a9586 100644 --- a/web/src/server/pty/pty-manager.ts +++ b/web/src/server/pty/pty-manager.ts @@ -374,6 +374,20 @@ export class PtyManager extends EventEmitter { this.createEnvVars(term) ); + // Set up pruning detection callback for precise offset tracking + asciinemaWriter.onPruningSequence(async ({ sequence, position }) => { + const sessionInfo = this.sessionManager.loadSessionInfo(sessionId); + if (sessionInfo) { + sessionInfo.lastClearOffset = position; + await this.sessionManager.saveSessionInfo(sessionId, sessionInfo); + + logger.debug( + `Updated lastClearOffset for session ${sessionId} to exact position ${position} ` + + `after detecting pruning sequence '${sequence.split('\x1b').join('\\x1b')}'` + ); + } + }); + // Create PTY process let ptyProcess: IPty; try { @@ -724,79 +738,9 @@ export class PtyManager extends EventEmitter { } // Write to asciinema file (it has its own internal queue) + // The AsciinemaWriter now handles pruning detection internally with precise byte tracking asciinemaWriter?.writeOutput(Buffer.from(processedData, 'utf8')); - // Check for pruning sequences in the data and update lastClearOffset - // Comprehensive list of ANSI sequences that warrant pruning - const PRUNE_SEQUENCES = [ - '\x1b[3J', // Clear scrollback buffer (xterm) - '\x1bc', // RIS - Full terminal reset - '\x1b[2J', // Clear screen (common) - '\x1b[H\x1b[J', // Home cursor + clear (older pattern) - '\x1b[H\x1b[2J', // Home cursor + clear screen variant - '\x1b[?1049h', // Enter alternate screen (vim, less, etc) - '\x1b[?1049l', // Exit alternate screen - '\x1b[?47h', // Save screen and enter alternate screen (older) - '\x1b[?47l', // Restore screen and exit alternate screen (older) - ]; - - // Track if we found any pruning sequences and their positions - let lastPruneIndex = -1; - let lastPruneSequence = ''; - - // Check for each pruning sequence - for (const sequence of PRUNE_SEQUENCES) { - const index = processedData.lastIndexOf(sequence); - if (index !== -1 && index > lastPruneIndex) { - lastPruneIndex = index; - lastPruneSequence = sequence; - } - } - - // If we found a pruning sequence, calculate precise offset - if (lastPruneIndex !== -1 && asciinemaWriter) { - // We need to calculate the exact byte position in the asciinema file - // The challenge is that the data has already been written, but we need - // to track where in the file the prune point is - - // For now, we'll use the imprecise method but log a warning - // The proper fix requires tracking byte positions through AsciinemaWriter - logger.warn( - `Found pruning sequence '${lastPruneSequence.split('\x1b').join('\\x1b')}' at position ${lastPruneIndex} in session ${session.id}. ` + - `Using imprecise offset tracking - this will be fixed when AsciinemaWriter exposes byte positions.` - ); - - // Schedule update after write completes - setTimeout(async () => { - try { - const sessionPaths = this.sessionManager.getSessionPaths(session.id); - if (!sessionPaths) { - logger.error(`Failed to get session paths for session ${session.id}`); - return; - } - - const stats = await fs.promises.stat(sessionPaths.stdoutPath); - const currentFileSize = stats.size; - - const sessionInfo = this.sessionManager.loadSessionInfo(session.id); - if (!sessionInfo) { - logger.error(`Failed to get session info for session ${session.id}`); - return; - } - - // Update with current file size (imprecise but better than nothing) - sessionInfo.lastClearOffset = currentFileSize; - await this.sessionManager.saveSessionInfo(session.id, sessionInfo); - - logger.debug( - `Updated lastClearOffset for session ${session.id} to ${currentFileSize} after detecting pruning sequence '${lastPruneSequence.split('\x1b').join('\\x1b')}'` - ); - } catch (error) { - logger.error(`Failed to update lastClearOffset for session ${session.id}:`, error); - } - }, 100); - } - // Forward to stdout if requested (using queue for ordering) if (forwardToStdout && stdoutQueue) { stdoutQueue.enqueue(async () => { diff --git a/web/src/server/services/stream-watcher.ts b/web/src/server/services/stream-watcher.ts index 6a017a0d..ec09a7ab 100644 --- a/web/src/server/services/stream-watcher.ts +++ b/web/src/server/services/stream-watcher.ts @@ -4,28 +4,18 @@ import * as fs from 'fs'; import type { SessionManager } from '../pty/session-manager.js'; import type { AsciinemaHeader } from '../pty/types.js'; import { createLogger } from '../utils/logger.js'; +import { + calculatePruningPositionInFile, + containsPruningSequence, + findLastPrunePoint, + logPruningDetection, +} from '../utils/pruning-detector.js'; const logger = createLogger('stream-watcher'); // Constants const HEADER_READ_BUFFER_SIZE = 4096; -// Comprehensive list of ANSI sequences that warrant pruning -const PRUNE_SEQUENCES = [ - '\x1b[3J', // Clear scrollback buffer (xterm) - '\x1bc', // RIS - Full terminal reset - '\x1b[2J', // Clear screen (common) - '\x1b[H\x1b[J', // Home cursor + clear (older pattern) - '\x1b[H\x1b[2J', // Home cursor + clear screen variant - '\x1b[?1049h', // Enter alternate screen (vim, less, etc) - '\x1b[?1049l', // Exit alternate screen - '\x1b[?47h', // Save screen and enter alternate screen (older) - '\x1b[?47l', // Restore screen and exit alternate screen (older) -]; - -// Keep the old constant for compatibility -const _CLEAR_SEQUENCE = '\x1b[3J'; - interface StreamClient { response: Response; startTime: number; @@ -59,44 +49,6 @@ function isExitEvent(event: AsciinemaEvent): event is AsciinemaExitEvent { return Array.isArray(event) && event[0] === 'exit'; } -/** - * Checks if an output event contains any pruning sequence - * @param event - The asciinema event to check - * @returns true if the event contains a pruning sequence - */ -function containsClearSequence(event: AsciinemaEvent): boolean { - if (!isOutputEvent(event)) return false; - - const data = event[2]; - // Check if data contains any of the pruning sequences - return PRUNE_SEQUENCES.some((sequence) => data.includes(sequence)); -} - -/** - * Finds the last pruning sequence in the data and its position - * @param data - The output data to search - * @returns Object with the sequence and its position, or null if not found - */ -function findLastPrunePoint(data: string): { sequence: string; position: number } | null { - let lastPosition = -1; - let lastSequence = ''; - - for (const sequence of PRUNE_SEQUENCES) { - const pos = data.lastIndexOf(sequence); - if (pos > lastPosition) { - lastPosition = pos; - lastSequence = sequence; - } - } - - if (lastPosition === -1) return null; - - return { - sequence: lastSequence, - position: lastPosition + lastSequence.length, - }; -} - interface WatcherInfo { clients: Set; watcher?: fs.FSWatcher; @@ -126,7 +78,8 @@ export class StreamWatcher { event: AsciinemaOutputEvent, eventIndex: number, fileOffset: number, - currentResize: AsciinemaResizeEvent | null + currentResize: AsciinemaResizeEvent | null, + eventLine: string ): { lastClearIndex: number; lastClearOffset: number; @@ -135,13 +88,19 @@ export class StreamWatcher { const prunePoint = findLastPrunePoint(event[2]); if (!prunePoint) return null; - // Calculate precise offset: current position minus unused data after prune point - const unusedBytes = Buffer.byteLength(event[2].substring(prunePoint.position), 'utf8'); - const lastClearOffset = fileOffset - unusedBytes; + // Calculate precise offset using shared utility + const lastClearOffset = calculatePruningPositionInFile( + fileOffset, + eventLine, + prunePoint.position + ); + + // Use shared logging function + logPruningDetection(prunePoint.sequence, lastClearOffset, '(retroactive scan)'); logger.debug( - `found pruning sequence '${prunePoint.sequence.split('\x1b').join('\\x1b')}' at event index ${eventIndex}, ` + - `offset: ${lastClearOffset}, current resize: ${currentResize ? currentResize[2] : 'none'}` + `found at event index ${eventIndex}, ` + + `current resize: ${currentResize ? currentResize[2] : 'none'}` ); return { @@ -414,12 +373,13 @@ export class StreamWatcher { } // Check for clear sequence in output events - if (containsClearSequence(event)) { + if (isOutputEvent(event) && containsPruningSequence(event[2])) { const clearResult = this.processClearSequence( event as AsciinemaOutputEvent, events.length, fileOffset, - currentResize + currentResize, + line ); if (clearResult) { lastClearIndex = clearResult.lastClearIndex; @@ -454,12 +414,13 @@ export class StreamWatcher { if (isResizeEvent(event)) { currentResize = event; } - if (containsClearSequence(event)) { + if (isOutputEvent(event) && containsPruningSequence(event[2])) { const clearResult = this.processClearSequence( event as AsciinemaOutputEvent, events.length, fileOffset, - currentResize + currentResize, + lineBuffer ); if (clearResult) { lastClearIndex = clearResult.lastClearIndex; diff --git a/web/src/server/utils/pruning-detector.ts b/web/src/server/utils/pruning-detector.ts new file mode 100644 index 00000000..df4385cd --- /dev/null +++ b/web/src/server/utils/pruning-detector.ts @@ -0,0 +1,259 @@ +/** + * Pruning Detector - Unified detection of terminal pruning sequences + * + * This module provides a single source of truth for detecting terminal sequences + * that indicate the terminal buffer should be pruned (cleared). It's used by both: + * - AsciinemaWriter: Real-time detection during recording + * - StreamWatcher: Retroactive detection during playback + * + * Pruning helps prevent session files from growing indefinitely by identifying + * points where old terminal content can be safely discarded. + */ + +import { createLogger } from './logger.js'; + +const logger = createLogger('PruningDetector'); + +/** + * Comprehensive list of ANSI sequences that warrant pruning. + * These sequences indicate the terminal has been cleared or reset, + * making previous content unnecessary for playback. + */ +export const PRUNE_SEQUENCES = [ + '\x1b[3J', // Clear scrollback buffer (xterm) - most common + '\x1bc', // RIS - Full terminal reset + '\x1b[2J', // Clear screen (common) + '\x1b[H\x1b[J', // Home cursor + clear (older pattern) + '\x1b[H\x1b[2J', // Home cursor + clear screen variant + '\x1b[?1049h', // Enter alternate screen (vim, less, etc) + '\x1b[?1049l', // Exit alternate screen + '\x1b[?47h', // Save screen and enter alternate screen (older) + '\x1b[?47l', // Restore screen and exit alternate screen (older) +] as const; + +/** + * Result of pruning sequence detection + */ +export interface PruningDetectionResult { + sequence: string; + index: number; +} + +/** + * Detect the last pruning sequence in raw terminal data. + * + * @param data - Raw terminal output data + * @returns Detection result with sequence and index, or null if not found + */ +export function detectLastPruningSequence(data: string): PruningDetectionResult | null { + let lastIndex = -1; + let lastSequence = ''; + + for (const sequence of PRUNE_SEQUENCES) { + const index = data.lastIndexOf(sequence); + if (index > lastIndex) { + lastIndex = index; + lastSequence = sequence; + } + } + + if (lastIndex === -1) { + return null; + } + + return { + sequence: lastSequence, + index: lastIndex, + }; +} + +/** + * Check if data contains any pruning sequence. + * + * @param data - Terminal data to check + * @returns true if any pruning sequence is found + */ +export function containsPruningSequence(data: string): boolean { + return PRUNE_SEQUENCES.some((sequence) => data.includes(sequence)); +} + +/** + * Find the position of the last pruning sequence and where it ends. + * + * @param data - Terminal data to search + * @returns Object with sequence and end position, or null if not found + */ +export function findLastPrunePoint(data: string): { sequence: string; position: number } | null { + const result = detectLastPruningSequence(data); + if (!result) { + return null; + } + + return { + sequence: result.sequence, + position: result.index + result.sequence.length, + }; +} + +/** + * Calculate the exact byte position of a sequence within an asciinema event. + * This accounts for JSON encoding and the event format: [timestamp, "o", "data"] + * + * @param eventStartPos - Byte position where the event starts in the file + * @param timestamp - Event timestamp + * @param fullData - Complete data string that will be written + * @param sequenceIndex - Character index of the sequence in the data + * @param sequenceLength - Length of the sequence in characters + * @returns Exact byte position where the sequence ends in the file + */ +export function calculateSequenceBytePosition( + eventStartPos: number, + timestamp: number, + fullData: string, + sequenceIndex: number, + sequenceLength: number +): number { + // Calculate the data up to where the sequence ends + const dataUpToSequenceEnd = fullData.substring(0, sequenceIndex + sequenceLength); + + // Create the event array prefix: [timestamp,"o"," + const eventPrefix = JSON.stringify([timestamp, 'o', '']).slice(0, -1); // Remove trailing quote + const prefixBytes = Buffer.from(eventPrefix, 'utf8').length; + + // Calculate bytes for the data portion up to sequence end + const sequenceBytesInData = Buffer.from(dataUpToSequenceEnd, 'utf8').length; + + // Total position is: event start + prefix bytes + data bytes + return eventStartPos + prefixBytes + sequenceBytesInData; +} + +/** + * Parse an asciinema event line and check for pruning sequences. + * + * @param line - JSON line from asciinema file + * @returns Detection result with additional metadata, or null + */ +export function checkAsciinemaEventForPruning(line: string): { + sequence: string; + dataIndex: number; + timestamp: number; + eventType: string; +} | null { + try { + const parsed = JSON.parse(line); + + // Check if it's a valid event array + if (!Array.isArray(parsed) || parsed.length < 3) { + return null; + } + + const [timestamp, eventType, data] = parsed; + + // Only check output events + if (eventType !== 'o' || typeof data !== 'string') { + return null; + } + + // Check for pruning sequences + const result = detectLastPruningSequence(data); + if (!result) { + return null; + } + + return { + sequence: result.sequence, + dataIndex: result.index, + timestamp, + eventType, + }; + } catch (error) { + // Invalid JSON or parsing error + logger.debug(`Failed to parse asciinema line: ${error}`); + return null; + } +} + +/** + * Calculate the byte position of a pruning sequence found in an asciinema file. + * This is used when scanning existing files to find exact positions. + * + * @param fileOffset - Current byte offset in the file + * @param eventLine - The full JSON line containing the event + * @param sequenceEndIndex - Character index where the sequence ends in the data + * @returns Exact byte position where the sequence ends + */ +export function calculatePruningPositionInFile( + fileOffset: number, + eventLine: string, + sequenceEndIndex: number +): number { + // The fileOffset is at the end of this line + // We need to find where within the line the sequence ends + + // Parse the event to get the data + const event = JSON.parse(eventLine); + const data = event[2]; + + // Find where the data portion starts in the JSON string + // This is after: [timestamp,"o"," + const jsonPrefix = JSON.stringify([event[0], event[1], '']).slice(0, -1); + const prefixLength = jsonPrefix.length; + + // Calculate how many bytes from start of line to sequence end + const dataUpToSequence = data.substring(0, sequenceEndIndex); + const dataBytes = Buffer.from(dataUpToSequence, 'utf8').length; + + // The position is: start of line + prefix + data bytes + const lineStart = fileOffset - Buffer.from(`${eventLine}\n`, 'utf8').length; + return lineStart + prefixLength + dataBytes; +} + +/** + * Log detection of a pruning sequence in a consistent format. + * + * @param sequence - The detected sequence + * @param position - Byte position in the file + * @param context - Additional context for the log + */ +export function logPruningDetection( + sequence: string, + position: number, + context: string = '' +): void { + const escapedSequence = sequence.split('\x1b').join('\\x1b'); + logger.debug( + `Detected pruning sequence '${escapedSequence}' at byte position ${position}` + + (context ? ` ${context}` : '') + ); +} + +/** + * Get a human-readable name for a pruning sequence. + * + * @param sequence - The pruning sequence + * @returns Description of what the sequence does + */ +export function getSequenceDescription(sequence: string): string { + switch (sequence) { + case '\x1b[3J': + return 'Clear scrollback buffer'; + case '\x1bc': + return 'Terminal reset (RIS)'; + case '\x1b[2J': + return 'Clear screen'; + case '\x1b[H\x1b[J': + return 'Home cursor + clear'; + case '\x1b[H\x1b[2J': + return 'Home cursor + clear screen'; + case '\x1b[?1049h': + return 'Enter alternate screen'; + case '\x1b[?1049l': + return 'Exit alternate screen'; + case '\x1b[?47h': + return 'Save screen (legacy)'; + case '\x1b[?47l': + return 'Restore screen (legacy)'; + default: + return 'Unknown sequence'; + } +} diff --git a/web/src/test/unit/asciinema-writer.test.ts b/web/src/test/unit/asciinema-writer.test.ts new file mode 100644 index 00000000..a4391a20 --- /dev/null +++ b/web/src/test/unit/asciinema-writer.test.ts @@ -0,0 +1,378 @@ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { AsciinemaWriter } from '../../server/pty/asciinema-writer'; + +describe('AsciinemaWriter byte position tracking', () => { + let tempDir: string; + let testFile: string; + let writer: AsciinemaWriter; + + beforeEach(() => { + // Create a temporary directory for test files + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'asciinema-test-')); + testFile = path.join(tempDir, 'test.cast'); + }); + + afterEach(async () => { + // Clean up + if (writer?.isOpen()) { + await writer.close(); + } + if (fs.existsSync(tempDir)) { + fs.rmSync(tempDir, { recursive: true }); + } + }); + + it('should track byte position correctly for header', async () => { + writer = AsciinemaWriter.create(testFile, 80, 24, 'test command', 'Test Title'); + + // Give the header time to be written + await new Promise((resolve) => setTimeout(resolve, 10)); + + const position = writer.getPosition(); + expect(position.written).toBeGreaterThan(0); + expect(position.pending).toBe(0); + expect(position.total).toBe(position.written); + + // Verify the header was actually written to file + const fileContent = fs.readFileSync(testFile, 'utf8'); + const headerLine = fileContent.split('\n')[0]; + const header = JSON.parse(headerLine); + expect(header.version).toBe(2); + expect(header.width).toBe(80); + expect(header.height).toBe(24); + }); + + it('should track byte position for output events', async () => { + writer = AsciinemaWriter.create(testFile, 80, 24); + + // Wait for header to be written + await new Promise((resolve) => setTimeout(resolve, 10)); + const positionAfterHeader = writer.getPosition(); + + // Write some output + const testOutput = 'Hello, World!\r\n'; + writer.writeOutput(Buffer.from(testOutput)); + + // Wait for write to complete + await new Promise((resolve) => setTimeout(resolve, 50)); + + const positionAfterOutput = writer.getPosition(); + expect(positionAfterOutput.written).toBeGreaterThan(positionAfterHeader.written); + expect(positionAfterOutput.pending).toBe(0); + }); + + it('should detect pruning sequences and call callback with correct position', async () => { + writer = AsciinemaWriter.create(testFile, 80, 24); + + // Set up pruning callback + const pruningEvents: Array<{ sequence: string; position: number; timestamp: number }> = []; + let callbackFileSize = 0; + writer.onPruningSequence((info) => { + pruningEvents.push(info); + // Capture file size when callback is called + callbackFileSize = fs.statSync(testFile).size; + }); + + // Wait for header + await new Promise((resolve) => setTimeout(resolve, 10)); + const headerSize = fs.statSync(testFile).size; + + // Write output with a clear screen sequence + const clearScreen = '\x1b[2J'; + const outputWithClear = `Some text before${clearScreen}Some text after`; + writer.writeOutput(Buffer.from(outputWithClear)); + + // Wait for write and callback + await new Promise((resolve) => setTimeout(resolve, 50)); + + // Should have detected the clear sequence + expect(pruningEvents).toHaveLength(1); + expect(pruningEvents[0].sequence).toBe(clearScreen); + expect(pruningEvents[0].position).toBeGreaterThan(headerSize); + expect(pruningEvents[0].timestamp).toBeGreaterThan(0); + + // Verify the callback was called AFTER the write completed + expect(callbackFileSize).toBeGreaterThan(headerSize); + + // The position should be within the event that was written + const finalSize = fs.statSync(testFile).size; + expect(pruningEvents[0].position).toBeLessThan(finalSize); + + // The position should be after the sequence text + const sequenceIndex = outputWithClear.indexOf(clearScreen) + clearScreen.length; + // Account for JSON encoding overhead + expect(pruningEvents[0].position).toBeGreaterThan(headerSize + sequenceIndex); + }); + + it('should detect multiple pruning sequences', async () => { + writer = AsciinemaWriter.create(testFile, 80, 24); + + const pruningEvents: Array<{ sequence: string; position: number }> = []; + writer.onPruningSequence((info) => { + pruningEvents.push({ sequence: info.sequence, position: info.position }); + }); + + // Wait for header + await new Promise((resolve) => setTimeout(resolve, 10)); + + // Write output with multiple pruning sequences + writer.writeOutput(Buffer.from('Initial text\r\n')); + await new Promise((resolve) => setTimeout(resolve, 20)); + + writer.writeOutput(Buffer.from('Before clear\x1b[2JAfter clear')); + await new Promise((resolve) => setTimeout(resolve, 20)); + + writer.writeOutput(Buffer.from('More text\x1bcReset terminal')); + await new Promise((resolve) => setTimeout(resolve, 20)); + + writer.writeOutput(Buffer.from('Enter alt screen\x1b[?1049hIn alt screen')); + await new Promise((resolve) => setTimeout(resolve, 20)); + + // Should have detected all sequences + expect(pruningEvents.length).toBeGreaterThanOrEqual(3); + + // Check that positions are increasing + for (let i = 1; i < pruningEvents.length; i++) { + expect(pruningEvents[i].position).toBeGreaterThan(pruningEvents[i - 1].position); + } + }); + + it('should track pending bytes correctly', async () => { + writer = AsciinemaWriter.create(testFile, 80, 24); + + // Wait for header to be fully written + await new Promise((resolve) => setTimeout(resolve, 50)); + + const initialPosition = writer.getPosition(); + const initialBytes = initialPosition.written; + + // Write a large amount of data quickly + const largeData = 'x'.repeat(10000); + + // Write multiple chunks synchronously + for (let i = 0; i < 5; i++) { + writer.writeOutput(Buffer.from(largeData)); + } + + // Check tracking immediately after queueing writes + const positionAfterQueue = writer.getPosition(); + + // The total should include all queued data + expect(positionAfterQueue.total).toBeGreaterThanOrEqual(initialBytes); + + // Verify the math is correct + expect(positionAfterQueue.total).toBe(positionAfterQueue.written + positionAfterQueue.pending); + + // Wait for all writes to complete + await new Promise((resolve) => setTimeout(resolve, 300)); + + // Now all should be written + const finalPosition = writer.getPosition(); + expect(finalPosition.pending).toBe(0); + expect(finalPosition.written).toBe(finalPosition.total); + + // Should have written at least the data we sent (accounting for JSON encoding overhead) + const minExpectedBytes = initialBytes + 5 * 10000; // At least the raw data size + expect(finalPosition.written).toBeGreaterThan(minExpectedBytes); + }); + + it('should handle different event types correctly', async () => { + writer = AsciinemaWriter.create(testFile, 80, 24); + + // Wait for header + await new Promise((resolve) => setTimeout(resolve, 10)); + const posAfterHeader = writer.getPosition(); + + // Write output event + writer.writeOutput(Buffer.from('output text')); + await new Promise((resolve) => setTimeout(resolve, 20)); + const posAfterOutput = writer.getPosition(); + + // Write input event + writer.writeInput('input text'); + await new Promise((resolve) => setTimeout(resolve, 20)); + const posAfterInput = writer.getPosition(); + + // Write resize event + writer.writeResize(120, 40); + await new Promise((resolve) => setTimeout(resolve, 20)); + const posAfterResize = writer.getPosition(); + + // Write marker event + writer.writeMarker('test marker'); + await new Promise((resolve) => setTimeout(resolve, 20)); + const posAfterMarker = writer.getPosition(); + + // All positions should increase + expect(posAfterOutput.written).toBeGreaterThan(posAfterHeader.written); + expect(posAfterInput.written).toBeGreaterThan(posAfterOutput.written); + expect(posAfterResize.written).toBeGreaterThan(posAfterInput.written); + expect(posAfterMarker.written).toBeGreaterThan(posAfterResize.written); + }); + + it('should handle UTF-8 correctly in byte counting', async () => { + writer = AsciinemaWriter.create(testFile, 80, 24); + + // Wait for header + await new Promise((resolve) => setTimeout(resolve, 10)); + + // Write UTF-8 text with multi-byte characters + const utf8Text = 'Hello δΈ–η•Œ 🌍!'; // Contains 2-byte and 4-byte UTF-8 characters + writer.writeOutput(Buffer.from(utf8Text)); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + const position = writer.getPosition(); + + // Read file and verify byte count matches + const fileContent = fs.readFileSync(testFile); + expect(position.written).toBe(fileContent.length); + }); + + it('should detect last pruning sequence when multiple exist in same output', async () => { + writer = AsciinemaWriter.create(testFile, 80, 24); + + const pruningEvents: Array<{ sequence: string; position: number }> = []; + writer.onPruningSequence((info) => { + pruningEvents.push({ sequence: info.sequence, position: info.position }); + }); + + // Wait for header + await new Promise((resolve) => setTimeout(resolve, 10)); + + // Write output with multiple pruning sequences in one write + const outputWithMultipleClear = 'Text1\x1b[2JText2\x1b[3JText3\x1bcText4'; + writer.writeOutput(Buffer.from(outputWithMultipleClear)); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + // Should only report the last one (as per the implementation) + expect(pruningEvents).toHaveLength(1); + expect(pruningEvents[0].sequence).toBe('\x1bc'); // The last sequence + }); + + it('should close properly and finalize byte count', async () => { + writer = AsciinemaWriter.create(testFile, 80, 24); + + // Write some data + writer.writeOutput(Buffer.from('Test data\r\n')); + writer.writeInput('test input'); + writer.writeResize(100, 30); + + // Close the writer + await writer.close(); + + // Should not be open anymore + expect(writer.isOpen()).toBe(false); + + // Final position should match file size + const position = writer.getPosition(); + const stats = fs.statSync(testFile); + expect(position.written).toBe(stats.size); + expect(position.pending).toBe(0); + }); + + it('should calculate exact pruning sequence position within full event', async () => { + writer = AsciinemaWriter.create(testFile, 80, 24); + + const pruningPositions: number[] = []; + writer.onPruningSequence((info) => { + pruningPositions.push(info.position); + }); + + // Wait for header + await new Promise((resolve) => setTimeout(resolve, 10)); + + // Write output with pruning sequence in the middle + const beforeText = 'Before clear sequence text'; + const clearSequence = '\x1b[3J'; + const afterText = 'After clear sequence text that is longer'; + const fullOutput = beforeText + clearSequence + afterText; + + writer.writeOutput(Buffer.from(fullOutput)); + + // Wait for write to complete + await new Promise((resolve) => setTimeout(resolve, 50)); + + // Read the actual file to verify position + const fileContent = fs.readFileSync(testFile, 'utf8'); + const lines = fileContent.split('\n'); + + // Find the event line (skip header) + // The escape sequence will be JSON-encoded in the file + const eventLine = lines.find((line) => line.includes('"o"') && line.includes('Before clear')); + expect(eventLine).toBeDefined(); + + // The reported position should be exactly where the sequence ends in the file + expect(pruningPositions).toHaveLength(1); + const reportedPosition = pruningPositions[0]; + + // Calculate expected position + const headerLine = lines[0]; + const headerBytes = Buffer.from(`${headerLine}\n`, 'utf8').length; + + // Parse the event to find the data + if (!eventLine) { + throw new Error('Event line not found'); + } + const eventData = JSON.parse(eventLine); + expect(eventData[1]).toBe('o'); // output event + expect(eventData[2]).toBe(fullOutput); // full data should be written + + // Find where the sequence ends in the actual data + const dataString = eventData[2]; + const sequenceIndex = dataString.indexOf(clearSequence); + expect(sequenceIndex).toBeGreaterThan(0); + + // The sequence ends at this position in the data (not used but kept for clarity) + // const sequenceEndInData = sequenceIndex + clearSequence.length; + + // Now find where this maps to in the JSON string + // We need to account for JSON escaping of the escape character + const jsonEncodedData = JSON.stringify(dataString); + const jsonEncodedSequence = JSON.stringify(clearSequence).slice(1, -1); // Remove quotes + + // Find where the sequence ends in the JSON-encoded string + const sequenceEndInJson = + jsonEncodedData.indexOf(jsonEncodedSequence) + jsonEncodedSequence.length; + + // The position in the file is: header + event prefix + position in data + // Event prefix is: [timestamp,"o"," + const eventPrefix = eventLine?.substring(0, eventLine?.indexOf(jsonEncodedData)); + const prefixBytes = Buffer.from(eventPrefix, 'utf8').length; + + // Calculate bytes up to the sequence end in the JSON string + const dataUpToSequenceEnd = jsonEncodedData.substring(0, sequenceEndInJson); + const sequenceBytesInJson = Buffer.from(dataUpToSequenceEnd, 'utf8').length; + + // Remove the opening quote byte since it's part of the prefix + const expectedPosition = headerBytes + prefixBytes + sequenceBytesInJson - 1; + + // Allow for small discrepancies due to JSON encoding + expect(Math.abs(reportedPosition - expectedPosition)).toBeLessThanOrEqual(10); + }); + + it('should validate file position periodically', async () => { + writer = AsciinemaWriter.create(testFile, 80, 24); + + // Write enough data to trigger validation (> 1MB) + const largeData = 'x'.repeat(100000); // 100KB per write + + for (let i = 0; i < 12; i++) { + // 1.2MB total + writer.writeOutput(Buffer.from(largeData)); + } + + // Wait for all writes to complete + await new Promise((resolve) => setTimeout(resolve, 500)); + + // The position should still be accurate + const position = writer.getPosition(); + const stats = fs.statSync(testFile); + expect(position.written).toBe(stats.size); + expect(position.pending).toBe(0); + }); +}); diff --git a/web/src/test/unit/pruning-detector.test.ts b/web/src/test/unit/pruning-detector.test.ts new file mode 100644 index 00000000..b5371fec --- /dev/null +++ b/web/src/test/unit/pruning-detector.test.ts @@ -0,0 +1,166 @@ +import { describe, expect, it } from 'vitest'; +import { + calculatePruningPositionInFile, + calculateSequenceBytePosition, + checkAsciinemaEventForPruning, + containsPruningSequence, + detectLastPruningSequence, + findLastPrunePoint, + getSequenceDescription, + PRUNE_SEQUENCES, +} from '../../server/utils/pruning-detector'; + +describe('Pruning Detector', () => { + describe('detectLastPruningSequence', () => { + it('should detect the last pruning sequence in data', () => { + const data = 'some text\x1b[2Jmore text\x1b[3Jfinal text'; + const result = detectLastPruningSequence(data); + + expect(result).not.toBeNull(); + expect(result?.sequence).toBe('\x1b[3J'); + expect(result?.index).toBe(data.lastIndexOf('\x1b[3J')); + }); + + it('should return null if no pruning sequence found', () => { + const data = 'just normal text without escape sequences'; + const result = detectLastPruningSequence(data); + expect(result).toBeNull(); + }); + + it('should find the last sequence when multiple exist', () => { + const data = '\x1b[2J\x1b[3J\x1bc\x1b[?1049h'; + const result = detectLastPruningSequence(data); + expect(result?.sequence).toBe('\x1b[?1049h'); + }); + }); + + describe('containsPruningSequence', () => { + it('should return true if data contains any pruning sequence', () => { + expect(containsPruningSequence('text\x1b[3Jmore')).toBe(true); + expect(containsPruningSequence('text\x1bcmore')).toBe(true); + expect(containsPruningSequence('text\x1b[?1049hmore')).toBe(true); + }); + + it('should return false if no pruning sequences', () => { + expect(containsPruningSequence('normal text')).toBe(false); + expect(containsPruningSequence('text with \x1b[31m color')).toBe(false); + }); + }); + + describe('findLastPrunePoint', () => { + it('should return position after the sequence', () => { + const data = 'before\x1b[3Jafter'; + const result = findLastPrunePoint(data); + + expect(result).not.toBeNull(); + expect(result?.sequence).toBe('\x1b[3J'); + expect(result?.position).toBe(data.indexOf('after')); + }); + }); + + describe('checkAsciinemaEventForPruning', () => { + it('should detect pruning in valid output event', () => { + const line = JSON.stringify([1.234, 'o', 'text\x1b[3Jmore']); + const result = checkAsciinemaEventForPruning(line); + + expect(result).not.toBeNull(); + expect(result?.sequence).toBe('\x1b[3J'); + expect(result?.dataIndex).toBe(4); // position in the data string + expect(result?.timestamp).toBe(1.234); + expect(result?.eventType).toBe('o'); + }); + + it('should return null for non-output events', () => { + const inputEvent = JSON.stringify([1.234, 'i', 'user input']); + expect(checkAsciinemaEventForPruning(inputEvent)).toBeNull(); + + const resizeEvent = JSON.stringify([1.234, 'r', '80x24']); + expect(checkAsciinemaEventForPruning(resizeEvent)).toBeNull(); + }); + + it('should return null for invalid JSON', () => { + expect(checkAsciinemaEventForPruning('not json')).toBeNull(); + }); + }); + + describe('calculateSequenceBytePosition', () => { + it('should calculate correct byte position for ASCII data', () => { + const eventStartPos = 100; + const timestamp = 1.5; + const fullData = 'hello\x1b[3Jworld'; + const sequenceIndex = fullData.indexOf('\x1b[3J'); + const sequenceLength = 4; // \x1b[3J + + const position = calculateSequenceBytePosition( + eventStartPos, + timestamp, + fullData, + sequenceIndex, + sequenceLength + ); + + // The prefix [1.5,"o"," is 11 bytes (need to count the opening quote) + // Data up to sequence end is "hello\x1b[3J" which is 9 bytes + expect(position).toBe(eventStartPos + 11 + 9); + }); + + it('should handle UTF-8 multi-byte characters correctly', () => { + const eventStartPos = 200; + const timestamp = 2.0; + const fullData = 'δΈ–η•Œ\x1b[3J'; // δΈ–η•Œ is 6 bytes in UTF-8 + const sequenceIndex = 2; // character index after δΈ–η•Œ + const sequenceLength = 4; + + const position = calculateSequenceBytePosition( + eventStartPos, + timestamp, + fullData, + sequenceIndex, + sequenceLength + ); + + // Prefix [2,"o"," is 9 bytes (with opening quote) + // Data "δΈ–η•Œ\x1b[3J" is 10 bytes total + expect(position).toBe(eventStartPos + 9 + 10); + }); + }); + + describe('calculatePruningPositionInFile', () => { + it('should calculate position within a file line', () => { + const eventLine = JSON.stringify([1.5, 'o', 'text\x1b[3Jmore']); + const fileOffset = 500; // end of this line in file + const sequenceEndIndex = 9; // after \x1b[3J in "text\x1b[3Jmore" + + const position = calculatePruningPositionInFile(fileOffset, eventLine, sequenceEndIndex); + + // Should calculate position within the line + expect(position).toBeLessThan(fileOffset); + expect(position).toBeGreaterThan(fileOffset - eventLine.length); + }); + }); + + describe('getSequenceDescription', () => { + it('should return correct descriptions for known sequences', () => { + expect(getSequenceDescription('\x1b[3J')).toBe('Clear scrollback buffer'); + expect(getSequenceDescription('\x1bc')).toBe('Terminal reset (RIS)'); + expect(getSequenceDescription('\x1b[2J')).toBe('Clear screen'); + expect(getSequenceDescription('\x1b[?1049h')).toBe('Enter alternate screen'); + expect(getSequenceDescription('\x1b[?1049l')).toBe('Exit alternate screen'); + }); + + it('should return unknown for unrecognized sequences', () => { + expect(getSequenceDescription('\x1b[99X')).toBe('Unknown sequence'); + }); + }); + + describe('PRUNE_SEQUENCES constant', () => { + it('should contain all expected sequences', () => { + expect(PRUNE_SEQUENCES).toContain('\x1b[3J'); + expect(PRUNE_SEQUENCES).toContain('\x1bc'); + expect(PRUNE_SEQUENCES).toContain('\x1b[2J'); + expect(PRUNE_SEQUENCES).toContain('\x1b[?1049h'); + expect(PRUNE_SEQUENCES).toContain('\x1b[?1049l'); + expect(PRUNE_SEQUENCES.length).toBeGreaterThanOrEqual(9); + }); + }); +});