diff --git a/guacamole-common-js/src/main/webapp/modules/ClipboardEventInterpreter.js b/guacamole-common-js/src/main/webapp/modules/ClipboardEventInterpreter.js new file mode 100644 index 0000000000..8af3082eb9 --- /dev/null +++ b/guacamole-common-js/src/main/webapp/modules/ClipboardEventInterpreter.js @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +var Guacamole = Guacamole || {}; + +/** + * An interpreter for clipboard events within a Guacamole session recording. + * Clipboard data arrives as a sequence of instructions: clipboard (declares + * stream and mimetype), blob (contains base64 data), and end (terminates stream). + * + * @constructor + * @param {number} [startTimestamp=0] + * The starting timestamp for the recording. Event timestamps will be + * relative to this value. + */ +Guacamole.ClipboardEventInterpreter = function ClipboardEventInterpreter(startTimestamp) { + + if (startTimestamp === undefined || startTimestamp === null) + startTimestamp = 0; + + /** + * All clipboard events parsed so far. + * + * @private + * @type {!Guacamole.ClipboardEventInterpreter.ClipboardEvent[]} + */ + var parsedEvents = []; + + /** + * Map of active clipboard streams, keyed by stream index. + * Each entry tracks the mimetype and accumulated base64 data. + * + * @private + * @type {Object.} + */ + var activeStreams = {}; + + /** + * The timestamp of the most recent instruction, used for events + * that don't have their own timestamp. + * + * @private + * @type {number} + */ + var lastTimestamp = 0; + + /** + * Updates the last known timestamp. + * + * @param {number} timestamp + * The absolute timestamp from a sync instruction or key event. + */ + this.setTimestamp = function setTimestamp(timestamp) { + lastTimestamp = timestamp; + }; + + /** + * Handles a clipboard instruction, which begins a new clipboard stream. + * + * @param {!string[]} args + * The arguments: [stream_index, mimetype] + */ + this.handleClipboard = function handleClipboard(args) { + var streamIndex = args[0]; + var mimetype = args[1]; + + activeStreams[streamIndex] = { + mimetype: mimetype, + data: '', + timestamp: lastTimestamp + }; + }; + + /** + * Handles a blob instruction, which contains base64-encoded data + * for an active stream. + * + * @param {!string[]} args + * The arguments: [stream_index, base64_data] + */ + this.handleBlob = function handleBlob(args) { + var streamIndex = args[0]; + var base64Data = args[1]; + + var stream = activeStreams[streamIndex]; + if (stream) + stream.data += base64Data; + }; + + /** + * Handles an end instruction, which completes a clipboard stream + * and creates the final clipboard event. + * + * @param {!string[]} args + * The arguments: [stream_index] + */ + this.handleEnd = function handleEnd(args) { + var streamIndex = args[0]; + + var stream = activeStreams[streamIndex]; + if (stream) { + // Decode the base64 data + var decodedData = ''; + try { + decodedData = atob(stream.data); + // Handle UTF-8 decoding + decodedData = decodeURIComponent(escape(decodedData)); + } catch (e) { + // If decoding fails, use raw decoded data or mark as binary + try { + decodedData = atob(stream.data); + } catch (e2) { + decodedData = '[Binary data]'; + } + } + + // Create the clipboard event + parsedEvents.push(new Guacamole.ClipboardEventInterpreter.ClipboardEvent({ + mimetype: stream.mimetype, + data: decodedData, + timestamp: stream.timestamp - startTimestamp + })); + + // Clean up the stream + delete activeStreams[streamIndex]; + } + }; + + /** + * Returns all parsed clipboard events. + * + * @returns {Guacamole.ClipboardEventInterpreter.ClipboardEvent[]} + * All clipboard events parsed so far. + */ + this.getEvents = function getEvents() { + return parsedEvents; + }; + +}; + +/** + * A single clipboard event from a recording. + * + * @constructor + * @param {Guacamole.ClipboardEventInterpreter.ClipboardEvent|object} [template={}] + * The object whose properties should be copied. + */ +Guacamole.ClipboardEventInterpreter.ClipboardEvent = function ClipboardEvent(template) { + + template = template || {}; + + /** + * The mimetype of the clipboard data. + * + * @type {!string} + */ + this.mimetype = template.mimetype || 'text/plain'; + + /** + * The clipboard content (decoded from base64). + * + * @type {!string} + */ + this.data = template.data || ''; + + /** + * The timestamp when this clipboard event occurred, relative to + * the start of the recording. + * + * @type {!number} + */ + this.timestamp = template.timestamp || 0; + +}; \ No newline at end of file diff --git a/guacamole-common-js/src/main/webapp/modules/SessionRecording.js b/guacamole-common-js/src/main/webapp/modules/SessionRecording.js index 819854b85c..1dc2e9acd2 100644 --- a/guacamole-common-js/src/main/webapp/modules/SessionRecording.js +++ b/guacamole-common-js/src/main/webapp/modules/SessionRecording.js @@ -428,16 +428,24 @@ Guacamole.SessionRecording = function SessionRecording(source, refreshInterval) var keyEventInterpreter = null; /** - * Initialize the key interpreter. This function should be called only once - * with the first timestamp in the recording as an argument. + * A clipboard event interpreter to extract all clipboard events from + * this recording. + * + * @type {Guacamole.ClipboardEventInterpreter} + */ + var clipboardEventInterpreter = null; + + /** + * Initialize the key and clipboard interpreters. This function should be + * called only once with the first timestamp in the recording. * * @private * @param {!number} startTimestamp - * The timestamp of the first frame in the recording, i.e. the start of - * the recording. + * The timestamp of the first frame in the recording. */ - function initializeKeyInterpreter(startTimestamp) { + function initializeInterpreters(startTimestamp) { keyEventInterpreter = new Guacamole.KeyEventInterpreter(startTimestamp); + clipboardEventInterpreter = new Guacamole.ClipboardEventInterpreter(startTimestamp); } /** @@ -466,6 +474,12 @@ Guacamole.SessionRecording = function SessionRecording(source, refreshInterval) // Parse frame timestamp from sync instruction var timestamp = parseInt(args[0]); + // Update the clipboard interpreter's timestamp so clipboard events + // are recorded with the correct time. Clipboard events don't have + // their own timestamps + if (clipboardEventInterpreter) + clipboardEventInterpreter.setTimestamp(timestamp); + // Add a new frame containing the instructions read since last frame var frame = new Guacamole.SessionRecording._Frame(timestamp, frameStart, frameEnd); frames.push(frame); @@ -474,7 +488,7 @@ Guacamole.SessionRecording = function SessionRecording(source, refreshInterval) // If this is the first frame, intialize the key event interpreter // with the timestamp of the first frame if (frames.length === 1) - initializeKeyInterpreter(timestamp); + initializeInterpreters(timestamp); // This frame should eventually become a keyframe if enough data // has been processed and enough recording time has elapsed, or if @@ -492,8 +506,27 @@ Guacamole.SessionRecording = function SessionRecording(source, refreshInterval) } - else if (opcode === 'key') + else if (opcode === 'key') { keyEventInterpreter.handleKeyEvent(args); + // The clipboard gets updated on Ctrl+C key events so we update the + // clipboard interpreter's timestamp to match the timestamp as + // clipboard events don't have own timestamps + if (clipboardEventInterpreter) { + var keyTimestamp = parseInt(args[2]); + clipboardEventInterpreter.setTimestamp(keyTimestamp); + } + } + + else if (opcode === 'clipboard' && clipboardEventInterpreter) + clipboardEventInterpreter.handleClipboard(args); + + // Handle blob data (may be clipboard data) + else if (opcode === 'blob' && clipboardEventInterpreter) + clipboardEventInterpreter.handleBlob(args); + + // Handle stream end (may complete clipboard stream) + else if (opcode === 'end' && clipboardEventInterpreter) + clipboardEventInterpreter.handleEnd(args); }; /** @@ -562,9 +595,14 @@ Guacamole.SessionRecording = function SessionRecording(source, refreshInterval) // Now that the recording is fully processed, and all key events // have been extracted, call the onkeyevents handler if defined - if (recording.onkeyevents) + if (recording.onkeyevents && keyEventInterpreter) recording.onkeyevents(keyEventInterpreter.getEvents()); + // call the onclipboardevents handler if defined with extracted + // clipboard events + if (recording.onclipboardevents && clipboardEventInterpreter) + recording.onclipboardevents(clipboardEventInterpreter.getEvents()); + // Consider recording loaded if tunnel has closed without errors if (!errorEncountered) notifyLoaded(); diff --git a/guacamole/src/main/frontend/src/app/player/directives/player.js b/guacamole/src/main/frontend/src/app/player/directives/player.js index 8b4adb4efe..fe2a0228b9 100644 --- a/guacamole/src/main/frontend/src/app/player/directives/player.js +++ b/guacamole/src/main/frontend/src/app/player/directives/player.js @@ -291,6 +291,22 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay */ var keyTimestamps = []; + /** + * Clipboard events extracted from the recording, stored for merging + * with key events. + * + * @type {!Guacamole.ClipboardEventInterpreter.ClipboardEvent[]} + */ + var clipboardEvents = []; + + /** + * Key events extracted from the recording, stored for merging + * with clipboard events. + * + * @type {!Guacamole.KeyEventInterpreter.KeyEvent[]} + */ + var keyEvents = []; + /** * Return true if any batches of key event logs are available for this * recording, or false otherwise. @@ -495,11 +511,25 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay // Extract key events from the recording $scope.recording.onkeyevents = function keyEventsReceived(events) { + keyEvents = events; + keyTimestamps = events.map(event => event.timestamp); + // Convert to a display-optimized format - $scope.textBatches = ( - keyEventDisplayService.parseEvents(events)); + $scope.textBatches = keyEventDisplayService.parseEventsWithClipboard( + keyEvents, clipboardEvents + ); - keyTimestamps = events.map(event => event.timestamp); + }; + + // Extract clipboard events from the recording + $scope.recording.onclipboardevents = function clipboardEventsReceived(events) { + + clipboardEvents = events; + + // Convert to a display-optimized format + $scope.textBatches = keyEventDisplayService.parseEventsWithClipboard( + keyEvents, clipboardEvents + ); }; diff --git a/guacamole/src/main/frontend/src/app/player/services/keyEventDisplayService.js b/guacamole/src/main/frontend/src/app/player/services/keyEventDisplayService.js index c362670a76..0a9ae6b439 100644 --- a/guacamole/src/main/frontend/src/app/player/services/keyEventDisplayService.js +++ b/guacamole/src/main/frontend/src/app/player/services/keyEventDisplayService.js @@ -190,32 +190,47 @@ angular.module('player').factory('keyEventDisplayService', }; /** - * Accepts key events in the format produced by KeyEventInterpreter and returns - * human readable text batches, seperated by at least `batchSeperation` milliseconds - * if provided. + * Accepts key events and clipboard events, merging them chronologically + * into human readable text batches, separated by at least `batchSeparation` + * milliseconds if provided. * * NOTE: The event processing logic and output format is based on the `guaclog` - * tool, with the addition of batching support. + * tool, with the addition of batching and clipboard support. * - * @param {Guacamole.KeyEventInterpreter.KeyEvent[]} [rawEvents] + * @param {Guacamole.KeyEventInterpreter.KeyEvent[]} [rawKeyEvents] * The raw key events to prepare for display. * - * @param {number} [batchSeperation=5000] + * @param {Guacamole.ClipboardEventInterpreter.ClipboardEvent[]} [clipboardEvents] + * The clipboard events to merge inline with key events. + * + * @param {number} [batchSeparation=5000] * The minimum number of milliseconds that must elapse between subsequent * batches of key-event-generated text. If 0 or negative, no splitting will - * occur, resulting in a single batch for all provided key events. + * occur, resulting in a single batch for all provided events. * * @param {boolean} [consolidateEvents=false] * Whether consecutive sequences of events with similar properties * should be consolidated into a single ConsolidatedKeyEvent object for * display performance reasons. */ - service.parseEvents = function parseEvents( - rawEvents, batchSeperation, consolidateEvents) { + service.parseEventsWithClipboard = function parseEventsWithClipboard( + rawKeyEvents, clipboardEvents, batchSeparation, consolidateEvents) { + + // Default to 5 seconds if the batch separation was not provided + if (batchSeparation === undefined || batchSeparation === null) + batchSeparation = 5000; + + // Convert clipboard events to unified format + const clipboardAsEvents = (clipboardEvents || []).map(event => ({ + isClipboard: true, + timestamp: event.timestamp, + data: event.data + })); + + // Merge and sort all events by timestamp + const allEvents = [...(rawKeyEvents || []), ...clipboardAsEvents] + .sort((a, b) => a.timestamp - b.timestamp); - // Default to 5 seconds if the batch seperation was not provided - if (batchSeperation === undefined || batchSeperation === null) - batchSeperation = 5000; /** * A map of X11 keysyms to a KeyDefinition object, if the corresponding * key is currently pressed. If a keysym has no entry in this map at all @@ -225,24 +240,22 @@ angular.module('player').factory('keyEventDisplayService', */ const pressedKeys = {}; - // The timestamp of the most recent key event processed - let lastKeyEvent = 0; + // The timestamp of the most recent event processed + let lastEventTime = 0; - // All text batches produced from the provided raw key events + // All text batches produced from the provided events const batches = [new service.TextBatch()]; - // Process every provided raw - _.forEach(rawEvents, event => { + // Process every provided event + allEvents.forEach(event => { - // Extract all fields from the raw event - const { definition, pressed, timestamp } = event; - const { keysym, name, value } = definition; + const { timestamp } = event; // Only switch to a new batch of text if sufficient time has passed - // since the last key event - const newBatch = (batchSeperation >= 0 - && (timestamp - lastKeyEvent) >= batchSeperation); - lastKeyEvent = timestamp; + // since the last event + const newBatch = (batchSeparation >= 0 + && (timestamp - lastEventTime) >= batchSeparation); + lastEventTime = timestamp; if (newBatch) batches.push(new service.TextBatch()); @@ -250,7 +263,7 @@ angular.module('player').factory('keyEventDisplayService', const currentBatch = _.last(batches); /** - * Either push the a new event constructed using the provided fields + * Either push a new event constructed using the provided fields * into the latest batch, or consolidate into the latest event as * appropriate given the consolidation configuration and event type. * @@ -274,94 +287,109 @@ angular.module('player').factory('keyEventDisplayService', // Otherwise, push a new event else { currentBatch.events.push(new service.ConsolidatedKeyEvent({ - text, typed, timestamp})); + text, typed, timestamp + })); currentBatch.simpleValue += text; } - } + }; - // Track modifier state - if (MODIFIER_KEYS[keysym]) { - if (pressed) - pressedKeys[keysym] = definition; - else - delete pressedKeys[keysym]; + // Handle clipboard events + if (event.isClipboard) { + const preview = (event.data || '').substring(0, 50); + pushEvent('[Clipboard: ' + preview + ']', false); } - // Append to the current typed value when a printable - // (non-modifier) key is pressed - else if (pressed) { + // Handle key events + else { - // If any shorcut keys are currently pressed - if (_.some(pressedKeys, (def, key) => SHORTCUT_KEYS[key])) { + const { definition, pressed } = event; + const { keysym, name, value } = definition; - var shortcutText = '<'; + // Track modifier state + if (MODIFIER_KEYS[keysym]) { + if (pressed) + pressedKeys[keysym] = definition; + else + delete pressedKeys[keysym]; + } - var firstKey = true; + // Append to the current typed value when a printable + // (non-modifier) key is pressed + else if (pressed) { - // Compose entry by inspecting the state of each tracked key. - // At least one key must be pressed when in a shortcut. - for (let pressedKeysym in pressedKeys) { + // If any shortcut keys are currently pressed + if (_.some(pressedKeys, (def, key) => SHORTCUT_KEYS[key])) { - var pressedKeyDefinition = pressedKeys[pressedKeysym]; + var shortcutText = '<'; - // Print name of key - if (firstKey) { - shortcutText += pressedKeyDefinition.name; - firstKey = false; - } + var firstKey = true; - else - shortcutText += ('+' + pressedKeyDefinition.name); + // Compose entry by inspecting the state of each tracked key. + // At least one key must be pressed when in a shortcut. + for (let pressedKeysym in pressedKeys) { - } + var pressedKeyDefinition = pressedKeys[pressedKeysym]; - // Finally, append the printable key to close the shortcut - shortcutText += ('+' + name + '>') + // Print name of key + if (firstKey) { + shortcutText += pressedKeyDefinition.name; + firstKey = false; + } - // Add the shortcut to the current batch - pushEvent(shortcutText, false); - } + else + shortcutText += ('+' + pressedKeyDefinition.name); - // Print the key itself - else { + } + + // Finally, append the printable key to close the shortcut + shortcutText += ('+' + name + '>'); - var keyText; - var typed; - - // Print the value if explicitly defined - if (value !== undefined) { - - keyText = value; - typed = true; - - // If the name should be printed in addition, add it as a - // seperate event before the actual character value - if (PRINT_NAME_TOO_KEYS[keysym]) - pushEvent(formatKeyName(name), false); - + // Add the shortcut to the current batch + pushEvent(shortcutText, false); } - - // Otherwise print the name + + // Print the key itself else { - - keyText = formatKeyName(name); - - // While this is a representation for a single character, - // the key text is the name of the key, not the actual - // character itself - typed = false; - - } - - // Add the key to the current batch - pushEvent(keyText, typed); + var keyText; + var typed; + + // Print the value if explicitly defined + if (value !== undefined) { + + keyText = value; + typed = true; + + // If the name should be printed in addition, add it as a + // separate event before the actual character value + if (PRINT_NAME_TOO_KEYS[keysym]) + pushEvent(formatKeyName(name), false); + + } + + // Otherwise print the name + else { + + keyText = formatKeyName(name); + + // While this is a representation for a single character, + // the key text is the name of the key, not the actual + // character itself + typed = false; + + } + + // Add the key to the current batch + pushEvent(keyText, typed); + + } } - } - // We ignore key release events here because in practice characters - // are printed when you press keys not release them. The release order - // can be different and lead to wrong character printing order in the log. + // We ignore key release events here because in practice characters + // are printed when you press keys not release them. The release order + // can be different and lead to wrong character printing order in the log. + + } }); @@ -372,4 +400,4 @@ angular.module('player').factory('keyEventDisplayService', return service; -}]); +}]); \ No newline at end of file