mirror of
https://github.com/xcat2/confluent.git
synced 2026-11-21 01:42:22 +00:00
184e198463
mouse can now set focus (click/ctrl-click) mouse movement and clicks can be sent through Still TODO: Do something with clicks in 'title' area, and see about fixing the misalignment due to image not filling the full cell geometry.
400 lines
17 KiB
Python
400 lines
17 KiB
Python
import asyncio
|
|
from PIL import Image
|
|
import io
|
|
import numpy as np
|
|
import queue
|
|
import threading
|
|
import time
|
|
import zlib
|
|
|
|
class VNCReceiverTerminated(Exception):
|
|
pass
|
|
# This results in an RGBA organization of pixels
|
|
MYPIXFORMAT = bytearray([
|
|
32, # bits per pixel
|
|
24, # depth
|
|
0, # big endian
|
|
1, # true color
|
|
0, 255, # red max
|
|
0, 255, # green max
|
|
0, 255, # blue max
|
|
0, 8, 16, # red shift, green shift, blue shift
|
|
0, 0, 0 # padding
|
|
])
|
|
|
|
class ByteStream:
|
|
def __init__(self):
|
|
self.buffer = b''
|
|
|
|
def add_number(self, number, num_bytes):
|
|
data = number.to_bytes(num_bytes, byteorder='big', signed=True)
|
|
self.buffer += data
|
|
|
|
def extend(self, data):
|
|
self.buffer += data
|
|
|
|
def get_bytes(self):
|
|
return self.buffer
|
|
|
|
def clear(self):
|
|
self.buffer = b''
|
|
|
|
def flush(self, writer):
|
|
writer.write(self.buffer)
|
|
self.clear()
|
|
|
|
class VNCClient:
|
|
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
|
await self.close()
|
|
return False
|
|
|
|
@classmethod
|
|
async def create(cls, url, outputfile=None, fps=15):
|
|
self = cls()
|
|
self.outputfile = outputfile
|
|
self.fps = fps
|
|
self.video_writer = None
|
|
self._video_size = None
|
|
self._last_frame = None
|
|
self._last_frame_time = None
|
|
self._video_queue = None
|
|
self._video_thread = None
|
|
self._cv2 = None
|
|
if outputfile:
|
|
try:
|
|
import cv2
|
|
except ImportError:
|
|
raise ImportError("OpenCV is required for video output but is not installed.")
|
|
self._cv2 = cv2
|
|
self._video_queue = queue.Queue()
|
|
self._video_thread = threading.Thread(
|
|
target=self._video_worker, daemon=True)
|
|
self._video_thread.start()
|
|
if url.startswith('unix://'):
|
|
url = url.replace('unix://', '')
|
|
if url.startswith('/'):
|
|
self.reader, self.writer = await asyncio.open_unix_connection(url)
|
|
elif url.startswith('@'):
|
|
url = '\0' + url[1:]
|
|
self.reader, self.writer = await asyncio.open_unix_connection(url)
|
|
elif url.startswith('tcp://'):
|
|
url = url.replace('tcp://', '')
|
|
host, port = url.split(':')
|
|
self.reader, self.writer = await asyncio.open_connection(host, int(port))
|
|
else:
|
|
raise ValueError('Unsupported URL: {}'.format(url))
|
|
self.receiver = None
|
|
self.framebuffer = None
|
|
self.copytext = None
|
|
self._updating = True
|
|
self.decompressor = zlib.decompressobj()
|
|
await self._do_vnc_handshake()
|
|
self._input_queue = asyncio.Queue()
|
|
self._input_task = asyncio.create_task(self._input_worker())
|
|
return self
|
|
|
|
async def _input_worker(self):
|
|
self.mouse_buttons = 0
|
|
while True:
|
|
if self.receiver is not None and self.receiver.done():
|
|
break
|
|
keys, modifierkeys, mouse_event = await self._input_queue.get()
|
|
payload = ByteStream()
|
|
for modkey in (modifierkeys or []):
|
|
payload.add_number(4, 1) # Key event
|
|
payload.add_number(1, 1) # Down
|
|
payload.add_number(0, 2) # Padding
|
|
payload.add_number(modkey.value, 4)
|
|
payload.flush(self.writer)
|
|
await self.writer.drain()
|
|
for key in keys:
|
|
keynumber = key.value if hasattr(key, 'value') else key
|
|
payload.add_number(4, 1) # Key event
|
|
payload.add_number(1, 1) # Down
|
|
payload.add_number(0, 2) # Padding
|
|
payload.add_number(keynumber, 4)
|
|
payload.add_number(4, 1) # Key event
|
|
payload.add_number(0, 1) # Up
|
|
payload.add_number(0, 2) # Padding
|
|
payload.add_number(keynumber, 4)
|
|
payload.flush(self.writer)
|
|
await self.writer.drain()
|
|
if mouse_event:
|
|
button, x, y, direction = mouse_event
|
|
if button is None:
|
|
self.mouse_buttons = 0
|
|
elif direction == 'release':
|
|
self.mouse_buttons &= ~(1 << button - 1)
|
|
else:
|
|
self.mouse_buttons |= (1 << button - 1)
|
|
payload.add_number(5, 1) # Pointer event
|
|
payload.add_number(self.mouse_buttons, 1)
|
|
payload.add_number(int(x), 2)
|
|
payload.add_number(int(y), 2)
|
|
payload.flush(self.writer)
|
|
await self.writer.drain()
|
|
self.mouse_buttons &= ~24 # clear scroll wheel buttons (bits 3 and 4)
|
|
for modkey in (modifierkeys or []):
|
|
payload.add_number(4, 1) # Key event
|
|
payload.add_number(0, 1) # Up
|
|
payload.add_number(0, 2) # Padding
|
|
payload.add_number(modkey.value, 4)
|
|
payload.flush(self.writer)
|
|
await self.writer.drain()
|
|
if keys:
|
|
await asyncio.sleep(0.01) # Have to slow down keypresses for some servers
|
|
# Still shouldn't be noticable interactively, but does slow down paste to a fast typist...
|
|
self._input_queue.task_done()
|
|
|
|
async def send_keypresses(self, keys, modifierkeys=None):
|
|
await self._input_queue.put((keys, modifierkeys, []))
|
|
|
|
async def send_pointer_event(self, button, x, y, direction, modifierkeys=None):
|
|
if isinstance(x, float):
|
|
x = self.width * x
|
|
if isinstance(y, float):
|
|
y = self.height * y
|
|
await self._input_queue.put(([], modifierkeys, [button, x, y, direction]))
|
|
|
|
async def _read_number(self, num_bytes):
|
|
data = await self.reader.readexactly(num_bytes)
|
|
return int.from_bytes(data, byteorder='big', signed=True)
|
|
|
|
def _write_number(self, number, num_bytes):
|
|
data = number.to_bytes(num_bytes, byteorder='big', signed=True)
|
|
self.writer.write(data)
|
|
return data
|
|
|
|
async def get_screenshot(self, oneshot=False):
|
|
# Oneshot is wanting a full screenshot, right now
|
|
# this means we wait up to half a second getting screenshots,
|
|
# then return whatever the result is after the accumulated updates over the half second
|
|
if oneshot:
|
|
await asyncio.sleep(0.5) # Wait up to half a second for updates
|
|
else:
|
|
while self._updating and not self.receiver.done():
|
|
await asyncio.sleep(0.1)
|
|
if self.receiver.done():
|
|
raise VNCReceiverTerminated('VNC receiver task has terminated unexpectedly')
|
|
await asyncio.sleep(0)
|
|
if self.framebuffer is None:
|
|
raise Exception('No framebuffer data available')
|
|
self._updating = True
|
|
return self.framebuffer.copy()
|
|
|
|
async def _do_vnc_handshake(self):
|
|
rfbver = await self.reader.readline()
|
|
if not rfbver:
|
|
raise Exception('Video not connected')
|
|
if not rfbver.startswith(b'RFB 003.008'):
|
|
self.writer.close()
|
|
await self.writer.wait_closed()
|
|
raise Exception('Unsupported RFB version')
|
|
self.writer.write(b'RFB 003.008\n')
|
|
numsectypes = await self._read_number(1)
|
|
if not numsectypes:
|
|
self.writer.close()
|
|
await self.writer.wait_closed()
|
|
raise Exception('No security types supported by the server')
|
|
sectypes = await self.reader.readexactly(numsectypes)
|
|
sectypes = bytearray(sectypes)
|
|
secresult = 1
|
|
if 1 in sectypes:
|
|
self.writer.write(b'\x01')
|
|
await self.writer.drain()
|
|
secresult = await self._read_number(4) # Security result
|
|
if secresult != 0:
|
|
self.writer.close()
|
|
await self.writer.wait_closed()
|
|
raise Exception('VNC authentication failed')
|
|
self.writer.write(b'\x01') # Share display
|
|
self.width = await self._read_number(2)
|
|
self.height = await self._read_number(2)
|
|
pixformat = await self.reader.readexactly(16)
|
|
name_length = await self._read_number(4)
|
|
self.name = await self.reader.readexactly(name_length)
|
|
payload = ByteStream()
|
|
if pixformat != MYPIXFORMAT:
|
|
payload.add_number(0, 1) # Set pixel format
|
|
payload.add_number(0, 3) # Padding
|
|
payload.extend(MYPIXFORMAT)
|
|
payload.flush(self.writer)
|
|
self.receiver = asyncio.create_task(self._receive_loop())
|
|
payload.add_number(2, 1) # Set encodings
|
|
payload.add_number(0, 1) # Padding
|
|
payload.add_number(5, 2) # Number of encodings
|
|
payload.add_number(6, 4) # zlib
|
|
payload.add_number(7, 4) # tight
|
|
payload.add_number(-239, 4) # cursor
|
|
payload.add_number(-223, 4) # desktopsize
|
|
payload.add_number(-308, 4) # extended desktopsize
|
|
payload.flush(self.writer)
|
|
self._request_screen_update(incremental=False)
|
|
|
|
def _request_screen_update(self, incremental=True):
|
|
incremental = 1 if incremental else 0
|
|
payload = ByteStream()
|
|
payload.add_number(3, 1) # Framebuffer update request
|
|
payload.add_number(incremental, 1) # Incremental
|
|
payload.add_number(0, 2) # x position
|
|
payload.add_number(0, 2) # y position
|
|
payload.add_number(self.width, 2) # width
|
|
payload.add_number(self.height, 2) # height
|
|
payload.flush(self.writer)
|
|
|
|
async def _receive_loop(self):
|
|
while True:
|
|
try:
|
|
message_type = await self._read_number(1)
|
|
if message_type == 0: # Framebuffer update
|
|
await self._handle_framebuffer_update()
|
|
elif message_type == 1: # Set color map entries
|
|
raise NotImplementedError('Set color map entries not implemented')
|
|
elif message_type == 2: # Bell
|
|
pass
|
|
elif message_type == 3: # Server cut text
|
|
padding = await self._read_number(3)
|
|
length = await self._read_number(4)
|
|
self.copytext = await self.reader.readexactly(length)
|
|
else:
|
|
raise Exception(f'Unknown message type: {message_type}')
|
|
except Exception as e:
|
|
#print(f"Error in receive loop: {e}")
|
|
break
|
|
|
|
async def _handle_framebuffer_update(self):
|
|
_ = await self._read_number(1) # Padding
|
|
num_rects = await self._read_number(2)
|
|
self._updating = True
|
|
for _ in range(num_rects):
|
|
await self._handle_rectangle()
|
|
self._updating = False
|
|
self._write_video_frame()
|
|
self._request_screen_update(incremental=True)
|
|
|
|
def _write_video_frame(self):
|
|
if not self._cv2 or self.framebuffer is None:
|
|
return
|
|
# Snapshot the framebuffer now and hand it to the writer thread. Frames
|
|
# captured while a write is in progress simply queue up behind it.
|
|
frame = np.ascontiguousarray(
|
|
np.array(self.framebuffer.convert('RGB'))[:, :, ::-1])
|
|
self._video_queue.put((frame, time.monotonic()))
|
|
|
|
def _video_worker(self):
|
|
cv2 = self._cv2
|
|
while True:
|
|
frame, now = self._video_queue.get()
|
|
if frame is None:
|
|
# Flush the final frame for the time it stayed on screen
|
|
if self.video_writer is not None and self._last_frame is not None:
|
|
nframes = max(1, round(
|
|
(now - self._last_frame_time) * self.fps))
|
|
for _ in range(nframes):
|
|
self.video_writer.write(self._last_frame)
|
|
if self.video_writer is not None:
|
|
self.video_writer.release()
|
|
self.video_writer = None
|
|
return
|
|
if self.video_writer is None:
|
|
self._video_size = (frame.shape[1], frame.shape[0])
|
|
fourcc = cv2.VideoWriter_fourcc(*'mp4v')
|
|
self.video_writer = cv2.VideoWriter(
|
|
self.outputfile, fourcc, self.fps, self._video_size)
|
|
if (frame.shape[1], frame.shape[0]) != self._video_size:
|
|
frame = cv2.resize(frame, self._video_size)
|
|
if self._last_frame is None:
|
|
self._last_frame = frame
|
|
self._last_frame_time = now
|
|
continue
|
|
# Hold the previous frame for the real time it was displayed
|
|
nframes = max(1, round((now - self._last_frame_time) * self.fps))
|
|
for _ in range(nframes):
|
|
self.video_writer.write(self._last_frame)
|
|
self._last_frame = frame
|
|
self._last_frame_time = now
|
|
|
|
async def _handle_rectangle(self):
|
|
if self.framebuffer is None:
|
|
self.framebuffer = Image.new('RGBA', (self.width, self.height))
|
|
x = await self._read_number(2)
|
|
y = await self._read_number(2)
|
|
width = await self._read_number(2)
|
|
height = await self._read_number(2)
|
|
encoding_type = await self._read_number(4)
|
|
pixel_data = None
|
|
if encoding_type == 6:
|
|
compressed_data_length = await self._read_number(4)
|
|
compressed_data = await self.reader.readexactly(compressed_data_length)
|
|
# Decompress the data using zlib and store it in the framebuffer
|
|
pixel_data = self.decompressor.decompress(compressed_data)
|
|
elif encoding_type == 0:
|
|
pixel_data = await self.reader.readexactly(width * height * 4) # Assuming 32 bits per pixel
|
|
if encoding_type in (-223, -308): # desktopsize
|
|
self.width = width
|
|
self.height = height
|
|
self.framebuffer = Image.new('RGBA', (self.width, self.height))
|
|
if encoding_type == -308:
|
|
nscreens = await self._read_number(1)
|
|
_ = await self._read_number(3) # padding
|
|
for _ in range(nscreens):
|
|
_ = await self.reader.readexactly(16) # screen info
|
|
elif pixel_data:
|
|
pixel_data = np.frombuffer(pixel_data, dtype=np.uint8).reshape((height, width, 4)).copy()
|
|
pixel_data[:, :, 3] = 0xff
|
|
img = Image.fromarray(pixel_data, 'RGBA')
|
|
self.framebuffer.paste(img, (x, y))
|
|
elif encoding_type == 7: # tight
|
|
# Best document I could see was:
|
|
# https://github.com/TurboVNC/tightvnc/blob/main/vnc_winsrc/rfb/rfbproto.h
|
|
tightheader = await self._read_number(1)
|
|
streamid = tightheader & 0x0F
|
|
if streamid:
|
|
raise NotImplementedError('tight encoding with streamid not implemented')
|
|
comptype = (tightheader >> 4) & 0x0F
|
|
if comptype != 9:
|
|
raise NotImplementedError(f'tight encoding with comptype {comptype} not implemented')
|
|
compressed_data_length = await self._read_tight_length()
|
|
compressed_data = await self.reader.readexactly(compressed_data_length)
|
|
with io.BytesIO(compressed_data) as jpgimg:
|
|
img = Image.open(jpgimg)
|
|
img.load()
|
|
self.framebuffer.paste(img, (x, y))
|
|
elif encoding_type == -239: # cursor
|
|
cursor_pixels = await self.reader.readexactly(width * height * 4)
|
|
mask_row_bytes = (width + 7) // 8
|
|
cursor_mask = await self.reader.readexactly(mask_row_bytes * height)
|
|
pixels = np.frombuffer(cursor_pixels, dtype=np.uint8).reshape(
|
|
(height, width, 4)).copy()
|
|
if width and height:
|
|
mask = np.unpackbits(
|
|
np.frombuffer(cursor_mask, dtype=np.uint8).reshape(
|
|
(height, mask_row_bytes)),
|
|
axis=1, bitorder='big')[:, :width]
|
|
pixels[:, :, 3] = mask * 0xff
|
|
self.cursor = Image.fromarray(pixels, 'RGBA')
|
|
self.cursor_hotspot = (x, y)
|
|
else:
|
|
raise Exception(f'Unsupported encoding type: {encoding_type}, x: {x}, y: {y}, width: {width}, height: {height}')
|
|
|
|
async def _read_tight_length(self):
|
|
length = 0
|
|
for i in range(3):
|
|
byte = await self._read_number(1)
|
|
length |= ((byte & 0x7F) << (i * 7))
|
|
if not (byte & 0x80):
|
|
break
|
|
return length
|
|
async def close(self):
|
|
if self._video_thread is not None:
|
|
# Signal the writer thread to flush and finalize the file
|
|
self._video_queue.put((None, time.monotonic()))
|
|
await asyncio.to_thread(self._video_thread.join)
|
|
self._video_thread = None
|
|
self.writer.close()
|
|
await self.writer.wait_closed()
|