From 3d6b5a8c7c5d262792a9e5f9cb745c17b7e2e993 Mon Sep 17 00:00:00 2001 From: Max Goodhart Date: Sun, 4 Oct 2020 22:05:27 -0700 Subject: [PATCH] Experimental offscreen rendering + iframe rearchitecture --- src/browser/background.html | 14 - src/browser/background.js | 48 --- src/browser/mediaPreload.js | 248 ++++++++++++++++ src/browser/{overlay.html => wall.html} | 2 +- src/browser/{overlay.js => wall.js} | 190 ++++++++---- src/browser/wallPreload.js | 8 + src/node/StreamWindow.js | 230 +++++++-------- src/node/TwitchBot.js | 2 +- src/node/data.js | 11 + src/node/index.js | 2 +- src/node/viewStateMachine.js | 377 ++++-------------------- src/web/control.js | 31 +- webpack.config.js | 8 +- 13 files changed, 592 insertions(+), 579 deletions(-) delete mode 100644 src/browser/background.html delete mode 100644 src/browser/background.js create mode 100644 src/browser/mediaPreload.js rename src/browser/{overlay.html => wall.html} (84%) rename src/browser/{overlay.js => wall.js} (51%) create mode 100644 src/browser/wallPreload.js diff --git a/src/browser/background.html b/src/browser/background.html deleted file mode 100644 index 95835d5..0000000 --- a/src/browser/background.html +++ /dev/null @@ -1,14 +0,0 @@ - - - - - Streamwall Stream Background - - - - - - diff --git a/src/browser/background.js b/src/browser/background.js deleted file mode 100644 index 6cd3d3d..0000000 --- a/src/browser/background.js +++ /dev/null @@ -1,48 +0,0 @@ -import { ipcRenderer } from 'electron' -import { h, render } from 'preact' -import { useEffect, useState } from 'preact/hooks' -import styled from 'styled-components' - -import '../index.css' - -function Background({ streams }) { - const backgrounds = streams.filter((s) => s.kind === 'background') - return ( -
- {backgrounds.map((s) => ( - - ))} -
- ) -} - -function App() { - const [state, setState] = useState({ - streams: [], - }) - - useEffect(() => { - ipcRenderer.on('state', (ev, state) => { - setState(state) - }) - }, []) - - const { streams } = state - return -} - -const BackgroundIFrame = styled.iframe` - position: fixed; - left: 0; - top: 0; - width: 100vw; - height: 100vh; - border: none; -` - -render(, document.body) diff --git a/src/browser/mediaPreload.js b/src/browser/mediaPreload.js new file mode 100644 index 0000000..dd7564b --- /dev/null +++ b/src/browser/mediaPreload.js @@ -0,0 +1,248 @@ +import { ipcRenderer, webFrame } from 'electron' + +const VIDEO_OVERRIDE_STYLE = ` + * { + pointer-events: none; + display: none !important; + position: static !important; + z-index: 0 !important; + } + html, body, video, audio { + display: block !important; + background: black !important; + } + html, body { + overflow: hidden !important; + background: black !important; + } + video, iframe.__video__, audio { + display: block !important; + position: fixed !important; + left: 0 !important; + right: 0 !important; + top: 0 !important; + bottom: 0 !important; + width: 100vw !important; + height: 100vh !important; + object-fit: cover !important; + transition: none !important; + z-index: 999999 !important; + } + audio { + z-index: 999998 !important; + } + .__video_parent__ { + display: block !important; + } + video.__rot180__ { + transform: rotate(180deg) !important; + } + /* For 90 degree rotations, we position the video with swapped width and height and rotate it into place. + It's helpful to offset the video so the transformation is centered in the viewport center. + We move the video top left corner to center of the page and then translate half the video dimensions up and left. + Note that the width and height are swapped in the translate because the video starts with the side dimensions swapped. */ + video.__rot90__ { + transform: translate(-50vh, -50vw) rotate(90deg) !important; + } + video.__rot270__ { + transform: translate(-50vh, -50vw) rotate(270deg) !important; + } + video.__rot90__, video.__rot270__ { + left: 50vw !important; + top: 50vh !important; + width: 100vh !important; + height: 100vw !important; + } +` +const NO_SCROLL_STYLE = ` + html, body { + overflow: hidden !important; + } +` + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + +function lockdownMediaTags() { + webFrame.executeJavaScript(` + for (const el of document.querySelectorAll('video, audio')) { + if (el.__sw) { + continue + } + el.muted = true + // Prevent sites from unmuting (Periscope, I'm looking at you!) + Object.defineProperty(el, 'muted', { writable: true, value: false }) + // Prevent Facebook from pausing the video after page load. + Object.defineProperty(el, 'pause', { writable: false, value: () => {} }) + el.__sw = true + } + `) +} + +// Watch for media tags and mute them as soon as possible. +function watchMediaTags(kind, onFirstOfKind) { + let foundMatch = false + const observer = new MutationObserver((mutationList) => { + if (kind) { + const el = document.querySelector(kind) + if (el && !foundMatch) { + onFirstOfKind(el) + foundMatch = true + } + } + lockdownMediaTags() + }) + document.addEventListener('DOMContentLoaded', () => { + observer.observe(document.body, { subtree: true, childList: true }) + }) +} + +async function waitForVideo(kind) { + const waitForTag = new Promise((resolve) => watchMediaTags(kind, resolve)) + let video = await Promise.race([waitForTag, sleep(10000)]) + if (video) { + return { video } + } + + let tries = 0 + let iframe + while ((!video || !video.src) && tries < 10) { + for (iframe of document.querySelectorAll('iframe')) { + if (!iframe.contentDocument) { + continue + } + video = iframe.contentDocument.querySelector(kind) + if (video) { + break + } + } + tries++ + await sleep(200) + } + if (video) { + return { video, iframe } + } + return {} +} + +const periscopeHacks = { + isMatch() { + return ( + location.host === 'www.pscp.tv' || location.host === 'www.periscope.tv' + ) + }, + onLoad() { + const playButton = document.querySelector('.PlayButton') + if (playButton) { + playButton.click() + } + }, + afterPlay(video) { + const baseVideoEl = document.querySelector('div.BaseVideo') + if (!baseVideoEl) { + return + } + + function positionPeriscopeVideo() { + // Periscope videos can be rotated using transform matrix. They need to be rotated correctly. + const tr = baseVideoEl.style.transform + if (tr.endsWith('matrix(0, 1, -1, 0, 0, 0)')) { + video.className = '__rot90__' + } else if (tr.endsWith('matrix(-1, 0, 0, -1, 0)')) { + video.className = '__rot180__' + } else if (tr.endsWith('matrix(0, -1, 1, 0, 0, 0)')) { + video.className = '__rot270__' + } + } + + positionPeriscopeVideo() + const obs = new MutationObserver((ml) => { + for (const m of ml) { + if (m.attributeName === 'style') { + positionPeriscopeVideo() + return + } + } + }) + obs.observe(baseVideoEl, { attributes: true }) + }, +} + +async function findVideo(kind) { + if (periscopeHacks.isMatch()) { + periscopeHacks.onLoad() + } + + const { video, iframe } = await waitForVideo(kind) + if (!video) { + throw new Error('could not find video') + } + if (iframe) { + // TODO: verify iframe still works + const style = iframe.contentDocument.createElement('style') + style.innerHTML = VIDEO_OVERRIDE_STYLE + iframe.contentDocument.head.appendChild(style) + iframe.className = '__video__' + let parentEl = iframe.parentElement + while (parentEl) { + parentEl.className = '__video_parent__' + parentEl = parentEl.parentElement + } + iframe.contentDocument.body.appendChild(video) + } else { + document.body.appendChild(video) + } + + const videoReady = new Promise((resolve) => + video.addEventListener('canplay', resolve, { once: true }), + ) + + video.play() + + if (periscopeHacks.isMatch()) { + periscopeHacks.afterPlay(video) + } + + if (!video.videoWidth) { + await videoReady + } + + const info = { + title: document.title, + intrinsicWidth: video.videoWidth, + intrinsicHeight: video.videoHeight, + } + return { info, video } +} + +async function main(viewId) { + const viewInit = ipcRenderer.invoke('view-init', { viewId }) + const pageReady = new Promise((resolve) => process.once('loaded', resolve)) + + const [{ content }] = await Promise.all([viewInit, pageReady]) + + if (content.kind === 'video' || content.kind === 'audio') { + webFrame.insertCSS(VIDEO_OVERRIDE_STYLE) + } else if (content.kind === 'web') { + webFrame.insertCSS(NO_SCROLL_STYLE) + } + + const { info, video } = await findVideo(content.kind) + ipcRenderer.send('view-loaded', { viewId, info }) + + ipcRenderer.on('mute', () => { + video.muted = true + }) + ipcRenderer.on('unmute', () => { + video.muted = false + }) +} + +if (window.name) { + try { + main(window.name) + } catch (err) { + ipcRenderer.send('view-error', { viewId, err }) + } +} else { + watchMediaTags() +} diff --git a/src/browser/overlay.html b/src/browser/wall.html similarity index 84% rename from src/browser/overlay.html rename to src/browser/wall.html index 4d29eed..372af05 100644 --- a/src/browser/overlay.html +++ b/src/browser/wall.html @@ -9,6 +9,6 @@ /> - + diff --git a/src/browser/overlay.js b/src/browser/wall.js similarity index 51% rename from src/browser/overlay.js rename to src/browser/wall.js index e88e3cf..bc90693 100644 --- a/src/browser/overlay.js +++ b/src/browser/wall.js @@ -1,6 +1,5 @@ -import { ipcRenderer } from 'electron' import { h, Fragment, render } from 'preact' -import { useEffect, useState } from 'preact/hooks' +import { useEffect, useState, useRef } from 'preact/hooks' import { State } from 'xstate' import styled from 'styled-components' import { useHotkeys } from 'react-hotkeys-hook' @@ -16,60 +15,75 @@ import TwitchIcon from '../static/twitch.svg' import YouTubeIcon from '../static/youtube.svg' import SoundIcon from '../static/volume-up-solid.svg' +const VIEW_POS_TRANSITION = '0.1s linear' +const VIEW_OPACITY_TRANSITION = '0.5s ease-out' + function Overlay({ config, views, streams }) { const { width, height, activeColor } = config const activeViews = views .map(({ state, context }) => State.from(state, context)) - .filter((s) => s.matches('displaying') && !s.matches('displaying.error')) + .filter((s) => !s.matches('error')) + const backgrounds = streams.filter((s) => s.kind === 'background') const overlays = streams.filter((s) => s.kind === 'overlay') + // TODO: prevent iframes from being reparented return (
+ {backgrounds.map((s) => ( + + ))} {activeViews.map((viewState) => { - const { content, pos } = viewState.context + const { viewId, content, pos, info } = viewState.context const data = streams.find((d) => content.url === d.link) - const isListening = viewState.matches( - 'displaying.running.audio.listening', - ) + const isListening = viewState.matches('running.audio.listening') const isBackgroundListening = viewState.matches( - 'displaying.running.audio.background', + 'running.audio.background', ) - const isBlurred = viewState.matches('displaying.running.video.blurred') - const isLoading = viewState.matches('displaying.loading') + const isBlurred = viewState.matches('running.video.blurred') + const isLoading = viewState.matches('loading') + const isRunning = viewState.matches('running') return ( - - - {data && ( - - - - {data.hasOwnProperty('label') ? ( - data.label - ) : ( - <> - {data.source} – {data.city} {data.state} - - )} - - {(isListening || isBackgroundListening) && } - - )} + + + + {data && ( + + + + {data.hasOwnProperty('label') ? ( + data.label + ) : ( + <> + {data.source} – {data.city} {data.state} + + )} + + {(isListening || isBackgroundListening) && } + + )} + {isLoading && } - + ) })} {overlays.map((s) => ( - + ))}
) @@ -84,13 +98,17 @@ function App() { }) useEffect(() => { - ipcRenderer.on('state', (ev, state) => { - setState(state) + streamwall.onState(setState) + streamwall.onReloadView(({ viewId }) => { + const viewFrame = document.querySelector(`iframe[name="${viewId}"]`) + if (viewFrame) { + viewFrame.src = viewFrame.src + } }) }, []) useHotkeys('ctrl+shift+i', () => { - ipcRenderer.send('devtools-overlay') + streamwall.openDevTools() }) const { config, views, streams, customStreams } = state @@ -132,16 +150,74 @@ function StreamIcon({ url, ...props }) { return null } -const SpaceBorder = styled.div.attrs((props) => ({ - borderWidth: 2, -}))` - display: flex; - align-items: flex-start; +function MediaIframe({ + isRunning, + pos, + intrinsicWidth, + intrinsicHeight, + ...props +}) { + const frameRef = useRef() + + // Set isMounted after render so the transition doesn't apply to the initial sizing of the frame. + useEffect(() => { + if (!isRunning) { + return + } + const { current: el } = frameRef + el.style.transition = `transform ${VIEW_POS_TRANSITION}, opacity ${VIEW_OPACITY_TRANSITION}` + el.style.opacity = 1 + }, [isRunning]) + + let style = { + position: 'absolute', + left: -9999, + width: pos.width, + height: pos.height, + opacity: 0, + } + if (isRunning) { + const scale = Math.max( + pos.width / intrinsicWidth, + pos.height / intrinsicHeight, + ) + const translateX = -(intrinsicWidth * scale - pos.width) / 2 + const translateY = -(intrinsicHeight * scale - pos.height) / 2 + const transform = `translate(${translateX}px, ${translateY}px) scale(${scale})` + style = { + opacity: 0, + transform, + transition: 'none', + // TODO: explore not oversampling as a perf improvement + width: intrinsicWidth, + height: intrinsicHeight, + } + } + return +} + +const ViewContainer = styled.div` position: fixed; left: ${({ pos }) => pos.x}px; top: ${({ pos }) => pos.y}px; width: ${({ pos }) => pos.width}px; height: ${({ pos }) => pos.height}px; + overflow: hidden; + transition: left ${VIEW_POS_TRANSITION}, top ${VIEW_POS_TRANSITION}, + width ${VIEW_POS_TRANSITION}, height ${VIEW_POS_TRANSITION}; + will-change: left, top, width, height; +` + +const SpaceBorder = styled.div.attrs((props) => ({ + borderWidth: 2, +}))` + position: absolute; + left: 0; + top: 0; + right: 0; + bottom: 0; + display: flex; + align-items: flex-start; border: 0 solid black; border-left-width: ${({ pos, borderWidth }) => pos.x === 0 ? 0 : borderWidth}px; @@ -220,14 +296,26 @@ const BlurCover = styled.div` backdrop-filter: ${({ isBlurred }) => (isBlurred ? 'blur(30px)' : 'blur(0)')}; ` -const OverlayIFrame = styled.iframe` +const Iframe = styled.iframe.attrs((props) => ({ + sandbox: 'allow-scripts allow-same-origin', + allow: 'autoplay', +}))` + border: none; + pointer-events: none; +` + +const StyledMediaIframe = styled(Iframe)` + position: absolute; + transform-origin: top left; + will-change: opacity, transform; +` + +const OverlayIframe = styled(Iframe)` position: fixed; left: 0; top: 0; width: 100vw; height: 100vh; - border: none; - pointer-events: none; ` render(, document.body) diff --git a/src/browser/wallPreload.js b/src/browser/wallPreload.js new file mode 100644 index 0000000..cd65890 --- /dev/null +++ b/src/browser/wallPreload.js @@ -0,0 +1,8 @@ +import { ipcRenderer, contextBridge } from 'electron' + +contextBridge.exposeInMainWorld('streamwall', { + openDevTools: () => ipcRenderer.send('devtools-overlay'), + onState: (handler) => ipcRenderer.on('state', (ev, state) => handler(state)), + onReloadView: (handler) => + ipcRenderer.on('reload-view', (ev, data) => handler(data)), +}) diff --git a/src/node/StreamWindow.js b/src/node/StreamWindow.js index 6e74eae..fb110cc 100644 --- a/src/node/StreamWindow.js +++ b/src/node/StreamWindow.js @@ -1,7 +1,10 @@ +import path from 'path' import isEqual from 'lodash/isEqual' +import sortBy from 'lodash/sortBy' import intersection from 'lodash/intersection' import EventEmitter from 'events' -import { BrowserView, BrowserWindow, ipcMain } from 'electron' +import net from 'net' +import { app, BrowserWindow, ipcMain } from 'electron' import { interpret } from 'xstate' import viewStateMachine from './viewStateMachine' @@ -20,8 +23,8 @@ export default class StreamWindow extends EventEmitter { this.offscreenWin = null this.backgroundView = null this.overlayView = null - this.views = [] - this.viewActions = null + this.views = new Map() + this.nextViewId = 0 } init() { @@ -45,135 +48,133 @@ export default class StreamWindow extends EventEmitter { backgroundColor, useContentSize: true, show: false, + webPreferences: { + offscreen: true, + sandbox: true, + nodeIntegration: true, + nodeIntegrationInSubFrames: true, + contextIsolation: true, + worldSafeExecuteJavaScript: true, + partition: 'persist:session', + preload: path.join(app.getAppPath(), 'wallPreload.js'), + }, }) win.removeMenu() - win.loadURL('about:blank') + // via https://github.com/electron/electron/pull/573#issuecomment-642216738 + win.webContents.session.webRequest.onHeadersReceived( + ({ responseHeaders }, callback) => { + for (const headerName of Object.keys(responseHeaders)) { + if (headerName.toLowerCase() === 'x-frame-options') { + delete responseHeaders[headerName] + } + if (headerName.toLowerCase() === 'content-security-policy') { + const csp = responseHeaders[headerName] + responseHeaders[headerName] = csp.map((val) => + val.replace(/frame-ancestors[^;]+;/, ''), + ) + } + } + callback({ responseHeaders }) + }, + ) + win.webContents.session.setPreloads([ + path.join(app.getAppPath(), 'mediaPreload.js'), + ]) + win.webContents.loadFile('wall.html') win.on('close', () => this.emit('close')) + let sock = net.connect(3000, '127.0.0.1') + let reconnectTimeout + sock.on('close', () => { + clearTimeout(reconnectTimeout) + reconnectTimeout = setTimeout(() => { + try { + sock.connect(3000, '127.0.0.1') + } catch (err) {} + }, 1000) + }) + sock.on('error', () => {}) + win.webContents.beginFrameSubscription((image) => { + if (sock.destroyed) { + return + } + try { + sock.write(image.getBitmap()) + } catch (err) {} + }) + win.webContents.setFrameRate(30) + // Work around https://github.com/electron/electron/issues/14308 // via https://github.com/lutzroeder/netron/commit/910ce67395130690ad76382c094999a4f5b51e92 win.once('ready-to-show', () => { win.resizable = false - win.show() }) this.win = win - const offscreenWin = new BrowserWindow({ - show: false, - webPreferences: { - offscreen: true, - }, - }) - this.offscreenWin = offscreenWin - - const backgroundView = new BrowserView({ - webPreferences: { - nodeIntegration: true, - }, - }) - win.addBrowserView(backgroundView) - backgroundView.setBounds({ - x: 0, - y: 0, - width, - height, - }) - backgroundView.webContents.loadFile('background.html') - this.backgroundView = backgroundView - - const overlayView = new BrowserView({ - webPreferences: { - nodeIntegration: true, - }, - }) - win.addBrowserView(overlayView) - overlayView.setBounds({ - x: 0, - y: 0, - width, - height, - }) - overlayView.webContents.loadFile('overlay.html') - this.overlayView = overlayView - - this.viewActions = { - offscreenView: (context, event) => { - const { view } = context - // It appears necessary to initialize the browser view by adding it to a window and setting bounds. Otherwise, some streaming sites like Periscope will not load their videos due to the Page Visibility API being hidden. - win.removeBrowserView(view) - offscreenWin.addBrowserView(view) - view.setBounds({ x: 0, y: 0, width: spaceWidth, height: spaceHeight }) - }, - positionView: (context, event) => { - const { pos, view } = context - win.addBrowserView(view) - - // It's necessary to remove and re-add the overlay view to ensure it's on top. - win.removeBrowserView(overlayView) - win.addBrowserView(overlayView) - - view.setBounds(pos) - }, - } - ipcMain.on('devtools-overlay', () => { - overlayView.webContents.openDevTools() + win.webContents.openDevTools() + }) + + ipcMain.handle('view-init', async (ev, { viewId }) => { + const view = this.views.get(viewId) + if (view) { + view.send({ + type: 'VIEW_INIT', + sendToFrame: (...args) => ev.sender.sendToFrame(ev.frameId, ...args), + }) + return { content: view.state.context.content } + } + }) + + ipcMain.on('view-loaded', (ev, { viewId, info }) => { + this.views.get(viewId)?.send?.({ type: 'VIEW_LOADED', info }) + }) + + ipcMain.on('view-error', (ev, { viewId, err }) => { + this.views.get(viewId)?.send?.({ type: 'VIEW_ERROR', err }) }) } createView() { - const { win, overlayView, viewActions } = this - const { backgroundColor } = this.config - const view = new BrowserView({ - webPreferences: { - nodeIntegration: false, - enableRemoteModule: false, - contextIsolation: true, - partition: 'persist:session', - sandbox: true, - }, - }) - view.setBackgroundColor(backgroundColor) - + // TODO: no parallel functionality in iframe? + /* // Prevent view pages from navigating away from the specified URL. view.webContents.on('will-navigate', (ev) => { ev.preventDefault() }) - - const machine = viewStateMachine - .withContext({ - ...viewStateMachine.context, - view, - parentWin: win, - overlayView, - }) - .withConfig({ actions: viewActions }) + */ + const machine = viewStateMachine.withContext({ + ...viewStateMachine.context, + id: `__view:${this.nextViewId}`, + sendToWall: (...args) => this.win.webContents.send(...args), + }) const service = interpret(machine).start() service.onTransition(this.emitState.bind(this)) + this.nextViewId++ + return service } emitState() { - this.emit( - 'state', - this.views.map(({ state }) => ({ - state: state.value, - context: { - content: state.context.content, - info: state.context.info, - pos: state.context.pos, - }, - })), - ) + const states = Array.from(this.views.values(), ({ state }) => ({ + state: state.value, + context: { + viewId: state.context.id, + content: state.context.content, + info: state.context.info, + pos: state.context.pos, + }, + })) + this.emit('state', sortBy(states, 'context.viewId')) } setViews(viewContentMap) { const { gridCount, spaceWidth, spaceHeight } = this.config - const { win, views } = this + const { views } = this const boxes = boxesFromViewContentMap(gridCount, gridCount, viewContentMap) const remainingBoxes = new Set(boxes) - const unusedViews = new Set(views) + const unusedViews = new Set(views.values()) const viewsToDisplay = [] // We try to find the best match for moving / reusing existing views to match the new positions. @@ -181,12 +182,11 @@ export default class StreamWindow extends EventEmitter { // First try to find a loaded view of the same URL in the same space... (v, content, spaces) => isEqual(v.state.context.content, content) && - v.state.matches('displaying.running') && + v.state.matches('running') && intersection(v.state.context.pos.spaces, spaces).length > 0, // Then try to find a loaded view of the same URL... (v, content) => - isEqual(v.state.context.content, content) && - v.state.matches('displaying.running'), + isEqual(v.state.context.content, content) && v.state.matches('running'), // Then try view with the same URL that is still loading... (v, content) => isEqual(v.state.context.content, content), ] @@ -214,7 +214,7 @@ export default class StreamWindow extends EventEmitter { viewsToDisplay.push({ box, view }) } - const newViews = [] + const newViews = new Map() for (const { box, view } of viewsToDisplay) { const { content, x, y, w, h, spaces } = box const pos = { @@ -225,12 +225,7 @@ export default class StreamWindow extends EventEmitter { spaces, } view.send({ type: 'DISPLAY', pos, content }) - newViews.push(view) - } - for (const view of unusedViews) { - const browserView = view.state.context.view - win.removeBrowserView(browserView) - browserView.destroy() + newViews.set(view.state.context.id, view) } this.views = newViews this.emitState() @@ -238,10 +233,7 @@ export default class StreamWindow extends EventEmitter { setListeningView(viewIdx) { const { views } = this - for (const view of views) { - if (!view.state.matches('displaying')) { - continue - } + for (const view of views.values()) { const { context } = view.state const isSelectedView = context.pos.spaces.includes(viewIdx) view.send(isSelectedView ? 'UNMUTE' : 'MUTE') @@ -249,10 +241,11 @@ export default class StreamWindow extends EventEmitter { } findViewByIdx(viewIdx) { - return this.views.find( - (v) => - v.state.context.pos && v.state.context.pos.spaces.includes(viewIdx), - ) + for (const view of this.views.values()) { + if (view.state.context.pos?.spaces?.includes?.(viewIdx)) { + return view + } + } } sendViewEvent(viewIdx, event) { @@ -279,7 +272,6 @@ export default class StreamWindow extends EventEmitter { } send(...args) { - this.overlayView.webContents.send(...args) - this.backgroundView.webContents.send(...args) + this.win.webContents.send(...args) } } diff --git a/src/node/TwitchBot.js b/src/node/TwitchBot.js index 0e804ef..62a24bc 100644 --- a/src/node/TwitchBot.js +++ b/src/node/TwitchBot.js @@ -57,7 +57,7 @@ export default class TwitchBot extends EventEmitter { this.streams = streams const listeningView = views.find(({ state, context }) => - State.from(state, context).matches('displaying.running.audio.listening'), + State.from(state, context).matches('running.audio.listening'), ) if (!listeningView) { return diff --git a/src/node/data.js b/src/node/data.js index 26bcc7f..fc2edce 100644 --- a/src/node/data.js +++ b/src/node/data.js @@ -71,6 +71,17 @@ export async function* combineDataSources(dataSources) { } } +export function filterStreams(streams) { + return streams.filter(({ link }) => { + const url = new URL(link) + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + console.warn(`filtering non-http stream URL '${urlStr}'`) + return false + } + return true + }) +} + export class StreamIDGenerator { constructor() { this.idMap = new Map() diff --git a/src/node/index.js b/src/node/index.js index 9b9a381..cb2e2f7 100644 --- a/src/node/index.js +++ b/src/node/index.js @@ -403,7 +403,7 @@ async function main() { if (require.main === module) { app.commandLine.appendSwitch('high-dpi-support', 1) app.commandLine.appendSwitch('force-device-scale-factor', 1) - + app.enableSandbox() app .whenReady() .then(main) diff --git a/src/node/viewStateMachine.js b/src/node/viewStateMachine.js index b111d80..ca5bb4d 100644 --- a/src/node/viewStateMachine.js +++ b/src/node/viewStateMachine.js @@ -1,360 +1,105 @@ -import isEqual from 'lodash/isEqual' import { Machine, assign } from 'xstate' -import { ensureValidURL } from '../util' - -const VIDEO_OVERRIDE_STYLE = ` - * { - pointer-events: none; - display: none !important; - position: static !important; - z-index: 0 !important; - } - html, body, video, audio { - display: block !important; - background: black !important; - } - html, body { - overflow: hidden !important; - background: black !important; - } - video, iframe.__video__, audio { - display: block !important; - position: fixed !important; - left: 0 !important; - right: 0 !important; - top: 0 !important; - bottom: 0 !important; - width: 100vw !important; - height: 100vh !important; - object-fit: cover !important; - transition: none !important; - z-index: 999999 !important; - } - audio { - z-index: 999998 !important; - } - .__video_parent__ { - display: block !important; - } - video.__rot180__ { - transform: rotate(180deg) !important; - } - /* For 90 degree rotations, we position the video with swapped width and height and rotate it into place. - It's helpful to offset the video so the transformation is centered in the viewport center. - We move the video top left corner to center of the page and then translate half the video dimensions up and left. - Note that the width and height are swapped in the translate because the video starts with the side dimensions swapped. */ - video.__rot90__ { - transform: translate(-50vh, -50vw) rotate(90deg) !important; - } - video.__rot270__ { - transform: translate(-50vh, -50vw) rotate(270deg) !important; - } - video.__rot90__, video.__rot270__ { - left: 50vw !important; - top: 50vh !important; - width: 100vh !important; - height: 100vw !important; - } -` - const viewStateMachine = Machine( { id: 'view', - initial: 'empty', + initial: 'loading', context: { - view: null, + id: null, pos: null, content: null, info: {}, + sendToWall: null, + sendToFrame: null, }, on: { - DISPLAY: 'displaying', - }, - states: { - empty: {}, - displaying: { - id: 'displaying', - initial: 'loading', - entry: assign({ + DISPLAY: { + actions: assign({ pos: (context, event) => event.pos, content: (context, event) => event.content, }), + }, + RELOAD: { + actions: 'reload', + }, + VIEW_INIT: { + target: '.loading', + actions: assign({ + sendToFrame: (context, event) => event.sendToFrame, + }), + }, + VIEW_ERROR: '.error', + }, + states: { + loading: { on: { - DISPLAY: { + VIEW_LOADED: { actions: assign({ - pos: (context, event) => event.pos, + info: (context, event) => event.info, }), - cond: 'contentUnchanged', - }, - RELOAD: '.loading', - DEVTOOLS: { - actions: 'openDevTools', + target: '#view.running', }, }, + }, + running: { + type: 'parallel', states: { - loading: { - initial: 'page', - entry: 'offscreenView', - states: { - page: { - invoke: { - src: 'loadPage', - onDone: { - target: 'video', - }, - onError: { - target: '#view.displaying.error', - }, - }, - }, - video: { - invoke: { - src: 'startVideo', - onDone: { - target: '#view.displaying.running', - actions: assign({ - info: (context, event) => event.data, - }), - }, - onError: { - target: '#view.displaying.error', - }, - }, - }, - }, - }, - running: { - type: 'parallel', - entry: 'positionView', + audio: { + initial: 'muted', on: { - DISPLAY: { - actions: [ - assign({ - pos: (context, event) => event.pos, - }), - 'positionView', - ], - cond: 'contentUnchanged', - }, + MUTE: '.muted', + UNMUTE: '.listening', + BACKGROUND: '.background', + UNBACKGROUND: '.muted', }, states: { - audio: { - initial: 'muted', - on: { - MUTE: '.muted', - UNMUTE: '.listening', - BACKGROUND: '.background', - UNBACKGROUND: '.muted', - }, - states: { - muted: { - entry: 'muteAudio', - }, - listening: { - entry: 'unmuteAudio', - }, - background: { - on: { - // Ignore normal audio swapping. - MUTE: {}, - }, - entry: 'unmuteAudio', - }, - }, + muted: { + entry: 'muteAudio', }, - video: { - initial: 'normal', + listening: { + entry: 'unmuteAudio', + }, + background: { on: { - BLUR: '.blurred', - UNBLUR: '.normal', - }, - states: { - normal: {}, - blurred: {}, + // Ignore normal audio swapping. + MUTE: {}, }, + entry: 'unmuteAudio', }, }, }, - error: { - entry: 'logError', + video: { + initial: 'normal', + on: { + BLUR: '.blurred', + UNBLUR: '.normal', + }, + states: { + normal: {}, + blurred: {}, + }, }, }, }, + error: { + entry: 'logError', + }, }, }, { actions: { logError: (context, event) => { - console.warn(event) + console.warn(event.err) + }, + reload: (context) => { + // TODO: keep muted over reload + context.sendToWall('reload-view', { viewId: context.id }) }, muteAudio: (context, event) => { - context.view.webContents.audioMuted = true + context.sendToFrame('mute') }, unmuteAudio: (context, event) => { - context.view.webContents.audioMuted = false - }, - openDevTools: (context, event) => { - const { view } = context - const { inWebContents } = event - view.webContents.setDevToolsWebContents(inWebContents) - view.webContents.openDevTools({ mode: 'detach' }) - }, - }, - guards: { - contentUnchanged: (context, event) => { - return isEqual(context.content, event.content) - }, - }, - services: { - loadPage: async (context, event) => { - const { content, view } = context - ensureValidURL(content.url) - const wc = view.webContents - wc.audioMuted = true - await wc.loadURL(content.url) - if (content.kind === 'video' || content.kind === 'audio') { - wc.insertCSS(VIDEO_OVERRIDE_STYLE, { cssOrigin: 'user' }) - } else if (content.kind === 'web') { - wc.insertCSS( - ` - html, body { - overflow: hidden !important; - } - `, - { cssOrigin: 'user' }, - ) - } - }, - startVideo: async (context, event) => { - const { content, view } = context - if (content.kind !== 'video' && content.kind !== 'audio') { - return - } - const wc = view.webContents - // TODO: generalize "video" terminology to "media" - const info = await wc.executeJavaScript(` - (function() { - const sleep = ms => new Promise((resolve) => setTimeout(resolve, ms)) - const kind = ${JSON.stringify( - content.kind === 'video' ? 'video' : 'audio', - )} - - async function waitForVideo() { - let tries = 0 - let video - while ((!video || !video.src) && tries < 10) { - video = document.querySelector(kind) - tries++ - await sleep(200) - } - if (video) { - return {video} - } - - tries = 0 - let iframe - while ((!video || !video.src) && tries < 10) { - for (iframe of document.querySelectorAll('iframe')) { - if (!iframe.contentDocument) { - continue - } - video = iframe.contentDocument.querySelector(kind) - if (video) { - break - } - } - tries++ - await sleep(200) - } - if (video) { - return { video, iframe } - } - return {} - } - - const periscopeHacks = { - isMatch() { - return location.host === 'www.pscp.tv' || location.host === 'www.periscope.tv' - }, - onLoad() { - const playButton = document.querySelector('.PlayButton') - if (playButton) { - playButton.click() - } - }, - afterPlay(video) { - const baseVideoEl = document.querySelector('div.BaseVideo') - if (!baseVideoEl) { - return - } - - function positionPeriscopeVideo() { - // Periscope videos can be rotated using transform matrix. They need to be rotated correctly. - const tr = baseVideoEl.style.transform - if (tr.endsWith('matrix(0, 1, -1, 0, 0, 0)')) { - video.className = '__rot90__' - } else if (tr.endsWith('matrix(-1, 0, 0, -1, 0)')) { - video.className = '__rot180__' - } else if (tr.endsWith('matrix(0, -1, 1, 0, 0, 0)')) { - video.className = '__rot270__' - } - } - - positionPeriscopeVideo() - const obs = new MutationObserver(ml => { - for (const m of ml) { - if (m.attributeName === 'style') { - positionPeriscopeVideo() - return - } - } - }) - obs.observe(baseVideoEl, {attributes: true}) - }, - } - - async function findVideo() { - if (periscopeHacks.isMatch()) { - periscopeHacks.onLoad() - } - - const { video, iframe } = await waitForVideo() - if (!video) { - throw new Error('could not find video') - } - if (iframe) { - const style = iframe.contentDocument.createElement('style') - style.innerHTML = \`${VIDEO_OVERRIDE_STYLE}\` - iframe.contentDocument.head.appendChild(style) - iframe.className = '__video__' - let parentEl = iframe.parentElement - while (parentEl) { - parentEl.className = '__video_parent__' - parentEl = parentEl.parentElement - } - iframe.contentDocument.body.appendChild(video) - } else { - document.body.appendChild(video) - } - video.muted = false - video.autoPlay = true - video.play() - - // Prevent sites from re-muting the video (Periscope, I'm looking at you!) - Object.defineProperty(video, 'muted', {writable: true, value: false}) - - if (periscopeHacks.isMatch()) { - periscopeHacks.afterPlay(video) - } - - const info = { title: document.title } - return info - } - findVideo() - }()) - `) - return info + context.sendToFrame('unmute') }, }, }, diff --git a/src/web/control.js b/src/web/control.js index dbeddf9..5cd102b 100644 --- a/src/web/control.js +++ b/src/web/control.js @@ -24,7 +24,6 @@ import SoundIcon from '../static/volume-up-solid.svg' import NoVideoIcon from '../static/video-slash-solid.svg' import ReloadIcon from '../static/redo-alt-solid.svg' import SwapIcon from '../static/exchange-alt-solid.svg' -import LifeRingIcon from '../static/life-ring-regular.svg' import WindowIcon from '../static/window-maximize-regular.svg' import { idColor } from './colors' @@ -137,13 +136,11 @@ function useStreamwallConnection(wsEndpoint) { for (const viewState of views) { const { pos } = viewState.context const state = State.from(viewState.state) - const isListening = state.matches( - 'displaying.running.audio.listening', - ) + const isListening = state.matches('running.audio.listening') const isBackgroundListening = state.matches( - 'displaying.running.audio.background', + 'running.audio.background', ) - const isBlurred = state.matches('displaying.running.video.blurred') + const isBlurred = state.matches('running.video.blurred') for (const space of pos.spaces) { if (!newStateIdxMap.has(space)) { newStateIdxMap.set(space, {}) @@ -374,13 +371,6 @@ function App({ wsEndpoint, role }) { [streams], ) - const handleDevTools = useCallback((idx) => { - send({ - type: 'dev-tools', - viewIdx: idx, - }) - }, []) - const handleClickId = useCallback( (streamId) => { try { @@ -543,8 +533,8 @@ function App({ wsEndpoint, role }) { ) })} @@ -789,7 +778,6 @@ function GridInput({ onReloadView, onSwapView, onBrowse, - onDevTools, }) { const [editingValue, setEditingValue] = useState() const handleFocus = useCallback( @@ -841,10 +829,6 @@ function GridInput({ spaceValue, onBrowse, ]) - const handleDevToolsClick = useCallback(() => onDevTools(idx), [ - idx, - onDevTools, - ]) const handleMouseDown = useCallback( (ev) => { onMouseDown(idx, ev) @@ -863,11 +847,6 @@ function GridInput({ )} - {roleCan(role, 'dev-tools') && ( - - - - )} ) : ( <> diff --git a/webpack.config.js b/webpack.config.js index 8bbd01d..22877f9 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -78,8 +78,12 @@ const browserConfig = { devtool: 'cheap-source-map', target: 'electron-renderer', entry: { - background: './src/browser/background.js', - overlay: './src/browser/overlay.js', + wall: './src/browser/wall.js', + wallPreload: './src/browser/wallPreload.js', + mediaPreload: './src/browser/mediaPreload.js', + }, + externals: { + events: 'commonjs events', }, plugins: [ new CopyPlugin({