Salastil/streamwall
1
0
Fork 0
mirror of https://github.com/streamwall/streamwall.git synced 2026-01-31 17:32:48 -05:00
Code Issues Packages Projects Releases Wiki Activity

Experimental offscreen rendering + iframe rearchitecture

Browse Source
This commit is contained in:
Max Goodhart
2020-10-04 22:05:27 -07:00
parent 4726954cb2
commit 3d6b5a8c7c
13 changed files with 592 additions and 579 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/browser/background.html
View File

@@ -1,14 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8" />
<title>Streamwall Stream Background</title>
<meta
http-equiv="Content-Security-Policy"
content="default-src 'self'; style-src 'self' 'unsafe-inline'; frame-src *"
/>
</head>
<body>
<script src="background.js" type="module"></script>
</body>
</html>

48
src/browser/background.js
View File

@@ -1,48 +0,0 @@
import { ipcRenderer } from 'electron'
import { h, render } from 'preact'
import { useEffect, useState } from 'preact/hooks'
import styled from 'styled-components'
import '../index.css'
function Background({ streams }) {
const backgrounds = streams.filter((s) => s.kind === 'background')
return (
<div>
{backgrounds.map((s) => (
<BackgroundIFrame
key={s._id}
src={s.link}
sandbox="allow-scripts allow-same-origin"
allow="autoplay"
/>
))}
</div>
)
}
function App() {
const [state, setState] = useState({
streams: [],
})
useEffect(() => {
ipcRenderer.on('state', (ev, state) => {
setState(state)
})
}, [])
const { streams } = state
return <Background streams={streams} />
}
const BackgroundIFrame = styled.iframe`
position: fixed;
left: 0;
top: 0;
width: 100vw;
height: 100vh;
border: none;
`
render(<App />, document.body)

248
src/browser/mediaPreload.js Normal file
View File

@@ -0,0 +1,248 @@
import { ipcRenderer, webFrame } from 'electron'
const VIDEO_OVERRIDE_STYLE = `
* {
pointer-events: none;
display: none !important;
position: static !important;
z-index: 0 !important;
}
html, body, video, audio {
display: block !important;
background: black !important;
}
html, body {
overflow: hidden !important;
background: black !important;
}
video, iframe.__video__, audio {
display: block !important;
position: fixed !important;
left: 0 !important;
right: 0 !important;
top: 0 !important;
bottom: 0 !important;
width: 100vw !important;
height: 100vh !important;
object-fit: cover !important;
transition: none !important;
z-index: 999999 !important;
}
audio {
z-index: 999998 !important;
}
.__video_parent__ {
display: block !important;
}
video.__rot180__ {
transform: rotate(180deg) !important;
}
/* For 90 degree rotations, we position the video with swapped width and height and rotate it into place.
It's helpful to offset the video so the transformation is centered in the viewport center.
We move the video top left corner to center of the page and then translate half the video dimensions up and left.
Note that the width and height are swapped in the translate because the video starts with the side dimensions swapped. */
video.__rot90__ {
transform: translate(-50vh, -50vw) rotate(90deg) !important;
}
video.__rot270__ {
transform: translate(-50vh, -50vw) rotate(270deg) !important;
}
video.__rot90__, video.__rot270__ {
left: 50vw !important;
top: 50vh !important;
width: 100vh !important;
height: 100vw !important;
}
`
const NO_SCROLL_STYLE = `
html, body {
overflow: hidden !important;
}
`
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
function lockdownMediaTags() {
webFrame.executeJavaScript(`
for (const el of document.querySelectorAll('video, audio')) {
if (el.__sw) {
continue
}
el.muted = true
// Prevent sites from unmuting (Periscope, I'm looking at you!)
Object.defineProperty(el, 'muted', { writable: true, value: false })
// Prevent Facebook from pausing the video after page load.
Object.defineProperty(el, 'pause', { writable: false, value: () => {} })
el.__sw = true
}
`)
}
// Watch for media tags and mute them as soon as possible.
function watchMediaTags(kind, onFirstOfKind) {
let foundMatch = false
const observer = new MutationObserver((mutationList) => {
if (kind) {
const el = document.querySelector(kind)
if (el && !foundMatch) {
onFirstOfKind(el)
foundMatch = true
}
}
lockdownMediaTags()
})
document.addEventListener('DOMContentLoaded', () => {
observer.observe(document.body, { subtree: true, childList: true })
})
}
async function waitForVideo(kind) {
const waitForTag = new Promise((resolve) => watchMediaTags(kind, resolve))
let video = await Promise.race([waitForTag, sleep(10000)])
if (video) {
return { video }
}
let tries = 0
let iframe
while ((!video || !video.src) && tries < 10) {
for (iframe of document.querySelectorAll('iframe')) {
if (!iframe.contentDocument) {
continue
}
video = iframe.contentDocument.querySelector(kind)
if (video) {
break
}
}
tries++
await sleep(200)
}
if (video) {
return { video, iframe }
}
return {}
}
const periscopeHacks = {
isMatch() {
return (
location.host === 'www.pscp.tv' || location.host === 'www.periscope.tv'
)
},
onLoad() {
const playButton = document.querySelector('.PlayButton')
if (playButton) {
playButton.click()
}
},
afterPlay(video) {
const baseVideoEl = document.querySelector('div.BaseVideo')
if (!baseVideoEl) {
return
}
function positionPeriscopeVideo() {
// Periscope videos can be rotated using transform matrix. They need to be rotated correctly.
const tr = baseVideoEl.style.transform
if (tr.endsWith('matrix(0, 1, -1, 0, 0, 0)')) {
video.className = '__rot90__'
} else if (tr.endsWith('matrix(-1, 0, 0, -1, 0)')) {
video.className = '__rot180__'
} else if (tr.endsWith('matrix(0, -1, 1, 0, 0, 0)')) {
video.className = '__rot270__'
}
}
positionPeriscopeVideo()
const obs = new MutationObserver((ml) => {
for (const m of ml) {
if (m.attributeName === 'style') {
positionPeriscopeVideo()
return
}
}
})
obs.observe(baseVideoEl, { attributes: true })
},
}
async function findVideo(kind) {
if (periscopeHacks.isMatch()) {
periscopeHacks.onLoad()
}
const { video, iframe } = await waitForVideo(kind)
if (!video) {
throw new Error('could not find video')
}
if (iframe) {
// TODO: verify iframe still works
const style = iframe.contentDocument.createElement('style')
style.innerHTML = VIDEO_OVERRIDE_STYLE
iframe.contentDocument.head.appendChild(style)
iframe.className = '__video__'
let parentEl = iframe.parentElement
while (parentEl) {
parentEl.className = '__video_parent__'
parentEl = parentEl.parentElement
}
iframe.contentDocument.body.appendChild(video)
} else {
document.body.appendChild(video)
}
const videoReady = new Promise((resolve) =>
video.addEventListener('canplay', resolve, { once: true }),
)
video.play()
if (periscopeHacks.isMatch()) {
periscopeHacks.afterPlay(video)
}
if (!video.videoWidth) {
await videoReady
}
const info = {
title: document.title,
intrinsicWidth: video.videoWidth,
intrinsicHeight: video.videoHeight,
}
return { info, video }
}
async function main(viewId) {
const viewInit = ipcRenderer.invoke('view-init', { viewId })
const pageReady = new Promise((resolve) => process.once('loaded', resolve))
const [{ content }] = await Promise.all([viewInit, pageReady])
if (content.kind === 'video' || content.kind === 'audio') {
webFrame.insertCSS(VIDEO_OVERRIDE_STYLE)
} else if (content.kind === 'web') {
webFrame.insertCSS(NO_SCROLL_STYLE)
}
const { info, video } = await findVideo(content.kind)
ipcRenderer.send('view-loaded', { viewId, info })
ipcRenderer.on('mute', () => {
video.muted = true
})
ipcRenderer.on('unmute', () => {
video.muted = false
})
}
if (window.name) {
try {
main(window.name)
} catch (err) {
ipcRenderer.send('view-error', { viewId, err })
}
} else {
watchMediaTags()
}

2
src/browser/overlay.html → src/browser/wall.html
View File

@@ -9,6 +9,6 @@
/> />
</head> </head>
<body> <body>
<script src="overlay.js" type="module"></script> <script src="wall.js" type="module"></script>
</body> </body>
</html> </html>

148
src/browser/overlay.js → src/browser/wall.js
View File

@@ -1,6 +1,5 @@
import { ipcRenderer } from 'electron'
import { h, Fragment, render } from 'preact' import { h, Fragment, render } from 'preact'
import { useEffect, useState } from 'preact/hooks' import { useEffect, useState, useRef } from 'preact/hooks'
import { State } from 'xstate' import { State } from 'xstate'
import styled from 'styled-components' import styled from 'styled-components'
import { useHotkeys } from 'react-hotkeys-hook' import { useHotkeys } from 'react-hotkeys-hook'
@@ -16,26 +15,43 @@ import TwitchIcon from '../static/twitch.svg'
import YouTubeIcon from '../static/youtube.svg' import YouTubeIcon from '../static/youtube.svg'
import SoundIcon from '../static/volume-up-solid.svg' import SoundIcon from '../static/volume-up-solid.svg'
const VIEW_POS_TRANSITION = '0.1s linear'
const VIEW_OPACITY_TRANSITION = '0.5s ease-out'
function Overlay({ config, views, streams }) { function Overlay({ config, views, streams }) {
const { width, height, activeColor } = config const { width, height, activeColor } = config
const activeViews = views const activeViews = views
.map(({ state, context }) => State.from(state, context)) .map(({ state, context }) => State.from(state, context))
.filter((s) => s.matches('displaying') && !s.matches('displaying.error')) .filter((s) => !s.matches('error'))
const backgrounds = streams.filter((s) => s.kind === 'background')
const overlays = streams.filter((s) => s.kind === 'overlay') const overlays = streams.filter((s) => s.kind === 'overlay')
// TODO: prevent iframes from being reparented
return ( return (
<div> <div>
{backgrounds.map((s) => (
<OverlayIframe key={s._id} src={s.link} />
))}
{activeViews.map((viewState) => { {activeViews.map((viewState) => {
const { content, pos } = viewState.context const { viewId, content, pos, info } = viewState.context
const data = streams.find((d) => content.url === d.link) const data = streams.find((d) => content.url === d.link)
const isListening = viewState.matches( const isListening = viewState.matches('running.audio.listening')
'displaying.running.audio.listening',
)
const isBackgroundListening = viewState.matches( const isBackgroundListening = viewState.matches(
'displaying.running.audio.background', 'running.audio.background',
) )
const isBlurred = viewState.matches('displaying.running.video.blurred') const isBlurred = viewState.matches('running.video.blurred')
const isLoading = viewState.matches('displaying.loading') const isLoading = viewState.matches('loading')
const isRunning = viewState.matches('running')
return ( return (
<ViewContainer key={viewId} pos={pos}>
<MediaIframe
key={viewId}
pos={pos}
intrinsicWidth={info.intrinsicWidth}
intrinsicHeight={info.intrinsicHeight}
isRunning={isRunning}
src={content.url}
name={viewId}
></MediaIframe>
<SpaceBorder <SpaceBorder
pos={pos} pos={pos}
windowWidth={width} windowWidth={width}
@@ -43,9 +59,11 @@ function Overlay({ config, views, streams }) {
activeColor={activeColor} activeColor={activeColor}
isListening={isListening} isListening={isListening}
> >
<BlurCover isBlurred={isBlurred} />
{data && ( {data && (
<StreamTitle activeColor={activeColor} isListening={isListening}> <StreamTitle
activeColor={activeColor}
isListening={isListening}
>
<StreamIcon url={content.url} /> <StreamIcon url={content.url} />
<span> <span>
{data.hasOwnProperty('label') ? ( {data.hasOwnProperty('label') ? (
@@ -59,17 +77,13 @@ function Overlay({ config, views, streams }) {
{(isListening || isBackgroundListening) && <SoundIcon />} {(isListening || isBackgroundListening) && <SoundIcon />}
</StreamTitle> </StreamTitle>
)} )}
{isLoading && <LoadingSpinner />}
</SpaceBorder> </SpaceBorder>
{isLoading && <LoadingSpinner />}
</ViewContainer>
) )
})} })}
{overlays.map((s) => ( {overlays.map((s) => (
<OverlayIFrame <OverlayIframe key={s._id} src={s.link} />
key={s._id}
src={s.link}
sandbox="allow-scripts allow-same-origin"
allow="autoplay"
/>
))} ))}
</div> </div>
) )
@@ -84,13 +98,17 @@ function App() {
}) })
useEffect(() => { useEffect(() => {
ipcRenderer.on('state', (ev, state) => { streamwall.onState(setState)
setState(state) streamwall.onReloadView(({ viewId }) => {
const viewFrame = document.querySelector(`iframe[name="${viewId}"]`)
if (viewFrame) {
viewFrame.src = viewFrame.src
}
}) })
}, []) }, [])
useHotkeys('ctrl+shift+i', () => { useHotkeys('ctrl+shift+i', () => {
ipcRenderer.send('devtools-overlay') streamwall.openDevTools()
}) })
const { config, views, streams, customStreams } = state const { config, views, streams, customStreams } = state
@@ -132,16 +150,74 @@ function StreamIcon({ url, ...props }) {
return null return null
} }
const SpaceBorder = styled.div.attrs((props) => ({ function MediaIframe({
borderWidth: 2, isRunning,
}))` pos,
display: flex; intrinsicWidth,
align-items: flex-start; intrinsicHeight,
...props
}) {
const frameRef = useRef()
// Set isMounted after render so the transition doesn't apply to the initial sizing of the frame.
useEffect(() => {
if (!isRunning) {
return
}
const { current: el } = frameRef
el.style.transition = `transform ${VIEW_POS_TRANSITION}, opacity ${VIEW_OPACITY_TRANSITION}`
el.style.opacity = 1
}, [isRunning])
let style = {
position: 'absolute',
left: -9999,
width: pos.width,
height: pos.height,
opacity: 0,
}
if (isRunning) {
const scale = Math.max(
pos.width / intrinsicWidth,
pos.height / intrinsicHeight,
)
const translateX = -(intrinsicWidth * scale - pos.width) / 2
const translateY = -(intrinsicHeight * scale - pos.height) / 2
const transform = `translate(${translateX}px, ${translateY}px) scale(${scale})`
style = {
opacity: 0,
transform,
transition: 'none',
// TODO: explore not oversampling as a perf improvement
width: intrinsicWidth,
height: intrinsicHeight,
}
}
return <StyledMediaIframe ref={frameRef} {...props} style={style} />
}
const ViewContainer = styled.div`
position: fixed; position: fixed;
left: ${({ pos }) => pos.x}px; left: ${({ pos }) => pos.x}px;
top: ${({ pos }) => pos.y}px; top: ${({ pos }) => pos.y}px;
width: ${({ pos }) => pos.width}px; width: ${({ pos }) => pos.width}px;
height: ${({ pos }) => pos.height}px; height: ${({ pos }) => pos.height}px;
overflow: hidden;
transition: left ${VIEW_POS_TRANSITION}, top ${VIEW_POS_TRANSITION},
width ${VIEW_POS_TRANSITION}, height ${VIEW_POS_TRANSITION};
will-change: left, top, width, height;
`
const SpaceBorder = styled.div.attrs((props) => ({
borderWidth: 2,
}))`
position: absolute;
left: 0;
top: 0;
right: 0;
bottom: 0;
display: flex;
align-items: flex-start;
border: 0 solid black; border: 0 solid black;
border-left-width: ${({ pos, borderWidth }) => border-left-width: ${({ pos, borderWidth }) =>
pos.x === 0 ? 0 : borderWidth}px; pos.x === 0 ? 0 : borderWidth}px;
@@ -220,14 +296,26 @@ const BlurCover = styled.div`
backdrop-filter: ${({ isBlurred }) => (isBlurred ? 'blur(30px)' : 'blur(0)')}; backdrop-filter: ${({ isBlurred }) => (isBlurred ? 'blur(30px)' : 'blur(0)')};
` `
const OverlayIFrame = styled.iframe` const Iframe = styled.iframe.attrs((props) => ({
sandbox: 'allow-scripts allow-same-origin',
allow: 'autoplay',
}))`
border: none;
pointer-events: none;
`
const StyledMediaIframe = styled(Iframe)`
position: absolute;
transform-origin: top left;
will-change: opacity, transform;
`
const OverlayIframe = styled(Iframe)`
position: fixed; position: fixed;
left: 0; left: 0;
top: 0; top: 0;
width: 100vw; width: 100vw;
height: 100vh; height: 100vh;
border: none;
pointer-events: none;
` `
render(<App />, document.body) render(<App />, document.body)

8
src/browser/wallPreload.js Normal file
View File

@@ -0,0 +1,8 @@
import { ipcRenderer, contextBridge } from 'electron'
contextBridge.exposeInMainWorld('streamwall', {
openDevTools: () => ipcRenderer.send('devtools-overlay'),
onState: (handler) => ipcRenderer.on('state', (ev, state) => handler(state)),
onReloadView: (handler) =>
ipcRenderer.on('reload-view', (ev, data) => handler(data)),
})

214
src/node/StreamWindow.js
View File

@@ -1,7 +1,10 @@
import path from 'path'
import isEqual from 'lodash/isEqual' import isEqual from 'lodash/isEqual'
import sortBy from 'lodash/sortBy'
import intersection from 'lodash/intersection' import intersection from 'lodash/intersection'
import EventEmitter from 'events' import EventEmitter from 'events'
import { BrowserView, BrowserWindow, ipcMain } from 'electron' import net from 'net'
import { app, BrowserWindow, ipcMain } from 'electron'
import { interpret } from 'xstate' import { interpret } from 'xstate'
import viewStateMachine from './viewStateMachine' import viewStateMachine from './viewStateMachine'
@@ -20,8 +23,8 @@ export default class StreamWindow extends EventEmitter {
this.offscreenWin = null this.offscreenWin = null
this.backgroundView = null this.backgroundView = null
this.overlayView = null this.overlayView = null
this.views = [] this.views = new Map()
this.viewActions = null this.nextViewId = 0
} }
init() { init() {
@@ -45,135 +48,133 @@ export default class StreamWindow extends EventEmitter {
backgroundColor, backgroundColor,
useContentSize: true, useContentSize: true,
show: false, show: false,
webPreferences: {
offscreen: true,
sandbox: true,
nodeIntegration: true,
nodeIntegrationInSubFrames: true,
contextIsolation: true,
worldSafeExecuteJavaScript: true,
partition: 'persist:session',
preload: path.join(app.getAppPath(), 'wallPreload.js'),
},
}) })
win.removeMenu() win.removeMenu()
win.loadURL('about:blank') // via https://github.com/electron/electron/pull/573#issuecomment-642216738
win.webContents.session.webRequest.onHeadersReceived(
({ responseHeaders }, callback) => {
for (const headerName of Object.keys(responseHeaders)) {
if (headerName.toLowerCase() === 'x-frame-options') {
delete responseHeaders[headerName]
}
if (headerName.toLowerCase() === 'content-security-policy') {
const csp = responseHeaders[headerName]
responseHeaders[headerName] = csp.map((val) =>
val.replace(/frame-ancestors[^;]+;/, ''),
)
}
}
callback({ responseHeaders })
},
)
win.webContents.session.setPreloads([
path.join(app.getAppPath(), 'mediaPreload.js'),
])
win.webContents.loadFile('wall.html')
win.on('close', () => this.emit('close')) win.on('close', () => this.emit('close'))
let sock = net.connect(3000, '127.0.0.1')
let reconnectTimeout
sock.on('close', () => {
clearTimeout(reconnectTimeout)
reconnectTimeout = setTimeout(() => {
try {
sock.connect(3000, '127.0.0.1')
} catch (err) {}
}, 1000)
})
sock.on('error', () => {})
win.webContents.beginFrameSubscription((image) => {
if (sock.destroyed) {
return
}
try {
sock.write(image.getBitmap())
} catch (err) {}
})
win.webContents.setFrameRate(30)
// Work around https://github.com/electron/electron/issues/14308 // Work around https://github.com/electron/electron/issues/14308
// via https://github.com/lutzroeder/netron/commit/910ce67395130690ad76382c094999a4f5b51e92 // via https://github.com/lutzroeder/netron/commit/910ce67395130690ad76382c094999a4f5b51e92
win.once('ready-to-show', () => { win.once('ready-to-show', () => {
win.resizable = false win.resizable = false
win.show()
}) })
this.win = win this.win = win
const offscreenWin = new BrowserWindow({
show: false,
webPreferences: {
offscreen: true,
},
})
this.offscreenWin = offscreenWin
const backgroundView = new BrowserView({
webPreferences: {
nodeIntegration: true,
},
})
win.addBrowserView(backgroundView)
backgroundView.setBounds({
x: 0,
y: 0,
width,
height,
})
backgroundView.webContents.loadFile('background.html')
this.backgroundView = backgroundView
const overlayView = new BrowserView({
webPreferences: {
nodeIntegration: true,
},
})
win.addBrowserView(overlayView)
overlayView.setBounds({
x: 0,
y: 0,
width,
height,
})
overlayView.webContents.loadFile('overlay.html')
this.overlayView = overlayView
this.viewActions = {
offscreenView: (context, event) => {
const { view } = context
// It appears necessary to initialize the browser view by adding it to a window and setting bounds. Otherwise, some streaming sites like Periscope will not load their videos due to the Page Visibility API being hidden.
win.removeBrowserView(view)
offscreenWin.addBrowserView(view)
view.setBounds({ x: 0, y: 0, width: spaceWidth, height: spaceHeight })
},
positionView: (context, event) => {
const { pos, view } = context
win.addBrowserView(view)
// It's necessary to remove and re-add the overlay view to ensure it's on top.
win.removeBrowserView(overlayView)
win.addBrowserView(overlayView)
view.setBounds(pos)
},
}
ipcMain.on('devtools-overlay', () => { ipcMain.on('devtools-overlay', () => {
overlayView.webContents.openDevTools() win.webContents.openDevTools()
})
ipcMain.handle('view-init', async (ev, { viewId }) => {
const view = this.views.get(viewId)
if (view) {
view.send({
type: 'VIEW_INIT',
sendToFrame: (...args) => ev.sender.sendToFrame(ev.frameId, ...args),
})
return { content: view.state.context.content }
}
})
ipcMain.on('view-loaded', (ev, { viewId, info }) => {
this.views.get(viewId)?.send?.({ type: 'VIEW_LOADED', info })
})
ipcMain.on('view-error', (ev, { viewId, err }) => {
this.views.get(viewId)?.send?.({ type: 'VIEW_ERROR', err })
}) })
} }
createView() { createView() {
const { win, overlayView, viewActions } = this // TODO: no parallel functionality in iframe?
const { backgroundColor } = this.config /*
const view = new BrowserView({
webPreferences: {
nodeIntegration: false,
enableRemoteModule: false,
contextIsolation: true,
partition: 'persist:session',
sandbox: true,
},
})
view.setBackgroundColor(backgroundColor)
// Prevent view pages from navigating away from the specified URL. // Prevent view pages from navigating away from the specified URL.
view.webContents.on('will-navigate', (ev) => { view.webContents.on('will-navigate', (ev) => {
ev.preventDefault() ev.preventDefault()
}) })
*/
const machine = viewStateMachine const machine = viewStateMachine.withContext({
.withContext({
...viewStateMachine.context, ...viewStateMachine.context,
view, id: `__view:${this.nextViewId}`,
parentWin: win, sendToWall: (...args) => this.win.webContents.send(...args),
overlayView,
}) })
.withConfig({ actions: viewActions })
const service = interpret(machine).start() const service = interpret(machine).start()
service.onTransition(this.emitState.bind(this)) service.onTransition(this.emitState.bind(this))
this.nextViewId++
return service return service
} }
emitState() { emitState() {
this.emit( const states = Array.from(this.views.values(), ({ state }) => ({
'state',
this.views.map(({ state }) => ({
state: state.value, state: state.value,
context: { context: {
viewId: state.context.id,
content: state.context.content, content: state.context.content,
info: state.context.info, info: state.context.info,
pos: state.context.pos, pos: state.context.pos,
}, },
})), }))
) this.emit('state', sortBy(states, 'context.viewId'))
} }
setViews(viewContentMap) { setViews(viewContentMap) {
const { gridCount, spaceWidth, spaceHeight } = this.config const { gridCount, spaceWidth, spaceHeight } = this.config
const { win, views } = this const { views } = this
const boxes = boxesFromViewContentMap(gridCount, gridCount, viewContentMap) const boxes = boxesFromViewContentMap(gridCount, gridCount, viewContentMap)
const remainingBoxes = new Set(boxes) const remainingBoxes = new Set(boxes)
const unusedViews = new Set(views) const unusedViews = new Set(views.values())
const viewsToDisplay = [] const viewsToDisplay = []
// We try to find the best match for moving / reusing existing views to match the new positions. // We try to find the best match for moving / reusing existing views to match the new positions.
@@ -181,12 +182,11 @@ export default class StreamWindow extends EventEmitter {
// First try to find a loaded view of the same URL in the same space... // First try to find a loaded view of the same URL in the same space...
(v, content, spaces) => (v, content, spaces) =>
isEqual(v.state.context.content, content) && isEqual(v.state.context.content, content) &&
v.state.matches('displaying.running') && v.state.matches('running') &&
intersection(v.state.context.pos.spaces, spaces).length > 0, intersection(v.state.context.pos.spaces, spaces).length > 0,
// Then try to find a loaded view of the same URL... // Then try to find a loaded view of the same URL...
(v, content) => (v, content) =>
isEqual(v.state.context.content, content) && isEqual(v.state.context.content, content) && v.state.matches('running'),
v.state.matches('displaying.running'),
// Then try view with the same URL that is still loading... // Then try view with the same URL that is still loading...
(v, content) => isEqual(v.state.context.content, content), (v, content) => isEqual(v.state.context.content, content),
] ]
@@ -214,7 +214,7 @@ export default class StreamWindow extends EventEmitter {
viewsToDisplay.push({ box, view }) viewsToDisplay.push({ box, view })
} }
const newViews = [] const newViews = new Map()
for (const { box, view } of viewsToDisplay) { for (const { box, view } of viewsToDisplay) {
const { content, x, y, w, h, spaces } = box const { content, x, y, w, h, spaces } = box
const pos = { const pos = {
@@ -225,12 +225,7 @@ export default class StreamWindow extends EventEmitter {
spaces, spaces,
} }
view.send({ type: 'DISPLAY', pos, content }) view.send({ type: 'DISPLAY', pos, content })
newViews.push(view) newViews.set(view.state.context.id, view)
}
for (const view of unusedViews) {
const browserView = view.state.context.view
win.removeBrowserView(browserView)
browserView.destroy()
} }
this.views = newViews this.views = newViews
this.emitState() this.emitState()
@@ -238,10 +233,7 @@ export default class StreamWindow extends EventEmitter {
setListeningView(viewIdx) { setListeningView(viewIdx) {
const { views } = this const { views } = this
for (const view of views) { for (const view of views.values()) {
if (!view.state.matches('displaying')) {
continue
}
const { context } = view.state const { context } = view.state
const isSelectedView = context.pos.spaces.includes(viewIdx) const isSelectedView = context.pos.spaces.includes(viewIdx)
view.send(isSelectedView ? 'UNMUTE' : 'MUTE') view.send(isSelectedView ? 'UNMUTE' : 'MUTE')
@@ -249,10 +241,11 @@ export default class StreamWindow extends EventEmitter {
} }
findViewByIdx(viewIdx) { findViewByIdx(viewIdx) {
return this.views.find( for (const view of this.views.values()) {
(v) => if (view.state.context.pos?.spaces?.includes?.(viewIdx)) {
v.state.context.pos && v.state.context.pos.spaces.includes(viewIdx), return view
) }
}
} }
sendViewEvent(viewIdx, event) { sendViewEvent(viewIdx, event) {
@@ -279,7 +272,6 @@ export default class StreamWindow extends EventEmitter {
} }
send(...args) { send(...args) {
this.overlayView.webContents.send(...args) this.win.webContents.send(...args)
this.backgroundView.webContents.send(...args)
} }
} }

2
src/node/TwitchBot.js
View File

@@ -57,7 +57,7 @@ export default class TwitchBot extends EventEmitter {
this.streams = streams this.streams = streams
const listeningView = views.find(({ state, context }) => const listeningView = views.find(({ state, context }) =>
State.from(state, context).matches('displaying.running.audio.listening'), State.from(state, context).matches('running.audio.listening'),
) )
if (!listeningView) { if (!listeningView) {
return return

11
src/node/data.js
View File

@@ -71,6 +71,17 @@ export async function* combineDataSources(dataSources) {
} }
} }
export function filterStreams(streams) {
return streams.filter(({ link }) => {
const url = new URL(link)
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
console.warn(`filtering non-http stream URL '${urlStr}'`)
return false
}
return true
})
}
export class StreamIDGenerator { export class StreamIDGenerator {
constructor() { constructor() {
this.idMap = new Map() this.idMap = new Map()

2
src/node/index.js
View File

@@ -403,7 +403,7 @@ async function main() {
if (require.main === module) { if (require.main === module) {
app.commandLine.appendSwitch('high-dpi-support', 1) app.commandLine.appendSwitch('high-dpi-support', 1)
app.commandLine.appendSwitch('force-device-scale-factor', 1) app.commandLine.appendSwitch('force-device-scale-factor', 1)
app.enableSandbox()
app app
.whenReady() .whenReady()
.then(main) .then(main)

305
src/node/viewStateMachine.js
View File

@@ -1,143 +1,48 @@
import isEqual from 'lodash/isEqual'
import { Machine, assign } from 'xstate' import { Machine, assign } from 'xstate'
import { ensureValidURL } from '../util'
const VIDEO_OVERRIDE_STYLE = `
* {
pointer-events: none;
display: none !important;
position: static !important;
z-index: 0 !important;
}
html, body, video, audio {
display: block !important;
background: black !important;
}
html, body {
overflow: hidden !important;
background: black !important;
}
video, iframe.__video__, audio {
display: block !important;
position: fixed !important;
left: 0 !important;
right: 0 !important;
top: 0 !important;
bottom: 0 !important;
width: 100vw !important;
height: 100vh !important;
object-fit: cover !important;
transition: none !important;
z-index: 999999 !important;
}
audio {
z-index: 999998 !important;
}
.__video_parent__ {
display: block !important;
}
video.__rot180__ {
transform: rotate(180deg) !important;
}
/* For 90 degree rotations, we position the video with swapped width and height and rotate it into place.
It's helpful to offset the video so the transformation is centered in the viewport center.
We move the video top left corner to center of the page and then translate half the video dimensions up and left.
Note that the width and height are swapped in the translate because the video starts with the side dimensions swapped. */
video.__rot90__ {
transform: translate(-50vh, -50vw) rotate(90deg) !important;
}
video.__rot270__ {
transform: translate(-50vh, -50vw) rotate(270deg) !important;
}
video.__rot90__, video.__rot270__ {
left: 50vw !important;
top: 50vh !important;
width: 100vh !important;
height: 100vw !important;
}
`
const viewStateMachine = Machine( const viewStateMachine = Machine(
{ {
id: 'view', id: 'view',
initial: 'empty', initial: 'loading',
context: { context: {
view: null, id: null,
pos: null, pos: null,
content: null, content: null,
info: {}, info: {},
sendToWall: null,
sendToFrame: null,
}, },
on: {
DISPLAY: 'displaying',
},
states: {
empty: {},
displaying: {
id: 'displaying',
initial: 'loading',
entry: assign({
pos: (context, event) => event.pos,
content: (context, event) => event.content,
}),
on: { on: {
DISPLAY: { DISPLAY: {
actions: assign({ actions: assign({
pos: (context, event) => event.pos, pos: (context, event) => event.pos,
content: (context, event) => event.content,
}), }),
cond: 'contentUnchanged',
}, },
RELOAD: '.loading', RELOAD: {
DEVTOOLS: { actions: 'reload',
actions: 'openDevTools',
}, },
VIEW_INIT: {
target: '.loading',
actions: assign({
sendToFrame: (context, event) => event.sendToFrame,
}),
},
VIEW_ERROR: '.error',
}, },
states: { states: {
loading: { loading: {
initial: 'page', on: {
entry: 'offscreenView', VIEW_LOADED: {
states: {
page: {
invoke: {
src: 'loadPage',
onDone: {
target: 'video',
},
onError: {
target: '#view.displaying.error',
},
},
},
video: {
invoke: {
src: 'startVideo',
onDone: {
target: '#view.displaying.running',
actions: assign({ actions: assign({
info: (context, event) => event.data, info: (context, event) => event.info,
}), }),
}, target: '#view.running',
onError: {
target: '#view.displaying.error',
},
},
}, },
}, },
}, },
running: { running: {
type: 'parallel', type: 'parallel',
entry: 'positionView',
on: {
DISPLAY: {
actions: [
assign({
pos: (context, event) => event.pos,
}),
'positionView',
],
cond: 'contentUnchanged',
},
},
states: { states: {
audio: { audio: {
initial: 'muted', initial: 'muted',
@@ -181,180 +86,20 @@ const viewStateMachine = Machine(
}, },
}, },
}, },
},
},
{ {
actions: { actions: {
logError: (context, event) => { logError: (context, event) => {
console.warn(event) console.warn(event.err)
},
reload: (context) => {
// TODO: keep muted over reload
context.sendToWall('reload-view', { viewId: context.id })
}, },
muteAudio: (context, event) => { muteAudio: (context, event) => {
context.view.webContents.audioMuted = true context.sendToFrame('mute')
}, },
unmuteAudio: (context, event) => { unmuteAudio: (context, event) => {
context.view.webContents.audioMuted = false context.sendToFrame('unmute')
},
openDevTools: (context, event) => {
const { view } = context
const { inWebContents } = event
view.webContents.setDevToolsWebContents(inWebContents)
view.webContents.openDevTools({ mode: 'detach' })
},
},
guards: {
contentUnchanged: (context, event) => {
return isEqual(context.content, event.content)
},
},
services: {
loadPage: async (context, event) => {
const { content, view } = context
ensureValidURL(content.url)
const wc = view.webContents
wc.audioMuted = true
await wc.loadURL(content.url)
if (content.kind === 'video' || content.kind === 'audio') {
wc.insertCSS(VIDEO_OVERRIDE_STYLE, { cssOrigin: 'user' })
} else if (content.kind === 'web') {
wc.insertCSS(
`
html, body {
overflow: hidden !important;
}
`,
{ cssOrigin: 'user' },
)
}
},
startVideo: async (context, event) => {
const { content, view } = context
if (content.kind !== 'video' && content.kind !== 'audio') {
return
}
const wc = view.webContents
// TODO: generalize "video" terminology to "media"
const info = await wc.executeJavaScript(`
(function() {
const sleep = ms => new Promise((resolve) => setTimeout(resolve, ms))
const kind = ${JSON.stringify(
content.kind === 'video' ? 'video' : 'audio',
)}
async function waitForVideo() {
let tries = 0
let video
while ((!video || !video.src) && tries < 10) {
video = document.querySelector(kind)
tries++
await sleep(200)
}
if (video) {
return {video}
}
tries = 0
let iframe
while ((!video || !video.src) && tries < 10) {
for (iframe of document.querySelectorAll('iframe')) {
if (!iframe.contentDocument) {
continue
}
video = iframe.contentDocument.querySelector(kind)
if (video) {
break
}
}
tries++
await sleep(200)
}
if (video) {
return { video, iframe }
}
return {}
}
const periscopeHacks = {
isMatch() {
return location.host === 'www.pscp.tv' || location.host === 'www.periscope.tv'
},
onLoad() {
const playButton = document.querySelector('.PlayButton')
if (playButton) {
playButton.click()
}
},
afterPlay(video) {
const baseVideoEl = document.querySelector('div.BaseVideo')
if (!baseVideoEl) {
return
}
function positionPeriscopeVideo() {
// Periscope videos can be rotated using transform matrix. They need to be rotated correctly.
const tr = baseVideoEl.style.transform
if (tr.endsWith('matrix(0, 1, -1, 0, 0, 0)')) {
video.className = '__rot90__'
} else if (tr.endsWith('matrix(-1, 0, 0, -1, 0)')) {
video.className = '__rot180__'
} else if (tr.endsWith('matrix(0, -1, 1, 0, 0, 0)')) {
video.className = '__rot270__'
}
}
positionPeriscopeVideo()
const obs = new MutationObserver(ml => {
for (const m of ml) {
if (m.attributeName === 'style') {
positionPeriscopeVideo()
return
}
}
})
obs.observe(baseVideoEl, {attributes: true})
},
}
async function findVideo() {
if (periscopeHacks.isMatch()) {
periscopeHacks.onLoad()
}
const { video, iframe } = await waitForVideo()
if (!video) {
throw new Error('could not find video')
}
if (iframe) {
const style = iframe.contentDocument.createElement('style')
style.innerHTML = \`${VIDEO_OVERRIDE_STYLE}\`
iframe.contentDocument.head.appendChild(style)
iframe.className = '__video__'
let parentEl = iframe.parentElement
while (parentEl) {
parentEl.className = '__video_parent__'
parentEl = parentEl.parentElement
}
iframe.contentDocument.body.appendChild(video)
} else {
document.body.appendChild(video)
}
video.muted = false
video.autoPlay = true
video.play()
// Prevent sites from re-muting the video (Periscope, I'm looking at you!)
Object.defineProperty(video, 'muted', {writable: true, value: false})
if (periscopeHacks.isMatch()) {
periscopeHacks.afterPlay(video)
}
const info = { title: document.title }
return info
}
findVideo()
}())
`)
return info
}, },
}, },
}, },

31
src/web/control.js
View File

@@ -24,7 +24,6 @@ import SoundIcon from '../static/volume-up-solid.svg'
import NoVideoIcon from '../static/video-slash-solid.svg' import NoVideoIcon from '../static/video-slash-solid.svg'
import ReloadIcon from '../static/redo-alt-solid.svg' import ReloadIcon from '../static/redo-alt-solid.svg'
import SwapIcon from '../static/exchange-alt-solid.svg' import SwapIcon from '../static/exchange-alt-solid.svg'
import LifeRingIcon from '../static/life-ring-regular.svg'
import WindowIcon from '../static/window-maximize-regular.svg' import WindowIcon from '../static/window-maximize-regular.svg'
import { idColor } from './colors' import { idColor } from './colors'
@@ -137,13 +136,11 @@ function useStreamwallConnection(wsEndpoint) {
for (const viewState of views) { for (const viewState of views) {
const { pos } = viewState.context const { pos } = viewState.context
const state = State.from(viewState.state) const state = State.from(viewState.state)
const isListening = state.matches( const isListening = state.matches('running.audio.listening')
'displaying.running.audio.listening',
)
const isBackgroundListening = state.matches( const isBackgroundListening = state.matches(
'displaying.running.audio.background', 'running.audio.background',
) )
const isBlurred = state.matches('displaying.running.video.blurred') const isBlurred = state.matches('running.video.blurred')
for (const space of pos.spaces) { for (const space of pos.spaces) {
if (!newStateIdxMap.has(space)) { if (!newStateIdxMap.has(space)) {
newStateIdxMap.set(space, {}) newStateIdxMap.set(space, {})
@@ -374,13 +371,6 @@ function App({ wsEndpoint, role }) {
[streams], [streams],
) )
const handleDevTools = useCallback((idx) => {
send({
type: 'dev-tools',
viewIdx: idx,
})
}, [])
const handleClickId = useCallback( const handleClickId = useCallback(
(streamId) => { (streamId) => {
try { try {
@@ -543,8 +533,8 @@ function App({ wsEndpoint, role }) {
<GridInput <GridInput
idx={idx} idx={idx}
spaceValue={streamId} spaceValue={streamId}
isError={state && state.matches('displaying.error')} isError={state && state.matches('error')}
isDisplaying={state && state.matches('displaying')} isDisplaying={!!state}
isListening={isListening} isListening={isListening}
isBackgroundListening={isBackgroundListening} isBackgroundListening={isBackgroundListening}
isBlurred={isBlurred} isBlurred={isBlurred}
@@ -563,7 +553,6 @@ function App({ wsEndpoint, role }) {
onReloadView={handleReloadView} onReloadView={handleReloadView}
onSwapView={handleSwapView} onSwapView={handleSwapView}
onBrowse={handleBrowse} onBrowse={handleBrowse}
onDevTools={handleDevTools}
/> />
) )
})} })}
@@ -789,7 +778,6 @@ function GridInput({
onReloadView, onReloadView,
onSwapView, onSwapView,
onBrowse, onBrowse,
onDevTools,
}) { }) {
const [editingValue, setEditingValue] = useState() const [editingValue, setEditingValue] = useState()
const handleFocus = useCallback( const handleFocus = useCallback(
@@ -841,10 +829,6 @@ function GridInput({
spaceValue, spaceValue,
onBrowse, onBrowse,
]) ])
const handleDevToolsClick = useCallback(() => onDevTools(idx), [
idx,
onDevTools,
])
const handleMouseDown = useCallback( const handleMouseDown = useCallback(
(ev) => { (ev) => {
onMouseDown(idx, ev) onMouseDown(idx, ev)
@@ -863,11 +847,6 @@ function GridInput({
<WindowIcon /> <WindowIcon />
</StyledSmallButton> </StyledSmallButton>
)} )}
{roleCan(role, 'dev-tools') && (
<StyledSmallButton onClick={handleDevToolsClick} tabIndex={1}>
<LifeRingIcon />
</StyledSmallButton>
)}
</> </>
) : ( ) : (
<> <>

8
webpack.config.js
View File

@@ -78,8 +78,12 @@ const browserConfig = {
devtool: 'cheap-source-map', devtool: 'cheap-source-map',
target: 'electron-renderer', target: 'electron-renderer',
entry: { entry: {
background: './src/browser/background.js', wall: './src/browser/wall.js',
overlay: './src/browser/overlay.js', wallPreload: './src/browser/wallPreload.js',
mediaPreload: './src/browser/mediaPreload.js',
},
externals: {
events: 'commonjs events',
}, },
plugins: [ plugins: [
new CopyPlugin({ new CopyPlugin({