Overhaul view positioning to enable animated transitions

Views are now positioned inside the web contents so we can use CSS
transitions to animate them. Since the overlay renders in a different
process, I needed to move the border and listening glow into the web
contents so that they'd move in sync.

packages/streamwall-shared/src/types.ts

@@ -15,6 +15,7 @@ export interface StreamWindowConfig {
 
 export interface ContentDisplayOptions {
   rotation?: number
+  glowColor?: string
 }
 
 /** Metadata scraped from a loaded view */

packages/streamwall/src/main/StreamWindow.ts

@@ -19,14 +19,6 @@ import { createActor, EventFrom, SnapshotFrom } from 'xstate'
 import { loadHTML } from './loadHTML'
 import viewStateMachine, { ViewActor } from './viewStateMachine'
 
-function getDisplayOptions(stream: StreamData): ContentDisplayOptions {
-  if (!stream) {
-    return {}
-  }
-  const { rotation } = stream
-  return { rotation }
-}
-
 export interface StreamWindowEventMap {
   load: []
   close: []
@@ -116,9 +108,10 @@ export default class StreamWindow extends EventEmitter<StreamWindowEventMap> {
       const view = this.views.get(ev.sender.id)
       if (view) {
         view.send({ type: 'VIEW_INIT' })
-        const { content, options } = view.getSnapshot().context
+        const { content, pos, options } = view.getSnapshot().context
         return {
           content,
+          pos,
           options,
         }
       }
@@ -146,7 +139,6 @@ export default class StreamWindow extends EventEmitter<StreamWindowEventMap> {
       config: { width, height },
     } = this
     assert(win != null, 'Window must be initialized')
-    const { backgroundColor } = this.config
     const view = new WebContentsView({
       webPreferences: {
         preload: path.join(__dirname, 'mediaPreload.js'),
@@ -156,7 +148,7 @@ export default class StreamWindow extends EventEmitter<StreamWindowEventMap> {
         partition: 'persist:session',
       },
     })
-    view.setBackgroundColor(backgroundColor)
+    view.setBackgroundColor('#00000000') // Transparent
 
     const viewId = view.webContents.id
 
@@ -215,6 +207,22 @@ export default class StreamWindow extends EventEmitter<StreamWindowEventMap> {
     this.emit('state', states)
   }
 
+  getDisplayOptions(
+    view: ViewActor,
+    stream: StreamData,
+  ): ContentDisplayOptions {
+    const { config } = this
+    const { rotation } = stream
+    return {
+      rotation,
+      glowColor: view
+        .getSnapshot()
+        .matches({ displaying: { running: { audio: 'listening' } } })
+        ? config.activeColor
+        : undefined,
+    }
+  }
+
   setViews(viewContentMap: ViewContentMap, streams: StreamList) {
     const { width, height, cols, rows } = this.config
     const spaceWidth = Math.floor(width / cols)
@@ -291,7 +299,10 @@ export default class StreamWindow extends EventEmitter<StreamWindowEventMap> {
       }
 
       view.send({ type: 'DISPLAY', pos, content })
-      view.send({ type: 'OPTIONS', options: getDisplayOptions(stream) })
+      view.send({
+        type: 'OPTIONS',
+        options: this.getDisplayOptions(view, stream),
+      })
       newViews.set(view.getSnapshot().context.id, view)
     }
     for (const view of unusedViews) {
@@ -370,7 +381,7 @@ export default class StreamWindow extends EventEmitter<StreamWindowEventMap> {
       if (stream) {
         view.send({
           type: 'OPTIONS',
-          options: getDisplayOptions(stream),
+          options: this.getDisplayOptions(view, stream),
         })
       }
     }

packages/streamwall/src/main/viewStateMachine.ts

@@ -31,7 +31,7 @@ const viewStateMachine = setup({
       view: WebContentsView
       pos: ViewPos | null
       content: ViewContent | null
-      options: ContentDisplayOptions | null
+      options: ContentDisplayOptions
       info: ContentViewInfo | null
     },
 
@@ -102,16 +102,19 @@ const viewStateMachine = setup({
 
       offscreenWin.contentView.removeChildView(view)
 
+      // Insert below the overlay (before the last view).
+      // This is a little tricky because if the view is already in the child array, re-inserting it will momentarily remove it from the array, so the length of the array is 1 shorter. We need to offset the insertion index to account for this.
+      // Note: We intentionally raise recently positioned views above the others, so they layer on top during transitions.
       const existingIdx = win.contentView.children.indexOf(view)
+      const insertIdxOffset = existingIdx !== -1 ? -2 : -1
       win.contentView.addChildView(
         view,
-        existingIdx !== -1
+        win.contentView.children.length + insertIdxOffset,
-          ? existingIdx
-          : // Insert below the overlay (end of the current list because once added, the overlay's index will increase by 1)
-            win.contentView.children.length - 1,
       )
 
-      view.setBounds(pos)
+      const { width, height } = win.getBounds()
+      view.setBounds({ x: 0, y: 0, width: width, height })
+      view.webContents.send('position', pos)
     },
   },
 
@@ -180,7 +183,7 @@ const viewStateMachine = setup({
     offscreenWin,
     pos: null,
     content: null,
-    options: null,
+    options: {},
     info: null,
   }),
   on: {

packages/streamwall/src/preload/mediaPreload.ts

@@ -1,6 +1,6 @@
 import { ipcRenderer, webFrame } from 'electron'
 import throttle from 'lodash/throttle'
-import { ContentDisplayOptions } from 'streamwall-shared'
+import { ContentDisplayOptions, ViewPos } from 'streamwall-shared'
 
 const SCAN_THROTTLE = 500
 const INITIAL_TIMEOUT = 10 * 1000
@@ -9,61 +9,46 @@ const VIDEO_OVERRIDE_STYLE = `
   * {
     pointer-events: none;
     display: none !important;
-    position: static !important;
     z-index: 0 !important;
   }
-  html, body, video, audio {
+  html, body, video, audio, body:after {
     display: block !important;
-    background: black !important;
+    background: transparent !important;
+    min-height: 0 !important;
+    min-width: 0 !important;
   }
   html, body {
     overflow: hidden !important;
-    background: black !important;
+    background: transparent !important;
   }
-  video, iframe.__video__, audio {
+  video, iframe.__video__, audio, body:after {
     display: block !important;
-    position: fixed !important;
+    position: absolute !important;
-    left: 0 !important;
-    right: 0 !important;
     top: 0 !important;
     bottom: 0 !important;
-    width: 100vw !important;
+    left: 0 !important;
-    height: 100vh !important;
+    right: 0 !important;
+    width: 100% !important;
+    height: 100% !important;
     object-fit: cover !important;
-    transition: none !important;
     z-index: 999999 !important;
   }
   audio {
     z-index: 999998 !important;
   }
+  /* deprecate? */
   .__video_parent__ {
     display: block !important;
   }
-  video.__rot180__ {
-    transform: rotate(180deg) !important;
-  }
-  /* For 90 degree rotations, we position the video with swapped width and height and rotate it into place.
-     It's helpful to offset the video so the transformation is centered in the viewport center.
-     We move the video top left corner to center of the page and then translate half the video dimensions up and left.
-     Note that the width and height are swapped in the translate because the video starts with the side dimensions swapped. */
-  video.__rot90__ {
-    transform: translate(-50vh, -50vw) rotate(90deg) !important;
-  }
-  video.__rot270__ {
-    transform: translate(-50vh, -50vw) rotate(270deg) !important;
-  }
-  video.__rot90__, video.__rot270__ {
-    left: 50vw !important;
-    top: 50vh !important;
-    width: 100vh !important;
-    height: 100vw !important;
-  }
 `
 
-const NO_SCROLL_STYLE = `
+const WEB_OVERRIDE_STYLE = `
   html, body {
     overflow: hidden !important;
   }
+  body {
+    background: white;
+  }
 `
 
 const sleep = (ms: number) =>
@@ -73,27 +58,98 @@ const pageReady = new Promise((resolve) =>
   document.addEventListener('DOMContentLoaded', resolve, { once: true }),
 )
 
-class RotationController {
+class BodyStyleController {
-  video: HTMLVideoElement
+  cssKey: string | undefined
-  siteRotation: number
+  pos: ViewPos
-  customRotation: number
+  options: ContentDisplayOptions
 
-  constructor(video: HTMLVideoElement) {
+  constructor(pos: ViewPos, options: ContentDisplayOptions) {
-    this.video = video
+    this.pos = pos
-    this.customRotation = 0
+    this.options = options
   }
 
-  _update() {
+  updatePosition(pos: ViewPos) {
-    const rotation = this.customRotation % 360
+    this.pos = pos
-    if (![0, 90, 180, 270].includes(rotation)) {
+    this.update()
-      console.warn('ignoring invalid rotation', rotation)
+  }
+
+  updateOptions(options: ContentDisplayOptions) {
+    this.options = options
+    this.update()
+  }
+
+  update() {
+    const { pos, options } = this
+    const { x, y, width, height } = pos
+    const { rotation, glowColor } = options
+    const borderWidth = 2
+    const windowWidth = window.innerWidth
+    const windowHeight = window.innerHeight
+
+    const styleParts = []
+    styleParts.push(`
+      body {
+        position: fixed !important;
+        contain: strict;
+        left: ${x}px !important;
+        top: ${y}px !important;
+        width: ${width}px !important;
+        height: ${height}px !important;
+        min-width: 0 !important;
+        min-height: 0 !important;
+        max-width: none !important;
+        max-height: none !important;
+        border: 0 solid black !important;
+        border-left-width: ${x === 0 ? 0 : borderWidth}px !important;
+        border-right-width: ${x + width === windowWidth ? 0 : borderWidth}px !important;
+        border-top-width: ${y === 0 ? 0 : borderWidth}px !important;
+        border-bottom-width: ${y + height === windowHeight ? 0 : borderWidth}px !important;
+        box-sizing: border-box !important;
+        transition: top 250ms ease, left 250ms ease, width 250ms ease, height 250ms ease, transform 250ms ease !important;
+        transform: rotate(0deg);
+      }
+    `)
+
+    if (rotation === 180) {
+      styleParts.push(`
+        body {
+          transform: rotate(180deg) !important;
+        }
+      `)
     }
-    this.video.className = `__rot${rotation}__`
-  }
 
-  setCustom(rotation = 0) {
+    if (rotation === 90 || rotation === 270) {
-    this.customRotation = rotation
+      // For 90 degree rotations, we position with swapped width and height and rotate it into place.
-    this._update()
+      // It's helpful to offset the position so the centered transform origin is in the center of the intended destination.
+      // Then we use translate to center on the position and rotate around the center.
+      // Note that the width and height are swapped in the translate because the video starts with the side dimensions swapped.
+      const halfWidth = width / 2
+      const halfHeight = height / 2
+      styleParts.push(`
+        body {
+          left: ${x + halfWidth}px !important;
+          top: ${y + halfHeight}px !important;
+          width: ${height}px !important;
+          height: ${width}px !important;
+          transform: translate(-${halfHeight}px, -${halfWidth}px) rotate(${rotation}deg) !important;
+        }
+      `)
+    }
+
+    if (glowColor) {
+      styleParts.push(`
+        body:after {
+          content: '';
+          box-shadow: 0 0 10px ${glowColor} inset !important;
+        }
+      `)
+    }
+
+    if (this.cssKey !== undefined) {
+      webFrame.removeInsertedCSS(this.cssKey)
+    }
+    // Note: we can't use 'user' origin here because it can't be removed (https://github.com/electron/electron/issues/27792)
+    this.cssKey = webFrame.insertCSS(styleParts.join('\n'))
   }
 }
 
@@ -141,7 +197,7 @@ class SnapshotController {
   }
 }
 
-// Watch for media tags and mute them as soon as possible.
+/** Watch for media tags and mute them as soon as possible. */
 async function lockdownMediaTags() {
   const lockdown = throttle(() => {
     webFrame.executeJavaScript(`
@@ -242,7 +298,6 @@ async function findMedia(
     throw new Error('could not find video')
   }
   if (iframe && iframe.contentDocument) {
-    // TODO: verify iframe still works
     const style = iframe.contentDocument.createElement('style')
     style.innerHTML = VIDEO_OVERRIDE_STYLE
     iframe.contentDocument.head.appendChild(style)
@@ -280,14 +335,14 @@ async function main() {
   const viewInit = ipcRenderer.invoke('view-init')
   const pageReady = new Promise((resolve) => process.once('loaded', resolve))
 
-  const [{ content, options: initialOptions }] = await Promise.all([
+  const [{ content, pos: initialPos, options: initialOptions }] =
-    viewInit,
+    await Promise.all([viewInit, pageReady])
-    pageReady,
+
-  ])
+  const styleController = new BodyStyleController(initialPos, initialOptions)
+  styleController.update()
 
   const snapshotController = new SnapshotController()
 
-  let rotationController: RotationController | undefined
   async function acquireMedia(elementTimeout: number) {
     let snapshotInterval: number | undefined
 
@@ -297,7 +352,6 @@ async function main() {
     ipcRenderer.send('view-loaded')
 
     if (content.kind === 'video' && media instanceof HTMLVideoElement) {
-      rotationController = new RotationController(media)
       snapshotInterval = window.setInterval(() => {
         snapshotController.snapshotVideo(media)
       }, 1000)
@@ -330,17 +384,14 @@ async function main() {
       },
     })
   } else if (content.kind === 'web') {
-    webFrame.insertCSS(NO_SCROLL_STYLE, { cssOrigin: 'user' })
+    webFrame.insertCSS(WEB_OVERRIDE_STYLE, { cssOrigin: 'user' })
     ipcRenderer.send('view-loaded')
   }
 
-  function updateOptions(options: ContentDisplayOptions) {
+  ipcRenderer.on('position', (_ev, pos) => styleController.updatePosition(pos))
-    if (rotationController) {
+  ipcRenderer.on('options', (_ev, options) =>
-      rotationController.setCustom(options.rotation)
+    styleController.updateOptions(options),
-    }
+  )
-  }
-  ipcRenderer.on('options', (ev, options) => updateOptions(options))
-  updateOptions(initialOptions)
 }
 
 main().catch((error) => {

packages/streamwall/src/renderer/overlay.tsx

@@ -12,7 +12,7 @@ import {
   FaYoutube,
 } from 'react-icons/fa'
 import { RiKickFill, RiTwitterXFill } from 'react-icons/ri'
-import { StreamwallState } from 'streamwall-shared'
+import { StreamwallState, ViewState } from 'streamwall-shared'
 import { styled } from 'styled-components'
 import { TailSpin } from 'svg-loaders-react'
 import { matchesState } from 'xstate'
@@ -33,12 +33,13 @@ function Overlay({
   views,
   streams,
 }: Pick<StreamwallState, 'config' | 'views' | 'streams'>) {
-  const { width, height, activeColor } = config
+  const { activeColor } = config
   const activeViews = views.filter(
     ({ state }) =>
       matchesState('displaying', state) &&
       !matchesState('displaying.error', state),
   )
+  activeViews.sort((a: ViewState, b: ViewState) => a.context.id - b.context.id)
   const overlays = streams.filter((s) => s.kind === 'overlay')
   return (
     <OverlayContainer>
@@ -62,20 +63,20 @@ function Overlay({
           'displaying.running.video.blurred',
           state,
         )
-        const isLoading =
+        const isLoading = matchesState('displaying.loading', state)
-          matchesState('displaying.loading', state) ||
+        const isStalled = matchesState(
-          matchesState('displaying.running.playback.stalled', state)
+          'displaying.running.playback.stalled',
+          state,
+        )
         const hasTitle = data && (data.label || data.source)
         const position = data?.labelPosition ?? 'top-left'
         return (
           <SpaceBorder
+            key={`view-${context.id}`}
             pos={pos}
-            windowWidth={width}
+            isLoading={isLoading}
-            windowHeight={height}
-            activeColor={activeColor}
-            isListening={isListening}
           >
-            <FilterCover isBlurred={isBlurred} isDesaturated={isLoading} />
+            <FilterCover isBlurred={isBlurred} isDesaturated={isStalled} />
             {hasTitle && (
               <StreamTitle
                 position={position}
@@ -95,7 +96,7 @@ function Overlay({
                 </span>
               </StreamLocation>
             )}
-            <LoadingSpinner isVisible={isLoading} />
+            <LoadingSpinner isVisible={isLoading || isStalled} />
           </SpaceBorder>
         )
       })}
@@ -197,20 +198,17 @@ const SpaceBorder = styled.div.attrs(() => ({
   top: ${({ pos }) => pos.y}px;
   width: ${({ pos }) => pos.width}px;
   height: ${({ pos }) => pos.height}px;
-  border: 0 solid black;
-  border-left-width: ${({ pos, borderWidth }) =>
-    pos.x === 0 ? 0 : borderWidth}px;
-  border-right-width: ${({ pos, borderWidth, windowWidth }) =>
-    pos.x + pos.width === windowWidth ? 0 : borderWidth}px;
-  border-top-width: ${({ pos, borderWidth }) =>
-    pos.y === 0 ? 0 : borderWidth}px;
-  border-bottom-width: ${({ pos, borderWidth, windowHeight }) =>
-    pos.y + pos.height === windowHeight ? 0 : borderWidth}px;
-  box-shadow: ${({ isListening, activeColor }) =>
-    isListening ? `0 0 10px ${activeColor} inset` : 'none'};
   box-sizing: border-box;
   pointer-events: none;
   user-select: none;
+  background-color: ${({ isLoading }) =>
+    isLoading ? 'rgba(0, 0, 0, .8)' : ''};
+  transition:
+    top 250ms ease,
+    left 250ms ease,
+    width 250ms ease,
+    height 250ms ease,
+    background-color 250ms ease;
 `
 
 const StreamTitle = styled.div`