Add support for displaying custom web content in views

src/browser/overlay.js

@@ -27,15 +27,17 @@ function Overlay({ views, streams, customStreams }) {
   return (
     <div>
       {activeViews.map((viewState) => {
-        const { url, pos } = viewState.context
+        const { content, pos } = viewState.context
-        const data = [...streams, ...customStreams].find((d) => url === d.Link)
+        const data = [...streams, ...customStreams].find(
+          (d) => content.url === d.Link,
+        )
         const isListening = viewState.matches('displaying.running.listening')
         const isLoading = viewState.matches('displaying.loading')
         return (
           <SpaceBorder pos={pos} isListening={isListening}>
             {data && (
               <StreamTitle isListening={isListening}>
-                <StreamIcon url={url} />
+                <StreamIcon url={content.url} />
                 <span>
                   {data.hasOwnProperty('Label') ? (
                     data.Label

src/node/StreamWindow.js

@@ -1,10 +1,11 @@
+import isEqual from 'lodash/isEqual'
 import intersection from 'lodash/intersection'
 import EventEmitter from 'events'
 import { BrowserView, BrowserWindow, ipcMain } from 'electron'
 import { interpret } from 'xstate'
 
 import viewStateMachine from './viewStateMachine'
-import { boxesFromViewURLMap } from './geometry'
+import { boxesFromViewContentMap } from './geometry'
 
 import {
   WIDTH,
@@ -118,7 +119,7 @@ export default class StreamWindow extends EventEmitter {
       this.views.map(({ state }) => ({
         state: state.value,
         context: {
-          url: state.context.url,
+          content: state.context.content,
           info: state.context.info,
           pos: state.context.pos,
         },
@@ -126,34 +127,38 @@ export default class StreamWindow extends EventEmitter {
     )
   }
 
-  setViews(viewURLMap) {
+  setViews(viewContentMap) {
     const { win, views } = this
-    const boxes = boxesFromViewURLMap(GRID_COUNT, GRID_COUNT, viewURLMap)
+    const boxes = boxesFromViewContentMap(
-    const remainingBoxes = new Set(boxes.filter(({ url }) => url))
+      GRID_COUNT,
-
+      GRID_COUNT,
+      viewContentMap,
+    )
+    const remainingBoxes = new Set(boxes)
     const unusedViews = new Set(views)
     const viewsToDisplay = []
 
     // We try to find the best match for moving / reusing existing views to match the new positions.
     const matchers = [
       // First try to find a loaded view of the same URL in the same space...
-      (v, url, spaces) =>
+      (v, content, spaces) =>
-        v.state.context.url === url &&
+        isEqual(v.state.context.content, content) &&
         v.state.matches('displaying.running') &&
         intersection(v.state.context.pos.spaces, spaces).length > 0,
       // Then try to find a loaded view of the same URL...
-      (v, url) =>
+      (v, content) =>
-        v.state.context.url === url && v.state.matches('displaying.running'),
+        isEqual(v.state.context.content, content) &&
+        v.state.matches('displaying.running'),
       // Then try view with the same URL that is still loading...
-      (v, url) => v.state.context.url === url,
+      (v, content) => isEqual(v.state.context.content, content),
     ]
 
     for (const matcher of matchers) {
       for (const box of remainingBoxes) {
-        const { url, spaces } = box
+        const { content, spaces } = box
         let foundView
         for (const view of unusedViews) {
-          if (matcher(view, url, spaces)) {
+          if (matcher(view, content, spaces)) {
             foundView = view
             break
           }
@@ -173,7 +178,7 @@ export default class StreamWindow extends EventEmitter {
 
     const newViews = []
     for (const { box, view } of viewsToDisplay) {
-      const { url, x, y, w, h, spaces } = box
+      const { content, x, y, w, h, spaces } = box
       const pos = {
         x: SPACE_WIDTH * x,
         y: SPACE_HEIGHT * y,
@@ -181,7 +186,7 @@ export default class StreamWindow extends EventEmitter {
         height: SPACE_HEIGHT * h,
         spaces,
       }
-      view.send({ type: 'DISPLAY', pos, url })
+      view.send({ type: 'DISPLAY', pos, content })
       newViews.push(view)
     }
     for (const view of unusedViews) {

src/node/geometry.js

@@ -1,16 +1,24 @@
-export function boxesFromViewURLMap(width, height, stateURLMap) {
+import isEqual from 'lodash/isEqual'
+
+export function boxesFromViewContentMap(width, height, viewContentMap) {
   const boxes = []
   const visited = new Set()
 
+  function isPosContent(x, y, content) {
+    const checkIdx = width * y + x
+    return (
+      !visited.has(checkIdx) && isEqual(viewContentMap.get(checkIdx), content)
+    )
+  }
+
   function findLargestBox(x, y) {
     const idx = width * y + x
     const spaces = [idx]
-    const url = stateURLMap.get(idx)
+    const content = viewContentMap.get(idx)
 
     let maxY
     for (maxY = y + 1; maxY < height; maxY++) {
-      const checkIdx = width * maxY + x
+      if (!isPosContent(x, maxY, content)) {
-      if (visited.has(checkIdx) || stateURLMap.get(checkIdx) !== url) {
         break
       }
       spaces.push(width * maxY + x)
@@ -20,8 +28,7 @@ export function boxesFromViewURLMap(width, height, stateURLMap) {
     let cy = y
     scan: for (cx = x + 1; cx < width; cx++) {
       for (cy = y; cy < maxY; cy++) {
-        const checkIdx = width * cy + cx
+        if (!isPosContent(cx, cy, content)) {
-        if (visited.has(checkIdx) || stateURLMap.get(checkIdx) !== url) {
           break scan
         }
       }
@@ -32,13 +39,13 @@ export function boxesFromViewURLMap(width, height, stateURLMap) {
     const w = cx - x
     const h = maxY - y
     spaces.sort()
-    return { url, x, y, w, h, spaces }
+    return { content, x, y, w, h, spaces }
   }
 
   for (let y = 0; y < width; y++) {
     for (let x = 0; x < height; x++) {
       const idx = width * y + x
-      if (visited.has(idx) || stateURLMap.get(idx) === undefined) {
+      if (visited.has(idx) || viewContentMap.get(idx) === undefined) {
         continue
       }
 

src/node/geometry.test.js

@@ -1,10 +1,10 @@
-import { boxesFromViewURLMap } from './geometry'
+import { boxesFromViewContentMap } from './geometry'
 
 function example([text]) {
   return text
     .replace(/\s/g, '')
     .split('')
-    .map((c) => (c === '.' ? undefined : c))
+    .map((c) => (c === '.' ? undefined : { url: c }))
 }
 
 const box1 = example`
@@ -41,8 +41,8 @@ describe.each([
     2,
     box1,
     [
-      { url: 'a', x: 0, y: 0, w: 1, h: 2, spaces: [0, 2] },
+      { content: { url: 'a' }, x: 0, y: 0, w: 1, h: 2, spaces: [0, 2] },
-      { url: 'b', x: 1, y: 0, w: 1, h: 2, spaces: [1, 3] },
+      { content: { url: 'b' }, x: 1, y: 0, w: 1, h: 2, spaces: [1, 3] },
     ],
   ],
   [
@@ -50,8 +50,8 @@ describe.each([
     2,
     box2,
     [
-      { url: 'a', x: 0, y: 0, w: 2, h: 1, spaces: [0, 1] },
+      { content: { url: 'a' }, x: 0, y: 0, w: 2, h: 1, spaces: [0, 1] },
-      { url: 'b', x: 0, y: 1, w: 2, h: 1, spaces: [2, 3] },
+      { content: { url: 'b' }, x: 0, y: 1, w: 2, h: 1, spaces: [2, 3] },
     ],
   ],
   [
@@ -59,28 +59,33 @@ describe.each([
     3,
     box3,
     [
-      { url: 'a', x: 0, y: 0, w: 2, h: 2, spaces: [0, 1, 3, 4] },
+      { content: { url: 'a' }, x: 0, y: 0, w: 2, h: 2, spaces: [0, 1, 3, 4] },
-      { url: 'c', x: 2, y: 0, w: 1, h: 1, spaces: [2] },
+      { content: { url: 'c' }, x: 2, y: 0, w: 1, h: 1, spaces: [2] },
-      { url: 'a', x: 2, y: 1, w: 1, h: 1, spaces: [5] },
+      { content: { url: 'a' }, x: 2, y: 1, w: 1, h: 1, spaces: [5] },
-      { url: 'd', x: 0, y: 2, w: 1, h: 1, spaces: [6] },
+      { content: { url: 'd' }, x: 0, y: 2, w: 1, h: 1, spaces: [6] },
-      { url: 'a', x: 1, y: 2, w: 1, h: 1, spaces: [7] },
+      { content: { url: 'a' }, x: 1, y: 2, w: 1, h: 1, spaces: [7] },
-      { url: 'e', x: 2, y: 2, w: 1, h: 1, spaces: [8] },
+      { content: { url: 'e' }, x: 2, y: 2, w: 1, h: 1, spaces: [8] },
     ],
   ],
-  [3, 3, box4, [{ url: 'a', x: 1, y: 1, w: 2, h: 2, spaces: [4, 5, 7, 8] }]],
+  [
+    3,
+    3,
+    box4,
+    [{ content: { url: 'a' }, x: 1, y: 1, w: 2, h: 2, spaces: [4, 5, 7, 8] }],
+  ],
   [
     3,
     3,
     box5,
     [
-      { url: 'a', x: 2, y: 0, w: 1, h: 3, spaces: [2, 5, 8] },
+      { content: { url: 'a' }, x: 2, y: 0, w: 1, h: 3, spaces: [2, 5, 8] },
-      { url: 'a', x: 1, y: 2, w: 1, h: 1, spaces: [7] },
+      { content: { url: 'a' }, x: 1, y: 2, w: 1, h: 1, spaces: [7] },
     ],
   ],
-])('boxesFromViewURLMap(%i, %i, %j)', (width, height, data, expected) => {
+])('boxesFromViewContentMap(%i, %i, %j)', (width, height, data, expected) => {
   test(`returns expected ${expected.length} boxes`, () => {
     const stateURLMap = new Map(data.map((v, idx) => [idx, v]))
-    const result = boxesFromViewURLMap(width, height, stateURLMap)
+    const result = boxesFromViewContentMap(width, height, stateURLMap)
     expect(result).toStrictEqual(expected)
   })
 })

src/node/viewStateMachine.js

@@ -1,3 +1,4 @@
+import isEqual from 'lodash/isEqual'
 import { Machine, assign } from 'xstate'
 
 const viewStateMachine = Machine(
@@ -7,7 +8,7 @@ const viewStateMachine = Machine(
     context: {
       view: null,
       pos: null,
-      url: null,
+      content: null,
       info: {},
     },
     on: {
@@ -20,14 +21,14 @@ const viewStateMachine = Machine(
         initial: 'loading',
         entry: assign({
           pos: (context, event) => event.pos,
-          url: (context, event) => event.url,
+          content: (context, event) => event.content,
         }),
         on: {
           DISPLAY: {
             actions: assign({
               pos: (context, event) => event.pos,
             }),
-            cond: 'urlUnchanged',
+            cond: 'contentUnchanged',
           },
           RELOAD: '.loading',
         },
@@ -38,7 +39,7 @@ const viewStateMachine = Machine(
             states: {
               page: {
                 invoke: {
-                  src: 'loadURL',
+                  src: 'loadPage',
                   onDone: {
                     target: 'video',
                   },
@@ -74,7 +75,7 @@ const viewStateMachine = Machine(
                   }),
                   'positionView',
                 ],
-                cond: 'urlUnchanged',
+                cond: 'contentUnchanged',
               },
               MUTE: '.muted',
               UNMUTE: '.listening',
@@ -108,16 +109,17 @@ const viewStateMachine = Machine(
       },
     },
     guards: {
-      urlUnchanged: (context, event) => {
+      contentUnchanged: (context, event) => {
-        return context.url === event.url
+        return isEqual(context.content, event.content)
       },
     },
     services: {
-      loadURL: async (context, event) => {
+      loadPage: async (context, event) => {
-        const { url, view } = context
+        const { content, view } = context
         const wc = view.webContents
         wc.audioMuted = true
-        await wc.loadURL(url)
+        await wc.loadURL(content.url)
+        if (content.kind === 'video') {
           wc.insertCSS(
             `
           * {
@@ -147,9 +149,14 @@ const viewStateMachine = Machine(
         `,
             { cssOrigin: 'user' },
           )
+        }
       },
       startVideo: async (context, event) => {
-        const wc = context.view.webContents
+        const { content, view } = context
+        if (content.kind !== 'video') {
+          return
+        }
+        const wc = view.webContents
         const info = await wc.executeJavaScript(`
           const sleep = ms => new Promise((resolve) => setTimeout(resolve, ms))
           async function waitForVideo() {

src/web/control.js

@@ -11,26 +11,12 @@ import SoundIcon from '../static/volume-up-solid.svg'
 import ReloadIcon from '../static/redo-alt-solid.svg'
 import LifeRingIcon from '../static/life-ring-regular.svg'
 
-function emptyStateIdxMap() {
-  return new Map(
-    range(GRID_COUNT * GRID_COUNT).map((idx) => [
-      idx,
-      {
-        streamId: null,
-        url: null,
-        state: State.from({}),
-        isListening: false,
-      },
-    ]),
-  )
-}
-
 function App({ wsEndpoint }) {
   const wsRef = useRef()
   const [isConnected, setIsConnected] = useState(false)
   const [streams, setStreams] = useState([])
   const [customStreams, setCustomStreams] = useState([])
-  const [stateIdxMap, setStateIdxMap] = useState(emptyStateIdxMap())
+  const [stateIdxMap, setStateIdxMap] = useState(new Map())
 
   useEffect(() => {
     const ws = new ReconnectingWebSocket(wsEndpoint, [], {
@@ -48,20 +34,21 @@ function App({ wsEndpoint }) {
           views,
           customStreams: newCustomStreams,
         } = msg.state
-        const newStateIdxMap = emptyStateIdxMap()
+        const newStateIdxMap = new Map()
         const allStreams = [...newStreams, ...newCustomStreams]
         for (const viewState of views) {
-          const { pos, url } = viewState.context
+          const { pos, content } = viewState.context
-          if (!url) {
+          const stream = allStreams.find((d) => d.Link === content.url)
-            continue
+          const streamId = stream?._id
-          }
-          const streamId = allStreams.find((d) => d.Link === url)?._id
           const state = State.from(viewState.state)
           const isListening = state.matches('displaying.running.listening')
           for (const space of pos.spaces) {
+            if (!newStateIdxMap.has(space)) {
+              newStateIdxMap.set(space, {})
+            }
             Object.assign(newStateIdxMap.get(space), {
               streamId,
-              url,
+              content,
               state,
               isListening,
             })
@@ -80,24 +67,25 @@ function App({ wsEndpoint }) {
   const handleSetView = useCallback(
     (idx, streamId) => {
       const newSpaceIdxMap = new Map(stateIdxMap)
-      const url = [...streams, ...customStreams].find((d) => d._id === streamId)
+      const stream = [...streams, ...customStreams].find(
-        ?.Link
+        (d) => d._id === streamId,
-      if (url) {
+      )
+      if (stream) {
+        const content = {
+          url: stream?.Link,
+          kind: stream?.Kind || 'video',
+        }
         newSpaceIdxMap.set(idx, {
           ...newSpaceIdxMap.get(idx),
           streamId,
-          url,
+          content,
         })
       } else {
-        newSpaceIdxMap.set(idx, {
+        newSpaceIdxMap.delete(idx)
-          ...newSpaceIdxMap.get(idx),
-          streamId: null,
-          url: null,
-        })
       }
-      const views = Array.from(newSpaceIdxMap, ([space, { url }]) => [
+      const views = Array.from(newSpaceIdxMap, ([space, { content }]) => [
         space,
-        url,
+        content,
       ])
       wsRef.current.send(JSON.stringify({ type: 'set-views', views }))
     },
@@ -155,18 +143,21 @@ function App({ wsEndpoint }) {
             <StyledGridLine>
               {range(0, 3).map((x) => {
                 const idx = 3 * y + x
-                const { streamId, isListening, url, state } = stateIdxMap.get(
+                const {
-                  idx,
+                  streamId = '',
-                )
+                  isListening = false,
+                  content = { url: '' },
+                  state,
+                } = stateIdxMap.get(idx) || {}
                 return (
                   <GridInput
                     idx={idx}
-                    url={url}
+                    url={content.url}
-                    onChangeSpace={handleSetView}
                     spaceValue={streamId}
-                    isError={state.matches('displaying.error')}
+                    isError={state && state.matches('displaying.error')}
-                    isDisplaying={state.matches('displaying')}
+                    isDisplaying={state && state.matches('displaying')}
                     isListening={isListening}
+                    onChangeSpace={handleSetView}
                     onSetListening={handleSetListening}
                     onReloadView={handleReloadView}
                     onBrowse={handleBrowse}
@@ -189,13 +180,14 @@ function App({ wsEndpoint }) {
             Include an empty object at the end to create an extra input for a new custom stream.
             We need it to be part of the array (rather than JSX below) for DOM diffing to match the key and retain focus.
            */}
-          {[...customStreams, { Link: '', Label: '' }].map(
+          {[...customStreams, { Link: '', Label: '', Kind: 'video' }].map(
-            ({ Link, Label }, idx) => (
+            ({ Link, Label, Kind }, idx) => (
               <CustomStreamInput
                 key={idx}
                 idx={idx}
                 Link={Link}
                 Label={Label}
+                Kind={Kind}
                 onChange={handleChangeCustomStream}
               />
             ),
@@ -311,6 +303,12 @@ function CustomStreamInput({ idx, onChange, ...props }) {
     },
     [onChange],
   )
+  const handleChangeKind = useCallback(
+    (ev) => {
+      onChange(idx, { ...props, Kind: ev.target.value })
+    },
+    [onChange],
+  )
   return (
     <div>
       <input
@@ -323,6 +321,10 @@ function CustomStreamInput({ idx, onChange, ...props }) {
         placeholder="Label (optional)"
         value={props.Label}
       />
+      <select onChange={handleChangeKind} value={props.Kind}>
+        <option value="video">video</option>
+        <option value="web">web</option>
+      </select>
     </div>
   )
 }