Add support for TOML file data source

2025-12-06 01:45:37 -05:00 · 2020-07-01 00:03:08 -07:00
parent cc14f2a17f
commit b2215aa42d
7 changed files with 135 additions and 64 deletions
--- a/README.md
+++ b/README.md
@@ -45,6 +45,14 @@ npm start -- --config="../streamwall.json"

 See `config.example.toml` for an example.

+## Data sources
+
+Streamwall can load stream data from both JSON APIs and TOML files. Data sources can be specified in a config file (see `config.example.toml` for an example) or the command line:
+
+```
+npm start -- --json-url="https://your-site/api/streams.json" --toml-file="./streams.toml"
+```
+
 ## Hotkeys

 The following hotkeys are available with the "control" webpage focused:
--- a/example.config.toml
+++ b/example.config.toml
@@ -13,6 +13,15 @@ address = "http://localhost:80"
 password = "woke"
 username = "woke"

+[data]
+# By default, we use the woke.net streams data source.
+# You can add other URLs to fetch here:
+json-url = ["https://woke.net/api/streams.json"]
+
+# You can also specify .toml files on disk. Streamwall will reload the data whenever the file changes.
+# See example.streams.toml for a sample.
+#toml-file = ["./example.streams.toml"]
+
 [cert]
 # SSL certificates (optional)
 # If you specify an https:// URL for the "webserver" option, a certificate will be automatically generated and signed by Let's Encrypt.
--- a/package-lock.json
+++ b/package-lock.json
@@ -1867,6 +1867,11 @@
        }
      }
    },
+    "@repeaterjs/repeater": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/@repeaterjs/repeater/-/repeater-3.0.1.tgz",
+      "integrity": "sha512-kqdrZRvTLFfvgPTeMgj8DpREYZXU+sz4/kdX2825zMNGmiaEUzM6u7A4YBoI4WzqXX4q123VftkllAw1C/4rMg=="
+    },
    "@sindresorhus/is": {
      "version": "0.14.0",
      "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz",
@@ -2509,7 +2514,6 @@
      "version": "3.1.1",
      "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz",
      "integrity": "sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg==",
-      "dev": true,
      "requires": {
        "normalize-path": "^3.0.0",
        "picomatch": "^2.0.4"
@@ -2956,11 +2960,9 @@
      "dev": true
    },
    "binary-extensions": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz",
-      "integrity": "sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==",
-      "dev": true,
-      "optional": true
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.1.0.tgz",
+      "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ=="
    },
    "bluebird": {
      "version": "3.7.2",
@@ -3339,8 +3341,6 @@
      "version": "3.4.0",
      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.0.tgz",
      "integrity": "sha512-aXAaho2VJtisB/1fg1+3nlLJqGOuewTzQpd/Tz0yTg2R0e4IGtshYvtjowyEumcBv2z+y4+kc75Mz7j5xJskcQ==",
-      "dev": true,
-      "optional": true,
      "requires": {
        "anymatch": "~3.1.1",
        "braces": "~3.0.2",
@@ -3356,8 +3356,6 @@
          "version": "3.0.2",
          "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
          "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
-          "dev": true,
-          "optional": true,
          "requires": {
            "fill-range": "^7.0.1"
          }
@@ -3366,8 +3364,6 @@
          "version": "7.0.1",
          "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
          "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
-          "dev": true,
-          "optional": true,
          "requires": {
            "to-regex-range": "^5.0.1"
          }
@@ -3375,16 +3371,12 @@
        "is-number": {
          "version": "7.0.0",
          "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
-          "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
-          "dev": true,
-          "optional": true
+          "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="
        },
        "to-regex-range": {
          "version": "5.0.1",
          "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
          "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
-          "dev": true,
-          "optional": true,
          "requires": {
            "is-number": "^7.0.0"
          }
@@ -5331,7 +5323,6 @@
      "version": "2.1.3",
      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.3.tgz",
      "integrity": "sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ==",
-      "dev": true,
      "optional": true
    },
    "function-bind": {
@@ -5412,7 +5403,6 @@
      "version": "5.1.1",
      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
      "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
-      "dev": true,
      "requires": {
        "is-glob": "^4.0.1"
      }
@@ -5912,8 +5902,6 @@
      "version": "2.1.0",
      "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
      "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==",
-      "dev": true,
-      "optional": true,
      "requires": {
        "binary-extensions": "^2.0.0"
      }
@@ -5998,8 +5986,7 @@
    "is-extglob": {
      "version": "2.1.1",
      "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
-      "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=",
-      "dev": true
+      "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI="
    },
    "is-fullwidth-code-point": {
      "version": "3.0.0",
@@ -6021,7 +6008,6 @@
      "version": "4.0.1",
      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz",
      "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==",
-      "dev": true,
      "requires": {
        "is-extglob": "^2.1.1"
      }
@@ -8972,8 +8958,7 @@
    "normalize-path": {
      "version": "3.0.0",
      "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
-      "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==",
-      "dev": true
+      "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA=="
    },
    "normalize-url": {
      "version": "4.5.0",
@@ -9418,8 +9403,7 @@
    "picomatch": {
      "version": "2.2.2",
      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz",
-      "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==",
-      "dev": true
+      "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg=="
    },
    "pify": {
      "version": "3.0.0",
@@ -9934,8 +9918,6 @@
      "version": "3.4.0",
      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.4.0.tgz",
      "integrity": "sha512-0xe001vZBnJEK+uKcj8qOhyAKPzIT+gStxWr3LCB0DwcXR5NZJ3IaC+yGnHCYzB/S7ov3m3EEbZI2zeNvX+hGQ==",
-      "dev": true,
-      "optional": true,
      "requires": {
        "picomatch": "^2.2.1"
      }
--- a/package.json
+++ b/package.json
@@ -13,6 +13,8 @@
  "license": "MIT",
  "dependencies": {
    "@iarna/toml": "^2.2.5",
+    "@repeaterjs/repeater": "^3.0.1",
+    "chokidar": "^3.4.0",
    "ejs": "^3.1.3",
    "electron": "^9.0.4",
    "koa": "^2.12.1",
--- a/src/node/data.js
+++ b/src/node/data.js
@@ -1,5 +1,10 @@
+import { once } from 'events'
+import { promises as fsPromises } from 'fs'
 import { promisify } from 'util'
+import { Repeater } from '@repeaterjs/repeater'
+import TOML from '@iarna/toml'
 import fetch from 'node-fetch'
+import chokidar from 'chokidar'

 const sleep = promisify(setTimeout)

@@ -7,14 +12,13 @@ function filterLive(data) {
  return data.filter(({ status }) => status === 'Live' || status === 'Unknown')
 }

-export async function* pollPublicData() {
-  const publicDataURL = 'https://woke.net/api/streams.json'
+export async function* pollDataURL(url) {
  const refreshInterval = 5 * 1000
  let lastData = []
  while (true) {
    let data = []
    try {
-      const resp = await fetch(publicDataURL)
+      const resp = await fetch(url)
      data = await resp.json()
    } catch (err) {
      console.warn('error loading stream data', err)
@@ -32,17 +36,52 @@ export async function* pollPublicData() {
  }
 }

+export async function* watchDataFile(path) {
+  const watcher = chokidar.watch(path)
+  while (true) {
+    let data
+    try {
+      const text = await fsPromises.readFile(path)
+      data = TOML.parse(text)
+    } catch (err) {
+      console.warn('error reading data file', err)
+    }
+    if (data) {
+      yield data.streams || []
+    }
+    await once(watcher, 'change')
+  }
+}
+
+export async function* markDataSource(dataSource, name) {
+  for await (const streamList of dataSource) {
+    for (const s of streamList) {
+      s._dataSource = name
+    }
+    yield streamList
+  }
+}
+
+export async function* combineDataSources(dataSources) {
+  for await (const streamLists of Repeater.latest(dataSources)) {
+    yield [].concat(...streamLists)
+  }
+}
+
 export class StreamIDGenerator {
-  constructor(parent) {
-    this.idMap = new Map(parent ? parent.idMap : null)
-    this.idSet = new Set(this.idMap.values())
+  constructor() {
+    this.idMap = new Map()
+    this.idSet = new Set()
  }

  process(streams) {
    const { idMap, idSet } = this
+    const localIdMap = new Map(idMap)
+    const localIdSet = new Set(idSet)
+
    for (const stream of streams) {
-      const { link, source, label } = stream
-      if (!idMap.has(link)) {
+      const { link, source, label, _dataSource } = stream
+      if (!localIdMap.has(link)) {
        let counter = 0
        let newId
        const normalizedText = (source || label || link)
@@ -54,11 +93,20 @@ export class StreamIDGenerator {
          const counterPart = counter === 0 && textPart ? '' : counter
          newId = `${textPart}${counterPart}`
          counter++
-        } while (idSet.has(newId))
+        } while (localIdSet.has(newId))
+
+        localIdMap.set(link, newId)
+        localIdSet.add(newId)
+
+        // Custom stream ids are not persisted so that editing them doesn't create a bunch of unused ids.
+        const persistId = _dataSource !== 'custom'
+        if (persistId) {
          idMap.set(link, newId)
          idSet.add(newId)
        }
-      stream._id = idMap.get(link)
+      }
+
+      stream._id = localIdMap.get(link)
    }
    return streams
  }
--- a/src/node/index.js
+++ b/src/node/index.js
@@ -1,10 +1,17 @@
 import fs from 'fs'
 import yargs from 'yargs'
 import TOML from '@iarna/toml'
+import { Repeater } from '@repeaterjs/repeater'
 import { app, shell, session, BrowserWindow } from 'electron'

 import { ensureValidURL } from '../util'
-import { pollPublicData, StreamIDGenerator } from './data'
+import {
+  pollDataURL,
+  watchDataFile,
+  StreamIDGenerator,
+  markDataSource,
+  combineDataSources,
+} from './data'
 import StreamWindow from './StreamWindow'
 import StreamdelayClient from './StreamdelayClient'
 import initWebServer from './server'
@@ -18,6 +25,18 @@ function parseArgs() {
      describe: 'Background color of wall (useful for chroma-keying)',
      default: '#000',
    })
+    .group(['data.json-url', 'data.toml-file'], 'Datasources')
+    .option('data.json-url', {
+      describe: 'Fetch streams from the specified URL(s)',
+      array: true,
+      default: ['https://woke.net/api/streams.json'],
+    })
+    .option('data.toml-file', {
+      describe: 'Fetch streams from the specified file(s)',
+      normalize: true,
+      array: true,
+      default: [],
+    })
    .group(
      [
        'control.username',
@@ -92,6 +111,11 @@ async function main() {
    })

  const idGen = new StreamIDGenerator()
+  let updateCustomStreams
+  const customStreamData = new Repeater(async (push) => {
+    await push([])
+    updateCustomStreams = push
+  })

  const streamWindow = new StreamWindow({
    backgroundColor: argv.backgroundColor,
@@ -117,10 +141,7 @@ async function main() {
    } else if (msg.type === 'set-view-blurred') {
      streamWindow.setViewBlurred(msg.viewIdx, msg.blurred)
    } else if (msg.type === 'set-custom-streams') {
-      const customIDGen = new StreamIDGenerator(idGen)
-      clientState.customStreams = customIDGen.process(msg.streams)
-      streamWindow.send('state', clientState)
-      broadcastState(clientState)
+      updateCustomStreams(msg.streams)
    } else if (msg.type === 'reload-view') {
      streamWindow.reloadView(msg.viewIdx)
    } else if (msg.type === 'browse' || msg.type === 'dev-tools') {
@@ -190,7 +211,17 @@ async function main() {
    broadcastState(clientState)
  })

-  for await (const rawStreams of pollPublicData()) {
+  const dataSources = [
+    ...argv.data['json-url'].map((url) =>
+      markDataSource(pollDataURL(url), 'json-url'),
+    ),
+    ...argv.data['toml-file'].map((path) =>
+      markDataSource(watchDataFile(path), 'toml-file'),
+    ),
+    markDataSource(customStreamData, 'custom'),
+  ]
+
+  for await (const rawStreams of combineDataSources(dataSources)) {
    const streams = idGen.process(rawStreams)
    clientState.streams = streams
    streamWindow.send('state', clientState)
--- a/src/web/control.js
+++ b/src/web/control.js
@@ -25,7 +25,6 @@ function App({ wsEndpoint }) {
  const [delayState, setDelayState] = useState({
    isConnected: false,
  })
-  const allStreams = sortBy([...streams, ...customStreams], ['_id'])

  useEffect(() => {
    const ws = new ReconnectingWebSocket(wsEndpoint, [], {
@@ -38,17 +37,11 @@ function App({ wsEndpoint }) {
    ws.addEventListener('message', (ev) => {
      const msg = JSON.parse(ev.data)
      if (msg.type === 'state') {
-        const {
-          streams: newStreams,
-          views,
-          customStreams: newCustomStreams,
-          streamdelay,
-        } = msg.state
+        const { streams: newStreams, views, streamdelay } = msg.state
        const newStateIdxMap = new Map()
-        const allStreams = [...newStreams, ...newCustomStreams]
        for (const viewState of views) {
          const { pos, content } = viewState.context
-          const stream = allStreams.find((d) => d.link === content.url)
+          const stream = newStreams.find((d) => d.link === content.url)
          const streamId = stream?._id
          const state = State.from(viewState.state)
          const isListening = state.matches(
@@ -69,8 +62,8 @@ function App({ wsEndpoint }) {
          }
        }
        setStateIdxMap(newStateIdxMap)
-        setStreams(newStreams)
-        setCustomStreams(newCustomStreams)
+        setStreams(sortBy(newStreams, ['_id']))
+        setCustomStreams(newStreams.filter((s) => s._dataSource === 'custom'))
        setDelayState(
          streamdelay.isConnected && {
            ...streamdelay,
@@ -87,9 +80,7 @@ function App({ wsEndpoint }) {
  const handleSetView = useCallback(
    (idx, streamId) => {
      const newSpaceIdxMap = new Map(stateIdxMap)
-      const stream = [...streams, ...customStreams].find(
-        (d) => d._id === streamId,
-      )
+      const stream = streams.find((d) => d._id === streamId)
      if (stream) {
        const content = {
          url: stream?.link,
@@ -109,7 +100,7 @@ function App({ wsEndpoint }) {
      ])
      wsRef.current.send(JSON.stringify({ type: 'set-views', views }))
    },
-    [streams, customStreams, stateIdxMap],
+    [streams, stateIdxMap],
  )

  const handleSetListening = useCallback((idx, listening) => {
@@ -270,7 +261,7 @@ function App({ wsEndpoint }) {
        </div>
        <div>
          {isConnected
-            ? allStreams.map((row) => (
+            ? streams.map((row) => (
                <StreamLine id={row._id} row={row} onClickId={handleClickId} />
              ))
            : 'loading...'}