Add support for TOML file data source

src/node/data.js

@@ -1,5 +1,10 @@
+import { once } from 'events'
+import { promises as fsPromises } from 'fs'
 import { promisify } from 'util'
+import { Repeater } from '@repeaterjs/repeater'
+import TOML from '@iarna/toml'
 import fetch from 'node-fetch'
+import chokidar from 'chokidar'
 
 const sleep = promisify(setTimeout)
 
@@ -7,14 +12,13 @@ function filterLive(data) {
   return data.filter(({ status }) => status === 'Live' || status === 'Unknown')
 }
 
-export async function* pollPublicData() {
-  const publicDataURL = 'https://woke.net/api/streams.json'
+export async function* pollDataURL(url) {
   const refreshInterval = 5 * 1000
   let lastData = []
   while (true) {
     let data = []
     try {
-      const resp = await fetch(publicDataURL)
+      const resp = await fetch(url)
       data = await resp.json()
     } catch (err) {
       console.warn('error loading stream data', err)
@@ -32,17 +36,52 @@ export async function* pollPublicData() {
   }
 }
 
+export async function* watchDataFile(path) {
+  const watcher = chokidar.watch(path)
+  while (true) {
+    let data
+    try {
+      const text = await fsPromises.readFile(path)
+      data = TOML.parse(text)
+    } catch (err) {
+      console.warn('error reading data file', err)
+    }
+    if (data) {
+      yield data.streams || []
+    }
+    await once(watcher, 'change')
+  }
+}
+
+export async function* markDataSource(dataSource, name) {
+  for await (const streamList of dataSource) {
+    for (const s of streamList) {
+      s._dataSource = name
+    }
+    yield streamList
+  }
+}
+
+export async function* combineDataSources(dataSources) {
+  for await (const streamLists of Repeater.latest(dataSources)) {
+    yield [].concat(...streamLists)
+  }
+}
+
 export class StreamIDGenerator {
-  constructor(parent) {
-    this.idMap = new Map(parent ? parent.idMap : null)
-    this.idSet = new Set(this.idMap.values())
+  constructor() {
+    this.idMap = new Map()
+    this.idSet = new Set()
   }
 
   process(streams) {
     const { idMap, idSet } = this
+    const localIdMap = new Map(idMap)
+    const localIdSet = new Set(idSet)
+
     for (const stream of streams) {
-      const { link, source, label } = stream
-      if (!idMap.has(link)) {
+      const { link, source, label, _dataSource } = stream
+      if (!localIdMap.has(link)) {
         let counter = 0
         let newId
         const normalizedText = (source || label || link)
@@ -54,11 +93,20 @@ export class StreamIDGenerator {
           const counterPart = counter === 0 && textPart ? '' : counter
           newId = `${textPart}${counterPart}`
           counter++
-        } while (idSet.has(newId))
-        idMap.set(link, newId)
-        idSet.add(newId)
+        } while (localIdSet.has(newId))
+
+        localIdMap.set(link, newId)
+        localIdSet.add(newId)
+
+        // Custom stream ids are not persisted so that editing them doesn't create a bunch of unused ids.
+        const persistId = _dataSource !== 'custom'
+        if (persistId) {
+          idMap.set(link, newId)
+          idSet.add(newId)
+        }
       }
-      stream._id = idMap.get(link)
+
+      stream._id = localIdMap.get(link)
     }
     return streams
   }

src/node/index.js

@@ -1,10 +1,17 @@
 import fs from 'fs'
 import yargs from 'yargs'
 import TOML from '@iarna/toml'
+import { Repeater } from '@repeaterjs/repeater'
 import { app, shell, session, BrowserWindow } from 'electron'
 
 import { ensureValidURL } from '../util'
-import { pollPublicData, StreamIDGenerator } from './data'
+import {
+  pollDataURL,
+  watchDataFile,
+  StreamIDGenerator,
+  markDataSource,
+  combineDataSources,
+} from './data'
 import StreamWindow from './StreamWindow'
 import StreamdelayClient from './StreamdelayClient'
 import initWebServer from './server'
@@ -18,6 +25,18 @@ function parseArgs() {
       describe: 'Background color of wall (useful for chroma-keying)',
       default: '#000',
     })
+    .group(['data.json-url', 'data.toml-file'], 'Datasources')
+    .option('data.json-url', {
+      describe: 'Fetch streams from the specified URL(s)',
+      array: true,
+      default: ['https://woke.net/api/streams.json'],
+    })
+    .option('data.toml-file', {
+      describe: 'Fetch streams from the specified file(s)',
+      normalize: true,
+      array: true,
+      default: [],
+    })
     .group(
       [
         'control.username',
@@ -92,6 +111,11 @@ async function main() {
     })
 
   const idGen = new StreamIDGenerator()
+  let updateCustomStreams
+  const customStreamData = new Repeater(async (push) => {
+    await push([])
+    updateCustomStreams = push
+  })
 
   const streamWindow = new StreamWindow({
     backgroundColor: argv.backgroundColor,
@@ -117,10 +141,7 @@ async function main() {
     } else if (msg.type === 'set-view-blurred') {
       streamWindow.setViewBlurred(msg.viewIdx, msg.blurred)
     } else if (msg.type === 'set-custom-streams') {
-      const customIDGen = new StreamIDGenerator(idGen)
-      clientState.customStreams = customIDGen.process(msg.streams)
-      streamWindow.send('state', clientState)
-      broadcastState(clientState)
+      updateCustomStreams(msg.streams)
     } else if (msg.type === 'reload-view') {
       streamWindow.reloadView(msg.viewIdx)
     } else if (msg.type === 'browse' || msg.type === 'dev-tools') {
@@ -190,7 +211,17 @@ async function main() {
     broadcastState(clientState)
   })
 
-  for await (const rawStreams of pollPublicData()) {
+  const dataSources = [
+    ...argv.data['json-url'].map((url) =>
+      markDataSource(pollDataURL(url), 'json-url'),
+    ),
+    ...argv.data['toml-file'].map((path) =>
+      markDataSource(watchDataFile(path), 'toml-file'),
+    ),
+    markDataSource(customStreamData, 'custom'),
+  ]
+
+  for await (const rawStreams of combineDataSources(dataSources)) {
     const streams = idGen.process(rawStreams)
     clientState.streams = streams
     streamWindow.send('state', clientState)