Add support for broadcasts

Fixes #303

public/js/hlsPlayback.js

@@ -3,6 +3,7 @@
 function playVideo(overlay) {
     const video = overlay.parentElement.querySelector('video');
     const url = video.getAttribute("data-url");
+    const startTime = parseFloat(video.getAttribute("data-start") || "0");
     video.setAttribute("controls", "");
     overlay.style.display = "none";
 
@@ -12,12 +13,13 @@ function playVideo(overlay) {
         hls.attachMedia(video);
         hls.on(Hls.Events.MANIFEST_PARSED, function () {
             hls.loadLevel = hls.levels.length - 1;
-            hls.startLoad();
+            hls.startLoad(startTime);
             video.play();
         });
     } else if (video.canPlayType('application/vnd.apple.mpegurl')) {
         video.src = url;
         video.addEventListener('canplay', function() {
+            if (startTime > 0) video.currentTime = startTime;
             video.play();
         });
     }

src/api.nim

@@ -73,6 +73,25 @@ proc getAboutAccount*(username: string): Future[AccountInfo] {.async.} =
     js = await fetch(url)
   result = parseAboutAccount(js)
 
+proc restReq(endpoint: string; params: seq[(string, string)] = @[]): ApiReq =
+  let url = ApiUrl(endpoint: endpoint, params: params)
+  ApiReq(cookie: url, oauth: url)
+
+proc getBroadcastInfo*(id: string): Future[Broadcast] {.async.} =
+  if id.len == 0: return
+  let
+    req = apiReq(graphBroadcast, """{"id":"$1"}""" % id)
+    js = await fetch(req)
+  result = parseBroadcastInfo(js)
+
+proc fetchBroadcastStream*(mediaKey: string): Future[string] {.async.} =
+  if mediaKey.len == 0: return
+  let
+    streamReq = restReq(restLiveStream & mediaKey)
+    streamJs = await fetch(streamReq)
+  result = streamJs{"source", "noRedirectPlaybackUrl"}.getStr(
+    streamJs{"source", "location"}.getStr)
+
 proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} =
   if id.len == 0: return
   let

src/apiutils.nim

@@ -34,13 +34,14 @@ proc setApiProxy*(url: string) =
       apiProxy = "http://" & apiProxy
 
 proc toUrl(req: ApiReq; sessionKind: SessionKind): Uri =
-  case sessionKind
+  let url = case sessionKind
-  of oauth:  
+    of oauth:  req.oauth
-    let o = req.oauth
+    of cookie: req.cookie
-    parseUri("https://api.x.com/graphql")   / o.endpoint ? o.params
+  let base = case sessionKind
-  of cookie: 
+    of oauth:  "https://api.x.com"
-    let c = req.cookie
+    of cookie: "https://x.com/i/api"
-    parseUri("https://x.com/i/api/graphql") / c.endpoint ? c.params
+  let prefix = if url.endpoint.startsWith("1.1/"): "" else: "graphql/"
+  parseUri(base) / (prefix & url.endpoint) ? url.params
 
 proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
   let
@@ -89,7 +90,7 @@ proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
     result["sec-fetch-dest"] = "empty"
     result["sec-fetch-mode"] = "cors"
     result["sec-fetch-site"] = "same-site"
-    if disableTid:
+    if disableTid or "/1.1/" in url.path:
       result["authorization"] = bearerToken2
     else:
       result["authorization"] = bearerToken
@@ -116,7 +117,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
     pool.use(await genHeaders(session, url)):
       template getContent =
         # TODO: this is a temporary simple implementation
-        if apiProxy.len > 0:
+        if apiProxy.len > 0 and "/1.1/" notin url.path:
           resp = await c.get(($url).replace("https://", apiProxy))
         else:
           resp = await c.get($url)

src/consts.nim

@@ -27,6 +27,9 @@ const
   graphListTweets* = "VQf8_XQynI3WzH6xopOMMQ/ListTimeline"
   graphAboutAccount* = "zs_jFPFT78rBpXv9Z3U2YQ/AboutAccountQuery"
 
+  graphBroadcast* = "0nMmbMh-_JwwRRFNXkyH3Q/BroadcastQuery"
+  restLiveStream* = "1.1/live_video_stream/status/"
+
   gqlFeatures* = """{
   "android_ad_formats_media_component_render_overlay_enabled": false,
   "android_graphql_skip_api_media_color_palette": false,

src/formatters.nim

@@ -91,7 +91,17 @@ proc getM3u8Url*(content: string): string =
   if re.find(content, m3u8Regex, matches) != -1:
     result = matches[0]
 
-proc proxifyVideo*(manifest: string; proxy: bool): string =
+proc proxifyVideo*(manifest: string; proxy: bool; manifestUrl = ""): string =
+  let (baseUrl, basePath) =
+    if manifestUrl.len > 0:
+      let
+        u = parseUri(manifestUrl)
+        origin = u.scheme & "://" & u.hostname
+        idx = manifestUrl.rfind('/')
+        dirPath = if idx > 8: manifestUrl[0 .. idx] else: ""
+      (origin, dirPath)
+    else:
+      ("https://video.twimg.com", "")
   var replacements: seq[(string, string)]
   for line in manifest.splitLines:
     let url =
@@ -99,9 +109,13 @@ proc proxifyVideo*(manifest: string; proxy: bool): string =
       elif line.startsWith("#EXT-X-MEDIA") and "URI=" in line:
         line[line.find("URI=") + 5 .. -1 + line.find("\"", start= 5 + line.find("URI="))]
       else: line
-    if url.startsWith('/'):
+    let resolved =
-      let path = "https://video.twimg.com" & url
+      if url.startsWith('/'): baseUrl & url
-      replacements.add (url, if proxy: path.getVidUrl else: path)
+      elif basePath.len > 0 and url.len > 0 and not url.startsWith('#') and
+           not url.startsWith("http") and ('.' in url): basePath & url
+      else: ""
+    if resolved.len > 0:
+      replacements.add (url, if proxy: resolved.getVidUrl else: resolved)
   return manifest.multiReplace(replacements)
 
 proc getUserPic*(userPic: string; style=""): string =
@@ -154,16 +168,18 @@ proc getShortTime*(tweet: Tweet): string =
   else:
     result = "now"
 
-proc getDuration*(video: Video): string =
+proc getDuration*(ms: int): string =
-  let 
+  let
-    ms = video.durationMs
     sec = int(round(ms / 1000))
     min = floorDiv(sec, 60)
     hour = floorDiv(min, 60)
   if hour > 0:
-    return &"{hour}:{min mod 60}:{sec mod 60:02}"
+    &"{hour}:{min mod 60:02}:{sec mod 60:02}"
   else:
-    return &"{min mod 60}:{sec mod 60:02}"
+    &"{min mod 60}:{sec mod 60:02}"
+
+proc getDuration*(video: Video): string =
+  getDuration(video.durationMs)
 
 proc getLink*(id: int64; username="i"; focus=true): string =
   var username = username

src/nitter.nim

@@ -10,7 +10,7 @@ import types, config, prefs, formatters, redis_cache, http_pool, auth, apiutils
 import views/[general, about]
 import routes/[
   preferences, timeline, status, media, search, rss, list, debug,
-  unsupported, embed, resolver, router_utils]
+  unsupported, embed, resolver, broadcast, router_utils]
 
 const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances"
 const issuesUrl = "https://github.com/zedeus/nitter/issues"
@@ -58,6 +58,7 @@ createSearchRouter(cfg)
 createMediaRouter(cfg)
 createEmbedRouter(cfg)
 createRssRouter(cfg)
+createBroadcastRouter(cfg)
 createDebugRouter(cfg)
 
 settings:
@@ -121,5 +122,6 @@ routes:
   extend preferences, ""
   extend resolver, ""
   extend embed, ""
+  extend broadcastRoute, ""
   extend debug, ""
   extend unsupported, ""

src/parser.nim

@@ -110,6 +110,22 @@ proc parseAboutAccount*(js: JsonNode): AccountInfo =
       with since, reason{"verified_since_msec"}:
         result.verifiedSince = since.getTimeFromMsStr
 
+proc parseBroadcastInfo*(js: JsonNode): Broadcast =
+  let bc = ? js{"data", "broadcast"}
+  result = Broadcast(
+    id: bc{"broadcast_id"}.getStr,
+    title: bc{"status"}.getStr,
+    state: bc{"state"}.getStr.toUpperAscii,
+    thumb: bc{"image_url"}.getStr,
+    mediaKey: bc{"media_key"}.getStr,
+    totalWatched: bc{"total_watched"}.getInt,
+    startTime: bc{"start_time"}.getTimeFromMs,
+    endTime: bc{"end_time"}.getTimeFromMs,
+    replayStart: bc{"edited_replay", "start_time"}.getInt,
+    availableForReplay: bc{"available_for_replay"}.getBool,
+    user: parseGraphUser(bc)
+  )
+
 proc parseGraphList*(js: JsonNode): List =
   if js.isNull: return
 
@@ -287,14 +303,23 @@ proc parsePromoVideo(js: JsonNode): Video =
   result.variants.add variant
 
 proc parseBroadcast(js: JsonNode): Card =
-  let image = js{"broadcast_thumbnail_large"}.getImageVal
+  let
+    image = js{"broadcast_thumbnail_large"}.getImageVal
+    broadcastUrl = js{"broadcast_url"}.getStrVal
+    broadcastId = broadcastUrl.rsplit('/', maxsplit=1)[^1]
+    streamUrl = "/i/broadcasts/" & broadcastId & "/stream"
   result = Card(
     kind: broadcast,
-    url: js{"broadcast_url"}.getStrVal,
+    url: "/i/broadcasts/" & broadcastId,
     title: js{"broadcaster_display_name"}.getStrVal,
     text: js{"broadcast_title"}.getStrVal,
     image: image,
-    video: some Video(thumb: image)
+    video: some Video(
+      thumb: image,
+      available: true,
+      playbackType: m3u8,
+      variants: @[VideoVariant(contentType: m3u8, url: streamUrl)]
+    )
   )
 
 proc parseCard(js: JsonNode; urls: JsonNode): Card =

src/redis_cache.nim

@@ -158,6 +158,20 @@ proc getCachedUsername*(userId: string): Future[string] {.async.} =
 #     if not result.isNil:
 #       await cache(result)
 
+proc cache*(data: Broadcast) {.async.} =
+  if data.id.len == 0: return
+  await setEx("bc:" & data.id, baseCacheTime, compress(toFlatty(data)))
+
+proc getCachedBroadcast*(id: string): Future[Broadcast] {.async.} =
+  if id.len == 0: return
+  let cached = await get("bc:" & id)
+  if cached != redisNil:
+    cached.deserialize(Broadcast)
+  else:
+    result = await getBroadcastInfo(id)
+    await cache(result)
+  result.m3u8Url = await fetchBroadcastStream(result.mediaKey)
+
 proc cache*(data: AccountInfo; name: string) {.async.} =
   await setEx("ai:" & toLower(name), baseCacheTime * 24, compress(toFlatty(data)))
 

src/routes/broadcast.nim

@@ -0,0 +1,44 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import asyncdispatch, strutils
+import jester
+
+import router_utils
+import ".."/[types, formatters, redis_cache]
+import ../views/[general, broadcast]
+import media
+
+export broadcast
+
+proc createBroadcastRouter*(cfg: Config) =
+  router broadcastRoute:
+    get "/i/broadcasts/@id":
+      cond @"id".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9'})
+      var bc: Broadcast
+      try:
+        bc = await getCachedBroadcast(@"id")
+      except:
+        discard
+
+      if bc.id.len == 0:
+        resp Http404, showError("Broadcast not found", cfg)
+
+      let prefs = requestPrefs()
+      resp renderMain(renderBroadcast(bc, prefs, request.path), request, cfg, prefs,
+                      bc.title, ogTitle=bc.title)
+
+    get "/i/broadcasts/@id/stream":
+      cond @"id".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9'})
+      var bc: Broadcast
+      try:
+        bc = await getCachedBroadcast(@"id")
+      except:
+        discard
+
+      if bc.m3u8Url.len == 0:
+        resp Http404
+
+      let manifest = await safeFetch(bc.m3u8Url)
+      if manifest.len == 0:
+        resp Http502
+
+      resp proxifyVideo(manifest, requestPrefs().proxyVideos, bc.m3u8Url), m3u8Mime

src/routes/media.nim

@@ -86,6 +86,12 @@ proc decoded*(req: jester.Request; index: int): string =
   if based: decode(encoded)
   else: decodeUrl(encoded)
 
+proc normalizeImgUrl*(url: var string) =
+  if not url.startsWith("http"):
+    if "twimg.com" notin url:
+      url.insert(twimg)
+    url.insert(https)
+
 proc createMediaRouter*(cfg: Config) =
   router media:
     get "/pic/?":
@@ -94,11 +100,7 @@ proc createMediaRouter*(cfg: Config) =
     get re"^\/pic\/orig\/(enc)?\/?(.+)":
       var url = decoded(request, 1)
       cond "/amplify_video/" notin url
-
+      normalizeImgUrl(url)
-      if "twimg.com" notin url:
-        url.insert(twimg)
-      if not url.startsWith(https):
-        url.insert(https)
       url.add("?name=orig")
 
       let uri = parseUri(url)
@@ -110,11 +112,7 @@ proc createMediaRouter*(cfg: Config) =
     get re"^\/pic\/(enc)?\/?(.+)":
       var url = decoded(request, 1)
       cond "/amplify_video/" notin url
-
+      normalizeImgUrl(url)
-      if "twimg.com" notin url:
-        url.insert(twimg)
-      if not url.startsWith(https):
-        url.insert(https)
 
       let uri = parseUri(url)
       cond isTwitterUrl(uri) == true
@@ -143,6 +141,6 @@ proc createMediaRouter*(cfg: Config) =
 
       if ".m3u8" in url:
         let vid = await safeFetch(url)
-        content = proxifyVideo(vid, requestPrefs().proxyVideos)
+        content = proxifyVideo(vid, requestPrefs().proxyVideos, url)
 
       resp content, m3u8Mime

src/sass/_broadcast.scss

@@ -0,0 +1,75 @@
+.broadcast-page {
+  max-width: 800px;
+  width: 100%;
+  margin: 20px auto 0;
+}
+
+.broadcast-panel {
+  background-color: var(--bg_panel);
+  border: 1px solid var(--border_grey);
+  border-radius: 8px;
+  overflow: hidden;
+}
+
+.broadcast-player {
+  position: relative;
+  background: black;
+
+  video,
+  img {
+    display: block;
+    width: 100%;
+  }
+}
+
+.broadcast-info {
+  padding: 14px 16px;
+}
+
+.broadcast-title {
+  font-size: 18px;
+  font-weight: bold;
+  margin: 0 0 12px;
+}
+
+.broadcast-user-row {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+}
+
+.broadcast-user {
+  display: flex;
+  align-items: center;
+  gap: 10px;
+  color: var(--fg_color);
+
+  img {
+    width: 40px;
+    height: 40px;
+    border-radius: 50%;
+  }
+}
+
+.broadcast-username {
+  color: var(--fg_dark);
+}
+
+.broadcast-meta {
+  color: var(--fg_faded);
+  font-size: 14px;
+  display: flex;
+  flex-direction: column;
+  align-items: flex-end;
+  flex-shrink: 0;
+  line-height: 1.5em;
+}
+
+.broadcast-live {
+  background: #e0245e;
+  color: white;
+  padding: 1px 6px;
+  border-radius: 3px;
+  font-weight: bold;
+  font-size: 12px;
+}

src/sass/index.scss

@@ -7,6 +7,7 @@
 @import "inputs";
 @import "timeline";
 @import "search";
+@import "broadcast";
 
 body {
   // colors
@@ -160,7 +161,7 @@ body.fixed-nav .container {
   display: inline-block;
   width: 14px;
   height: 14px;
-  margin-left: 2px;
+  margin-bottom: 2px;
 
   .verified-icon-circle {
     position: absolute;

src/sass/tweet/_base.scss

@@ -44,6 +44,10 @@
   padding: 0;
   display: flex;
   justify-content: space-between;
+
+  .verified-icon {
+    margin-left: 2px;
+  }
 }
 
 .fullname-and-username {

src/types.nim

@@ -113,6 +113,20 @@ type
     verifiedSince*: DateTime
     overrideVerifiedYear*: int
 
+  Broadcast* = object
+    id*: string
+    title*: string
+    state*: string
+    thumb*: string
+    mediaKey*: string
+    m3u8Url*: string
+    totalWatched*: int
+    startTime*: DateTime
+    endTime*: DateTime
+    replayStart*: int
+    availableForReplay*: bool
+    user*: User
+
   VideoType* = enum
     m3u8 = "application/x-mpegURL"
     mp4 = "video/mp4"

src/utils.nim

@@ -17,7 +17,9 @@ const
     "abs.twimg.com",
     "pbs.twimg.com",
     "video.twimg.com",
-    "x.com"
+    "x.com",
+    "pscp.tv",
+    "video.pscp.tv"
   ]
 
 proc setHmacKey*(key: string) =
@@ -55,7 +57,8 @@ proc filterParams*(params: Table): seq[(string, string)] =
       result.add p
 
 proc isTwitterUrl*(uri: Uri): bool =
-  uri.hostname in twitterDomains
+  uri.hostname in twitterDomains or
+    uri.hostname.endsWith(".video.pscp.tv")
 
 proc isTwitterUrl*(url: string): bool =
   isTwitterUrl(parseUri(url))

src/views/broadcast.nim

@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import strutils, strformat, times
+import karax/[karaxdsl, vdom]
+
+import renderutils
+import ".."/[types, utils, formatters]
+
+proc renderBroadcast*(bc: Broadcast; prefs: Prefs; path: string): VNode =
+  let
+    isLive = bc.state == "RUNNING"
+    thumb = getPicUrl(bc.thumb)
+    source = if prefs.proxyVideos and bc.m3u8Url.startsWith("http"):
+               getVidUrl(bc.m3u8Url) else: bc.m3u8Url
+    stateText =
+      if isLive: "LIVE"
+      elif bc.endTime.year > 1: "Ended " & bc.endTime.format("MMM d, YYYY")
+      elif bc.state.len > 0: bc.state
+      else: "Ended"
+    durationMs =
+      if bc.startTime.year > 1 and bc.endTime.year > 1:
+        int((bc.endTime - bc.startTime).inMilliseconds) - bc.replayStart * 1000
+      else: 0
+    duration = if durationMs > 0: getDuration(durationMs) else: ""
+
+  buildHtml(tdiv(class="broadcast-page")):
+    tdiv(class="broadcast-panel"):
+      tdiv(class="broadcast-player"):
+        if bc.m3u8Url.len > 0 and prefs.hlsPlayback:
+          video(poster=thumb, data-url=source, data-autoload="false",
+                data-start=($bc.replayStart), muted=prefs.muteVideos)
+          verbatim "<div class=\"video-overlay\" onclick=\"playVideo(this)\">"
+          tdiv(class="overlay-circle"): span(class="overlay-triangle")
+          if isLive:
+            tdiv(class="broadcast-live"): text "LIVE"
+          elif duration.len > 0:
+            tdiv(class="overlay-duration"): text duration
+          verbatim "</div>"
+        elif bc.m3u8Url.len > 0:
+          img(src=thumb, alt=bc.title)
+          tdiv(class="video-overlay"):
+            buttonReferer "/enablehls", "Enable hls playback", path
+            if isLive:
+              tdiv(class="broadcast-live"): text "LIVE"
+            elif duration.len > 0:
+              tdiv(class="overlay-duration"): text duration
+        elif bc.thumb.len > 0:
+          img(src=thumb, alt=bc.title)
+          tdiv(class="video-overlay"):
+            if bc.availableForReplay:
+              p: text "Stream unavailable"
+            else:
+              p: text "Replay is not available"
+        else:
+          tdiv(class="video-overlay"):
+            p: text "Broadcast not found"
+
+      tdiv(class="broadcast-info"):
+        h2(class="broadcast-title"): text bc.title
+
+        tdiv(class="broadcast-user-row"):
+          a(class="broadcast-user", href=("/" & bc.user.username)):
+            genImg(getUserPic(bc.user.userPic, "_bigger"))
+            tdiv:
+              tdiv:
+                strong: text bc.user.fullname
+                verifiedIcon(bc.user)
+              span(class="broadcast-username"): text "@" & bc.user.username
+
+          tdiv(class="broadcast-meta"):
+            if bc.totalWatched > 0: 
+              span: text insertSep($bc.totalWatched, ',') & " views"
+            if isLive:
+              span(class="broadcast-live"): text stateText
+            else:
+              span: text stateText

src/views/general.nim

@@ -50,7 +50,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
   let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
 
   buildHtml(head):
-    link(rel="stylesheet", type="text/css", href="/css/style.css?v=34")
+    link(rel="stylesheet", type="text/css", href="/css/style.css?v=35")
     link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=5")
 
     if theme.len > 0:

src/views/tweet.nim

@@ -95,8 +95,8 @@ proc renderVideoAttachment(videoData: Video; prefs: Prefs; path=""; bigThumb=fal
       let
         vars = videoData.variants.filterIt(it.contentType == playbackType)
         vidUrl = vars.sortedByIt(it.resolution)[^1].url
-        source = if prefs.proxyVideos: getVidUrl(vidUrl)
+        source = if prefs.proxyVideos and vidUrl.startsWith("http"):
-                 else: vidUrl
+                   getVidUrl(vidUrl) else: vidUrl
       case playbackType
       of mp4:
         video(poster=thumb, controls="", muted=prefs.muteVideos):
@@ -105,7 +105,8 @@ proc renderVideoAttachment(videoData: Video; prefs: Prefs; path=""; bigThumb=fal
         video(poster=thumb, data-url=source, data-autoload="false", muted=prefs.muteVideos)
         verbatim "<div class=\"video-overlay\" onclick=\"playVideo(this)\">"
         tdiv(class="overlay-circle"): span(class="overlay-triangle")
-        tdiv(class="overlay-duration"): text getDuration(videoData)
+        if videoData.durationMs > 0:
+          tdiv(class="overlay-duration"): text getDuration(videoData)
         verbatim "</div>"
 
 proc renderVideo*(video: Video; prefs: Prefs; path: string; bigThumb=false): VNode =