Merge branch 'master' into feature/mp4-streaming

2026-01-28 14:22:48 -05:00 · 2023-05-20 22:23:12 +02:00
parent 6e490c2dd9 e3b3b38a2d
commit 93208908e6
50 changed files with 842 additions and 492 deletions
--- a/src/api.nim
+++ b/src/api.nim
@@ -4,123 +4,143 @@ import packedjson
 import types, query, formatters, consts, apiutils, parser
 import experimental/parser as newParser

-proc getGraphUser*(id: string): Future[User] {.async.} =
+proc getGraphUser*(username: string): Future[User] {.async.} =
+  if username.len == 0: return
+  let
+    variables = %*{"screen_name": username}
+    params = {"variables": $variables, "features": gqlFeatures}
+    js = await fetchRaw(graphUser ? params, Api.userScreenName)
+  result = parseGraphUser(js)
+
+proc getGraphUserById*(id: string): Future[User] {.async.} =
  if id.len == 0 or id.any(c => not c.isDigit): return
  let
-    variables = %*{"userId": id, "withSuperFollowsUserFields": true}
-    js = await fetchRaw(graphUser ? {"variables": $variables}, Api.userRestId)
+    variables = %*{"userId": id}
+    params = {"variables": $variables, "features": gqlFeatures}
+    js = await fetchRaw(graphUserById ? params, Api.userRestId)
  result = parseGraphUser(js)

+proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Timeline] {.async.} =
+  if id.len == 0: return
+  let
+    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+    variables = userTweetsVariables % [id, cursor]
+    params = {"variables": variables, "features": gqlFeatures}
+    (url, apiId) = case kind
+                   of TimelineKind.tweets: (graphUserTweets, Api.userTweets)
+                   of TimelineKind.replies: (graphUserTweetsAndReplies, Api.userTweetsAndReplies)
+                   of TimelineKind.media: (graphUserMedia, Api.userMedia)
+    js = await fetch(url ? params, apiId)
+  result = parseGraphTimeline(js, "user", after)
+
+proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
+  if id.len == 0: return
+  let
+    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+    variables = listTweetsVariables % [id, cursor]
+    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetch(graphListTweets ? params, Api.listTweets)
+  result = parseGraphTimeline(js, "list", after)
+
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
  let
-    variables = %*{"screenName": name, "listSlug": list, "withHighlightedLabel": false}
-    url = graphListBySlug ? {"variables": $variables}
-  result = parseGraphList(await fetch(url, Api.listBySlug))
+    variables = %*{"screenName": name, "listSlug": list}
+    params = {"variables": $variables, "features": gqlFeatures}
+  result = parseGraphList(await fetch(graphListBySlug ? params, Api.listBySlug))

 proc getGraphList*(id: string): Future[List] {.async.} =
  let
-    variables = %*{"listId": id, "withHighlightedLabel": false}
-    url = graphList ? {"variables": $variables}
-  result = parseGraphList(await fetch(url, Api.list))
+    variables = %*{"listId": id}
+    params = {"variables": $variables, "features": gqlFeatures}
+  result = parseGraphList(await fetch(graphListById ? params, Api.list))

 proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
  if list.id.len == 0: return
  var
    variables = %*{
      "listId": list.id,
-      "withSuperFollowsUserFields": false,
      "withBirdwatchPivots": false,
      "withDownvotePerspective": false,
      "withReactionsMetadata": false,
-      "withReactionsPerspective": false,
-      "withSuperFollowsTweetFields": false
+      "withReactionsPerspective": false
    }
  if after.len > 0:
    variables["cursor"] = % after
-  let url = graphListMembers ? {"variables": $variables}
+  let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures}
  result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)

-proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
+proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
  if id.len == 0: return
  let
-    ps = genParams({"list_id": id, "ranking_mode": "reverse_chronological"}, after)
-    url = listTimeline ? ps
-  result = parseTimeline(await fetch(url, Api.timeline), after)
+    variables = tweetResultVariables % id
+    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetch(graphTweetResult ? params, Api.tweetResult)
+  result = parseGraphTweetResult(js)

-proc getUser*(username: string): Future[User] {.async.} =
-  if username.len == 0: return
-  let
-    ps = genParams({"screen_name": username})
-    json = await fetchRaw(userShow ? ps, Api.userShow)
-  result = parseUser(json, username)
-
-proc getUserById*(userId: string): Future[User] {.async.} =
-  if userId.len == 0: return
-  let
-    ps = genParams({"user_id": userId})
-    json = await fetchRaw(userShow ? ps, Api.userShow)
-  result = parseUser(json)
-
-proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
+proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
  if id.len == 0: return
  let
-    ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
-    url = timeline / (id & ".json") ? ps
-  result = parseTimeline(await fetch(url, Api.timeline), after)
+    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+    variables = tweetVariables % [id, cursor]
+    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetch(graphTweet ? params, Api.tweetDetail)
+  result = parseGraphConversation(js, id)

-proc getMediaTimeline*(id: string; after=""): Future[Timeline] {.async.} =
-  if id.len == 0: return
-  let url = mediaTimeline / (id & ".json") ? genParams(cursor=after)
-  result = parseTimeline(await fetch(url, Api.timeline), after)
+proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
+  result = (await getGraphTweet(id, after)).replies
+  result.beginning = after.len == 0
+
+proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
+  result = await getGraphTweet(id)
+  if after.len > 0:
+    result.replies = await getReplies(id, after)
+
+proc getGraphSearch*(query: Query; after=""): Future[Result[Tweet]] {.async.} =
+  let q = genQueryParam(query)
+  if q.len == 0 or q == emptyQuery:
+    return Result[Tweet](query: query, beginning: true)
+
+  var
+    variables = %*{
+      "rawQuery": q,
+      "count": 20,
+      "product": "Latest",
+      "withDownvotePerspective": false,
+      "withReactionsMetadata": false,
+      "withReactionsPerspective": false
+    }
+  if after.len > 0:
+    variables["cursor"] = % after
+  let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
+  result = parseGraphSearch(await fetch(url, Api.search), after)
+  result.query = query
+
+proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} =
+  if query.text.len == 0:
+    return Result[User](query: query, beginning: true)
+
+  var url = userSearch ? {
+    "q": query.text,
+    "skip_status": "1",
+    "count": "20",
+    "page": page
+  }
+
+  result = parseUsers(await fetchRaw(url, Api.userSearch))
+  result.query = query
+  if page.len == 0:
+    result.bottom = "2"
+  elif page.allCharsInSet(Digits):
+    result.bottom = $(parseInt(page) + 1)

 proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
  if name.len == 0: return
  let
    ps = genParams({"screen_name": name, "trim_user": "true"},
-                   count="18", ext=false)
+                    count="18", ext=false)
    url = photoRail ? ps
  result = parsePhotoRail(await fetch(url, Api.timeline))

-proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
-  when T is User:
-    const
-      searchMode = ("result_filter", "user")
-      parse = parseUsers
-      fetchFunc = fetchRaw
-  else:
-    const
-      searchMode = ("tweet_search_mode", "live")
-      parse = parseTimeline
-      fetchFunc = fetch
-
-  let q = genQueryParam(query)
-  if q.len == 0 or q == emptyQuery:
-    return Result[T](beginning: true, query: query)
-
-  let url = search ? genParams(searchParams & @[("q", q), searchMode], after)
-  try:
-    result = parse(await fetchFunc(url, Api.search), after)
-    result.query = query
-  except InternalError:
-    return Result[T](beginning: true, query: query)
-
-proc getTweetImpl(id: string; after=""): Future[Conversation] {.async.} =
-  let url = tweet / (id & ".json") ? genParams(cursor=after)
-  result = parseConversation(await fetch(url, Api.tweet), id)
-
-proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
-  result = (await getTweetImpl(id, after)).replies
-  result.beginning = after.len == 0
-
-proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
-  result = await getTweetImpl(id)
-  if after.len > 0:
-    result.replies = await getReplies(id, after)
-
-proc getStatus*(id: string): Future[Tweet] {.async.} =
-  let url = status / (id & ".json") ? genParams()
-  result = parseStatus(await fetch(url, Api.status))
-
 proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
  let client = newAsyncHttpClient(maxRedirects=0)
  try:
--- a/src/apiutils.nim
+++ b/src/apiutils.nim
@@ -17,13 +17,13 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
    result &= p
  if ext:
    result &= ("ext", "mediaStats")
-    result &= ("include_ext_alt_text", "true")
-    result &= ("include_ext_media_availability", "true")
+    result &= ("include_ext_alt_text", "1")
+    result &= ("include_ext_media_availability", "1")
  if count.len > 0:
    result &= ("count", count)
  if cursor.len > 0:
    # The raw cursor often has plus signs, which sometimes get turned into spaces,
-    # so we need to them back into a plus
+    # so we need to turn them back into a plus
    if " " in cursor:
      result &= ("cursor", cursor.replace(" ", "+"))
    else:
@@ -44,7 +44,7 @@ proc genHeaders*(token: Token = nil): HttpHeaders =
  })

 template updateToken() =
-  if api != Api.search and resp.headers.hasKey(rlRemaining):
+  if resp.headers.hasKey(rlRemaining):
    let
      remaining = parseInt(resp.headers[rlRemaining])
      reset = parseInt(resp.headers[rlReset])
@@ -61,12 +61,15 @@ template fetchImpl(result, fetchBody) {.dirty.} =
  try:
    var resp: AsyncResponse
    pool.use(genHeaders(token)):
-      resp = await c.get($url)
-      result = await resp.body
+      template getContent =
+        resp = await c.get($url)
+        result = await resp.body
+
+      getContent()

      if resp.status == $Http503:
        badClient = true
-        raise newException(InternalError, result)
+        raise newException(BadClientError, "Bad client")

    if result.len > 0:
      if resp.headers.getOrDefault("content-encoding") == "gzip":
@@ -82,6 +85,9 @@ template fetchImpl(result, fetchBody) {.dirty.} =
      raise newException(InternalError, $url)
  except InternalError as e:
    raise e
+  except BadClientError as e:
+    release(token, used=true)
+    raise e
  except Exception as e:
    echo "error: ", e.name, ", msg: ", e.msg, ", token: ", token[], ", url: ", url
    if "length" notin e.msg and "descriptor" notin e.msg:
@@ -100,7 +106,7 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
    updateToken()

    let error = result.getError
-    if error in {invalidToken, forbidden, badToken}:
+    if error in {invalidToken, badToken}:
      echo "fetch error: ", result.getError
      release(token, invalid=true)
      raise rateLimitError()
@@ -115,7 +121,7 @@ proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =

    if result.startsWith("{\"errors"):
      let errors = result.fromJson(Errors)
-      if errors in {invalidToken, forbidden, badToken}:
+      if errors in {invalidToken, badToken}:
        echo "fetch error: ", errors
        release(token, invalid=true)
        raise rateLimitError()
--- a/src/consts.nim
+++ b/src/consts.nim
@@ -1,28 +1,28 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import uri, sequtils
+import uri, sequtils, strutils

 const
-  auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
+  auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"

  api = parseUri("https://api.twitter.com")
  activate* = $(api / "1.1/guest/activate.json")

-  userShow* = api / "1.1/users/show.json"
  photoRail* = api / "1.1/statuses/media_timeline.json"
-  status* = api / "1.1/statuses/show"
-  search* = api / "2/search/adaptive.json"
-
-  timelineApi = api / "2/timeline"
-  timeline* = timelineApi / "profile"
-  mediaTimeline* = timelineApi / "media"
-  listTimeline* = timelineApi / "list.json"
-  tweet* = timelineApi / "conversation"
+  userSearch* = api / "1.1/users/search.json"

  graphql = api / "graphql"
-  graphUser* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
-  graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
-  graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
-  graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers"
+  graphUser* = graphql / "pVrmNaXcxPjisIvKtLDMEA/UserByScreenName"
+  graphUserById* = graphql / "1YAM811Q8Ry4XyPpJclURQ/UserByRestId"
+  graphUserTweets* = graphql / "WzJjibAcDa-oCjCcLOotcg/UserTweets"
+  graphUserTweetsAndReplies* = graphql / "fn9oRltM1N4thkh5CVusPg/UserTweetsAndReplies"
+  graphUserMedia* = graphql / "qQoeS7szGavsi8-ehD2AWg/UserMedia"
+  graphTweet* = graphql / "miKSMGb2R1SewIJv2-ablQ/TweetDetail"
+  graphTweetResult* = graphql / "0kc0a_7TTr3dvweZlMslsQ/TweetResultByRestId"
+  graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline"
+  graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId"
+  graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug"
+  graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers"
+  graphListTweets* = graphql / "jZntL0oVJSdjhmPcdbw_eA/ListLatestTweetsTimeline"

  timelineParams* = {
    "include_profile_interstitial_type": "0",
@@ -33,27 +33,89 @@ const
    "include_mute_edge": "0",
    "include_can_dm": "0",
    "include_can_media_tag": "1",
+    "include_ext_is_blue_verified": "1",
    "skip_status": "1",
    "cards_platform": "Web-12",
    "include_cards": "1",
-    "include_composer_source": "false",
+    "include_composer_source": "0",
    "include_reply_count": "1",
    "tweet_mode": "extended",
-    "include_entities": "true",
-    "include_user_entities": "true",
-    "include_ext_media_color": "false",
-    "send_error_codes": "true",
-    "simple_quoted_tweet": "true",
-    "include_quote_count": "true"
+    "include_entities": "1",
+    "include_user_entities": "1",
+    "include_ext_media_color": "0",
+    "send_error_codes": "1",
+    "simple_quoted_tweet": "1",
+    "include_quote_count": "1"
  }.toSeq

-  searchParams* = {
-    "query_source": "typed_query",
-    "pc": "1",
-    "spelling_corrections": "1"
-  }.toSeq
-  ## top: nothing
-  ## latest: "tweet_search_mode: live"
-  ## user:   "result_filter: user"
-  ## photos: "result_filter: photos"
-  ## videos: "result_filter: videos"
+  gqlFeatures* = """{
+  "blue_business_profile_image_shape_enabled": false,
+  "creator_subscriptions_tweet_preview_api_enabled": true,
+  "freedom_of_speech_not_reach_fetch_enabled": false,
+  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
+  "highlights_tweets_tab_ui_enabled": false,
+  "interactive_text_enabled": false,
+  "longform_notetweets_consumption_enabled": true,
+  "longform_notetweets_inline_media_enabled": false,
+  "longform_notetweets_richtext_consumption_enabled": true,
+  "longform_notetweets_rich_text_read_enabled": false,
+  "responsive_web_edit_tweet_api_enabled": false,
+  "responsive_web_enhance_cards_enabled": false,
+  "responsive_web_graphql_exclude_directive_enabled": true,
+  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+  "responsive_web_graphql_timeline_navigation_enabled": false,
+  "responsive_web_text_conversations_enabled": false,
+  "responsive_web_twitter_blue_verified_badge_is_enabled": true,
+  "rweb_lists_timeline_redesign_enabled": true,
+  "spaces_2022_h2_clipping": true,
+  "spaces_2022_h2_spaces_communities": true,
+  "standardized_nudges_misinfo": false,
+  "tweet_awards_web_tipping_enabled": false,
+  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
+  "tweetypie_unmention_optimization_enabled": false,
+  "verified_phone_label_enabled": false,
+  "vibe_api_enabled": false,
+  "view_counts_everywhere_api_enabled": false
+}""".replace(" ", "").replace("\n", "")
+
+  tweetVariables* = """{
+  "focalTweetId": "$1",
+  $2
+  "withBirdwatchNotes": false,
+  "includePromotedContent": false,
+  "withDownvotePerspective": false,
+  "withReactionsMetadata": false,
+  "withReactionsPerspective": false,
+  "withVoice": false
+}"""
+
+  tweetResultVariables* = """{
+  "tweetId": "$1",
+  "includePromotedContent": false,
+  "withDownvotePerspective": false,
+  "withReactionsMetadata": false,
+  "withReactionsPerspective": false,
+  "withVoice": false,
+  "withCommunity": false
+}"""
+
+  userTweetsVariables* = """{
+  "userId": "$1", $2
+  "count": 20,
+  "includePromotedContent": false,
+  "withDownvotePerspective": false,
+  "withReactionsMetadata": false,
+  "withReactionsPerspective": false,
+  "withVoice": false,
+  "withV2Timeline": true
+}"""
+
+  listTweetsVariables* = """{
+  "listId": "$1", $2
+  "count": 20,
+  "includePromotedContent": false,
+  "withDownvotePerspective": false,
+  "withReactionsMetadata": false,
+  "withReactionsPerspective": false,
+  "withVoice": false
+}"""
--- a/src/experimental/parser.nim
+++ b/src/experimental/parser.nim
@@ -1,2 +1,2 @@
-import parser/[user, graphql, timeline]
-export user, graphql, timeline
+import parser/[user, graphql]
+export user, graphql
--- a/src/experimental/parser/graphql.nim
+++ b/src/experimental/parser/graphql.nim
@@ -1,11 +1,17 @@
+import options
 import jsony
 import user, ../types/[graphuser, graphlistmembers]
 from ../../types import User, Result, Query, QueryKind

 proc parseGraphUser*(json: string): User =
  let raw = json.fromJson(GraphUser)
+
+  if raw.data.user.result.reason.get("") == "Suspended":
+    return User(suspended: true)
+
  result = toUser raw.data.user.result.legacy
  result.id = raw.data.user.result.restId
+  result.verified = result.verified or raw.data.user.result.isBlueVerified

 proc parseGraphListMembers*(json, cursor: string): Result[User] =
  result = Result[User](
--- a/src/experimental/parser/timeline.nim
+++ b/src/experimental/parser/timeline.nim
@@ -1,30 +0,0 @@
-import std/[strutils, tables]
-import jsony
-import user, ../types/timeline
-from ../../types import Result, User
-
-proc getId(id: string): string {.inline.} =
-  let start = id.rfind("-")
-  if start < 0: return id
-  id[start + 1 ..< id.len]
-
-proc parseUsers*(json: string; after=""): Result[User] =
-  result = Result[User](beginning: after.len == 0)
-
-  let raw = json.fromJson(Search)
-  if raw.timeline.instructions.len == 0:
-    return
-
-  for i in raw.timeline.instructions:
-    if i.addEntries.entries.len > 0:
-      for e in i.addEntries.entries:
-        let id = e.entryId.getId
-        if e.entryId.startsWith("user"):
-         if id in raw.globalObjects.users:
-           result.content.add toUser raw.globalObjects.users[id]
-        elif e.entryId.startsWith("cursor"):
-         let cursor = e.content.operation.cursor
-         if cursor.cursorType == "Top":
-           result.top = cursor.value
-         elif cursor.cursorType == "Bottom":
-           result.bottom = cursor.value
--- a/src/experimental/parser/unifiedcard.nim
+++ b/src/experimental/parser/unifiedcard.nim
@@ -66,6 +66,8 @@ proc parseMedia(component: Component; card: UnifiedCard; result: var Card) =
      durationMs: videoInfo.durationMillis,
      variants: videoInfo.variants
    )
+  of model3d:
+    result.title = "Unsupported 3D model ad"

 proc parseUnifiedCard*(json: string): Card =
  let card = json.fromJson(UnifiedCard)
@@ -82,6 +84,10 @@ proc parseUnifiedCard*(json: string): Card =
      component.parseMedia(card, result)
    of buttonGroup:
      discard
+    of ComponentType.hidden:
+      result.kind = CardKind.hidden
+    of ComponentType.unknown:
+      echo "ERROR: Unknown component type: ", json

    case component.kind
    of twitterListDetails:
--- a/src/experimental/parser/user.nim
+++ b/src/experimental/parser/user.nim
@@ -2,7 +2,7 @@ import std/[algorithm, unicode, re, strutils, strformat, options, nre]
 import jsony
 import utils, slices
 import ../types/user as userType
-from ../../types import User, Error
+from ../../types import Result, User, Error

 let
  unRegex = re.re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
@@ -76,3 +76,12 @@ proc parseUser*(json: string; username=""): User =
    else: echo "[error - parseUser]: ", error

  result = toUser json.fromJson(RawUser)
+
+proc parseUsers*(json: string; after=""): Result[User] =
+  result = Result[User](beginning: after.len == 0)
+
+  # starting with '{' means it's an error
+  if json[0] == '[':
+    let raw = json.fromJson(seq[RawUser])
+    for user in raw:
+      result.content.add user.toUser
--- a/src/experimental/types/graphuser.nim
+++ b/src/experimental/types/graphuser.nim
@@ -1,3 +1,4 @@
+import options
 import user

 type
@@ -10,3 +11,5 @@ type
  UserResult = object
    legacy*: RawUser
    restId*: string
+    isBlueVerified*: bool
+    reason*: Option[string]
--- a/src/experimental/types/unifiedcard.nim
+++ b/src/experimental/types/unifiedcard.nim
@@ -17,6 +17,8 @@ type
    twitterListDetails
    communityDetails
    mediaWithDetailsHorizontal
+    hidden
+    unknown

  Component* = object
    kind*: ComponentType
@@ -47,7 +49,7 @@ type
    vanity*: string

  MediaType* = enum
-    photo, video
+    photo, video, model3d

  MediaEntity* = object
    kind*: MediaType
@@ -70,10 +72,37 @@ type
  Text = object
    content: string

-  HasTypeField = Component | Destination | MediaEntity | AppStoreData
+  TypeField = Component | Destination | MediaEntity | AppStoreData

 converter fromText*(text: Text): string = text.content

-proc renameHook*(v: var HasTypeField; fieldName: var string) =
+proc renameHook*(v: var TypeField; fieldName: var string) =
  if fieldName == "type":
    fieldName = "kind"
+
+proc enumHook*(s: string; v: var ComponentType) =
+  v = case s
+      of "details": details
+      of "media": media
+      of "swipeable_media": swipeableMedia
+      of "button_group": buttonGroup
+      of "app_store_details": appStoreDetails
+      of "twitter_list_details": twitterListDetails
+      of "community_details": communityDetails
+      of "media_with_details_horizontal": mediaWithDetailsHorizontal
+      of "commerce_drop_details": hidden
+      else: echo "ERROR: Unknown enum value (ComponentType): ", s; unknown
+
+proc enumHook*(s: string; v: var AppType) =
+  v = case s
+      of "android_app": androidApp
+      of "iphone_app": iPhoneApp
+      of "ipad_app": iPadApp
+      else: echo "ERROR: Unknown enum value (AppType): ", s; androidApp
+
+proc enumHook*(s: string; v: var MediaType) =
+  v = case s
+      of "video": video
+      of "photo": photo
+      of "model3d": model3d
+      else: echo "ERROR: Unknown enum value (MediaType): ", s; photo
--- a/src/formatters.nim
+++ b/src/formatters.nim
@@ -12,8 +12,7 @@ let
  twRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?twitter\.com"
  twLinkRegex = re"""<a href="https:\/\/twitter.com([^"]+)">twitter\.com(\S+)</a>"""

-  ytRegex = re"([A-z.]+\.)?youtu(be\.com|\.be)"
-  igRegex = re"(www\.)?instagram\.com"
+  ytRegex = re(r"([A-z.]+\.)?youtu(be\.com|\.be)", {reStudy, reIgnoreCase})

  rdRegex = re"(?<![.b])((www|np|new|amp|old)\.)?reddit.com"
  rdShortRegex = re"(?<![.b])redd\.it\/"
@@ -56,15 +55,13 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =

  if prefs.replaceYouTube.len > 0 and "youtu" in result:
    result = result.replace(ytRegex, prefs.replaceYouTube)
-    if prefs.replaceYouTube in result:
-      result = result.replace("/c/", "/")

  if prefs.replaceTwitter.len > 0 and ("twitter.com" in body or tco in body):
-    result = result.replace(tco, &"{https}{prefs.replaceTwitter}/t.co")
+    result = result.replace(tco, https & prefs.replaceTwitter & "/t.co")
    result = result.replace(cards, prefs.replaceTwitter & "/cards")
    result = result.replace(twRegex, prefs.replaceTwitter)
    result = result.replacef(twLinkRegex, a(
-      prefs.replaceTwitter & "$2", href = &"{https}{prefs.replaceTwitter}$1"))
+      prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))

  if prefs.replaceReddit.len > 0 and ("reddit.com" in result or "redd.it" in result):
    result = result.replace(rdShortRegex, prefs.replaceReddit & "/comments/")
@@ -72,9 +69,6 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
    if prefs.replaceReddit in result and "/gallery/" in result:
      result = result.replace("/gallery/", "/comments/")

-  if prefs.replaceInstagram.len > 0 and "instagram.com" in result:
-    result = result.replace(igRegex, prefs.replaceInstagram)
-
  if absolute.len > 0 and "href" in result:
    result = result.replace("href=\"/", &"href=\"{absolute}/")

--- a/src/http_pool.nim
+++ b/src/http_pool.nim
@@ -42,5 +42,11 @@ template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped =
  except ProtocolError:
    # Twitter closed the connection, retry
    body
+  except BadClientError:
+    # Twitter returned 503, we need a new client
+    pool.release(c, true)
+    badClient = false
+    c = pool.acquire(heads)
+    body
  finally:
    pool.release(c, badClient)
--- a/src/nitter.nim
+++ b/src/nitter.nim
@@ -56,6 +56,7 @@ settings:
  port = Port(cfg.port)
  staticDir = cfg.staticDir
  bindAddr = cfg.address
+  reusePort = true

 routes:
  get "/":
@@ -84,19 +85,23 @@ routes:
    resp Http500, showError(
      &"An error occurred, please {link} with the URL you tried to visit.", cfg)

+  error BadClientError:
+    echo error.exc.name, ": ", error.exc.msg
+    resp Http500, showError("Network error occured, please try again.", cfg)
+
  error RateLimitError:
    const link = a("another instance", href = instancesUrl)
    resp Http429, showError(
      &"Instance has been rate limited.<br>Use {link} or try again later.", cfg)

-  extend unsupported, ""
-  extend preferences, ""
-  extend resolver, ""
  extend rss, ""
+  extend status, ""
  extend search, ""
  extend timeline, ""
-  extend list, ""
-  extend status, ""
  extend media, ""
+  extend list, ""
+  extend preferences, ""
+  extend resolver, ""
  extend embed, ""
  extend debug, ""
+  extend unsupported, ""
--- a/src/parser.nim
+++ b/src/parser.nim
@@ -4,6 +4,8 @@ import packedjson, packedjson/deserialiser
 import types, parserutils, utils
 import experimental/parser/unifiedcard

+proc parseGraphTweet(js: JsonNode): Tweet
+
 proc parseUser(js: JsonNode; id=""): User =
  if js.isNull: return
  result = User(
@@ -19,13 +21,20 @@ proc parseUser(js: JsonNode; id=""): User =
    tweets: js{"statuses_count"}.getInt,
    likes: js{"favourites_count"}.getInt,
    media: js{"media_count"}.getInt,
-    verified: js{"verified"}.getBool,
+    verified: js{"verified"}.getBool or js{"ext_is_blue_verified"}.getBool,
    protected: js{"protected"}.getBool,
    joinDate: js{"created_at"}.getTime
  )

  result.expandUserEntities(js)

+proc parseGraphUser(js: JsonNode): User =
+  let user = ? js{"user_results", "result"}
+  result = parseUser(user{"legacy"})
+
+  if "is_blue_verified" in user:
+    result.verified = true
+
 proc parseGraphList*(js: JsonNode): List =
  if js.isNull: return

@@ -38,14 +47,13 @@ proc parseGraphList*(js: JsonNode): List =
  result = List(
    id: list{"id_str"}.getStr,
    name: list{"name"}.getStr,
-    username: list{"user", "legacy", "screen_name"}.getStr,
-    userId: list{"user", "rest_id"}.getStr,
+    username: list{"user_results", "result", "legacy", "screen_name"}.getStr,
+    userId: list{"user_results", "result", "rest_id"}.getStr,
    description: list{"description"}.getStr,
    members: list{"member_count"}.getInt,
-    banner: list{"custom_banner_media", "media_info", "url"}.getImageStr
+    banner: list{"custom_banner_media", "media_info", "original_img_url"}.getImageStr
  )

-
 proc parsePoll(js: JsonNode): Poll =
  let vals = js{"binding_values"}
  # name format is pollNchoice_*
@@ -73,8 +81,8 @@ proc parseGif(js: JsonNode): Gif =
 proc parseVideo(js: JsonNode): Video =
  result = Video(
    thumb: js{"media_url_https"}.getImageStr,
-    views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr,
-    available: js{"ext_media_availability", "status"}.getStr == "available",
+    views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr($js{"mediaStats", "viewCount"}.getInt),
+    available: js{"ext_media_availability", "status"}.getStr.toLowerAscii == "available",
    title: js{"ext_alt_text"}.getStr,
    durationMs: js{"video_info", "duration_millis"}.getInt
    # playbackType: mp4
@@ -186,7 +194,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
     result.url.len == 0 or result.url.startsWith("card://"):
    result.url = getPicUrl(result.image)

-proc parseTweet(js: JsonNode): Tweet =
+proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
  if js.isNull: return
  result = Tweet(
    id: js{"id_str"}.getId,
@@ -194,7 +202,6 @@ proc parseTweet(js: JsonNode): Tweet =
    replyId: js{"in_reply_to_status_id_str"}.getId,
    text: js{"full_text"}.getStr,
    time: js{"created_at"}.getTime,
-    source: getSource(js),
    hasThread: js{"self_thread"}.notNull,
    available: true,
    user: User(id: js{"user_id_str"}.getStr),
@@ -208,14 +215,26 @@ proc parseTweet(js: JsonNode): Tweet =

  result.expandTweetEntities(js)

+  # fix for pinned threads
+  if result.hasThread and result.threadId == 0:
+    result.threadId = js{"self_thread", "id_str"}.getId
+
  if js{"is_quote_status"}.getBool:
    result.quote = some Tweet(id: js{"quoted_status_id_str"}.getId)

+  # legacy
  with rt, js{"retweeted_status_id_str"}:
    result.retweet = some Tweet(id: rt.getId)
    return

-  with jsCard, js{"card"}:
+  # graphql
+  with rt, js{"retweeted_status_result", "result"}:
+    # needed due to weird edgecase where the actual tweet data isn't included
+    if "legacy" in rt:
+      result.retweet = some parseGraphTweet(rt)
+      return
+
+  if jsCard.kind != JNull:
    let name = jsCard{"name"}.getStr
    if "poll" in name:
      if "image" in name:
@@ -235,7 +254,10 @@ proc parseTweet(js: JsonNode): Tweet =
      of "video":
        result.video = some(parseVideo(m))
        with user, m{"additional_media_info", "source_user"}:
-          result.attribution = some(parseUser(user))
+          if user{"id"}.getInt > 0:
+            result.attribution = some(parseUser(user))
+          else:
+            result.attribution = some(parseGraphUser(user))
      of "animated_gif":
        result.gif = some(parseGif(m))
      else: discard
@@ -292,70 +314,11 @@ proc parseGlobalObjects(js: JsonNode): GlobalObjects =
    result.users[k] = parseUser(v, k)

  for k, v in tweets:
-    var tweet = parseTweet(v)
+    var tweet = parseTweet(v, v{"card"})
    if tweet.user.id in result.users:
      tweet.user = result.users[tweet.user.id]
    result.tweets[k] = tweet

-proc parseThread(js: JsonNode; global: GlobalObjects): tuple[thread: Chain, self: bool] =
-  result.thread = Chain()
-
-  let thread = js{"content", "item", "content", "conversationThread"}
-  with cursor, thread{"showMoreCursor"}:
-    result.thread.cursor = cursor{"value"}.getStr
-    result.thread.hasMore = true
-
-  for t in thread{"conversationComponents"}:
-    let content = t{"conversationTweetComponent", "tweet"}
-
-    if content{"displayType"}.getStr == "SelfThread":
-      result.self = true
-
-    var tweet = finalizeTweet(global, content{"id"}.getStr)
-    if not tweet.available:
-      tweet.tombstone = getTombstone(content{"tombstone"})
-    result.thread.content.add tweet
-
-proc parseConversation*(js: JsonNode; tweetId: string): Conversation =
-  result = Conversation(replies: Result[Chain](beginning: true))
-  let global = parseGlobalObjects(? js)
-
-  let instructions = ? js{"timeline", "instructions"}
-  if instructions.len == 0:
-    return
-
-  for e in instructions[0]{"addEntries", "entries"}:
-    let entry = e{"entryId"}.getStr
-    if "tweet" in entry or "tombstone" in entry:
-      let tweet = finalizeTweet(global, e.getEntryId)
-      if $tweet.id != tweetId:
-        result.before.content.add tweet
-      else:
-        result.tweet = tweet
-    elif "conversationThread" in entry:
-      let (thread, self) = parseThread(e, global)
-      if thread.content.len > 0:
-        if self:
-          result.after = thread
-        else:
-          result.replies.content.add thread
-    elif "cursor-showMore" in entry:
-      result.replies.bottom = e.getCursor
-    elif "cursor-bottom" in entry:
-      result.replies.bottom = e.getCursor
-
-proc parseStatus*(js: JsonNode): Tweet =
-  with e, js{"errors"}:
-    if e.getError == tweetNotFound:
-      return
-
-  result = parseTweet(js)
-  if not result.isNil:
-    result.user = parseUser(js{"user"})
-
-    with quote, js{"quoted_status"}:
-      result.quote = some parseStatus(js{"quoted_status"})
-
 proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNode) =
  if js.kind != JArray or js.len == 0:
    return
@@ -396,7 +359,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
      result.top = e.getCursor
    elif "cursor-bottom" in entry:
      result.bottom = e.getCursor
-    elif entry.startsWith("sq-C"):
+    elif entry.startsWith("sq-cursor"):
      with cursor, e{"content", "operation", "cursor"}:
        if cursor{"cursorType"}.getStr == "Bottom":
          result.bottom = cursor{"value"}.getStr
@@ -406,7 +369,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
 proc parsePhotoRail*(js: JsonNode): PhotoRail =
  for tweet in js:
    let
-      t = parseTweet(tweet)
+      t = parseTweet(tweet, js{"card"})
      url = if t.photos.len > 0: t.photos[0]
            elif t.video.isSome: get(t.video).thumb
            elif t.gif.isSome: get(t.gif).thumb
@@ -415,3 +378,141 @@ proc parsePhotoRail*(js: JsonNode): PhotoRail =

    if url.len == 0: continue
    result.add GalleryPhoto(url: url, tweetId: $t.id)
+
+proc parseGraphTweet(js: JsonNode): Tweet =
+  if js.kind == JNull:
+    return Tweet()
+
+  case js{"__typename"}.getStr
+  of "TweetUnavailable":
+    return Tweet()
+  of "TweetTombstone":
+    return Tweet(text: js{"tombstone", "text"}.getTombstone)
+  of "TweetPreviewDisplay":
+    return Tweet(text: "You're unable to view this Tweet because it's only available to the Subscribers of the account owner.")
+  of "TweetWithVisibilityResults":
+    return parseGraphTweet(js{"tweet"})
+
+  var jsCard = copy(js{"card", "legacy"})
+  if jsCard.kind != JNull:
+    var values = newJObject()
+    for val in jsCard["binding_values"]:
+      values[val["key"].getStr] = val["value"]
+    jsCard["binding_values"] = values
+
+  result = parseTweet(js{"legacy"}, jsCard)
+  result.user = parseGraphUser(js{"core"})
+
+  with noteTweet, js{"note_tweet", "note_tweet_results", "result"}:
+    result.expandNoteTweetEntities(noteTweet)
+
+  if result.quote.isSome:
+    result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}))
+
+proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
+  let thread = js{"content", "items"}
+  for t in js{"content", "items"}:
+    let entryId = t{"entryId"}.getStr
+    if "cursor-showmore" in entryId:
+      let cursor = t{"item", "itemContent", "value"}
+      result.thread.cursor = cursor.getStr
+      result.thread.hasMore = true
+    elif "tweet" in entryId:
+      let tweet = parseGraphTweet(t{"item", "itemContent", "tweet_results", "result"})
+      result.thread.content.add tweet
+
+      if t{"item", "itemContent", "tweetDisplayType"}.getStr == "SelfThread":
+        result.self = true
+
+proc parseGraphTweetResult*(js: JsonNode): Tweet =
+  with tweet, js{"data", "tweetResult", "result"}:
+    result = parseGraphTweet(tweet)
+
+proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
+  result = Conversation(replies: Result[Chain](beginning: true))
+
+  let instructions = ? js{"data", "threaded_conversation_with_injections", "instructions"}
+  if instructions.len == 0:
+    return
+
+  for e in instructions[0]{"entries"}:
+    let entryId = e{"entryId"}.getStr
+    # echo entryId
+    if entryId.startsWith("tweet"):
+      with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
+        let tweet = parseGraphTweet(tweetResult)
+
+        if not tweet.available:
+          tweet.id = parseBiggestInt(entryId.getId())
+
+        if $tweet.id == tweetId:
+          result.tweet = tweet
+        else:
+          result.before.content.add tweet
+    elif entryId.startsWith("tombstone"):
+      let id = entryId.getId()
+      let tweet = Tweet(
+        id: parseBiggestInt(id),
+        available: false,
+        text: e{"content", "itemContent", "tombstoneInfo", "richText"}.getTombstone
+      )
+
+      if id == tweetId:
+        result.tweet = tweet
+      else:
+        result.before.content.add tweet
+    elif entryId.startsWith("conversationthread"):
+      let (thread, self) = parseGraphThread(e)
+      if self:
+        result.after = thread
+      else:
+        result.replies.content.add thread
+    elif entryId.startsWith("cursor-bottom"):
+      result.replies.bottom = e{"content", "itemContent", "value"}.getStr
+
+proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Timeline =
+  result = Timeline(beginning: after.len == 0)
+
+  let instructions =
+    if root == "list": ? js{"data", "list", "tweets_timeline", "timeline", "instructions"}
+    else: ? js{"data", "user", "result", "timeline_v2", "timeline", "instructions"}
+
+  if instructions.len == 0:
+    return
+
+  for i in instructions:
+    if i{"type"}.getStr == "TimelineAddEntries":
+      for e in i{"entries"}:
+        let entryId = e{"entryId"}.getStr
+        if entryId.startsWith("tweet"):
+          with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
+            let tweet = parseGraphTweet(tweetResult)
+            if not tweet.available:
+              tweet.id = parseBiggestInt(entryId.getId())
+            result.content.add tweet
+        elif entryId.startsWith("cursor-bottom"):
+          result.bottom = e{"content", "value"}.getStr
+
+proc parseGraphSearch*(js: JsonNode; after=""): Timeline =
+  result = Timeline(beginning: after.len == 0)
+
+  let instructions = js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"}
+  if instructions.len == 0:
+    return
+
+  for instruction in instructions:
+    let typ = instruction{"type"}.getStr
+    if typ == "TimelineAddEntries":
+      for e in instructions[0]{"entries"}:
+        let entryId = e{"entryId"}.getStr
+        if entryId.startsWith("tweet"):
+          with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
+            let tweet = parseGraphTweet(tweetResult)
+            if not tweet.available:
+              tweet.id = parseBiggestInt(entryId.getId())
+            result.content.add tweet
+        elif entryId.startsWith("cursor-bottom"):
+          result.bottom = e{"content", "value"}.getStr
+    elif typ == "TimelineReplaceEntry":
+      if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"):
+        result.bottom = instruction{"entry", "content", "value"}.getStr
--- a/src/parserutils.nim
+++ b/src/parserutils.nim
@@ -28,13 +28,13 @@ template `?`*(js: JsonNode): untyped =
  if j.isNull: return
  j

-template `with`*(ident, value, body): untyped =
-  block:
+template with*(ident, value, body): untyped =
+  if true:
    let ident {.inject.} = value
    if ident != nil: body

-template `with`*(ident; value: JsonNode; body): untyped =
-  block:
+template with*(ident; value: JsonNode; body): untyped =
+  if true:
    let ident {.inject.} = value
    if value.notNull: body

@@ -130,13 +130,9 @@ proc getBanner*(js: JsonNode): string =
      return

 proc getTombstone*(js: JsonNode): string =
-  result = js{"tombstoneInfo", "richText", "text"}.getStr
+  result = js{"text"}.getStr
  result.removeSuffix(" Learn more")

-proc getSource*(js: JsonNode): string =
-  let src = js{"source"}.getStr
-  result = src.substr(src.find('>') + 1, src.rfind('<') - 1)
-
 proc getMp4Resolution*(url: string): int =
  # parses the height out of a URL like this one:
  # https://video.twimg.com/ext_tw_video/<tweet-id>/pu/vid/720x1280/<random>.mp4
@@ -234,47 +230,37 @@ proc expandUserEntities*(user: var User; js: JsonNode) =
  user.bio = user.bio.replacef(unRegex, unReplace)
                     .replacef(htRegex, htReplace)

-proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
-  let
-    orig = tweet.text.toRunes
-    textRange = js{"display_text_range"}
-    textSlice = textRange{0}.getInt .. textRange{1}.getInt
-    hasQuote = js{"is_quote_status"}.getBool
-    hasCard = tweet.card.isSome
-
-  var replyTo = ""
-  if tweet.replyId != 0:
-    with reply, js{"in_reply_to_screen_name"}:
-      tweet.reply.add reply.getStr
-      replyTo = reply.getStr
-
-  let ent = ? js{"entities"}
+proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlice: Slice[int];
+                        replyTo=""; hasQuote=false) =
+  let hasCard = tweet.card.isSome

  var replacements = newSeq[ReplaceSlice]()

-  with urls, ent{"urls"}:
+  with urls, entities{"urls"}:
    for u in urls:
      let urlStr = u["url"].getStr
-      if urlStr.len == 0 or urlStr notin tweet.text:
+      if urlStr.len == 0 or urlStr notin text:
        continue
+
      replacements.extractUrls(u, textSlice.b, hideTwitter = hasQuote)
+
      if hasCard and u{"url"}.getStr == get(tweet.card).url:
        get(tweet.card).url = u{"expanded_url"}.getStr

-  with media, ent{"media"}:
+  with media, entities{"media"}:
    for m in media:
      replacements.extractUrls(m, textSlice.b, hideTwitter = true)

-  if "hashtags" in ent:
-    for hashtag in ent["hashtags"]:
+  if "hashtags" in entities:
+    for hashtag in entities["hashtags"]:
      replacements.extractHashtags(hashtag)

-  if "symbols" in ent:
-    for symbol in ent["symbols"]:
+  if "symbols" in entities:
+    for symbol in entities["symbols"]:
      replacements.extractHashtags(symbol)

-  if "user_mentions" in ent:
-    for mention in ent["user_mentions"]:
+  if "user_mentions" in entities:
+    for mention in entities["user_mentions"]:
      let
        name = mention{"screen_name"}.getStr
        slice = mention.extractSlice
@@ -291,5 +277,27 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
  replacements.deduplicate
  replacements.sort(cmp)

-  tweet.text = orig.replacedWith(replacements, textSlice)
-                   .strip(leading=false)
+  tweet.text = text.toRunes.replacedWith(replacements, textSlice).strip(leading=false)
+
+proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
+  let
+    entities = ? js{"entities"}
+    hasQuote = js{"is_quote_status"}.getBool
+    textRange = js{"display_text_range"}
+    textSlice = textRange{0}.getInt .. textRange{1}.getInt
+
+  var replyTo = ""
+  if tweet.replyId != 0:
+    with reply, js{"in_reply_to_screen_name"}:
+      replyTo = reply.getStr
+      tweet.reply.add replyTo
+
+  tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote)
+
+proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) =
+  let
+    entities = ? js{"entity_set"}
+    text = js{"text"}.getStr
+    textSlice = 0..text.runeLen
+
+  tweet.expandTextEntities(entities, text, textSlice)
--- a/src/prefs_impl.nim
+++ b/src/prefs_impl.nim
@@ -83,7 +83,7 @@ genPrefs:
      "Enable mp4 video playback"

    hlsPlayback(checkbox, false):
-      "Enable hls video streaming (requires JavaScript)"
+      "Enable HLS video streaming (requires JavaScript)"

    proxyVideos(checkbox, true):
      "Proxy video streaming through the server (might be slow)"
@@ -107,10 +107,6 @@ genPrefs:
      "Reddit -> Teddit/Libreddit"
      placeholder: "Teddit hostname"

-    replaceInstagram(input, ""):
-      "Instagram -> Bibliogram"
-      placeholder: "Bibliogram hostname"
-
 iterator allPrefs*(): Pref =
  for k, v in prefList:
    for pref in v:
--- a/src/redis_cache.nim
+++ b/src/redis_cache.nim
@@ -118,11 +118,11 @@ proc getUserId*(username: string): Future[string] {.async.} =
  pool.withAcquire(r):
    result = await r.hGet(name.uidKey, name)
    if result == redisNil:
-      let user = await getUser(username)
+      let user = await getGraphUser(username)
      if user.suspended:
        return "suspended"
      else:
-        await cacheUserId(name, user.id)
+        await all(cacheUserId(name, user.id), cache(user))
        return user.id

 proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
@@ -130,8 +130,7 @@ proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
  if prof != redisNil:
    prof.deserialize(User)
  elif fetch:
-    let userId = await getUserId(username)
-    result = await getGraphUser(userId)
+    result = await getGraphUser(username)
    await cache(result)

 proc getCachedUsername*(userId: string): Future[string] {.async.} =
@@ -142,9 +141,11 @@ proc getCachedUsername*(userId: string): Future[string] {.async.} =
  if username != redisNil:
    result = username
  else:
-    let user = await getUserById(userId)
+    let user = await getGraphUserById(userId)
    result = user.username
    await setEx(key, baseCacheTime, result)
+    if result.len > 0 and user.id.len > 0:
+      await all(cacheUserId(result, user.id), cache(user))

 proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
  if id == 0: return
@@ -152,8 +153,8 @@ proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
  if tweet != redisNil:
    tweet.deserialize(Tweet)
  else:
-    result = await getStatus($id)
-    if result.isNil:
+    result = await getGraphTweetResult($id)
+    if not result.isNil:
      await cache(result)

 proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
--- a/src/routes/embed.nim
+++ b/src/routes/embed.nim
@@ -25,7 +25,7 @@ proc createEmbedRouter*(cfg: Config) =
      if convo == nil or convo.tweet == nil:
        resp Http404

-      resp $renderTweetEmbed(convo.tweet, path, prefs, cfg, request)
+      resp renderTweetEmbed(convo.tweet, path, prefs, cfg, request)

    get "/embed/Tweet.html":
      let id = @"id"
--- a/src/routes/list.nim
+++ b/src/routes/list.nim
@@ -6,7 +6,6 @@ import jester
 import router_utils
 import ".."/[types, redis_cache, api]
 import ../views/[general, timeline, list]
-export getListTimeline, getGraphList

 template respList*(list, timeline, title, vnode: typed) =
  if list.id.len == 0 or list.name.len == 0:
@@ -39,7 +38,7 @@ proc createListRouter*(cfg: Config) =
      let
        prefs = cookiePrefs()
        list = await getCachedList(id=(@"id"))
-        timeline = await getListTimeline(list.id, getCursor())
+        timeline = await getGraphListTweets(list.id, getCursor())
        vnode = renderTimelineTweets(timeline, prefs, request.path)
      respList(list, timeline, list.title, vnode)

--- a/src/routes/rss.nim
+++ b/src/routes/rss.nim
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, strformat, tables, times, hashes, uri
+import asyncdispatch, tables, times, hashes, uri

 import jester

@@ -10,6 +10,11 @@ include "../views/rss.nimf"

 export times, hashes

+proc redisKey*(page, name, cursor: string): string =
+  result = page & ":" & name
+  if cursor.len > 0:
+    result &= ":" & cursor
+
 proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.} =
  var profile: Profile
  let
@@ -23,7 +28,7 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
    var q = query
    q.fromUser = names
    profile = Profile(
-      tweets: await getSearch[Tweet](q, after),
+      tweets: await getGraphSearch(q, after),
      # this is kinda dumb
      user: User(
        username: name,
@@ -42,8 +47,8 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
 template respRss*(rss, page) =
  if rss.cursor.len == 0:
    let info = case page
-               of "User": &""" "{@"name"}" """
-               of "List": &""" "{@"id"}" """
+               of "User": " \"" & @"name" & "\" "
+               of "List": " \"" & @"id" & "\" "
               else: " "

    resp Http404, showError(page & info & "not found", cfg)
@@ -67,13 +72,13 @@ proc createRssRouter*(cfg: Config) =

      let
        cursor = getCursor()
-        key = &"search:{hash(genQueryUrl(query))}:cursor"
+        key = redisKey("search", $hash(genQueryUrl(query)), cursor)

      var rss = await getCachedRss(key)
      if rss.cursor.len > 0:
        respRss(rss, "Search")

-      let tweets = await getSearch[Tweet](query, cursor)
+      let tweets = await getGraphSearch(query, cursor)
      rss.cursor = tweets.bottom
      rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg)

@@ -84,9 +89,8 @@ proc createRssRouter*(cfg: Config) =
      cond cfg.enableRss
      cond '.' notin @"name"
      let
-        cursor = getCursor()
        name = @"name"
-        key = &"twitter:{name}:{cursor}"
+        key = redisKey("twitter", name, getCursor())

      var rss = await getCachedRss(key)
      if rss.cursor.len > 0:
@@ -101,18 +105,20 @@ proc createRssRouter*(cfg: Config) =
      cond cfg.enableRss
      cond '.' notin @"name"
      cond @"tab" in ["with_replies", "media", "search"]
-      let name = @"name"
-      let query =
-        case @"tab"
-        of "with_replies": getReplyQuery(name)
-        of "media": getMediaQuery(name)
-        of "search": initQuery(params(request), name=name)
-        else: Query(fromUser: @[name])
+      let
+        name = @"name"
+        tab = @"tab"
+        query =
+          case tab
+          of "with_replies": getReplyQuery(name)
+          of "media": getMediaQuery(name)
+          of "search": initQuery(params(request), name=name)
+          else: Query(fromUser: @[name])

-      var key = &"""{@"tab"}:{@"name"}:"""
-      if @"tab" == "search":
-        key &= $hash(genQueryUrl(query)) & ":"
-      key &= getCursor()
+      let searchKey = if tab != "search": ""
+                      else: ":" & $hash(genQueryUrl(query))
+
+      let key = redisKey(tab, name & searchKey, getCursor())

      var rss = await getCachedRss(key)
      if rss.cursor.len > 0:
@@ -132,29 +138,28 @@ proc createRssRouter*(cfg: Config) =
        cursor = getCursor()

      if list.id.len == 0:
-        resp Http404, showError(&"""List "{@"slug"}" not found""", cfg)
+        resp Http404, showError("List \"" & @"slug" & "\" not found", cfg)

-      let url = &"/i/lists/{list.id}/rss"
+      let url = "/i/lists/" & list.id & "/rss"
      if cursor.len > 0:
-        redirect(&"{url}?cursor={encodeUrl(cursor, false)}")
+        redirect(url & "?cursor=" & encodeUrl(cursor, false))
      else:
        redirect(url)

    get "/i/lists/@id/rss":
      cond cfg.enableRss
      let
+        id = @"id"
        cursor = getCursor()
-        key =
-          if cursor.len == 0: "lists:" & @"id"
-          else: &"""lists:{@"id"}:{cursor}"""
+        key = redisKey("lists", id, cursor)

      var rss = await getCachedRss(key)
      if rss.cursor.len > 0:
        respRss(rss, "List")

      let
-        list = await getCachedList(id=(@"id"))
-        timeline = await getListTimeline(list.id, cursor)
+        list = await getCachedList(id=id)
+        timeline = await getGraphListTweets(list.id, cursor)
      rss.cursor = timeline.bottom
      rss.feed = renderListRss(timeline.content, list, cfg)

--- a/src/routes/search.nim
+++ b/src/routes/search.nim
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, strformat, uri
+import strutils, uri

 import jester

@@ -14,32 +14,38 @@ export search
 proc createSearchRouter*(cfg: Config) =
  router search:
    get "/search/?":
-      if @"q".len > 500:
+      let q = @"q"
+      if q.len > 500:
        resp Http400, showError("Search input too long.", cfg)

      let
        prefs = cookiePrefs()
        query = initQuery(params(request))
+        title = "Search" & (if q.len > 0: " (" & q & ")" else: "")

      case query.kind
      of users:
-        if "," in @"q":
-          redirect("/" & @"q")
-        let users = await getSearch[User](query, getCursor())
-        resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs)
+        if "," in q:
+          redirect("/" & q)
+        var users: Result[User]
+        try:
+          users = await getUserSearch(query, getCursor())
+        except InternalError:
+          users = Result[User](beginning: true, query: query)
+        resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title)
      of tweets:
        let
-          tweets = await getSearch[Tweet](query, getCursor())
+          tweets = await getGraphSearch(query, getCursor())
          rss = "/search/rss?" & genQueryUrl(query)
        resp renderMain(renderTweetSearch(tweets, prefs, getPath()),
-                        request, cfg, prefs, rss=rss)
+                        request, cfg, prefs, title, rss=rss)
      else:
        resp Http404, showError("Invalid search", cfg)

    get "/hashtag/@hash":
-      redirect(&"""/search?q={encodeUrl("#" & @"hash")}""")
+      redirect("/search?q=" & encodeUrl("#" & @"hash"))

    get "/opensearch":
      let url = getUrlPrefix(cfg) & "/search?q="
      resp Http200, {"Content-Type": "application/opensearchdescription+xml"},
-                    generateOpenSearchXML(cfg.title, cfg.hostname, url)
+                     generateOpenSearchXML(cfg.title, cfg.hostname, url)
--- a/src/routes/status.nim
+++ b/src/routes/status.nim
@@ -16,17 +16,21 @@ proc createStatusRouter*(cfg: Config) =
  router status:
    get "/@name/status/@id/?":
      cond '.' notin @"name"
-      cond not @"id".any(c => not c.isDigit)
+      let id = @"id"
+
+      if id.len > 19 or id.any(c => not c.isDigit):
+        resp Http404, showError("Invalid tweet ID", cfg)
+
      let prefs = cookiePrefs()

      # used for the infinite scroll feature
      if @"scroll".len > 0:
-        let replies = await getReplies(@"id", getCursor())
+        let replies = await getReplies(id, getCursor())
        if replies.content.len == 0:
          resp Http404, ""
        resp $renderReplies(replies, prefs, getPath())

-      let conv = await getTweet(@"id", getCursor())
+      let conv = await getTweet(id, getCursor())
      if conv == nil:
        echo "nil conv"

@@ -72,3 +76,6 @@ proc createStatusRouter*(cfg: Config) =

    get "/i/web/status/@id":
      redirect("/i/status/" & @"id")
+      
+    get "/@name/thread/@id/?":
+      redirect("/$1/status/$2" % [@"name", @"id"])
--- a/src/routes/timeline.nim
+++ b/src/routes/timeline.nim
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, strformat, sequtils, uri, options, times
+import asyncdispatch, strutils, sequtils, uri, options, times
 import jester, karax/vdom

 import router_utils
@@ -47,10 +47,10 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
  let
    timeline =
      case query.kind
-      of posts: getTimeline(userId, after)
-      of replies: getTimeline(userId, after, replies=true)
-      of media: getMediaTimeline(userId, after)
-      else: getSearch[Tweet](query, after)
+      of posts: getGraphUserTweets(userId, TimelineKind.tweets, after)
+      of replies: getGraphUserTweets(userId, TimelineKind.replies, after)
+      of media: getGraphUserTweets(userId, TimelineKind.media, after)
+      else: getGraphSearch(query, after)

    rail =
      skipIf(skipRail or query.kind == media, @[]):
@@ -64,6 +64,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
    let tweet = await getCachedTweet(user.pinnedTweet)
    if not tweet.isNil:
      tweet.pinned = true
+      tweet.user = user
      pinned = some tweet

  result = Profile(
@@ -82,7 +83,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
                   rss, after: string): Future[string] {.async.} =
  if query.fromUser.len != 1:
    let
-      timeline = await getSearch[Tweet](query, after)
+      timeline = await getGraphSearch(query, after)
      html = renderTweetSearch(timeline, prefs, getPath())
    return renderMain(html, request, cfg, prefs, "Multi", rss=rss)

@@ -102,7 +103,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
 template respTimeline*(timeline: typed) =
  let t = timeline
  if t.len == 0:
-    resp Http404, showError(&"""User "{@"name"}" not found""", cfg)
+    resp Http404, showError("User \"" & @"name" & "\" not found", cfg)
  resp t

 template respUserId*() =
@@ -123,7 +124,7 @@ proc createTimelineRouter*(cfg: Config) =

    get "/@name/?@tab?/?":
      cond '.' notin @"name"
-      cond @"name" notin ["pic", "gif", "video"]
+      cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]
      cond @"tab" in ["with_replies", "media", "search", ""]
      let
        prefs = cookiePrefs()
@@ -137,7 +138,7 @@ proc createTimelineRouter*(cfg: Config) =
      # used for the infinite scroll feature
      if @"scroll".len > 0:
        if query.fromUser.len != 1:
-          var timeline = await getSearch[Tweet](query, after)
+          var timeline = await getGraphSearch(query, after)
          if timeline.content.len == 0: resp Http404
          timeline.beginning = true
          resp $renderTweetSearch(timeline, prefs, getPath())
--- a/src/sass/profile/card.scss
+++ b/src/sass/profile/card.scss
@@ -73,9 +73,9 @@
        }
    }

-    .profile-joindate, .profile-location, profile-website {
+    .profile-joindate, .profile-location, .profile-website {
        color: var(--fg_faded);
-        margin: 2px 0;
+        margin: 1px 0;
        width: 100%;
    }
 }
--- a/src/sass/tweet/_base.scss
+++ b/src/sass/tweet/_base.scss
@@ -100,6 +100,7 @@
 .avatar {
    &.round {
        border-radius: 50%;
+        -webkit-user-select: none;
    }
    
    &.mini {
@@ -137,7 +138,6 @@
    }
 }

-
 .attribution {
    display: flex;
    pointer-events: all;
@@ -200,6 +200,7 @@

 .tweet-stats {
    margin-bottom: -3px;
+    -webkit-user-select: none;
 }

 .tweet-stat {
@@ -231,6 +232,7 @@
    left: 0;
    top: 0;
    position: absolute;
+    -webkit-user-select: none;

    &:hover {
        background-color: var(--bg_hover);
--- a/src/sass/tweet/thread.scss
+++ b/src/sass/tweet/thread.scss
@@ -23,7 +23,6 @@
    font-size: 18px;
 }

-
@media(max-width: 600px) {
    .main-tweet .tweet-content {
        font-size: 16px;
--- a/src/sass/tweet/video.scss
+++ b/src/sass/tweet/video.scss
@@ -3,7 +3,7 @@

 video {
    max-height: 100%;
-    max-width: 100%;
+    width: 100%;
 }

 .gallery-video {
--- a/src/tokens.nim
+++ b/src/tokens.nim
@@ -1,8 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import asyncdispatch, httpclient, times, sequtils, json, random
 import strutils, tables
-import zippy
-import types, consts, http_pool
+import types, consts

 const
  maxConcurrentReqs = 5  # max requests at a time per token, to avoid race conditions
@@ -11,11 +10,12 @@ const
  failDelay = initDuration(minutes=30)

 var
-  clientPool: HttpPool
  tokenPool: seq[Token]
  lastFailed: Time
  enableLogging = false

+let headers = newHttpHeaders({"authorization": auth})
+
 template log(str) =
  if enableLogging: echo "[tokens] ", str

@@ -41,9 +41,12 @@ proc getPoolJson*(): JsonNode =
      let
        maxReqs =
          case api
-          of Api.listMembers, Api.listBySlug, Api.list, Api.userRestId: 500
          of Api.timeline: 187
-          else: 180
+          of Api.listMembers, Api.listBySlug, Api.list, Api.listTweets,
+             Api.userTweets, Api.userTweetsAndReplies, Api.userMedia,
+             Api.userRestId, Api.userScreenName,
+             Api.tweetDetail, Api.tweetResult, Api.search: 500
+          of Api.userSearch: 900
        reqs = maxReqs - token.apis[api].remaining

      reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
@@ -64,18 +67,12 @@ proc fetchToken(): Future[Token] {.async.} =
  if getTime() - lastFailed < failDelay:
    raise rateLimitError()

-  let headers = newHttpHeaders({
-    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-    "accept-encoding": "gzip",
-    "accept-language": "en-US,en;q=0.5",
-    "connection": "keep-alive",
-    "authorization": auth
-  })
+  let client = newAsyncHttpClient(headers=headers)

  try:
    let
-      resp = clientPool.use(headers): await c.postContent(activate)
-      tokNode = parseJson(uncompress(resp))["guest_token"]
+      resp = await client.postContent(activate)
+      tokNode = parseJson(resp)["guest_token"]
      tok = tokNode.getStr($(tokNode.getInt))
      time = getTime()

@@ -85,6 +82,8 @@ proc fetchToken(): Future[Token] {.async.} =
    if "Try again" notin e.msg:
      echo "[tokens] fetching tokens paused, resuming in 30 minutes"
      lastFailed = getTime()
+  finally:
+    client.close()

 proc expired(token: Token): bool =
  let time = getTime()
@@ -157,7 +156,6 @@ proc poolTokens*(amount: int) {.async.} =
      tokenPool.add newToken

 proc initTokenPool*(cfg: Config) {.async.} =
-  clientPool = HttpPool()
  enableLogging = cfg.enableDebug

  while true:
--- a/src/types.nim
+++ b/src/types.nim
@@ -7,17 +7,28 @@ genPrefsType()
 type
  RateLimitError* = object of CatchableError
  InternalError* = object of CatchableError
+  BadClientError* = object of CatchableError
+
+  TimelineKind* {.pure.} = enum
+    tweets
+    replies
+    media

  Api* {.pure.} = enum
-    userShow
+    tweetDetail
+    tweetResult
    timeline
    search
-    tweet
+    userSearch
    list
    listBySlug
    listMembers
+    listTweets
    userRestId
-    status
+    userScreenName
+    userTweets
+    userTweetsAndReplies
+    userMedia

  RateLimit* = object
    remaining*: int
@@ -34,17 +45,22 @@ type
    null = 0
    noUserMatches = 17
    protectedUser = 22
+    missingParams = 25
    couldntAuth = 32
    doesntExist = 34
+    invalidParam = 47
    userNotFound = 50
    suspended = 63
    rateLimited = 88
    invalidToken = 89
    listIdOrSlug = 112
    tweetNotFound = 144
+    tweetNotAuthorized = 179
    forbidden = 200
    badToken = 239
    noCsrf = 353
+    tweetUnavailable = 421
+    tweetCensored = 422

  User* = object
    id*: string
@@ -145,6 +161,7 @@ type
    imageDirectMessage = "image_direct_message"
    audiospace = "audiospace"
    newsletterPublication = "newsletter_publication"
+    hidden
    unknown
    
  Card* = object
@@ -175,6 +192,7 @@ type
    available*: bool
    tombstone*: string
    location*: string
+    # Unused, needed for backwards compat
    source*: string
    stats*: TweetStats
    retweet*: Option[Tweet]
--- a/src/views/embed.nim
+++ b/src/views/embed.nim
@@ -15,7 +15,8 @@ proc renderVideoEmbed*(tweet: Tweet; cfg: Config; req: Request): string =
  let node = buildHtml(html(lang="en")):
    renderHead(prefs, cfg, req, video=vidUrl, images=(@[thumb]))

-    tdiv(class="embed-video"):
-      renderVideo(get(tweet.video), prefs, "")
+    body:
+      tdiv(class="embed-video"):
+        renderVideo(get(tweet.video), prefs, "")

  result = doctype & $node
--- a/src/views/general.nim
+++ b/src/views/general.nim
@@ -81,7 +81,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";

    title:
      if titleText.len > 0:
-        text &"{titleText}|{cfg.title}"
+        text titleText & " | " & cfg.title
      else:
        text cfg.title

@@ -98,9 +98,8 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
      link(rel="preload", type="image/png", href=bannerUrl, `as`="image")

    for url in images:
-      let suffix = if "400x400" in url or url.endsWith("placeholder.png"): ""
-                   else: "?name=small"
-      let preloadUrl = getPicUrl(url & suffix)
+      let preloadUrl = if "400x400" in url: getPicUrl(url)
+                       else: getSmallPic(url)
      link(rel="preload", type="image/png", href=preloadUrl, `as`="image")

      let image = getUrlPrefix(cfg) & getPicUrl(url)
--- a/src/views/profile.nim
+++ b/src/views/profile.nim
@@ -50,7 +50,7 @@ proc renderUserCard*(user: User; prefs: Prefs): VNode =
          span:
            let url = replaceUrls(user.website, prefs)
            icon "link"
-            a(href=url): text shortLink(url)
+            a(href=url): text url.shortLink

      tdiv(class="profile-joindate"):
        span(title=getJoinDateFull(user)):
@@ -108,7 +108,7 @@ proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode =
        renderBanner(profile.user.banner)

    let sticky = if prefs.stickyProfile: " sticky" else: ""
-    tdiv(class=(&"profile-tab{sticky}")):
+    tdiv(class=("profile-tab" & sticky)):
      renderUserCard(profile.user, prefs)
      if profile.photoRail.len > 0:
        renderPhotoRail(profile)
--- a/src/views/renderutils.nim
+++ b/src/views/renderutils.nim
@@ -3,6 +3,14 @@ import strutils, strformat
 import karax/[karaxdsl, vdom, vstyles]
 import ".."/[types, utils]

+const smallWebp* = "?name=small&format=webp"
+
+proc getSmallPic*(url: string): string =
+  result = url
+  if "?" notin url and not url.endsWith("placeholder.png"):
+    result &= smallWebp
+  result = getPicUrl(result)
+
 proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
  var c = "icon-" & icon
  if class.len > 0: c = &"{c} {class}"
@@ -51,29 +59,23 @@ proc buttonReferer*(action, text, path: string; class=""; `method`="post"): VNod
 proc genCheckbox*(pref, label: string; state: bool): VNode =
  buildHtml(label(class="pref-group checkbox-container")):
    text label
-    if state: input(name=pref, `type`="checkbox", checked="")
-    else: input(name=pref, `type`="checkbox")
+    input(name=pref, `type`="checkbox", checked=state)
    span(class="checkbox")

-proc genInput*(pref, label, state, placeholder: string; class=""): VNode =
+proc genInput*(pref, label, state, placeholder: string; class=""; autofocus=true): VNode =
  let p = placeholder
  buildHtml(tdiv(class=("pref-group pref-input " & class))):
    if label.len > 0:
      label(`for`=pref): text label
-    if state.len == 0:
-      input(name=pref, `type`="text", placeholder=p, value=state, autofocus="")
-    else:
-      input(name=pref, `type`="text", placeholder=p, value=state)
+    input(name=pref, `type`="text", placeholder=p, value=state, autofocus=(autofocus and state.len == 0))

 proc genSelect*(pref, label, state: string; options: seq[string]): VNode =
  buildHtml(tdiv(class="pref-group pref-input")):
    label(`for`=pref): text label
    select(name=pref):
      for opt in options:
-        if opt == state:
-          option(value=opt, selected=""): text opt
-        else:
-          option(value=opt): text opt
+        option(value=opt, selected=(opt == state)):
+          text opt

 proc genDate*(pref, state: string): VNode =
  buildHtml(span(class="date-input")):
@@ -85,12 +87,9 @@ proc genImg*(url: string; class=""): VNode =
    img(src=getPicUrl(url), class=class, alt="", loading="lazy", decoding="async")

 proc getTabClass*(query: Query; tab: QueryKind): string =
-  result = "tab-item"
-  if query.kind == tab:
-    result &= " active"
+  if query.kind == tab: "tab-item active"
+  else: "tab-item"

 proc getAvatarClass*(prefs: Prefs): string =
-  if prefs.squareAvatars:
-    "avatar"
-  else:
-    "avatar round"
+  if prefs.squareAvatars: "avatar"
+  else: "avatar round"
--- a/src/views/search.nim
+++ b/src/views/search.nim
@@ -63,12 +63,10 @@ proc renderSearchPanel*(query: Query): VNode =
    hiddenField("f", "tweets")
    genInput("q", "", query.text, "Enter search...", class="pref-inline")
    button(`type`="submit"): icon "search"
-    if isPanelOpen(query):
-      input(id="search-panel-toggle", `type`="checkbox", checked="")
-    else:
-      input(id="search-panel-toggle", `type`="checkbox")
-    label(`for`="search-panel-toggle"):
-      icon "down"
+
+    input(id="search-panel-toggle", `type`="checkbox", checked=isPanelOpen(query))
+    label(`for`="search-panel-toggle"): icon "down"
+
    tdiv(class="search-panel"):
      for f in @["filter", "exclude"]:
        span(class="search-title"): text capitalize(f)
@@ -88,7 +86,7 @@ proc renderSearchPanel*(query: Query): VNode =
            genDate("until", query.until)
        tdiv:
          span(class="search-title"): text "Near"
-          genInput("near", "", query.near, placeholder="Location...")
+          genInput("near", "", query.near, "Location...", autofocus=false)

 proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string;
                        pinned=none(Tweet)): VNode =
--- a/src/views/tweet.nim
+++ b/src/views/tweet.nim
@@ -7,11 +7,7 @@ import renderutils
 import ".."/[types, utils, formatters]
 import general

-proc getSmallPic(url: string): string =
-  result = url
-  if "?" notin url and not url.endsWith("placeholder.png"):
-    result &= "?name=small"
-  result = getPicUrl(result)
+const doctype = "<!DOCTYPE html>\n"

 proc renderMiniAvatar(user: User; prefs: Prefs): VNode =
  let url = getPicUrl(user.getUserPic("_mini"))
@@ -57,9 +53,8 @@ proc renderAlbum(tweet: Tweet): VNode =
          tdiv(class="attachment image"):
            let
              named = "name=" in photo
-              orig = photo
-              small = if named: photo else: photo & "?name=small"
-            a(href=getOrigPicUrl(orig), class="still-image", target="_blank"):
+              small = if named: photo else: photo & smallWebp
+            a(href=getOrigPicUrl(photo), class="still-image", target="_blank"):
              genImg(small)

 proc isPlaybackEnabled(prefs: Prefs; playbackType: VideoType): bool =
@@ -106,12 +101,10 @@ proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode =
                     else: vidUrl
          case playbackType
          of mp4:
-            if prefs.muteVideos:
-              video(src=source, poster=thumb, controls="", preload="none", muted=""):
-            else:
-              video(src=source, poster=thumb, controls="", preload="none"):
+            video(poster=thumb, controls="", muted=prefs.muteVideos):
+              source(src=source, `type`="video/mp4")
          of m3u8, vmap:
-            video(poster=thumb, data-url=source, data-autoload="false")
+            video(poster=thumb, data-url=source, data-autoload="false", muted=prefs.muteVideos)
            verbatim "<div class=\"video-overlay\" onclick=\"playVideo(this)\">"
            tdiv(class="overlay-circle"): span(class="overlay-triangle")
            verbatim "</div>"
@@ -132,12 +125,9 @@ proc renderGif(gif: Gif; prefs: Prefs): VNode =
  buildHtml(tdiv(class="attachments media-gif")):
    tdiv(class="gallery-gif", style={maxHeight: "unset"}):
      tdiv(class="attachment"):
-        let thumb = getSmallPic(gif.thumb)
-        let url = getPicUrl(gif.url)
-        if prefs.autoplayGifs:
-          video(src=url, class="gif", poster=thumb, controls="", muted="", loop="", playsinline="", autoplay="")
-        else:
-          video(src=url, class="gif", poster=thumb, controls="", muted="", loop="", playsinline="")
+        video(class="gif", poster=getSmallPic(gif.thumb), autoplay=prefs.autoplayGifs,
+              controls="", muted="", loop=""):
+          source(src=getPicUrl(gif.url), `type`="video/mp4")

 proc renderPoll(poll: Poll): VNode =
  buildHtml(tdiv(class="poll")):
@@ -331,7 +321,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
      if tweet.attribution.isSome:
        renderAttribution(tweet.attribution.get(), prefs)

-      if tweet.card.isSome:
+      if tweet.card.isSome and tweet.card.get().kind != hidden:
        renderCard(tweet.card.get(), prefs, path)

      if tweet.photos.len > 0:
@@ -350,7 +340,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
        renderQuote(tweet.quote.get(), prefs, path)

      if mainTweet:
-        p(class="tweet-published"): text &"{getTime(tweet)} · {tweet.source}"
+        p(class="tweet-published"): text &"{getTime(tweet)}"

      if tweet.mediaTags.len > 0:
        renderMediaTags(tweet.mediaTags)
@@ -362,7 +352,12 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
        a(class="show-thread", href=("/i/status/" & $tweet.threadId)):
          text "Show this thread"

-proc renderTweetEmbed*(tweet: Tweet; path: string; prefs: Prefs; cfg: Config; req: Request): VNode =
-  buildHtml(tdiv(class="tweet-embed")):
+proc renderTweetEmbed*(tweet: Tweet; path: string; prefs: Prefs; cfg: Config; req: Request): string =
+  let node = buildHtml(html(lang="en")):
    renderHead(prefs, cfg, req)
-    renderTweet(tweet, prefs, path, mainTweet=true)
+
+    body:
+      tdiv(class="tweet-embed"):
+        renderTweet(tweet, prefs, path, mainTweet=true)
+
+  result = doctype & $node