Update endpoints, fix parser, remove quotes stat

src/api.nim

@@ -1,16 +1,23 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, httpclient, uri, strutils, sequtils, sugar
+import asyncdispatch, httpclient, uri, strutils, sequtils, sugar, tables
 import packedjson
 import types, query, formatters, consts, apiutils, parser
 import experimental/parser as newParser
 
+# Helper to generate params object for GraphQL requests
+proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] =
+  result.add ("variables", variables)
+  result.add ("features", gqlFeatures)
+  if fieldToggles.len > 0:
+    result.add ("fieldToggles", fieldToggles)
+
 proc mediaUrl(id: string; cursor: string): SessionAwareUrl =
   let
     cookieVariables = userMediaVariables % [id, cursor]
     oauthVariables = restIdVariables % [id, cursor]
   result = SessionAwareUrl(
-    cookieUrl: graphUserMedia ? {"variables": cookieVariables, "features": gqlFeatures},
-    oauthUrl: graphUserMediaV2 ? {"variables": oauthVariables, "features": gqlFeatures}
+    cookieUrl: graphUserMedia ? genParams(cookieVariables),
+    oauthUrl: graphUserMediaV2 ? genParams(oauthVariables)
   )
 
 proc userTweetsUrl(id: string; cursor: string): SessionAwareUrl =
@@ -18,17 +25,19 @@ proc userTweetsUrl(id: string; cursor: string): SessionAwareUrl =
     cookieVariables = userTweetsVariables % [id, cursor]
     oauthVariables = restIdVariables % [id, cursor]
   result = SessionAwareUrl(
-    cookieUrl: graphUserTweets ? {"variables": cookieVariables, "features": gqlFeatures, "fieldToggles": fieldToggles},
-    oauthUrl: graphUserTweetsV2 ? {"variables": oauthVariables, "features": gqlFeatures}
+    # cookieUrl: graphUserTweets ? genParams(cookieVariables, fieldToggles),
+    oauthUrl: graphUserTweetsV2 ? genParams(oauthVariables)
   )
+  # might change this in the future pending testing
+  result.cookieUrl = result.oauthUrl
 
 proc userTweetsAndRepliesUrl(id: string; cursor: string): SessionAwareUrl =
   let
     cookieVariables = userTweetsAndRepliesVariables % [id, cursor]
     oauthVariables = restIdVariables % [id, cursor]
   result = SessionAwareUrl(
-    cookieUrl: graphUserTweetsAndReplies ? {"variables": cookieVariables, "features": gqlFeatures, "fieldToggles": fieldToggles},
-    oauthUrl: graphUserTweetsAndRepliesV2 ? {"variables": oauthVariables, "features": gqlFeatures}
+    cookieUrl: graphUserTweetsAndReplies ? genParams(cookieVariables, fieldToggles),
+    oauthUrl: graphUserTweetsAndRepliesV2 ? genParams(oauthVariables)
   )
 
 proc tweetDetailUrl(id: string; cursor: string): SessionAwareUrl =
@@ -36,24 +45,22 @@ proc tweetDetailUrl(id: string; cursor: string): SessionAwareUrl =
     cookieVariables = tweetDetailVariables % [id, cursor]
     oauthVariables = tweetVariables % [id, cursor]
   result = SessionAwareUrl(
-    cookieUrl: graphTweetDetail ? {"variables": cookieVariables, "features": gqlFeatures, "fieldToggles": tweetDetailFieldToggles},
-    oauthUrl: graphTweet ? {"variables": oauthVariables, "features": gqlFeatures}
+    cookieUrl: graphTweetDetail ? genParams(cookieVariables, tweetDetailFieldToggles),
+    oauthUrl: graphTweet ? genParams(oauthVariables)
   )
 
 proc getGraphUser*(username: string): Future[User] {.async.} =
   if username.len == 0: return
   let
-    variables = """{"screen_name": "$1"}""" % username
-    params = {"variables": variables, "features": gqlFeatures}
-    js = await fetchRaw(graphUser ? params, Api.userScreenName)
+    url = graphUser ? genParams("""{"screen_name": "$1"}""" % username)
+    js = await fetchRaw(url, Api.userScreenName)
   result = parseGraphUser(js)
 
 proc getGraphUserById*(id: string): Future[User] {.async.} =
   if id.len == 0 or id.any(c => not c.isDigit): return
   let
-    variables = """{"rest_id": "$1"}""" % id
-    params = {"variables": variables, "features": gqlFeatures}
-    js = await fetchRaw(graphUserById ? params, Api.userRestId)
+    url = graphUserById ? genParams("""{"rest_id": "$1"}""" % id)
+    js = await fetchRaw(url, Api.userRestId)
   result = parseGraphUser(js)
 
 proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} =
@@ -73,23 +80,18 @@ proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
   if id.len == 0: return
   let
     cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = restIdVariables % [id, cursor]
-    params = {"variables": variables, "features": gqlFeatures}
-    js = await fetch(graphListTweets ? params, Api.listTweets)
-  result = parseGraphTimeline(js, after).tweets
+    url = graphListTweets ? genParams(restIdVariables % [id, cursor])
+  result = parseGraphTimeline(await fetch(url, Api.listTweets), after).tweets
 
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
   let
     variables = %*{"screenName": name, "listSlug": list}
-    params = {"variables": $variables, "features": gqlFeatures}
-    url = graphListBySlug ? params
+    url = graphListBySlug ? genParams($variables)
   result = parseGraphList(await fetch(url, Api.listBySlug))
 
 proc getGraphList*(id: string): Future[List] {.async.} =
   let
-    variables = """{"listId": "$1"}""" % id
-    params = {"variables": variables, "features": gqlFeatures}
-    url = graphListById ? params
+    url = graphListById ? genParams("""{"listId": "$1"}""" % id)
   result = parseGraphList(await fetch(url, Api.list))
 
 proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
@@ -104,7 +106,7 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
     }
   if after.len > 0:
     variables["cursor"] = % after
-  let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures}
+  let url = graphListMembers ? genParams($variables)
   result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
 
 proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
@@ -139,6 +141,7 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
   var
     variables = %*{
       "rawQuery": q,
+      "query_source": "typedQuery",
       "count": 20,
       "product": "Latest",
       "withDownvotePerspective": false,
@@ -147,7 +150,7 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
     }
   if after.len > 0:
     variables["cursor"] = % after
-  let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
+  let url = graphSearchTimeline ? genParams($variables)
   result = parseGraphSearch[Tweets](await fetch(url, Api.search), after)
   result.query = query
 
@@ -158,6 +161,7 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
   var
     variables = %*{
       "rawQuery": query.text,
+      "query_source": "typedQuery",
       "count": 20,
       "product": "People",
       "withDownvotePerspective": false,
@@ -168,7 +172,7 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
     variables["cursor"] = % after
     result.beginning = false
 
-  let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
+  let url = graphSearchTimeline ? genParams($variables)
   result = parseGraphSearch[User](await fetch(url, Api.search), after)
   result.query = query
 

src/consts.nim

@@ -7,26 +7,29 @@ const
 
   gql = parseUri("https://api.x.com") / "graphql"
 
-  graphUser* = gql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery"
-  graphUserById* = gql / "oPppcargziU1uDQHAUmH-A/UserResultByIdQuery"
-  graphUserTweetsV2* = gql / "JLApJKFY0MxGTzCoK6ps8Q/UserWithProfileTweetsQueryV2"
-  graphUserTweetsAndRepliesV2* = gql / "Y86LQY7KMvxn5tu3hFTyPg/UserWithProfileTweetsAndRepliesQueryV2"
+  graphUser* = gql / "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery"
+  graphUserById* = gql / "VN33vKXrPT7p35DgNR27aw/UserResultByIdQuery"
+  graphUserTweetsV2* = gql / "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2"
+  graphUserTweetsAndRepliesV2* = gql / "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2"
   graphUserTweets* = gql / "oRJs8SLCRNRbQzuZG93_oA/UserTweets"
   graphUserTweetsAndReplies* = gql / "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies"
   graphUserMedia* = gql / "36oKqyQ7E_9CmtONGjJRsA/UserMedia"
-  graphUserMediaV2* = gql / "PDfFf8hGeJvUCiTyWtw4wQ/MediaTimelineV2"
-  graphTweet* = gql / "Vorskcd2tZ-tc4Gx3zbk4Q/ConversationTimelineV2"
+  graphUserMediaV2* = gql / "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2"
+  graphTweet* = gql / "Y4Erk_-0hObvLpz0Iw3bzA/ConversationTimeline"
   graphTweetDetail* = gql / "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail"
-  graphTweetResult* = gql / "sITyJdhRPpvpEjg4waUmTA/TweetResultByIdQuery"
-  graphSearchTimeline* = gql / "7r8ibjHuK3MWUyzkzHNMYQ/SearchTimeline"
+  graphTweetResult* = gql / "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery"
+  graphSearchTimeline* = gql / "bshMIjqDk8LTXTq4w91WKw/SearchTimeline"
   graphListById* = gql / "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId"
   graphListBySlug* = gql / "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug"
   graphListMembers* = gql / "fuVHh5-gFn8zDBBxb8wOMA/ListMembers"
-  graphListTweets* = gql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline"
+  graphListTweets* = gql / "VQf8_XQynI3WzH6xopOMMQ/ListTimeline"
 
   gqlFeatures* = """{
+  "android_ad_formats_media_component_render_overlay_enabled": false,
   "android_graphql_skip_api_media_color_palette": false,
+  "android_professional_link_spotlight_display_enabled": false,
   "blue_business_profile_image_shape_enabled": false,
+  "commerce_android_shop_module_enabled": false,
   "creator_subscriptions_subscription_count_enabled": false,
   "creator_subscriptions_tweet_preview_api_enabled": true,
   "freedom_of_speech_not_reach_fetch_enabled": true,
@@ -36,8 +39,9 @@ const
   "interactive_text_enabled": false,
   "longform_notetweets_consumption_enabled": true,
   "longform_notetweets_inline_media_enabled": true,
-  "longform_notetweets_richtext_consumption_enabled": true,
   "longform_notetweets_rich_text_read_enabled": true,
+  "longform_notetweets_richtext_consumption_enabled": true,
+  "mobile_app_spotlight_module_enabled": false,
   "responsive_web_edit_tweet_api_enabled": true,
   "responsive_web_enhance_cards_enabled": false,
   "responsive_web_graphql_exclude_directive_enabled": true,
@@ -46,6 +50,7 @@ const
   "responsive_web_media_download_video_enabled": false,
   "responsive_web_text_conversations_enabled": false,
   "responsive_web_twitter_article_tweet_consumption_enabled": true,
+  "unified_cards_destination_url_params_enabled": false,
   "responsive_web_twitter_blue_verified_badge_is_enabled": true,
   "rweb_lists_timeline_redesign_enabled": true,
   "spaces_2022_h2_clipping": true,
@@ -86,11 +91,17 @@ const
   "payments_enabled": false,
   "responsive_web_profile_redirect_enabled": false,
   "responsive_web_grok_show_grok_translated_post": false,
-  "responsive_web_grok_community_note_auto_translation_is_enabled": false
+  "responsive_web_grok_community_note_auto_translation_is_enabled": false,
+  "profile_label_improvements_pcf_label_in_profile_enabled": false,
+  "grok_android_analyze_trend_fetch_enabled": false,
+  "grok_translations_community_note_auto_translation_is_enabled": false,
+  "grok_translations_post_auto_translation_is_enabled": false,
+  "grok_translations_community_note_translation_is_enabled": false,
+  "grok_translations_timeline_user_bio_auto_translation_is_enabled": false
 }""".replace(" ", "").replace("\n", "")
 
   tweetVariables* = """{
-  "focalTweetId": "$1",
+  "postId": "$1",
   $2
   "includeHasBirdwatchNotes": false,
   "includePromotedContent": false,

src/parser.nim

@@ -1,10 +1,10 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, options, times, math
+import strutils, options, times, math, tables
 import packedjson, packedjson/deserialiser
 import types, parserutils, utils
 import experimental/parser/unifiedcard
 
-proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet
+proc parseGraphTweet(js: JsonNode): Tweet
 
 proc parseUser(js: JsonNode; id=""): User =
   if js.isNull: return
@@ -46,6 +46,9 @@ proc parseGraphUser(js: JsonNode): User =
 
   result = parseUser(user{"legacy"}, user{"rest_id"}.getStr)
 
+  if result.verifiedType == none and user{"is_blue_verified"}.getBool(false):
+    result.verifiedType = blue
+
   # fallback to support UserMedia/recent GraphQL updates
   if result.username.len == 0:
     result.username = user{"core", "screen_name"}.getStr
@@ -95,16 +98,24 @@ proc parsePoll(js: JsonNode): Poll =
   result.leader = result.values.find(max(result.values))
   result.votes = result.values.sum
 
-proc parseGif(js: JsonNode): Gif =
-  result = Gif(
-    url: js{"video_info", "variants"}[0]{"url"}.getImageStr,
-    thumb: js{"media_url_https"}.getImageStr
-  )
+proc parseVideoVariants(variants: JsonNode): seq[VideoVariant] =
+  result = @[]
+  for v in variants:
+    let
+      url = v{"url"}.getStr
+      contentType = parseEnum[VideoType](v{"content_type"}.getStr("video/mp4"))
+      bitrate = v{"bit_rate"}.getInt(v{"bitrate"}.getInt(0))
+
+    result.add VideoVariant(
+      contentType: contentType,
+      bitrate: bitrate,
+      url: url,
+      resolution: if contentType == mp4: getMp4Resolution(url) else: 0
+    )
 
 proc parseVideo(js: JsonNode): Video =
   result = Video(
     thumb: js{"media_url_https"}.getImageStr,
-    views: getVideoViewCount(js),
     available: true,
     title: js{"ext_alt_text"}.getStr,
     durationMs: js{"video_info", "duration_millis"}.getInt
@@ -121,17 +132,62 @@ proc parseVideo(js: JsonNode): Video =
   with description, js{"additional_media_info", "description"}:
     result.description = description.getStr
 
-  for v in js{"video_info", "variants"}:
-    let
-      contentType = parseEnum[VideoType](v{"content_type"}.getStr("summary"))
-      url = v{"url"}.getStr
+  result.variants = parseVideoVariants(js{"video_info", "variants"})
 
-    result.variants.add VideoVariant(
-      contentType: contentType,
-      bitrate: v{"bitrate"}.getInt,
-      url: url,
-      resolution: if contentType == mp4: getMp4Resolution(url) else: 0
-    )
+proc parseLegacyMediaEntities(js: JsonNode; result: var Tweet) =
+  with jsMedia, js{"extended_entities", "media"}:
+    for m in jsMedia:
+      case m.getTypeName:
+      of "photo":
+        result.photos.add m{"media_url_https"}.getImageStr
+      of "video":
+        result.video = some(parseVideo(m))
+        with user, m{"additional_media_info", "source_user"}:
+          if user{"id"}.getInt > 0:
+            result.attribution = some(parseUser(user))
+          else:
+            result.attribution = some(parseGraphUser(user))
+      of "animated_gif":
+        result.gif = some Gif(
+          url: m{"video_info", "variants"}[0]{"url"}.getImageStr,
+          thumb: m{"media_url_https"}.getImageStr
+        )
+      else: discard
+
+      with url, m{"url"}:
+        if result.text.endsWith(url.getStr):
+          result.text.removeSuffix(url.getStr)
+          result.text = result.text.strip()
+
+proc parseMediaEntities(js: JsonNode; result: var Tweet) =
+  with mediaEntities, js{"media_entities"}:
+    for mediaEntity in mediaEntities:
+      with mediaInfo, mediaEntity{"media_results", "result", "media_info"}:
+        case mediaInfo.getTypeName
+        of "ApiImage":
+          result.photos.add mediaInfo{"original_img_url"}.getImageStr
+        of "ApiVideo":
+          let status = mediaEntity{"media_results", "result", "media_availability_v2", "status"}
+          result.video = some Video(
+            available: status.getStr == "Available",
+            thumb: mediaInfo{"preview_image", "original_img_url"}.getImageStr,
+            durationMs: mediaInfo{"duration_millis"}.getInt,
+            variants: parseVideoVariants(mediaInfo{"variants"})
+          )
+        of "ApiGif":
+          result.gif = some Gif(
+            url: mediaInfo{"variants"}[0]{"url"}.getImageStr,
+            thumb: mediaInfo{"preview_image", "original_img_url"}.getImageStr
+          )
+        else: discard
+
+  # Remove media URLs from text
+  with mediaList, js{"legacy", "entities", "media"}:
+    for url in mediaList:
+      let expandedUrl = url{"expanded_url"}.getStr
+      if result.text.endsWith(expandedUrl):
+        result.text.removeSuffix(expandedUrl)
+        result.text = result.text.strip()
 
 proc parsePromoVideo(js: JsonNode): Video =
   result = Video(
@@ -223,12 +279,17 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
 
 proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
   if js.isNull: return
+
+  let time =
+    if js{"created_at"}.notNull: js{"created_at"}.getTime
+    else: js{"created_at_ms"}.getTimeFromMs
+
   result = Tweet(
     id: js{"id_str"}.getId,
     threadId: js{"conversation_id_str"}.getId,
     replyId: js{"in_reply_to_status_id_str"}.getId,
     text: js{"full_text"}.getStr,
-    time: js{"created_at"}.getTime,
+    time: time,
     hasThread: js{"self_thread"}.notNull,
     available: true,
     user: User(id: js{"user_id_str"}.getStr),
@@ -236,7 +297,6 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
       replies: js{"reply_count"}.getInt,
       retweets: js{"retweet_count"}.getInt,
       likes: js{"favorite_count"}.getInt,
-      quotes: js{"quote_count"}.getInt,
       views: js{"views_count"}.getInt
     )
   )
@@ -262,6 +322,12 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
       result.retweet = some parseGraphTweet(rt)
       return
 
+  with reposts, js{"repostedStatusResults"}:
+    with rt, reposts{"result"}:
+      if "legacy" in rt:
+        result.retweet = some parseGraphTweet(rt)
+        return
+
   if jsCard.kind != JNull:
     let name = jsCard{"name"}.getStr
     if "poll" in name:
@@ -275,27 +341,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
       result.card = some parseCard(jsCard, js{"entities", "urls"})
 
   result.expandTweetEntities(js)
-
-  with jsMedia, js{"extended_entities", "media"}:
-    for m in jsMedia:
-      case m{"type"}.getStr
-      of "photo":
-        result.photos.add m{"media_url_https"}.getImageStr
-      of "video":
-        result.video = some(parseVideo(m))
-        with user, m{"additional_media_info", "source_user"}:
-          if user{"id"}.getInt > 0:
-            result.attribution = some(parseUser(user))
-          else:
-            result.attribution = some(parseGraphUser(user))
-      of "animated_gif":
-        result.gif = some(parseGif(m))
-      else: discard
-
-      with url, m{"url"}:
-        if result.text.endsWith(url.getStr):
-          result.text.removeSuffix(url.getStr)
-          result.text = result.text.strip()
+  parseLegacyMediaEntities(js, result)
 
   with jsWithheld, js{"withheld_in_countries"}:
     let withheldInCountries: seq[string] =
@@ -311,95 +357,108 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
       result.text.removeSuffix(" Learn more.")
       result.available = false
 
-proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet =
+proc parseGraphTweet(js: JsonNode): Tweet =
   if js.kind == JNull:
     return Tweet()
 
-  case js{"__typename"}.getStr
+  case js.getTypeName:
   of "TweetUnavailable":
     return Tweet()
   of "TweetTombstone":
-    with text, js{"tombstone", "richText"}:
-      return Tweet(text: text.getTombstone)
-    with text, js{"tombstone", "text"}:
+    with text, select(js{"tombstone", "richText"}, js{"tombstone", "text"}):
       return Tweet(text: text.getTombstone)
     return Tweet()
   of "TweetPreviewDisplay":
     return Tweet(text: "You're unable to view this Tweet because it's only available to the Subscribers of the account owner.")
   of "TweetWithVisibilityResults":
-    return parseGraphTweet(js{"tweet"}, isLegacy)
+    return parseGraphTweet(js{"tweet"})
   else:
     discard
 
   if not js.hasKey("legacy"):
     return Tweet()
 
-  var jsCard = copy(js{if isLegacy: "card" else: "tweet_card", "legacy"})
+  var jsCard = select(js{"card"}, js{"tweet_card"}, js{"legacy", "tweet_card"})
   if jsCard.kind != JNull:
-    var values = newJObject()
-    for val in jsCard["binding_values"]:
-      values[val["key"].getStr] = val["value"]
-    jsCard["binding_values"] = values
+    let legacyCard = jsCard{"legacy"}
+    if legacyCard.kind != JNull:
+      let bindingArray = legacyCard{"binding_values"}
+      if bindingArray.kind == JArray:
+        var bindingObj: seq[(string, JsonNode)]
+        for item in bindingArray:
+          bindingObj.add((item{"key"}.getStr, item{"value"}))
+        # Create a new card object with flattened structure
+        jsCard = %*{
+          "name": legacyCard{"name"},
+          "url": legacyCard{"url"},
+          "binding_values": %bindingObj
+        }
 
   result = parseTweet(js{"legacy"}, jsCard)
   result.id = js{"rest_id"}.getId
   result.user = parseGraphUser(js{"core"})
 
+  if result.replyId == 0:
+    result.replyId = js{"reply_to_results", "rest_id"}.getId
+
   with count, js{"views", "count"}:
     result.stats.views = count.getStr("0").parseInt
 
   with noteTweet, js{"note_tweet", "note_tweet_results", "result"}:
     result.expandNoteTweetEntities(noteTweet)
 
+  parseMediaEntities(js, result)
+
   if result.quote.isSome:
-    result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}, isLegacy))
+    result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}))
+
+  with quoted, js{"quotedPostResults", "result"}:
+    result.quote = some(parseGraphTweet(quoted))
 
 proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
-  for t in js{"content", "items"}:
-    let entryId = t{"entryId"}.getStr
+  for t in ? js{"content", "items"}:
+    let entryId = t.getEntryId
     if "cursor-showmore" in entryId:
       let cursor = t{"item", "content", "value"}
       result.thread.cursor = cursor.getStr
       result.thread.hasMore = true
     elif "tweet" in entryId and "promoted" notin entryId:
-      let
-        isLegacy = t{"item"}.hasKey("itemContent")
-        (contentKey, resultKey) = if isLegacy: ("itemContent", "tweet_results")
-                                  else: ("content", "tweetResult")
+      with tweet, t.getTweetResult("item"):
+        result.thread.content.add parseGraphTweet(tweet)
 
-      with content, t{"item", contentKey}:
-        result.thread.content.add parseGraphTweet(content{resultKey, "result"}, isLegacy)
-
-        if content{"tweetDisplayType"}.getStr == "SelfThread":
+        let tweetDisplayType = select(
+          t{"item", "content", "tweet_display_type"},
+          t{"item", "itemContent", "tweetDisplayType"}
+        )
+        if tweetDisplayType.getStr == "SelfThread":
           result.self = true
 
 proc parseGraphTweetResult*(js: JsonNode): Tweet =
   with tweet, js{"data", "tweet_result", "result"}:
-    result = parseGraphTweet(tweet, false)
+    result = parseGraphTweet(tweet)
 
 proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
   result = Conversation(replies: Result[Chain](beginning: true))
-  let
-    v2 = js{"data", "timeline_response"}.notNull
-    rootKey = if v2: "timeline_response" else: "threaded_conversation_with_injections_v2"
-    contentKey = if v2: "content" else: "itemContent"
-    resultKey = if v2: "tweetResult" else: "tweet_results"
 
-  let instructions = ? js{"data", rootKey, "instructions"}
+  let instructions = ? select(
+    js{"data", "timelineResponse", "instructions"},
+    js{"data", "timeline_response", "instructions"},
+    js{"data", "threaded_conversation_with_injections_v2", "instructions"}
+  )
   if instructions.len == 0:
     return
 
   for i in instructions:
-    let instrType = i{"type"}.getStr(i{"__typename"}.getStr)
-    if instrType == "TimelineAddEntries":
+    if i.getTypeName == "TimelineAddEntries":
       for e in i{"entries"}:
-        let entryId = e{"entryId"}.getStr
+        let entryId = e.getEntryId
         if entryId.startsWith("tweet"):
-          with tweetResult, e{"content", contentKey, resultKey, "result"}:
-            let tweet = parseGraphTweet(tweetResult, not v2)
+          let tweetResult = getTweetResult(e)
+          if tweetResult.notNull:
+            let tweet = parseGraphTweet(tweetResult)
 
             if not tweet.available:
-              tweet.id = parseBiggestInt(entryId.getId())
+              tweet.id = entryId.getId
 
             if $tweet.id == tweetId:
               result.tweet = tweet
@@ -412,67 +471,64 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
           elif thread.content.len > 0:
             result.replies.content.add thread
         elif entryId.startsWith("tombstone"):
-          let id = entryId.getId()
-          let tweet = Tweet(
-            id: parseBiggestInt(id),
-            available: false,
-            text: e{"content", contentKey, "tombstoneInfo", "richText"}.getTombstone
-          )
+          let
+            content = select(e{"content", "content"}, e{"content", "itemContent"})
+            tweet = Tweet(
+              id: entryId.getId,
+              available: false,
+              text: content{"tombstoneInfo", "richText"}.getTombstone
+            )
 
-          if id == tweetId:
+          if $tweet.id == tweetId:
             result.tweet = tweet
           else:
             result.before.content.add tweet
         elif entryId.startsWith("cursor-bottom"):
-          result.replies.bottom = e{"content", contentKey, "value"}.getStr
+          var cursorValue = select(
+            e{"content", "content", "value"},
+            e{"content", "itemContent", "value"}
+          )
+          result.replies.bottom = cursorValue.getStr
 
 proc extractTweetsFromEntry*(e: JsonNode): seq[Tweet] =
-  var tweetResult = e{"content", "itemContent", "tweet_results", "result"}
-  if tweetResult.isNull:
-    tweetResult = e{"content", "content", "tweetResult", "result"}
-
-  if tweetResult.notNull:
-    var tweet = parseGraphTweet(tweetResult, false)
+  with tweetResult, getTweetResult(e):
+    var tweet = parseGraphTweet(tweetResult)
     if not tweet.available:
-      tweet.id = parseBiggestInt(e.getEntryId())
+      tweet.id = e.getEntryId.getId
     result.add tweet
     return
 
   for item in e{"content", "items"}:
-    with tweetResult, item{"item", "itemContent", "tweet_results", "result"}:
-      var tweet = parseGraphTweet(tweetResult, false)
+    with tweetResult, item.getTweetResult("item"):
+      var tweet = parseGraphTweet(tweetResult)
       if not tweet.available:
-        tweet.id = parseBiggestInt(item{"entryId"}.getStr.getId())
+        tweet.id = item.getEntryId.getId
       result.add tweet
 
 proc parseGraphTimeline*(js: JsonNode; after=""): Profile =
   result = Profile(tweets: Timeline(beginning: after.len == 0))
 
-  let instructions =
-    if js{"data", "list"}.notNull:
-      ? js{"data", "list", "timeline_response", "timeline", "instructions"}
-    elif js{"data", "user"}.notNull:
-      ? js{"data", "user", "result", "timeline", "timeline", "instructions"}
-    else:
-      ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
-
+  let instructions = ? select(
+    js{"data", "list", "timeline_response", "timeline", "instructions"},
+    js{"data", "user", "result", "timeline", "timeline", "instructions"},
+    js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
+  )
   if instructions.len == 0:
     return
 
   for i in instructions:
-    # TimelineAddToModule instruction is used by UserMedia
     if i{"moduleItems"}.notNull:
       for item in i{"moduleItems"}:
-        with tweetResult, item{"item", "itemContent", "tweet_results", "result"}:
-          let tweet = parseGraphTweet(tweetResult, false)
+        with tweetResult, item.getTweetResult("item"):
+          let tweet = parseGraphTweet(tweetResult)
           if not tweet.available:
-            tweet.id = parseBiggestInt(item{"entryId"}.getStr.getId())
+            tweet.id = item.getEntryId.getId
           result.tweets.content.add tweet
       continue
 
     if i{"entries"}.notNull:
       for e in i{"entries"}:
-        let entryId = e{"entryId"}.getStr
+        let entryId = e.getEntryId
         if entryId.startsWith("tweet") or entryId.startsWith("profile-grid"):
           for tweet in extractTweetsFromEntry(e):
             result.tweets.content.add tweet
@@ -483,8 +539,7 @@ proc parseGraphTimeline*(js: JsonNode; after=""): Profile =
           result.tweets.bottom = e{"content", "value"}.getStr
 
     if after.len == 0:
-      let instrType = i{"type"}.getStr(i{"__typename"}.getStr)
-      if instrType == "TimelinePinEntry":
+      if i.getTypeName == "TimelinePinEntry":
         let tweets = extractTweetsFromEntry(i{"entry"})
         if tweets.len > 0:
           var tweet = tweets[0]
@@ -494,23 +549,20 @@ proc parseGraphTimeline*(js: JsonNode; after=""): Profile =
 proc parseGraphPhotoRail*(js: JsonNode): PhotoRail =
   result = @[]
 
-  let instructions =
-    if js{"data", "user"}.notNull:
-      ? js{"data", "user", "result", "timeline", "timeline", "instructions"}
-    else:
-      ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
-
+  let instructions = select(
+    js{"data", "user", "result", "timeline", "timeline", "instructions"},
+    js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
+  )
   if instructions.len == 0:
     return
 
   for i in instructions:
-    # TimelineAddToModule instruction is used by MediaTimelineV2
     if i{"moduleItems"}.notNull:
       for item in i{"moduleItems"}:
-        with tweetResult, item{"item", "itemContent", "tweet_results", "result"}:
-          let t = parseGraphTweet(tweetResult, false)
+        with tweetResult, item.getTweetResult("item"):
+          let t = parseGraphTweet(tweetResult)
           if not t.available:
-            t.id = parseBiggestInt(item{"entryId"}.getStr.getId())
+            t.id = item.getEntryId.getId
 
           let photo = extractGalleryPhoto(t)
           if photo.url.len > 0:
@@ -520,12 +572,11 @@ proc parseGraphPhotoRail*(js: JsonNode): PhotoRail =
             return
       continue
 
-    let instrType = i{"type"}.getStr(i{"__typename"}.getStr)
-    if instrType != "TimelineAddEntries":
+    if i.getTypeName != "TimelineAddEntries":
       continue
 
     for e in i{"entries"}:
-      let entryId = e{"entryId"}.getStr
+      let entryId = e.getEntryId
       if entryId.startsWith("tweet") or entryId.startsWith("profile-grid"):
         for t in extractTweetsFromEntry(e):
           let photo = extractGalleryPhoto(t)
@@ -538,21 +589,24 @@ proc parseGraphPhotoRail*(js: JsonNode): PhotoRail =
 proc parseGraphSearch*[T: User | Tweets](js: JsonNode; after=""): Result[T] =
   result = Result[T](beginning: after.len == 0)
 
-  let instructions = js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"}
+  let instructions = select(
+    js{"data", "search", "timeline_response", "timeline", "instructions"},
+    js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"}
+  )
   if instructions.len == 0:
     return
 
   for instruction in instructions:
-    let typ = instruction{"type"}.getStr
+    let typ = getTypeName(instruction)
     if typ == "TimelineAddEntries":
       for e in instruction{"entries"}:
-        let entryId = e{"entryId"}.getStr
+        let entryId = e.getEntryId
         when T is Tweets:
           if entryId.startsWith("tweet"):
-            with tweetRes, e{"content", "itemContent", "tweet_results", "result"}:
+            with tweetRes, getTweetResult(e):
               let tweet = parseGraphTweet(tweetRes)
               if not tweet.available:
-                tweet.id = parseBiggestInt(entryId.getId())
+                tweet.id = entryId.getId
               result.content.add tweet
         elif T is User:
           if entryId.startsWith("user"):

src/parserutils.nim

@@ -36,6 +36,12 @@ template `?`*(js: JsonNode): untyped =
   if j.isNull: return
   j
 
+template select*(a, b: JsonNode): untyped =
+  if a.notNull: a else: b
+
+template select*(a, b, c: JsonNode): untyped =
+  if a.notNull: a elif b.notNull: b else: c
+
 template with*(ident, value, body): untyped =
   if true:
     let ident {.inject.} = value
@@ -54,6 +60,20 @@ template getError*(js: JsonNode): Error =
   if js.kind != JArray or js.len == 0: null
   else: Error(js[0]{"code"}.getInt)
 
+proc getTweetResult*(js: JsonNode; root="content"): JsonNode =
+  select(
+    js{root, "content", "tweet_results", "result"},
+    js{root, "itemContent", "tweet_results", "result"},
+    js{root, "content", "tweetResult", "result"}
+  )
+
+template getTypeName*(js: JsonNode): string =
+  js{"__typename"}.getStr(js{"type"}.getStr)
+
+template getEntryId*(e: JsonNode): string =
+  e{"entryId"}.getStr(e{"entry_id"}.getStr)
+
+
 template parseTime(time: string; f: static string; flen: int): DateTime =
   if time.len != flen: return
   parse(time, f, utc())
@@ -64,29 +84,24 @@ proc getDateTime*(js: JsonNode): DateTime =
 proc getTime*(js: JsonNode): DateTime =
   parseTime(js.getStr, "ddd MMM dd hh:mm:ss \'+0000\' yyyy", 30)
 
-proc getId*(id: string): string {.inline.} =
+proc getTimeFromMs*(js: JsonNode): DateTime =
+  let ms = js.getInt(0)
+  if ms == 0: return
+  let seconds = ms div 1000
+  return fromUnix(seconds).utc()
+
+proc getId*(id: string): int64 {.inline.} =
   let start = id.rfind("-")
-  if start < 0: return id
-  id[start + 1 ..< id.len]
+  if start < 0:
+    return parseBiggestInt(id)
+  return parseBiggestInt(id[start + 1 ..< id.len])
 
 proc getId*(js: JsonNode): int64 {.inline.} =
   case js.kind
-  of JString: return parseBiggestInt(js.getStr("0"))
+  of JString: return js.getStr("0").getId
   of JInt: return js.getBiggestInt()
   else: return 0
 
-proc getEntryId*(js: JsonNode): string {.inline.} =
-  let entry = js{"entryId"}.getStr
-  if entry.len == 0: return
-
-  if "tweet" in entry or "sq-I-t" in entry:
-    return entry.getId
-  elif "tombstone" in entry:
-    return js{"content", "item", "content", "tombstone", "tweet", "id"}.getStr
-  else:
-    echo "unknown entry: ", entry
-    return
-
 template getStrVal*(js: JsonNode; default=""): string =
   js{"string_value"}.getStr(default)
 
@@ -157,12 +172,6 @@ proc getMp4Resolution*(url: string): int =
     # cannot determine resolution (e.g. m3u8/non-mp4 video)
     return 0
 
-proc getVideoViewCount*(js: JsonNode): string =
-  with stats, js{"ext_media_stats"}:
-    return stats{"view_count"}.getStr($stats{"viewCount"}.getInt)
-
-  return $js{"mediaStats", "viewCount"}.getInt(0)
-
 proc extractSlice(js: JsonNode): Slice[int] =
   result = js["indices"][0].getInt ..< js["indices"][1].getInt
 

src/types.nim

@@ -121,7 +121,6 @@ type
     durationMs*: int
     url*: string
     thumb*: string
-    views*: string
     available*: bool
     reason*: string
     title*: string
@@ -202,7 +201,6 @@ type
     replies*: int
     retweets*: int
     likes*: int
-    quotes*: int
     views*: int
 
   Tweet* = ref object

src/views/tweet.nim

@@ -178,16 +178,12 @@ func formatStat(stat: int): string =
   if stat > 0: insertSep($stat, ',')
   else: ""
 
-proc renderStats(stats: TweetStats; views: string): VNode =
+proc renderStats(stats: TweetStats): VNode =
   buildHtml(tdiv(class="tweet-stats")):
     span(class="tweet-stat"): icon "comment", formatStat(stats.replies)
     span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets)
-    span(class="tweet-stat"): icon "quote", formatStat(stats.quotes)
     span(class="tweet-stat"): icon "heart", formatStat(stats.likes)
-    if stats.views > 0:
-      span(class="tweet-stat"): icon "views", formatStat(stats.views)
-    if views.len > 0:
-      span(class="tweet-stat"): icon "play", insertSep(views, ',')
+    span(class="tweet-stat"): icon "views", formatStat(stats.views)
 
 proc renderReply(tweet: Tweet): VNode =
   buildHtml(tdiv(class="replying-to")):
@@ -303,7 +299,6 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
       a(class="tweet-link", href=getLink(tweet))
 
     tdiv(class="tweet-body"):
-      var views = ""
       renderHeader(tweet, retweet, pinned, prefs)
 
       if not afterTweet and index == 0 and tweet.reply.len > 0 and
@@ -327,10 +322,8 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
         renderAlbum(tweet)
       elif tweet.video.isSome:
         renderVideo(tweet.video.get(), prefs, path)
-        views = tweet.video.get().views
       elif tweet.gif.isSome:
         renderGif(tweet.gif.get(), prefs)
-        views = "GIF"
 
       if tweet.poll.isSome:
         renderPoll(tweet.poll.get())
@@ -345,7 +338,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
         renderMediaTags(tweet.mediaTags)
 
       if not prefs.hideTweetStats:
-        renderStats(tweet.stats, views)
+        renderStats(tweet.stats)
 
       if showThread:
         a(class="show-thread", href=("/i/status/" & $tweet.threadId)):