Bump API versions, use more SessionAwareUrls

src/api.nim

@@ -7,12 +7,39 @@ import experimental/parser as newParser
 proc mediaUrl(id: string; cursor: string): SessionAwareUrl =
   let
     cookieVariables = userMediaVariables % [id, cursor]
-    oauthVariables = userTweetsVariables % [id, cursor]
+    oauthVariables = restIdVariables % [id, cursor]
   result = SessionAwareUrl(
     cookieUrl: graphUserMedia ? {"variables": cookieVariables, "features": gqlFeatures},
     oauthUrl: graphUserMediaV2 ? {"variables": oauthVariables, "features": gqlFeatures}
   )
 
+proc userTweetsUrl(id: string; cursor: string): SessionAwareUrl =
+  let
+    cookieVariables = userTweetsVariables % [id, cursor]
+    oauthVariables = restIdVariables % [id, cursor]
+  result = SessionAwareUrl(
+    cookieUrl: graphUserTweets ? {"variables": cookieVariables, "features": gqlFeatures, "fieldToggles": fieldToggles},
+    oauthUrl: graphUserTweetsV2 ? {"variables": oauthVariables, "features": gqlFeatures}
+  )
+
+proc userTweetsAndRepliesUrl(id: string; cursor: string): SessionAwareUrl =
+  let
+    cookieVariables = userTweetsAndRepliesVariables % [id, cursor]
+    oauthVariables = restIdVariables % [id, cursor]
+  result = SessionAwareUrl(
+    cookieUrl: graphUserTweetsAndReplies ? {"variables": cookieVariables, "features": gqlFeatures, "fieldToggles": fieldToggles},
+    oauthUrl: graphUserTweetsAndRepliesV2 ? {"variables": oauthVariables, "features": gqlFeatures}
+  )
+
+proc tweetDetailUrl(id: string; cursor: string): SessionAwareUrl =
+  let
+    cookieVariables = tweetDetailVariables % [id, cursor]
+    oauthVariables = tweetVariables % [id, cursor]
+  result = SessionAwareUrl(
+    cookieUrl: graphTweetDetail ? {"variables": cookieVariables, "features": gqlFeatures, "fieldToggles": tweetDetailFieldToggles},
+    oauthUrl: graphTweet ? {"variables": oauthVariables, "features": gqlFeatures}
+  )
+
 proc getGraphUser*(username: string): Future[User] {.async.} =
   if username.len == 0: return
   let
@@ -33,13 +60,11 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi
   if id.len == 0: return
   let
     cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = userTweetsVariables % [id, cursor]
-    params = {"variables": variables, "features": gqlFeatures}
     js = case kind
       of TimelineKind.tweets:
-        await fetch(graphUserTweets ? params, Api.userTweets)
+        await fetch(userTweetsUrl(id, cursor), Api.userTweets)
       of TimelineKind.replies:
-        await fetch(graphUserTweetsAndReplies ? params, Api.userTweetsAndReplies)
+        await fetch(userTweetsAndRepliesUrl(id, cursor), Api.userTweetsAndReplies)
       of TimelineKind.media:
         await fetch(mediaUrl(id, cursor), Api.userMedia)
   result = parseGraphTimeline(js, after)
@@ -48,7 +73,7 @@ proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
   if id.len == 0: return
   let
     cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = listTweetsVariables % [id, cursor]
+    variables = restIdVariables % [id, cursor]
     params = {"variables": variables, "features": gqlFeatures}
     js = await fetch(graphListTweets ? params, Api.listTweets)
   result = parseGraphTimeline(js, after).tweets
@@ -94,9 +119,7 @@ proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
   if id.len == 0: return
   let
     cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = tweetVariables % [id, cursor]
+    js = await fetch(tweetDetailUrl(id, cursor), Api.tweetDetail)
-    params = {"variables": variables, "features": gqlFeatures}
-    js = await fetch(graphTweet ? params, Api.tweetDetail)
   result = parseGraphConversation(js, id)
 
 proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =

src/consts.nim

@@ -9,16 +9,19 @@ const
 
   graphUser* = gql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery"
   graphUserById* = gql / "oPppcargziU1uDQHAUmH-A/UserResultByIdQuery"
-  graphUserTweets* = gql / "JLApJKFY0MxGTzCoK6ps8Q/UserWithProfileTweetsQueryV2"
+  graphUserTweetsV2* = gql / "JLApJKFY0MxGTzCoK6ps8Q/UserWithProfileTweetsQueryV2"
-  graphUserTweetsAndReplies* = gql / "Y86LQY7KMvxn5tu3hFTyPg/UserWithProfileTweetsAndRepliesQueryV2"
+  graphUserTweetsAndRepliesV2* = gql / "Y86LQY7KMvxn5tu3hFTyPg/UserWithProfileTweetsAndRepliesQueryV2"
+  graphUserTweets* = gql / "oRJs8SLCRNRbQzuZG93_oA/UserTweets"
+  graphUserTweetsAndReplies* = gql / "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies"
   graphUserMedia* = gql / "36oKqyQ7E_9CmtONGjJRsA/UserMedia"
   graphUserMediaV2* = gql / "PDfFf8hGeJvUCiTyWtw4wQ/MediaTimelineV2"
   graphTweet* = gql / "Vorskcd2tZ-tc4Gx3zbk4Q/ConversationTimelineV2"
+  graphTweetDetail* = gql / "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail"
   graphTweetResult* = gql / "sITyJdhRPpvpEjg4waUmTA/TweetResultByIdQuery"
-  graphSearchTimeline* = gql / "KI9jCXUx3Ymt-hDKLOZb9Q/SearchTimeline"
+  graphSearchTimeline* = gql / "7r8ibjHuK3MWUyzkzHNMYQ/SearchTimeline"
-  graphListById* = gql / "oygmAig8kjn0pKsx_bUadQ/ListByRestId"
+  graphListById* = gql / "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId"
-  graphListBySlug* = gql / "88GTz-IPPWLn1EiU8XoNVg/ListBySlug"
+  graphListBySlug* = gql / "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug"
-  graphListMembers* = gql / "kSmxeqEeelqdHSR7jMnb_w/ListMembers"
+  graphListMembers* = gql / "fuVHh5-gFn8zDBBxb8wOMA/ListMembers"
   graphListTweets* = gql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline"
 
   gqlFeatures* = """{
@@ -96,24 +99,20 @@ const
   "withV2Timeline": true
 }""".replace(" ", "").replace("\n", "")
 
-#   oldUserTweetsVariables* = """{
+  tweetDetailVariables* = """{
-#   "userId": "$1", $2
+  "focalTweetId": "$1",
-#   "count": 20,
+  $2
-#   "includePromotedContent": false,
+  "referrer": "profile",
-#   "withDownvotePerspective": false,
+  "with_rux_injections": false,
-#   "withReactionsMetadata": false,
+  "rankingMode": "Relevance",
-#   "withReactionsPerspective": false,
+  "includePromotedContent": true,
-#   "withVoice": false,
+  "withCommunity": true,
-#   "withV2Timeline": true
+  "withQuickPromoteEligibilityTweetFields": true,
-# }
+  "withBirdwatchNotes": true,
-# """
+  "withVoice": true
+}""".replace(" ", "").replace("\n", "")
 
-  userTweetsVariables* = """{
+  restIdVariables* = """{
-  "rest_id": "$1", $2
-  "count": 20
-}"""
-
-  listTweetsVariables* = """{
   "rest_id": "$1", $2
   "count": 20
 }"""
@@ -126,3 +125,22 @@ const
   "withBirdwatchNotes": false,
   "withVoice": true
 }""".replace(" ", "").replace("\n", "")
+
+  userTweetsVariables* = """{
+  "userId": "$1", $2
+  "count": 20,
+  "includePromotedContent": false,
+  "withQuickPromoteEligibilityTweetFields": true,
+  "withVoice": true
+}""".replace(" ", "").replace("\n", "")
+
+  userTweetsAndRepliesVariables* = """{
+  "userId": "$1", $2
+  "count": 20,
+  "includePromotedContent": false,
+  "withCommunity": true,
+  "withVoice": true
+}""".replace(" ", "").replace("\n", "")
+
+  fieldToggles* = """{"withArticlePlainText":false}"""
+  tweetDetailFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false}"""

src/experimental/parser/graphql.nim

@@ -1,21 +1,39 @@
-import options
+import options, strutils
 import jsony
 import user, ../types/[graphuser, graphlistmembers]
 from ../../types import User, VerifiedType, Result, Query, QueryKind
 
+proc parseUserResult*(userResult: UserResult): User =
+  result = userResult.legacy
+
+  if result.verifiedType == none and userResult.isBlueVerified:
+    result.verifiedType = blue
+
+  if result.username.len == 0 and userResult.core.screenName.len > 0:
+    result.id = userResult.restId
+    result.username = userResult.core.screenName
+    result.fullname = userResult.core.name
+    result.userPic = userResult.avatar.imageUrl.replace("_normal", "")
+
+    if userResult.verification.isSome:
+      let v = userResult.verification.get
+      if v.verifiedType != VerifiedType.none:
+        result.verifiedType = v.verifiedType
+
+    if userResult.profileBio.isSome:
+      result.bio = userResult.profileBio.get.description
+
 proc parseGraphUser*(json: string): User =
   if json.len == 0 or json[0] != '{':
     return
 
   let raw = json.fromJson(GraphUser)
+  let userResult = raw.data.userResult.result
 
-  if raw.data.userResult.result.unavailableReason.get("") == "Suspended":
+  if userResult.unavailableReason.get("") == "Suspended":
     return User(suspended: true)
 
-  result = raw.data.userResult.result.legacy
+  result = parseUserResult(userResult)
-  result.id = raw.data.userResult.result.restId
-  if result.verifiedType == VerifiedType.none and raw.data.userResult.result.isBlueVerified:
-    result.verifiedType = blue
 
 proc parseGraphListMembers*(json, cursor: string): Result[User] =
   result = Result[User](
@@ -31,7 +49,7 @@ proc parseGraphListMembers*(json, cursor: string): Result[User] =
         of TimelineTimelineItem:
           let userResult = entry.content.itemContent.userResults.result
           if userResult.restId.len > 0:
-            result.content.add userResult.legacy
+            result.content.add parseUserResult(userResult)
         of TimelineTimelineCursor:
           if entry.content.cursorType == "Bottom":
             result.bottom = entry.content.value

src/experimental/types/graphuser.nim

@@ -1,5 +1,5 @@
-import options
+import options, strutils
-from ../../types import User
+from ../../types import User, VerifiedType
 
 type
   GraphUser* = object
@@ -8,8 +8,32 @@ type
   UserData* = object
     result*: UserResult
 
-  UserResult = object
+  UserCore* = object
+    name*: string
+    screenName*: string
+    createdAt*: string
+
+  UserBio* = object
+    description*: string
+
+  UserAvatar* = object
+    imageUrl*: string
+
+  Verification* = object
+    verifiedType*: VerifiedType
+
+  UserResult* = object
     legacy*: User
     restId*: string
     isBlueVerified*: bool
     unavailableReason*: Option[string]
+    core*: UserCore
+    avatar*: UserAvatar
+    profileBio*: Option[UserBio]
+    verification*: Option[Verification]
+
+proc enumHook*(s: string; v: var VerifiedType) =
+  v = try:
+    parseEnum[VerifiedType](s)
+  except:
+    VerifiedType.none

src/parser.nim

@@ -42,16 +42,16 @@ proc parseGraphUser(js: JsonNode): User =
   result = parseUser(user{"legacy"}, user{"rest_id"}.getStr)
 
   # fallback to support UserMedia/recent GraphQL updates
-  if result.username.len == 0 and user{"core", "screen_name"}.notNull:
+  if result.username.len == 0:
     result.username = user{"core", "screen_name"}.getStr
     result.fullname = user{"core", "name"}.getStr
     result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "")
 
     if user{"is_blue_verified"}.getBool(false):
       result.verifiedType = blue
-    elif user{"verification", "verified_type"}.notNull:
+
-      let verifiedType = user{"verification", "verified_type"}.getStr("None")
+    with verifiedType, user{"verification", "verified_type"}:
-      result.verifiedType = parseEnum[VerifiedType](verifiedType)
+      result.verifiedType = parseEnum[VerifiedType](verifiedType.getStr)
 
 proc parseGraphList*(js: JsonNode): List =
   if js.isNull: return
@@ -372,10 +372,10 @@ proc parseGraphTweetResult*(js: JsonNode): Tweet =
   with tweet, js{"data", "tweet_result", "result"}:
     result = parseGraphTweet(tweet, false)
 
-proc parseGraphConversation*(js: JsonNode; tweetId: string; v2=true): Conversation =
+proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
   result = Conversation(replies: Result[Chain](beginning: true))
-
   let
+    v2 = js{"data", "timeline_response"}.notNull
     rootKey = if v2: "timeline_response" else: "threaded_conversation_with_injections_v2"
     contentKey = if v2: "content" else: "itemContent"
     resultKey = if v2: "tweetResult" else: "tweet_results"
@@ -385,7 +385,8 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string; v2=true): Conversati
     return
 
   for i in instructions:
-    if i{"__typename"}.getStr == "TimelineAddEntries":
+    let instrType = i{"__typename"}.getStr(i{"type"}.getStr)
+    if instrType == "TimelineAddEntries":
       for e in i{"entries"}:
         let entryId = e{"entryId"}.getStr
         if entryId.startsWith("tweet"):
@@ -421,20 +422,23 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string; v2=true): Conversati
           result.replies.bottom = e{"content", contentKey, "value"}.getStr
 
 proc extractTweetsFromEntry*(e: JsonNode; entryId: string): seq[Tweet] =
-  if e{"content", "items"}.notNull:
+  var tweetResult = e{"content", "itemContent", "tweet_results", "result"}
-    for item in e{"content", "items"}:
+  if tweetResult.isNull:
-      with tweetResult, item{"item", "itemContent", "tweet_results", "result"}:
+    tweetResult = e{"content", "content", "tweetResult", "result"}
-        var tweet = parseGraphTweet(tweetResult, false)
-        if not tweet.available:
-          tweet.id = parseBiggestInt(item{"entryId"}.getStr.getId())
-        result.add tweet
-    return
 
-  with tweetResult, e{"content", "content", "tweetResult", "result"}:
+  if tweetResult.notNull:
     var tweet = parseGraphTweet(tweetResult, false)
     if not tweet.available:
       tweet.id = parseBiggestInt(entryId.getId())
     result.add tweet
+    return
+
+  for item in e{"content", "items"}:
+    with tweetResult, item{"item", "itemContent", "tweet_results", "result"}:
+      var tweet = parseGraphTweet(tweetResult, false)
+      if not tweet.available:
+        tweet.id = parseBiggestInt(item{"entryId"}.getStr.getId())
+      result.add tweet
 
 proc parseGraphTimeline*(js: JsonNode; after=""): Profile =
   result = Profile(tweets: Timeline(beginning: after.len == 0))