diff --git a/src/api.nim b/src/api.nim index c0efa58..aeb0f17 100644 --- a/src/api.nim +++ b/src/api.nim @@ -7,12 +7,39 @@ import experimental/parser as newParser proc mediaUrl(id: string; cursor: string): SessionAwareUrl = let cookieVariables = userMediaVariables % [id, cursor] - oauthVariables = userTweetsVariables % [id, cursor] + oauthVariables = restIdVariables % [id, cursor] result = SessionAwareUrl( cookieUrl: graphUserMedia ? {"variables": cookieVariables, "features": gqlFeatures}, oauthUrl: graphUserMediaV2 ? {"variables": oauthVariables, "features": gqlFeatures} ) +proc userTweetsUrl(id: string; cursor: string): SessionAwareUrl = + let + cookieVariables = userTweetsVariables % [id, cursor] + oauthVariables = restIdVariables % [id, cursor] + result = SessionAwareUrl( + cookieUrl: graphUserTweets ? {"variables": cookieVariables, "features": gqlFeatures, "fieldToggles": fieldToggles}, + oauthUrl: graphUserTweetsV2 ? {"variables": oauthVariables, "features": gqlFeatures} + ) + +proc userTweetsAndRepliesUrl(id: string; cursor: string): SessionAwareUrl = + let + cookieVariables = userTweetsAndRepliesVariables % [id, cursor] + oauthVariables = restIdVariables % [id, cursor] + result = SessionAwareUrl( + cookieUrl: graphUserTweetsAndReplies ? {"variables": cookieVariables, "features": gqlFeatures, "fieldToggles": fieldToggles}, + oauthUrl: graphUserTweetsAndRepliesV2 ? {"variables": oauthVariables, "features": gqlFeatures} + ) + +proc tweetDetailUrl(id: string; cursor: string): SessionAwareUrl = + let + cookieVariables = tweetDetailVariables % [id, cursor] + oauthVariables = tweetVariables % [id, cursor] + result = SessionAwareUrl( + cookieUrl: graphTweetDetail ? {"variables": cookieVariables, "features": gqlFeatures, "fieldToggles": tweetDetailFieldToggles}, + oauthUrl: graphTweet ? {"variables": oauthVariables, "features": gqlFeatures} + ) + proc getGraphUser*(username: string): Future[User] {.async.} = if username.len == 0: return let @@ -33,13 +60,11 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi if id.len == 0: return let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" - variables = userTweetsVariables % [id, cursor] - params = {"variables": variables, "features": gqlFeatures} js = case kind of TimelineKind.tweets: - await fetch(graphUserTweets ? params, Api.userTweets) + await fetch(userTweetsUrl(id, cursor), Api.userTweets) of TimelineKind.replies: - await fetch(graphUserTweetsAndReplies ? params, Api.userTweetsAndReplies) + await fetch(userTweetsAndRepliesUrl(id, cursor), Api.userTweetsAndReplies) of TimelineKind.media: await fetch(mediaUrl(id, cursor), Api.userMedia) result = parseGraphTimeline(js, after) @@ -48,7 +73,7 @@ proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} = if id.len == 0: return let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" - variables = listTweetsVariables % [id, cursor] + variables = restIdVariables % [id, cursor] params = {"variables": variables, "features": gqlFeatures} js = await fetch(graphListTweets ? params, Api.listTweets) result = parseGraphTimeline(js, after).tweets @@ -94,9 +119,7 @@ proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} = if id.len == 0: return let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" - variables = tweetVariables % [id, cursor] - params = {"variables": variables, "features": gqlFeatures} - js = await fetch(graphTweet ? params, Api.tweetDetail) + js = await fetch(tweetDetailUrl(id, cursor), Api.tweetDetail) result = parseGraphConversation(js, id) proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} = diff --git a/src/consts.nim b/src/consts.nim index c8ae8d2..2623484 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -9,16 +9,19 @@ const graphUser* = gql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery" graphUserById* = gql / "oPppcargziU1uDQHAUmH-A/UserResultByIdQuery" - graphUserTweets* = gql / "JLApJKFY0MxGTzCoK6ps8Q/UserWithProfileTweetsQueryV2" - graphUserTweetsAndReplies* = gql / "Y86LQY7KMvxn5tu3hFTyPg/UserWithProfileTweetsAndRepliesQueryV2" + graphUserTweetsV2* = gql / "JLApJKFY0MxGTzCoK6ps8Q/UserWithProfileTweetsQueryV2" + graphUserTweetsAndRepliesV2* = gql / "Y86LQY7KMvxn5tu3hFTyPg/UserWithProfileTweetsAndRepliesQueryV2" + graphUserTweets* = gql / "oRJs8SLCRNRbQzuZG93_oA/UserTweets" + graphUserTweetsAndReplies* = gql / "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies" graphUserMedia* = gql / "36oKqyQ7E_9CmtONGjJRsA/UserMedia" graphUserMediaV2* = gql / "PDfFf8hGeJvUCiTyWtw4wQ/MediaTimelineV2" graphTweet* = gql / "Vorskcd2tZ-tc4Gx3zbk4Q/ConversationTimelineV2" + graphTweetDetail* = gql / "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail" graphTweetResult* = gql / "sITyJdhRPpvpEjg4waUmTA/TweetResultByIdQuery" - graphSearchTimeline* = gql / "KI9jCXUx3Ymt-hDKLOZb9Q/SearchTimeline" - graphListById* = gql / "oygmAig8kjn0pKsx_bUadQ/ListByRestId" - graphListBySlug* = gql / "88GTz-IPPWLn1EiU8XoNVg/ListBySlug" - graphListMembers* = gql / "kSmxeqEeelqdHSR7jMnb_w/ListMembers" + graphSearchTimeline* = gql / "7r8ibjHuK3MWUyzkzHNMYQ/SearchTimeline" + graphListById* = gql / "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId" + graphListBySlug* = gql / "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug" + graphListMembers* = gql / "fuVHh5-gFn8zDBBxb8wOMA/ListMembers" graphListTweets* = gql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline" gqlFeatures* = """{ @@ -96,24 +99,20 @@ const "withV2Timeline": true }""".replace(" ", "").replace("\n", "") -# oldUserTweetsVariables* = """{ -# "userId": "$1", $2 -# "count": 20, -# "includePromotedContent": false, -# "withDownvotePerspective": false, -# "withReactionsMetadata": false, -# "withReactionsPerspective": false, -# "withVoice": false, -# "withV2Timeline": true -# } -# """ + tweetDetailVariables* = """{ + "focalTweetId": "$1", + $2 + "referrer": "profile", + "with_rux_injections": false, + "rankingMode": "Relevance", + "includePromotedContent": true, + "withCommunity": true, + "withQuickPromoteEligibilityTweetFields": true, + "withBirdwatchNotes": true, + "withVoice": true +}""".replace(" ", "").replace("\n", "") - userTweetsVariables* = """{ - "rest_id": "$1", $2 - "count": 20 -}""" - - listTweetsVariables* = """{ + restIdVariables* = """{ "rest_id": "$1", $2 "count": 20 }""" @@ -126,3 +125,22 @@ const "withBirdwatchNotes": false, "withVoice": true }""".replace(" ", "").replace("\n", "") + + userTweetsVariables* = """{ + "userId": "$1", $2 + "count": 20, + "includePromotedContent": false, + "withQuickPromoteEligibilityTweetFields": true, + "withVoice": true +}""".replace(" ", "").replace("\n", "") + + userTweetsAndRepliesVariables* = """{ + "userId": "$1", $2 + "count": 20, + "includePromotedContent": false, + "withCommunity": true, + "withVoice": true +}""".replace(" ", "").replace("\n", "") + + fieldToggles* = """{"withArticlePlainText":false}""" + tweetDetailFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false}""" diff --git a/src/experimental/parser/graphql.nim b/src/experimental/parser/graphql.nim index 69837ab..045a5d6 100644 --- a/src/experimental/parser/graphql.nim +++ b/src/experimental/parser/graphql.nim @@ -1,21 +1,39 @@ -import options +import options, strutils import jsony import user, ../types/[graphuser, graphlistmembers] from ../../types import User, VerifiedType, Result, Query, QueryKind +proc parseUserResult*(userResult: UserResult): User = + result = userResult.legacy + + if result.verifiedType == none and userResult.isBlueVerified: + result.verifiedType = blue + + if result.username.len == 0 and userResult.core.screenName.len > 0: + result.id = userResult.restId + result.username = userResult.core.screenName + result.fullname = userResult.core.name + result.userPic = userResult.avatar.imageUrl.replace("_normal", "") + + if userResult.verification.isSome: + let v = userResult.verification.get + if v.verifiedType != VerifiedType.none: + result.verifiedType = v.verifiedType + + if userResult.profileBio.isSome: + result.bio = userResult.profileBio.get.description + proc parseGraphUser*(json: string): User = if json.len == 0 or json[0] != '{': return let raw = json.fromJson(GraphUser) + let userResult = raw.data.userResult.result - if raw.data.userResult.result.unavailableReason.get("") == "Suspended": + if userResult.unavailableReason.get("") == "Suspended": return User(suspended: true) - result = raw.data.userResult.result.legacy - result.id = raw.data.userResult.result.restId - if result.verifiedType == VerifiedType.none and raw.data.userResult.result.isBlueVerified: - result.verifiedType = blue + result = parseUserResult(userResult) proc parseGraphListMembers*(json, cursor: string): Result[User] = result = Result[User]( @@ -31,7 +49,7 @@ proc parseGraphListMembers*(json, cursor: string): Result[User] = of TimelineTimelineItem: let userResult = entry.content.itemContent.userResults.result if userResult.restId.len > 0: - result.content.add userResult.legacy + result.content.add parseUserResult(userResult) of TimelineTimelineCursor: if entry.content.cursorType == "Bottom": result.bottom = entry.content.value diff --git a/src/experimental/types/graphuser.nim b/src/experimental/types/graphuser.nim index 08100f9..d732b4e 100644 --- a/src/experimental/types/graphuser.nim +++ b/src/experimental/types/graphuser.nim @@ -1,5 +1,5 @@ -import options -from ../../types import User +import options, strutils +from ../../types import User, VerifiedType type GraphUser* = object @@ -8,8 +8,32 @@ type UserData* = object result*: UserResult - UserResult = object + UserCore* = object + name*: string + screenName*: string + createdAt*: string + + UserBio* = object + description*: string + + UserAvatar* = object + imageUrl*: string + + Verification* = object + verifiedType*: VerifiedType + + UserResult* = object legacy*: User restId*: string isBlueVerified*: bool unavailableReason*: Option[string] + core*: UserCore + avatar*: UserAvatar + profileBio*: Option[UserBio] + verification*: Option[Verification] + +proc enumHook*(s: string; v: var VerifiedType) = + v = try: + parseEnum[VerifiedType](s) + except: + VerifiedType.none diff --git a/src/parser.nim b/src/parser.nim index f132dea..c4ccab0 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -42,16 +42,16 @@ proc parseGraphUser(js: JsonNode): User = result = parseUser(user{"legacy"}, user{"rest_id"}.getStr) # fallback to support UserMedia/recent GraphQL updates - if result.username.len == 0 and user{"core", "screen_name"}.notNull: + if result.username.len == 0: result.username = user{"core", "screen_name"}.getStr result.fullname = user{"core", "name"}.getStr result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "") if user{"is_blue_verified"}.getBool(false): result.verifiedType = blue - elif user{"verification", "verified_type"}.notNull: - let verifiedType = user{"verification", "verified_type"}.getStr("None") - result.verifiedType = parseEnum[VerifiedType](verifiedType) + + with verifiedType, user{"verification", "verified_type"}: + result.verifiedType = parseEnum[VerifiedType](verifiedType.getStr) proc parseGraphList*(js: JsonNode): List = if js.isNull: return @@ -372,10 +372,10 @@ proc parseGraphTweetResult*(js: JsonNode): Tweet = with tweet, js{"data", "tweet_result", "result"}: result = parseGraphTweet(tweet, false) -proc parseGraphConversation*(js: JsonNode; tweetId: string; v2=true): Conversation = +proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation = result = Conversation(replies: Result[Chain](beginning: true)) - let + v2 = js{"data", "timeline_response"}.notNull rootKey = if v2: "timeline_response" else: "threaded_conversation_with_injections_v2" contentKey = if v2: "content" else: "itemContent" resultKey = if v2: "tweetResult" else: "tweet_results" @@ -385,7 +385,8 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string; v2=true): Conversati return for i in instructions: - if i{"__typename"}.getStr == "TimelineAddEntries": + let instrType = i{"__typename"}.getStr(i{"type"}.getStr) + if instrType == "TimelineAddEntries": for e in i{"entries"}: let entryId = e{"entryId"}.getStr if entryId.startsWith("tweet"): @@ -421,20 +422,23 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string; v2=true): Conversati result.replies.bottom = e{"content", contentKey, "value"}.getStr proc extractTweetsFromEntry*(e: JsonNode; entryId: string): seq[Tweet] = - if e{"content", "items"}.notNull: - for item in e{"content", "items"}: - with tweetResult, item{"item", "itemContent", "tweet_results", "result"}: - var tweet = parseGraphTweet(tweetResult, false) - if not tweet.available: - tweet.id = parseBiggestInt(item{"entryId"}.getStr.getId()) - result.add tweet - return + var tweetResult = e{"content", "itemContent", "tweet_results", "result"} + if tweetResult.isNull: + tweetResult = e{"content", "content", "tweetResult", "result"} - with tweetResult, e{"content", "content", "tweetResult", "result"}: + if tweetResult.notNull: var tweet = parseGraphTweet(tweetResult, false) if not tweet.available: tweet.id = parseBiggestInt(entryId.getId()) result.add tweet + return + + for item in e{"content", "items"}: + with tweetResult, item{"item", "itemContent", "tweet_results", "result"}: + var tweet = parseGraphTweet(tweetResult, false) + if not tweet.available: + tweet.id = parseBiggestInt(item{"entryId"}.getStr.getId()) + result.add tweet proc parseGraphTimeline*(js: JsonNode; after=""): Profile = result = Profile(tweets: Timeline(beginning: after.len == 0))