From b0d9c1d51a4039cecbb58ab0180883baf5325d59 Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 22 Nov 2025 21:21:10 +0100 Subject: [PATCH] Update endpoints, fix parser, remove quotes stat --- src/api.nim | 58 +++++---- src/consts.nim | 35 +++-- src/parser.nim | 308 ++++++++++++++++++++++++++------------------ src/parserutils.nim | 53 ++++---- src/types.nim | 2 - src/views/tweet.nim | 13 +- 6 files changed, 269 insertions(+), 200 deletions(-) diff --git a/src/api.nim b/src/api.nim index aeb0f17..ef3a0f9 100644 --- a/src/api.nim +++ b/src/api.nim @@ -1,16 +1,23 @@ # SPDX-License-Identifier: AGPL-3.0-only -import asyncdispatch, httpclient, uri, strutils, sequtils, sugar +import asyncdispatch, httpclient, uri, strutils, sequtils, sugar, tables import packedjson import types, query, formatters, consts, apiutils, parser import experimental/parser as newParser +# Helper to generate params object for GraphQL requests +proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] = + result.add ("variables", variables) + result.add ("features", gqlFeatures) + if fieldToggles.len > 0: + result.add ("fieldToggles", fieldToggles) + proc mediaUrl(id: string; cursor: string): SessionAwareUrl = let cookieVariables = userMediaVariables % [id, cursor] oauthVariables = restIdVariables % [id, cursor] result = SessionAwareUrl( - cookieUrl: graphUserMedia ? {"variables": cookieVariables, "features": gqlFeatures}, - oauthUrl: graphUserMediaV2 ? {"variables": oauthVariables, "features": gqlFeatures} + cookieUrl: graphUserMedia ? genParams(cookieVariables), + oauthUrl: graphUserMediaV2 ? genParams(oauthVariables) ) proc userTweetsUrl(id: string; cursor: string): SessionAwareUrl = @@ -18,17 +25,19 @@ proc userTweetsUrl(id: string; cursor: string): SessionAwareUrl = cookieVariables = userTweetsVariables % [id, cursor] oauthVariables = restIdVariables % [id, cursor] result = SessionAwareUrl( - cookieUrl: graphUserTweets ? {"variables": cookieVariables, "features": gqlFeatures, "fieldToggles": fieldToggles}, - oauthUrl: graphUserTweetsV2 ? {"variables": oauthVariables, "features": gqlFeatures} + # cookieUrl: graphUserTweets ? genParams(cookieVariables, fieldToggles), + oauthUrl: graphUserTweetsV2 ? genParams(oauthVariables) ) + # might change this in the future pending testing + result.cookieUrl = result.oauthUrl proc userTweetsAndRepliesUrl(id: string; cursor: string): SessionAwareUrl = let cookieVariables = userTweetsAndRepliesVariables % [id, cursor] oauthVariables = restIdVariables % [id, cursor] result = SessionAwareUrl( - cookieUrl: graphUserTweetsAndReplies ? {"variables": cookieVariables, "features": gqlFeatures, "fieldToggles": fieldToggles}, - oauthUrl: graphUserTweetsAndRepliesV2 ? {"variables": oauthVariables, "features": gqlFeatures} + cookieUrl: graphUserTweetsAndReplies ? genParams(cookieVariables, fieldToggles), + oauthUrl: graphUserTweetsAndRepliesV2 ? genParams(oauthVariables) ) proc tweetDetailUrl(id: string; cursor: string): SessionAwareUrl = @@ -36,24 +45,22 @@ proc tweetDetailUrl(id: string; cursor: string): SessionAwareUrl = cookieVariables = tweetDetailVariables % [id, cursor] oauthVariables = tweetVariables % [id, cursor] result = SessionAwareUrl( - cookieUrl: graphTweetDetail ? {"variables": cookieVariables, "features": gqlFeatures, "fieldToggles": tweetDetailFieldToggles}, - oauthUrl: graphTweet ? {"variables": oauthVariables, "features": gqlFeatures} + cookieUrl: graphTweetDetail ? genParams(cookieVariables, tweetDetailFieldToggles), + oauthUrl: graphTweet ? genParams(oauthVariables) ) proc getGraphUser*(username: string): Future[User] {.async.} = if username.len == 0: return let - variables = """{"screen_name": "$1"}""" % username - params = {"variables": variables, "features": gqlFeatures} - js = await fetchRaw(graphUser ? params, Api.userScreenName) + url = graphUser ? genParams("""{"screen_name": "$1"}""" % username) + js = await fetchRaw(url, Api.userScreenName) result = parseGraphUser(js) proc getGraphUserById*(id: string): Future[User] {.async.} = if id.len == 0 or id.any(c => not c.isDigit): return let - variables = """{"rest_id": "$1"}""" % id - params = {"variables": variables, "features": gqlFeatures} - js = await fetchRaw(graphUserById ? params, Api.userRestId) + url = graphUserById ? genParams("""{"rest_id": "$1"}""" % id) + js = await fetchRaw(url, Api.userRestId) result = parseGraphUser(js) proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} = @@ -73,23 +80,18 @@ proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} = if id.len == 0: return let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" - variables = restIdVariables % [id, cursor] - params = {"variables": variables, "features": gqlFeatures} - js = await fetch(graphListTweets ? params, Api.listTweets) - result = parseGraphTimeline(js, after).tweets + url = graphListTweets ? genParams(restIdVariables % [id, cursor]) + result = parseGraphTimeline(await fetch(url, Api.listTweets), after).tweets proc getGraphListBySlug*(name, list: string): Future[List] {.async.} = let variables = %*{"screenName": name, "listSlug": list} - params = {"variables": $variables, "features": gqlFeatures} - url = graphListBySlug ? params + url = graphListBySlug ? genParams($variables) result = parseGraphList(await fetch(url, Api.listBySlug)) proc getGraphList*(id: string): Future[List] {.async.} = let - variables = """{"listId": "$1"}""" % id - params = {"variables": variables, "features": gqlFeatures} - url = graphListById ? params + url = graphListById ? genParams("""{"listId": "$1"}""" % id) result = parseGraphList(await fetch(url, Api.list)) proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} = @@ -104,7 +106,7 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} } if after.len > 0: variables["cursor"] = % after - let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures} + let url = graphListMembers ? genParams($variables) result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after) proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} = @@ -139,6 +141,7 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} = var variables = %*{ "rawQuery": q, + "query_source": "typedQuery", "count": 20, "product": "Latest", "withDownvotePerspective": false, @@ -147,7 +150,7 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} = } if after.len > 0: variables["cursor"] = % after - let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures} + let url = graphSearchTimeline ? genParams($variables) result = parseGraphSearch[Tweets](await fetch(url, Api.search), after) result.query = query @@ -158,6 +161,7 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} var variables = %*{ "rawQuery": query.text, + "query_source": "typedQuery", "count": 20, "product": "People", "withDownvotePerspective": false, @@ -168,7 +172,7 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} variables["cursor"] = % after result.beginning = false - let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures} + let url = graphSearchTimeline ? genParams($variables) result = parseGraphSearch[User](await fetch(url, Api.search), after) result.query = query diff --git a/src/consts.nim b/src/consts.nim index 2623484..792a519 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -7,26 +7,29 @@ const gql = parseUri("https://api.x.com") / "graphql" - graphUser* = gql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery" - graphUserById* = gql / "oPppcargziU1uDQHAUmH-A/UserResultByIdQuery" - graphUserTweetsV2* = gql / "JLApJKFY0MxGTzCoK6ps8Q/UserWithProfileTweetsQueryV2" - graphUserTweetsAndRepliesV2* = gql / "Y86LQY7KMvxn5tu3hFTyPg/UserWithProfileTweetsAndRepliesQueryV2" + graphUser* = gql / "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery" + graphUserById* = gql / "VN33vKXrPT7p35DgNR27aw/UserResultByIdQuery" + graphUserTweetsV2* = gql / "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2" + graphUserTweetsAndRepliesV2* = gql / "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2" graphUserTweets* = gql / "oRJs8SLCRNRbQzuZG93_oA/UserTweets" graphUserTweetsAndReplies* = gql / "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies" graphUserMedia* = gql / "36oKqyQ7E_9CmtONGjJRsA/UserMedia" - graphUserMediaV2* = gql / "PDfFf8hGeJvUCiTyWtw4wQ/MediaTimelineV2" - graphTweet* = gql / "Vorskcd2tZ-tc4Gx3zbk4Q/ConversationTimelineV2" + graphUserMediaV2* = gql / "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2" + graphTweet* = gql / "Y4Erk_-0hObvLpz0Iw3bzA/ConversationTimeline" graphTweetDetail* = gql / "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail" - graphTweetResult* = gql / "sITyJdhRPpvpEjg4waUmTA/TweetResultByIdQuery" - graphSearchTimeline* = gql / "7r8ibjHuK3MWUyzkzHNMYQ/SearchTimeline" + graphTweetResult* = gql / "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery" + graphSearchTimeline* = gql / "bshMIjqDk8LTXTq4w91WKw/SearchTimeline" graphListById* = gql / "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId" graphListBySlug* = gql / "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug" graphListMembers* = gql / "fuVHh5-gFn8zDBBxb8wOMA/ListMembers" - graphListTweets* = gql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline" + graphListTweets* = gql / "VQf8_XQynI3WzH6xopOMMQ/ListTimeline" gqlFeatures* = """{ + "android_ad_formats_media_component_render_overlay_enabled": false, "android_graphql_skip_api_media_color_palette": false, + "android_professional_link_spotlight_display_enabled": false, "blue_business_profile_image_shape_enabled": false, + "commerce_android_shop_module_enabled": false, "creator_subscriptions_subscription_count_enabled": false, "creator_subscriptions_tweet_preview_api_enabled": true, "freedom_of_speech_not_reach_fetch_enabled": true, @@ -36,8 +39,9 @@ const "interactive_text_enabled": false, "longform_notetweets_consumption_enabled": true, "longform_notetweets_inline_media_enabled": true, - "longform_notetweets_richtext_consumption_enabled": true, "longform_notetweets_rich_text_read_enabled": true, + "longform_notetweets_richtext_consumption_enabled": true, + "mobile_app_spotlight_module_enabled": false, "responsive_web_edit_tweet_api_enabled": true, "responsive_web_enhance_cards_enabled": false, "responsive_web_graphql_exclude_directive_enabled": true, @@ -46,6 +50,7 @@ const "responsive_web_media_download_video_enabled": false, "responsive_web_text_conversations_enabled": false, "responsive_web_twitter_article_tweet_consumption_enabled": true, + "unified_cards_destination_url_params_enabled": false, "responsive_web_twitter_blue_verified_badge_is_enabled": true, "rweb_lists_timeline_redesign_enabled": true, "spaces_2022_h2_clipping": true, @@ -86,11 +91,17 @@ const "payments_enabled": false, "responsive_web_profile_redirect_enabled": false, "responsive_web_grok_show_grok_translated_post": false, - "responsive_web_grok_community_note_auto_translation_is_enabled": false + "responsive_web_grok_community_note_auto_translation_is_enabled": false, + "profile_label_improvements_pcf_label_in_profile_enabled": false, + "grok_android_analyze_trend_fetch_enabled": false, + "grok_translations_community_note_auto_translation_is_enabled": false, + "grok_translations_post_auto_translation_is_enabled": false, + "grok_translations_community_note_translation_is_enabled": false, + "grok_translations_timeline_user_bio_auto_translation_is_enabled": false }""".replace(" ", "").replace("\n", "") tweetVariables* = """{ - "focalTweetId": "$1", + "postId": "$1", $2 "includeHasBirdwatchNotes": false, "includePromotedContent": false, diff --git a/src/parser.nim b/src/parser.nim index 8712a5f..700e896 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -1,10 +1,10 @@ # SPDX-License-Identifier: AGPL-3.0-only -import strutils, options, times, math +import strutils, options, times, math, tables import packedjson, packedjson/deserialiser import types, parserutils, utils import experimental/parser/unifiedcard -proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet +proc parseGraphTweet(js: JsonNode): Tweet proc parseUser(js: JsonNode; id=""): User = if js.isNull: return @@ -46,6 +46,9 @@ proc parseGraphUser(js: JsonNode): User = result = parseUser(user{"legacy"}, user{"rest_id"}.getStr) + if result.verifiedType == none and user{"is_blue_verified"}.getBool(false): + result.verifiedType = blue + # fallback to support UserMedia/recent GraphQL updates if result.username.len == 0: result.username = user{"core", "screen_name"}.getStr @@ -95,16 +98,24 @@ proc parsePoll(js: JsonNode): Poll = result.leader = result.values.find(max(result.values)) result.votes = result.values.sum -proc parseGif(js: JsonNode): Gif = - result = Gif( - url: js{"video_info", "variants"}[0]{"url"}.getImageStr, - thumb: js{"media_url_https"}.getImageStr - ) +proc parseVideoVariants(variants: JsonNode): seq[VideoVariant] = + result = @[] + for v in variants: + let + url = v{"url"}.getStr + contentType = parseEnum[VideoType](v{"content_type"}.getStr("video/mp4")) + bitrate = v{"bit_rate"}.getInt(v{"bitrate"}.getInt(0)) + + result.add VideoVariant( + contentType: contentType, + bitrate: bitrate, + url: url, + resolution: if contentType == mp4: getMp4Resolution(url) else: 0 + ) proc parseVideo(js: JsonNode): Video = result = Video( thumb: js{"media_url_https"}.getImageStr, - views: getVideoViewCount(js), available: true, title: js{"ext_alt_text"}.getStr, durationMs: js{"video_info", "duration_millis"}.getInt @@ -121,17 +132,62 @@ proc parseVideo(js: JsonNode): Video = with description, js{"additional_media_info", "description"}: result.description = description.getStr - for v in js{"video_info", "variants"}: - let - contentType = parseEnum[VideoType](v{"content_type"}.getStr("summary")) - url = v{"url"}.getStr + result.variants = parseVideoVariants(js{"video_info", "variants"}) - result.variants.add VideoVariant( - contentType: contentType, - bitrate: v{"bitrate"}.getInt, - url: url, - resolution: if contentType == mp4: getMp4Resolution(url) else: 0 - ) +proc parseLegacyMediaEntities(js: JsonNode; result: var Tweet) = + with jsMedia, js{"extended_entities", "media"}: + for m in jsMedia: + case m.getTypeName: + of "photo": + result.photos.add m{"media_url_https"}.getImageStr + of "video": + result.video = some(parseVideo(m)) + with user, m{"additional_media_info", "source_user"}: + if user{"id"}.getInt > 0: + result.attribution = some(parseUser(user)) + else: + result.attribution = some(parseGraphUser(user)) + of "animated_gif": + result.gif = some Gif( + url: m{"video_info", "variants"}[0]{"url"}.getImageStr, + thumb: m{"media_url_https"}.getImageStr + ) + else: discard + + with url, m{"url"}: + if result.text.endsWith(url.getStr): + result.text.removeSuffix(url.getStr) + result.text = result.text.strip() + +proc parseMediaEntities(js: JsonNode; result: var Tweet) = + with mediaEntities, js{"media_entities"}: + for mediaEntity in mediaEntities: + with mediaInfo, mediaEntity{"media_results", "result", "media_info"}: + case mediaInfo.getTypeName + of "ApiImage": + result.photos.add mediaInfo{"original_img_url"}.getImageStr + of "ApiVideo": + let status = mediaEntity{"media_results", "result", "media_availability_v2", "status"} + result.video = some Video( + available: status.getStr == "Available", + thumb: mediaInfo{"preview_image", "original_img_url"}.getImageStr, + durationMs: mediaInfo{"duration_millis"}.getInt, + variants: parseVideoVariants(mediaInfo{"variants"}) + ) + of "ApiGif": + result.gif = some Gif( + url: mediaInfo{"variants"}[0]{"url"}.getImageStr, + thumb: mediaInfo{"preview_image", "original_img_url"}.getImageStr + ) + else: discard + + # Remove media URLs from text + with mediaList, js{"legacy", "entities", "media"}: + for url in mediaList: + let expandedUrl = url{"expanded_url"}.getStr + if result.text.endsWith(expandedUrl): + result.text.removeSuffix(expandedUrl) + result.text = result.text.strip() proc parsePromoVideo(js: JsonNode): Video = result = Video( @@ -223,12 +279,17 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card = proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = if js.isNull: return + + let time = + if js{"created_at"}.notNull: js{"created_at"}.getTime + else: js{"created_at_ms"}.getTimeFromMs + result = Tweet( id: js{"id_str"}.getId, threadId: js{"conversation_id_str"}.getId, replyId: js{"in_reply_to_status_id_str"}.getId, text: js{"full_text"}.getStr, - time: js{"created_at"}.getTime, + time: time, hasThread: js{"self_thread"}.notNull, available: true, user: User(id: js{"user_id_str"}.getStr), @@ -236,7 +297,6 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = replies: js{"reply_count"}.getInt, retweets: js{"retweet_count"}.getInt, likes: js{"favorite_count"}.getInt, - quotes: js{"quote_count"}.getInt, views: js{"views_count"}.getInt ) ) @@ -262,6 +322,12 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = result.retweet = some parseGraphTweet(rt) return + with reposts, js{"repostedStatusResults"}: + with rt, reposts{"result"}: + if "legacy" in rt: + result.retweet = some parseGraphTweet(rt) + return + if jsCard.kind != JNull: let name = jsCard{"name"}.getStr if "poll" in name: @@ -275,27 +341,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = result.card = some parseCard(jsCard, js{"entities", "urls"}) result.expandTweetEntities(js) - - with jsMedia, js{"extended_entities", "media"}: - for m in jsMedia: - case m{"type"}.getStr - of "photo": - result.photos.add m{"media_url_https"}.getImageStr - of "video": - result.video = some(parseVideo(m)) - with user, m{"additional_media_info", "source_user"}: - if user{"id"}.getInt > 0: - result.attribution = some(parseUser(user)) - else: - result.attribution = some(parseGraphUser(user)) - of "animated_gif": - result.gif = some(parseGif(m)) - else: discard - - with url, m{"url"}: - if result.text.endsWith(url.getStr): - result.text.removeSuffix(url.getStr) - result.text = result.text.strip() + parseLegacyMediaEntities(js, result) with jsWithheld, js{"withheld_in_countries"}: let withheldInCountries: seq[string] = @@ -311,95 +357,108 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = result.text.removeSuffix(" Learn more.") result.available = false -proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet = +proc parseGraphTweet(js: JsonNode): Tweet = if js.kind == JNull: return Tweet() - case js{"__typename"}.getStr + case js.getTypeName: of "TweetUnavailable": return Tweet() of "TweetTombstone": - with text, js{"tombstone", "richText"}: - return Tweet(text: text.getTombstone) - with text, js{"tombstone", "text"}: + with text, select(js{"tombstone", "richText"}, js{"tombstone", "text"}): return Tweet(text: text.getTombstone) return Tweet() of "TweetPreviewDisplay": return Tweet(text: "You're unable to view this Tweet because it's only available to the Subscribers of the account owner.") of "TweetWithVisibilityResults": - return parseGraphTweet(js{"tweet"}, isLegacy) + return parseGraphTweet(js{"tweet"}) else: discard if not js.hasKey("legacy"): return Tweet() - var jsCard = copy(js{if isLegacy: "card" else: "tweet_card", "legacy"}) + var jsCard = select(js{"card"}, js{"tweet_card"}, js{"legacy", "tweet_card"}) if jsCard.kind != JNull: - var values = newJObject() - for val in jsCard["binding_values"]: - values[val["key"].getStr] = val["value"] - jsCard["binding_values"] = values + let legacyCard = jsCard{"legacy"} + if legacyCard.kind != JNull: + let bindingArray = legacyCard{"binding_values"} + if bindingArray.kind == JArray: + var bindingObj: seq[(string, JsonNode)] + for item in bindingArray: + bindingObj.add((item{"key"}.getStr, item{"value"})) + # Create a new card object with flattened structure + jsCard = %*{ + "name": legacyCard{"name"}, + "url": legacyCard{"url"}, + "binding_values": %bindingObj + } result = parseTweet(js{"legacy"}, jsCard) result.id = js{"rest_id"}.getId result.user = parseGraphUser(js{"core"}) + if result.replyId == 0: + result.replyId = js{"reply_to_results", "rest_id"}.getId + with count, js{"views", "count"}: result.stats.views = count.getStr("0").parseInt with noteTweet, js{"note_tweet", "note_tweet_results", "result"}: result.expandNoteTweetEntities(noteTweet) + parseMediaEntities(js, result) + if result.quote.isSome: - result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}, isLegacy)) + result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"})) + + with quoted, js{"quotedPostResults", "result"}: + result.quote = some(parseGraphTweet(quoted)) proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] = - for t in js{"content", "items"}: - let entryId = t{"entryId"}.getStr + for t in ? js{"content", "items"}: + let entryId = t.getEntryId if "cursor-showmore" in entryId: let cursor = t{"item", "content", "value"} result.thread.cursor = cursor.getStr result.thread.hasMore = true elif "tweet" in entryId and "promoted" notin entryId: - let - isLegacy = t{"item"}.hasKey("itemContent") - (contentKey, resultKey) = if isLegacy: ("itemContent", "tweet_results") - else: ("content", "tweetResult") + with tweet, t.getTweetResult("item"): + result.thread.content.add parseGraphTweet(tweet) - with content, t{"item", contentKey}: - result.thread.content.add parseGraphTweet(content{resultKey, "result"}, isLegacy) - - if content{"tweetDisplayType"}.getStr == "SelfThread": + let tweetDisplayType = select( + t{"item", "content", "tweet_display_type"}, + t{"item", "itemContent", "tweetDisplayType"} + ) + if tweetDisplayType.getStr == "SelfThread": result.self = true proc parseGraphTweetResult*(js: JsonNode): Tweet = with tweet, js{"data", "tweet_result", "result"}: - result = parseGraphTweet(tweet, false) + result = parseGraphTweet(tweet) proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation = result = Conversation(replies: Result[Chain](beginning: true)) - let - v2 = js{"data", "timeline_response"}.notNull - rootKey = if v2: "timeline_response" else: "threaded_conversation_with_injections_v2" - contentKey = if v2: "content" else: "itemContent" - resultKey = if v2: "tweetResult" else: "tweet_results" - let instructions = ? js{"data", rootKey, "instructions"} + let instructions = ? select( + js{"data", "timelineResponse", "instructions"}, + js{"data", "timeline_response", "instructions"}, + js{"data", "threaded_conversation_with_injections_v2", "instructions"} + ) if instructions.len == 0: return for i in instructions: - let instrType = i{"type"}.getStr(i{"__typename"}.getStr) - if instrType == "TimelineAddEntries": + if i.getTypeName == "TimelineAddEntries": for e in i{"entries"}: - let entryId = e{"entryId"}.getStr + let entryId = e.getEntryId if entryId.startsWith("tweet"): - with tweetResult, e{"content", contentKey, resultKey, "result"}: - let tweet = parseGraphTweet(tweetResult, not v2) + let tweetResult = getTweetResult(e) + if tweetResult.notNull: + let tweet = parseGraphTweet(tweetResult) if not tweet.available: - tweet.id = parseBiggestInt(entryId.getId()) + tweet.id = entryId.getId if $tweet.id == tweetId: result.tweet = tweet @@ -412,67 +471,64 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation = elif thread.content.len > 0: result.replies.content.add thread elif entryId.startsWith("tombstone"): - let id = entryId.getId() - let tweet = Tweet( - id: parseBiggestInt(id), - available: false, - text: e{"content", contentKey, "tombstoneInfo", "richText"}.getTombstone - ) + let + content = select(e{"content", "content"}, e{"content", "itemContent"}) + tweet = Tweet( + id: entryId.getId, + available: false, + text: content{"tombstoneInfo", "richText"}.getTombstone + ) - if id == tweetId: + if $tweet.id == tweetId: result.tweet = tweet else: result.before.content.add tweet elif entryId.startsWith("cursor-bottom"): - result.replies.bottom = e{"content", contentKey, "value"}.getStr + var cursorValue = select( + e{"content", "content", "value"}, + e{"content", "itemContent", "value"} + ) + result.replies.bottom = cursorValue.getStr proc extractTweetsFromEntry*(e: JsonNode): seq[Tweet] = - var tweetResult = e{"content", "itemContent", "tweet_results", "result"} - if tweetResult.isNull: - tweetResult = e{"content", "content", "tweetResult", "result"} - - if tweetResult.notNull: - var tweet = parseGraphTweet(tweetResult, false) + with tweetResult, getTweetResult(e): + var tweet = parseGraphTweet(tweetResult) if not tweet.available: - tweet.id = parseBiggestInt(e.getEntryId()) + tweet.id = e.getEntryId.getId result.add tweet return for item in e{"content", "items"}: - with tweetResult, item{"item", "itemContent", "tweet_results", "result"}: - var tweet = parseGraphTweet(tweetResult, false) + with tweetResult, item.getTweetResult("item"): + var tweet = parseGraphTweet(tweetResult) if not tweet.available: - tweet.id = parseBiggestInt(item{"entryId"}.getStr.getId()) + tweet.id = item.getEntryId.getId result.add tweet proc parseGraphTimeline*(js: JsonNode; after=""): Profile = result = Profile(tweets: Timeline(beginning: after.len == 0)) - let instructions = - if js{"data", "list"}.notNull: - ? js{"data", "list", "timeline_response", "timeline", "instructions"} - elif js{"data", "user"}.notNull: - ? js{"data", "user", "result", "timeline", "timeline", "instructions"} - else: - ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"} - + let instructions = ? select( + js{"data", "list", "timeline_response", "timeline", "instructions"}, + js{"data", "user", "result", "timeline", "timeline", "instructions"}, + js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"} + ) if instructions.len == 0: return for i in instructions: - # TimelineAddToModule instruction is used by UserMedia if i{"moduleItems"}.notNull: for item in i{"moduleItems"}: - with tweetResult, item{"item", "itemContent", "tweet_results", "result"}: - let tweet = parseGraphTweet(tweetResult, false) + with tweetResult, item.getTweetResult("item"): + let tweet = parseGraphTweet(tweetResult) if not tweet.available: - tweet.id = parseBiggestInt(item{"entryId"}.getStr.getId()) + tweet.id = item.getEntryId.getId result.tweets.content.add tweet continue if i{"entries"}.notNull: for e in i{"entries"}: - let entryId = e{"entryId"}.getStr + let entryId = e.getEntryId if entryId.startsWith("tweet") or entryId.startsWith("profile-grid"): for tweet in extractTweetsFromEntry(e): result.tweets.content.add tweet @@ -483,8 +539,7 @@ proc parseGraphTimeline*(js: JsonNode; after=""): Profile = result.tweets.bottom = e{"content", "value"}.getStr if after.len == 0: - let instrType = i{"type"}.getStr(i{"__typename"}.getStr) - if instrType == "TimelinePinEntry": + if i.getTypeName == "TimelinePinEntry": let tweets = extractTweetsFromEntry(i{"entry"}) if tweets.len > 0: var tweet = tweets[0] @@ -494,23 +549,20 @@ proc parseGraphTimeline*(js: JsonNode; after=""): Profile = proc parseGraphPhotoRail*(js: JsonNode): PhotoRail = result = @[] - let instructions = - if js{"data", "user"}.notNull: - ? js{"data", "user", "result", "timeline", "timeline", "instructions"} - else: - ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"} - + let instructions = select( + js{"data", "user", "result", "timeline", "timeline", "instructions"}, + js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"} + ) if instructions.len == 0: return for i in instructions: - # TimelineAddToModule instruction is used by MediaTimelineV2 if i{"moduleItems"}.notNull: for item in i{"moduleItems"}: - with tweetResult, item{"item", "itemContent", "tweet_results", "result"}: - let t = parseGraphTweet(tweetResult, false) + with tweetResult, item.getTweetResult("item"): + let t = parseGraphTweet(tweetResult) if not t.available: - t.id = parseBiggestInt(item{"entryId"}.getStr.getId()) + t.id = item.getEntryId.getId let photo = extractGalleryPhoto(t) if photo.url.len > 0: @@ -520,12 +572,11 @@ proc parseGraphPhotoRail*(js: JsonNode): PhotoRail = return continue - let instrType = i{"type"}.getStr(i{"__typename"}.getStr) - if instrType != "TimelineAddEntries": + if i.getTypeName != "TimelineAddEntries": continue for e in i{"entries"}: - let entryId = e{"entryId"}.getStr + let entryId = e.getEntryId if entryId.startsWith("tweet") or entryId.startsWith("profile-grid"): for t in extractTweetsFromEntry(e): let photo = extractGalleryPhoto(t) @@ -538,21 +589,24 @@ proc parseGraphPhotoRail*(js: JsonNode): PhotoRail = proc parseGraphSearch*[T: User | Tweets](js: JsonNode; after=""): Result[T] = result = Result[T](beginning: after.len == 0) - let instructions = js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"} + let instructions = select( + js{"data", "search", "timeline_response", "timeline", "instructions"}, + js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"} + ) if instructions.len == 0: return for instruction in instructions: - let typ = instruction{"type"}.getStr + let typ = getTypeName(instruction) if typ == "TimelineAddEntries": for e in instruction{"entries"}: - let entryId = e{"entryId"}.getStr + let entryId = e.getEntryId when T is Tweets: if entryId.startsWith("tweet"): - with tweetRes, e{"content", "itemContent", "tweet_results", "result"}: + with tweetRes, getTweetResult(e): let tweet = parseGraphTweet(tweetRes) if not tweet.available: - tweet.id = parseBiggestInt(entryId.getId()) + tweet.id = entryId.getId result.content.add tweet elif T is User: if entryId.startsWith("user"): diff --git a/src/parserutils.nim b/src/parserutils.nim index 7e246dd..f40082c 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -36,6 +36,12 @@ template `?`*(js: JsonNode): untyped = if j.isNull: return j +template select*(a, b: JsonNode): untyped = + if a.notNull: a else: b + +template select*(a, b, c: JsonNode): untyped = + if a.notNull: a elif b.notNull: b else: c + template with*(ident, value, body): untyped = if true: let ident {.inject.} = value @@ -54,6 +60,20 @@ template getError*(js: JsonNode): Error = if js.kind != JArray or js.len == 0: null else: Error(js[0]{"code"}.getInt) +proc getTweetResult*(js: JsonNode; root="content"): JsonNode = + select( + js{root, "content", "tweet_results", "result"}, + js{root, "itemContent", "tweet_results", "result"}, + js{root, "content", "tweetResult", "result"} + ) + +template getTypeName*(js: JsonNode): string = + js{"__typename"}.getStr(js{"type"}.getStr) + +template getEntryId*(e: JsonNode): string = + e{"entryId"}.getStr(e{"entry_id"}.getStr) + + template parseTime(time: string; f: static string; flen: int): DateTime = if time.len != flen: return parse(time, f, utc()) @@ -64,29 +84,24 @@ proc getDateTime*(js: JsonNode): DateTime = proc getTime*(js: JsonNode): DateTime = parseTime(js.getStr, "ddd MMM dd hh:mm:ss \'+0000\' yyyy", 30) -proc getId*(id: string): string {.inline.} = +proc getTimeFromMs*(js: JsonNode): DateTime = + let ms = js.getInt(0) + if ms == 0: return + let seconds = ms div 1000 + return fromUnix(seconds).utc() + +proc getId*(id: string): int64 {.inline.} = let start = id.rfind("-") - if start < 0: return id - id[start + 1 ..< id.len] + if start < 0: + return parseBiggestInt(id) + return parseBiggestInt(id[start + 1 ..< id.len]) proc getId*(js: JsonNode): int64 {.inline.} = case js.kind - of JString: return parseBiggestInt(js.getStr("0")) + of JString: return js.getStr("0").getId of JInt: return js.getBiggestInt() else: return 0 -proc getEntryId*(js: JsonNode): string {.inline.} = - let entry = js{"entryId"}.getStr - if entry.len == 0: return - - if "tweet" in entry or "sq-I-t" in entry: - return entry.getId - elif "tombstone" in entry: - return js{"content", "item", "content", "tombstone", "tweet", "id"}.getStr - else: - echo "unknown entry: ", entry - return - template getStrVal*(js: JsonNode; default=""): string = js{"string_value"}.getStr(default) @@ -157,12 +172,6 @@ proc getMp4Resolution*(url: string): int = # cannot determine resolution (e.g. m3u8/non-mp4 video) return 0 -proc getVideoViewCount*(js: JsonNode): string = - with stats, js{"ext_media_stats"}: - return stats{"view_count"}.getStr($stats{"viewCount"}.getInt) - - return $js{"mediaStats", "viewCount"}.getInt(0) - proc extractSlice(js: JsonNode): Slice[int] = result = js["indices"][0].getInt ..< js["indices"][1].getInt diff --git a/src/types.nim b/src/types.nim index 5a08bb7..f16fe6f 100644 --- a/src/types.nim +++ b/src/types.nim @@ -121,7 +121,6 @@ type durationMs*: int url*: string thumb*: string - views*: string available*: bool reason*: string title*: string @@ -202,7 +201,6 @@ type replies*: int retweets*: int likes*: int - quotes*: int views*: int Tweet* = ref object diff --git a/src/views/tweet.nim b/src/views/tweet.nim index 6d76755..8ff8cb1 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -178,16 +178,12 @@ func formatStat(stat: int): string = if stat > 0: insertSep($stat, ',') else: "" -proc renderStats(stats: TweetStats; views: string): VNode = +proc renderStats(stats: TweetStats): VNode = buildHtml(tdiv(class="tweet-stats")): span(class="tweet-stat"): icon "comment", formatStat(stats.replies) span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets) - span(class="tweet-stat"): icon "quote", formatStat(stats.quotes) span(class="tweet-stat"): icon "heart", formatStat(stats.likes) - if stats.views > 0: - span(class="tweet-stat"): icon "views", formatStat(stats.views) - if views.len > 0: - span(class="tweet-stat"): icon "play", insertSep(views, ',') + span(class="tweet-stat"): icon "views", formatStat(stats.views) proc renderReply(tweet: Tweet): VNode = buildHtml(tdiv(class="replying-to")): @@ -303,7 +299,6 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0; a(class="tweet-link", href=getLink(tweet)) tdiv(class="tweet-body"): - var views = "" renderHeader(tweet, retweet, pinned, prefs) if not afterTweet and index == 0 and tweet.reply.len > 0 and @@ -327,10 +322,8 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0; renderAlbum(tweet) elif tweet.video.isSome: renderVideo(tweet.video.get(), prefs, path) - views = tweet.video.get().views elif tweet.gif.isSome: renderGif(tweet.gif.get(), prefs) - views = "GIF" if tweet.poll.isSome: renderPoll(tweet.poll.get()) @@ -345,7 +338,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0; renderMediaTags(tweet.mediaTags) if not prefs.hideTweetStats: - renderStats(tweet.stats, views) + renderStats(tweet.stats) if showThread: a(class="show-thread", href=("/i/status/" & $tweet.threadId)):