diff --git a/src/api.nim b/src/api.nim index f3c6c12..c66ca3a 100644 --- a/src/api.nim +++ b/src/api.nim @@ -25,27 +25,27 @@ proc mediaUrl(id, cursor: string; count=20): ApiReq = ) proc userTweetsUrl(id: string; cursor: string): ApiReq = - result = ApiReq( - # cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles), - oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor, "20"]) - ) - # might change this in the future pending testing - result.cookie = result.oauth + return apiReq(graphUserTweetsV2, restIdVars % [id, cursor, "20"]) + # result = ApiReq( + # cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles), + # oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor, "20"]) + # ) proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq = - let cookieVars = userTweetsAndRepliesVars % [id, cursor] - result = ApiReq( - cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles), - oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"]) - ) + return apiReq(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"]) + #let cookieVars = userTweetsAndRepliesVars % [id, cursor] + # result = ApiReq( + # cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles), + # oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"]) + # ) proc tweetDetailUrl(id: string; cursor: string): ApiReq = - let cookieVars = tweetDetailVars % [id, cursor] - result = ApiReq( - # cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles), - cookie: apiUrl(graphTweet, tweetVars % [id, cursor]), - oauth: apiUrl(graphTweet, tweetVars % [id, cursor]) - ) + return apiReq(graphTweet, tweetVars % [id, cursor]) + # let cookieVars = tweetDetailVars % [id, cursor] + # result = ApiReq( + # cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles), + # oauth: apiUrl(graphTweet, tweetVars % [id, cursor]) + # ) proc userUrl(username: string): ApiReq = let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username @@ -184,13 +184,13 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} = var variables = %*{ "rawQuery": q, - "query_source": "typedQuery", "count": 20, + "querySource": "typed_query", "product": "Latest", - "withDownvotePerspective": false, - "withReactionsMetadata": false, - "withReactionsPerspective": false + "withGrokTranslatedBio":true, + "withQuickPromoteEligibilityTweetFields":false } + if after.len > 0 and maxId.len == 0: variables["cursor"] = % after let @@ -212,12 +212,11 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} var variables = %*{ "rawQuery": query.text, - "query_source": "typedQuery", "count": 20, + "querySource": "typed_query", "product": "People", - "withDownvotePerspective": false, - "withReactionsMetadata": false, - "withReactionsPerspective": false + "withGrokTranslatedBio":true, + "withQuickPromoteEligibilityTweetFields":false } if after.len > 0: variables["cursor"] = % after diff --git a/src/apiutils.nim b/src/apiutils.nim index 6f4f727..bf6c4b9 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -84,12 +84,13 @@ proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} = result["x-twitter-auth-type"] = "OAuth2Session" result["x-csrf-token"] = session.ct0 result["cookie"] = getCookieHeader(session.authToken, session.ct0) + result["referer"] = "https://x.com/" result["sec-ch-ua"] = """"Google Chrome";v="142", "Chromium";v="142", "Not A(Brand";v="24"""" result["sec-ch-ua-mobile"] = "?0" result["sec-ch-ua-platform"] = "Windows" result["sec-fetch-dest"] = "empty" result["sec-fetch-mode"] = "cors" - result["sec-fetch-site"] = "same-site" + result["sec-fetch-site"] = "same-origin" if disableTid or "/1.1/" in url.path: result["authorization"] = bearerToken2 else: @@ -114,7 +115,9 @@ template fetchImpl(result, fetchBody) {.dirty.} = try: var resp: AsyncResponse - pool.use(await genHeaders(session, url)): + let headers = await genHeaders(session, url) + + pool.use(headers): template getContent = # TODO: this is a temporary simple implementation if apiProxy.len > 0 and "/1.1/" notin url.path: diff --git a/src/consts.nim b/src/consts.nim index 29a582b..f903528 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -7,109 +7,70 @@ const bearerToken* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" bearerToken2* = "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F" - graphUser* = "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName" - graphUserV2* = "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery" - graphUserById* = "VN33vKXrPT7p35DgNR27aw/UserResultByIdQuery" - graphUserTweetsV2* = "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2" - graphUserTweetsAndRepliesV2* = "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2" - graphUserTweets* = "oRJs8SLCRNRbQzuZG93_oA/UserTweets" - graphUserTweetsAndReplies* = "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies" - graphUserMedia* = "36oKqyQ7E_9CmtONGjJRsA/UserMedia" - graphUserMediaV2* = "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2" - graphTweet* = "b4pV7sWOe97RncwHcGESUA/ConversationTimeline" - graphTweetDetail* = "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail" - graphTweetResult* = "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery" - graphTweetEditHistory* = "upS9teTSG45aljmP9oTuXA/TweetEditHistory" - graphSearchTimeline* = "bshMIjqDk8LTXTq4w91WKw/SearchTimeline" - graphListById* = "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId" - graphListBySlug* = "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug" - graphListMembers* = "fuVHh5-gFn8zDBBxb8wOMA/ListMembers" - graphListTweets* = "VQf8_XQynI3WzH6xopOMMQ/ListTimeline" - graphAboutAccount* = "zs_jFPFT78rBpXv9Z3U2YQ/AboutAccountQuery" + graphUser* = "IGgvgiOx4QZndDHuD3x9TQ/UserByScreenName" + graphUserV2* = "-ZzAG_Bckx16LMbEvHC3lg/UserResultByScreenNameQuery" + graphUserById* = "-DAaa9jPxPswYeI2fZ9rug/UserResultByIdQuery" + graphUserTweetsV2* = "PHTSTXqZYuHIeK4B1HQprQ/UserWithProfileTweetsQueryV2" + graphUserTweetsAndRepliesV2* = "AcYHjc_YAx-9_rKWdMsKvA/UserWithProfileTweetsAndRepliesQueryV2" + graphUserTweets* = "PNd0vlufvrcIwrAnBYKE9g/UserTweets" + graphUserTweetsAndReplies* = "EqtpEwt0CoQXmDfq5DKH0A/UserTweetsAndReplies" + graphUserMedia* = "g_rGPF0fLON-M9cyVjXuzA/UserMedia" + graphUserMediaV2* = "WK111rbR0vM0ZX4lyZCYjw/MediaTimelineV2" + graphTweet* = "OZMbEnEa96AN8Pq6HyTWdw/ConversationTimeline" + graphTweetDetail* = "6uCvnic3m5reVuehkvHa3w/TweetDetail" + graphTweetResult* = "xYOrBQoTlfKJJPsX76MZEw/TweetResultByIdQuery" + graphTweetEditHistory* = "MGElmrYILE8wUfI8GorUYA/TweetEditHistory" + graphSearchTimeline* = "-TFXKoMnMTKdEXcCn-eahw/SearchTimeline" - graphBroadcast* = "0nMmbMh-_JwwRRFNXkyH3Q/BroadcastQuery" + graphListById* = "t9AbdyHaJVfjL9jsODwgpQ/ListByRestId" + graphListBySlug* = "LDQpQ89B5ipR8izCKrWU0g/ListBySlug" + graphListMembers* = "EM7YRaM3gCnzDESmchA7RA/ListMembers" + graphListTweets* = "0QJtcuMzVywHGAWD6Dtjlw/ListTimeline" + graphAboutAccount* = "zUnx-DLN9dkwOkNhTLySjg/AboutAccountQuery" + + graphBroadcast* = "FJLCzpXCLPM1jUZqmM7oEA/BroadcastQuery" restLiveStream* = "1.1/live_video_stream/status/" gqlFeatures* = """{ - "android_ad_formats_media_component_render_overlay_enabled": false, - "android_graphql_skip_api_media_color_palette": false, - "android_professional_link_spotlight_display_enabled": false, - "articles_api_enabled": false, - "articles_preview_enabled": true, - "blue_business_profile_image_shape_enabled": false, - "c9s_tweet_anatomy_moderator_badge_enabled": true, - "commerce_android_shop_module_enabled": false, - "communities_web_enable_tweet_community_results_fetch": true, - "creator_subscriptions_quote_tweet_preview_enabled": false, - "creator_subscriptions_subscription_count_enabled": false, - "creator_subscriptions_tweet_preview_api_enabled": true, - "freedom_of_speech_not_reach_fetch_enabled": true, - "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, - "grok_android_analyze_trend_fetch_enabled": false, - "grok_translations_community_note_auto_translation_is_enabled": false, - "grok_translations_community_note_translation_is_enabled": false, - "grok_translations_post_auto_translation_is_enabled": false, - "grok_translations_timeline_user_bio_auto_translation_is_enabled": false, - "hidden_profile_likes_enabled": false, - "highlights_tweets_tab_ui_enabled": false, - "immersive_video_status_linkable_timestamps": false, - "interactive_text_enabled": false, - "longform_notetweets_consumption_enabled": true, - "longform_notetweets_inline_media_enabled": true, - "longform_notetweets_richtext_consumption_enabled": true, - "longform_notetweets_rich_text_read_enabled": true, - "mobile_app_spotlight_module_enabled": false, - "payments_enabled": false, - "post_ctas_fetch_enabled": true, - "premium_content_api_read_enabled": false, + "rweb_video_screen_enabled": false, + "rweb_cashtags_enabled": true, "profile_label_improvements_pcf_label_in_post_enabled": true, - "profile_label_improvements_pcf_label_in_profile_enabled": false, - "responsive_web_edit_tweet_api_enabled": true, - "responsive_web_enhance_cards_enabled": false, - "responsive_web_graphql_exclude_directive_enabled": true, - "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, + "responsive_web_profile_redirect_enabled": false, + "rweb_tipjar_consumption_enabled": false, + "verified_phone_label_enabled": false, + "creator_subscriptions_tweet_preview_api_enabled": true, "responsive_web_graphql_timeline_navigation_enabled": true, - "responsive_web_grok_analysis_button_from_backend": true, + "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, + "premium_content_api_read_enabled": false, + "communities_web_enable_tweet_community_results_fetch": true, + "c9s_tweet_anatomy_moderator_badge_enabled": true, "responsive_web_grok_analyze_button_fetch_trends_enabled": false, "responsive_web_grok_analyze_post_followups_enabled": true, + "rweb_cashtags_composer_attachment_enabled": true, + "responsive_web_jetfuel_frame": true, + "responsive_web_grok_share_attachment_enabled": true, "responsive_web_grok_annotations_enabled": true, - "responsive_web_grok_community_note_auto_translation_is_enabled": false, + "articles_preview_enabled": true, + "responsive_web_edit_tweet_api_enabled": true, + "rweb_conversational_replies_downvote_enabled": false, + "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, + "view_counts_everywhere_api_enabled": true, + "longform_notetweets_consumption_enabled": true, + "responsive_web_twitter_article_tweet_consumption_enabled": true, + "content_disclosure_indicator_enabled": true, + "content_disclosure_ai_generated_indicator_enabled": true, + "responsive_web_grok_show_grok_translated_post": true, + "responsive_web_grok_analysis_button_from_backend": true, + "post_ctas_fetch_enabled": true, + "freedom_of_speech_not_reach_fetch_enabled": true, + "standardized_nudges_misinfo": true, + "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, + "longform_notetweets_rich_text_read_enabled": true, + "longform_notetweets_inline_media_enabled": false, "responsive_web_grok_image_annotation_enabled": true, "responsive_web_grok_imagine_annotation_enabled": true, - "responsive_web_grok_share_attachment_enabled": true, - "responsive_web_grok_show_grok_translated_post": false, - "responsive_web_jetfuel_frame": true, - "responsive_web_media_download_video_enabled": false, - "responsive_web_profile_redirect_enabled": false, - "responsive_web_text_conversations_enabled": false, - "responsive_web_twitter_article_notes_tab_enabled": false, - "responsive_web_twitter_article_tweet_consumption_enabled": true, - "responsive_web_twitter_blue_verified_badge_is_enabled": true, - "rweb_lists_timeline_redesign_enabled": true, - "rweb_tipjar_consumption_enabled": true, - "rweb_video_screen_enabled": false, - "rweb_video_timestamps_enabled": false, - "spaces_2022_h2_clipping": true, - "spaces_2022_h2_spaces_communities": true, - "standardized_nudges_misinfo": true, - "subscriptions_feature_can_gift_premium": false, - "subscriptions_verification_info_enabled": true, - "subscriptions_verification_info_is_identity_verified_enabled": false, - "subscriptions_verification_info_reason_enabled": true, - "subscriptions_verification_info_verified_since_enabled": true, - "super_follow_badge_privacy_enabled": false, - "super_follow_exclusive_tweet_notifications_enabled": false, - "super_follow_tweet_api_enabled": false, - "super_follow_user_api_enabled": false, - "tweet_awards_web_tipping_enabled": false, - "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, - "tweetypie_unmention_optimization_enabled": false, - "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false, - "unified_cards_destination_url_params_enabled": false, - "verified_phone_label_enabled": false, - "vibe_api_enabled": false, - "view_counts_everywhere_api_enabled": true, - "hidden_profile_subscriptions_enabled": false + "responsive_web_grok_community_note_auto_translation_is_enabled": true, + "responsive_web_enhance_cards_enabled": false }""".replace(" ", "").replace("\n", "") tweetVars* = """{ diff --git a/src/parser.nim b/src/parser.nim index d55c546..902aebd 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -4,7 +4,7 @@ import packedjson, packedjson/deserialiser import types, parserutils, utils import experimental/parser/unifiedcard -proc parseGraphTweet(js: JsonNode): Tweet +proc parseGraphTweet*(js: JsonNode): Tweet proc parseVerifiedType(s: string; current: VerifiedType): VerifiedType = try: parseEnum[VerifiedType](s) @@ -46,10 +46,10 @@ proc parseUser(js: JsonNode; id=""): User = proc parseGraphUser(js: JsonNode): User = var user = js{"user_result", "result"} if user.isNull: - user = ? js{"user_results", "result"} + user = js{"user_results", "result"} if user.isNull: - if js{"core"}.notNull and js{"legacy"}.notNull: + if js{"core"}.notNull: user = js else: return @@ -61,6 +61,7 @@ proc parseGraphUser(js: JsonNode): User = # fallback to support UserMedia/recent GraphQL updates if result.username.len == 0: + result.id = user{"rest_id"}.getStr result.username = user{"core", "screen_name"}.getStr result.fullname = user{"core", "name"}.getStr result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "") @@ -261,6 +262,20 @@ proc parseMediaEntities(js: JsonNode; result: var Tweet) = durationMs: mediaInfo{"duration_millis"}.getInt, variants: parseVideoVariants(mediaInfo{"variants"}) )) + + # Parse source user for video attribution + with sourceUser, mediaEntity{"source_user_results", "result"}: + if result.attribution.isNone: + let + expanded = mediaEntity{"expanded_url"}.getStr + pathStart = expanded.find('/', expanded.find("://") + 3) + if pathStart >= 0: + result.attributionLink = expanded[pathStart .. ^1].replace("/video/1", "") + result.attribution = some(User( + id: sourceUser{"rest_id"}.getStr, + fullname: sourceUser{"core", "name"}.getStr, + userPic: sourceUser{"avatar", "image_url"}.getImageStr.replace("_normal", "") + )) of "ApiGif": parsedMedia.addMedia(Gif( url: mediaInfo{"variants"}[0]{"url"}.getImageStr, @@ -428,13 +443,13 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull(); # graphql with rt, js{"retweeted_status_result", "result"}: # needed due to weird edgecase where the actual tweet data isn't included - if "legacy" in rt: + if "legacy" in rt or "rest_id" in rt: result.retweet = some parseGraphTweet(rt) return with reposts, js{"repostedStatusResults"}: with rt, reposts{"result"}: - if "legacy" in rt: + if "legacy" in rt or "rest_id" in rt: result.retweet = some parseGraphTweet(rt) return @@ -449,7 +464,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull(); result.poll = some parsePoll(jsCard) elif name == "amplify": result.media.addMedia(parsePromoVideo(jsCard{"binding_values"})) - else: + elif name.len > 0 and jsCard{"binding_values"}.notNull: result.card = some parseCard(jsCard, js{"entities", "urls"}) result.expandTweetEntities(js) @@ -469,7 +484,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull(); result.text.removeSuffix(" Learn more.") result.available = false -proc parseGraphTweet(js: JsonNode): Tweet = +proc parseGraphTweet*(js: JsonNode): Tweet = if js.kind == JNull: return Tweet() @@ -537,10 +552,21 @@ proc parseGraphTweet(js: JsonNode): Tweet = result.poll = some parsePoll(jsCard) elif name == "amplify": result.media.addMedia(parsePromoVideo(jsCard{"binding_values"})) - else: + elif name.len > 0 and jsCard{"binding_values"}.notNull: result.card = some parseCard(jsCard, js{"url_entities"}) result.expandTweetEntitiesV2(js) + + # Strip video source URL from text (for videos from other tweets) + with mediaEntities, js{"media_entities"}: + for m in mediaEntities: + if "source_status_id_str" in m: + let mediaUrl = m{"url"}.getStr + if mediaUrl.len > 0: + let idx = result.text.rfind(mediaUrl) + if idx >= 0: + result.text = result.text[0 ..< idx].strip() + break else: result = parseTweet(js{"legacy"}, jsCard, replyId) result.id = js{"rest_id"}.getId @@ -559,6 +585,12 @@ proc parseGraphTweet(js: JsonNode): Tweet = parseMediaEntities(js, result) + # Handle retweets - check both legacy and top-level paths + with reposts, js{"legacy", "repostedStatusResults"}: + with rt, reposts{"result"}: + if "legacy" in rt or "rest_id" in rt: + result.retweet = some parseGraphTweet(rt) + with quoted, js{"quoted_status_result", "result"}: result.quote = some(parseGraphTweet(quoted))