Support tweet content disclosures (AI and ads)

Fixes #1374

src/consts.nim

@@ -16,7 +16,7 @@ const
   graphUserTweetsAndReplies* = "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies"
   graphUserMedia* = "36oKqyQ7E_9CmtONGjJRsA/UserMedia"
   graphUserMediaV2* = "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2"
-  graphTweet* = "Y4Erk_-0hObvLpz0Iw3bzA/ConversationTimeline"
+  graphTweet* = "b4pV7sWOe97RncwHcGESUA/ConversationTimeline"
   graphTweetDetail* = "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail"
   graphTweetResult* = "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery"
   graphTweetEditHistory* = "upS9teTSG45aljmP9oTuXA/TweetEditHistory"

src/parser.nim

@@ -61,7 +61,8 @@ proc parseGraphUser(js: JsonNode): User =
     result.fullname = user{"core", "name"}.getStr
     result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "")
 
-    if user{"is_blue_verified"}.getBool(false):
+    if user{"is_blue_verified"}.getBool(
+       user{"verification", "is_blue_verified"}.getBool(false)):
       result.verifiedType = blue
 
     with verifiedType, user{"verification", "verified_type"}:
@@ -206,6 +207,12 @@ proc parseMediaEntities(js: JsonNode; result: var Tweet) =
           ))
         else: discard
 
+      if "expanded_url" in mediaEntity:
+        let expandedUrl = js.getExpandedUrl
+        if result.text.endsWith(expandedUrl):
+          result.text.removeSuffix(expandedUrl)
+          result.text = result.text.strip()
+
     if mediaEntities.len > 0 and parsedMedia.len == mediaEntities.len:
       result.media = parsedMedia
 
@@ -409,7 +416,7 @@ proc parseGraphTweet(js: JsonNode): Tweet =
   else:
     discard
 
-  if not js.hasKey("legacy"):
+  if "legacy" notin js and "rest_id" notin js:
     return Tweet()
 
   var jsCard = select(js{"card"}, js{"tweet_card"}, js{"legacy", "tweet_card"})
@@ -432,8 +439,41 @@ proc parseGraphTweet(js: JsonNode): Tweet =
   with restId, js{"reply_to_results", "rest_id"}:
     replyId = restId.getId
 
-  result = parseTweet(js{"legacy"}, jsCard, replyId)
-  result.id = js{"rest_id"}.getId
+  if "details" in js:
+    result = Tweet(
+      id: js{"rest_id"}.getId,
+      available: true,
+      text: js{"details", "full_text"}.getStr,
+      time: js{"details", "created_at_ms"}.getTimeFromMs,
+      replyId: js{"reply_to_results", "rest_id"}.getId,
+      isAd: js{"content_disclosure", "advertising_disclosure", "is_paid_promotion"}.getBool,
+      isAI: js{"content_disclosure", "ai_generated_disclosure", "has_ai_generated_media"}.getBool,
+      stats: TweetStats(
+        replies: js{"counts", "reply_count"}.getInt,
+        retweets: js{"counts", "retweet_count"}.getInt,
+        likes: js{"counts", "favorite_count"}.getInt,
+      )
+    )
+
+    if jsCard.kind != JNull:
+      let name = jsCard{"name"}.getStr
+      if "poll" in name:
+        if "image" in name:
+          result.media.addMedia(Photo(
+            url: jsCard{"binding_values", "image_large"}.getImageVal
+          ))
+
+        result.poll = some parsePoll(jsCard)
+      elif name == "amplify":
+        result.media.addMedia(parsePromoVideo(jsCard{"binding_values"}))
+      else:
+        result.card = some parseCard(jsCard, js{"url_entities"})
+
+    result.expandTweetEntitiesV2(js)
+  else:
+    result = parseTweet(js{"legacy"}, jsCard, replyId)
+    result.id = js{"rest_id"}.getId
+
   result.user = parseGraphUser(js{"core"})
 
   if result.reply.len == 0:

src/parserutils.nim

@@ -320,6 +320,58 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
 
   tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote or hasJobCard)
 
+proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: Slice[int];
+                          hasRedundantLink=false) =
+  let hasCard = tweet.card.isSome
+
+  var replacements = newSeq[ReplaceSlice]()
+
+  with urls, js{"url_entities"}:
+    for u in urls:
+      let urlStr = u["url"].getStr
+      if urlStr.len == 0 or urlStr notin text:
+        continue
+
+      replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink)
+
+      if hasCard and u{"url"}.getStr == get(tweet.card).url:
+        get(tweet.card).url = u.getExpandedUrl
+
+  with hashtags, js{"details", "hashtag_entities"}:
+    for hashtag in hashtags:
+      replacements.extractHashtags(hashtag)
+
+  with cashtags, js{"details", "cashtag_entities"}:
+    for cashtag in cashtags:
+      replacements.extractHashtags(cashtag)
+
+  with mentions, js{"mention_entities"}:
+    for mention in mentions:
+      let
+        name = mention{"screen_name"}.getStr
+        slice = mention.extractSlice
+        idx = tweet.reply.find(name)
+
+      if slice.a >= textSlice.a:
+        replacements.add ReplaceSlice(kind: rkMention, slice: slice,
+          url: "/" & name, display: mention["name"].getStr)
+      elif idx == -1 and tweet.replyId != 0:
+        tweet.reply.add name
+
+  replacements.deduplicate
+  replacements.sort(cmp)
+
+  tweet.text = text.toRunes.replacedWith(replacements, textSlice).strip(leading=false)
+
+proc expandTweetEntitiesV2*(tweet: Tweet; js: JsonNode) =
+  let
+    textRange = js{"details", "display_text_range"}
+    textSlice = textRange{0}.getInt .. textRange{1}.getInt
+    hasQuote = "quoted_tweet_results" in js
+    hasJobCard = tweet.card.isSome and get(tweet.card).kind == jobDetails
+
+  tweet.expandTextEntitiesV2(js, tweet.text, textSlice, hasQuote or hasJobCard)
+
 proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) =
   let
     entities = ? js{"entity_set"}

src/sass/tweet/_base.scss

@@ -80,8 +80,8 @@
 }
 
 .tweet-published {
-  margin-top: 10px;
-  margin-bottom: 3px;
+  margin-top: 6px;
+  margin-bottom: 0px;
   color: var(--grey);
   pointer-events: all;
 }
@@ -292,3 +292,16 @@
   padding: 10px 10px;
   padding-top: 6px;
 }
+
+.disclosures {
+  display: flex;
+  flex-direction: column;
+  color: var(--grey);
+  font-size: 14px;
+  margin-top: 4px;
+  margin-bottom: -2px;
+
+  .icon-attention {
+    margin-right: -3px;
+  }
+}

src/types.nim

@@ -240,6 +240,8 @@ type
     media*: MediaEntities
     history*: seq[int64]
     note*: string
+    isAd*: bool
+    isAI*: bool
 
   Tweets* = seq[Tweet]
 

src/views/tweet.nim

@@ -257,10 +257,6 @@ proc renderLatestPost(username: string; id: int64): VNode =
     a(href=getLink(id, username)):
       text "See the latest post"
 
-proc renderQuoteMedia(quote: Tweet; prefs: Prefs; path: string): VNode =
-  buildHtml(tdiv(class="quote-media-container")):
-    renderMedia(quote.media, prefs, path)
-
 proc renderCommunityNote(note: string; prefs: Prefs): VNode =
   buildHtml(tdiv(class="community-note")):
     tdiv(class="community-note-header"):
@@ -269,6 +265,10 @@ proc renderCommunityNote(note: string; prefs: Prefs): VNode =
     tdiv(class="community-note-text", dir="auto"):
       verbatim replaceUrls(note, prefs)
 
+proc renderQuoteMedia(quote: Tweet; prefs: Prefs; path: string): VNode =
+  buildHtml(tdiv(class="quote-media-container")):
+    renderMedia(quote.media, prefs, path)
+
 proc renderQuote(quote: Tweet; prefs: Prefs; path: string): VNode =
   if not quote.available:
     return buildHtml(tdiv(class="quote unavailable")):
@@ -315,6 +315,15 @@ proc renderQuote(quote: Tweet; prefs: Prefs; path: string): VNode =
       tdiv(class="quote-latest"):
         text "There's a new version of this post"
 
+proc renderDisclosures*(tweet: Tweet): VNode =
+  buildHtml(tdiv(class="disclosures")):
+    if tweet.isAI:
+      span(data-disclosure="ai"):
+        icon "attention", "Made with AI"
+    if tweet.isAd:
+      span(data-disclosure="ad"):
+        icon "attention", "Paid partnership (ad)"
+
 proc renderLocation*(tweet: Tweet): string =
   let (place, url) = tweet.getLocation()
   if place.len == 0: return
@@ -391,6 +400,9 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
       if tweet.note.len > 0 and not prefs.hideCommunityNotes:
         renderCommunityNote(tweet.note, prefs)
 
+      if tweet.isAI or tweet.isAd:
+        renderDisclosures(tweet)
+
       let
         hasEdits = tweet.history.len > 1
         isLatest = hasEdits and tweet.id == max(tweet.history)