Support tweet content disclosures (AI and ads)

Fixes #1374

public/css/fontello.css

@@ -1,12 +1,12 @@
 @font-face {
   font-family: "fontello";
-  src: url("/fonts/fontello.eot?42791196");
+  src: url("/fonts/fontello.eot?49059696");
   src:
-    url("/fonts/fontello.eot?42791196#iefix") format("embedded-opentype"),
-    url("/fonts/fontello.woff2?42791196") format("woff2"),
-    url("/fonts/fontello.woff?42791196") format("woff"),
-    url("/fonts/fontello.ttf?42791196") format("truetype"),
-    url("/fonts/fontello.svg?42791196#fontello") format("svg");
+    url("/fonts/fontello.eot?49059696#iefix") format("embedded-opentype"),
+    url("/fonts/fontello.woff2?49059696") format("woff2"),
+    url("/fonts/fontello.woff?49059696") format("woff"),
+    url("/fonts/fontello.ttf?49059696") format("truetype"),
+    url("/fonts/fontello.svg?49059696#fontello") format("svg");
   font-weight: normal;
   font-style: normal;
 }
@@ -126,6 +126,11 @@
 }
 
 /* '' */
+.icon-attention:before {
+  content: "\e812";
+}
+
+/* '' */
 .icon-circle:before {
   content: "\f111";
 }

public/fonts/fontello.svg

@@ -42,6 +42,8 @@
 
 <glyph glyph-name="ok" unicode="&#xe811;" d="M933 534q0-22-16-38l-404-404-76-76q-16-15-38-15t-38 15l-76 76-202 202q-15 16-15 38t15 38l76 76q16 16 38 16t38-16l164-165 366 367q16 16 38 16t38-16l76-76q16-15 16-38z" horiz-adv-x="1000" />
 
+<glyph glyph-name="attention-circled" unicode="&#xe812;" d="M429 779q116 0 215-58t156-156 57-215-57-215-156-156-215-58-216 58-155 156-58 215 58 215 155 156 216 58z m71-696v106q0 8-5 13t-12 5h-107q-8 0-13-5t-6-13v-106q0-8 6-13t13-6h107q7 0 12 6t5 13z m-1 192l10 346q0 7-6 10-5 5-13 5h-123q-8 0-13-5-6-3-6-10l10-346q0-6 5-10t14-4h103q8 0 13 4t6 10z" horiz-adv-x="857.1" />
+
 <glyph glyph-name="circle" unicode="&#xf111;" d="M857 350q0-117-57-215t-156-156-215-58-216 58-155 156-58 215 58 215 155 156 216 58 215-58 156-156 57-215z" horiz-adv-x="857.1" />
 
 <glyph glyph-name="info" unicode="&#xf128;" d="M393 149v-134q0-9-7-15t-15-7h-134q-9 0-16 7t-7 15v134q0 9 7 16t16 6h134q9 0 15-6t7-16z m176 335q0-30-8-56t-20-43-31-33-32-25-34-19q-23-13-38-37t-15-37q0-10-7-18t-16-9h-134q-8 0-14 11t-6 20v26q0 46 37 87t79 60q33 16 47 32t14 42q0 24-26 41t-60 18q-36 0-60-16-20-14-60-64-7-9-17-9-7 0-14 4l-91 70q-8 6-9 14t3 16q89 148 259 148 45 0 90-17t81-46 59-72 23-88z" horiz-adv-x="571.4" />

src/consts.nim

@@ -16,7 +16,7 @@ const
   graphUserTweetsAndReplies* = "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies"
   graphUserMedia* = "36oKqyQ7E_9CmtONGjJRsA/UserMedia"
   graphUserMediaV2* = "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2"
-  graphTweet* = "Y4Erk_-0hObvLpz0Iw3bzA/ConversationTimeline"
+  graphTweet* = "b4pV7sWOe97RncwHcGESUA/ConversationTimeline"
   graphTweetDetail* = "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail"
   graphTweetResult* = "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery"
   graphTweetEditHistory* = "upS9teTSG45aljmP9oTuXA/TweetEditHistory"

src/parser.nim

@@ -61,7 +61,8 @@ proc parseGraphUser(js: JsonNode): User =
     result.fullname = user{"core", "name"}.getStr
     result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "")
 
-    if user{"is_blue_verified"}.getBool(false):
+    if user{"is_blue_verified"}.getBool(
+       user{"verification", "is_blue_verified"}.getBool(false)):
       result.verifiedType = blue
 
     with verifiedType, user{"verification", "verified_type"}:
@@ -206,6 +207,12 @@ proc parseMediaEntities(js: JsonNode; result: var Tweet) =
           ))
         else: discard
 
+      if "expanded_url" in mediaEntity:
+        let expandedUrl = js.getExpandedUrl
+        if result.text.endsWith(expandedUrl):
+          result.text.removeSuffix(expandedUrl)
+          result.text = result.text.strip()
+
     if mediaEntities.len > 0 and parsedMedia.len == mediaEntities.len:
       result.media = parsedMedia
 
@@ -409,7 +416,7 @@ proc parseGraphTweet(js: JsonNode): Tweet =
   else:
     discard
 
-  if not js.hasKey("legacy"):
+  if "legacy" notin js and "rest_id" notin js:
     return Tweet()
 
   var jsCard = select(js{"card"}, js{"tweet_card"}, js{"legacy", "tweet_card"})
@@ -432,8 +439,41 @@ proc parseGraphTweet(js: JsonNode): Tweet =
   with restId, js{"reply_to_results", "rest_id"}:
     replyId = restId.getId
 
+  if "details" in js:
+    result = Tweet(
+      id: js{"rest_id"}.getId,
+      available: true,
+      text: js{"details", "full_text"}.getStr,
+      time: js{"details", "created_at_ms"}.getTimeFromMs,
+      replyId: js{"reply_to_results", "rest_id"}.getId,
+      isAd: js{"content_disclosure", "advertising_disclosure", "is_paid_promotion"}.getBool,
+      isAI: js{"content_disclosure", "ai_generated_disclosure", "has_ai_generated_media"}.getBool,
+      stats: TweetStats(
+        replies: js{"counts", "reply_count"}.getInt,
+        retweets: js{"counts", "retweet_count"}.getInt,
+        likes: js{"counts", "favorite_count"}.getInt,
+      )
+    )
+
+    if jsCard.kind != JNull:
+      let name = jsCard{"name"}.getStr
+      if "poll" in name:
+        if "image" in name:
+          result.media.addMedia(Photo(
+            url: jsCard{"binding_values", "image_large"}.getImageVal
+          ))
+
+        result.poll = some parsePoll(jsCard)
+      elif name == "amplify":
+        result.media.addMedia(parsePromoVideo(jsCard{"binding_values"}))
+      else:
+        result.card = some parseCard(jsCard, js{"url_entities"})
+
+    result.expandTweetEntitiesV2(js)
+  else:
     result = parseTweet(js{"legacy"}, jsCard, replyId)
     result.id = js{"rest_id"}.getId
+
   result.user = parseGraphUser(js{"core"})
 
   if result.reply.len == 0:

src/parserutils.nim

@@ -320,6 +320,58 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
 
   tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote or hasJobCard)
 
+proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: Slice[int];
+                          hasRedundantLink=false) =
+  let hasCard = tweet.card.isSome
+
+  var replacements = newSeq[ReplaceSlice]()
+
+  with urls, js{"url_entities"}:
+    for u in urls:
+      let urlStr = u["url"].getStr
+      if urlStr.len == 0 or urlStr notin text:
+        continue
+
+      replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink)
+
+      if hasCard and u{"url"}.getStr == get(tweet.card).url:
+        get(tweet.card).url = u.getExpandedUrl
+
+  with hashtags, js{"details", "hashtag_entities"}:
+    for hashtag in hashtags:
+      replacements.extractHashtags(hashtag)
+
+  with cashtags, js{"details", "cashtag_entities"}:
+    for cashtag in cashtags:
+      replacements.extractHashtags(cashtag)
+
+  with mentions, js{"mention_entities"}:
+    for mention in mentions:
+      let
+        name = mention{"screen_name"}.getStr
+        slice = mention.extractSlice
+        idx = tweet.reply.find(name)
+
+      if slice.a >= textSlice.a:
+        replacements.add ReplaceSlice(kind: rkMention, slice: slice,
+          url: "/" & name, display: mention["name"].getStr)
+      elif idx == -1 and tweet.replyId != 0:
+        tweet.reply.add name
+
+  replacements.deduplicate
+  replacements.sort(cmp)
+
+  tweet.text = text.toRunes.replacedWith(replacements, textSlice).strip(leading=false)
+
+proc expandTweetEntitiesV2*(tweet: Tweet; js: JsonNode) =
+  let
+    textRange = js{"details", "display_text_range"}
+    textSlice = textRange{0}.getInt .. textRange{1}.getInt
+    hasQuote = "quoted_tweet_results" in js
+    hasJobCard = tweet.card.isSome and get(tweet.card).kind == jobDetails
+
+  tweet.expandTextEntitiesV2(js, tweet.text, textSlice, hasQuote or hasJobCard)
+
 proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) =
   let
     entities = ? js{"entity_set"}

src/sass/tweet/_base.scss

@@ -80,8 +80,8 @@
 }
 
 .tweet-published {
-  margin-top: 10px;
-  margin-bottom: 3px;
+  margin-top: 6px;
+  margin-bottom: 0px;
   color: var(--grey);
   pointer-events: all;
 }
@@ -292,3 +292,16 @@
   padding: 10px 10px;
   padding-top: 6px;
 }
+
+.disclosures {
+  display: flex;
+  flex-direction: column;
+  color: var(--grey);
+  font-size: 14px;
+  margin-top: 4px;
+  margin-bottom: -2px;
+
+  .icon-attention {
+    margin-right: -3px;
+  }
+}

src/types.nim

@@ -240,6 +240,8 @@ type
     media*: MediaEntities
     history*: seq[int64]
     note*: string
+    isAd*: bool
+    isAI*: bool
 
   Tweets* = seq[Tweet]
 

src/views/tweet.nim

@@ -257,10 +257,6 @@ proc renderLatestPost(username: string; id: int64): VNode =
     a(href=getLink(id, username)):
       text "See the latest post"
 
-proc renderQuoteMedia(quote: Tweet; prefs: Prefs; path: string): VNode =
-  buildHtml(tdiv(class="quote-media-container")):
-    renderMedia(quote.media, prefs, path)
-
 proc renderCommunityNote(note: string; prefs: Prefs): VNode =
   buildHtml(tdiv(class="community-note")):
     tdiv(class="community-note-header"):
@@ -269,6 +265,10 @@ proc renderCommunityNote(note: string; prefs: Prefs): VNode =
     tdiv(class="community-note-text", dir="auto"):
       verbatim replaceUrls(note, prefs)
 
+proc renderQuoteMedia(quote: Tweet; prefs: Prefs; path: string): VNode =
+  buildHtml(tdiv(class="quote-media-container")):
+    renderMedia(quote.media, prefs, path)
+
 proc renderQuote(quote: Tweet; prefs: Prefs; path: string): VNode =
   if not quote.available:
     return buildHtml(tdiv(class="quote unavailable")):
@@ -315,6 +315,15 @@ proc renderQuote(quote: Tweet; prefs: Prefs; path: string): VNode =
       tdiv(class="quote-latest"):
         text "There's a new version of this post"
 
+proc renderDisclosures*(tweet: Tweet): VNode =
+  buildHtml(tdiv(class="disclosures")):
+    if tweet.isAI:
+      span(data-disclosure="ai"):
+        icon "attention", "Made with AI"
+    if tweet.isAd:
+      span(data-disclosure="ad"):
+        icon "attention", "Paid partnership (ad)"
+
 proc renderLocation*(tweet: Tweet): string =
   let (place, url) = tweet.getLocation()
   if place.len == 0: return
@@ -391,6 +400,9 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
       if tweet.note.len > 0 and not prefs.hideCommunityNotes:
         renderCommunityNote(tweet.note, prefs)
 
+      if tweet.isAI or tweet.isAd:
+        renderDisclosures(tweet)
+
       let
         hasEdits = tweet.history.len > 1
         isLatest = hasEdits and tweet.id == max(tweet.history)