Update and speed up GitHub workflows (#1368)

* Update actions and switch to GitHub runners

* Bump workflow Python version to 3.14

* Reuse nitter build for integration test

* Add missing libpcre3 installation to workflow

* Consolidate workflow runtime deps install

* Make nitter binary executable

* Run nimble md and scss simultaneously in workflow

* Run tests with 4 workers in workflow

* Rerun failing integration tests

* Bump integration test workers to 5

* Improve python dep install and run less workers

* Use native GitHub Actions Redis service

* Lower integration test workers to 2

* Switch to poetry to cache venv

* Ensure poetry is installed before setup-python

* Fix poetry sync command

* Switch back to 3 workers

* Cache poetry install

* WIP

* WIP

* Fix poetry/pipx caching

* Speed up integration test significantly

* WIP

* Cleanup

.github/workflows/build-docker.yml

@@ -11,20 +11,19 @@ jobs:
   tests:
     uses: ./.github/workflows/run-tests.yml
     secrets: inherit
+
   build-docker-amd64:
     needs: [tests]
-    runs-on: buildjet-2vcpu-ubuntu-2204
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
+      - uses: actions/checkout@v6
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           version: latest
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
@@ -36,20 +35,19 @@ jobs:
           platforms: linux/amd64
           push: true
           tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}
+
   build-docker-arm64:
     needs: [tests]
-    runs-on: buildjet-2vcpu-ubuntu-2204-arm
+    runs-on: ubuntu-24.04-arm
     steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
+      - uses: actions/checkout@v6
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           version: latest
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/run-tests.yml

@@ -20,19 +20,17 @@ defaults:
 jobs:
   build-test:
     name: Build and test
-    runs-on: buildjet-2vcpu-ubuntu-2204
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         nim: ["2.0.x", "2.2.x", "devel"]
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
+        uses: actions/checkout@v6
 
       - name: Cache Nimble Dependencies
         id: cache-nimble
-        uses: buildjet/cache@v4
+        uses: actions/cache@v5
         with:
           path: ~/.nimble
           key: ${{ matrix.nim }}-nimble-v2-${{ hashFiles('*.nimble') }}
@@ -47,62 +45,100 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build Project
-        run: nimble build -d:release -Y
+        run: nimble build -Y
+
+      - name: Upload 2.2.x build artifact
+        if: matrix.nim == '2.2.x'
+        uses: actions/upload-artifact@v6
+        with:
+          name: nitter-linux-nim-2.2.x-${{ github.sha }}
+          path: |
+            ./nitter
+          if-no-files-found: error
 
   integration-test:
     needs: [build-test]
     name: Integration test
-    runs-on: buildjet-2vcpu-ubuntu-2204
+    runs-on: ubuntu-24.04
+
+    services:
+      redis:
+        image: redis:7
+        ports:
+          - 6379:6379
+
     steps:
+      - name: Install runtime deps
+        run: |
+          sudo apt-get install -y --no-install-recommends libsass-dev libpcre3
+
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
+
+      - name: Cache pipx (poetry)
+        uses: actions/cache@v5
         with:
-          fetch-depth: 0
+          path: |
+            ~/.local/pipx
+            ~/.local/bin
+          key: pipx-poetry-${{ runner.os }}
+
+      - name: Install poetry
+        env:
+          PIPX_HOME: ~/.local/pipx
+          PIPX_BIN_DIR: ~/.local/bin
+        run: command -v poetry >/dev/null 2>&1 || pipx install poetry
+
+      - name: Setup Python (3.14) with Poetry cache
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.14"
+          cache: poetry
+          cache-dependency-path: tests/poetry.lock
+
+      - name: Install Python deps
+        working-directory: tests
+        run: poetry sync
 
       - name: Cache Nimble Dependencies
-        id: cache-nimble
-        uses: buildjet/cache@v4
+        uses: actions/cache@v5
         with:
           path: ~/.nimble
-          key: devel-nimble-v2-${{ hashFiles('*.nimble') }}
+          key: 2.2.x-nimble-v2-${{ hashFiles('*.nimble') }}
           restore-keys: |
-            devel-nimble-v2-
-
-      - name: Setup Python (3.10) with pip cache
-        uses: buildjet/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: pip
+            2.2.x-nimble-v2-
 
       - name: Setup Nim
         uses: jiro4989/setup-nim-action@v2
         with:
-          nim-version: devel
+          nim-version: 2.2.x
           use-nightlies: true
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build Project
-        run: nimble build -d:release -Y
+      - name: Download 2.2.x build artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: nitter-linux-nim-2.2.x-${{ github.sha }}
+          path: .
 
-      - name: Install SeleniumBase and Chromedriver
-        run: |
-          pip install seleniumbase
-          seleniumbase install chromedriver
-
-      - name: Start Redis Service
-        uses: supercharge/redis-github-action@1.5.0
+      - name: Make nitter binary executable
+        run: chmod +x ./nitter
 
       - name: Prepare Nitter Environment
         run: |
-          sudo apt-get update && sudo apt-get install -y libsass-dev
           cp nitter.example.conf nitter.conf
           sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf
-          nimble md
-          nimble scss
+
+          # Run both Nimble tasks concurrently
+          nim r tools/rendermd.nim &
+          nim r tools/gencss.nim &
+          wait
+
           echo '${{ secrets.SESSIONS }}' | head -n1
           echo '${{ secrets.SESSIONS }}' > ./sessions.jsonl
 
       - name: Run Tests
         run: |
           ./nitter &
-          pytest -n1 tests
+          cd tests
+          poetry run pytest -n3 --reruns=3 --rs .

nitter.nimble

@@ -28,7 +28,7 @@ requires "oauth#b8c163b"
 # Tasks
 
 task scss, "Generate css":
-  exec "nimble c --hint[Processing]:off -d:danger -r tools/gencss"
+  exec "nim r --hint[Processing]:off tools/gencss"
 
 task md, "Render md":
-  exec "nimble c --hint[Processing]:off -d:danger -r tools/rendermd"
+  exec "nim r --hint[Processing]:off tools/rendermd"

public/css/fontello.css

@@ -1,12 +1,12 @@
 @font-face {
   font-family: "fontello";
-  src: url("/fonts/fontello.eot?77185648");
+  src: url("/fonts/fontello.eot?42791196");
   src:
-    url("/fonts/fontello.eot?77185648#iefix") format("embedded-opentype"),
-    url("/fonts/fontello.woff2?77185648") format("woff2"),
-    url("/fonts/fontello.woff?77185648") format("woff"),
-    url("/fonts/fontello.ttf?77185648") format("truetype"),
-    url("/fonts/fontello.svg?77185648#fontello") format("svg");
+    url("/fonts/fontello.eot?42791196#iefix") format("embedded-opentype"),
+    url("/fonts/fontello.woff2?42791196") format("woff2"),
+    url("/fonts/fontello.woff?42791196") format("woff"),
+    url("/fonts/fontello.ttf?42791196") format("truetype"),
+    url("/fonts/fontello.svg?42791196#fontello") format("svg");
   font-weight: normal;
   font-style: normal;
 }
@@ -56,6 +56,11 @@
 }
 
 /* '' */
+.icon-group:before {
+  content: "\e804";
+}
+
+/* '' */
 .icon-play:before {
   content: "\e805";
 }

public/fonts/fontello.svg

@@ -1,7 +1,7 @@
 <?xml version="1.0" standalone="no"?>
 <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
 <svg xmlns="http://www.w3.org/2000/svg">
-<metadata>Copyright (C) 2025 by original authors @ fontello.com</metadata>
+<metadata>Copyright (C) 2026 by original authors @ fontello.com</metadata>
 <defs>
 <font id="fontello" horiz-adv-x="1000" >
 <font-face font-family="fontello" font-weight="400" font-stretch="normal" units-per-em="1000" ascent="850" descent="-150" />
@@ -14,6 +14,8 @@
 
 <glyph glyph-name="comment" unicode="&#xe803;" d="M1000 350q0-97-67-179t-182-130-251-48q-39 0-81 4-110-97-257-135-27-8-63-12-10-1-17 5t-10 16v1q-2 2 0 6t1 6 2 5l4 5t4 5 4 5q4 5 17 19t20 22 17 22 18 28 15 33 15 42q-88 50-138 123t-51 157q0 73 40 139t106 114 160 76 194 28q136 0 251-48t182-130 67-179z" horiz-adv-x="1000" />
 
+<glyph glyph-name="group" unicode="&#xe804;" d="M0 106l0 134q0 26 18 32l171 80q-66 39-68 131 0 56 35 103 37 41 90 43 31 0 63-19-49-125 23-237-12-11-25-19l-114-55q-48-23-52-84l0-143-114 0q-25 0-27 34z m193-59l0 168q0 27 22 37l152 70 57 28q-37 23-60 66t-22 94q0 76 46 130t110 54 109-54 45-130q0-105-78-158l61-30 146-70q24-10 24-37l0-168q-2-37-37-41l-541 0q-14 2-24 14t-10 27z m473 330q68 106 22 231 31 19 66 21 49 0 90-43 35-41 35-103 0-82-65-131l168-80q18-10 18-32l0-134q0-32-27-34l-118 0 0 143q0 57-50 84l-110 53q-15 8-29 25z" horiz-adv-x="1000" />
+
 <glyph glyph-name="play" unicode="&#xe805;" d="M772 333l-741-412q-13-7-22-2t-9 20v822q0 14 9 20t22-2l741-412q13-7 13-17t-13-17z" horiz-adv-x="785.7" />
 
 <glyph glyph-name="link" unicode="&#xe806;" d="M294 116q14 14 34 14t36-14q32-34 0-70l-42-40q-56-56-132-56-78 0-134 56t-56 132q0 78 56 134l148 148q70 68 144 77t128-43q16-16 16-36t-16-36q-36-32-70 0-50 48-132-34l-148-146q-26-26-26-64t26-62q26-26 63-26t63 26z m450 574q56-56 56-132 0-78-56-134l-158-158q-74-72-150-72-62 0-112 50-14 14-14 34t14 36q14 14 35 14t35-14q50-48 122 24l158 156q28 28 28 64 0 38-28 62-24 26-56 31t-60-21l-50-50q-16-14-36-14t-34 14q-34 34 0 70l50 50q54 54 127 51t129-61z" horiz-adv-x="800" />

src/consts.nim

@@ -113,7 +113,7 @@ const
   $2
   "includeHasBirdwatchNotes": false,
   "includePromotedContent": false,
-  "withBirdwatchNotes": false,
+  "withBirdwatchNotes": true,
   "withVoice": false,
   "withV2Timeline": true
 }""".replace(" ", "").replace("\n", "")

src/parser.nim

@@ -6,6 +6,12 @@ import experimental/parser/unifiedcard
 
 proc parseGraphTweet(js: JsonNode): Tweet
 
+proc parseCommunityNote(js: JsonNode): string =
+  let subtitle = js{"subtitle"}
+  result = subtitle{"text"}.getStr
+  with entities, subtitle{"entities"}:
+    result = expandBirdwatchEntities(result, entities)
+
 proc parseUser(js: JsonNode; id=""): User =
   if js.isNull: return
   result = User(
@@ -439,6 +445,9 @@ proc parseGraphTweet(js: JsonNode): Tweet =
     for id in ids:
       result.history.add parseBiggestInt(id.getStr)
 
+  with birdwatch, js{"birdwatch_pivot"}:
+    result.note = parseCommunityNote(birdwatch)
+
 proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
   for t in ? js{"content", "items"}:
     let entryId = t.getEntryId

src/parserutils.nim

@@ -330,6 +330,26 @@ proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) =
 
   tweet.text = tweet.text.multiReplace((unicodeOpen, xmlOpen), (unicodeClose, xmlClose))
 
+proc expandBirdwatchEntities*(text: string; entities: JsonNode): string =
+  let runes = text.toRunes
+  var replacements: seq[ReplaceSlice]
+
+  for entity in entities:
+    let
+      fromIdx = entity{"from_index"}.getInt
+      toIdx = entity{"to_index"}.getInt
+      url = entity{"ref", "url"}.getStr
+    if url.len > 0:
+      replacements.add ReplaceSlice(
+        kind: rkUrl,
+        slice: fromIdx ..< toIdx,
+        url: url,
+        display: $runes[fromIdx ..< min(toIdx, runes.len)]
+      )
+
+  replacements.sort(cmp)
+  result = runes.replacedWith(replacements, 0 ..< runes.len)
+
 proc extractGalleryPhoto*(t: Tweet): GalleryPhoto =
   let url =
     if t.photos.len > 0: t.photos[0].url

src/prefs_impl.nim

@@ -78,6 +78,9 @@ genPrefs:
     hideReplies(checkbox, false):
       "Hide tweet replies"
 
+    hideCommunityNotes(checkbox, false):
+      "Hide community notes"
+
     squareAvatars(checkbox, false):
       "Square profile pictures"
 

src/sass/tweet/_base.scss

@@ -254,3 +254,38 @@
     pointer-events: all;
   }
 }
+
+.community-note {
+  background-color: var(--bg_elements);
+  margin-top: 10px;
+  border: solid 1px var(--dark_grey);
+  border-radius: 10px;
+  overflow: hidden;
+  pointer-events: all;
+
+  &:hover {
+    background-color: var(--bg_panel);
+    border-color: var(--grey);
+  }
+}
+
+.community-note-header {
+  background-color: var(--bg_hover);
+  font-weight: 700;
+  padding: 8px 10px;
+  padding-top: 6px;
+  display: flex;
+  align-items: center;
+  gap: 2px;
+
+  .icon-container {
+    flex-shrink: 0;
+    color: var(--accent);
+  }
+}
+
+.community-note-text {
+  white-space: pre-line;
+  padding: 10px 10px;
+  padding-top: 6px;
+}

src/sass/tweet/card.scss

@@ -1,119 +1,119 @@
-@import '_variables';
-@import '_mixins';
+@import "_variables";
+@import "_mixins";
 
 .card {
-    margin: 5px 0;
-    pointer-events: all;
-    max-height: unset;
+  margin: 5px 0;
+  pointer-events: all;
+  max-height: unset;
 }
 
 .card-container {
-    border-radius: 10px;
-    border-width: 1px;
-    border-style: solid;
-    border-color: var(--dark_grey);
-    background-color: var(--bg_elements);
-    overflow: hidden;
-    color: inherit;
-    display: flex;
-    flex-direction: row;
-    text-decoration: none !important;
+  border: solid 1px var(--dark_grey);
+  border-radius: 10px;
+  background-color: var(--bg_elements);
+  overflow: hidden;
+  color: inherit;
+  display: flex;
+  flex-direction: row;
+  text-decoration: none !important;
 
-    &:hover {
-        border-color: var(--grey);
-    }
+  &:hover {
+    border-color: var(--grey);
+  }
 
-    .attachments {
-        margin: 0;
-        border-radius: 0;
-    }
+  .attachments {
+    margin: 0;
+    border-radius: 0;
+  }
 }
 
 .card-content {
-    padding: 0.5em;
+  padding: 0.5em;
 }
 
 .card-title {
-    @include ellipsis;
-    white-space: unset;
-    font-weight: bold;
-    font-size: 1.1em;
+  @include ellipsis;
+  white-space: unset;
+  font-weight: bold;
+  font-size: 1.1em;
 }
 
 .card-description {
-    margin: 0.3em 0;
-    white-space: pre-wrap;
+  margin: 0.3em 0;
+  white-space: pre-wrap;
 }
 
 .card-destination {
-    @include ellipsis;
-    color: var(--grey);
-    display: block;
+  @include ellipsis;
+  color: var(--grey);
+  display: block;
 }
 
 .card-content-container {
-    color: unset;
-    overflow: auto;
-    &:hover {
-        text-decoration: none;
-    }
+  color: unset;
+  overflow: auto;
+
+  &:hover {
+    text-decoration: none;
+  }
 }
 
 .card-image-container {
-    width: 98px;
-    flex-shrink: 0;
-    position: relative;
-    overflow: hidden;
-    &:before {
-        content: "";
-        display: block;
-        padding-top: 100%;
-    }
+  width: 98px;
+  flex-shrink: 0;
+  position: relative;
+  overflow: hidden;
+
+  &:before {
+    content: "";
+    display: block;
+    padding-top: 100%;
+  }
 }
 
 .card-image {
-    position: absolute;
-    top: 0;
-    left: 0;
-    bottom: 0;
-    right: 0;
-    background-color: var(--bg_overlays);
+  position: absolute;
+  top: 0;
+  left: 0;
+  bottom: 0;
+  right: 0;
+  background-color: var(--bg_overlays);
 
-    img {
-        width: 100%;
-        height: 100%;
-        max-height: 400px;
-        display: block;
-        object-fit: cover;
-    }
+  img {
+    width: 100%;
+    height: 100%;
+    max-height: 400px;
+    display: block;
+    object-fit: cover;
+  }
 }
 
 .card-overlay {
-    @include play-button;
-    opacity: 0.8;
-    display: flex;
-    justify-content: center;
-    align-items: center;
+  @include play-button;
+  opacity: 0.8;
+  display: flex;
+  justify-content: center;
+  align-items: center;
 }
 
 .large {
-    .card-container {
-        display: block;
-    }
+  .card-container {
+    display: block;
+  }
 
-    .card-image-container {
-        width: unset;
+  .card-image-container {
+    width: unset;
 
-        &:before {
-            display: none;
-        }
+    &:before {
+      display: none;
     }
+  }
 
-    .card-image {
-        position: unset;
-        border-style: solid;
-        border-color: var(--dark_grey);
-        border-width: 0;
-        border-bottom-width: 1px;
-    }
+  .card-image {
+    position: unset;
+    border-style: solid;
+    border-color: var(--dark_grey);
+    border-width: 0;
+    border-bottom-width: 1px;
+  }
 }

src/sass/tweet/quote.scss

@@ -19,31 +19,49 @@
   }
 
   .tweet-name-row {
-    padding: 6px 8px;
-    margin-top: 1px;
+    padding: 8px 10px 6px 10px;
   }
 
   .quote-text {
     overflow: hidden;
     white-space: pre-wrap;
     word-wrap: break-word;
-    padding: 0px 8px 8px 8px;
+    padding: 10px;
+    padding-top: 0;
   }
 
   .show-thread {
-    padding: 0px 8px 6px 8px;
+    padding: 0px 10px 6px 10px;
     margin-top: -6px;
   }
 
   .quote-latest {
-    padding: 0px 8px 6px 8px;
+    padding: 0px 10px 6px 10px;
     color: var(--grey);
   }
 
   .replying-to {
-    padding: 0px 8px;
+    padding: 0px 10px;
+    padding-bottom: 4px;
     margin: unset;
   }
+
+  .community-note {
+    background-color: var(--bg_panel);
+    border: unset;
+    border-top: solid 1px var(--dark_grey);
+    border-radius: unset;
+    margin-top: 0;
+
+    &:hover {
+      border-top-color: var(--grey);
+    }
+
+    .community-note-header {
+      background-color: var(--bg_panel);
+      padding-bottom: 0;
+    }
+  }
 }
 
 .unavailable-quote {

src/types.nim

@@ -223,6 +223,7 @@ type
     video*: Option[Video]
     photos*: seq[Photo]
     history*: seq[int64]
+    note*: string
 
   Tweets* = seq[Tweet]
 

src/views/general.nim

@@ -50,7 +50,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
   let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
 
   buildHtml(head):
-    link(rel="stylesheet", type="text/css", href="/css/style.css?v=27")
+    link(rel="stylesheet", type="text/css", href="/css/style.css?v=28")
     link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=4")
 
     if theme.len > 0:

src/views/rss.nimf

@@ -74,6 +74,9 @@ Twitter feed for: ${desc}. Generated by ${getUrlPrefix(cfg)}
 <img src="${urlPrefix}${getPicUrl(card.image)}" style="max-width:250px;" />
 #  end if
 #end if
+#if tweet.note.len > 0 and not prefs.hideCommunityNotes:
+<p><b>Community note:</b> ${replaceUrls(tweet.note, prefs, absolute=urlPrefix)}</p>
+#end if
 #if tweet.quote.isSome and get(tweet.quote).available:
 #  let quoteTweet = get(tweet.quote)
 #  let quoteLink = urlPrefix & getLink(quoteTweet)

src/views/tweet.nim

@@ -226,6 +226,14 @@ proc renderQuoteMedia(quote: Tweet; prefs: Prefs; path: string): VNode =
     elif quote.gif.isSome:
       renderGif(quote.gif.get(), prefs)
 
+proc renderCommunityNote(note: string; prefs: Prefs): VNode =
+  buildHtml(tdiv(class="community-note")):
+    tdiv(class="community-note-header"):
+      icon "group"
+      span: text "Community note"
+    tdiv(class="community-note-text", dir="auto"):
+      verbatim replaceUrls(note, prefs)
+
 proc renderQuote(quote: Tweet; prefs: Prefs; path: string): VNode =
   if not quote.available:
     return buildHtml(tdiv(class="quote unavailable")):
@@ -261,6 +269,9 @@ proc renderQuote(quote: Tweet; prefs: Prefs; path: string): VNode =
     if quote.photos.len > 0 or quote.video.isSome or quote.gif.isSome:
       renderQuoteMedia(quote, prefs, path)
 
+    if quote.note.len > 0 and not prefs.hideCommunityNotes:
+      renderCommunityNote(quote.note, prefs)
+
     if quote.hasThread:
       a(class="show-thread", href=getLink(quote)):
         text "Show this thread"
@@ -346,6 +357,9 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
       if tweet.quote.isSome:
         renderQuote(tweet.quote.get(), prefs, path)
 
+      if tweet.note.len > 0 and not prefs.hideCommunityNotes:
+        renderCommunityNote(tweet.note, prefs)
+
       let
         hasEdits = tweet.history.len > 1
         isLatest = hasEdits and tweet.id == max(tweet.history)

tests/poetry.toml

@@ -0,0 +1,2 @@
+[virtualenvs]
+in-project = true

tests/pyproject.toml

@@ -0,0 +1,8 @@
+[tool.poetry]
+name = "nitter-tests"
+version = "0.0.0"
+package-mode = false
+
+[tool.poetry.dependencies]
+python = "^3.14"
+seleniumbase = "4.46.5"

tests/requirements.txt

@@ -1 +1 @@
-seleniumbase
+seleniumbase==4.46.5