From f8fe16738eb96eb79584eec4c8f14f9c836018b7 Mon Sep 17 00:00:00 2001 From: bbedward Date: Fri, 17 Jul 2026 13:31:23 -0400 Subject: [PATCH] screenshot: initial support for scroll capture mode fixes #2871 port 1.5 --- core/cmd/dms/commands_screenshot.go | 43 +- core/internal/screenshot/compositor.go | 55 +- core/internal/screenshot/encode.go | 52 ++ core/internal/screenshot/niri_window.go | 164 ++++++ core/internal/screenshot/region.go | 35 +- core/internal/screenshot/region_input.go | 73 ++- core/internal/screenshot/region_render.go | 80 ++- core/internal/screenshot/screenshot.go | 6 +- core/internal/screenshot/scroll.go | 574 ++++++++++++++++++++ core/internal/screenshot/scroll_sim_test.go | 278 ++++++++++ core/internal/screenshot/stitch.go | 407 ++++++++++++++ core/internal/screenshot/stitch_test.go | 169 ++++++ core/internal/screenshot/types.go | 2 + 13 files changed, 1866 insertions(+), 72 deletions(-) create mode 100644 core/internal/screenshot/niri_window.go create mode 100644 core/internal/screenshot/scroll.go create mode 100644 core/internal/screenshot/scroll_sim_test.go create mode 100644 core/internal/screenshot/stitch.go create mode 100644 core/internal/screenshot/stitch_test.go diff --git a/core/cmd/dms/commands_screenshot.go b/core/cmd/dms/commands_screenshot.go index 93a35943c..5c626f72a 100644 --- a/core/cmd/dms/commands_screenshot.go +++ b/core/cmd/dms/commands_screenshot.go @@ -51,8 +51,9 @@ Modes: full - Capture the focused output all - Capture all outputs combined output - Capture a specific output by name - window - Capture the focused window (Hyprland/Mango) + window - Capture the focused window (Hyprland/Mango/niri) last - Capture the last selected region + scroll - Select a region, then scroll to capture a stitched tall image Output format (--format): png - PNG format (default) @@ -72,7 +73,9 @@ Examples: dms screenshot --no-confirm # Region capture on mouse release dms screenshot --cursor=on # Include cursor dms screenshot -f jpg -q 85 # JPEG with quality 85 - dms screenshot --json # Print capture metadata as JSON`, + dms screenshot --json # Print capture metadata as JSON + dms screenshot scroll # Scroll capture, Enter finishes / Esc cancels + dms screenshot scroll --interval 250`, } var ssRegionCmd = &cobra.Command{ @@ -110,10 +113,33 @@ If no previous region exists, falls back to interactive selection.`, var ssWindowCmd = &cobra.Command{ Use: "window", Short: "Capture the focused window", - Long: `Capture the currently focused window. Supported on Hyprland and Mango.`, + Long: `Capture the currently focused window. Supported on Hyprland, Mango, and niri.`, Run: runScreenshotWindow, } +var ssScrollInterval int + +var ssScrollCmd = &cobra.Command{ + Use: "scroll", + Short: "Capture a scrolling region stitched into one tall image", + Long: `Select a region, then scroll the content beneath with the mouse wheel or +touchpad while frames are captured and stitched vertically. Finish with the +on-screen done button; cancel with the cancel button. Enter and Esc work +everywhere: most compositors hold the keyboard on the overlay (keyboard +scrolling does not reach the app there), while Hyprland leaves the keyboard +with the application — keyboard scrolling works, and Enter/Esc act through +temporary global binds for the session. The cursor is never included in +frames. + +Frames are stitched continuously while scrolling, and revisited content is +never duplicated — scrolling up past the starting point extends the image +upward. Content jumped past faster than capture can follow is skipped rather +than stitched incorrectly. + +Rotated outputs are not supported.`, + Run: runScreenshotScroll, +} + var ssListCmd = &cobra.Command{ Use: "list", Short: "List available outputs", @@ -143,7 +169,10 @@ func init() { screenshotCmd.PersistentFlags().BoolVar(&ssStdout, "stdout", false, "Output image to stdout (for piping to swappy, etc.)") screenshotCmd.PersistentFlags().BoolVar(&ssJSON, "json", false, "Print capture metadata as JSON") + ssScrollCmd.Flags().IntVar(&ssScrollInterval, "interval", 45, "Capture interval in milliseconds (30-1000)") + screenshotCmd.AddCommand(ssRegionCmd) + screenshotCmd.AddCommand(ssScrollCmd) screenshotCmd.AddCommand(ssFullCmd) screenshotCmd.AddCommand(ssAllCmd) screenshotCmd.AddCommand(ssOutputCmd) @@ -249,7 +278,7 @@ func runScreenshot(config screenshot.Config) { // Region select needs the keyboard; drop popout grabs for its duration. result, err := func() (*screenshot.CaptureResult, error) { - interactive := config.Mode == screenshot.ModeRegion || config.Mode == screenshot.ModeLastRegion + interactive := config.Mode == screenshot.ModeRegion || config.Mode == screenshot.ModeLastRegion || config.Mode == screenshot.ModeScroll if interactive { setPopoutScreenshotMode(true) defer setPopoutScreenshotMode(false) @@ -437,6 +466,12 @@ func runScreenshotRegion(cmd *cobra.Command, args []string) { runScreenshot(config) } +func runScreenshotScroll(cmd *cobra.Command, args []string) { + config := getScreenshotConfig(screenshot.ModeScroll) + config.IntervalMs = min(max(ssScrollInterval, 30), 1000) + runScreenshot(config) +} + func runScreenshotFull(cmd *cobra.Command, args []string) { config := getScreenshotConfig(screenshot.ModeFullScreen) runScreenshot(config) diff --git a/core/internal/screenshot/compositor.go b/core/internal/screenshot/compositor.go index 1f107d23a..17debf0f2 100644 --- a/core/internal/screenshot/compositor.go +++ b/core/internal/screenshot/compositor.go @@ -30,41 +30,30 @@ func DetectCompositor() Compositor { return detectedCompositor } - hyprlandSig := os.Getenv("HYPRLAND_INSTANCE_SIGNATURE") - niriSocket := os.Getenv("NIRI_SOCKET") - swaySocket := os.Getenv("SWAYSOCK") - scrollSocket := os.Getenv("SCROLLSOCK") - miracleSocket := os.Getenv("MIRACLESOCK") - mangoSocket := os.Getenv("MANGO_INSTANCE_SIGNATURE") + candidates := []struct { + socket string + needsStat bool + compositor Compositor + }{ + {os.Getenv("MANGO_INSTANCE_SIGNATURE"), true, CompositorMango}, + {os.Getenv("NIRI_SOCKET"), true, CompositorNiri}, + {os.Getenv("SCROLLSOCK"), true, CompositorScroll}, + {os.Getenv("MIRACLESOCK"), true, CompositorMiracle}, + {os.Getenv("SWAYSOCK"), true, CompositorSway}, + {os.Getenv("HYPRLAND_INSTANCE_SIGNATURE"), false, CompositorHyprland}, + } - switch { - case mangoSocket != "": - if _, err := os.Stat(mangoSocket); err == nil { - detectedCompositor = CompositorMango - return detectedCompositor + // A stale env var from a previous session must not mask the live compositor + for _, c := range candidates { + if c.socket == "" { + continue } - case niriSocket != "": - if _, err := os.Stat(niriSocket); err == nil { - detectedCompositor = CompositorNiri - return detectedCompositor + if c.needsStat { + if _, err := os.Stat(c.socket); err != nil { + continue + } } - case scrollSocket != "": - if _, err := os.Stat(scrollSocket); err == nil { - detectedCompositor = CompositorScroll - return detectedCompositor - } - case miracleSocket != "": - if _, err := os.Stat(miracleSocket); err == nil { - detectedCompositor = CompositorMiracle - return detectedCompositor - } - case swaySocket != "": - if _, err := os.Stat(swaySocket); err == nil { - detectedCompositor = CompositorSway - return detectedCompositor - } - case hyprlandSig != "": - detectedCompositor = CompositorHyprland + detectedCompositor = c.compositor return detectedCompositor } @@ -90,7 +79,7 @@ func GetActiveWindow() (*WindowGeometry, error) { case CompositorMango: return getMangoActiveWindow() default: - return nil, fmt.Errorf("window capture requires Hyprland or Mango") + return nil, fmt.Errorf("window capture requires Hyprland, Mango, or niri") } } diff --git a/core/internal/screenshot/encode.go b/core/internal/screenshot/encode.go index 0256716aa..d425001e1 100644 --- a/core/internal/screenshot/encode.go +++ b/core/internal/screenshot/encode.go @@ -54,6 +54,58 @@ func BufferToImageWithFormat(buf *ShmBuffer, format uint32) *image.RGBA { return img } +func ImageToBuffer(img image.Image) (*ShmBuffer, error) { + bounds := img.Bounds() + w, h := bounds.Dx(), bounds.Dy() + buf, err := CreateShmBuffer(w, h, w*4) + if err != nil { + return nil, err + } + + data := buf.Data() + switch src := img.(type) { + case *image.NRGBA: + for y := range h { + srcOff := y * src.Stride + dstOff := y * buf.Stride + for x := range w { + si, di := srcOff+x*4, dstOff+x*4 + a := uint32(src.Pix[si+3]) + data[di+0] = uint8(uint32(src.Pix[si+2]) * a / 255) + data[di+1] = uint8(uint32(src.Pix[si+1]) * a / 255) + data[di+2] = uint8(uint32(src.Pix[si+0]) * a / 255) + data[di+3] = uint8(a) + } + } + case *image.RGBA: + for y := range h { + srcOff := y * src.Stride + dstOff := y * buf.Stride + for x := range w { + si, di := srcOff+x*4, dstOff+x*4 + data[di+0] = src.Pix[si+2] + data[di+1] = src.Pix[si+1] + data[di+2] = src.Pix[si+0] + data[di+3] = src.Pix[si+3] + } + } + default: + for y := range h { + dstOff := y * buf.Stride + for x := range w { + cr, cg, cb, ca := img.At(bounds.Min.X+x, bounds.Min.Y+y).RGBA() + di := dstOff + x*4 + data[di+0] = uint8(cb >> 8) + data[di+1] = uint8(cg >> 8) + data[di+2] = uint8(cr >> 8) + data[di+3] = uint8(ca >> 8) + } + } + } + buf.Format = FormatARGB8888 + return buf, nil +} + func EncodePNG(w io.Writer, img image.Image) error { enc := png.Encoder{CompressionLevel: png.BestSpeed} return enc.Encode(w, img) diff --git a/core/internal/screenshot/niri_window.go b/core/internal/screenshot/niri_window.go new file mode 100644 index 000000000..88a03056e --- /dev/null +++ b/core/internal/screenshot/niri_window.go @@ -0,0 +1,164 @@ +package screenshot + +import ( + "bufio" + "encoding/json" + "fmt" + "image" + "image/png" + "net" + "os" + "path/filepath" + "time" +) + +const niriScreenshotTimeout = 5 * time.Second + +// CaptureNiriWindowImage captures the focused window through niri's +// ScreenshotWindow action; niri replies before writing the file, so a second +// event-stream connection waits for ScreenshotCaptured. niri also copies the +// capture to its own clipboard, which cannot be disabled. +func CaptureNiriWindowImage(showPointer bool) (image.Image, error) { + socket := os.Getenv("NIRI_SOCKET") + if socket == "" { + return nil, fmt.Errorf("NIRI_SOCKET not set") + } + + dir := os.Getenv("XDG_RUNTIME_DIR") + if dir == "" { + dir = os.TempDir() + } + path := filepath.Join(dir, fmt.Sprintf("dms-window-%d.png", os.Getpid())) + + events, err := subscribeNiriEvents(socket) + if err != nil { + return nil, err + } + defer events.Close() + + if err := requestNiriWindowScreenshot(socket, path, showPointer); err != nil { + return nil, err + } + defer os.Remove(path) + + if err := awaitNiriScreenshot(events, path); err != nil { + return nil, err + } + + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open niri screenshot: %w", err) + } + defer f.Close() + + img, err := png.Decode(f) + if err != nil { + return nil, fmt.Errorf("decode niri screenshot: %w", err) + } + return img, nil +} + +func subscribeNiriEvents(socket string) (net.Conn, error) { + conn, err := net.DialTimeout("unix", socket, 2*time.Second) + if err != nil { + return nil, fmt.Errorf("connect niri socket: %w", err) + } + _ = conn.SetDeadline(time.Now().Add(niriScreenshotTimeout)) + + if _, err := conn.Write([]byte("\"EventStream\"\n")); err != nil { + conn.Close() + return nil, fmt.Errorf("subscribe niri events: %w", err) + } + return conn, nil +} + +func awaitNiriScreenshot(events net.Conn, path string) error { + scanner := bufio.NewScanner(events) + scanner.Buffer(make([]byte, 0, 64<<10), 1<<20) + + for scanner.Scan() { + var event struct { + ScreenshotCaptured *struct { + Path string `json:"path"` + } `json:"ScreenshotCaptured"` + } + if json.Unmarshal(scanner.Bytes(), &event) != nil { + continue + } + if event.ScreenshotCaptured != nil && event.ScreenshotCaptured.Path == path { + return nil + } + } + + if err := scanner.Err(); err != nil { + return fmt.Errorf("await niri screenshot: %w", err) + } + return fmt.Errorf("niri event stream closed before screenshot completed") +} + +func requestNiriWindowScreenshot(socket, path string, showPointer bool) error { + conn, err := net.DialTimeout("unix", socket, 2*time.Second) + if err != nil { + return fmt.Errorf("connect niri socket: %w", err) + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(3 * time.Second)) + + request := map[string]any{ + "Action": map[string]any{ + "ScreenshotWindow": map[string]any{ + "id": nil, + "write_to_disk": true, + "show_pointer": showPointer, + "path": path, + }, + }, + } + payload, err := json.Marshal(request) + if err != nil { + return err + } + if _, err := conn.Write(append(payload, '\n')); err != nil { + return fmt.Errorf("niri request: %w", err) + } + + line, err := bufio.NewReader(conn).ReadBytes('\n') + if err != nil { + return fmt.Errorf("niri reply: %w", err) + } + + var reply map[string]json.RawMessage + if err := json.Unmarshal(line, &reply); err != nil { + return fmt.Errorf("parse niri reply: %w", err) + } + if raw, ok := reply["Err"]; ok { + var msg string + _ = json.Unmarshal(raw, &msg) + return fmt.Errorf("niri screenshot: %s", msg) + } + return nil +} + +func (s *Screenshoter) captureNiriWindow() (*CaptureResult, error) { + img, err := CaptureNiriWindowImage(s.config.Cursor == CursorOn) + if err != nil { + return nil, err + } + + buf, err := ImageToBuffer(img) + if err != nil { + return nil, err + } + + scale := 1.0 + if output := s.findOutputByName(GetFocusedMonitor()); output != nil { + scale = output.effectiveScale() + } + + return &CaptureResult{ + Buffer: buf, + YInverted: false, + Format: uint32(FormatARGB8888), + Scale: scale, + }, nil +} diff --git a/core/internal/screenshot/region.go b/core/internal/screenshot/region.go index c8c392938..d9e4215e4 100644 --- a/core/internal/screenshot/region.go +++ b/core/internal/screenshot/region.go @@ -95,6 +95,9 @@ type RegionSelector struct { showCapturedCursor bool shiftHeld bool + phase selectorPhase + scroll *scrollSession + running bool cancelled bool result Region @@ -167,11 +170,15 @@ func (r *RegionSelector) Run() (*CaptureResult, bool, error) { r.running = true for r.running { - if err := r.ctx.Dispatch(); err != nil { + if err := r.dispatchOrTick(); err != nil { return nil, false, fmt.Errorf("dispatch: %w", err) } } + if r.scroll != nil && r.scroll.abortErr != nil { + return nil, false, r.scroll.abortErr + } + if r.cancelled || r.capturedBuffer == nil { return nil, r.cancelled, nil } @@ -186,6 +193,10 @@ func (r *RegionSelector) Run() (*CaptureResult, bool, error) { scale = s } } + if r.scroll != nil { + yInverted = false + format = uint32(r.scroll.format) + } return &CaptureResult{ Buffer: r.capturedBuffer, @@ -696,7 +707,9 @@ func (r *RegionSelector) initRenderBuffer(os *OutputSurface) { } slot.pool = pool - wlBuf, err := pool.CreateBuffer(0, int32(buf.Width), int32(buf.Height), int32(buf.Stride), os.screenFormat) + // niri latches surface opacity from the first buffer's format + // (observed), so slots are ARGB from the start with A=255 when opaque + wlBuf, err := pool.CreateBuffer(0, int32(buf.Width), int32(buf.Height), int32(buf.Stride), alphaFormat(os.screenFormat)) if err != nil { log.Error("create render slot wl_buffer failed", "err", err) pool.Destroy() @@ -738,8 +751,9 @@ func (r *RegionSelector) applyPreSelection(os *OutputSurface) { x1 := float64(r.preSelect.X-os.output.x) * scaleX y1 := float64(r.preSelect.Y-os.output.y) * scaleY - x2 := float64(r.preSelect.X-os.output.x+r.preSelect.Width) * scaleX - y2 := float64(r.preSelect.Y-os.output.y+r.preSelect.Height) * scaleY + // selection edges are inclusive; the exclusive width edge is one device px past it + x2 := float64(r.preSelect.X-os.output.x+r.preSelect.Width)*scaleX - scaleX + y2 := float64(r.preSelect.Y-os.output.y+r.preSelect.Height)*scaleY - scaleY r.selection.hasSelection = true r.selection.dragging = false @@ -769,10 +783,13 @@ func (r *RegionSelector) redrawSurface(os *OutputSurface) { return } - slot.shm.CopyFrom(srcBuf) - - // Draw overlay (dimming + selection) into this slot - r.drawOverlay(os, slot.shm) + switch r.phase { + case phaseScroll: + r.drawScrollOverlay(os, slot.shm) + default: + slot.shm.CopyFrom(srcBuf) + r.drawOverlay(os, slot.shm) + } if os.viewport != nil { _ = os.wlSurface.SetBufferScale(1) @@ -808,6 +825,8 @@ func (r *RegionSelector) cleanup() { r.cursorBuffer.Close() } + r.cleanupScroll() + for _, os := range r.surfaces { for _, slot := range os.slots { if slot == nil { diff --git a/core/internal/screenshot/region_input.go b/core/internal/screenshot/region_input.go index e2434435e..629f67082 100644 --- a/core/internal/screenshot/region_input.go +++ b/core/internal/screenshot/region_input.go @@ -94,6 +94,20 @@ func (r *RegionSelector) setupPointerHandlers() { return } + if r.phase == phaseScroll { + if e.Button != 0x110 || e.State != 1 || r.activeSurface != r.selection.surface { + return + } + switch r.scrollBarHit(r.pointerX, r.pointerY) { + case "done": + r.finishScroll() + case "cancel": + r.cancelled = true + r.running = false + } + return + } + switch e.Button { case 0x110: // BTN_LEFT switch e.State { @@ -135,6 +149,17 @@ func (r *RegionSelector) setupKeyboardHandlers() { return } + if r.phase == phaseScroll { + switch e.Key { + case 1: + r.cancelled = true + r.running = false + case 28, 96: + r.finishScroll() + } + return + } + switch e.Key { case 1: r.cancelled = true @@ -152,17 +177,15 @@ func (r *RegionSelector) setupKeyboardHandlers() { }) } -func (r *RegionSelector) finishSelection() { +func (r *RegionSelector) selectionDeviceRect() (*OutputSurface, int, int, int, int) { if r.selection.surface == nil { - r.running = false - return + return nil, 0, 0, 0, 0 } os := r.selection.surface srcBuf := r.getSourceBuffer(os) if srcBuf == nil { - r.running = false - return + return nil, 0, 0, 0, 0 } x1, y1 := r.selection.anchorX, r.selection.anchorY @@ -181,24 +204,10 @@ func (r *RegionSelector) finishSelection() { scaleY = float64(srcBuf.Height) / float64(os.logicalH) } - bx1 := int(x1 * scaleX) - by1 := int(y1 * scaleY) - bx2 := int(x2 * scaleX) - by2 := int(y2 * scaleY) - - // Clamp to buffer bounds - if bx1 < 0 { - bx1 = 0 - } - if by1 < 0 { - by1 = 0 - } - if bx2 > srcBuf.Width { - bx2 = srcBuf.Width - } - if by2 > srcBuf.Height { - by2 = srcBuf.Height - } + bx1 := clamp(int(x1*scaleX), 0, srcBuf.Width) + by1 := clamp(int(y1*scaleY), 0, srcBuf.Height) + bx2 := clamp(int(x2*scaleX), 0, srcBuf.Width) + by2 := clamp(int(y2*scaleY), 0, srcBuf.Height) w, h := bx2-bx1+1, by2-by1+1 if r.shiftHeld && w != h { @@ -215,7 +224,23 @@ func (r *RegionSelector) finishSelection() { h = 1 } - // Create cropped buffer and copy pixels directly + return os, bx1, by1, w, h +} + +func (r *RegionSelector) finishSelection() { + os, bx1, by1, w, h := r.selectionDeviceRect() + if os == nil { + r.running = false + return + } + + if r.screenshoter != nil && r.screenshoter.config.Mode == ModeScroll { + r.enterScrollPhase(os, bx1, by1, w, h) + return + } + + srcBuf := r.getSourceBuffer(os) + cropped, err := CreateShmBuffer(w, h, w*4) if err != nil { r.running = false diff --git a/core/internal/screenshot/region_render.go b/core/internal/screenshot/region_render.go index 4071f8a92..732681273 100644 --- a/core/internal/screenshot/region_render.go +++ b/core/internal/screenshot/region_render.go @@ -57,7 +57,7 @@ func (r *RegionSelector) drawOverlay(os *OutputSurface, renderBuf *ShmBuffer) { w, h := renderBuf.Width, renderBuf.Height format := os.screenFormat - // Dim the entire buffer + // dim, forcing alpha: the X-format source's padding byte is undefined for y := 0; y < h; y++ { off := y * stride for x := 0; x < w; x++ { @@ -68,6 +68,7 @@ func (r *RegionSelector) drawOverlay(os *OutputSurface, renderBuf *ShmBuffer) { data[i+0] = uint8(int(data[i+0]) * 3 / 5) data[i+1] = uint8(int(data[i+1]) * 3 / 5) data[i+2] = uint8(int(data[i+2]) * 3 / 5) + data[i+3] = 255 } } @@ -110,7 +111,7 @@ func (r *RegionSelector) drawOverlay(os *OutputSurface, renderBuf *ShmBuffer) { data[di+0] = srcData[si+0] data[di+1] = srcData[si+1] data[di+2] = srcData[si+2] - data[di+3] = srcData[si+3] + data[di+3] = 255 } } @@ -126,6 +127,81 @@ func (r *RegionSelector) drawOverlay(os *OutputSurface, renderBuf *ShmBuffer) { r.drawDimensions(data, stride, w, h, bx1, by1, selW, selH, format) } +func (r *RegionSelector) drawScrollOverlay(os *OutputSurface, renderBuf *ShmBuffer) { + data := renderBuf.Data() + stride := renderBuf.Stride + w, h := renderBuf.Width, renderBuf.Height + + // 40% premultiplied scrim + for y := 0; y < h; y++ { + off := y * stride + for x := 0; x < w; x++ { + i := off + x*4 + if i+3 >= len(data) { + continue + } + data[i+0], data[i+1], data[i+2], data[i+3] = 0, 0, 0, 102 + } + } + + s := r.scroll + if s == nil || r.selection.surface != os { + return + } + + // hole oversized 2px so overlay pixels never land in captured frames + holeX := s.holeX - 2 + holeY := s.holeY - 2 + holeW := s.holeW + 4 + holeH := s.holeH + 4 + + x1 := clamp(holeX, 0, w) + y1 := clamp(holeY, 0, h) + x2 := clamp(holeX+holeW, 0, w) + y2 := clamp(holeY+holeH, 0, h) + + for y := y1; y < y2; y++ { + off := y * stride + for x := x1; x < x2; x++ { + i := off + x*4 + if i+3 >= len(data) { + continue + } + data[i+0], data[i+1], data[i+2], data[i+3] = 0, 0, 0, 0 + } + } + + r.drawBorder(data, stride, w, h, holeX-1, holeY-1, holeW+2, holeH+2, os.screenFormat) + r.drawScrollBar(data, stride, w, h, os.screenFormat) +} + +func (r *RegionSelector) drawScrollBar(data []byte, stride, bufW, bufH int, format uint32) { + s := r.scroll + style := LoadOverlayStyle() + const charH = 12 + + r.fillRect(data, stride, bufW, bufH, s.barX, s.barY, s.barW, s.barH, + style.BackgroundR, style.BackgroundG, style.BackgroundB, 245, format) + + labelY := s.doneY + (s.btnH-charH)/2 + r.fillRect(data, stride, bufW, bufH, s.doneX, s.doneY, s.doneW, s.btnH, + style.AccentR, style.AccentG, style.AccentB, 255, format) + r.drawText(data, stride, bufW, bufH, s.doneX+12, labelY, "done", 10, 10, 10, format) + + r.fillRect(data, stride, bufW, bufH, s.cancelX, s.cancelY, s.cancelW, s.btnH, + 70, 70, 70, 255, format) + r.drawText(data, stride, bufW, bufH, s.cancelX+12, labelY, "cancel", + style.TextR, style.TextG, style.TextB, format) + + rows := 0 + if s.st != nil { + rows = s.st.rows() + } + counter := fmt.Sprintf("%d shots %dpx", s.kept, rows) + r.drawText(data, stride, bufW, bufH, s.cancelX+s.cancelW+16, labelY, counter, + style.TextR, style.TextG, style.TextB, format) +} + func (r *RegionSelector) drawHUD(data []byte, stride, bufW, bufH int, format uint32) { if r.selection.dragging { return diff --git a/core/internal/screenshot/screenshot.go b/core/internal/screenshot/screenshot.go index cdf7fd44a..0ba50b8a7 100644 --- a/core/internal/screenshot/screenshot.go +++ b/core/internal/screenshot/screenshot.go @@ -92,7 +92,7 @@ func (s *Screenshoter) Run() (*CaptureResult, error) { switch s.config.Mode { case ModeLastRegion: return s.captureLastRegion() - case ModeRegion: + case ModeRegion, ModeScroll: return s.captureRegion() case ModeWindow: return s.captureWindow() @@ -145,6 +145,10 @@ func (s *Screenshoter) captureRegion() (*CaptureResult, error) { } func (s *Screenshoter) captureWindow() (*CaptureResult, error) { + if DetectCompositor() == CompositorNiri { + return s.captureNiriWindow() + } + geom, err := GetActiveWindow() if err != nil { return nil, err diff --git a/core/internal/screenshot/scroll.go b/core/internal/screenshot/scroll.go new file mode 100644 index 000000000..16cdb8f9d --- /dev/null +++ b/core/internal/screenshot/scroll.go @@ -0,0 +1,574 @@ +package screenshot + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "os/signal" + "time" + + "github.com/AvengeMedia/DankMaterialShell/core/internal/log" + "github.com/AvengeMedia/DankMaterialShell/core/internal/proto/wlr_layer_shell" + "github.com/AvengeMedia/DankMaterialShell/core/internal/proto/wlr_screencopy" + "github.com/AvengeMedia/DankMaterialShell/core/pkg/go-wayland/wayland/client" + "golang.org/x/sys/unix" +) + +type selectorPhase int + +const ( + phaseSelect selectorPhase = iota + phaseScroll +) + +const ( + scrollMaxFailures = 5 + scrollSeamTicks = 4 +) + +type scrollSession struct { + output *WaylandOutput + // wire coords for CaptureOutputRegion (logical or device px per compositor) + capX, capY, capW, capH int32 + // device-pixel rect in the overlay buffer, for hole/border drawing + holeX, holeY, holeW, holeH int + + interval time.Duration + nextTick time.Time + inFlight bool + failures int + kept int + abortErr error + + buf *ShmBuffer + pool *client.ShmPool + wlBuf *client.Buffer + frame *wlr_screencopy.ZwlrScreencopyFrameV1 + format PixelFormat + frameW, frameH int + yInverted bool + + prevSig []float32 + prevPlaced bool + unmatched bool + unmatchedTicks int + + // control bar geometry in overlay buffer pixels + barX, barY, barW, barH int + doneX, doneY, doneW int + cancelX, cancelY int + cancelW int + btnH int + + sigCh chan os.Signal + keysBound bool + + st *stitcher +} + +func (r *RegionSelector) dispatchOrTick() error { + timeout := -1 + if s := r.scroll; r.phase == phaseScroll && s != nil && s.sigCh != nil { + select { + case sig := <-s.sigCh: + switch sig { + case unix.SIGUSR2: + r.cancelled = true + r.running = false + default: + r.finishScroll() + } + return nil + default: + } + } + if s := r.scroll; r.phase == phaseScroll && s.abortErr == nil && (s.st == nil || !s.st.full) { + timeout = max(int(time.Until(s.nextTick).Milliseconds()), 0) + } + + fds := []unix.PollFd{{Fd: int32(r.ctx.Fd()), Events: unix.POLLIN}} + n, err := unix.Poll(fds, timeout) + switch { + case err == unix.EINTR: + return nil + case err != nil: + return err + case n > 0: + return r.ctx.Dispatch() + } + + r.scrollTick() + return nil +} + +func (r *RegionSelector) scrollTick() { + s := r.scroll + if s == nil { + return + } + if s.inFlight || (s.st != nil && s.st.full) { + s.nextTick = time.Now().Add(s.interval) + return + } + r.startScrollCapture() +} + +func (r *RegionSelector) enterScrollPhase(os *OutputSurface, x, y, w, h int) { + switch { + case os.output.transform != TransformNormal: + r.abortScroll(fmt.Errorf("scroll capture does not support rotated outputs")) + return + case w < 1 || h < 1: + r.abortScroll(fmt.Errorf("empty scroll capture region")) + return + } + + interval := 45 + if r.screenshoter != nil && r.screenshoter.config.IntervalMs > 0 { + interval = r.screenshoter.config.IntervalMs + } + + capX, capY, capW, capH := x, y, w, h + switch DetectCompositor() { + case CompositorHyprland, CompositorMango: + // both take device pixels, deviating from spec (observed) + default: + // spec: logical coordinates, scaled by the compositor + // https://wayland.app/protocols/wlr-screencopy-unstable-v1#zwlr_screencopy_manager_v1:request:capture_output_region + if scale := os.output.fractionalScale; scale > 1 { + capX = int(float64(x)/scale + 0.5) + capY = int(float64(y)/scale + 0.5) + capW = int(float64(w)/scale + 0.5) + capH = int(float64(h)/scale + 0.5) + } + } + + r.scroll = &scrollSession{ + output: os.output, + capX: int32(capX), + capY: int32(capY), + capW: int32(capW), + capH: int32(capH), + holeX: x, + holeY: y, + holeW: w, + holeH: h, + interval: time.Duration(interval) * time.Millisecond, + nextTick: time.Now(), + } + + r.layoutScrollBar(os) + + for _, surf := range r.surfaces { + r.setInputPassthrough(surf, surf == os) + } + + // Hyprland routes all pointer input to exclusive-keyboard layers + // (https://github.com/hyprwm/Hyprland/discussions/14136), so the keyboard + // is released there and Enter/Esc come back via temporary global binds + if DetectCompositor() == CompositorHyprland { + r.enterHyprlandScrollInput(os) + } + + r.phase = phaseScroll + for _, surf := range r.surfaces { + r.redrawSurface(surf) + } +} + +// sized for the worst-case counter so the input region is set once +func (r *RegionSelector) layoutScrollBar(os *OutputSurface) { + s := r.scroll + const charAdv, pad, gap = 9, 12, 16 + + s.btnH = 24 + s.doneW = len("done")*charAdv + 24 + s.cancelW = len("cancel")*charAdv + 24 + counterW := len("99999 shots 999999px") * charAdv + s.barW = pad + s.doneW + gap + s.cancelW + gap + counterW + pad + s.barH = s.btnH + 24 + + bufW, bufH := os.screenBuf.Width, os.screenBuf.Height + s.barX = (bufW - s.barW) / 2 + s.barY = bufH - s.barH - 24 + + borderX1, borderY1 := s.holeX-3, s.holeY-3 + borderX2, borderY2 := s.holeX+s.holeW+3, s.holeY+s.holeH+3 + overlaps := s.barX < borderX2 && s.barX+s.barW > borderX1 && + s.barY < borderY2 && s.barY+s.barH > borderY1 + if overlaps { + s.barY = 24 + } + + s.doneX = s.barX + pad + s.doneY = s.barY + (s.barH-s.btnH)/2 + s.cancelX = s.doneX + s.doneW + gap + s.cancelY = s.doneY +} + +func (r *RegionSelector) setInputPassthrough(os *OutputSurface, withBar bool) { + reg, err := r.compositor.CreateRegion() + if err != nil { + return + } + if withBar && os.screenBuf != nil && os.logicalW > 0 { + s := r.scroll + scaleX := float64(os.logicalW) / float64(os.screenBuf.Width) + scaleY := float64(os.logicalH) / float64(os.screenBuf.Height) + _ = reg.Add(int32(float64(s.barX)*scaleX), int32(float64(s.barY)*scaleY), + int32(float64(s.barW)*scaleX)+1, int32(float64(s.barH)*scaleY)+1) + } + _ = os.wlSurface.SetInputRegion(reg) + _ = reg.Destroy() +} + +func (r *RegionSelector) enterHyprlandScrollInput(osurf *OutputSurface) { + for _, surf := range r.surfaces { + _ = surf.layerSurf.SetKeyboardInteractivity(uint32(wlr_layer_shell.ZwlrLayerSurfaceV1KeyboardInteractivityNone)) + } + if r.shortcutsInhibitor != nil { + _ = r.shortcutsInhibitor.Destroy() + r.shortcutsInhibitor = nil + } + + s := r.scroll + scale := osurf.output.fractionalScale + if scale <= 0 { + scale = 1 + } + cx := int(float64(osurf.output.x) + float64(s.holeX+s.holeW/2)/scale) + cy := int(float64(osurf.output.y) + float64(s.holeY+s.holeH/2)/scale) + hyprlandFocusWindowAt(cx, cy) + + s.sigCh = make(chan os.Signal, 2) + signal.Notify(s.sigCh, unix.SIGUSR1, unix.SIGUSR2) + s.keysBound = hyprlandBindScrollKeys(os.Getpid()) +} + +func hyprlandFocusWindowAt(x, y int) { + out, err := exec.Command("hyprctl", "-j", "clients").Output() + if err != nil { + return + } + var clients []struct { + Address string `json:"address"` + At [2]int `json:"at"` + Size [2]int `json:"size"` + Mapped bool `json:"mapped"` + Hidden bool `json:"hidden"` + FocusHistoryID int `json:"focusHistoryID"` + } + if json.Unmarshal(out, &clients) != nil { + return + } + + best := -1 + for i, c := range clients { + if !c.Mapped || c.Hidden { + continue + } + if x < c.At[0] || x >= c.At[0]+c.Size[0] || y < c.At[1] || y >= c.At[1]+c.Size[1] { + continue + } + if best < 0 || c.FocusHistoryID < clients[best].FocusHistoryID { + best = i + } + } + if best < 0 { + return + } + _ = exec.Command("hyprctl", "dispatch", "focuswindow", "address:"+clients[best].Address).Run() +} + +func hyprlandBindScrollKeys(pid int) bool { + batch := fmt.Sprintf("keyword bind ,Return,exec,kill -USR1 %d ; keyword bind ,Escape,exec,kill -USR2 %d", pid, pid) + return exec.Command("hyprctl", "--batch", batch).Run() == nil +} + +func hyprlandUnbindScrollKeys() { + _ = exec.Command("hyprctl", "--batch", "keyword unbind ,Return ; keyword unbind ,Escape").Run() +} + +func (r *RegionSelector) scrollBarHit(x, y float64) string { + s := r.scroll + os := r.selection.surface + if s == nil || os == nil || os.screenBuf == nil || os.logicalW == 0 { + return "" + } + bx := int(x * float64(os.screenBuf.Width) / float64(os.logicalW)) + by := int(y * float64(os.screenBuf.Height) / float64(os.logicalH)) + + switch { + case bx >= s.doneX && bx < s.doneX+s.doneW && by >= s.doneY && by < s.doneY+s.btnH: + return "done" + case bx >= s.cancelX && bx < s.cancelX+s.cancelW && by >= s.cancelY && by < s.cancelY+s.btnH: + return "cancel" + default: + return "" + } +} + +func alphaFormat(format uint32) uint32 { + switch format { + case uint32(FormatXRGB8888): + return uint32(FormatARGB8888) + case uint32(FormatXBGR8888): + return uint32(FormatABGR8888) + default: + return format + } +} + +func (r *RegionSelector) startScrollCapture() { + s := r.scroll + frame, err := r.screencopy.CaptureOutputRegion(0, s.output.wlOutput, s.capX, s.capY, s.capW, s.capH) + if err != nil { + r.abortScroll(fmt.Errorf("scroll capture: %w", err)) + return + } + s.inFlight = true + s.frame = frame + s.nextTick = time.Now().Add(s.interval) + + frame.SetBufferHandler(func(e wlr_screencopy.ZwlrScreencopyFrameV1BufferEvent) { + if err := s.ensureCaptureBuffer(r, e); err != nil { + r.abortScroll(err) + return + } + if err := frame.Copy(s.wlBuf); err != nil { + log.Error("scroll frame copy failed", "err", err) + } + }) + + frame.SetFlagsHandler(func(e wlr_screencopy.ZwlrScreencopyFrameV1FlagsEvent) { + s.yInverted = (e.Flags & 1) != 0 + }) + + frame.SetReadyHandler(func(e wlr_screencopy.ZwlrScreencopyFrameV1ReadyEvent) { + frame.Destroy() + s.frame = nil + s.inFlight = false + s.failures = 0 + s.nextTick = time.Now().Add(s.interval) + r.handleScrollFrame() + }) + + frame.SetFailedHandler(func(e wlr_screencopy.ZwlrScreencopyFrameV1FailedEvent) { + frame.Destroy() + s.frame = nil + s.inFlight = false + s.failures++ + s.nextTick = time.Now().Add(s.interval) + if s.failures >= scrollMaxFailures { + r.abortScroll(fmt.Errorf("screencopy failed %d consecutive times", s.failures)) + } + }) +} + +func (s *scrollSession) ensureCaptureBuffer(r *RegionSelector, e wlr_screencopy.ZwlrScreencopyFrameV1BufferEvent) error { + if s.buf != nil { + if int(e.Width) != s.frameW || int(e.Height) != s.frameH || PixelFormat(e.Format) != s.format { + return fmt.Errorf("output changed during scroll capture") + } + return nil + } + + format := PixelFormat(e.Format) + if int(e.Stride) < int(e.Width)*format.BytesPerPixel() { + return fmt.Errorf("invalid stride from compositor: %d for width %d", e.Stride, e.Width) + } + + buf, err := CreateShmBuffer(int(e.Width), int(e.Height), int(e.Stride)) + if err != nil { + return fmt.Errorf("create scroll buffer: %w", err) + } + buf.Format = format + + pool, err := r.shm.CreatePool(buf.Fd(), int32(buf.Size())) + if err != nil { + buf.Close() + return fmt.Errorf("create scroll pool: %w", err) + } + + wlBuf, err := pool.CreateBuffer(0, int32(buf.Width), int32(buf.Height), int32(buf.Stride), e.Format) + if err != nil { + pool.Destroy() + buf.Close() + return fmt.Errorf("create scroll wl_buffer: %w", err) + } + + s.buf = buf + s.pool = pool + s.wlBuf = wlBuf + s.format = format + s.frameW = int(e.Width) + s.frameH = int(e.Height) + return nil +} + +func (r *RegionSelector) handleScrollFrame() { + s := r.scroll + if s == nil || s.buf == nil { + return + } + + rows, err := s.extractRows() + if err != nil { + r.abortScroll(err) + return + } + + if s.st == nil { + s.st = newStitcher(s.frameW * 4) + } + + cols := s.st.rowSamples(rows) + sig := s.st.frameSig(rows) + dup := duplicateFrame(sig, s.prevSig) + s.prevSig = sig + + // moving content: recapture at compositor speed, the timer paces idle only + if !dup { + s.nextTick = time.Now() + } + + var added int + switch { + case dup && s.unmatched: + // settled somewhere unreachable: seam a new segment after a few ticks + s.unmatchedTicks++ + if s.unmatchedTicks < scrollSeamTicks { + return + } + var placed bool + added, placed = s.st.pushFrame(rows, cols) + if !placed { + added = s.st.seamAppend(rows, cols) + } + s.prevPlaced = true + s.unmatched = false + s.unmatchedTicks = 0 + case dup && s.prevPlaced: + return + default: + var placed bool + added, placed = s.st.pushFrame(rows, cols) + s.prevPlaced = placed + s.unmatched = !placed + s.unmatchedTicks = 0 + } + + if scrollDebug { + log.Error("scroll frame", "dup", dup, "unmatched", s.unmatched, + "placed", s.prevPlaced, "added", added, "canvas", s.st.rows(), "kept", s.kept) + } + if added == 0 { + return + } + s.kept++ + if r.selection.surface != nil { + r.redrawSurface(r.selection.surface) + } +} + +var scrollDebug = os.Getenv("DMS_SCROLL_DEBUG") != "" + +func (s *scrollSession) extractRows() ([]byte, error) { + src := s.buf + format := s.format + + if format.Is24Bit() { + converted, newFormat, err := src.ConvertTo32Bit(format) + if err != nil { + return nil, fmt.Errorf("convert scroll frame: %w", err) + } + defer converted.Close() + src = converted + s.format = newFormat + } + + rows := make([]byte, s.frameW*4*s.frameH) + data := src.Data() + for y := 0; y < s.frameH; y++ { + srcY := y + if s.yInverted { + srcY = s.frameH - 1 - y + } + srcOff := srcY * src.Stride + dstOff := y * s.frameW * 4 + if srcOff+s.frameW*4 > len(data) { + continue + } + copy(rows[dstOff:dstOff+s.frameW*4], data[srcOff:srcOff+s.frameW*4]) + } + return rows, nil +} + +func (r *RegionSelector) finishScroll() { + s := r.scroll + if s == nil || s.st == nil || s.st.rows() == 0 { + r.cancelled = true + r.running = false + return + } + + buf, err := CreateShmBuffer(s.frameW, s.st.rows(), s.frameW*4) + if err != nil { + r.abortScroll(fmt.Errorf("create stitched buffer: %w", err)) + return + } + copy(buf.Data(), s.st.canvas) + buf.Format = s.format + + r.capturedBuffer = buf + r.capturedRegion = Region{ + X: int32(s.holeX), + Y: int32(s.holeY), + Width: int32(s.holeW), + Height: int32(s.holeH), + Output: s.output.name, + } + // same convention as finishSelection or preselect breaks on scaled outputs + r.result = Region{ + X: int32(s.holeX) + s.output.x, + Y: int32(s.holeY) + s.output.y, + Width: int32(s.holeW), + Height: int32(s.holeH), + Output: s.output.name, + } + r.running = false +} + +func (r *RegionSelector) abortScroll(err error) { + if r.scroll == nil { + r.scroll = &scrollSession{} + } + r.scroll.abortErr = err + r.running = false +} + +func (r *RegionSelector) cleanupScroll() { + s := r.scroll + if s == nil { + return + } + if s.keysBound { + hyprlandUnbindScrollKeys() + } + if s.sigCh != nil { + signal.Stop(s.sigCh) + } + if s.frame != nil { + s.frame.Destroy() + } + if s.wlBuf != nil { + s.wlBuf.Destroy() + } + if s.pool != nil { + s.pool.Destroy() + } + if s.buf != nil { + s.buf.Close() + } +} diff --git a/core/internal/screenshot/scroll_sim_test.go b/core/internal/screenshot/scroll_sim_test.go new file mode 100644 index 000000000..64b30c382 --- /dev/null +++ b/core/internal/screenshot/scroll_sim_test.go @@ -0,0 +1,278 @@ +package screenshot + +import ( + "math/rand" + "slices" + "testing" +) + +// mirrors handleScrollFrame's stitch logic so glides run without a compositor +type simSession struct { + prevSig []float32 + prevPlaced bool + unmatched bool + unmatchedTicks int + st *stitcher +} + +func (s *simSession) observe(rows []byte) { + cols := s.st.rowSamples(rows) + sig := s.st.frameSig(rows) + dup := duplicateFrame(sig, s.prevSig) + s.prevSig = sig + + switch { + case dup && s.unmatched: + s.unmatchedTicks++ + if s.unmatchedTicks < scrollSeamTicks { + return + } + if _, placed := s.st.pushFrame(rows, cols); !placed { + s.st.seamAppend(rows, cols) + } + s.prevPlaced = true + s.unmatched = false + s.unmatchedTicks = 0 + case dup && s.prevPlaced: + return + default: + _, placed := s.st.pushFrame(rows, cols) + s.prevPlaced = placed + s.unmatched = !placed + s.unmatchedTicks = 0 + } +} + +// the page at a fractional scroll offset, as a compositor renders mid-glide +func fractionalFrame(page []byte, stride, frameH int, offset float64) []byte { + top := int(offset) + frac := offset - float64(top) + out := make([]byte, frameH*stride) + for y := 0; y < frameH; y++ { + a := page[(top+y)*stride : (top+y+1)*stride] + b := page[(top+y+1)*stride : (top+y+2)*stride] + row := out[y*stride : (y+1)*stride] + for x := range row { + row[x] = byte(float64(a[x])*(1-frac) + float64(b[x])*frac) + } + } + return out +} + +// blank gaps between paragraphs plus identical card blocks repeated around +func webbyPage(rng *rand.Rand, stride, rows int) []byte { + page := make([]byte, rows*stride) + card := make([]byte, 40*stride) + rng.Read(card) + + row := 0 + for row < rows { + switch rng.Intn(4) { + case 0: // blank gap + row += 10 + rng.Intn(20) + case 1: // repeated card block + n := copy(page[row*stride:], card) + row += n / stride + default: // paragraph of distinct rows + n := (8 + rng.Intn(22)) * stride + if row*stride+n > len(page) { + n = len(page) - row*stride + } + rng.Read(page[row*stride : row*stride+n]) + row += n / stride + } + } + return page +} + +// screen-fixed sidebar in the unsampled outer 8% plus per-frame hover noise +func addFixedChrome(rng *rand.Rand, frame []byte, stride, frameH int, sidebar []byte) { + sbw := len(sidebar) / frameH + for y := 0; y < frameH; y++ { + copy(frame[y*stride:y*stride+sbw], sidebar[y*sbw:(y+1)*sbw]) + } + hoverTop := 40 + rng.Intn(frameH-80) + for y := hoverTop; y < hoverTop+24; y++ { + off := y*stride + stride/3 + for x := 0; x < 60; x++ { + frame[off+x] ^= 0x08 + } + } +} + +// starting at the page bottom and scrolling up must prepend, never stall +func TestScrollSimulationBottomUp(t *testing.T) { + const stride = 2048 + const frameH = 240 + rng := rand.New(rand.NewSource(99)) + page := webbyPage(rng, stride, 4000) + + st := newStitcher(stride) + sidebar := make([]byte, frameH*140) + rng.Read(sidebar) + + sim := &simSession{st: st} + + pos := 3700.0 + capture := func() []byte { + f := fractionalFrame(page, stride, frameH, pos) + addFixedChrome(rng, f, stride, frameH, sidebar) + return f + } + glide := func(target float64) { + for i := 0; ; i++ { + step := (target - pos) * 0.45 + if step > -1 && step < 1 { + break + } + pos += step + if i%4 != 3 { + pos = float64(int(pos)) + } + sim.observe(capture()) + } + pos = target + sim.observe(capture()) + sim.observe(capture()) + } + + sim.observe(capture()) + for _, target := range []float64{3640, 3560, 3460, 3340, 3240} { + glide(target) + } + + wantRows := (3700 + frameH) - 3240 + got := sim.st.rows() + if got < wantRows-stitchMinAppend || got > wantRows+2 { + t.Fatalf("canvas has %d rows, want ~%d (upward scrolling must prepend)", got, wantRows) + } + topPage := 3240 + (wantRows - got) + for _, cr := range []int{0, 100, 300} { + if !rowMatchesPage(sim.st.canvas, page, stride, cr, topPage+cr) { + t.Fatalf("canvas row %d does not map onto page row %d", cr, topPage+cr) + } + } +} + +// exact page row or a blend of neighbors, allowing a one-row offset +func rowMatchesPage(canvas, page []byte, stride, canvasRow, pageRow int) bool { + for x := 200; x < stride-1400; x++ { + c := int(canvas[canvasRow*stride+x]) + lo, hi := 255, 0 + for k := pageRow - 1; k <= pageRow+1; k++ { + v := int(page[k*stride+x]) + lo, hi = min(lo, v), max(hi, v) + } + if c < lo-1 || c > hi+1 { + return false + } + } + return true +} + +// a fling past a full frame height must seam a new segment, not go dead +func TestScrollSimulationFastFlingRecovers(t *testing.T) { + const stride = 2048 + const frameH = 240 + rng := rand.New(rand.NewSource(7)) + page := webbyPage(rng, stride, 4000) + + sim := &simSession{st: newStitcher(stride)} + frame := func(top int) []byte { + return slices.Clone(page[top*stride : (top+frameH)*stride]) + } + rest := func(top int) { + for range scrollSeamTicks + 2 { + sim.observe(frame(top)) + } + } + + rest(0) + sim.observe(frame(60)) + sim.observe(frame(130)) + rest(130) + firstRange := 130 + frameH + + sim.observe(frame(900)) + sim.observe(frame(1400)) + rest(1800) + + sim.observe(frame(1860)) + sim.observe(frame(1930)) + rest(1930) + + wantRows := firstRange + (1930 - 1800) + frameH + if got := sim.st.rows(); got != wantRows { + t.Fatalf("canvas has %d rows, want %d (first range %d + new segment)", got, wantRows, firstRange) + } + seamStart := firstRange + if !slices.Equal(sim.st.canvas[seamStart*stride:], page[1800*stride:(1930+frameH)*stride]) { + t.Fatal("new segment content wrong after fling recovery") + } +} + +// eased glides with up/down scrubbing must cover the range exactly once +func TestScrollSimulationSmoothGlide(t *testing.T) { + const stride = 2048 + const frameH = 240 + rng := rand.New(rand.NewSource(99)) + page := webbyPage(rng, stride, 4000) + + st := newStitcher(stride) + sidebar := make([]byte, frameH*140) + rng.Read(sidebar) + + sim := &simSession{st: st} + + pos := 0.0 + capture := func() []byte { + f := fractionalFrame(page, stride, frameH, pos) + addFixedChrome(rng, f, stride, frameH, sidebar) + return f + } + glide := func(target float64) { + for i := 0; ; i++ { + step := (target - pos) * 0.45 + if step > -1 && step < 1 { + break + } + pos += step + // mostly snapped to device pixels, with the odd fractional frame + if i%4 != 3 { + pos = float64(int(pos)) + } + sim.observe(capture()) + } + pos = target + sim.observe(capture()) + sim.observe(capture()) + } + + sim.observe(capture()) + for _, target := range []float64{160, 330, 480, 650, 800, 960, 1100} { + glide(target) + } + for _, target := range []float64{700, 300, 900, 1100} { + glide(target) + } + + wantRows := 1100 + frameH + got := sim.st.rows() + if got < wantRows-stitchMinAppend || got > wantRows+2 { + t.Fatalf("canvas has %d rows, want ~%d (more = duplicated bands, fewer = gaps)", got, wantRows) + } + + hoverLo, hoverHi := stride/3, stride/3+60 + mismatched := 0 + for row := 0; row < min(got, wantRows); row += 7 { + off := row * stride + a1, b1 := sim.st.canvas[off+200:off+hoverLo], page[off+200:off+hoverLo] + a2, b2 := sim.st.canvas[off+hoverHi:off+stride], page[off+hoverHi:off+stride] + if !slices.Equal(a1, b1) || !slices.Equal(a2, b2) { + mismatched++ + } + } + if mismatched > (wantRows/7)/20 { + t.Fatalf("%d of %d sampled rows mismatch page content (mid-animation pixels baked in)", mismatched, wantRows/7) + } +} diff --git a/core/internal/screenshot/stitch.go b/core/internal/screenshot/stitch.go new file mode 100644 index 000000000..60bc03d30 --- /dev/null +++ b/core/internal/screenshot/stitch.go @@ -0,0 +1,407 @@ +package screenshot + +// Frame stitcher after mark-shot's column-sampling design +// (https://github.com/jswysnemc/mark-shot, src/scroll/stitcher_algorithm.cpp). +// Only rows overhanging the captured range are committed; frames that match +// nothing are dropped without touching state. + +const ( + stitchMaxCanvasBytes = 256 << 20 + stitchMaxRowsCap = 30000 + + // mark-shot: StitchConfig{100, 9.0f, 15, 1.0f} + stitchAcceptDiff = 9.0 + stitchApproxDiff = 1.0 + stitchMinCompare = 50 + stitchMinCanvas = 100 + stitchMinAppend = 15 + stitchCoarseStep = 8 + stitchPredictWindow = 160 + stitchBandSamples = 17 + + // mark-shot: kDuplicateAvgDiff=1.1f, kDuplicateMaxDiff=4, 18x24 grid + stitchDupAvgDiff = 1.1 + stitchDupMaxDiff = 4.0 + stitchSigCols = 18 + stitchSigRows = 24 + + // blank rows agree at every offset and must not decide a match + stitchActivityMin = 2.0 + stitchRowMatchTol = 4.0 + stitchMinActive = 12 +) + +// mean luminance per band (8-32%, 34-66%, 68-92%); the outer 8% is chrome +type rowCols [3]float32 + +type stitcher struct { + stride int + sampleOffs [3][]int + + canvas []byte + cols []rowCols + + anchor int + last []rowCols + lastOffset int + + maxRows int + full bool +} + +func newStitcher(stride int) *stitcher { + px := stride / 4 + st := &stitcher{ + stride: stride, + maxRows: min(stitchMaxCanvasBytes/stride, stitchMaxRowsCap), + } + bands := [3][2]float64{{0.08, 0.32}, {0.34, 0.66}, {0.68, 0.92}} + for b, band := range bands { + lo := int(float64(px) * band[0]) + hi := max(int(float64(px)*band[1]), lo+1) + n := min(stitchBandSamples, hi-lo) + for s := range n { + st.sampleOffs[b] = append(st.sampleOffs[b], (lo+(hi-lo)*s/n)*4) + } + } + return st +} + +func (st *stitcher) rowSamples(data []byte) []rowCols { + rows := len(data) / st.stride + cols := make([]rowCols, rows) + for y := range rows { + row := data[y*st.stride:] + for b := range 3 { + var sum float32 + for _, off := range st.sampleOffs[b] { + sum += 0.114*float32(row[off]) + 0.587*float32(row[off+1]) + 0.299*float32(row[off+2]) + } + cols[y][b] = sum / float32(len(st.sampleOffs[b])) + } + } + return cols +} + +func (st *stitcher) frameSig(data []byte) []float32 { + rows := len(data) / st.stride + px := st.stride / 4 + sig := make([]float32, 0, stitchSigCols*stitchSigRows) + for gy := range stitchSigRows { + y := (2*gy + 1) * rows / (2 * stitchSigRows) + for gx := range stitchSigCols { + x := (2*gx + 1) * px / (2 * stitchSigCols) + off := y*st.stride + x*4 + sig = append(sig, 0.114*float32(data[off])+0.587*float32(data[off+1])+0.299*float32(data[off+2])) + } + } + return sig +} + +func (st *stitcher) rows() int { + return len(st.cols) +} + +func rowColsDiff(a, b rowCols) float32 { + return (abs32(a[0]-b[0]) + abs32(a[1]-b[1]) + abs32(a[2]-b[2])) / 3 +} + +func duplicateFrame(a, b []float32) bool { + if len(a) != len(b) || len(a) == 0 { + return false + } + var sum, maxDiff float32 + for i := range a { + d := abs32(a[i] - b[i]) + sum += d + maxDiff = max(maxDiff, d) + } + return sum/float32(len(a)) <= stitchDupAvgDiff && maxDiff <= stitchDupMaxDiff +} + +// sticky header/footer zones, per mark-shot: 10% top, 8% bottom, min 16px +func matchIgnores(h int) (top, bottom int) { + if h < 80 { + return 0, 0 + } + return clamp(h/10, 16, h/4), clamp(h*8/100, 16, h/4) +} + +func activity(f []rowCols) []bool { + active := make([]bool, len(f)) + for i := 1; i < len(f); i++ { + active[i] = rowColsDiff(f[i], f[i-1]) > stitchActivityMin + } + return active +} + +func (st *stitcher) pushFrame(frame []byte, f []rowCols) (int, bool) { + if st.full || len(f) == 0 { + return 0, true + } + + h := len(f) + if len(st.cols) == 0 { + n := st.appendRows(frame, f, 0) + st.anchor = 0 + st.last = f + st.lastOffset = 0 + return n, true + } + + pos, ok := st.locateFrame(f, activity(f)) + if !ok { + return 0, false + } + + delta := pos - st.anchor + added := 0 + if over := pos + h - len(st.cols); over >= stitchMinAppend { + added += st.appendRows(frame, f, h-over) + } + if over := -pos; over >= stitchMinAppend { + n := st.prependRows(frame, f, over) + added += n + pos += n + } + + st.anchor = pos + st.last = f + st.lastOffset = delta + return added, true +} + +// seamAppend starts a new segment after a jump capture couldn't follow. +func (st *stitcher) seamAppend(frame []byte, f []rowCols) int { + if st.full || len(f) == 0 { + return 0 + } + pos := len(st.cols) + n := st.appendRows(frame, f, 0) + st.anchor = pos + st.last = f + st.lastOffset = 0 + return n +} + +func (st *stitcher) locateFrame(f []rowCols, active []bool) (int, bool) { + d, diff := st.adjacentOffset(f, active) + pred := st.anchor + d + + if diff <= stitchAcceptDiff { + if _, ok := st.verifyAt(f, active, pred); ok { + return pred, true + } + } + if pos, _, ok := st.scanPositions(f, active, pred, true); ok { + return pos, true + } + pos, _, ok := st.scanPositions(f, active, pred, false) + return pos, ok +} + +func (st *stitcher) verifyAt(f []rowCols, active []bool, pos int) (float32, bool) { + diff, count, activeMatches := st.canvasDiff(f, active, pos) + ok := count >= stitchMinCanvas && diff <= stitchAcceptDiff && activeMatches >= stitchMinActive + return diff, ok +} + +// signed deltas searched outward from the previous one (mark-shot's +// predictOffsetIter), early-exiting once a diff beats approxDiff +func (st *stitcher) adjacentOffset(f []rowCols, active []bool) (int, float32) { + h := len(f) + if len(st.last) != h { + return 0, float32(1e9) + } + limit := max(h-stitchMinCompare-1, 0) + + bestD, bestDiff := 0, float32(1e9) + countdown := -1 + try := func(d int) bool { + if d < -limit || d > limit { + return false + } + diff, activeMatches := st.pairDiff(f, active, d) + if activeMatches >= stitchMinActive && diff < bestDiff { + bestDiff, bestD = diff, d + } + switch { + case bestDiff < stitchApproxDiff/4: + return true + case bestDiff < stitchApproxDiff && countdown < 0: + countdown = 10 + } + if countdown > 0 { + countdown-- + } + return countdown == 0 + } + + if try(st.lastOffset) { + return bestD, bestDiff + } + for k := 1; ; k++ { + lo, hi := st.lastOffset-k, st.lastOffset+k + if lo < -limit && hi > limit { + break + } + if try(hi) || try(lo) { + break + } + } + return bestD, bestDiff +} + +func (st *stitcher) pairDiff(f []rowCols, active []bool, d int) (float32, int) { + h := len(f) + top, bottom := matchIgnores(h) + lo := max(top, -d) + hi := min(h-bottom, h-d) + + count := hi - lo + if count < stitchMinCompare { + return float32(1e9), 0 + } + var sum float32 + activeMatches := 0 + for i := lo; i < hi; i++ { + rd := rowColsDiff(f[i], st.last[i+d]) + sum += rd + if active[i] && rd <= stitchRowMatchTol { + activeMatches++ + } + } + return sum / float32(count), activeMatches +} + +func (st *stitcher) canvasDiff(f []rowCols, active []bool, pos int) (float32, int, int) { + h := len(f) + top, bottom := matchIgnores(h) + lo := max(top, -pos) + hi := min(h-bottom, len(st.cols)-pos) + + count := hi - lo + if count < 1 { + return float32(1e9), 0, 0 + } + var sum float32 + activeMatches := 0 + for i := lo; i < hi; i++ { + rd := rowColsDiff(f[i], st.cols[pos+i]) + sum += rd + if active[i] && rd <= stitchRowMatchTol { + activeMatches++ + } + } + return sum / float32(count), count, activeMatches +} + +// mark-shot's findEdgePosition (nearOnly: edges + prediction window, 1px) and +// findKnownPosition (coarse sweep refined around the winner) +func (st *stitcher) scanPositions(f []rowCols, active []bool, pred int, nearOnly bool) (int, float32, bool) { + h := len(f) + C := len(st.cols) + minPos := stitchMinCanvas - h + maxPos := C - stitchMinCanvas + + bestPos, bestDiff := 0, float32(1e9) + bestDist := 1 << 30 + consider := func(pos int) { + if pos < minPos || pos > maxPos { + return + } + diff, ok := st.verifyAt(f, active, pos) + if !ok { + return + } + dist := pos - pred + if dist < 0 { + dist = -dist + } + better := diff < bestDiff + if !nearOnly { + better = dist < bestDist || dist == bestDist && diff < bestDiff + } + if better { + bestPos, bestDiff, bestDist = pos, diff, dist + } + } + + if nearOnly { + for pos := pred - stitchPredictWindow; pos <= pred+stitchPredictWindow; pos++ { + consider(pos) + } + for pos := C - h; pos <= maxPos; pos++ { + consider(pos) + } + for pos := minPos; pos <= 0; pos++ { + consider(pos) + } + if bestDiff > stitchAcceptDiff { + return 0, 0, false + } + return bestPos, bestDiff, true + } + + for pos := minPos; pos <= maxPos; pos += stitchCoarseStep { + consider(pos) + } + if bestDiff > stitchAcceptDiff { + return 0, 0, false + } + + refined, refinedDiff := bestPos, bestDiff + for pos := bestPos - stitchCoarseStep + 1; pos < bestPos+stitchCoarseStep; pos++ { + if pos == bestPos { + continue + } + if diff, ok := st.verifyAt(f, active, pos); ok && diff < refinedDiff { + refined, refinedDiff = pos, diff + } + } + return refined, refinedDiff, true +} + +func (st *stitcher) appendRows(frame []byte, f []rowCols, from int) int { + n := len(f) - from + if room := st.maxRows - len(st.cols); n > room { + n = room + st.full = true + } + if n <= 0 { + st.full = true + return 0 + } + + st.canvas = append(st.canvas, frame[from*st.stride:(from+n)*st.stride]...) + st.cols = append(st.cols, f[from:from+n]...) + return n +} + +func (st *stitcher) prependRows(frame []byte, f []rowCols, n int) int { + if room := st.maxRows - len(st.cols); n > room { + n = room + st.full = true + } + if n <= 0 { + st.full = true + return 0 + } + + canvas := make([]byte, n*st.stride+len(st.canvas)) + copy(canvas, frame[:n*st.stride]) + copy(canvas[n*st.stride:], st.canvas) + st.canvas = canvas + + cols := make([]rowCols, 0, n+len(st.cols)) + cols = append(cols, f[:n]...) + st.cols = append(cols, st.cols...) + return n +} + +func abs32(f float32) float32 { + if f < 0 { + return -f + } + return f +} diff --git a/core/internal/screenshot/stitch_test.go b/core/internal/screenshot/stitch_test.go new file mode 100644 index 000000000..f0b2ba9c5 --- /dev/null +++ b/core/internal/screenshot/stitch_test.go @@ -0,0 +1,169 @@ +package screenshot + +import ( + "bytes" + "math/rand" + "slices" + "testing" +) + +const ( + testStride = 512 + testFrameH = 240 +) + +func makePage(t *testing.T, rows int) []byte { + t.Helper() + rng := rand.New(rand.NewSource(42)) + page := make([]byte, rows*testStride) + rng.Read(page) + return page +} + +func frameAt(page []byte, top int) []byte { + return page[top*testStride : (top+testFrameH)*testStride] +} + +func pushFrame(st *stitcher, frame []byte) int { + n, _ := st.pushFrame(frame, st.rowSamples(frame)) + return n +} + +func TestStitchSlidingWindows(t *testing.T) { + page := makePage(t, 1000) + + for _, delta := range []int{20, 60, 110} { + st := newStitcher(testStride) + lastTop := 0 + for top := 0; top+testFrameH <= 900; top += delta { + lastTop = top + pushFrame(st, frameAt(page, top)) + } + + wantRows := lastTop + testFrameH + if st.rows() != wantRows { + t.Fatalf("delta %d: got %d rows, want %d", delta, st.rows(), wantRows) + } + if !bytes.Equal(st.canvas, page[:wantRows*testStride]) { + t.Fatalf("delta %d: canvas does not match source rows", delta) + } + } +} + +func TestStitchDropsNoOverlap(t *testing.T) { + page := makePage(t, 1000) + st := newStitcher(testStride) + + pushFrame(st, frameAt(page, 0)) + if appended := pushFrame(st, frameAt(page, testFrameH+50)); appended != 0 { + t.Fatalf("unmatched jump appended %d rows", appended) + } + if !bytes.Equal(st.canvas, page[:testFrameH*testStride]) { + t.Fatal("canvas changed on unmatched frame") + } +} + +func TestStitchNoGrowthCases(t *testing.T) { + page := makePage(t, 1000) + blank := make([]byte, testFrameH*testStride) + + cases := []struct { + name string + first, second []byte + }{ + {"identical frame", frameAt(page, 0), frameAt(page, 0)}, + {"jitter below min append", frameAt(page, 0), frameAt(page, stitchMinAppend-5)}, + {"blank on blank", blank, blank}, + } + for _, tc := range cases { + st := newStitcher(testStride) + pushFrame(st, tc.first) + if appended := pushFrame(st, tc.second); appended != 0 { + t.Fatalf("%s: appended %d rows", tc.name, appended) + } + if st.rows() != testFrameH { + t.Fatalf("%s: got %d rows, want %d", tc.name, st.rows(), testFrameH) + } + } +} + +func TestStitchRevisitNeverDuplicates(t *testing.T) { + page := makePage(t, 1000) + st := newStitcher(testStride) + + pushFrame(st, frameAt(page, 0)) + pushFrame(st, frameAt(page, 100)) + pushFrame(st, frameAt(page, 200)) + + for _, top := range []int{150, 60, 0, 80, 190} { + if appended := pushFrame(st, frameAt(page, top)); appended != 0 { + t.Fatalf("revisited frame at %d appended %d rows", top, appended) + } + } + pushFrame(st, frameAt(page, 300)) + + wantRows := 300 + testFrameH + if st.rows() != wantRows { + t.Fatalf("got %d rows, want %d", st.rows(), wantRows) + } + if !bytes.Equal(st.canvas, page[:wantRows*testStride]) { + t.Fatal("canvas corrupted by revisited frames") + } +} + +func TestStitchScrollUpPrepends(t *testing.T) { + page := makePage(t, 1000) + st := newStitcher(testStride) + + pushFrame(st, frameAt(page, 500)) + if appended := pushFrame(st, frameAt(page, 420)); appended != 80 { + t.Fatalf("upward frame appended %d rows, want 80", appended) + } + pushFrame(st, frameAt(page, 560)) + + if !bytes.Equal(st.canvas, page[420*testStride:(560+testFrameH)*testStride]) { + t.Fatal("canvas does not match page range after prepend + append") + } +} + +func TestStitchNoisyChromeStillMatches(t *testing.T) { + page := makePage(t, 1000) + st := newStitcher(testStride) + + addChrome := func(frame []byte, seed byte) []byte { + f := slices.Clone(frame) + for y := range testFrameH { + for x := range 32 { + f[y*testStride+x] = seed + byte(y) + } + } + for y := 100; y < 124; y++ { + for x := testStride / 2; x < testStride/2+40; x++ { + f[y*testStride+x] ^= 0x08 + } + } + return f + } + + pushFrame(st, addChrome(frameAt(page, 0), 1)) + if appended := pushFrame(st, addChrome(frameAt(page, 90), 2)); appended != 90 { + t.Fatalf("appended %d rows, want 90", appended) + } +} + +func TestStitchMaxRowsCap(t *testing.T) { + page := makePage(t, 1000) + st := newStitcher(testStride) + st.maxRows = testFrameH + 10 + + pushFrame(st, frameAt(page, 0)) + if appended := pushFrame(st, frameAt(page, 100)); appended != 10 { + t.Fatalf("appended %d rows past cap, want 10", appended) + } + if !st.full { + t.Fatal("stitcher not marked full at cap") + } + if pushFrame(st, frameAt(page, 300)) != 0 { + t.Fatal("push after full appended rows") + } +} diff --git a/core/internal/screenshot/types.go b/core/internal/screenshot/types.go index 00cbe1c97..54739b12a 100644 --- a/core/internal/screenshot/types.go +++ b/core/internal/screenshot/types.go @@ -9,6 +9,7 @@ const ( ModeAllScreens ModeOutput ModeLastRegion + ModeScroll ) type Format int @@ -62,6 +63,7 @@ type Config struct { SaveFile bool Notify bool Stdout bool + IntervalMs int } func DefaultConfig() Config {