Modularized

2025-10-18 23:35:51 -04:00
parent d76672ae80
commit a219a426c9
14 changed files with 1066 additions and 1930 deletions
--- a/cookie/fetcher.go
+++ b/cookie/fetcher.go
@@ -0,0 +1,302 @@
+package cookie
+
+import (
+	"compress/gzip"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"log"
+	"math/rand"
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	CookieRetryDelay    = 5 * time.Second
+	MaxCookieRetryDelay = 60 * time.Second
+	CookieRefreshEvery  = 4 * time.Hour
+)
+
+type RefreshService struct {
+	username, password, domain string
+	client                     *http.Client
+
+	cookieMu      sync.RWMutex
+	currentCookie string
+
+	readyOnce sync.Once
+	readyCh   chan struct{}
+
+	stopCh chan struct{}
+	wg     sync.WaitGroup
+}
+
+func NewRefreshService(username, password, domain string) *RefreshService {
+	jar, _ := cookiejar.New(nil)
+	tr := &http.Transport{
+		// Force HTTP/1.1 (avoid ALPN h2 differences)
+		TLSNextProto: make(map[string]func(string, *tls.Conn) http.RoundTripper),
+	}
+	client := &http.Client{
+		Jar:       jar,
+		Timeout:   30 * time.Second,
+		Transport: tr,
+	}
+	return &RefreshService{
+		username: username,
+		password: password,
+		domain:   domain,
+		client:   client,
+		readyCh:  make(chan struct{}),
+		stopCh:   make(chan struct{}),
+	}
+}
+
+func (r *RefreshService) Start() {
+	r.wg.Add(1)
+	go r.loop()
+}
+
+func (r *RefreshService) Stop() {
+	close(r.stopCh)
+	r.wg.Wait()
+}
+
+func (r *RefreshService) WaitForCookie() { <-r.readyCh }
+
+func (r *RefreshService) GetCurrentCookie() string {
+	r.cookieMu.RLock()
+	defer r.cookieMu.RUnlock()
+	return r.currentCookie
+}
+
+func (r *RefreshService) loop() {
+	defer r.wg.Done()
+
+	log.Println("🔑 Fetching initial cookie...")
+	c, err := r.FetchFreshCookie()
+	if err != nil {
+		log.Printf("❌ Failed to acquire initial cookie: %v", err)
+		return
+	}
+	r.cookieMu.Lock()
+	r.currentCookie = c
+	r.cookieMu.Unlock()
+	r.readyOnce.Do(func() { close(r.readyCh) })
+	log.Println("✅ Initial cookie acquired")
+}
+
+func (r *RefreshService) FetchFreshCookie() (string, error) {
+	attempt := 0
+	delay := CookieRetryDelay
+	for {
+		select {
+		case <-r.stopCh:
+			return "", fmt.Errorf("stopped")
+		default:
+		}
+
+		attempt++
+		if attempt > 1 {
+			log.Printf("🔄 Cookie fetch retry attempt %d (waiting %v)...", attempt, delay)
+			time.Sleep(delay)
+			delay *= 2
+			if delay > MaxCookieRetryDelay {
+				delay = MaxCookieRetryDelay
+			}
+		}
+
+		c, err := r.attemptFetchCookie()
+		if err != nil {
+			log.Printf("⚠️ Cookie fetch attempt %d failed: %v", attempt, err)
+			continue
+		}
+		if strings.Contains(c, "xf_user=") {
+			log.Printf("✅ Successfully fetched fresh cookie with xf_user (attempt %d)", attempt)
+			r.cookieMu.Lock()
+			r.currentCookie = c
+			r.cookieMu.Unlock()
+			return c, nil
+		}
+		log.Printf("❌ Cookie fetch attempt %d missing xf_user — retrying...", attempt)
+	}
+}
+
+func (r *RefreshService) attemptFetchCookie() (string, error) {
+	// Step 1: KiwiFlare
+	log.Println("Step 1: Checking for KiwiFlare challenge...")
+	clearance, err := r.getClearanceToken()
+	if err != nil {
+		return "", fmt.Errorf("clearance token error: %w", err)
+	}
+	if clearance != "" {
+		log.Println("✅ KiwiFlare challenge solved")
+		log.Println("⏳ Waiting 2 seconds for cookie propagation...")
+		time.Sleep(2 * time.Second)
+	}
+
+	// Step 2: GET /login
+	log.Println("Step 2: Fetching login page...")
+	loginURL := fmt.Sprintf("https://%s/login/", r.domain)
+	req, _ := http.NewRequest("GET", loginURL, nil)
+	req.Header.Set("User-Agent", randomUserAgent())
+	req.Header.Set("Cache-Control", "no-cache")
+	req.Header.Set("Pragma", "no-cache")
+	req.URL.RawQuery = fmt.Sprintf("r=%d", rand.Intn(1_000_000))
+
+	resp, err := r.client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("failed to get login page: %w", err)
+	}
+	defer resp.Body.Close()
+	log.Printf("→ Using protocol for login page: %s", resp.Proto)
+
+	body, _ := io.ReadAll(resp.Body)
+	bodyStr := string(body)
+
+	log.Println("⏳ Waiting 1 second before processing login page...")
+	time.Sleep(1 * time.Second)
+
+	// Step 3: Extract CSRF
+	log.Println("Step 3: Extracting CSRF token...")
+	var csrf string
+	for _, pat := range []*regexp.Regexp{
+		regexp.MustCompile(`<html[^>]*data-csrf=["']([^"']+)["']`),
+		regexp.MustCompile(`name="_xfToken" value="([^"]+)"`),
+		regexp.MustCompile(`data-csrf=["']([^"']+)["']`),
+		regexp.MustCompile(`"csrf":"([^"]+)"`),
+		regexp.MustCompile(`XF\.config\.csrf\s*=\s*"([^"]+)"`),
+	} {
+		if m := pat.FindStringSubmatch(bodyStr); len(m) >= 2 {
+			csrf = m[1]
+			break
+		}
+	}
+	if csrf == "" {
+		log.Printf("⚠️ CSRF token not found. Partial HTML:\n%s", bodyStr[:min(800, len(bodyStr))])
+		return "", fmt.Errorf("CSRF token not found in login page")
+	}
+	log.Printf("✅ Found CSRF token: %s...", csrf[:min(10, len(csrf))])
+
+	// Step 4: POST /login/login
+	log.Println("Step 4: Submitting login credentials...")
+	postURL := fmt.Sprintf("https://%s/login/login", r.domain)
+	form := url.Values{
+		"_xfToken":      {csrf},
+		"_xfRequestUri": {"/"},
+		"_xfWithData":   {"1"},
+		"login":         {r.username},
+		"password":      {r.password},
+		"_xfRedirect":   {fmt.Sprintf("https://%s/", r.domain)},
+		"remember":      {"1"},
+	}
+
+	// ensure GET cookies are kept
+	cookieURL, _ := url.Parse(fmt.Sprintf("https://%s/", r.domain))
+	if resp.Cookies() != nil {
+		r.client.Jar.SetCookies(cookieURL, resp.Cookies())
+	}
+
+	postReq, _ := http.NewRequest("POST", postURL, strings.NewReader(form.Encode()))
+	postReq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	postReq.Header.Set("User-Agent", randomUserAgent())
+	postReq.Header.Set("Referer", loginURL)
+	postReq.Header.Set("Origin", fmt.Sprintf("https://%s", r.domain))
+	postReq.Header.Set("X-XF-Token", csrf)
+	postReq.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+	postReq.Header.Set("Accept-Language", "en-US,en;q=0.9")
+	postReq.Header.Set("Accept-Encoding", "gzip, deflate") // avoid br
+
+	loginResp, err := r.client.Do(postReq)
+	if err != nil {
+		return "", fmt.Errorf("login POST failed: %w", err)
+	}
+	defer loginResp.Body.Close()
+	log.Printf("Login response status: %d", loginResp.StatusCode)
+
+	// Follow a single redirect (XenForo usually sets xf_user on redirect target)
+	if loginResp.StatusCode >= 300 && loginResp.StatusCode < 400 {
+		if loc := loginResp.Header.Get("Location"); loc != "" {
+			log.Printf("Following redirect to %s to check for xf_user...", loc)
+			url2 := loc
+			if !strings.HasPrefix(loc, "http") {
+				url2 = fmt.Sprintf("https://%s%s", r.domain, loc)
+			}
+			time.Sleep(1 * time.Second)
+			if fr, err := r.client.Get(url2); err == nil {
+				fr.Body.Close()
+				time.Sleep(500 * time.Millisecond)
+			}
+		}
+	}
+
+	// Decode response (gzip)
+	var reader io.ReadCloser
+	if loginResp.Header.Get("Content-Encoding") == "gzip" {
+		gz, ge := gzip.NewReader(loginResp.Body)
+		if ge == nil {
+			reader = gz
+			defer gz.Close()
+		} else {
+			reader = io.NopCloser(loginResp.Body)
+		}
+	} else {
+		reader = io.NopCloser(loginResp.Body)
+	}
+	respHTML, _ := io.ReadAll(reader)
+	if strings.Contains(string(respHTML), `data-logged-in="false"`) {
+		log.Println("⚠️ HTML indicates still logged out (data-logged-in=false)")
+		time.Sleep(1 * time.Second)
+		return r.retryWithFreshCSRF()
+	}
+
+	// Normalize cookie domains and compose cookie string
+	cookies := r.client.Jar.Cookies(cookieURL)
+	for _, c := range cookies {
+		c.Domain = strings.TrimPrefix(c.Domain, ".")
+	}
+	r.client.Jar.SetCookies(cookieURL, cookies)
+
+	want := map[string]bool{
+		"xf_user":        true,
+		"xf_toggle":      true,
+		"xf_csrf":        true,
+		"xf_session":     true,
+		"sssg_clearance": true,
+	}
+	var parts []string
+	hasUser := false
+	for _, c := range cookies {
+		if want[c.Name] {
+			parts = append(parts, fmt.Sprintf("%s=%s", c.Name, c.Value))
+			if c.Name == "xf_user" {
+				hasUser = true
+			}
+		}
+	}
+	if !hasUser {
+		return "", fmt.Errorf("xf_user cookie missing — authentication failed, will retry")
+	}
+	return strings.Join(parts, "; "), nil
+}
+
+func randomUserAgent() string {
+	agents := []string{
+		"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0",
+		"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
+		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15",
+	}
+	return agents[rand.Intn(len(agents))]
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
--- a/cookie/retry.go
+++ b/cookie/retry.go
@@ -0,0 +1,78 @@
+package cookie
+
+import (
+	"compress/gzip"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+)
+
+func (r *RefreshService) retryWithFreshCSRF() (string, error) {
+	loginURL := fmt.Sprintf("https://%s/login/", r.domain)
+	resp, err := r.client.Get(loginURL)
+	if err != nil {
+		return "", fmt.Errorf("failed to refetch login page: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, _ := io.ReadAll(resp.Body)
+	re := regexp.MustCompile(`name="_xfToken" value="([^"]+)"`)
+	m := re.FindSubmatch(body)
+	if len(m) < 2 {
+		return "", fmt.Errorf("csrf retry token not found")
+	}
+	csrf := string(m[1])
+	log.Printf("✅ Retry CSRF token: %.10s...", csrf)
+
+	postURL := fmt.Sprintf("https://%s/login/login", r.domain)
+	form := url.Values{
+		"login":       {r.username},
+		"password":    {r.password},
+		"_xfToken":    {csrf},
+		"_xfRedirect": {"/"},
+	}
+	req, _ := http.NewRequest("POST", postURL, strings.NewReader(form.Encode()))
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	req.Header.Set("User-Agent", randomUserAgent())
+	req.Header.Set("Referer", loginURL)
+	req.Header.Set("Origin", fmt.Sprintf("https://%s", r.domain))
+	req.Header.Set("X-XF-Token", csrf)
+	req.Header.Set("Accept-Encoding", "gzip, deflate")
+
+	resp2, err := r.client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("retry POST failed: %v", err)
+	}
+	defer resp2.Body.Close()
+
+	var reader io.ReadCloser
+	if resp2.Header.Get("Content-Encoding") == "gzip" {
+		gz, ge := gzip.NewReader(resp2.Body)
+		if ge == nil {
+			reader = gz
+			defer gz.Close()
+		} else {
+			reader = io.NopCloser(resp2.Body)
+		}
+	} else {
+		reader = io.NopCloser(resp2.Body)
+	}
+	b, _ := io.ReadAll(reader)
+	if strings.Contains(string(b), `data-logged-in="true"`) {
+		log.Println("✅ Retry indicates logged in successfully")
+	}
+
+	cookieURL, _ := url.Parse(fmt.Sprintf("https://%s/", r.domain))
+	for _, c := range r.client.Jar.Cookies(cookieURL) {
+		if c.Name == "xf_user" {
+			log.Printf("✅ Successfully fetched fresh cookie with xf_user: %.12s...", c.Value)
+			// Rebuild cookie header with known-good set (reuse attemptFetchCookie’s logic if you want)
+			return "xf_user=" + c.Value, nil
+		}
+	}
+	return "", fmt.Errorf("retry still missing xf_user cookie")
+}
--- a/cookie/solver.go
+++ b/cookie/solver.go
@@ -0,0 +1,138 @@
+package cookie
+
+import (
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"math/rand"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func (r *RefreshService) getClearanceToken() (string, error) {
+	baseURL := fmt.Sprintf("https://%s/", r.domain)
+	req, _ := http.NewRequest("GET", baseURL, nil)
+	req.Header.Set("User-Agent", randomUserAgent())
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+	req.Header.Set("Accept-Language", "en-US,en;q=0.9")
+	req.Header.Set("Connection", "keep-alive")
+
+	resp, err := r.client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	body, _ := io.ReadAll(resp.Body)
+
+	// Detect challenge (several patterns)
+	patterns := []*regexp.Regexp{
+		regexp.MustCompile(`<html[^>]*id=["']sssg["'][^>]*data-sssg-challenge=["']([^"']+)["'][^>]*data-sssg-difficulty=["'](\d+)["']`),
+		regexp.MustCompile(`<html[^>]*id=["']sssg["'][^>]*data-sssg-difficulty=["'](\d+)["'][^>]*data-sssg-challenge=["']([^"']+)["']`),
+		regexp.MustCompile(`data-sssg-challenge=["']([^"']+)["'][^>]*data-sssg-difficulty=["'](\d+)["']`),
+	}
+	var salt string
+	var difficulty int
+	found := false
+	for i, p := range patterns {
+		if m := p.FindStringSubmatch(string(body)); len(m) >= 3 {
+			if i == 1 {
+				difficulty, _ = strconv.Atoi(m[1])
+				salt = m[2]
+			} else {
+				salt = m[1]
+				difficulty, _ = strconv.Atoi(m[2])
+			}
+			found = true
+			break
+		}
+	}
+	if !found || difficulty == 0 || salt == "" {
+		return "", nil
+	}
+
+	log.Printf("Solving KiwiFlare challenge (difficulty=%d)", difficulty)
+	time.Sleep(time.Duration(500+rand.Intn(750)) * time.Millisecond)
+
+	nonce, err := r.solvePoW(salt, difficulty)
+	if err != nil {
+		return "", err
+	}
+
+	time.Sleep(time.Duration(700+rand.Intn(900)) * time.Millisecond)
+
+	submitURL := fmt.Sprintf("https://%s/.sssg/api/answer", r.domain)
+	form := url.Values{"a": {salt}, "b": {nonce}}
+	post, _ := http.NewRequest("POST", submitURL, strings.NewReader(form.Encode()))
+	post.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	post.Header.Set("User-Agent", randomUserAgent())
+	post.Header.Set("Origin", baseURL)
+	post.Header.Set("Referer", baseURL)
+
+	resp2, err := r.client.Do(post)
+	if err != nil {
+		return "", err
+	}
+	defer resp2.Body.Close()
+
+	// Some deployments return JSON like {"auth":"..."}
+	var result map[string]any
+	_ = json.NewDecoder(resp2.Body).Decode(&result)
+
+	time.Sleep(time.Duration(1200+rand.Intn(800)) * time.Millisecond)
+
+	cookieURL, _ := url.Parse(baseURL)
+	for _, c := range r.client.Jar.Cookies(cookieURL) {
+		if c.Name == "sssg_clearance" {
+			log.Printf("✅ KiwiFlare clearance cookie confirmed: %s...", c.Value[:min(10, len(c.Value))])
+			return c.Value, nil
+		}
+	}
+	if v, ok := result["auth"].(string); ok && v != "" {
+		// Fallback: manually add
+		r.client.Jar.SetCookies(cookieURL, []*http.Cookie{{
+			Name:   "sssg_clearance",
+			Value:  v,
+			Path:   "/",
+			Domain: r.domain,
+		}})
+		return v, nil
+	}
+	return "", fmt.Errorf("clearance cookie missing after solve")
+}
+
+func (r *RefreshService) solvePoW(salt string, difficulty int) (string, error) {
+	start := time.Now()
+	bytes := difficulty / 8
+	bits := difficulty % 8
+
+	for nonce := rand.Int63(); ; nonce++ {
+		sum := sha256.Sum256([]byte(fmt.Sprintf("%s%d", salt, nonce)))
+		ok := true
+		for i := 0; i < bytes; i++ {
+			if sum[i] != 0 {
+				ok = false
+				break
+			}
+		}
+		if ok && bits > 0 && bytes < len(sum) {
+			mask := byte(0xFF << (8 - bits))
+			if sum[bytes]&mask != 0 {
+				ok = false
+			}
+		}
+		if ok {
+			delay := time.Duration(2+rand.Intn(3))*time.Second - time.Since(start)
+			if delay > 0 {
+				time.Sleep(delay)
+			}
+			return fmt.Sprintf("%d", nonce), nil
+		}
+	}
+}