mirror of
https://gitgud.io/yats/libkiwi.git
synced 2026-06-15 15:55:28 -04:00
Refactoring n shit
This commit is contained in:
+99
@@ -0,0 +1,99 @@
|
||||
package libkiwi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/url"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type LoginResp struct {
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message"`
|
||||
Redirect *url.URL `json:"redirect"`
|
||||
Visitor Visitor `json:"visitor"`
|
||||
}
|
||||
|
||||
func (lr *LoginResp) UnmarshalJSON(b []byte) error {
|
||||
var jsonMap map[string]any
|
||||
err := json.Unmarshal(b, &jsonMap)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for k, v := range jsonMap {
|
||||
switch k {
|
||||
case "status":
|
||||
lr.Status = v.(string)
|
||||
case "message":
|
||||
lr.Message = v.(string)
|
||||
case "redirect":
|
||||
u, err := url.Parse(v.(string))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lr.Redirect = u
|
||||
case "visitor":
|
||||
vst, err := parseVisitorMap(v.(map[string]any))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lr.Visitor = vst
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var ErrNoXFToken = errors.New("Failed to locate xfToken on page.")
|
||||
|
||||
type Visitor struct {
|
||||
ConversationsUnread uint32 `json:"conversations_unread"`
|
||||
AlertsUnviewed uint32 `json:"alerts_unviewed"`
|
||||
TotalUnread uint32 `json:"total_unread"`
|
||||
}
|
||||
|
||||
func (vst *Visitor) UnmarshalJSON(b []byte) error {
|
||||
var jsonMap map[string]any
|
||||
err := json.Unmarshal(b, &jsonMap)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
newVst, err := parseVisitorMap(jsonMap)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*vst = newVst
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseVisitorMap(jsonMap map[string]any) (Visitor, error) {
|
||||
var out Visitor
|
||||
|
||||
for k, v := range jsonMap {
|
||||
switch k {
|
||||
case "conversations_unread":
|
||||
convos, err := strconv.Atoi(v.(string))
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.ConversationsUnread = uint32(convos)
|
||||
case "alerts_unviewed":
|
||||
alerts, err := strconv.Atoi(v.(string))
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.AlertsUnviewed = uint32(alerts)
|
||||
case "total_unread":
|
||||
unread, err := strconv.Atoi(v.(string))
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.ConversationsUnread = uint32(unread)
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
@@ -2,129 +2,52 @@ package libkiwi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
"golang.org/x/net/html/atom"
|
||||
)
|
||||
|
||||
type LoginResp struct {
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message"`
|
||||
Redirect *url.URL `json:"redirect"`
|
||||
Visitor struct {
|
||||
ConversationsUnread uint32 `json:"conversations_unread"`
|
||||
AlertsUnviewed uint32 `json:"alerts_unviewed"`
|
||||
TotalUnread uint32 `json:"total_unread"`
|
||||
} `json:"visitor"`
|
||||
type Credentials struct {
|
||||
Username string
|
||||
Password string
|
||||
Remember bool
|
||||
}
|
||||
|
||||
func (lr *LoginResp) UnmarshalJSON(b []byte) error {
|
||||
var jsonMap map[string]any
|
||||
err := json.Unmarshal(b, &jsonMap)
|
||||
if err != nil {
|
||||
return err
|
||||
func loginForm(xfToken string, creds Credentials) url.Values {
|
||||
rem := []byte{'0'}
|
||||
if creds.Remember {
|
||||
rem[0] = '1'
|
||||
}
|
||||
|
||||
for k, v := range jsonMap {
|
||||
switch k {
|
||||
case "status":
|
||||
lr.Status = v.(string)
|
||||
case "message":
|
||||
lr.Message = v.(string)
|
||||
case "redirect":
|
||||
u, err := url.Parse(v.(string))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lr.Redirect = u
|
||||
case "visitor":
|
||||
for k, v := range v.(map[string]any) {
|
||||
switch k {
|
||||
case "conversations_unread":
|
||||
convos, err := strconv.Atoi(v.(string))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lr.Visitor.ConversationsUnread = uint32(convos)
|
||||
case "alerts_unviewed":
|
||||
alerts, err := strconv.Atoi(v.(string))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lr.Visitor.AlertsUnviewed = uint32(alerts)
|
||||
case "total_unread":
|
||||
unread, err := strconv.Atoi(v.(string))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lr.Visitor.ConversationsUnread = uint32(unread)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var ErrNoXFToken = errors.New("Failed to locate xfToken on page.")
|
||||
|
||||
func xfToken(page io.Reader) (string, error) {
|
||||
z := html.NewTokenizer(page)
|
||||
for i := z.Next(); i != html.ErrorToken; i = z.Next() {
|
||||
tk := z.Token()
|
||||
if tk.DataAtom == atom.Html {
|
||||
for _, a := range tk.Attr {
|
||||
switch a.Key {
|
||||
case "data-csrf":
|
||||
return a.Val, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "", ErrNoXFToken
|
||||
}
|
||||
|
||||
func loginURL(host *url.URL) *url.URL {
|
||||
return &url.URL{
|
||||
Scheme: "https",
|
||||
Host: host.Hostname(),
|
||||
Path: "login",
|
||||
return url.Values{
|
||||
"_xfToken": {xfToken},
|
||||
"login": {creds.Username},
|
||||
"password": {creds.Password},
|
||||
"remember": {string(rem)},
|
||||
}
|
||||
}
|
||||
|
||||
func (kf *KF) newLoginRequest(ctx context.Context, user string, pass string) (*http.Request, error) {
|
||||
u := loginURL(kf.domain)
|
||||
func (kf *KF) newLoginRequest(ctx context.Context, creds Credentials) (*http.Request, error) {
|
||||
u := kf.urlFromPath("login")
|
||||
|
||||
resp, err := kf.GetPage(ctx, u)
|
||||
resp, err := kf.Get(ctx, u)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
xfToken, err := xfToken(resp.Body)
|
||||
xfToken, err := XFToken(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
form := url.Values{
|
||||
"_xfToken": {xfToken},
|
||||
"login": {user},
|
||||
"password": {pass},
|
||||
"remember": {"1"},
|
||||
}
|
||||
form := loginForm(xfToken, creds)
|
||||
|
||||
// i.e. https://kiwifarms.net/login/login
|
||||
postURL := fmt.Sprintf("%s/login", u.String())
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", postURL, strings.NewReader(form.Encode()))
|
||||
reqURL := fmt.Sprintf("%s/login", u.String())
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", reqURL, strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -133,13 +56,13 @@ func (kf *KF) newLoginRequest(ctx context.Context, user string, pass string) (*h
|
||||
return req, nil
|
||||
}
|
||||
|
||||
func (kf *KF) Login(ctx context.Context, user string, pass string) (*http.Response, error) {
|
||||
req, err := kf.newLoginRequest(ctx, user, pass)
|
||||
func (kf *KF) Login(ctx context.Context, creds Credentials) (*http.Response, error) {
|
||||
req, err := kf.newLoginRequest(ctx, creds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := kf.client.Do(req)
|
||||
resp, err := kf.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -151,7 +74,7 @@ func (kf *KF) Login(ctx context.Context, user string, pass string) (*http.Respon
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func TwoFactorForm(xfToken string, code uint32, u *url.URL) url.Values {
|
||||
func twoFactorForm(xfToken string, code uint32, u *url.URL) url.Values {
|
||||
return url.Values{
|
||||
"code": {fmt.Sprint(code)},
|
||||
"provider": {"totp"},
|
||||
@@ -166,19 +89,19 @@ func TwoFactorForm(xfToken string, code uint32, u *url.URL) url.Values {
|
||||
}
|
||||
}
|
||||
|
||||
func (kf *KF) twoFactorAuthForm(ctx context.Context, resp *http.Response, code uint32) (*http.Request, error) {
|
||||
func (kf *KF) newTwoFactorRequest(ctx context.Context, resp *http.Response, code uint32) (*http.Request, error) {
|
||||
u := resp.Request.URL
|
||||
if !strings.Contains(u.Path, "two-step") {
|
||||
// TODO: move err to proper type with url included.
|
||||
return nil, errors.New("Did not redirect to two-step page.")
|
||||
}
|
||||
|
||||
xfToken, err := xfToken(resp.Body)
|
||||
xfToken, err := XFToken(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
form := TwoFactorForm(xfToken, code, u)
|
||||
form := twoFactorForm(xfToken, code, u)
|
||||
reqURL := fmt.Sprintf("https://%s/login/two-step", u.Hostname())
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", reqURL, strings.NewReader(form.Encode()))
|
||||
@@ -191,28 +114,12 @@ func (kf *KF) twoFactorAuthForm(ctx context.Context, resp *http.Response, code u
|
||||
}
|
||||
|
||||
func (kf *KF) TwoFactorAuth(ctx context.Context, resp *http.Response, code uint32) (*http.Response, error) {
|
||||
req, err := kf.twoFactorAuthForm(ctx, resp, code)
|
||||
req, err := kf.newTwoFactorRequest(ctx, resp, code)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return kf.client.Do(req)
|
||||
}
|
||||
|
||||
func setCookie(u *url.URL, jar http.CookieJar, newCookie *http.Cookie) {
|
||||
cookies := jar.Cookies(u)
|
||||
|
||||
for i, c := range cookies {
|
||||
if c.Name == newCookie.Name {
|
||||
cookies[i] = newCookie
|
||||
jar.SetCookies(u, cookies)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Append if not already existing.
|
||||
cookies = append(cookies, newCookie)
|
||||
jar.SetCookies(u, cookies)
|
||||
return kf.Do(req)
|
||||
}
|
||||
|
||||
func (kf *KF) RefreshSession(ctx context.Context) error {
|
||||
@@ -222,7 +129,7 @@ func (kf *KF) RefreshSession(ctx context.Context) error {
|
||||
Value: "",
|
||||
})
|
||||
|
||||
resp, err := kf.GetPage(ctx, kf.domain)
|
||||
resp, err := kf.Get(ctx, kf.domain)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package libkiwi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/pquerna/otp/totp"
|
||||
)
|
||||
|
||||
func TestLogin(t *testing.T) {
|
||||
user, pass := os.Getenv("TEST_USER"), os.Getenv("TEST_PASS")
|
||||
if user == "" || pass == "" {
|
||||
t.Log("TEST_USER and/or TEST_PASS not set in env. Skipping.\n")
|
||||
return
|
||||
}
|
||||
|
||||
kf, err := newTestKF()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
ctx := t.Context()
|
||||
|
||||
resp, err := kf.Login(ctx, Credentials{
|
||||
Username: user,
|
||||
Password: pass,
|
||||
Remember: false, // Don't wastefully create dangling sessions.
|
||||
})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
t.Logf("Login response: %+v\n", resp)
|
||||
|
||||
if strings.Contains(resp.Request.URL.Path, "two-step") {
|
||||
code, err := genTest2FACode()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
resp, err := kf.TwoFactorAuth(ctx, resp, code)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var lr LoginResp
|
||||
|
||||
err = json.NewDecoder(resp.Body).Decode(&lr)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
t.Logf("2FA response: %+v\n", lr)
|
||||
}
|
||||
}
|
||||
|
||||
func genTest2FACode() (uint32, error) {
|
||||
secret := os.Getenv("TEST_TOTP")
|
||||
if secret == "" {
|
||||
return 0, errors.New("TEST_TOTP not set in env.")
|
||||
}
|
||||
|
||||
codeStr, err := totp.GenerateCode(secret, time.Now())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
code, err := strconv.Atoi(codeStr)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return uint32(code), nil
|
||||
}
|
||||
@@ -5,11 +5,13 @@ go 1.26.1
|
||||
require (
|
||||
gitgud.io/yats/cerberus v0.0.0-20260214165307-66e6f74a4be9
|
||||
github.com/PuerkitoBio/goquery v1.12.0
|
||||
github.com/pquerna/otp v1.5.0
|
||||
golang.org/x/net v0.55.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
|
||||
github.com/minio/sha256-simd v1.0.1 // indirect
|
||||
golang.org/x/sys v0.45.0 // indirect
|
||||
|
||||
@@ -4,11 +4,22 @@ github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO
|
||||
github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ=
|
||||
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
||||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI=
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
|
||||
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
|
||||
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs=
|
||||
github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package libkiwi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
gq "github.com/PuerkitoBio/goquery"
|
||||
"golang.org/x/net/html"
|
||||
"golang.org/x/net/html/atom"
|
||||
)
|
||||
|
||||
// Get XFToken (data-csrf) from page html.
|
||||
func XFToken(page io.Reader) (string, error) {
|
||||
z := html.NewTokenizer(page)
|
||||
for i := z.Next(); i != html.ErrorToken; i = z.Next() {
|
||||
tk := z.Token()
|
||||
if tk.DataAtom == atom.Html {
|
||||
for _, a := range tk.Attr {
|
||||
switch a.Key {
|
||||
case "data-csrf":
|
||||
return a.Val, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "", ErrNoXFToken
|
||||
}
|
||||
|
||||
func setCookie(u *url.URL, jar http.CookieJar, newCookie *http.Cookie) {
|
||||
cookies := jar.Cookies(u)
|
||||
|
||||
for i, c := range cookies {
|
||||
if c.Name == newCookie.Name {
|
||||
cookies[i] = newCookie
|
||||
jar.SetCookies(u, cookies)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Append if not already existing.
|
||||
cookies = append(cookies, newCookie)
|
||||
jar.SetCookies(u, cookies)
|
||||
}
|
||||
|
||||
func (kf *KF) Do(req *http.Request) (*http.Response, error) {
|
||||
var (
|
||||
ctx = req.Context()
|
||||
bb bytes.Buffer
|
||||
)
|
||||
|
||||
if req.Body != nil {
|
||||
defer req.Body.Close()
|
||||
|
||||
r := io.TeeReader(req.Body, &bb)
|
||||
req.Body = io.NopCloser(r)
|
||||
}
|
||||
|
||||
resp, err := kf.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// KiwiFlare redirect is signaled by 203 status.
|
||||
if resp.StatusCode == 203 {
|
||||
defer resp.Body.Close()
|
||||
|
||||
err = kf.solveKiwiFlare(ctx, resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if req.Body != nil {
|
||||
req.Body = io.NopCloser(&bb)
|
||||
}
|
||||
|
||||
// Try request again now that we're authed.
|
||||
return kf.Do(req)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (kf *KF) Get(ctx context.Context, u *url.URL) (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := kf.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (kf *KF) GetPost(ctx context.Context, postID uint32) (Post, error) {
|
||||
// Example post goto link: https://kiwifarms.st/goto/post?id=22058462
|
||||
u, err := url.Parse(fmt.Sprintf("%s/goto/post?id=%d", kf.domain.String(), postID))
|
||||
if err != nil {
|
||||
return Post{}, err
|
||||
}
|
||||
|
||||
resp, err := kf.Get(ctx, u)
|
||||
if err != nil {
|
||||
return Post{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
doc, err := gq.NewDocumentFromReader(resp.Body)
|
||||
if err != nil {
|
||||
return Post{}, err
|
||||
}
|
||||
|
||||
// Selector: #js-post-22058462
|
||||
article := doc.Find(fmt.Sprintf("article#js-post-%d", postID))
|
||||
|
||||
body := article.Find("div.message-content article.message-body")
|
||||
if body.Length() == 0 {
|
||||
return Post{}, errors.New("Failed to parse post message body.")
|
||||
}
|
||||
|
||||
bh, err := body.Html()
|
||||
if err != nil {
|
||||
return Post{}, err
|
||||
}
|
||||
|
||||
author, err := parsePostAuthor(article)
|
||||
if err != nil {
|
||||
return Post{}, err
|
||||
}
|
||||
|
||||
post := Post{
|
||||
Author: author,
|
||||
Text: bytes.TrimSpace([]byte(body.Text())),
|
||||
HTML: bytes.TrimSpace([]byte(bh)),
|
||||
|
||||
article: article,
|
||||
body: body,
|
||||
}
|
||||
|
||||
return post, nil
|
||||
}
|
||||
+23
-97
@@ -1,7 +1,6 @@
|
||||
package libkiwi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -14,6 +13,8 @@ import (
|
||||
gq "github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
const _USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0"
|
||||
|
||||
type KF struct {
|
||||
client http.Client
|
||||
domain *url.URL
|
||||
@@ -38,47 +39,9 @@ func NewKF(hc http.Client, host *url.URL) (*KF, error) {
|
||||
domain: u,
|
||||
}
|
||||
|
||||
// Update host url in case we get redirected across domains.
|
||||
hc.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
||||
reqHost := req.URL.Hostname()
|
||||
if reqHost != u.Hostname() {
|
||||
// Deliberately set to Hostname() and not Host.
|
||||
// This excludes any extra shit like ports.
|
||||
u.Host = reqHost
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return kf, nil
|
||||
}
|
||||
|
||||
func (kf *KF) GetPage(ctx context.Context, u *url.URL) (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0")
|
||||
|
||||
resp, err := kf.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// KiwiFlare redirect is signaled by 203 status.
|
||||
if resp.StatusCode == 203 {
|
||||
err = kf.solveKiwiFlare(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Try fetching the page again now that we're authed.
|
||||
return kf.GetPage(ctx, u)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
type User struct {
|
||||
ID uint32
|
||||
Name string
|
||||
@@ -87,94 +50,57 @@ type User struct {
|
||||
}
|
||||
|
||||
func parsePostAuthor(article *gq.Selection) (User, error) {
|
||||
user := User{}
|
||||
|
||||
userBlock := article.Find("section.message-user")
|
||||
|
||||
idStr, ok := userBlock.Attr("data-user-id")
|
||||
if !ok {
|
||||
// TODO: Proper error types.
|
||||
return User{}, errors.New("Failed to parse post author attr.")
|
||||
return user, errors.New("Failed to parse post author attr.")
|
||||
}
|
||||
|
||||
id, err := strconv.Atoi(idStr)
|
||||
if err != nil {
|
||||
return User{}, err
|
||||
return user, err
|
||||
}
|
||||
user.ID = uint32(id)
|
||||
|
||||
name, ok := article.Attr("data-author")
|
||||
if !ok {
|
||||
return User{}, errors.New("Failed to parse post author attr.")
|
||||
return user, errors.New("Failed to parse post author attr.")
|
||||
}
|
||||
user.Name = name
|
||||
|
||||
title := userBlock.Find(".message-userTitle").Text()
|
||||
user.Title = userBlock.Find(".message-userTitle").Text()
|
||||
|
||||
urlStr, ok := userBlock.Attr("itemid")
|
||||
if !ok {
|
||||
return User{}, errors.New("Failed to parse post author attr.")
|
||||
return user, errors.New("Failed to parse post author attr.")
|
||||
}
|
||||
|
||||
u, err := url.Parse(urlStr)
|
||||
if err != nil {
|
||||
return User{}, err
|
||||
}
|
||||
|
||||
user := User{
|
||||
ID: uint32(id),
|
||||
Name: name,
|
||||
Title: title,
|
||||
URL: u,
|
||||
return user, err
|
||||
}
|
||||
user.URL = u
|
||||
|
||||
return user, nil
|
||||
}
|
||||
|
||||
type Post struct {
|
||||
Author User
|
||||
Body io.Reader
|
||||
Text []byte
|
||||
|
||||
HTML []byte
|
||||
|
||||
article *gq.Selection
|
||||
body *gq.Selection
|
||||
}
|
||||
|
||||
func (kf *KF) GetPost(ctx context.Context, postID uint32) (Post, error) {
|
||||
// Example post goto link: https://kiwifarms.st/goto/post?id=22058462
|
||||
gtl := fmt.Sprintf("%s/goto/post?id=%d", kf.domain.String(), postID)
|
||||
u, err := url.Parse(gtl)
|
||||
func (post *Post) TextContent() (io.Reader, error) {
|
||||
postHTML, err := post.body.Html()
|
||||
if err != nil {
|
||||
return Post{}, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := kf.GetPage(ctx, u)
|
||||
if err != nil {
|
||||
return Post{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
doc, err := gq.NewDocumentFromReader(resp.Body)
|
||||
if err != nil {
|
||||
return Post{}, err
|
||||
}
|
||||
|
||||
// Selector: #js-post-22058462
|
||||
article := doc.Find(fmt.Sprintf("article#js-post-%d", postID))
|
||||
|
||||
body := article.Find("div.message-content article.message-body")
|
||||
if body.Length() == 0 {
|
||||
return Post{}, errors.New("Failed to parse post message body.")
|
||||
}
|
||||
|
||||
bh, err := body.Html()
|
||||
if err != nil {
|
||||
return Post{}, err
|
||||
}
|
||||
r := strings.NewReader(bh)
|
||||
|
||||
author, err := parsePostAuthor(article)
|
||||
if err != nil {
|
||||
return Post{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
post := Post{
|
||||
Author: author,
|
||||
Body: r,
|
||||
}
|
||||
|
||||
return post, nil
|
||||
return strings.NewReader(postHTML), nil
|
||||
}
|
||||
|
||||
+2
-27
@@ -21,7 +21,7 @@ func TestGetPage(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
|
||||
t.Log("Getting homepage\n")
|
||||
resp, err := kf.GetPage(ctx, kf.domain)
|
||||
resp, err := kf.Get(ctx, kf.domain)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
@@ -58,7 +58,7 @@ func TestGetPost(t *testing.T) {
|
||||
}
|
||||
|
||||
func newTestKF() (*KF, error) {
|
||||
host := os.Getenv("KF_HOST")
|
||||
host := os.Getenv("TEST_HOST")
|
||||
if host == "" {
|
||||
host = _TEST_HOST
|
||||
}
|
||||
@@ -85,28 +85,3 @@ func newTestKF() (*KF, error) {
|
||||
|
||||
return NewKF(hc, u)
|
||||
}
|
||||
|
||||
func TestLogin(t *testing.T) {
|
||||
user, pass := os.Getenv("TEST_USER"), os.Getenv("TEST_PASS")
|
||||
if user == "" || pass == "" {
|
||||
t.Log("TEST_USER and/or TEST_PASS empty. Skipping.\n")
|
||||
return
|
||||
}
|
||||
|
||||
kf, err := newTestKF()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
ctx := t.Context()
|
||||
|
||||
resp, err := kf.Login(ctx, user, pass)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
t.Logf("Login response: %+v\n", resp)
|
||||
}
|
||||
|
||||
@@ -1,16 +1,33 @@
|
||||
package libkiwi
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
|
||||
"gitgud.io/yats/cerberus"
|
||||
)
|
||||
|
||||
func (kf *KF) solveKiwiFlare(ctx context.Context) error {
|
||||
c, err := cerberus.NewChallenge(ctx, kf.client, kf.domain.String())
|
||||
func (kf *KF) solveKiwiFlare(ctx context.Context, page io.Reader) error {
|
||||
var (
|
||||
c cerberus.Challenge
|
||||
err error
|
||||
|
||||
host = fmt.Sprintf("https://%s/", kf.domain.Hostname())
|
||||
)
|
||||
|
||||
switch {
|
||||
case page != nil:
|
||||
c, err = cerberus.ParseChallenge(page, host)
|
||||
default:
|
||||
c, err = cerberus.NewChallenge(ctx, kf.client, host)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s, err := cerberus.Solve(ctx, c)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -22,3 +39,11 @@ func (kf *KF) solveKiwiFlare(ctx context.Context) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (kf *KF) urlFromPath(path string) *url.URL {
|
||||
return &url.URL{
|
||||
Scheme: "https",
|
||||
Host: kf.domain.Hostname(),
|
||||
Path: path,
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user