diff --git a/auth.go b/auth.go new file mode 100644 index 0000000..1230740 --- /dev/null +++ b/auth.go @@ -0,0 +1,232 @@ +package libkiwi + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + + "golang.org/x/net/html" + "golang.org/x/net/html/atom" +) + +type LoginResp struct { + Status string `json:"status"` + Message string `json:"message"` + Redirect *url.URL `json:"redirect"` + Visitor struct { + ConversationsUnread uint32 `json:"conversations_unread"` + AlertsUnviewed uint32 `json:"alerts_unviewed"` + TotalUnread uint32 `json:"total_unread"` + } `json:"visitor"` +} + +func (lr *LoginResp) UnmarshalJSON(b []byte) error { + var jsonMap map[string]any + err := json.Unmarshal(b, &jsonMap) + if err != nil { + return err + } + + for k, v := range jsonMap { + switch k { + case "status": + lr.Status = v.(string) + case "message": + lr.Message = v.(string) + case "redirect": + u, err := url.Parse(v.(string)) + if err != nil { + return err + } + lr.Redirect = u + case "visitor": + for k, v := range v.(map[string]any) { + switch k { + case "conversations_unread": + convos, err := strconv.Atoi(v.(string)) + if err != nil { + return err + } + lr.Visitor.ConversationsUnread = uint32(convos) + case "alerts_unviewed": + alerts, err := strconv.Atoi(v.(string)) + if err != nil { + return err + } + lr.Visitor.AlertsUnviewed = uint32(alerts) + case "total_unread": + unread, err := strconv.Atoi(v.(string)) + if err != nil { + return err + } + lr.Visitor.ConversationsUnread = uint32(unread) + } + } + } + } + + return nil +} + +var ErrNoXFToken = errors.New("Failed to locate xfToken on page.") + +func xfToken(page io.Reader) (string, error) { + z := html.NewTokenizer(page) + for i := z.Next(); i != html.ErrorToken; i = z.Next() { + tk := z.Token() + if tk.DataAtom == atom.Html { + for _, a := range tk.Attr { + switch a.Key { + case "data-csrf": + return a.Val, nil + } + } + } + } + + return "", ErrNoXFToken +} + +func loginURL(host *url.URL) *url.URL { + return &url.URL{ + Scheme: "https", + Host: host.Hostname(), + Path: "login", + } +} + +func (kf *KF) newLoginRequest(ctx context.Context, user string, pass string) (*http.Request, error) { + u := loginURL(kf.domain) + + resp, err := kf.GetPage(ctx, u) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + xfToken, err := xfToken(resp.Body) + if err != nil { + return nil, err + } + + form := url.Values{ + "_xfToken": {xfToken}, + "login": {user}, + "password": {pass}, + "remember": {"1"}, + } + + // i.e. https://kiwifarms.net/login/login + postURL := fmt.Sprintf("%s/login", u.String()) + req, err := http.NewRequestWithContext(ctx, "POST", postURL, strings.NewReader(form.Encode())) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + return req, nil +} + +func (kf *KF) Login(ctx context.Context, user string, pass string) (*http.Response, error) { + req, err := kf.newLoginRequest(ctx, user, pass) + if err != nil { + return nil, err + } + + resp, err := kf.client.Do(req) + if err != nil { + return nil, err + } + + if resp.StatusCode != 200 { + // TODO + } + + return resp, nil +} + +func TwoFactorForm(xfToken string, code uint32, u *url.URL) url.Values { + return url.Values{ + "code": {fmt.Sprint(code)}, + "provider": {"totp"}, + "trust": {"1"}, + "confirm": {"1"}, + "remember": {"1"}, + "_xfRedirect": {fmt.Sprintf("https://%s/", u.Hostname())}, + "_xfResponseType": {"json"}, + "_xfToken": {xfToken}, + "_xfWithData": {"1"}, + "_xfRequestUri": {u.RequestURI()}, + } +} + +func (kf *KF) twoFactorAuthForm(ctx context.Context, resp *http.Response, code uint32) (*http.Request, error) { + u := resp.Request.URL + if !strings.Contains(u.Path, "two-step") { + // TODO: move err to proper type with url included. + return nil, errors.New("Did not redirect to two-step page.") + } + + xfToken, err := xfToken(resp.Body) + if err != nil { + return nil, err + } + + form := TwoFactorForm(xfToken, code, u) + reqURL := fmt.Sprintf("https://%s/login/two-step", u.Hostname()) + + req, err := http.NewRequestWithContext(ctx, "POST", reqURL, strings.NewReader(form.Encode())) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + return req, nil +} + +func (kf *KF) TwoFactorAuth(ctx context.Context, resp *http.Response, code uint32) (*http.Response, error) { + req, err := kf.twoFactorAuthForm(ctx, resp, code) + if err != nil { + return nil, err + } + + return kf.client.Do(req) +} + +func setCookie(u *url.URL, jar http.CookieJar, newCookie *http.Cookie) { + cookies := jar.Cookies(u) + + for i, c := range cookies { + if c.Name == newCookie.Name { + cookies[i] = newCookie + jar.SetCookies(u, cookies) + return + } + } + + // Append if not already existing. + cookies = append(cookies, newCookie) + jar.SetCookies(u, cookies) +} + +func (kf *KF) RefreshSession(ctx context.Context) error { + // Clear any existing session token to request a new one. + setCookie(kf.domain, kf.client.Jar, &http.Cookie{ + Name: "xf_session", + Value: "", + }) + + resp, err := kf.GetPage(ctx, kf.domain) + if err != nil { + return err + } + resp.Body.Close() + + return nil +} diff --git a/go.mod b/go.mod index 7666816..29d3b21 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,16 @@ module gitgud.io/yats/libkiwi -go 1.25.6 - -require gitgud.io/yats/cerberus v0.0.0-20260214165307-66e6f74a4be9 +go 1.26.1 require ( + gitgud.io/yats/cerberus v0.0.0-20260214165307-66e6f74a4be9 + github.com/PuerkitoBio/goquery v1.12.0 + golang.org/x/net v0.55.0 +) + +require ( + github.com/andybalholm/cascadia v1.3.3 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect github.com/minio/sha256-simd v1.0.1 // indirect - golang.org/x/net v0.49.0 // indirect - golang.org/x/sys v0.40.0 // indirect + golang.org/x/sys v0.45.0 // indirect ) diff --git a/go.sum b/go.sum index 4fef71d..ba960c8 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,80 @@ gitgud.io/yats/cerberus v0.0.0-20260214165307-66e6f74a4be9 h1:OSYrnxTeCuvaX6O8/AHUE4Xndb76vtcVwdvdLtGfp4Q= gitgud.io/yats/cerberus v0.0.0-20260214165307-66e6f74a4be9/go.mod h1:WVfXXYUHR8x5hX0cpRUOlaeRqxR/9JxYhLbjFSb/jjc= +github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo= +github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ= +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= -golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/libkiwi.go b/libkiwi.go index ac9a547..1ced1f6 100644 --- a/libkiwi.go +++ b/libkiwi.go @@ -4,60 +4,53 @@ import ( "context" "errors" "fmt" + "io" "net/http" "net/http/cookiejar" "net/url" - "regexp" + "strconv" "strings" - "gitgud.io/yats/cerberus" + gq "github.com/PuerkitoBio/goquery" ) type KF struct { - Client http.Client + client http.Client domain *url.URL } // Supply your own http.Client to route through any proxies. -func NewKF(hc http.Client, host string, cookies string) (KF, error) { - u, err := parseHost(host) +func NewKF(hc http.Client, host *url.URL) (*KF, error) { + u, err := url.Parse(fmt.Sprintf("https://%s", host.Hostname())) if err != nil { - return KF{}, err + return nil, err } - if hc.Jar == nil { - jar, err := cookiejar.New(nil) - if err != nil { - return KF{}, err - } - hc.Jar = jar + jar, err := cookiejar.New(nil) + if err != nil { + return nil, err } - if cookies != "" { - cs, err := parseCookieString(u, cookies) - if err != nil { - return KF{}, err - } + hc.Jar = jar - hc.Jar.SetCookies(u, cs) + kf := &KF{ + client: hc, + domain: u, } // Update host url in case we get redirected across domains. hc.CheckRedirect = func(req *http.Request, via []*http.Request) error { - hn := req.URL.Hostname() - if hn != u.Hostname() { + reqHost := req.URL.Hostname() + if reqHost != u.Hostname() { // Deliberately set to Hostname() and not Host. // This excludes any extra shit like ports. - u.Host = hn + u.Host = reqHost } return nil } - return KF{ - Client: hc, - domain: u, - }, nil + return kf, nil } func (kf *KF) GetPage(ctx context.Context, u *url.URL) (*http.Response, error) { @@ -65,24 +58,20 @@ func (kf *KF) GetPage(ctx context.Context, u *url.URL) (*http.Response, error) { if err != nil { return nil, err } + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0") - resp, err := kf.Client.Do(req) + resp, err := kf.client.Do(req) if err != nil { return nil, err } - hn := resp.Request.URL.Hostname() - if hn != kf.domain.Hostname() { - jar := kf.Client.Jar - jar.SetCookies(resp.Request.URL, jar.Cookies(kf.domain)) - kf.domain.Host = hn - } // KiwiFlare redirect is signaled by 203 status. if resp.StatusCode == 203 { - err = kf.SolveKiwiFlare(ctx) + err = kf.solveKiwiFlare(ctx) if err != nil { return nil, err } + // Try fetching the page again now that we're authed. return kf.GetPage(ctx, u) } @@ -90,90 +79,102 @@ func (kf *KF) GetPage(ctx context.Context, u *url.URL) (*http.Response, error) { return resp, nil } -func parseCookieString(u *url.URL, cookies string) ([]*http.Cookie, error) { - sp := strings.Split(cookies, "; ") - cs := make([]*http.Cookie, len(sp)) - - for i, c := range sp { - kv := strings.Split(c, "=") - if len(kv) != 2 { - return nil, errors.New("Invalid cookie string: " + cookies) - } - cs[i] = &http.Cookie{ - Domain: u.Hostname(), - Name: kv[0], - Path: "/", - Value: kv[1], - } - } - - return cs, nil +type User struct { + ID uint32 + Name string + Title string + URL *url.URL } -func setCookie(jar http.CookieJar, u *url.URL, cookie *http.Cookie) { - cookies := jar.Cookies(u) +func parsePostAuthor(article *gq.Selection) (User, error) { + userBlock := article.Find("section.message-user") - for i, c := range cookies { - if c.Name == cookie.Name { - cookies[i] = cookie - // Store changes and stop looping since it's found. - jar.SetCookies(u, cookies) - return - } + idStr, ok := userBlock.Attr("data-user-id") + if !ok { + // TODO: Proper error types. + return User{}, errors.New("Failed to parse post author attr.") } - // Append new cookie if existing one not found. - cookies = append(cookies, cookie) - jar.SetCookies(u, cookies) -} - -func (kf *KF) CookieString() string { - var b strings.Builder - for _, c := range kf.Client.Jar.Cookies(kf.domain) { - fmt.Fprintf(&b, "%s: %s; ", c.Name, c.Value) - } - return strings.TrimSuffix(b.String(), "; ") -} - -func (kf *KF) RefreshSession(ctx context.Context) (string, error) { - // Clear any existing session token to request a new one. - setCookie(kf.Client.Jar, kf.domain, &http.Cookie{ - Name: "xf_session", - Value: "", - }) - - resp, err := kf.GetPage(ctx, kf.domain) + id, err := strconv.Atoi(idStr) if err != nil { - return "", err + return User{}, err + } + + name, ok := article.Attr("data-author") + if !ok { + return User{}, errors.New("Failed to parse post author attr.") + } + + title := userBlock.Find(".message-userTitle").Text() + + urlStr, ok := userBlock.Attr("itemid") + if !ok { + return User{}, errors.New("Failed to parse post author attr.") + } + + u, err := url.Parse(urlStr) + if err != nil { + return User{}, err + } + + user := User{ + ID: uint32(id), + Name: name, + Title: title, + URL: u, + } + + return user, nil +} + +type Post struct { + Author User + Body io.Reader +} + +func (kf *KF) GetPost(ctx context.Context, postID uint32) (Post, error) { + // Example post goto link: https://kiwifarms.st/goto/post?id=22058462 + gtl := fmt.Sprintf("%s/goto/post?id=%d", kf.domain.String(), postID) + u, err := url.Parse(gtl) + if err != nil { + return Post{}, err + } + + resp, err := kf.GetPage(ctx, u) + if err != nil { + return Post{}, err } defer resp.Body.Close() - return regexp.MustCompile(`xf_session=([^;]*)`).FindString(resp.Header.Get("Set-Cookie")), nil -} + doc, err := gq.NewDocumentFromReader(resp.Body) + if err != nil { + return Post{}, err + } -func (kf *KF) SolveKiwiFlare(ctx context.Context) error { - c, err := cerberus.NewChallenge(ctx, kf.Client, kf.domain.String()) - if err != nil { - return err + // Selector: #js-post-22058462 + article := doc.Find(fmt.Sprintf("article#js-post-%d", postID)) + + body := article.Find("div.message-content article.message-body") + if body.Length() == 0 { + return Post{}, errors.New("Failed to parse post message body.") } - s, err := cerberus.Solve(ctx, c) + + bh, err := body.Html() if err != nil { - return err + return Post{}, err } - resp, err := cerberus.Submit(ctx, kf.Client, s, "") + r := strings.NewReader(bh) + + author, err := parsePostAuthor(article) if err != nil { - return err + return Post{}, err } defer resp.Body.Close() - return nil -} - -func parseHost(host string) (*url.URL, error) { - // Try prepending protocol if it seems to be missing. - if !strings.Contains(host, "://") { - host = "https://" + host + post := Post{ + Author: author, + Body: r, } - return url.Parse(host) + return post, nil } diff --git a/libkiwi_test.go b/libkiwi_test.go index c66131c..cebf9cb 100644 --- a/libkiwi_test.go +++ b/libkiwi_test.go @@ -1,52 +1,112 @@ package libkiwi import ( - "log" "net/http" + "net/http/cookiejar" + "net/url" "os" "testing" + + "golang.org/x/net/proxy" ) -const TEST_HOST = "kiwifarms.st" +const _TEST_HOST = "kiwifarms.st" func TestGetPage(t *testing.T) { - cookies := os.Getenv("TEST_COOKIES") - kf, err := NewKF(http.Client{}, TEST_HOST, cookies) + kf, err := newTestKF() if err != nil { t.Error(err) } - log.Println("Getting homepage") - resp, err := kf.GetPage(t.Context(), kf.domain) + ctx := t.Context() + + t.Log("Getting homepage\n") + resp, err := kf.GetPage(ctx, kf.domain) if err != nil { t.Error(err) return } defer resp.Body.Close() - log.Printf("Response status code: %d\n\n", resp.StatusCode) + t.Logf("Response status code: %d\n\n", resp.StatusCode) for k, v := range resp.Header { if len(v) > 0 { - log.Printf("%s: %s\n", k, v[0]) + t.Logf("%s: %s\n", k, v[0]) } } - log.Printf("Response host: %s\n\n", kf.domain) - log.Printf("Cookies: %+v\n", resp.Cookies()) + t.Logf("Response host: %s\n\n", kf.domain) } -func TestRefreshSession(t *testing.T) { - cookies := os.Getenv("TEST_COOKIES") - kf, err := NewKF(http.Client{}, TEST_HOST, cookies) - if err != nil { - t.Error(err) - } - - log.Println("Refreshing xf_session") - tk, err := kf.RefreshSession(t.Context()) +func TestGetPost(t *testing.T) { + kf, err := newTestKF() if err != nil { t.Error(err) return } - log.Println("New xf_session token: " + tk) + const POST_ID = 22058462 + ctx := t.Context() + + t.Logf("Getting post #%d\n", POST_ID) + post, err := kf.GetPost(ctx, POST_ID) + if err != nil { + t.Error(err) + return + } + + t.Logf("Post author: %+v\n", post.Author) +} + +func newTestKF() (*KF, error) { + host := os.Getenv("KF_HOST") + if host == "" { + host = _TEST_HOST + } + + u, err := url.Parse("https://" + host) + if err != nil { + return nil, err + } + + p := proxy.FromEnvironment() + + tr := http.DefaultTransport.(*http.Transport).Clone() + tr.DialContext = p.(proxy.ContextDialer).DialContext + + jar, err := cookiejar.New(nil) + if err != nil { + return nil, err + } + + hc := http.Client{ + Jar: jar, + Transport: tr, + } + + return NewKF(hc, u) +} + +func TestLogin(t *testing.T) { + user, pass := os.Getenv("TEST_USER"), os.Getenv("TEST_PASS") + if user == "" || pass == "" { + t.Log("TEST_USER and/or TEST_PASS empty. Skipping.\n") + return + } + + kf, err := newTestKF() + if err != nil { + t.Error(err) + return + } + + ctx := t.Context() + + resp, err := kf.Login(ctx, user, pass) + if err != nil { + t.Error(err) + return + } + defer resp.Body.Close() + + t.Logf("Login response: %+v\n", resp) } diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..0dc4afc --- /dev/null +++ b/utils.go @@ -0,0 +1,24 @@ +package libkiwi + +import ( + "context" + + "gitgud.io/yats/cerberus" +) + +func (kf *KF) solveKiwiFlare(ctx context.Context) error { + c, err := cerberus.NewChallenge(ctx, kf.client, kf.domain.String()) + if err != nil { + return err + } + s, err := cerberus.Solve(ctx, c) + if err != nil { + return err + } + _, err = cerberus.Submit(ctx, kf.client, s, "/") + if err != nil { + return err + } + + return nil +}