From 75e9ecd9dcbfb9ed24e69a48233ad55ad8ac7f8c Mon Sep 17 00:00:00 2001 From: y a t s <140337963+y-a-t-s@users.noreply.github.com> Date: Mon, 26 Aug 2024 09:56:44 -0400 Subject: [PATCH] Initial commit. --- .gitignore | 13 +++++ README.md | 3 ++ UNLICENSE | 24 +++++++++ go.mod | 7 +++ go.sum | 4 ++ jar.go | 128 ++++++++++++++++++++++++++++++++++++++++++++++++ libkiwi.go | 104 +++++++++++++++++++++++++++++++++++++++ libkiwi_test.go | 45 +++++++++++++++++ utils.go | 44 +++++++++++++++++ 9 files changed, 372 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 UNLICENSE create mode 100644 go.mod create mode 100644 go.sum create mode 100644 jar.go create mode 100644 libkiwi.go create mode 100644 libkiwi_test.go create mode 100644 utils.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5932476 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +.* +!.gitignore + +*.json +*.log + +build/ +data-dir-*/ +tmp/ + +GPATH +GRTAGS +GTAGS diff --git a/README.md b/README.md new file mode 100644 index 0000000..3518867 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# libkiwi + +A WIP library for interacting with the KF site. diff --git a/UNLICENSE b/UNLICENSE new file mode 100644 index 0000000..68a49da --- /dev/null +++ b/UNLICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..82d955d --- /dev/null +++ b/go.mod @@ -0,0 +1,7 @@ +module github.com/y-a-t-s/libkiwi + +go 1.23.0 + +require github.com/y-a-t-s/firebird v0.0.0-20240702142028-fc2f2361cb95 + +require golang.org/x/net v0.20.0 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..9f90a1d --- /dev/null +++ b/go.sum @@ -0,0 +1,4 @@ +github.com/y-a-t-s/firebird v0.0.0-20240702142028-fc2f2361cb95 h1:fEW4GHrwit5YdsY6GhNYjmHSHcS1fKvs1DwLvD4Vd4s= +github.com/y-a-t-s/firebird v0.0.0-20240702142028-fc2f2361cb95/go.mod h1:8S1hx2sI8odLiO0KNDpGLzH5MePuTco7u8q29wlFzGQ= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= diff --git a/jar.go b/jar.go new file mode 100644 index 0000000..7b88fc4 --- /dev/null +++ b/jar.go @@ -0,0 +1,128 @@ +package libkiwi + +import ( + "net/http" + "net/url" + "sync" +) + +type cookieMap map[string]map[string]*http.Cookie + +type KiwiJar struct { + cookieMap + mutex *sync.Mutex +} + +func newCookieMap() cookieMap { + return make(cookieMap, 2) +} + +func NewKiwiJar(domain *url.URL, cookies string) (kj *KiwiJar, err error) { + kj = &KiwiJar{ + cookieMap: newCookieMap(), + mutex: &sync.Mutex{}, + } + kj.newDomain(domain) + + if cookies == "" { + return + } + + cs, err := parseCookieString(cookies) + if err != nil { + return + } + kj.SetCookies(domain, cs) + + return +} + +func (kj *KiwiJar) checkAlloc(u *url.URL) { + if kj.cookieMap == nil || kj.cookieMap[u.Host] == nil { + kj.newDomain(u) + } +} + +func (kj *KiwiJar) SetCookies(u *url.URL, cookies []*http.Cookie) { + kj.checkAlloc(u) + + for _, c := range cookies { + kj.SetCookie(u, c) + } +} + +func (kj *KiwiJar) Cookies(u *url.URL) []*http.Cookie { + kj.checkAlloc(u) + + res := make(chan []*http.Cookie, 1) + + go func() { + kj.mutex.Lock() + defer kj.mutex.Unlock() + + cl := len(kj.cookieMap[u.Host]) + cs := make([]*http.Cookie, cl) + i := 0 + for _, c := range kj.cookieMap[u.Host] { + if i >= cl { + break + } + + cs[i] = c + i++ + } + + res <- cs + }() + + return <-res +} + +func (kj *KiwiJar) GetCookie(u *url.URL, name string) *http.Cookie { + kj.checkAlloc(u) + + res := make(chan *http.Cookie, 1) + + go func() { + kj.mutex.Lock() + defer kj.mutex.Unlock() + + res <- kj.cookieMap[u.Host][name] + }() + + return <-res +} + +func (kj *KiwiJar) SetCookie(u *url.URL, cookie *http.Cookie) { + kj.checkAlloc(u) + + done := make(chan bool, 1) + + go func() { + kj.mutex.Lock() + defer kj.mutex.Unlock() + + kj.cookieMap[u.Host][cookie.Name] = cookie + done <- true + }() + + <-done +} + +func (kj *KiwiJar) newDomain(domain *url.URL) { + if kj.cookieMap == nil { + kj.cookieMap = newCookieMap() + } + + done := make(chan bool, 1) + + go func() { + kj.mutex.Lock() + defer kj.mutex.Unlock() + + kj.cookieMap[domain.Host] = make(map[string]*http.Cookie, 16) + done <- true + }() + + <-done +} diff --git a/libkiwi.go b/libkiwi.go new file mode 100644 index 0000000..44308c8 --- /dev/null +++ b/libkiwi.go @@ -0,0 +1,104 @@ +package libkiwi + +import ( + "errors" + "net/http" + "net/url" + "regexp" + + "github.com/y-a-t-s/firebird" +) + +type KF struct { + client http.Client + domain *url.URL +} + +// Supply your own http.Client to route through any proxies. +func NewKF(hc http.Client, host string, cookies string) (kf *KF, err error) { + _, host, err = splitProtocol(host) + if err != nil { + return + } + u, err := url.Parse("https://" + host) + if err != nil { + return + } + + jar, err := NewKiwiJar(u, cookies) + if err != nil { + return + } + hc.Jar = jar + + kf = &KF{ + client: hc, + domain: u, + } + + return +} + +func (kf *KF) GetPage(u *url.URL) (resp *http.Response, err error) { + if u == nil { + err = errors.New("Received nil URL.") + return + } + + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return + } + // req.Header.Set("Cookie", kf.cookies) + + resp, err = kf.client.Do(req) + if err != nil { + return + } + + // KiwiFlare redirect is signaled by 203 status. + if resp.StatusCode == 203 { + err = kf.solveKiwiFlare() + if err != nil { + return + } + // Try fetching the page again now that we're authed. + return kf.GetPage(u) + } + + return +} + +func (kf *KF) RefreshSession() (tk string, err error) { + // Clear any existing session token to request a new one. + kf.client.Jar.(*KiwiJar).SetCookie(kf.domain, &http.Cookie{ + Name: "xf_session", + Value: "", + }) + + resp, err := kf.GetPage(kf.domain) + if err != nil { + return + } + defer resp.Body.Close() + + tk = regexp.MustCompile(`xf_session=([^;]*)`).FindString(resp.Header.Get("Set-Cookie")) + return +} + +func (kf *KF) solveKiwiFlare() error { + c, err := firebird.NewChallenge(kf.client) + if err != nil { + return err + } + s, err := firebird.Solve(c) + if err != nil { + return err + } + _, err = firebird.Submit(kf.client, s) + if err != nil { + return err + } + + return nil +} diff --git a/libkiwi_test.go b/libkiwi_test.go new file mode 100644 index 0000000..0502a31 --- /dev/null +++ b/libkiwi_test.go @@ -0,0 +1,45 @@ +package libkiwi + +import ( + "log" + "net/http" + "os" + "testing" +) + +const HOST string = "kiwifarms.st" + +func TestGetPage(t *testing.T) { + cookies := os.Getenv("TEST_COOKIES") + kf, err := NewKF(http.Client{}, HOST, cookies) + if err != nil { + t.Error(err) + } + log.Println("Getting homepage") + resp, err := kf.GetPage(kf.domain) + if err != nil { + t.Error(err) + } + defer resp.Body.Close() + + log.Printf("Response status code: %d\n", resp.StatusCode) + for k, v := range resp.Header { + if len(v) > 0 { + log.Printf("%s: %s\n", k, v[0]) + } + } +} + +func TestRefreshSession(t *testing.T) { + cookies := os.Getenv("TEST_COOKIES") + kf, err := NewKF(http.Client{}, HOST, cookies) + if err != nil { + t.Error(err) + } + log.Println("Refreshing xf_session") + tk, err := kf.RefreshSession() + if err != nil { + t.Error(err) + } + log.Println("New xf_session token: " + tk) +} diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..0f13375 --- /dev/null +++ b/utils.go @@ -0,0 +1,44 @@ +package libkiwi + +import ( + "errors" + "net/http" + "regexp" + "strings" +) + +func parseCookieString(cookies string) ([]*http.Cookie, error) { + sp := strings.Split(cookies, "; ") + cs := make([]*http.Cookie, len(sp)) + + for i, c := range sp { + kv := strings.Split(c, "=") + if len(kv) != 2 { + return nil, errors.New("Invalid cookie string: " + cookies) + } + cs[i] = &http.Cookie{ + Name: kv[0], + Value: kv[1], + } + } + + return cs, nil +} + +func splitProtocol(addr string) (proto string, host string, err error) { + // FindStringSubmatch is used to capture the groups. + // Index 0 is the full matching string with all groups. + // The rest are numbered by the order of the opening parens. + // Here, we want the last 2 groups (indexes 1 and 2, requiring length 3). + tmp := regexp.MustCompile(`^([\w-]+://)?([^/]+)`).FindStringSubmatch(addr) + // At the very least, we need the hostname part (index 2). + if len(tmp) < 3 || tmp[2] == "" { + err = errors.New("Failed to parse address: " + addr) + return + } + + proto = tmp[1] + host = tmp[2] + + return +}