commit 75e9ecd9dcbfb9ed24e69a48233ad55ad8ac7f8c Author: y a t s <140337963+y-a-t-s@users.noreply.github.com> Date: Mon Aug 26 09:56:44 2024 -0400 Initial commit. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5932476 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +.* +!.gitignore + +*.json +*.log + +build/ +data-dir-*/ +tmp/ + +GPATH +GRTAGS +GTAGS diff --git a/README.md b/README.md new file mode 100644 index 0000000..3518867 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# libkiwi + +A WIP library for interacting with the KF site. diff --git a/UNLICENSE b/UNLICENSE new file mode 100644 index 0000000..68a49da --- /dev/null +++ b/UNLICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..82d955d --- /dev/null +++ b/go.mod @@ -0,0 +1,7 @@ +module github.com/y-a-t-s/libkiwi + +go 1.23.0 + +require github.com/y-a-t-s/firebird v0.0.0-20240702142028-fc2f2361cb95 + +require golang.org/x/net v0.20.0 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..9f90a1d --- /dev/null +++ b/go.sum @@ -0,0 +1,4 @@ +github.com/y-a-t-s/firebird v0.0.0-20240702142028-fc2f2361cb95 h1:fEW4GHrwit5YdsY6GhNYjmHSHcS1fKvs1DwLvD4Vd4s= +github.com/y-a-t-s/firebird v0.0.0-20240702142028-fc2f2361cb95/go.mod h1:8S1hx2sI8odLiO0KNDpGLzH5MePuTco7u8q29wlFzGQ= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= diff --git a/jar.go b/jar.go new file mode 100644 index 0000000..7b88fc4 --- /dev/null +++ b/jar.go @@ -0,0 +1,128 @@ +package libkiwi + +import ( + "net/http" + "net/url" + "sync" +) + +type cookieMap map[string]map[string]*http.Cookie + +type KiwiJar struct { + cookieMap + mutex *sync.Mutex +} + +func newCookieMap() cookieMap { + return make(cookieMap, 2) +} + +func NewKiwiJar(domain *url.URL, cookies string) (kj *KiwiJar, err error) { + kj = &KiwiJar{ + cookieMap: newCookieMap(), + mutex: &sync.Mutex{}, + } + kj.newDomain(domain) + + if cookies == "" { + return + } + + cs, err := parseCookieString(cookies) + if err != nil { + return + } + kj.SetCookies(domain, cs) + + return +} + +func (kj *KiwiJar) checkAlloc(u *url.URL) { + if kj.cookieMap == nil || kj.cookieMap[u.Host] == nil { + kj.newDomain(u) + } +} + +func (kj *KiwiJar) SetCookies(u *url.URL, cookies []*http.Cookie) { + kj.checkAlloc(u) + + for _, c := range cookies { + kj.SetCookie(u, c) + } +} + +func (kj *KiwiJar) Cookies(u *url.URL) []*http.Cookie { + kj.checkAlloc(u) + + res := make(chan []*http.Cookie, 1) + + go func() { + kj.mutex.Lock() + defer kj.mutex.Unlock() + + cl := len(kj.cookieMap[u.Host]) + cs := make([]*http.Cookie, cl) + i := 0 + for _, c := range kj.cookieMap[u.Host] { + if i >= cl { + break + } + + cs[i] = c + i++ + } + + res <- cs + }() + + return <-res +} + +func (kj *KiwiJar) GetCookie(u *url.URL, name string) *http.Cookie { + kj.checkAlloc(u) + + res := make(chan *http.Cookie, 1) + + go func() { + kj.mutex.Lock() + defer kj.mutex.Unlock() + + res <- kj.cookieMap[u.Host][name] + }() + + return <-res +} + +func (kj *KiwiJar) SetCookie(u *url.URL, cookie *http.Cookie) { + kj.checkAlloc(u) + + done := make(chan bool, 1) + + go func() { + kj.mutex.Lock() + defer kj.mutex.Unlock() + + kj.cookieMap[u.Host][cookie.Name] = cookie + done <- true + }() + + <-done +} + +func (kj *KiwiJar) newDomain(domain *url.URL) { + if kj.cookieMap == nil { + kj.cookieMap = newCookieMap() + } + + done := make(chan bool, 1) + + go func() { + kj.mutex.Lock() + defer kj.mutex.Unlock() + + kj.cookieMap[domain.Host] = make(map[string]*http.Cookie, 16) + done <- true + }() + + <-done +} diff --git a/libkiwi.go b/libkiwi.go new file mode 100644 index 0000000..44308c8 --- /dev/null +++ b/libkiwi.go @@ -0,0 +1,104 @@ +package libkiwi + +import ( + "errors" + "net/http" + "net/url" + "regexp" + + "github.com/y-a-t-s/firebird" +) + +type KF struct { + client http.Client + domain *url.URL +} + +// Supply your own http.Client to route through any proxies. +func NewKF(hc http.Client, host string, cookies string) (kf *KF, err error) { + _, host, err = splitProtocol(host) + if err != nil { + return + } + u, err := url.Parse("https://" + host) + if err != nil { + return + } + + jar, err := NewKiwiJar(u, cookies) + if err != nil { + return + } + hc.Jar = jar + + kf = &KF{ + client: hc, + domain: u, + } + + return +} + +func (kf *KF) GetPage(u *url.URL) (resp *http.Response, err error) { + if u == nil { + err = errors.New("Received nil URL.") + return + } + + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return + } + // req.Header.Set("Cookie", kf.cookies) + + resp, err = kf.client.Do(req) + if err != nil { + return + } + + // KiwiFlare redirect is signaled by 203 status. + if resp.StatusCode == 203 { + err = kf.solveKiwiFlare() + if err != nil { + return + } + // Try fetching the page again now that we're authed. + return kf.GetPage(u) + } + + return +} + +func (kf *KF) RefreshSession() (tk string, err error) { + // Clear any existing session token to request a new one. + kf.client.Jar.(*KiwiJar).SetCookie(kf.domain, &http.Cookie{ + Name: "xf_session", + Value: "", + }) + + resp, err := kf.GetPage(kf.domain) + if err != nil { + return + } + defer resp.Body.Close() + + tk = regexp.MustCompile(`xf_session=([^;]*)`).FindString(resp.Header.Get("Set-Cookie")) + return +} + +func (kf *KF) solveKiwiFlare() error { + c, err := firebird.NewChallenge(kf.client) + if err != nil { + return err + } + s, err := firebird.Solve(c) + if err != nil { + return err + } + _, err = firebird.Submit(kf.client, s) + if err != nil { + return err + } + + return nil +} diff --git a/libkiwi_test.go b/libkiwi_test.go new file mode 100644 index 0000000..0502a31 --- /dev/null +++ b/libkiwi_test.go @@ -0,0 +1,45 @@ +package libkiwi + +import ( + "log" + "net/http" + "os" + "testing" +) + +const HOST string = "kiwifarms.st" + +func TestGetPage(t *testing.T) { + cookies := os.Getenv("TEST_COOKIES") + kf, err := NewKF(http.Client{}, HOST, cookies) + if err != nil { + t.Error(err) + } + log.Println("Getting homepage") + resp, err := kf.GetPage(kf.domain) + if err != nil { + t.Error(err) + } + defer resp.Body.Close() + + log.Printf("Response status code: %d\n", resp.StatusCode) + for k, v := range resp.Header { + if len(v) > 0 { + log.Printf("%s: %s\n", k, v[0]) + } + } +} + +func TestRefreshSession(t *testing.T) { + cookies := os.Getenv("TEST_COOKIES") + kf, err := NewKF(http.Client{}, HOST, cookies) + if err != nil { + t.Error(err) + } + log.Println("Refreshing xf_session") + tk, err := kf.RefreshSession() + if err != nil { + t.Error(err) + } + log.Println("New xf_session token: " + tk) +} diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..0f13375 --- /dev/null +++ b/utils.go @@ -0,0 +1,44 @@ +package libkiwi + +import ( + "errors" + "net/http" + "regexp" + "strings" +) + +func parseCookieString(cookies string) ([]*http.Cookie, error) { + sp := strings.Split(cookies, "; ") + cs := make([]*http.Cookie, len(sp)) + + for i, c := range sp { + kv := strings.Split(c, "=") + if len(kv) != 2 { + return nil, errors.New("Invalid cookie string: " + cookies) + } + cs[i] = &http.Cookie{ + Name: kv[0], + Value: kv[1], + } + } + + return cs, nil +} + +func splitProtocol(addr string) (proto string, host string, err error) { + // FindStringSubmatch is used to capture the groups. + // Index 0 is the full matching string with all groups. + // The rest are numbered by the order of the opening parens. + // Here, we want the last 2 groups (indexes 1 and 2, requiring length 3). + tmp := regexp.MustCompile(`^([\w-]+://)?([^/]+)`).FindStringSubmatch(addr) + // At the very least, we need the hostname part (index 2). + if len(tmp) < 3 || tmp[2] == "" { + err = errors.New("Failed to parse address: " + addr) + return + } + + proto = tmp[1] + host = tmp[2] + + return +}