Removed Puppeteer and logging in by POSTing the logon form now that we can get clearance tokens.

KfChatDotNetBot/ChatBot.cs

@@ -17,7 +17,6 @@ public class ChatBot
 {
     internal readonly ChatClient KfClient;
     private readonly Logger _logger = LogManager.GetCurrentClassLogger();
-    private string _xfSessionToken;
     // Oh no it's an ever expanding list that may never get cleaned up!
     // BUY MORE RAM
     private readonly List<int> _seenMsgIds = [];
@@ -29,35 +28,27 @@ public class ChatBot
     internal bool GambaSeshPresent;
     internal readonly BotServices BotServices;
     private Task _kfChatPing;
+    private KfTokenService _kfTokenService;
     
     public ChatBot()
     {
         _logger.Info("Bot starting!");
 
         var settings = Helpers.GetMultipleValues([
-            BuiltIn.Keys.KiwiFarmsWsEndpoint, BuiltIn.Keys.KiwiFarmsDomain, BuiltIn.Keys.PusherEndpoint,
-            BuiltIn.Keys.Proxy, BuiltIn.Keys.PusherReconnectTimeout, BuiltIn.Keys.PusherChannels,
-            BuiltIn.Keys.TwitchBossmanJackId, BuiltIn.Keys.DiscordToken, BuiltIn.Keys.KiwiFarmsWsReconnectTimeout,
-            BuiltIn.Keys.KiwiFarmsToken, BuiltIn.Keys.KickEnabled
-        ]).Result;
+            BuiltIn.Keys.KiwiFarmsWsEndpoint, BuiltIn.Keys.KiwiFarmsDomain,
+            BuiltIn.Keys.Proxy, BuiltIn.Keys.KiwiFarmsWsReconnectTimeout]).Result;
 
-        _xfSessionToken = settings[BuiltIn.Keys.KiwiFarmsToken].Value ?? "unset";
-        if (_xfSessionToken == "unset")
+        _kfTokenService = new KfTokenService(settings[BuiltIn.Keys.KiwiFarmsDomain].Value!,
+            settings[BuiltIn.Keys.Proxy].Value, _cancellationToken);
+        if (_kfTokenService.GetXfSessionCookie() == null)
         {
             RefreshXfToken().Wait(_cancellationToken);
         }
-
-        // var kiwiflare = new KiwiFlare(settings[BuiltIn.Keys.KiwiFarmsDomain].Value,
-        //     settings[BuiltIn.Keys.Proxy].Value, _cancellationToken);
-        // var challenge = kiwiflare.GetChallenge().Result;
-        // var solution = kiwiflare.SolveChallenge(challenge).Result;
-        // var token = kiwiflare.SubmitAnswer(solution).Result;
-        // var test = kiwiflare.CheckAuth(token).Result;
         
         KfClient = new ChatClient(new ChatClientConfigModel
         {
             WsUri = new Uri(settings[BuiltIn.Keys.KiwiFarmsWsEndpoint].Value ?? throw new InvalidOperationException($"{BuiltIn.Keys.KiwiFarmsWsEndpoint} cannot be null")),
-            XfSessionToken = _xfSessionToken,
+            XfSessionToken = _kfTokenService.GetXfSessionCookie(),
             CookieDomain = settings[BuiltIn.Keys.KiwiFarmsDomain].Value ?? throw new InvalidOperationException($"{BuiltIn.Keys.KiwiFarmsDomain} cannot be null"),
             Proxy = settings[BuiltIn.Keys.Proxy].Value,
             ReconnectTimeout = settings[BuiltIn.Keys.KiwiFarmsWsReconnectTimeout].ToType<int>()
@@ -92,7 +83,9 @@ public class ChatBot
         _logger.Error($"Couldn't join the room. KF returned: {message}");
         _logger.Error("This is likely due to the session cookie expiring. Retrieving a new one.");
         RefreshXfToken().Wait(_cancellationToken);
-        KfClient.UpdateToken(_xfSessionToken);
+        // Shouldn't be null if we've just refreshed the token
+        // It's only null if a logon has never been attempted since the cookie DB entry was created
+        KfClient.UpdateToken(_kfTokenService.GetXfSessionCookie()!);
         _logger.Info("Retrieved fresh token. Reconnecting.");
         KfClient.Disconnect();
         KfClient.StartWsClient().Wait(_cancellationToken);
@@ -122,15 +115,19 @@ public class ChatBot
 
     private async Task RefreshXfToken()
     {
-        var settings = Helpers.GetMultipleValues([BuiltIn.Keys.KiwiFarmsDomain,
-        BuiltIn.Keys.KiwiFarmsUsername, BuiltIn.Keys.KiwiFarmsPassword, BuiltIn.Keys.KiwiFarmsChromiumPath,
-        BuiltIn.Keys.Proxy]).Result;
-        var cookie = await KfTokenService.FetchSessionTokenAsync(settings[BuiltIn.Keys.KiwiFarmsDomain].Value!,
-            settings[BuiltIn.Keys.KiwiFarmsUsername].Value!, settings[BuiltIn.Keys.KiwiFarmsPassword].Value!,
-            settings[BuiltIn.Keys.KiwiFarmsChromiumPath].Value!, settings[BuiltIn.Keys.Proxy].Value);
-        _logger.Debug($"FetchSessionTokenAsync returned {cookie}");
-        _xfSessionToken = cookie;
-        await Helpers.SetValue(BuiltIn.Keys.KiwiFarmsToken, _xfSessionToken);
+        if (await _kfTokenService.IsLoggedIn())
+        {
+            _logger.Info("We were already logged in and should have a fresh cookie for chat now");
+            await _kfTokenService.SaveCookies();
+            return;
+        }
+
+        var settings =
+            await Helpers.GetMultipleValues([BuiltIn.Keys.KiwiFarmsUsername, BuiltIn.Keys.KiwiFarmsPassword]);
+        await _kfTokenService.PerformLogin(settings[BuiltIn.Keys.KiwiFarmsUsername].Value!,
+            settings[BuiltIn.Keys.KiwiFarmsPassword].Value!);
+        await _kfTokenService.SaveCookies();
+        _logger.Info("Successfully logged in");
     }
 
     private void OnKfChatMessage(object sender, List<MessageModel> messages, MessagesJsonModel jsonPayload)

KfChatDotNetBot/KfChatDotNetBot.csproj

@@ -18,7 +18,6 @@
       </PackageReference>
       <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.8" />
       <PackageReference Include="NLog" Version="5.3.3" />
-      <PackageReference Include="PuppeteerSharp" Version="19.0.1" />
       <PackageReference Include="System.Runtime.Caching" Version="8.0.0" />
       <PackageReference Include="System.Text.Json" Version="8.0.4" />
     </ItemGroup>

KfChatDotNetBot/Services/KfTokenService.cs

@@ -1,83 +1,167 @@
 using System.Net;
-using System.Text.Json;
+using HtmlAgilityPack;
+using KfChatDotNetBot.Settings;
 using NLog;
-using PuppeteerSharp;
 
 namespace KfChatDotNetBot.Services;
 
 public class KfTokenService
 {
-    // Shout out Gamba Sesh for open sourcing his token retriever which heavily inspired this implementation
-    public static async Task<string> FetchSessionTokenAsync(string domain, string username, string password, string browserPath, string? proxy = null)
+    private readonly Logger _logger = LogManager.GetCurrentClassLogger();
+    private readonly CancellationToken _ctx;
+    private CookieContainer _cookies = new();
+    private KiwiFlare _kiwiFlare;
+    private readonly string _kfDomain;
+    private readonly string? _proxy;
+
+    public KfTokenService(string kfDomain, string? proxy = null, CancellationToken? cancellationToken = null)
     {
-        var logger = LogManager.GetCurrentClassLogger();
-        var browserFetcher = new BrowserFetcher(new BrowserFetcherOptions
-            { Browser = SupportedBrowser.Chromium, Path = browserPath });
-        if (proxy != null)
+        _ctx = cancellationToken ?? CancellationToken.None;
+        _kiwiFlare = new KiwiFlare(kfDomain, proxy, cancellationToken);
+        _proxy = proxy;
+        _kfDomain = kfDomain;
+        var cachedCookies = Helpers.GetValue(BuiltIn.Keys.KiwiFarmsCookies).Result
+            .JsonDeserialize<Dictionary<string, string>>();
+        // This shouldn't happen as the setting's default value is {}, but I'm just doing it to shut the IDE up
+        if (cachedCookies == null) return;
+        foreach (var key in cachedCookies.Keys)
         {
-            browserFetcher.WebProxy = new WebProxy(proxy);
-            logger.Debug($"Detected proxy settings for browser download: {proxy}");
-        }
-
-        var installedBrowser = await browserFetcher.DownloadAsync();
-        logger.Debug("Downloaded browser");
-        List<string> launchArgs = ["--no-sandbox"];
-        if (proxy != null)
-        {
-            logger.Debug($"Configuring Chromium to use proxy {proxy}");
-            launchArgs.Add($"--proxy-server=\"{proxy}\"");
-        }
-
-        var launchOptions = new LaunchOptions
-        {
-            Headless = false, ExecutablePath = installedBrowser.GetExecutablePath(), UserDataDir = "kf_profile",
-            Args = launchArgs.ToArray()
-        };
-
-        await using var browser = await Puppeteer.LaunchAsync(launchOptions);
-        await using var page = await browser.NewPageAsync();
-        await page.GoToAsync($"https://{domain}/login");
-        await page.WaitForSelectorAsync("img[alt=\"Kiwi Farms\"]");
-        if (await page.QuerySelectorAsync("html[data-template=\"login\"]") == null)
-        {
-            logger.Debug("Page template is not login. This is expected if we're already logged in. Reloading page to get the freshest cookies then retrieving");
-            await page.ReloadAsync();
-            return await GetXfSessionCookie();
-        }
-        
-        var usernameFieldSelector = await page.QuerySelectorAsync("input[autocomplete=\"username\"]");
-        var passwordFieldSelector = await page.QuerySelectorAsync("input[autocomplete=\"current-password\"]");
-        var loginButtonSelector = await page.QuerySelectorAsync("div[class=\"formSubmitRow-controls\"] > button[type=\"submit\"]");
-        if (usernameFieldSelector == null || passwordFieldSelector == null || loginButtonSelector == null)
-        {
-            // Realistically this shouldn't happen unless Null changes the login template in a big way
-            logger.Error("Username/password fields could not be found");
-            throw new MissingLoginElementsException();
-        }
-
-        await usernameFieldSelector.TypeAsync(username);
-        await passwordFieldSelector.TypeAsync(password);
-        await loginButtonSelector.ClickAsync();
-        logger.Debug("Login fields have been filled out and button clicked. Awaiting page navigation.");
-        await page.WaitForNavigationAsync();
-        logger.Debug("Navigation completed. Doing the cookie needful");
-        return await GetXfSessionCookie();
-        
-        async Task<string> GetXfSessionCookie()
-        {
-            var cookies = await page.GetCookiesAsync();
-            var xfSession = cookies.FirstOrDefault(x => x.Name == "xf_session");
-            if (xfSession == null)
-            {
-                logger.Error("xf_session cookie not set. Cookie data follows");
-                logger.Error(JsonSerializer.Serialize(cookies));
-                throw new MissingSessionCookieException();
-            }
-            logger.Debug($"Returning xf_session value: {xfSession.Value}");
-            return xfSession.Value;
+            _cookies.Add(new Cookie(key, cachedCookies[key], "/", _kfDomain));
         }
     }
+    
+    private HttpClientHandler GetHttpClientHandler()
+    {
+        var handler = new HttpClientHandler { AutomaticDecompression = DecompressionMethods.All };
+        if (_proxy != null)
+        {
+            handler.Proxy = new WebProxy(_proxy);
+            handler.UseProxy = true;
+        }
+        handler.CookieContainer = _cookies;
+        return handler;
+    }
 
-    public class MissingSessionCookieException : Exception;
-    public class MissingLoginElementsException : Exception;
+    private async Task CheckClearanceToken()
+    {
+        var clearanceCookie = _cookies.GetAllCookies()["sssg_clearance"];
+        _logger.Debug($"Got clearance cookie with value: {clearanceCookie}");
+        if (clearanceCookie != null)
+        {
+            if (await _kiwiFlare.CheckAuth(clearanceCookie.Value)) return;
+            _logger.Debug("Cookie is no longer valid, removing");
+            _cookies.GetAllCookies().Remove(clearanceCookie);
+        }
+        _logger.Debug("Getting a new clearance token");
+        var i = 0;
+        // Shitty retry logic as the forum is still annoyingly unstable
+        while (i < 10)
+        {
+            i++;
+            try
+            {
+                var challenge = await _kiwiFlare.GetChallenge();
+                var solution = await _kiwiFlare.SolveChallenge(challenge);
+                var token = await _kiwiFlare.SubmitAnswer(solution);
+                _cookies.Add(new Cookie("sssg_clearance", token, "/", _kfDomain));
+                _logger.Debug("Successfully retrieved a new token and added to the cookie container");
+                return;
+            }
+            catch (Exception e)
+            {
+                _logger.Error($"Failed to solve the KiwiFlare challenge, attempt {i} of 10");
+                _logger.Error(e);
+            }
+        }
+        _logger.Error("Ran out of attempts");
+        throw new Exception("Failed to solve the challenge");
+    }
+
+    public async Task<bool> IsLoggedIn()
+    {
+        _logger.Debug("Checking clearance token is actually valid first");
+        await CheckClearanceToken();
+        using var client = new HttpClient(GetHttpClientHandler());
+        var response = await client.GetAsync($"https://{_kfDomain}/login", _ctx);
+        if (response.StatusCode == HttpStatusCode.NonAuthoritativeInformation)
+        {
+            _logger.Error("Caught a 203 response when trying to load logon page which means we were KiwiFlare challenged");
+            throw new KiwiFlareChallengedException();
+        }
+        response.EnsureSuccessStatusCode();
+        var document = new HtmlDocument();
+        document.Load(await response.Content.ReadAsStreamAsync(_ctx));
+        var html = document.DocumentNode.SelectSingleNode("//html");
+        if (html == null) throw new Exception("Caught a null when retrieving html element");
+        if (!html.Attributes.Contains("data-logged-in"))
+        {
+            throw new Exception("data-logged-in attribute missing");
+        }
+
+        return html.Attributes["data-logged-in"].Value == "true";
+    }
+
+    public async Task PerformLogin(string username, string password)
+    {
+        _logger.Debug("Checking clearance token is actually valid first");
+        await CheckClearanceToken();
+        using var client = new HttpClient(GetHttpClientHandler());
+        var response = await client.GetAsync($"https://{_kfDomain}/login", _ctx);
+        if (response.StatusCode == HttpStatusCode.NonAuthoritativeInformation)
+        {
+            _logger.Error("Caught a 203 response when trying to load logon page which means we were KiwiFlare challenged");
+            throw new KiwiFlareChallengedException();
+        }
+        response.EnsureSuccessStatusCode();
+        var document = new HtmlDocument();
+        document.Load(await response.Content.ReadAsStreamAsync(_ctx));
+        var html = document.DocumentNode.SelectSingleNode("//html");
+        if (html == null) throw new Exception("Caught a null when retrieving html element");
+        // Already logged in
+        if (html.GetAttributeValue("data-logged-in", "false") == "true") return;
+        if (!html.Attributes.Contains("data-csrf")) throw new Exception("data-csrf missing from html element");
+        var csrf = html.GetAttributeValue("data-csrf", string.Empty);
+        var formData = new FormUrlEncodedContent(new List<KeyValuePair<string, string>>
+        {
+            new("_xfToken", csrf),
+            new("login", username),
+            new("password", password),
+            new("_xfRedirect", $"https://{_kfDomain}/")
+        });
+        var postResponse = await client.PostAsync($"https://{_kfDomain}/login/login", formData, _ctx);
+        if (postResponse.StatusCode == HttpStatusCode.SeeOther)
+        {
+            _logger.Debug("Got HTTP response 303. Success!");
+            return;
+        }
+        postResponse.EnsureSuccessStatusCode();
+        _logger.Error($"Received HTTP response {postResponse.StatusCode}, checking to see if we're logged in");
+        var postDocument = new HtmlDocument();
+        postDocument.Load(await postResponse.Content.ReadAsStreamAsync(_ctx));
+        html = postDocument.DocumentNode.SelectSingleNode("//html");
+        if (html == null) throw new Exception("Caught a null when retrieving html element");
+        // Logged in!
+        if (html.GetAttributeValue("data-logged-in", "false") == "true") return;
+        _logger.Error("Not logged in :(");
+        var message =
+            postDocument.DocumentNode.SelectSingleNode(
+                "//div[@class=\"blockMessage blockMessage--error blockMessage--iconic\"]");
+        _logger.Error($"Error from the page was {message?.InnerText}");
+        throw new KiwiFarmsLogonFailedException();
+    }
+
+    public string? GetXfSessionCookie()
+    {
+        var cookie = _cookies.GetAllCookies()["xf_session"];
+        return cookie?.Value;
+    }
+
+    public async Task SaveCookies()
+    {
+        var cookiesToSave = _cookies.GetAllCookies().ToDictionary(cookie => cookie.Name, cookie => cookie.Value);
+        await Helpers.SetValueAsJsonObject(BuiltIn.Keys.KiwiFarmsCookies, cookiesToSave);
+    }
+    
+    public class KiwiFlareChallengedException : Exception;
+    public class KiwiFarmsLogonFailedException : Exception;
 }

KfChatDotNetBot/Settings/BuiltIn.cs

@@ -429,6 +429,16 @@ public static class BuiltIn
             Default = "951905",
             IsSecret = false,
             CacheDuration = TimeSpan.FromHours(1)
+        },
+        new BuiltInSettingsModel
+        {
+            Key = Keys.KiwiFarmsCookies,
+            Regex = ".+",
+            Description = "Kiwi Farms cookies in key-value pair format",
+            // Empty JSON object as it's a Dictionary<string, string> object
+            Default = "{}",
+            IsSecret = true,
+            CacheDuration = TimeSpan.FromHours(1)
         }
     ];
     
@@ -458,7 +468,6 @@ public static class BuiltIn
         public static string ShuffleBmjUsername = "Shuffle.BmjUsername";
         public static string JuiceCooldown = "Juice.Cooldown";
         public static string JuiceAmount = "Juice.Amount";
-        public static string KiwiFarmsToken = "KiwiFarms.Token";
         public static string KickEnabled = "Kick.Enabled";
         public static string HowlggDivisionAmount = "Howlgg.DivisionAmount";
         public static string HowlggBmjUserId = "Howlgg.BmjUserId";
@@ -470,5 +479,6 @@ public static class BuiltIn
         public static string FlareSolverrProxy = "FlareSolverr.Proxy";
         public static string ChipsggBmjUsername = "Chipsgg.BmjUsername";
         public static string RestreamUrl = "RestreamUrl";
+        public static string KiwiFarmsCookies = "KiwiFarms.Cookies";
     }
 }