Webdriver error handling and use native files

scraper/twitter_scraper.py

@@ -135,76 +135,80 @@ class Twitter_Scraper:
 			self.router = self.go_to_home
 		pass
 	
-    def _get_driver(
-        self,
-        proxy=None,
-    ):
+	def _get_driver(self, proxy=None):
 		print("Setup WebDriver...")
-        # header = Headers().generate()["User-Agent"] 
 
-        # User agent of a andoird smartphone device
-        header="Mozilla/5.0 (Linux; Android 11; SM-G998B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.5414.87 Mobile Safari/537.36"
-
-        # browser_option = ChromeOptions()
-        browser_option = FirefoxOptions()
-        browser_option.add_argument("--no-sandbox")
-        browser_option.add_argument("--disable-dev-shm-usage")
-        browser_option.add_argument("--ignore-certificate-errors")
-        browser_option.add_argument("--disable-gpu")
-        browser_option.add_argument("--log-level=3")
-        browser_option.add_argument("--disable-notifications")
-        browser_option.add_argument("--disable-popup-blocking")
-        browser_option.add_argument("--user-agent={}".format(header))
-        if proxy is not None:
-            browser_option.add_argument("--proxy-server=%s" % proxy)
-
-        # Option to hide browser or not
-        # If not yes then skips the headless
-        if self.headlessState == 'yes':
-            # For Hiding Browser
-            browser_option.add_argument("--headless")
-
-        try:
-            # print("Initializing ChromeDriver...")
-            # driver = webdriver.Chrome(
-            #     options=browser_option,
-            # )
-
-            print("Initializing FirefoxDriver...")
-            driver = webdriver.Firefox(
-                options=browser_option,
+		header = (
+			"Mozilla/5.0 (Linux; Android 11; SM-G998B) "
+			"AppleWebKit/537.36 (KHTML, like Gecko) "
+			"Chrome/109.0.5414.87 Mobile Safari/537.36"
 		)
 
-            print("WebDriver Setup Complete")
-            return driver
-        except WebDriverException:
-            try:
-                # print("Downloading ChromeDriver...")
-                # chromedriver_path = ChromeDriverManager().install()
-                # chrome_service = ChromeService(executable_path=chromedriver_path)
+		def create_firefox_driver():
+			firefox_options = FirefoxOptions()
+			firefox_options.add_argument("--no-sandbox")
+			firefox_options.add_argument("--disable-dev-shm-usage")
+			firefox_options.add_argument("--ignore-certificate-errors")
+			firefox_options.add_argument("--disable-gpu")
+			firefox_options.add_argument("--log-level=3")
+			firefox_options.add_argument("--disable-notifications")
+			firefox_options.add_argument("--disable-popup-blocking")
+			firefox_options.add_argument(f"--user-agent={header}")
+			if proxy:
+				firefox_options.add_argument(f"--proxy-server={proxy}")
+			if self.headlessState in ["yes", "true"]:
+				firefox_options.add_argument("--headless")
 
-                print("Downloading FirefoxDriver...")
-                firefoxdriver_path = GeckoDriverManager().install()
+			# Fix Invalid Host header: force IPv4
+			firefox_options.set_preference("network.dns.disableIPv6", True)
+
+			print("Initializing FirefoxDriver with system binary...")
+			firefoxdriver_path = "/usr/bin/geckodriver"  # Native geckodriver
 			firefox_service = FirefoxService(executable_path=firefoxdriver_path)
 
-                # print("Initializing ChromeDriver...")
-                # driver = webdriver.Chrome(
-                #     service=chrome_service,
-                #     options=browser_option,
-                # )
-
-                print("Initializing FirefoxDriver...")
-                driver = webdriver.Firefox(
-                    service=firefox_service,
-                    options=browser_option,
-                )
-
-                print("WebDriver Setup Complete")
+			driver = webdriver.Firefox(service=firefox_service, options=firefox_options)
+			print("Firefox WebDriver Setup Complete")
 			return driver
-            except Exception as e:
-                print(f"Error setting up WebDriver: {e}")
+
+		def create_chrome_driver():
+			chrome_options = ChromeOptions()
+			chrome_options.add_argument("--no-sandbox")
+			chrome_options.add_argument("--disable-dev-shm-usage")
+			chrome_options.add_argument("--ignore-certificate-errors")
+			chrome_options.add_argument("--disable-gpu")
+			chrome_options.add_argument("--log-level=3")
+			chrome_options.add_argument("--disable-notifications")
+			chrome_options.add_argument("--disable-popup-blocking")
+			chrome_options.add_argument(f"--user-agent={header}")
+
+			# Use Arch Chromium binary
+			chrome_options.binary_location = "/usr/sbin/chromium"
+
+			if proxy:
+				chrome_options.add_argument(f"--proxy-server={proxy}")
+			if self.headlessState in ["yes", "true"]:
+				chrome_options.add_argument("--headless")
+
+			print("Initializing ChromeDriver with system binary...")
+			chromedriver_path = "/usr/bin/chromedriver" # Native chromedriver
+			chrome_service = ChromeService(executable_path=chromedriver_path)
+
+			driver = webdriver.Chrome(service=chrome_service, options=chrome_options)
+			print("Chrome WebDriver Setup Complete")
+			return driver
+
+		try:
+			driver = create_firefox_driver()
+			return driver
+		except Exception as firefox_error:
+			print(f"Firefox WebDriver failed: {firefox_error}")
+			print("Attempting Chromium fallback...")
+			try:
+				driver = create_chrome_driver()
+				return driver
+			except Exception as chrome_error:
+				print(f"Error setting up Chrome WebDriver: {chrome_error}")
 				sys.exit(1)
-        pass
 	
 	def login(self):
 		print()