From ec1309885a41804d48f17ae59e7f7701aa2e03c4 Mon Sep 17 00:00:00 2001 From: Jarrian Date: Sat, 9 Sep 2023 12:46:38 +0800 Subject: [PATCH] conditional ChromeDriver Installation --- requirements.txt | 10 +++++----- scraper/twitter_scraper.py | 32 +++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/requirements.txt b/requirements.txt index 25fc613..0b553fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -fake_headers==1.0.2 -pandas==2.0.3 -python-dotenv==1.0.0 -selenium==4.12.0 -webdriver_manager==4.0.0 +fake_headers>=1.0.2 +pandas>=2.0.3 +python-dotenv>=1.0.0 +selenium>=4.12.0 +webdriver_manager>=4.0.0 diff --git a/scraper/twitter_scraper.py b/scraper/twitter_scraper.py index cc50efc..9d7ad82 100644 --- a/scraper/twitter_scraper.py +++ b/scraper/twitter_scraper.py @@ -52,20 +52,29 @@ class Twitter_Scraper(): browser_option.add_argument("--headless") try: - print("Downloading ChromeDriver...") - chromedriver_path = ChromeDriverManager().install() - chrome_service = ChromeService(executable_path=chromedriver_path) - print("Initializing ChromeDriver...") driver = webdriver.Chrome( - service=chrome_service, options=browser_option, ) return driver - except Exception as e: - print(f"Error setting up WebDriver: {e}") - sys.exit(1) + except WebDriverException: + try: + print("Downloading ChromeDriver...") + chromedriver_path = ChromeDriverManager().install() + chrome_service = ChromeService( + executable_path=chromedriver_path) + + print("Initializing ChromeDriver...") + driver = webdriver.Chrome( + service=chrome_service, + options=browser_option, + ) + + return driver + except Exception as e: + print(f"Error setting up WebDriver: {e}") + sys.exit(1) def _login(self): print("Logging in to Twitter...") @@ -76,6 +85,9 @@ class Twitter_Scraper(): self._input_username() self._input_unusual_activity() self._input_password() + + print("Login Successful") + print() pass def _input_username(self): @@ -96,7 +108,7 @@ class Twitter_Scraper(): input_attempt += 1 if input_attempt >= 3: print() - print("""" + print(""" There was an error inputting the username. It may be due to the following: @@ -177,8 +189,6 @@ It may be due to the following: callback() print("Scraping Tweets...") - print() - self.progress.print_progress(0) try: