add error handling and script runs until unexpected error or max reached

This commit is contained in:
Jarrian
2023-09-09 14:19:13 +08:00
parent 8843b4d81e
commit b43fb72dbd
5 changed files with 142 additions and 127 deletions
+36 -23
View File
@@ -1,6 +1,7 @@
import os
import sys
import argparse
import getpass
from twitter_scraper import Twitter_Scraper
try:
@@ -16,37 +17,49 @@ except Exception as e:
def main():
try:
USER_UNAME = os.getenv('TWITTER_USERNAME')
USER_PASSWORD = os.getenv('TWITTER_PASSWORD')
except Exception as e:
print(f"Error retrieving environment variables: {e}")
USER_UNAME = None
USER_PASSWORD = None
sys.exit(1)
parser = argparse.ArgumentParser(description='Twitter Scraper')
parser.add_argument('--tweets', type=int, default=50,
help='Number of tweets to scrape (default: 50)')
args = parser.parse_args()
if USER_UNAME is not None and USER_PASSWORD is not None:
try:
USER_UNAME = os.getenv("TWITTER_USERNAME")
USER_PASSWORD = os.getenv("TWITTER_PASSWORD")
except Exception as e:
print(f"Error retrieving environment variables: {e}")
USER_UNAME = None
USER_PASSWORD = None
sys.exit(1)
if USER_UNAME is None:
USER_UNAME = input("Twitter Username: ")
if USER_PASSWORD is None:
USER_PASSWORD = getpass.getpass("Enter Password: ")
print()
parser = argparse.ArgumentParser(description="Twitter Scraper")
parser.add_argument(
"--tweets",
type=int,
default=50,
help="Number of tweets to scrape (default: 50)",
)
args = parser.parse_args()
if USER_UNAME is not None and USER_PASSWORD is not None:
scraper = Twitter_Scraper(
username=USER_UNAME,
password=USER_PASSWORD,
max_tweets=args.tweets
username=USER_UNAME, password=USER_PASSWORD, max_tweets=args.tweets
)
scraper.scrape_tweets()
scraper.driver.close()
scraper.save_to_csv()
except KeyboardInterrupt:
print("\nScript Interrupted by user. Exiting...")
scraper.driver.close()
else:
print(
"Missing Twitter username or password environment variables. Please check your .env file."
)
sys.exit(1)
else:
print("Missing Twitter username or password environment variables. Please check your .env file.")
except KeyboardInterrupt:
print("\nScript Interrupted by user. Exiting...")
sys.exit(1)
if __name__ == '__main__':
if __name__ == "__main__":
main()