add error handling and script runs until unexpected error or max reached
This commit is contained in:
+36
-23
@@ -1,6 +1,7 @@
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import getpass
|
||||
from twitter_scraper import Twitter_Scraper
|
||||
|
||||
try:
|
||||
@@ -16,37 +17,49 @@ except Exception as e:
|
||||
|
||||
def main():
|
||||
try:
|
||||
USER_UNAME = os.getenv('TWITTER_USERNAME')
|
||||
USER_PASSWORD = os.getenv('TWITTER_PASSWORD')
|
||||
except Exception as e:
|
||||
print(f"Error retrieving environment variables: {e}")
|
||||
USER_UNAME = None
|
||||
USER_PASSWORD = None
|
||||
sys.exit(1)
|
||||
|
||||
parser = argparse.ArgumentParser(description='Twitter Scraper')
|
||||
parser.add_argument('--tweets', type=int, default=50,
|
||||
help='Number of tweets to scrape (default: 50)')
|
||||
args = parser.parse_args()
|
||||
|
||||
if USER_UNAME is not None and USER_PASSWORD is not None:
|
||||
try:
|
||||
USER_UNAME = os.getenv("TWITTER_USERNAME")
|
||||
USER_PASSWORD = os.getenv("TWITTER_PASSWORD")
|
||||
except Exception as e:
|
||||
print(f"Error retrieving environment variables: {e}")
|
||||
USER_UNAME = None
|
||||
USER_PASSWORD = None
|
||||
sys.exit(1)
|
||||
|
||||
if USER_UNAME is None:
|
||||
USER_UNAME = input("Twitter Username: ")
|
||||
|
||||
if USER_PASSWORD is None:
|
||||
USER_PASSWORD = getpass.getpass("Enter Password: ")
|
||||
|
||||
print()
|
||||
|
||||
parser = argparse.ArgumentParser(description="Twitter Scraper")
|
||||
parser.add_argument(
|
||||
"--tweets",
|
||||
type=int,
|
||||
default=50,
|
||||
help="Number of tweets to scrape (default: 50)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if USER_UNAME is not None and USER_PASSWORD is not None:
|
||||
scraper = Twitter_Scraper(
|
||||
username=USER_UNAME,
|
||||
password=USER_PASSWORD,
|
||||
max_tweets=args.tweets
|
||||
username=USER_UNAME, password=USER_PASSWORD, max_tweets=args.tweets
|
||||
)
|
||||
|
||||
scraper.scrape_tweets()
|
||||
scraper.driver.close()
|
||||
scraper.save_to_csv()
|
||||
except KeyboardInterrupt:
|
||||
print("\nScript Interrupted by user. Exiting...")
|
||||
scraper.driver.close()
|
||||
else:
|
||||
print(
|
||||
"Missing Twitter username or password environment variables. Please check your .env file."
|
||||
)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("Missing Twitter username or password environment variables. Please check your .env file.")
|
||||
except KeyboardInterrupt:
|
||||
print("\nScript Interrupted by user. Exiting...")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user