diff --git a/.env.example b/.env.example index e9510eb..217e8b8 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,3 @@ TWITTER_USERNAME=# Your Twitter Handle TWITTER_PASSWORD=# Your Twitter Password +HEADLESS=# Headless Mode "yes" or "no" diff --git a/scraper/__main__.py b/scraper/__main__.py index 7676189..b9349d9 100644 --- a/scraper/__main__.py +++ b/scraper/__main__.py @@ -44,6 +44,13 @@ def main(): default=os.getenv("TWITTER_PASSWORD"), help="Your Twitter password.", ) + + parser.add_argument( + "--headlessState", + type=str, + default=os.getenv("HEADLESS"), + help="Headless mode? [yes/no]" + ) except Exception as e: print(f"Error retrieving environment variables: {e}") sys.exit(1) @@ -113,6 +120,7 @@ def main(): USER_MAIL = args.mail USER_UNAME = args.user USER_PASSWORD = args.password + HEADLESS_MODE= args.headlessState if USER_UNAME is None: USER_UNAME = input("Twitter Username: ") @@ -120,6 +128,9 @@ def main(): if USER_PASSWORD is None: USER_PASSWORD = getpass.getpass("Enter Password: ") + if HEADLESS_MODE is None: + HEADLESS_MODE - str(input("Headless?[Yes/No]")).lower() + print() tweet_type_args = [] @@ -146,6 +157,7 @@ def main(): mail=USER_MAIL, username=USER_UNAME, password=USER_PASSWORD, + headlessState=HEADLESS_MODE ) scraper.login() scraper.scrape_tweets( diff --git a/scraper/twitter_scraper.py b/scraper/twitter_scraper.py index 4fd0fcd..9aa3883 100644 --- a/scraper/twitter_scraper.py +++ b/scraper/twitter_scraper.py @@ -37,6 +37,7 @@ class Twitter_Scraper: mail, username, password, + headlessState, max_tweets=50, scrape_username=None, scrape_hashtag=None, @@ -50,6 +51,7 @@ class Twitter_Scraper: self.mail = mail self.username = username self.password = password + self.headlessState = headlessState self.interrupted = False self.tweet_ids = set() self.data = [] @@ -125,7 +127,10 @@ class Twitter_Scraper: proxy=None, ): print("Setup WebDriver...") - header = Headers().generate()["User-Agent"] + # header = Headers().generate()["User-Agent"] + + # User agent of a andoird smartphone device + header="Mozilla/5.0 (Linux; Android 11; SM-G998B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.5414.87 Mobile Safari/537.36" # browser_option = ChromeOptions() browser_option = FirefoxOptions() @@ -140,8 +145,11 @@ class Twitter_Scraper: if proxy is not None: browser_option.add_argument("--proxy-server=%s" % proxy) - # For Hiding Browser - browser_option.add_argument("--headless") + # Option to hide browser or not + # If not yes then skips the headless + if self.headlessState == 'yes': + # For Hiding Browser + browser_option.add_argument("--headless") try: # print("Initializing ChromeDriver...")