Update dlc_fetcher.py

Propose changes for dramatically faster scanning and fallback options

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,31 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: "[BUG]"
+labels: bug
+assignees: Novattz
+
+---
+
+
+**Before submitting, have you tried:**
+- Using smokeapi with Proton? [ ] Yes [ ] No
+- Checking if `LD_PRELOAD` is blocked on your system? [ ] Yes [ ] No
+
+**Describe the bug**
+- A clear and concise description of what the bug is.
+
+**Terminal output** 
+- Copy and paste the entire terminal output when running the script.
+
+**Log File Output**
+- If the script logged any errors, attach the log file **`script.log`** to this issue.
+
+**Debug Output**
+- Run the script with the **`--debug`** argument and attach the log file **`debug_script.log`** to this issue.
+
+**Steam library path**
+- Provide the path where your steam library is.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: "[ Feature Request ]"
+labels: enhancement
+assignees: Novattz
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

README.md

@@ -1,35 +1,39 @@
-# Steam DLC Fetcher and installer for Linux
-- Python script designed for linux to automate fetching of DLC id's for steam games and the installation of creamlinux automatically.
 
-# Features
+# Steam DLC Fetcher and installer for Linux
+- Python script designed for linux to automate fetching of DLC id's for steam games and the installation of creamlinux automatically. [Demo/Tutorial](https://www.youtube.com/watch?v=Y1E15rUsdDw)
+### Features
 - Automatically fetches and lists DLC's for selected steam game(s) installed on the computer.
 - Automatically installs creamlinux and its components into selected steam games, excluding and handling specific config files.
 - Provides a simple cli to navigate and operate the entire process.
 
-# Demo
+## Usage
-https://www.youtube.com/watch?v=22LDDUoBvus&ab_channel=Nova
+### Prerequisites
-
-# To do
-- Cross reference dlc files and dlc id's. Incase dlc id's and dlc files differ in terms of quantity it will notify the user.
-- Possibly add functionality to search for dlc files/automatically installing them.
-- Add the possibility to install cream/smokeapi for games running proton.
-- Check if the game already has dlc files installed
-- Gui?
-
-# Prerequisites
 - `python 3.x`
 - `requests` library
 - `zipfile` library
 
-# Usage
+
+### Installation
+
 - Clone the repo or download the script.
 - Navigate to the directory containing the script.
 - Run the script using python.
-```bash
-python dlc_fetcher.py
-```
 
-# Issues?
+#### OR
+Use this one-line shell script.
+```bash
+git clone https://github.com/Novattz/creamlinux-installer;cd creamlinux-installer;python dlc_fetcher.py
+```
+## TODO
+- [ ] Cross reference dlc files and dlc id's. Incase dlc id's and dlc files differ in terms of quantity it will notify the user.
+- [ ] Possibly add functionality to search for dlc files/automatically installing them.
+- [ ] Add the possibility to install cream/smokeapi for games running proton.
+- [ ] Check if the game already has dlc files installed
+- [ ] Gui?
+- [ ] Add checker for configs already applied to games. (i.e script will check for new dlc id's for already applied games.)
+- [ ] Add a way to check if a game blocks LD_PRELOAD.
+
+### Issues?
 - Open a issue and attach all relevant errors/logs.
 
 # Credits

dlc_fetcher.py

@@ -4,24 +4,59 @@ import requests
 import zipfile
 import time
 import stat
+import subprocess
+from collections import defaultdict
+import logging
+import argparse
+from concurrent.futures import ThreadPoolExecutor
+from tqdm import tqdm
 
 LOG_FILE = 'script.log'
+DEBUG_FILE = 'debug_script.log'
+TIMEOUT = 180  # Timeout in seconds (3 minutes)
+
+def setup_logging(debug):
+    log_format = '%(asctime)s [%(levelname)s] %(message)s'
+    date_format = '%m-%d %H:%M:%S'
+    if debug:
+        logging.basicConfig(filename=DEBUG_FILE, level=logging.DEBUG, format=log_format, datefmt=date_format)
+    else:
+        logging.basicConfig(filename=LOG_FILE, level=logging.ERROR, format=log_format, datefmt=date_format)
 
 def clear_screen():
     os.system('cls' if os.name == 'nt' else 'clear')
 
+def log_error(message):
+    logging.error(message)
+    print(message)
+
+def log_debug(message):
+    logging.debug(message)
+
+def read_steam_registry():
+    registry_path = os.path.expanduser('~/.steam/registry.vdf')
+    if os.path.exists(registry_path):
+        log_debug(f"Found Steam registry file: {registry_path}")
+        with open(registry_path, 'r') as f:
+            content = f.read()
+            install_path = re.search(r'"InstallPath"\s*"([^"]+)"', content)
+            if install_path:
+                return install_path.group(1)
+    return None
+
 def fetch_latest_version():
     try:
         response = requests.get("https://api.github.com/repos/Novattz/creamlinux-installer/releases/latest")
         data = response.json()
         return data['tag_name']
     except requests.exceptions.RequestException as e:
-        log_message(f"Failed to fetch latest version: {str(e)}")
+        log_error(f"Failed to fetch latest version: {str(e)}")
         return "Unknown"
 
-def show_header(app_version):
+def show_header(app_version, debug_mode):
     clear_screen()
     cyan = '\033[96m'
+    red = '\033[91m'
     reset = '\033[0m'
     print(f"{cyan}")
     print(r"""
@@ -39,13 +74,12 @@ def show_header(app_version):
     > Version: {app_version}
     {reset}
     """)
+    if debug_mode:
+        print(f"{red}    [Running in DEBUG mode]{reset}\n")
+    print()
 
 app_version = fetch_latest_version()
-
+#app_version = "TESTING / LETS NOT OVERLOAD GITHUB FOR NO REASON"
-def log_message(message):
-    with open(LOG_FILE, 'a') as log_file:
-        log_file.write(f"{message}\n")
-    print(message)
 
 def parse_vdf(file_path):
     library_paths = []
@@ -55,61 +89,113 @@ def parse_vdf(file_path):
             paths = re.findall(r'"path"\s*"(.*?)"', content, re.IGNORECASE)
             library_paths.extend([os.path.normpath(path) for path in paths])
     except Exception as e:
-        log_message(f"Failed to read {file_path}: {str(e)}")
+        log_error(f"Failed to read {file_path}: {str(e)}")
     return library_paths
 
-def find_steam_library_folders():
+def find_steam_binary():
-    base_paths = [
+    try:
+        result = subprocess.run(['which', 'steam'], stdout=subprocess.PIPE)
+        steam_path = result.stdout.decode('utf-8').strip()
+        if steam_path:
+            return os.path.dirname(steam_path)
+    except Exception as e:
+        log_error(f"Failed to locate steam binary: {str(e)}")
+    return None
+
+def find_steam_library_folders(manual_path=""):
+    search_list = [
+        # Default
         os.path.expanduser('~/.steam/steam'),
         os.path.expanduser('~/.local/share/Steam'),
-        os.path.expanduser('~/home/deck/.steam/steam'),
+
-        os.path.expanduser('~/home/deck/.local/share/Steam'),
+        # Steam Deck
-        '/mnt', '/media',
+        os.path.expanduser('/home/deck/.steam/steam'),
-        '/run/media/mmcblk0p1/steamapps'
+        os.path.expanduser('/home/deck/.local/share/Steam'),
+
+        # Others
+        '/mnt/Jogos/Steam',
+        '/run/media/mmcblk0p1',
+
+        # Flatpak
+        os.path.expanduser('~/.var/app/com.valvesoftware.Steam/.local/share/Steam'),
+        os.path.expanduser('~/.var/app/com.valvesoftware.Steam/data/Steam/steamapps/common')
     ]
+
     library_folders = []
     try:
-        for base_path in base_paths:
+        if manual_path:
-            if os.path.exists(base_path):
+            log_debug(f"Manual game path set to \"{manual_path}\" skipping path lookup")
-                for root, dirs, files in os.walk(base_path, topdown=True):
+            library_folders.append(manual_path)
-                    if 'steamapps' in dirs:
+        else:
-                        steamapps_path = os.path.join(root, 'steamapps')
+            steam_binary_path = find_steam_binary()
+            if steam_binary_path and steam_binary_path not in search_list:
+                search_list.append(steam_binary_path)
+
+            steam_install_path = read_steam_registry()
+            if steam_install_path and steam_install_path not in search_list:
+                search_list.append(steam_install_path)
+
+            log_debug(f"Paths that will be searched: {search_list}")
+
+            for search_path in search_list:
+                if os.path.exists(search_path):
+                    log_debug(f"Scanning path: {search_path}")
+                    steamapps_path = str(os.path.normpath(f"{search_path}/steamapps"))
+                    if os.path.exists(steamapps_path):
                         library_folders.append(steamapps_path)
+                        log_debug(f"Found steamapps folder: {steamapps_path}")
+
                         vdf_path = os.path.join(steamapps_path, 'libraryfolders.vdf')
                         if os.path.exists(vdf_path):
+                            log_debug(f"Found libraryfolders.vdf: {vdf_path}")
                             additional_paths = parse_vdf(vdf_path)
                             for path in additional_paths:
                                 new_steamapps_path = os.path.join(path, 'steamapps')
                                 if os.path.exists(new_steamapps_path):
                                     library_folders.append(new_steamapps_path)
-                            dirs[:] = []  # Prevent further scanning into subdirectories
+                                    log_debug(f"Added additional steamapps folder: {new_steamapps_path}")
-        if not library_folders:
+
-            raise FileNotFoundError("No Steam library folders found.")
+            if not library_folders:
+                raise FileNotFoundError("No Steam library folders found.")
     except Exception as e:
-        log_message(f"Error finding Steam library folders: {e}")
+        log_error(f"Error finding Steam library folders: {e}")
+        log_error("Scanned paths:")
+        for path in search_list:
+            log_error(f"  - {path}")
     return library_folders
 
+def process_acf_file(folder, item):
+    try:
+        app_id, game_name, install_dir = parse_acf(os.path.join(folder, item))
+        if app_id and game_name:
+            install_path = os.path.join(folder, 'common', install_dir)
+            if os.path.exists(install_path):
+                return app_id, game_name, install_path
+    except Exception as e:
+        log_error(f"Error processing {item}: {e}")
+    return None
+
 def find_steam_apps(library_folders):
     acf_pattern = re.compile(r'^appmanifest_(\d+)\.acf$')
     games = {}
-    try:
+    
+    with ThreadPoolExecutor() as executor:
+        futures = []
         for folder in library_folders:
             if os.path.exists(folder):
                 for item in os.listdir(folder):
                     if acf_pattern.match(item):
-                        try:
+                        futures.append(
-                            app_id, game_name, install_dir = parse_acf(os.path.join(folder, item))
+                            executor.submit(process_acf_file, folder, item)
-                            if app_id and game_name:
+                        )
-                                install_path = os.path.join(folder, 'common', install_dir)
+        
-                                if os.path.exists(install_path):
+        for future in futures:
-                                    cream_installed = 'Cream installed' if 'cream.sh' in os.listdir(install_path) else ''
+            result = future.result()
-                                    games[app_id] = (game_name, cream_installed, install_path)
+            if result:
-                        except Exception as e:
+                app_id, game_name, install_path = result
-                            log_message(f"Error parsing {item}: {e}")
+                cream_installed = 'Cream installed' if os.path.exists(os.path.join(install_path, 'cream.sh')) else ''
-        if not games:
+                games[app_id] = (game_name, cream_installed, install_path)
-            raise FileNotFoundError("No Steam games found.")
+    
-    except Exception as e:
-        log_message(f"Error finding Steam apps: {e}")
     return games
 
 def parse_acf(file_path):
@@ -121,7 +207,7 @@ def parse_acf(file_path):
         install_dir = re.search(r'"installdir"\s+"([^"]+)"', data)
         return app_id.group(1), name.group(1), install_dir.group(1)
     except Exception as e:
-        log_message(f"Error reading ACF file {file_path}: {e}")
+        log_error(f"Error reading ACF file {file_path}: {e}")
         return None, None, None
 
 def fetch_dlc_details(app_id):
@@ -130,36 +216,41 @@ def fetch_dlc_details(app_id):
         response = requests.get(base_url)
         data = response.json()
         if app_id not in data or "data" not in data[app_id]:
-            log_message("Error: Unable to fetch game details.")
             return []
+        
         game_data = data[app_id]["data"]
         dlcs = game_data.get("dlc", [])
         dlc_details = []
-        for dlc_id in dlcs:
+        
-            try:
+        with tqdm(total=len(dlcs), desc="Fetching DLC details") as pbar:
-                time.sleep(0.3)
+            for dlc_id in dlcs:
-                dlc_url = f"https://store.steampowered.com/api/appdetails?appids={dlc_id}"
+                try:
-                dlc_response = requests.get(dlc_url)
+                    time.sleep(0.3)
-                if dlc_response.status_code == 200:
+                    dlc_url = f"https://store.steampowered.com/api/appdetails?appids={dlc_id}"
-                    dlc_data = dlc_response.json()
+                    dlc_response = requests.get(dlc_url)
-                    if str(dlc_id) in dlc_data and "data" in dlc_data[str(dlc_id)]:
+                    
-                        dlc_name = dlc_data[str(dlc_id)]["data"].get("name", "Unknown DLC")
+                    if dlc_response.status_code == 200:
-                        dlc_details.append({"appid": dlc_id, "name": dlc_name})
+                        dlc_data = dlc_response.json()
-                    else:
+                        if str(dlc_id) in dlc_data and "data" in dlc_data[str(dlc_id)]:
-                        log_message(f"Data missing for DLC {dlc_id}")
+                            dlc_name = dlc_data[str(dlc_id)]["data"].get("name", "Unknown DLC")
-                elif dlc_response.status_code == 429:
+                            dlc_details.append({"appid": dlc_id, "name": dlc_name})
-                    log_message("Rate limited! Please wait before trying again.")
+                    elif dlc_response.status_code == 429:
-                    time.sleep(10)
+                        time.sleep(10)
-                else:
+                        
-                    log_message(f"Failed to fetch details for DLC {dlc_id}, Status Code: {dlc_response.status_code}")
+                    pbar.update(1)
-            except Exception as e:
+                except Exception as e:
-                log_message(f"Exception for DLC {dlc_id}: {str(e)}")
+                    log_error(f"Exception for DLC {dlc_id}: {str(e)}")
+                    
         return dlc_details
+        
     except requests.exceptions.RequestException as e:
-        log_message(f"Failed to fetch DLC details for {app_id}: {e}")
+        log_error(f"Failed to fetch DLC details for {app_id}: {e}")
         return []
 
 def install_files(app_id, game_install_dir, dlcs, game_name):
+    GREEN = '\033[92m'
+    YELLOW = '\033[93m'
+    RESET = '\033[0m'
     zip_url = "https://github.com/anticitizn/creamlinux/releases/latest/download/creamlinux.zip"
     zip_path = os.path.join(game_install_dir, 'creamlinux.zip')
     try:
@@ -179,44 +270,99 @@ def install_files(app_id, game_install_dir, dlcs, game_name):
             with open(cream_api_path, 'w') as f:
                 f.write(f"APPID = {app_id}\n[config]\nissubscribedapp_on_false_use_real = true\n[methods]\ndisable_steamapps_issubscribedapp = false\n[dlc]\n{dlc_list}")
             print(f"Custom cream_api.ini has been written to {game_install_dir}.")
-            print(f"Installation complete. Set launch options in Steam: 'sh ./cream.sh %command%' for {game_name}.")
+            print(f"\n{GREEN}Installation complete!{RESET}")
+            print(f"\n{YELLOW}Set launch options in Steam:{RESET}")
+            print(f"{GREEN}'sh ./cream.sh %command%'{RESET} for {game_name}")
         else:
-            log_message("Failed to download the files needed for installation.")
+            log_error("Failed to download the files needed for installation.")
     except Exception as e:
-        log_message(f"Failed to install files for {game_name}: {e}")
+        log_error(f"Failed to install files for {game_name}: {e}")
+
+def uninstall_creamlinux(install_path, game_name):
+    YELLOW = '\033[93m'
+    GREEN = '\033[92m'
+    RESET = '\033[0m'
+    try:
+        files_to_remove = ['cream.sh', 'cream_api.ini', 'cream_api.so', 'lib32Creamlinux.so', 'lib64Creamlinux.so']
+        for file in files_to_remove:
+            file_path = os.path.join(install_path, file)
+            if os.path.exists(file_path):
+                os.remove(file_path)
+        print(f"\n{GREEN}Successfully uninstalled CreamLinux from {game_name}{RESET}")
+        print(f"\n{YELLOW}Make sure to remove{RESET} {GREEN}'sh ./cream.sh %command%'{RESET} {YELLOW}from launch options{RESET}")
+    except Exception as e:
+        log_error(f"Failed to uninstall CreamLinux: {e}")
 
 def main():
-    show_header(app_version)
+    parser = argparse.ArgumentParser(description="Steam DLC Fetcher")
+    parser.add_argument("--manual", metavar='steamapps_path', help="Sets the steamapps path for faster operation", required=False)
+    parser.add_argument("--debug", action="store_true", help="Enable debug logging")
+    args = parser.parse_args()
+
+    setup_logging(args.debug)
+    show_header(app_version, args.debug)
+
     try:
-        library_folders = find_steam_library_folders()
+        library_folders = find_steam_library_folders(args.manual)
+        if not library_folders:
+            print("Falling back to Manual Method since no library folder was found")
+            steamapps_path = input("Steamapps Path: ")
+            if len(steamapps_path) > 3:
+                library_folders = [steamapps_path]
+            else:
+                print("Invalid path! Closing the program...")
+                return
+
         games = find_steam_apps(library_folders)
         if games:
-            print("Select the game for which you want to fetch DLCs:")
+            print("\nSelect the game for which you want to fetch DLCs:")
             games_list = list(games.items())
             GREEN = '\033[92m'
             RESET = '\033[0m'
+            
             for idx, (app_id, (name, cream_status, _)) in enumerate(games_list, 1):
+                status = []
                 if cream_status:
-                    print(f"{idx}. {GREEN}{name} (App ID: {app_id}) - Cream installed{RESET}")
+                    status.append(f"{GREEN}Cream installed{RESET}")
+                
+                status_str = f" ({', '.join(status)})" if status else ""
+                print(f"{idx}. {name} (App ID: {app_id}){status_str}")
+
+            choice = int(input("\nEnter the number of the game: ")) - 1
+            if 0 <= choice < len(games_list):
+                selected_app_id, (selected_game_name, cream_status, selected_install_dir) = games_list[choice]
+                
+                if cream_status:
+                    print("\nCreamLinux is installed. What would you like to do?")
+                    print("1. Fetch DLC IDs")
+                    print("2. Uninstall CreamLinux")
+                    action = input("Enter your choice (1-2): ")
+                    
+                    if action == "1":
+                        print(f"\nSelected: {selected_game_name} (App ID: {selected_app_id})")
+                        dlcs = fetch_dlc_details(selected_app_id)
+                        if dlcs:
+                            install_files(selected_app_id, selected_install_dir, dlcs, selected_game_name)
+                        else:
+                            print("No DLCs found for the selected game.")
+                    elif action == "2":
+                        uninstall_creamlinux(selected_install_dir, selected_game_name)
+                    else:
+                        print("Invalid choice.")
                 else:
-                    print(f"{idx}. {name} (App ID: {app_id})")
+                    print(f"\nSelected: {selected_game_name} (App ID: {selected_app_id})")
-
+                    dlcs = fetch_dlc_details(selected_app_id)
-            choice = int(input("Enter the number of the game: ")) - 1
+                    if dlcs:
-            if choice < 0 or choice >= len(games_list):
+                        install_files(selected_app_id, selected_install_dir, dlcs, selected_game_name)
-                raise ValueError("Invalid selection.")
+                    else:
-            selected_app_id, (selected_game_name, _, selected_install_dir) = games_list[choice]
+                        print("No DLCs found for the selected game.")
-            print(f"You selected: {selected_game_name} (App ID: {selected_app_id})")
-
-            dlcs = fetch_dlc_details(selected_app_id)
-            if dlcs:
-                print("DLC IDs found:", [dlc['appid'] for dlc in dlcs])  # Only print app IDs for clarity
-                install_files(selected_app_id, selected_install_dir, dlcs, selected_game_name)
             else:
-                print("No DLCs found for the selected game.")
+                print("Invalid selection.")
         else:
             print("No Steam games found on this computer or connected drives.")
     except Exception as e:
-        log_message(f"An error occurred: {e}")
+        logging.exception("An error occurred:")
+        log_error(f"An error occurred: {e}")
 
 if __name__ == "__main__":
     main()