Update dlc_fetcher.py

Propose changes for dramatically faster scanning and fallback options

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,31 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: "[BUG]"
+labels: bug
+assignees: Novattz
+
+---
+
+
+**Before submitting, have you tried:**
+- Using smokeapi with Proton? [ ] Yes [ ] No
+- Checking if `LD_PRELOAD` is blocked on your system? [ ] Yes [ ] No
+
+**Describe the bug**
+- A clear and concise description of what the bug is.
+
+**Terminal output** 
+- Copy and paste the entire terminal output when running the script.
+
+**Log File Output**
+- If the script logged any errors, attach the log file **`script.log`** to this issue.
+
+**Debug Output**
+- Run the script with the **`--debug`** argument and attach the log file **`debug_script.log`** to this issue.
+
+**Steam library path**
+- Provide the path where your steam library is.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: "[ Feature Request ]"
+labels: enhancement
+assignees: Novattz
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

README.md

@@ -1,23 +1,40 @@
-# Steam DLC Fetcher and installer for Linux
-- Python script designed for linux to automate fetching of DLC id's for steam games and the installation of creamlinux automatically.
 
-# Features
+# Steam DLC Fetcher and installer for Linux
+- Python script designed for linux to automate fetching of DLC id's for steam games and the installation of creamlinux automatically. [Demo/Tutorial](https://www.youtube.com/watch?v=Y1E15rUsdDw)
+### Features
 - Automatically fetches and lists DLC's for selected steam game(s) installed on the computer.
 - Automatically installs creamlinux and its components into selected steam games, excluding and handling specific config files.
 - Provides a simple cli to navigate and operate the entire process.
 
-# Prerequisites
+## Usage
+### Prerequisites
 - `python 3.x`
 - `requests` library
 - `zipfile` library
 
-# Usage
+
+### Installation
+
 - Clone the repo or download the script.
 - Navigate to the directory containing the script.
 - Run the script using python.
+
+#### OR
+Use this one-line shell script.
 ```bash
-python steam_dlc_fetcher.py
+git clone https://github.com/Novattz/creamlinux-installer;cd creamlinux-installer;python dlc_fetcher.py
 ```
+## TODO
+- [ ] Cross reference dlc files and dlc id's. Incase dlc id's and dlc files differ in terms of quantity it will notify the user.
+- [ ] Possibly add functionality to search for dlc files/automatically installing them.
+- [ ] Add the possibility to install cream/smokeapi for games running proton.
+- [ ] Check if the game already has dlc files installed
+- [ ] Gui?
+- [ ] Add checker for configs already applied to games. (i.e script will check for new dlc id's for already applied games.)
+- [ ] Add a way to check if a game blocks LD_PRELOAD.
+
+### Issues?
+- Open a issue and attach all relevant errors/logs.
 
 # Credits
 - [All credits for creamlinux go to its original author and contributors.](https://github.com/anticitizn/creamlinux)

dlc_fetcher.py

@@ -4,6 +4,82 @@ import requests
 import zipfile
 import time
 import stat
+import subprocess
+from collections import defaultdict
+import logging
+import argparse
+from concurrent.futures import ThreadPoolExecutor
+from tqdm import tqdm
+
+LOG_FILE = 'script.log'
+DEBUG_FILE = 'debug_script.log'
+TIMEOUT = 180  # Timeout in seconds (3 minutes)
+
+def setup_logging(debug):
+    log_format = '%(asctime)s [%(levelname)s] %(message)s'
+    date_format = '%m-%d %H:%M:%S'
+    if debug:
+        logging.basicConfig(filename=DEBUG_FILE, level=logging.DEBUG, format=log_format, datefmt=date_format)
+    else:
+        logging.basicConfig(filename=LOG_FILE, level=logging.ERROR, format=log_format, datefmt=date_format)
+
+def clear_screen():
+    os.system('cls' if os.name == 'nt' else 'clear')
+
+def log_error(message):
+    logging.error(message)
+    print(message)
+
+def log_debug(message):
+    logging.debug(message)
+
+def read_steam_registry():
+    registry_path = os.path.expanduser('~/.steam/registry.vdf')
+    if os.path.exists(registry_path):
+        log_debug(f"Found Steam registry file: {registry_path}")
+        with open(registry_path, 'r') as f:
+            content = f.read()
+            install_path = re.search(r'"InstallPath"\s*"([^"]+)"', content)
+            if install_path:
+                return install_path.group(1)
+    return None
+
+def fetch_latest_version():
+    try:
+        response = requests.get("https://api.github.com/repos/Novattz/creamlinux-installer/releases/latest")
+        data = response.json()
+        return data['tag_name']
+    except requests.exceptions.RequestException as e:
+        log_error(f"Failed to fetch latest version: {str(e)}")
+        return "Unknown"
+
+def show_header(app_version, debug_mode):
+    clear_screen()
+    cyan = '\033[96m'
+    red = '\033[91m'
+    reset = '\033[0m'
+    print(f"{cyan}")
+    print(r"""
+  _    _ _   _ _      ____   _____ _  ________ _____  
+ | |  | | \ | | |    / __ \ / ____| |/ /  ____|  __ \ 
+ | |  | |  \| | |   | |  | | |    | ' /| |__  | |__) |
+ | |  | | . ` | |   | |  | | |    |  < |  __| |  _  / 
+ | |__| | |\  | |___| |__| | |____| . \| |____| | \ \ 
+  \____/|_| \_|______\____/ \_____|_|\_\______|_|  \_\
+                                                      
+    """)
+    print(f"""
+    > Made by Tickbase
+    > GitHub: https://github.com/Novattz/creamlinux-installer
+    > Version: {app_version}
+    {reset}
+    """)
+    if debug_mode:
+        print(f"{red}    [Running in DEBUG mode]{reset}\n")
+    print()
+
+app_version = fetch_latest_version()
+#app_version = "TESTING / LETS NOT OVERLOAD GITHUB FOR NO REASON"
 
 def parse_vdf(file_path):
     library_paths = []
@@ -13,128 +89,280 @@ def parse_vdf(file_path):
             paths = re.findall(r'"path"\s*"(.*?)"', content, re.IGNORECASE)
             library_paths.extend([os.path.normpath(path) for path in paths])
     except Exception as e:
-        print(f"Failed to read {file_path}: {str(e)}")
+        log_error(f"Failed to read {file_path}: {str(e)}")
     return library_paths
 
-def find_steam_library_folders():
+def find_steam_binary():
-    base_paths = [os.path.expanduser('~/.steam/steam'), os.path.expanduser('~/.local/share/Steam'), '/mnt', '/media']
+    try:
+        result = subprocess.run(['which', 'steam'], stdout=subprocess.PIPE)
+        steam_path = result.stdout.decode('utf-8').strip()
+        if steam_path:
+            return os.path.dirname(steam_path)
+    except Exception as e:
+        log_error(f"Failed to locate steam binary: {str(e)}")
+    return None
+
+def find_steam_library_folders(manual_path=""):
+    search_list = [
+        # Default
+        os.path.expanduser('~/.steam/steam'),
+        os.path.expanduser('~/.local/share/Steam'),
+
+        # Steam Deck
+        os.path.expanduser('/home/deck/.steam/steam'),
+        os.path.expanduser('/home/deck/.local/share/Steam'),
+
+        # Others
+        '/mnt/Jogos/Steam',
+        '/run/media/mmcblk0p1',
+
+        # Flatpak
+        os.path.expanduser('~/.var/app/com.valvesoftware.Steam/.local/share/Steam'),
+        os.path.expanduser('~/.var/app/com.valvesoftware.Steam/data/Steam/steamapps/common')
+    ]
+
     library_folders = []
-    for base_path in base_paths:
+    try:
-        if os.path.exists(base_path):
+        if manual_path:
-            for root, dirs, files in os.walk(base_path, topdown=True):
+            log_debug(f"Manual game path set to \"{manual_path}\" skipping path lookup")
-                if 'steamapps' in dirs:
+            library_folders.append(manual_path)
-                    steamapps_path = os.path.join(root, 'steamapps')
+        else:
-                    library_folders.append(steamapps_path)
+            steam_binary_path = find_steam_binary()
-                    vdf_path = os.path.join(steamapps_path, 'libraryfolders.vdf')
+            if steam_binary_path and steam_binary_path not in search_list:
-                    if os.path.exists(vdf_path):
+                search_list.append(steam_binary_path)
-                        additional_paths = parse_vdf(vdf_path)
+
-                        for path in additional_paths:
+            steam_install_path = read_steam_registry()
-                            new_steamapps_path = os.path.join(path, 'steamapps')
+            if steam_install_path and steam_install_path not in search_list:
-                            if os.path.exists(new_steamapps_path):
+                search_list.append(steam_install_path)
-                                library_folders.append(new_steamapps_path)
+
-                    dirs[:] = []  # Prevent further scanning into subdirectories
+            log_debug(f"Paths that will be searched: {search_list}")
+
+            for search_path in search_list:
+                if os.path.exists(search_path):
+                    log_debug(f"Scanning path: {search_path}")
+                    steamapps_path = str(os.path.normpath(f"{search_path}/steamapps"))
+                    if os.path.exists(steamapps_path):
+                        library_folders.append(steamapps_path)
+                        log_debug(f"Found steamapps folder: {steamapps_path}")
+
+                        vdf_path = os.path.join(steamapps_path, 'libraryfolders.vdf')
+                        if os.path.exists(vdf_path):
+                            log_debug(f"Found libraryfolders.vdf: {vdf_path}")
+                            additional_paths = parse_vdf(vdf_path)
+                            for path in additional_paths:
+                                new_steamapps_path = os.path.join(path, 'steamapps')
+                                if os.path.exists(new_steamapps_path):
+                                    library_folders.append(new_steamapps_path)
+                                    log_debug(f"Added additional steamapps folder: {new_steamapps_path}")
+
+            if not library_folders:
+                raise FileNotFoundError("No Steam library folders found.")
+    except Exception as e:
+        log_error(f"Error finding Steam library folders: {e}")
+        log_error("Scanned paths:")
+        for path in search_list:
+            log_error(f"  - {path}")
     return library_folders
 
+def process_acf_file(folder, item):
+    try:
+        app_id, game_name, install_dir = parse_acf(os.path.join(folder, item))
+        if app_id and game_name:
+            install_path = os.path.join(folder, 'common', install_dir)
+            if os.path.exists(install_path):
+                return app_id, game_name, install_path
+    except Exception as e:
+        log_error(f"Error processing {item}: {e}")
+    return None
+
 def find_steam_apps(library_folders):
     acf_pattern = re.compile(r'^appmanifest_(\d+)\.acf$')
     games = {}
-    for folder in library_folders:
+    
-        if os.path.exists(folder):
+    with ThreadPoolExecutor() as executor:
-            for item in os.listdir(folder):
+        futures = []
-                if acf_pattern.match(item):
+        for folder in library_folders:
-                    app_id, game_name, install_dir = parse_acf(os.path.join(folder, item))
+            if os.path.exists(folder):
-                    if app_id and game_name:
+                for item in os.listdir(folder):
-                        install_path = os.path.join(folder, 'common', install_dir)
+                    if acf_pattern.match(item):
-                        if os.path.exists(install_path):
+                        futures.append(
-                            cream_installed = 'Cream installed' if 'cream.sh' in os.listdir(install_path) else ''
+                            executor.submit(process_acf_file, folder, item)
-                            games[app_id] = (game_name, cream_installed, install_path)
+                        )
+        
+        for future in futures:
+            result = future.result()
+            if result:
+                app_id, game_name, install_path = result
+                cream_installed = 'Cream installed' if os.path.exists(os.path.join(install_path, 'cream.sh')) else ''
+                games[app_id] = (game_name, cream_installed, install_path)
+    
     return games
 
 def parse_acf(file_path):
-    with open(file_path, 'r', encoding='utf-8') as file:
+    try:
-        data = file.read()
+        with open(file_path, 'r', encoding='utf-8') as file:
-    app_id = re.search(r'"appid"\s+"(\d+)"', data)
+            data = file.read()
-    name = re.search(r'"name"\s+"([^"]+)"', data)
+        app_id = re.search(r'"appid"\s+"(\d+)"', data)
-    install_dir = re.search(r'"installdir"\s+"([^"]+)"', data)
+        name = re.search(r'"name"\s+"([^"]+)"', data)
-    return app_id.group(1), name.group(1), install_dir.group(1)
+        install_dir = re.search(r'"installdir"\s+"([^"]+)"', data)
+        return app_id.group(1), name.group(1), install_dir.group(1)
+    except Exception as e:
+        log_error(f"Error reading ACF file {file_path}: {e}")
+        return None, None, None
 
 def fetch_dlc_details(app_id):
     base_url = f"https://store.steampowered.com/api/appdetails?appids={app_id}"
-    response = requests.get(base_url)
+    try:
-    data = response.json()
+        response = requests.get(base_url)
-    if app_id not in data or "data" not in data[app_id]:
+        data = response.json()
-        print("Error: Unable to fetch game details.")
+        if app_id not in data or "data" not in data[app_id]:
+            return []
+        
+        game_data = data[app_id]["data"]
+        dlcs = game_data.get("dlc", [])
+        dlc_details = []
+        
+        with tqdm(total=len(dlcs), desc="Fetching DLC details") as pbar:
+            for dlc_id in dlcs:
+                try:
+                    time.sleep(0.3)
+                    dlc_url = f"https://store.steampowered.com/api/appdetails?appids={dlc_id}"
+                    dlc_response = requests.get(dlc_url)
+                    
+                    if dlc_response.status_code == 200:
+                        dlc_data = dlc_response.json()
+                        if str(dlc_id) in dlc_data and "data" in dlc_data[str(dlc_id)]:
+                            dlc_name = dlc_data[str(dlc_id)]["data"].get("name", "Unknown DLC")
+                            dlc_details.append({"appid": dlc_id, "name": dlc_name})
+                    elif dlc_response.status_code == 429:
+                        time.sleep(10)
+                        
+                    pbar.update(1)
+                except Exception as e:
+                    log_error(f"Exception for DLC {dlc_id}: {str(e)}")
+                    
+        return dlc_details
+        
+    except requests.exceptions.RequestException as e:
+        log_error(f"Failed to fetch DLC details for {app_id}: {e}")
         return []
-    game_data = data[app_id]["data"]
-    dlcs = game_data.get("dlc", [])
-    dlc_details = []
-    for dlc_id in dlcs:
-        try:
-            time.sleep(0.3)
-            dlc_url = f"https://store.steampowered.com/api/appdetails?appids={dlc_id}"
-            dlc_response = requests.get(dlc_url)
-            if dlc_response.status_code == 200:
-                dlc_data = dlc_response.json()
-                if str(dlc_id) in dlc_data and "data" in dlc_data[str(dlc_id)]:
-                    dlc_name = dlc_data[str(dlc_id)]["data"].get("name", "Unknown DLC")
-                    dlc_details.append({"appid": dlc_id, "name": dlc_name})
-                else:
-                    print(f"Data missing for DLC {dlc_id}")
-            elif dlc_response.status_code == 429:
-                print("Rate limited! Please wait before trying again.")
-                time.sleep(10)
-            else:
-                print(f"Failed to fetch details for DLC {dlc_id}, Status Code: {dlc_response.status_code}")
-        except Exception as e:
-            print(f"Exception for DLC {dlc_id}: {str(e)}")
-    return dlc_details
 
 def install_files(app_id, game_install_dir, dlcs, game_name):
+    GREEN = '\033[92m'
+    YELLOW = '\033[93m'
+    RESET = '\033[0m'
     zip_url = "https://github.com/anticitizn/creamlinux/releases/latest/download/creamlinux.zip"
     zip_path = os.path.join(game_install_dir, 'creamlinux.zip')
-    response = requests.get(zip_url)
+    try:
-    if response.status_code == 200:
+        response = requests.get(zip_url)
-        with open(zip_path, 'wb') as f:
+        if response.status_code == 200:
-            f.write(response.content)
+            with open(zip_path, 'wb') as f:
-        with zipfile.ZipFile(zip_path, 'r') as zip_ref:
+                f.write(response.content)
-            zip_ref.extractall(game_install_dir)
+            with zipfile.ZipFile(zip_path, 'r') as zip_ref:
-        os.remove(zip_path)
+                zip_ref.extractall(game_install_dir)
+            os.remove(zip_path)
 
-        cream_sh_path = os.path.join(game_install_dir, 'cream.sh')
+            cream_sh_path = os.path.join(game_install_dir, 'cream.sh')
-        os.chmod(cream_sh_path, os.stat(cream_sh_path).st_mode | stat.S_IEXEC)
+            os.chmod(cream_sh_path, os.stat(cream_sh_path).st_mode | stat.S_IEXEC)
 
-        dlc_list = "\n".join([f"{dlc['appid']} = {dlc['name']}" for dlc in dlcs])
+            dlc_list = "\n".join([f"{dlc['appid']} = {dlc['name']}" for dlc in dlcs])
-        cream_api_path = os.path.join(game_install_dir, 'cream_api.ini')
+            cream_api_path = os.path.join(game_install_dir, 'cream_api.ini')
-        with open(cream_api_path, 'w') as f:
+            with open(cream_api_path, 'w') as f:
-            f.write(f"APPID = {app_id}\n[config]\nissubscribedapp_on_false_use_real = true\n[methods]\ndisable_steamapps_issubscribedapp = false\n[dlc]\n{dlc_list}")
+                f.write(f"APPID = {app_id}\n[config]\nissubscribedapp_on_false_use_real = true\n[methods]\ndisable_steamapps_issubscribedapp = false\n[dlc]\n{dlc_list}")
-        print(f"Custom cream_api.ini has been written to {game_install_dir}.")
+            print(f"Custom cream_api.ini has been written to {game_install_dir}.")
-        print(f"Installation complete. Set launch options in Steam: 'sh ./cream.sh %command%' for {game_name}.")
+            print(f"\n{GREEN}Installation complete!{RESET}")
-    else:
+            print(f"\n{YELLOW}Set launch options in Steam:{RESET}")
-        print("Failed to download the files needed for installation.")
+            print(f"{GREEN}'sh ./cream.sh %command%'{RESET} for {game_name}")
+        else:
+            log_error("Failed to download the files needed for installation.")
+    except Exception as e:
+        log_error(f"Failed to install files for {game_name}: {e}")
+
+def uninstall_creamlinux(install_path, game_name):
+    YELLOW = '\033[93m'
+    GREEN = '\033[92m'
+    RESET = '\033[0m'
+    try:
+        files_to_remove = ['cream.sh', 'cream_api.ini', 'cream_api.so', 'lib32Creamlinux.so', 'lib64Creamlinux.so']
+        for file in files_to_remove:
+            file_path = os.path.join(install_path, file)
+            if os.path.exists(file_path):
+                os.remove(file_path)
+        print(f"\n{GREEN}Successfully uninstalled CreamLinux from {game_name}{RESET}")
+        print(f"\n{YELLOW}Make sure to remove{RESET} {GREEN}'sh ./cream.sh %command%'{RESET} {YELLOW}from launch options{RESET}")
+    except Exception as e:
+        log_error(f"Failed to uninstall CreamLinux: {e}")
 
 def main():
-    library_folders = find_steam_library_folders()
+    parser = argparse.ArgumentParser(description="Steam DLC Fetcher")
-    games = find_steam_apps(library_folders)
+    parser.add_argument("--manual", metavar='steamapps_path', help="Sets the steamapps path for faster operation", required=False)
-    if games:
+    parser.add_argument("--debug", action="store_true", help="Enable debug logging")
-        print("Select the game for which you want to fetch DLCs:")
+    args = parser.parse_args()
-        games_list = list(games.items())
+
-        GREEN = '\033[92m'
+    setup_logging(args.debug)
-        RESET = '\033[0m'
+    show_header(app_version, args.debug)
-        for idx, (app_id, (name, cream_status, _)) in enumerate(games_list, 1):
+
-            if cream_status:
+    try:
-                print(f"{idx}. {GREEN}{name} (App ID: {app_id}) - Cream installed{RESET}")
+        library_folders = find_steam_library_folders(args.manual)
+        if not library_folders:
+            print("Falling back to Manual Method since no library folder was found")
+            steamapps_path = input("Steamapps Path: ")
+            if len(steamapps_path) > 3:
+                library_folders = [steamapps_path]
             else:
-                print(f"{idx}. {name} (App ID: {app_id})")
+                print("Invalid path! Closing the program...")
+                return
 
-        choice = int(input("Enter the number of the game: ")) - 1
+        games = find_steam_apps(library_folders)
-        selected_app_id, (selected_game_name, _, selected_install_dir) = games_list[choice]
+        if games:
-        print(f"You selected: {selected_game_name} (App ID: {selected_app_id})")
+            print("\nSelect the game for which you want to fetch DLCs:")
+            games_list = list(games.items())
+            GREEN = '\033[92m'
+            RESET = '\033[0m'
             
-        dlcs = fetch_dlc_details(selected_app_id)
+            for idx, (app_id, (name, cream_status, _)) in enumerate(games_list, 1):
-        print("DLC IDs found:", [dlc['appid'] for dlc in dlcs])  # Only print app IDs for clarity
+                status = []
-        install_files(selected_app_id, selected_install_dir, dlcs, selected_game_name)
+                if cream_status:
-    else:
+                    status.append(f"{GREEN}Cream installed{RESET}")
-        print("No Steam games found on this computer or connected drives.")
+                
+                status_str = f" ({', '.join(status)})" if status else ""
+                print(f"{idx}. {name} (App ID: {app_id}){status_str}")
+
+            choice = int(input("\nEnter the number of the game: ")) - 1
+            if 0 <= choice < len(games_list):
+                selected_app_id, (selected_game_name, cream_status, selected_install_dir) = games_list[choice]
+                
+                if cream_status:
+                    print("\nCreamLinux is installed. What would you like to do?")
+                    print("1. Fetch DLC IDs")
+                    print("2. Uninstall CreamLinux")
+                    action = input("Enter your choice (1-2): ")
+                    
+                    if action == "1":
+                        print(f"\nSelected: {selected_game_name} (App ID: {selected_app_id})")
+                        dlcs = fetch_dlc_details(selected_app_id)
+                        if dlcs:
+                            install_files(selected_app_id, selected_install_dir, dlcs, selected_game_name)
+                        else:
+                            print("No DLCs found for the selected game.")
+                    elif action == "2":
+                        uninstall_creamlinux(selected_install_dir, selected_game_name)
+                    else:
+                        print("Invalid choice.")
+                else:
+                    print(f"\nSelected: {selected_game_name} (App ID: {selected_app_id})")
+                    dlcs = fetch_dlc_details(selected_app_id)
+                    if dlcs:
+                        install_files(selected_app_id, selected_install_dir, dlcs, selected_game_name)
+                    else:
+                        print("No DLCs found for the selected game.")
+            else:
+                print("Invalid selection.")
+        else:
+            print("No Steam games found on this computer or connected drives.")
+    except Exception as e:
+        logging.exception("An error occurred:")
+        log_error(f"An error occurred: {e}")
 
 if __name__ == "__main__":
     main()