fix: only do 8 jobs at once in update & verify

Prevent lag which can lead to unusable system (especially when using hdiffpatch)

Also print more

setup.py

@@ -10,12 +10,12 @@ README = (HERE / "README.md").read_text()
 
 setup(
     name='worthless',
-    version='2.2.13',
+    version='2.2.14',
     packages=['worthless', 'worthless.classes', 'worthless.classes.launcher', 'worthless.classes.installer'],
     url='https://git.froggi.es/tretrauit/worthless-launcher',
     license='MIT License',
     author='tretrauit',
-    author_email='tretrauit@cachyos.org',
+    author_email='tretrauit@gmail.org',
     description='A worthless CLI launcher written in Python.',
     long_description=README,
     long_description_content_type="text/markdown",

worthless/__init__.py

@@ -4,4 +4,4 @@ Launcher = launcher.Launcher
 Installer = installer.Installer
 
 
-__version__ = "2.2.13"
+__version__ = "2.2.14"

worthless/installer.py

@@ -56,7 +56,7 @@ def calculate_md5(file_to_calculate):
         return ""
     with file_to_calculate.open("rb") as f:
         file_hash = hashlib.md5()
-        while chunk := f.read(8192):
+        while chunk := f.read(1024 * 1024):
             file_hash.update(chunk)
         return file_hash.hexdigest()
 
@@ -181,7 +181,7 @@ class Installer:
         Path(self.temp_path).mkdir(parents=True, exist_ok=True)
         config_file = self._gamedir.joinpath("config.ini")
         self._config_file = config_file
-        self._download_chunk = 8192
+        self._download_chunk = 1024 * 1024
         self._overseas = overseas
         self._version = None
         self._launcher = Launcher(self._gamedir, overseas=self._overseas)
@@ -347,8 +347,10 @@ class Installer:
             hdifffiles = []
             for x in (await asyncio.to_thread(archive.read, "hdifffiles.txt")).decode().split("\n"):
                 if x:
-                    hdifffiles.append(json.loads(x)["remoteName"])
+                    hdifffiles.append(json.loads(x.strip())["remoteName"])
             patch_jobs = []
+            cur_jobs = []
+            count = 0
             for file in hdifffiles:
                 current_game_file = self._gamedir.joinpath(file)
                 if not await current_game_file.exists():
@@ -381,9 +383,19 @@ class Installer:
                     await old_file.rename(old_file.with_suffix(old_suffix))
 
                 files.remove(patch_file)
-                patch_jobs.append(extract_and_patch(current_game_file, patch_file))
+                # Limit to 8 process running so it doesn't hang the PC.
+                if count == 7:
+                    patch_jobs.append(cur_jobs)
+                    cur_jobs = []
+                    count = 0
+                cur_jobs.append(extract_and_patch(current_game_file, patch_file))
+                count += 1
+
+            # The last list may have count < 7 and the above code will not add them
+            patch_jobs.append(cur_jobs)
+            for jobs in patch_jobs:
+                await asyncio.gather(*jobs)
 
-            await asyncio.gather(*patch_jobs)
         except Exception as e:
             print(f"Error while reading hdifffiles.txt: {e}")
 
@@ -411,7 +423,7 @@ class Installer:
         if not game_archive.exists():
             raise FileNotFoundError(f"Update archive {game_archive} not found")
 
-        self._update(game_archive=game_archive)
+        await self._update(game_archive=game_archive)
         # Update game version on local variable.
         self._version = await self.get_game_version()
         self.set_version_config()
@@ -544,21 +556,33 @@ class Installer:
         contents = await pkg_version.read_text()
 
         async def verify_file(file_to_verify, md5):
+            print("Verifying file:", file_to_verify)
             file_md5 = await asyncio.to_thread(calculate_md5, file_to_verify)
             if file_md5 == md5:
                 return None
             if ignore_mismatch:
+                # print(f"MD5 does not match for {file_to_verify}, expected md5: {md5}, actual md5: {file_md5}")
                 return file_to_verify, md5, file_md5
             raise ValueError(f"MD5 does not match for {file_to_verify}, expected md5: {md5}, actual md5: {file_md5}")
 
         verify_jobs = []
+        cur_jobs = []
+        count = 0
         for content in contents.split("\r\n"):
             if not content.strip():
                 continue
+            if count >= 7:
+                verify_jobs.append(cur_jobs)
+                cur_jobs = []
+                count = 0
             info = json.loads(content)
-            verify_jobs.append(verify_file(self._gamedir.joinpath(info["remoteName"]), info["md5"]))
+            cur_jobs.append(verify_file(self._gamedir.joinpath(info["remoteName"]), info["md5"]))
+            count += 1
 
-        verify_result = await asyncio.gather(*verify_jobs)
+        verify_jobs.append(cur_jobs)
+        verify_result = []
+        for jobs in verify_jobs:
+            verify_result.extend(await asyncio.gather(*jobs))
         failed_files = []
         for file in verify_result:
             if file is not None: