FFMpeg-Compressor: Show only completed tasks

FFMpeg-Compressor/README.md

@@ -36,5 +36,5 @@ Python utility uses ffmpeg to compress Visual Novel Resources
 * [x] Recreate whole game directory with compressed files
 * [x] Cross-platform (Easy Windows usage and binaries, macOS binaries)
 * [x] Use ffmpeg python bindings instead of cli commands
+* [x] Multithread
 * [ ] Reorganize code
-* [ ] Multithread

FFMpeg-Compressor/ffmpeg-comp.toml

@@ -4,6 +4,7 @@ ForceCompress = false
 MimicMode = true
 HideErrors = true
 WebpRGBA = true
+Workers = 16
 
 [AUDIO]
 Extension = "opus"

FFMpeg-Compressor/main.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
-from modules import configloader
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from modules.configloader import config
 from modules import compressor
 from modules import printer
 from modules import utils
@@ -10,70 +11,49 @@ import sys
 import os
 
 
-def get_file_type(filename):
-    audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma']
-    image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png']
-    video_ext = ['.3gp' '.amv', '.avi', '.gif', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv',
-                 '.webm', '.ogv']
-
-    if os.path.splitext(filename)[1] in audio_ext:
-        return "audio"
-    elif os.path.splitext(filename)[1] in image_ext:
-        return "image"
-    elif os.path.splitext(filename)[1] in video_ext:
-        return "video"
+def get_args():
+    try:
+        if sys.argv[1][len(sys.argv[1])-1] == "/":
+            path = sys.argv[1][:len(sys.argv[1])-1]
         else:
-        return "unknown"
+            path = sys.argv[1]
+        return path
+    except IndexError:
+        print(utils.help_message())
+        exit()
+
+
+def compress_worker(folder, file, target_folder, req_folder):
+    if os.path.isfile(f'{folder}/{file}'):
+        compressor.compress_file(folder, file, target_folder, req_folder)
 
 
 if __name__ == "__main__":
     start_time = datetime.now()
     printer.win_ascii_esc()
-    try:
-        if sys.argv[1][len(sys.argv[1])-1] == "/":
-            arg_path = sys.argv[1][:len(sys.argv[1])-1]
-        else:
-            arg_path = sys.argv[1]
-    except IndexError:
-        print(utils.help_message())
-        exit()
+    req_folder = os.path.abspath(get_args())
 
-    orig_folder = os.path.abspath(arg_path)
-    printer.orig_folder = os.path.abspath(arg_path)
+    printer.bar_init(req_folder)
 
-    printer.bar_init(orig_folder)
-
-    if os.path.exists(f"{orig_folder}_compressed"):
-        shutil.rmtree(f"{orig_folder}_compressed")
+    if os.path.exists(f"{req_folder}_compressed"):
+        shutil.rmtree(f"{req_folder}_compressed")
 
     printer.info("Creating folders...")
-    for folder, folders, files in os.walk(orig_folder):
-        if not os.path.exists(folder.replace(orig_folder, f"{orig_folder}_compressed")):
-            os.mkdir(folder.replace(orig_folder, f"{orig_folder}_compressed"))
+    for folder, folders, files in os.walk(req_folder):
+        if not os.path.exists(folder.replace(req_folder, f"{req_folder}_compressed")):
+            os.mkdir(folder.replace(req_folder, f"{req_folder}_compressed"))
 
-        printer.info(f"Compressing \"{folder.replace(orig_folder, orig_folder.split('/').pop())}\" folder...")
-        target_folder = folder.replace(orig_folder, f"{orig_folder}_compressed")
-        for file in os.listdir(folder):
-            if os.path.isfile(f'{folder}/{file}'):
-                match get_file_type(file):
-                    case "audio":
-                        comp_file = compressor.compress_audio(folder, file, target_folder,
-                                                              configloader.config['AUDIO']['Extension'])
-                    case "image":
-                        comp_file = compressor.compress_image(folder, file, target_folder,
-                                                              configloader.config['IMAGE']['Extension'])
-                    case "video":
-                        comp_file = compressor.compress_video(folder, file, target_folder,
-                                                              configloader.config['VIDEO']['Extension'])
-                    case "unknown":
-                        comp_file = compressor.compress(folder, file, target_folder)
+        printer.info(f"Compressing \"{folder.replace(req_folder, req_folder.split('/').pop())}\" folder...")
+        target_folder = folder.replace(req_folder, f"{req_folder}_compressed")
 
-                if configloader.config['FFMPEG']['MimicMode']:
-                    try:
-                        os.rename(comp_file, f'{folder}/{file}'.replace(orig_folder, f"{orig_folder}_compressed"))
-                    except FileNotFoundError:
-                        pass
+        with ThreadPoolExecutor(max_workers=config["FFMPEG"]["Workers"]) as executor:
+            futures = [
+                executor.submit(compress_worker, folder, file, target_folder, req_folder)
+                for file in files
+            ]
+            for future in as_completed(futures):
+                future.result()
 
-    utils.get_compression_status(orig_folder)
+    utils.get_compression_status(req_folder)
     utils.sys_pause()
     print(f"Time taken: {datetime.now() - start_time}")

FFMpeg-Compressor/modules/compressor.py

@@ -7,6 +7,22 @@ from ffmpeg import FFmpeg, FFmpegError
 import os
 
 
+def get_file_type(filename):
+    audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma']
+    image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png']
+    video_ext = ['.3gp' '.amv', '.avi', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv',
+                 '.webm', '.ogv']
+
+    if os.path.splitext(filename)[1] in audio_ext:
+        return "audio"
+    elif os.path.splitext(filename)[1] in image_ext:
+        return "image"
+    elif os.path.splitext(filename)[1] in video_ext:
+        return "video"
+    else:
+        return "unknown"
+
+
 def has_transparency(img):
     if img.info.get("transparency", None) is not None:
         return True
@@ -25,13 +41,12 @@ def has_transparency(img):
 
 def compress_audio(folder, file, target_folder, extension):
     bitrate = configloader.config['AUDIO']['BitRate']
-
-    printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}")
     try:
         (FFmpeg()
          .input(f'{folder}/{file}')
+         .option("hide_banner")
          .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'),
-                 {"b:a": bitrate})
+                 {"b:a": bitrate, "loglevel": "error"})
          .execute()
          )
     except FFmpegError as e:
@@ -39,6 +54,7 @@ def compress_audio(folder, file, target_folder, extension):
         utils.errors_count += 1
         if not configloader.config['FFMPEG']['HideErrors']:
             printer.error(f"File {file} can't be processed! Error: {e}")
+    printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}")
     return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'
 
 
@@ -47,14 +63,16 @@ def compress_video(folder, file, target_folder, extension):
         codec = configloader.config['VIDEO']['Codec']
         crf = configloader.config['VIDEO']['CRF']
 
-        printer.files(file, os.path.splitext(file)[0], extension, codec)
         try:
             (FFmpeg()
              .input(f'{folder}/{file}')
+             .option("hide_banner")
+             .option("hwaccel", "auto")
              .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'),
-                     {"codec:v": codec, "v:b": 0}, crf=crf)
+                     {"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf)
              .execute()
              )
+            printer.files(file, os.path.splitext(file)[0], extension, codec)
         except FFmpegError as e:
             utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
             utils.errors_count += 1
@@ -67,7 +85,6 @@ def compress_video(folder, file, target_folder, extension):
 
 def compress_image(folder, file, target_folder, extension):
     quality = configloader.config['IMAGE']['Quality']
-    printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%")
     try:
         image = Image.open(f'{folder}/{file}')
 
@@ -77,6 +94,9 @@ def compress_image(folder, file, target_folder, extension):
                 printer.warning(f"{file} has transparency. Changing to fallback...")
                 extension = configloader.config['IMAGE']['FallBackExtension']
 
+        if has_transparency(image):
+            image.convert('RGBA')
+
         res_downscale = configloader.config['IMAGE']['ResDownScale']
         if res_downscale != 1:
             width, height = image.size
@@ -88,6 +108,7 @@ def compress_image(folder, file, target_folder, extension):
                    lossless=configloader.config['IMAGE']['Lossless'],
                    quality=quality,
                    minimize_size=True)
+        printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%")
     except Exception as e:
         utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
         utils.errors_count += 1
@@ -113,3 +134,27 @@ def compress(folder, file, target_folder):
     else:
         utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
     return f'{target_folder}/{file}'
+
+
+def compress_file(_dir, filename, target_dir, source):
+    match get_file_type(filename):
+        case "audio":
+            comp_file = compress_audio(_dir, filename, target_dir,
+                                       configloader.config['AUDIO']['Extension'])
+        case "image":
+            comp_file = compress_image(_dir, filename, target_dir,
+                                       configloader.config['IMAGE']['Extension'])
+        case "video":
+            comp_file = compress_video(_dir, filename, target_dir,
+                                       configloader.config['VIDEO']['Extension'])
+        case "unknown":
+            comp_file = compress(_dir, filename, target_dir)
+
+    if configloader.config['FFMPEG']['MimicMode']:
+        try:
+            os.rename(comp_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed"))
+        except FileNotFoundError:
+            pass
+
+    printer.bar.update()
+    printer.bar.next()

FFMpeg-Compressor/modules/printer.py

@@ -4,42 +4,46 @@ import sys
 import os
 
 
-# Fill whole string with spaces for cleaning progress bar
-def clean_str(string):
-    return string + " " * (os.get_terminal_size().columns - len(string))
-
-
-def info(string):
-    print(clean_str(f"\r\033[100mI {string}\033[49m"))
-
-
-def warning(string):
-    print(clean_str(f"\r\033[93mW\033[0m {string}\033[49m"))
-
-
-def error(string):
-    print(clean_str(f"\r\033[31mE\033[0m {string}\033[49m"))
-
-
 def bar_init(folder):
     file_count = 0
     for folder, folders, file in os.walk(folder):
         file_count += len(file)
     global bar
     bar = IncrementalBar('Compressing', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)')
+    bar.update()
+
+
+def bar_print(string):
+    print(string)
+    bar.update()
+
+
+# Fill whole string with spaces for cleaning progress bar
+def clean_str(string):
+    return string + " " * (os.get_terminal_size().columns - len(string))
+
+
+def info(string):
+    bar_print(clean_str(f"\r\033[100m- {string}\033[49m"))
+
+
+def warning(string):
+    bar_print(clean_str(f"\r\033[93m!\033[0m {string}\033[49m"))
+
+
+def error(string):
+    bar_print(clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m"))
 
 
 def files(source, dest, dest_ext, comment):
     source_ext = os.path.splitext(source)[1]
     source_name = os.path.splitext(source)[0]
 
-    print(clean_str(f"\r* \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m"))
-    bar.next()
+    bar_print(clean_str(f"\r\033[0;32m\u2713\033[0m \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m"))
 
 
 def unknown_file(file):
-    print(clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)"))
-    bar.next()
+    bar_print(clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)"))
 
 
 def win_ascii_esc():

FFMpeg-Compressor/modules/utils.py

@@ -25,7 +25,7 @@ def get_compression(orig, comp):
         orig = get_dir_size(orig, processed_files)
         comp = get_dir_size(comp, processed_files)
 
-        print(f"Result: {orig/1024/1024:.2f}MB -> {comp/1024/1024:.2f}MB Δ {(orig - comp)/1024/1024:.2f}MB")
+        print(f"\nResult: {orig/1024/1024:.2f}MB -> {comp/1024/1024:.2f}MB ({(comp - orig)/1024/1024:.2f}MB)")
     except ZeroDivisionError:
         printer.warning("Nothing compressed!")
 
@@ -55,7 +55,7 @@ def get_compression_status(orig_folder):
 
 def add_unprocessed_file(orig_folder, new_folder):
     if configloader.config['FFMPEG']['CopyUnprocessed']:
-        filename = orig_folder.split().pop()
+        filename = orig_folder.split("/").pop()
         copyfile(orig_folder, new_folder)
         printer.info(f"File {filename} copied to compressed folder.")
 

RenPy-Unpacker/unpack.rpy

@@ -2,8 +2,13 @@ init 4 python:
     import os
 
     for asset in renpy.list_files():
-        if os.path.splitext(asset)[1] != ".rpa" and asset != "unpack.rpyc":
-            output = "unpack/game/" + asset
+        if os.path.splitext(asset)[1] != ".rpa" and not asset.count("unpack.rpy"): # Ignore .rpa and script itself
+            if renpy.macintosh:
+                game_path = os.path.expanduser('~') + "/" + config.name # Unpack assets to home folder (on mac you cant get cwd)
+                output = game_path + "/game/" + asset
+            else:
+                output = "unpack/game/" + asset # Unpack assets to game folder
+
             if not os.path.exists(os.path.dirname(output)):
                 os.makedirs(os.path.dirname(output))