diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index 496d47c..40b37d8 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -36,5 +36,5 @@ Python utility uses ffmpeg to compress Visual Novel Resources * [x] Recreate whole game directory with compressed files * [x] Cross-platform (Easy Windows usage and binaries, macOS binaries) * [x] Use ffmpeg python bindings instead of cli commands -* [ ] Reorganize code -* [ ] Multithread \ No newline at end of file +* [x] Multithread +* [ ] Reorganize code \ No newline at end of file diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index 7005024..15551da 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -4,6 +4,7 @@ ForceCompress = false MimicMode = true HideErrors = true WebpRGBA = true +Workers = 16 [AUDIO] Extension = "opus" diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py index 2721629..7c05dc5 100755 --- a/FFMpeg-Compressor/main.py +++ b/FFMpeg-Compressor/main.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -from modules import configloader +from concurrent.futures import ThreadPoolExecutor, as_completed +from modules.configloader import config from modules import compressor from modules import printer from modules import utils @@ -10,70 +11,49 @@ import sys import os -def get_file_type(filename): - audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] - image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'] - video_ext = ['.3gp' '.amv', '.avi', '.gif', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', - '.webm', '.ogv'] +def get_args(): + try: + if sys.argv[1][len(sys.argv[1])-1] == "/": + path = sys.argv[1][:len(sys.argv[1])-1] + else: + path = sys.argv[1] + return path + except IndexError: + print(utils.help_message()) + exit() - if os.path.splitext(filename)[1] in audio_ext: - return "audio" - elif os.path.splitext(filename)[1] in image_ext: - return "image" - elif os.path.splitext(filename)[1] in video_ext: - return "video" - else: - return "unknown" + +def compress_worker(folder, file, target_folder, req_folder): + if os.path.isfile(f'{folder}/{file}'): + compressor.compress_file(folder, file, target_folder, req_folder) if __name__ == "__main__": start_time = datetime.now() printer.win_ascii_esc() - try: - if sys.argv[1][len(sys.argv[1])-1] == "/": - arg_path = sys.argv[1][:len(sys.argv[1])-1] - else: - arg_path = sys.argv[1] - except IndexError: - print(utils.help_message()) - exit() + req_folder = os.path.abspath(get_args()) - orig_folder = os.path.abspath(arg_path) - printer.orig_folder = os.path.abspath(arg_path) + printer.bar_init(req_folder) - printer.bar_init(orig_folder) - - if os.path.exists(f"{orig_folder}_compressed"): - shutil.rmtree(f"{orig_folder}_compressed") + if os.path.exists(f"{req_folder}_compressed"): + shutil.rmtree(f"{req_folder}_compressed") printer.info("Creating folders...") - for folder, folders, files in os.walk(orig_folder): - if not os.path.exists(folder.replace(orig_folder, f"{orig_folder}_compressed")): - os.mkdir(folder.replace(orig_folder, f"{orig_folder}_compressed")) + for folder, folders, files in os.walk(req_folder): + if not os.path.exists(folder.replace(req_folder, f"{req_folder}_compressed")): + os.mkdir(folder.replace(req_folder, f"{req_folder}_compressed")) - printer.info(f"Compressing \"{folder.replace(orig_folder, orig_folder.split('/').pop())}\" folder...") - target_folder = folder.replace(orig_folder, f"{orig_folder}_compressed") - for file in os.listdir(folder): - if os.path.isfile(f'{folder}/{file}'): - match get_file_type(file): - case "audio": - comp_file = compressor.compress_audio(folder, file, target_folder, - configloader.config['AUDIO']['Extension']) - case "image": - comp_file = compressor.compress_image(folder, file, target_folder, - configloader.config['IMAGE']['Extension']) - case "video": - comp_file = compressor.compress_video(folder, file, target_folder, - configloader.config['VIDEO']['Extension']) - case "unknown": - comp_file = compressor.compress(folder, file, target_folder) + printer.info(f"Compressing \"{folder.replace(req_folder, req_folder.split('/').pop())}\" folder...") + target_folder = folder.replace(req_folder, f"{req_folder}_compressed") - if configloader.config['FFMPEG']['MimicMode']: - try: - os.rename(comp_file, f'{folder}/{file}'.replace(orig_folder, f"{orig_folder}_compressed")) - except FileNotFoundError: - pass + with ThreadPoolExecutor(max_workers=config["FFMPEG"]["Workers"]) as executor: + futures = [ + executor.submit(compress_worker, folder, file, target_folder, req_folder) + for file in files + ] + for future in as_completed(futures): + future.result() - utils.get_compression_status(orig_folder) + utils.get_compression_status(req_folder) utils.sys_pause() - print(f"Time taken: {datetime.now() - start_time}") + print(f"Time taken: {datetime.now() - start_time}") \ No newline at end of file diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 7260787..728953c 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -7,6 +7,22 @@ from ffmpeg import FFmpeg, FFmpegError import os +def get_file_type(filename): + audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] + image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'] + video_ext = ['.3gp' '.amv', '.avi', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', + '.webm', '.ogv'] + + if os.path.splitext(filename)[1] in audio_ext: + return "audio" + elif os.path.splitext(filename)[1] in image_ext: + return "image" + elif os.path.splitext(filename)[1] in video_ext: + return "video" + else: + return "unknown" + + def has_transparency(img): if img.info.get("transparency", None) is not None: return True @@ -25,13 +41,12 @@ def has_transparency(img): def compress_audio(folder, file, target_folder, extension): bitrate = configloader.config['AUDIO']['BitRate'] - - printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}") try: (FFmpeg() .input(f'{folder}/{file}') + .option("hide_banner") .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), - {"b:a": bitrate}) + {"b:a": bitrate, "loglevel": "error"}) .execute() ) except FFmpegError as e: @@ -39,6 +54,7 @@ def compress_audio(folder, file, target_folder, extension): utils.errors_count += 1 if not configloader.config['FFMPEG']['HideErrors']: printer.error(f"File {file} can't be processed! Error: {e}") + printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}") return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' @@ -47,14 +63,16 @@ def compress_video(folder, file, target_folder, extension): codec = configloader.config['VIDEO']['Codec'] crf = configloader.config['VIDEO']['CRF'] - printer.files(file, os.path.splitext(file)[0], extension, codec) try: (FFmpeg() .input(f'{folder}/{file}') + .option("hide_banner") + .option("hwaccel", "auto") .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), - {"codec:v": codec, "v:b": 0}, crf=crf) + {"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) .execute() ) + printer.files(file, os.path.splitext(file)[0], extension, codec) except FFmpegError as e: utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') utils.errors_count += 1 @@ -67,7 +85,6 @@ def compress_video(folder, file, target_folder, extension): def compress_image(folder, file, target_folder, extension): quality = configloader.config['IMAGE']['Quality'] - printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") try: image = Image.open(f'{folder}/{file}') @@ -77,6 +94,9 @@ def compress_image(folder, file, target_folder, extension): printer.warning(f"{file} has transparency. Changing to fallback...") extension = configloader.config['IMAGE']['FallBackExtension'] + if has_transparency(image): + image.convert('RGBA') + res_downscale = configloader.config['IMAGE']['ResDownScale'] if res_downscale != 1: width, height = image.size @@ -88,6 +108,7 @@ def compress_image(folder, file, target_folder, extension): lossless=configloader.config['IMAGE']['Lossless'], quality=quality, minimize_size=True) + printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") except Exception as e: utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') utils.errors_count += 1 @@ -113,3 +134,27 @@ def compress(folder, file, target_folder): else: utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') return f'{target_folder}/{file}' + + +def compress_file(_dir, filename, target_dir, source): + match get_file_type(filename): + case "audio": + comp_file = compress_audio(_dir, filename, target_dir, + configloader.config['AUDIO']['Extension']) + case "image": + comp_file = compress_image(_dir, filename, target_dir, + configloader.config['IMAGE']['Extension']) + case "video": + comp_file = compress_video(_dir, filename, target_dir, + configloader.config['VIDEO']['Extension']) + case "unknown": + comp_file = compress(_dir, filename, target_dir) + + if configloader.config['FFMPEG']['MimicMode']: + try: + os.rename(comp_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed")) + except FileNotFoundError: + pass + + printer.bar.update() + printer.bar.next() diff --git a/FFMpeg-Compressor/modules/printer.py b/FFMpeg-Compressor/modules/printer.py index 5b63a56..0fdce24 100644 --- a/FFMpeg-Compressor/modules/printer.py +++ b/FFMpeg-Compressor/modules/printer.py @@ -4,42 +4,46 @@ import sys import os -# Fill whole string with spaces for cleaning progress bar -def clean_str(string): - return string + " " * (os.get_terminal_size().columns - len(string)) - - -def info(string): - print(clean_str(f"\r\033[100mI {string}\033[49m")) - - -def warning(string): - print(clean_str(f"\r\033[93mW\033[0m {string}\033[49m")) - - -def error(string): - print(clean_str(f"\r\033[31mE\033[0m {string}\033[49m")) - - def bar_init(folder): file_count = 0 for folder, folders, file in os.walk(folder): file_count += len(file) global bar bar = IncrementalBar('Compressing', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)') + bar.update() + + +def bar_print(string): + print(string) + bar.update() + + +# Fill whole string with spaces for cleaning progress bar +def clean_str(string): + return string + " " * (os.get_terminal_size().columns - len(string)) + + +def info(string): + bar_print(clean_str(f"\r\033[100m- {string}\033[49m")) + + +def warning(string): + bar_print(clean_str(f"\r\033[93m!\033[0m {string}\033[49m")) + + +def error(string): + bar_print(clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m")) def files(source, dest, dest_ext, comment): source_ext = os.path.splitext(source)[1] source_name = os.path.splitext(source)[0] - print(clean_str(f"\r* \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m")) - bar.next() + bar_print(clean_str(f"\r\033[0;32m\u2713\033[0m \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m")) def unknown_file(file): - print(clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)")) - bar.next() + bar_print(clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)")) def win_ascii_esc(): diff --git a/FFMpeg-Compressor/modules/utils.py b/FFMpeg-Compressor/modules/utils.py index f1a3256..907f9b1 100644 --- a/FFMpeg-Compressor/modules/utils.py +++ b/FFMpeg-Compressor/modules/utils.py @@ -25,7 +25,7 @@ def get_compression(orig, comp): orig = get_dir_size(orig, processed_files) comp = get_dir_size(comp, processed_files) - print(f"Result: {orig/1024/1024:.2f}MB -> {comp/1024/1024:.2f}MB Δ {(orig - comp)/1024/1024:.2f}MB") + print(f"\nResult: {orig/1024/1024:.2f}MB -> {comp/1024/1024:.2f}MB ({(comp - orig)/1024/1024:.2f}MB)") except ZeroDivisionError: printer.warning("Nothing compressed!") @@ -55,7 +55,7 @@ def get_compression_status(orig_folder): def add_unprocessed_file(orig_folder, new_folder): if configloader.config['FFMPEG']['CopyUnprocessed']: - filename = orig_folder.split().pop() + filename = orig_folder.split("/").pop() copyfile(orig_folder, new_folder) printer.info(f"File {filename} copied to compressed folder.") diff --git a/RenPy-Unpacker/unpack.rpy b/RenPy-Unpacker/unpack.rpy index 005e855..905102f 100644 --- a/RenPy-Unpacker/unpack.rpy +++ b/RenPy-Unpacker/unpack.rpy @@ -2,8 +2,13 @@ init 4 python: import os for asset in renpy.list_files(): - if os.path.splitext(asset)[1] != ".rpa" and asset != "unpack.rpyc": - output = "unpack/game/" + asset + if os.path.splitext(asset)[1] != ".rpa" and not asset.count("unpack.rpy"): # Ignore .rpa and script itself + if renpy.macintosh: + game_path = os.path.expanduser('~') + "/" + config.name # Unpack assets to home folder (on mac you cant get cwd) + output = game_path + "/game/" + asset + else: + output = "unpack/game/" + asset # Unpack assets to game folder + if not os.path.exists(os.path.dirname(output)): os.makedirs(os.path.dirname(output))