From a69b17c624af1df1a9256a7ff5b577380d7b245e Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Tue, 3 Sep 2024 22:54:59 +0300 Subject: [PATCH 01/10] vnrecode: improve naming, fix mimic mode errors --- vnrecode/compress.py | 65 ++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 7dbc6fc..25d3177 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -45,54 +45,56 @@ class Compress: self.printer = printer self.utils = utils - def audio(self, folder, file, target_folder, extension): - bitrate = self.params.audio_bitrate + def audio(self, in_dir, file, out_dir, extension): + bit_rate = self.params.audio_bitrate + out_file = self.utils.check_duplicates(f'{out_dir}/{os.path.splitext(file)[0]}.{extension}') try: (FFmpeg() - .input(f'{folder}/{file}') + .input(f'{in_dir}/{file}') .option("hide_banner") - .output(self.utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), - {"b:a": bitrate, "loglevel": "error"}) + .output(out_file,{"b:a": bit_rate, "loglevel": "error"}) .execute() ) except FFmpegError as e: - self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') self.utils.errors += 1 if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") - self.printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}") - return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' + self.printer.files(file, os.path.splitext(file)[0], extension, f"{bit_rate}") + return out_file - def video(self, folder, file, target_folder, extension): + def video(self, in_dir, file, out_dir, extension): if not self.params.video_skip: + out_file = self.utils.check_duplicates(f'{out_dir}/{os.path.splitext(file)[0]}.{extension}') codec = self.params.video_codec crf = self.params.video_crf try: (FFmpeg() - .input(f'{folder}/{file}') + .input(f'{in_dir}/{file}') .option("hide_banner") .option("hwaccel", "auto") - .output(self.utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), - {"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) + .output(out_file,{"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) .execute() ) self.printer.files(file, os.path.splitext(file)[0], extension, codec) except FFmpegError as e: - self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') self.utils.errors += 1 if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") + return out_file else: - self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' + self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') + return f'{out_dir}/{os.path.splitext(file)[0]}.{extension}' - def image(self, folder, file, target_folder, extension): + def image(self, in_dir, file, out_dir, extension): quality = self.params.image_quality + out_file = self.utils.check_duplicates(f"{out_dir}/{os.path.splitext(file)[0]}.{extension}") try: - image = Image.open(f'{folder}/{file}') + image = Image.open(f'{in_dir}/{file}') if (extension == "jpg" or extension == "jpeg" or (extension == "webp" and not self.params.webp_rgba)): @@ -109,37 +111,39 @@ class Compress: new_size = (int(width / res_downscale), int(height / res_downscale)) image = image.resize(new_size) - image.save(self.utils.check_duplicates(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}"), + image.save(out_file, optimize=True, lossless=self.params.image_lossless, quality=quality, minimize_size=True) self.printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") except Exception as e: - self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') self.utils.errors += 1 if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") - return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' + return out_file - def unknown(self, folder, file, target_folder): + def unknown(self, in_dir, filename, out_dir): if self.params.force_compress: - self.printer.unknown_file(file) + self.printer.unknown_file(filename) + out_file = self.utils.check_duplicates(f'{out_dir}/{filename}') try: (FFmpeg() - .input(f'{folder}/{file}') - .output(self.utils.check_duplicates(f'{target_folder}/{file}')) + .input(f'{in_dir}/{filename}') + .output(out_file) .execute() ) except FFmpegError as e: - self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.add_unprocessed_file(f'{in_dir}/{filename}', f'{out_dir}/{filename}') self.utils.errors += 1 if not self.params.hide_errors: - self.printer.error(f"File {file} can't be processed! Error: {e}") + self.printer.error(f"File {filename} can't be processed! Error: {e}") + return out_file else: - self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - return f'{target_folder}/{file}' + self.utils.add_unprocessed_file(f'{in_dir}/{filename}', f'{out_dir}/{filename}') + return f'{out_dir}/{filename}' def compress(self, _dir, filename, source, output): @@ -154,10 +158,7 @@ class Compress: out_file = self.unknown(_dir, filename, output) if self.params.mimic_mode: - try: - os.rename(out_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed")) - except FileNotFoundError: - self.printer.warning(f"File {out_file} failed to copy to out dir") + os.rename(out_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed")) self.printer.bar.update() self.printer.bar.next() From 03647d4b8446402d2a05b65afbdffc512750a449 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Tue, 3 Sep 2024 23:00:19 +0300 Subject: [PATCH 02/10] vnrecode: rewrite get_type method --- vnrecode/compress.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 25d3177..75202ee 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -8,19 +8,18 @@ class File: @staticmethod def get_type(filename): - audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] - image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'] - video_ext = ['.3gp' '.amv', '.avi', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', - '.webm', '.ogv'] - if os.path.splitext(filename)[1] in audio_ext: - return "audio" - elif os.path.splitext(filename)[1] in image_ext: - return "image" - elif os.path.splitext(filename)[1] in video_ext: - return "video" - else: - return "unknown" + extensions = { + "audio": ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'], + "image": ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'], + "video": ['.3gp' '.amv', '.avi', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', + '.webm', '.ogv'] + } + + for file_type in extensions: + if os.path.splitext(filename)[1] in extensions[file_type]: + return file_type + return "unknown" @staticmethod def has_transparency(img: Image) -> bool: From 92474b4aa41bbbf555fcf72cefed768758868cdc Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 01:21:59 +0300 Subject: [PATCH 03/10] vnrecode: improve duplications check for mt --- vnrecode/application.py | 1 + vnrecode/compress.py | 11 +++++------ vnrecode/utils.py | 27 +++++++++++++++++++++------ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/vnrecode/application.py b/vnrecode/application.py index d32a3cf..c7cb7cd 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -42,6 +42,7 @@ class Application: for future in as_completed(futures): future.result() + self.utils.print_duplicates() self.utils.get_compression_status(source) self.utils.sys_pause() print(f"Time taken: {datetime.now() - start_time}") \ No newline at end of file diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 75202ee..249061e 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -46,7 +46,7 @@ class Compress: def audio(self, in_dir, file, out_dir, extension): bit_rate = self.params.audio_bitrate - out_file = self.utils.check_duplicates(f'{out_dir}/{os.path.splitext(file)[0]}.{extension}') + out_file = self.utils.check_duplicates(in_dir, out_dir, f'{os.path.splitext(file)[0]}.{extension}') try: (FFmpeg() .input(f'{in_dir}/{file}') @@ -65,7 +65,7 @@ class Compress: def video(self, in_dir, file, out_dir, extension): if not self.params.video_skip: - out_file = self.utils.check_duplicates(f'{out_dir}/{os.path.splitext(file)[0]}.{extension}') + out_file = self.utils.check_duplicates(in_dir, out_dir, f'{os.path.splitext(file)[0]}.{extension}') codec = self.params.video_codec crf = self.params.video_crf @@ -91,7 +91,7 @@ class Compress: def image(self, in_dir, file, out_dir, extension): quality = self.params.image_quality - out_file = self.utils.check_duplicates(f"{out_dir}/{os.path.splitext(file)[0]}.{extension}") + out_file = self.utils.check_duplicates(in_dir, out_dir, f"{os.path.splitext(file)[0]}.{extension}") try: image = Image.open(f'{in_dir}/{file}') @@ -127,7 +127,7 @@ class Compress: def unknown(self, in_dir, filename, out_dir): if self.params.force_compress: self.printer.unknown_file(filename) - out_file = self.utils.check_duplicates(f'{out_dir}/{filename}') + out_file = self.utils.check_duplicates(in_dir, out_dir, filename) try: (FFmpeg() .input(f'{in_dir}/{filename}') @@ -144,7 +144,6 @@ class Compress: self.utils.add_unprocessed_file(f'{in_dir}/{filename}', f'{out_dir}/{filename}') return f'{out_dir}/{filename}' - def compress(self, _dir, filename, source, output): match File.get_type(filename): case "audio": @@ -157,7 +156,7 @@ class Compress: out_file = self.unknown(_dir, filename, output) if self.params.mimic_mode: - os.rename(out_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed")) + self.utils.mimic_rename(out_file, f'{_dir}/{filename}', source) self.printer.bar.update() self.printer.bar.next() diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 040da09..96b7db9 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -1,4 +1,5 @@ from shutil import copyfile +from glob import glob import sys import os @@ -8,6 +9,7 @@ class Utils: self.errors = 0 self.params = params self.printer = printer + self.duplicates = [] @staticmethod def sys_pause(): @@ -61,11 +63,24 @@ class Utils: copyfile(orig_folder, new_folder) self.printer.info(f"File {filename} copied to compressed folder.") - def check_duplicates(self, new_folder): - filename = new_folder.split().pop() - if os.path.exists(new_folder): + def check_duplicates(self, in_dir, out_dir, filename): + duplicates = glob(f"{in_dir}/{os.path.splitext(filename)[0]}.*") + if len(duplicates) > 1: + if filename in self.duplicates: + new_name = os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1] + return f"{out_dir}/{new_name}" + self.duplicates.append(filename) + return f"{out_dir}/{filename}" + + def print_duplicates(self): + for filename in self.duplicates: self.printer.warning( f'Duplicate file has been found! Check manually this files - "{filename}", ' - f'"{os.path.splitext(filename)[0] + "(copy)" + os.path.splitext(filename)[1]}"') - return os.path.splitext(new_folder)[0] + "(copy)" + os.path.splitext(new_folder)[1] - return new_folder + f'"{os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1]}"') + + @staticmethod + def mimic_rename(filename, target, source): + if filename.count("(vncopy)"): + target = os.path.splitext(target)[0] + "(vncopy)" + os.path.splitext(target)[1] + + os.rename(filename, target.replace(source, f"{source}_compressed")) \ No newline at end of file From 44c12a568885826539efe78dcbca0e5f758e4c4a Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 01:22:42 +0300 Subject: [PATCH 04/10] Update requirements --- pyproject.toml | 3 ++- requirements.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4d8aa85..f2c379f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,5 +27,6 @@ dependencies = [ "pillow-avif-plugin==1.4.3", "python-ffmpeg==2.0.12", "progress==1.6", - "colorama==0.4.6" + "colorama==0.4.6", + "argparse~=1.4.0" ] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index c3c571c..16bd1e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ Pillow==10.3.0 pillow-avif-plugin==1.4.3 python-ffmpeg==2.0.12 progress==1.6 -colorama==0.4.6 \ No newline at end of file +colorama==0.4.6 +argparse~=1.4.0 \ No newline at end of file From a9aeb5250622845a8fb36995a31befba3bdcd41f Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 01:52:18 +0300 Subject: [PATCH 05/10] vnrecode: improve cli parameters --- vnrecode/params.py | 82 +++++++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 37 deletions(-) diff --git a/vnrecode/params.py b/vnrecode/params.py index 789cac5..c8474c2 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -32,30 +32,7 @@ class Params: @classmethod def setup(cls) -> Self: - parser = ArgumentParser(prog="vnrecode", - description="Python utility to compress Visual Novel Resources" - ) - parser.add_argument("source", help="SourceDir") - parser.add_argument("-c", "--config", help="ConfigFile") - parser.add_argument("-u", type=bool, help="CopyUnprocessed", default=True) - parser.add_argument("-f", "--force", type=bool, help="ForceCompress", default=False) - parser.add_argument("-m", "--mimic", type=bool, help="MimicMode", default=True) - parser.add_argument("-s", "--silent", type=bool, help="HideErrors", default=True) - parser.add_argument("--webprgba", type=bool, help="WebpRGBA", default=True) - parser.add_argument("-j", "--jobs", type=int, help="Workers", default=16) - parser.add_argument("-ae", "--aext", help="Audio Extension", default="opus") - parser.add_argument("-ab", "--abit", help="Audio Bitrate", default="128k") - parser.add_argument("-id", "--idown", type=int, help="Image Downscale", default=1) - parser.add_argument("-ie", "--iext", help="Image Extension", default="avif") - parser.add_argument("-ife", "--ifallext", help="Image Fallback Extension", default="webp") - parser.add_argument("-il", "--ilossless", type=bool, help="Image Lossless", default=True) - parser.add_argument("-iq", "--iquality", type=int, help="Image Quality", default=100) - parser.add_argument("--vcrf", help="Video CRF", type=int, default=27) - parser.add_argument("-vs", "--vskip", help="Video Skip", default=False) - parser.add_argument("-ve", "--vext", help="Video Extension", default="webm") - parser.add_argument("-vc", "--vcodec", help="Video Codec", default="libvpx-vp9") - args = parser.parse_args() - + args = cls.get_args() if args.config is not None: if os.path.isfile(args.config): with open(args.config, "rb") as cfile: @@ -67,20 +44,20 @@ class Params: copy_unprocessed = config["FFMPEG"]["CopyUnprocessed"] if args.config else args.u force_compress = config["FFMPEG"]["ForceCompress"] if args.config else args.force mimic_mode = config["FFMPEG"]["MimicMode"] if args.config else args.mimic - hide_errors = config["FFMPEG"]["HideErrors"] if args.config else args.silent + hide_errors = config["FFMPEG"]["HideErrors"] if args.config else args.show_errors workers = config["FFMPEG"]["Workers"] if args.config else args.jobs - webp_rgba = config["FFMPEG"]["WebpRGBA"] if args.config else args.webprgba - audio_ext = config["AUDIO"]["Extension"] if args.config else args.aext - audio_bitrate = config["AUDIO"]["BitRate"] if args.config else args.abit - image_downscale = config["IMAGE"]["ResDownScale"] if args.config else args.idown - image_ext = config["IMAGE"]["Extension"] if args.config else args.iext - image_fall_ext = config["IMAGE"]["FallBackExtension"] if args.config else args.ifallext - image_lossless = config["IMAGE"]["Lossless"] if args.config else args.ilossless - image_quality = config["IMAGE"]["Quality"] if args.config else args.iquality - video_crf = config["VIDEO"]["CRF"] if args.config else args.vcrf - video_skip = config["VIDEO"]["SkipVideo"] if args.config else args.vskip - video_ext = config["VIDEO"]["Extension"] if args.config else args.vext - video_codec = config["VIDEO"]["Codec"] if args.config else args.vcodec + webp_rgba = config["FFMPEG"]["WebpRGBA"] if args.config else args.webp_rgba + audio_ext = config["AUDIO"]["Extension"] if args.config else args.a_ext + audio_bitrate = config["AUDIO"]["BitRate"] if args.config else args.a_bit + image_downscale = config["IMAGE"]["ResDownScale"] if args.config else args.i_down + image_ext = config["IMAGE"]["Extension"] if args.config else args.i_ext + image_fall_ext = config["IMAGE"]["FallBackExtension"] if args.config else args.i_fallext + image_lossless = config["IMAGE"]["Lossless"] if args.config else args.i_lossless + image_quality = config["IMAGE"]["Quality"] if args.config else args.i_quality + video_crf = config["VIDEO"]["CRF"] if args.config else args.v_crf + video_skip = config["VIDEO"]["SkipVideo"] if args.config else args.v_skip + video_ext = config["VIDEO"]["Extension"] if args.config else args.v_ext + video_codec = config["VIDEO"]["Codec"] if args.config else args.v_codec source = args.source return cls( @@ -89,3 +66,34 @@ class Params: image_downscale, image_ext, image_fall_ext, image_lossless, image_quality, video_crf, video_skip, video_ext, video_codec, source ) + + @staticmethod + def get_args(): + parser = ArgumentParser(prog="vnrecode", + description="Python utility to compress Visual Novel Resources" + ) + parser.add_argument("source", help="Directory with game files to recode") + parser.add_argument("-c", "--config", help="Utility config file") + parser.add_argument("-u", action='store_true', help="Copy unprocessed filed", default=True) + parser.add_argument("-nu", dest='u', action='store_false', help="Don't copy unprocessed") + parser.add_argument("-f", "--force", action='store_true', help="Try to recode unknown files") + parser.add_argument("-m", "--mimic", action='store_true', help="Enable mimic mode", default=True) + parser.add_argument("-nm", "--no-mimic", dest='mimic', action='store_false', help="Disable mimic mode") + parser.add_argument("-v", "--show_errors", action='store_false', help="Show recode errors") + parser.add_argument("--webp_rgba", action='store_true', help="Recode .webp with alpha channel", default=True) + parser.add_argument("--webp_rgb", dest='webp_rgba', action='store_false', help="Recode .webp without alpha channel") + parser.add_argument("-j", "--jobs", type=int, help="Number of threads", default=16) + parser.add_argument("-ae", "--a_ext", help="Audio extension", default="opus") + parser.add_argument("-ab", "--a_bit", help="Audio bit rate", default="128k") + parser.add_argument("-id", "--i_down", type=int, help="Image resolution downscale multiplier", default=1) + parser.add_argument("-ie", "--i_ext", help="Image extension", default="avif") + parser.add_argument("-ife", "--i_fallext", help="Image fallback extension", default="webp") + parser.add_argument("-il", "--i_lossless", action='store_true', help="Image lossless compression mode", default=True) + parser.add_argument("-ilo", "--i_losing", dest='ilossless', action='store_false', help="Image losing compression mode") + parser.add_argument("-iq", "--i_quality", type=int, help="Image quality", default=100) + parser.add_argument("--v_crf", help="Video CRF number", type=int, default=27) + parser.add_argument("-vs", "--v_skip", action='store_true', help="Skip video recoding") + parser.add_argument("-ve", "--v_ext", help="Video extension", default="webm") + parser.add_argument("-vc", "--v_codec", help="Video codec name", default="libvpx-vp9") + args = parser.parse_args() + return args \ No newline at end of file From b534214be9e42ef58dce9c2f199b19ac1fdd67c9 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 02:07:17 +0300 Subject: [PATCH 06/10] vnrecode: accurate duplicate messages --- vnrecode/utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 96b7db9..8acd4dc 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -66,10 +66,10 @@ class Utils: def check_duplicates(self, in_dir, out_dir, filename): duplicates = glob(f"{in_dir}/{os.path.splitext(filename)[0]}.*") if len(duplicates) > 1: - if filename in self.duplicates: + if filename not in self.duplicates: + self.duplicates.append(filename) new_name = os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1] return f"{out_dir}/{new_name}" - self.duplicates.append(filename) return f"{out_dir}/{filename}" def print_duplicates(self): @@ -78,9 +78,11 @@ class Utils: f'Duplicate file has been found! Check manually this files - "{filename}", ' f'"{os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1]}"') - @staticmethod - def mimic_rename(filename, target, source): + def mimic_rename(self, filename, target, source): if filename.count("(vncopy)"): + orig_name = filename.replace("(vncopy)", "") + index = self.duplicates.index(os.path.split(orig_name)[-1]) + self.duplicates[index] = os.path.split(target)[-1] target = os.path.splitext(target)[0] + "(vncopy)" + os.path.splitext(target)[1] os.rename(filename, target.replace(source, f"{source}_compressed")) \ No newline at end of file From 8f9db132e65fdff3b99c9e8aa728f8204d6e9ec9 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 02:12:49 +0300 Subject: [PATCH 07/10] vnrecode: remove unneeded cli parameters --- vnrecode/params.py | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/vnrecode/params.py b/vnrecode/params.py index c8474c2..89fe28f 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -41,7 +41,7 @@ class Params: print("Failed to find config. Check `vnrecode -h` to more info") exit(255) - copy_unprocessed = config["FFMPEG"]["CopyUnprocessed"] if args.config else args.u + copy_unprocessed = config["FFMPEG"]["CopyUnprocessed"] if args.config else args.unproc force_compress = config["FFMPEG"]["ForceCompress"] if args.config else args.force mimic_mode = config["FFMPEG"]["MimicMode"] if args.config else args.mimic hide_errors = config["FFMPEG"]["HideErrors"] if args.config else args.show_errors @@ -73,27 +73,23 @@ class Params: description="Python utility to compress Visual Novel Resources" ) parser.add_argument("source", help="Directory with game files to recode") - parser.add_argument("-c", "--config", help="Utility config file") - parser.add_argument("-u", action='store_true', help="Copy unprocessed filed", default=True) - parser.add_argument("-nu", dest='u', action='store_false', help="Don't copy unprocessed") + parser.add_argument("-c", dest='config', help="Utility config file") + parser.add_argument("-nu", dest='unproc', action='store_false', help="Don't copy unprocessed") parser.add_argument("-f", "--force", action='store_true', help="Try to recode unknown files") - parser.add_argument("-m", "--mimic", action='store_true', help="Enable mimic mode", default=True) parser.add_argument("-nm", "--no-mimic", dest='mimic', action='store_false', help="Disable mimic mode") parser.add_argument("-v", "--show_errors", action='store_false', help="Show recode errors") - parser.add_argument("--webp_rgba", action='store_true', help="Recode .webp with alpha channel", default=True) - parser.add_argument("--webp_rgb", dest='webp_rgba', action='store_false', help="Recode .webp without alpha channel") + parser.add_argument("--webp-rgb", dest='webp_rgba', action='store_false', help="Recode .webp without alpha channel") parser.add_argument("-j", "--jobs", type=int, help="Number of threads", default=16) - parser.add_argument("-ae", "--a_ext", help="Audio extension", default="opus") - parser.add_argument("-ab", "--a_bit", help="Audio bit rate", default="128k") - parser.add_argument("-id", "--i_down", type=int, help="Image resolution downscale multiplier", default=1) - parser.add_argument("-ie", "--i_ext", help="Image extension", default="avif") - parser.add_argument("-ife", "--i_fallext", help="Image fallback extension", default="webp") - parser.add_argument("-il", "--i_lossless", action='store_true', help="Image lossless compression mode", default=True) - parser.add_argument("-ilo", "--i_losing", dest='ilossless', action='store_false', help="Image losing compression mode") - parser.add_argument("-iq", "--i_quality", type=int, help="Image quality", default=100) + parser.add_argument("-ae", dest="a_ext", help="Audio extension", default="opus") + parser.add_argument("-ab", dest="a_bit", help="Audio bit rate", default="128k") + parser.add_argument("-id", dest="i_down", type=int, help="Image resolution downscale multiplier", default=1) + parser.add_argument("-ie", dest="i_ext", help="Image extension", default="avif") + parser.add_argument("-ife", dest="i_fallext", help="Image fallback extension", default="webp") + parser.add_argument("-il", dest='i_lossless', action='store_false', help="Image losing compression mode") + parser.add_argument("-iq", dest="i_quality", type=int, help="Image quality", default=100) parser.add_argument("--v_crf", help="Video CRF number", type=int, default=27) - parser.add_argument("-vs", "--v_skip", action='store_true', help="Skip video recoding") - parser.add_argument("-ve", "--v_ext", help="Video extension", default="webm") - parser.add_argument("-vc", "--v_codec", help="Video codec name", default="libvpx-vp9") + parser.add_argument("-vs", dest="v_skip", action='store_true', help="Skip video recoding") + parser.add_argument("-ve", dest="v_ext", help="Video extension", default="webm") + parser.add_argument("-vc", dest="v_codec", help="Video codec name", default="libvpx-vp9") args = parser.parse_args() return args \ No newline at end of file From 7433027cf3c31f9ca52c502ad3ca4bc034c82cf4 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 02:40:47 +0300 Subject: [PATCH 08/10] vnrecode: add type definitions --- vnrecode/application.py | 11 +++++--- vnrecode/compress.py | 59 +++++++++++++++++++++-------------------- vnrecode/params.py | 4 +-- vnrecode/printer.py | 12 ++++----- vnrecode/utils.py | 46 ++++++++++++++++---------------- 5 files changed, 69 insertions(+), 63 deletions(-) diff --git a/vnrecode/application.py b/vnrecode/application.py index c7cb7cd..68fa45f 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -4,17 +4,22 @@ from datetime import datetime import shutil import os +from .compress import Compress +from .printer import Printer +from .params import Params +from .utils import Utils + class Application: - def __init__(self, params, compress, printer, utils): + def __init__(self, params: Params, compress: Compress, printer: Printer, utils: Utils): self.params = params self.compress = compress.compress self.printer = printer self.utils = utils - def compress_worker(self, folder, file, source, output): - if os.path.isfile(f'{folder}/{file}'): + def compress_worker(self, folder: str, file: str, source: str, output: str): + if os.path.isfile(os.path.join(folder, file)): self.compress(folder, file, source, output) def run(self): diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 249061e..18fb20e 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -1,13 +1,17 @@ from ffmpeg import FFmpeg, FFmpegError from PIL import Image +from os import path import pillow_avif -import os + +from .printer import Printer +from .params import Params +from .utils import Utils class File: @staticmethod - def get_type(filename): + def get_type(filename: str) -> str: extensions = { "audio": ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'], @@ -17,7 +21,7 @@ class File: } for file_type in extensions: - if os.path.splitext(filename)[1] in extensions[file_type]: + if path.splitext(filename)[1] in extensions[file_type]: return file_type return "unknown" @@ -39,45 +43,44 @@ class File: class Compress: - def __init__(self, params, printer, utils): + def __init__(self, params: Params, printer: Printer, utils: Utils): self.params = params self.printer = printer self.utils = utils - def audio(self, in_dir, file, out_dir, extension): + def audio(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: bit_rate = self.params.audio_bitrate - out_file = self.utils.check_duplicates(in_dir, out_dir, f'{os.path.splitext(file)[0]}.{extension}') + out_file = self.utils.check_duplicates(in_dir, out_dir, f'{path.splitext(file)[0]}.{extension}') try: (FFmpeg() - .input(f'{in_dir}/{file}') + .input(path.join(in_dir, file)) .option("hide_banner") .output(out_file,{"b:a": bit_rate, "loglevel": "error"}) .execute() ) except FFmpegError as e: - self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') + self.utils.add_unprocessed_file(path.join(in_dir, file), path.join(out_dir, file)) self.utils.errors += 1 if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") - self.printer.files(file, os.path.splitext(file)[0], extension, f"{bit_rate}") + self.printer.files(file, path.splitext(file)[0], extension, f"{bit_rate}") return out_file - - def video(self, in_dir, file, out_dir, extension): + def video(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: if not self.params.video_skip: - out_file = self.utils.check_duplicates(in_dir, out_dir, f'{os.path.splitext(file)[0]}.{extension}') + out_file = self.utils.check_duplicates(in_dir, out_dir, f'{path.splitext(file)[0]}.{extension}') codec = self.params.video_codec crf = self.params.video_crf try: (FFmpeg() - .input(f'{in_dir}/{file}') + .input(path.join(in_dir, file)) .option("hide_banner") .option("hwaccel", "auto") .output(out_file,{"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) .execute() ) - self.printer.files(file, os.path.splitext(file)[0], extension, codec) + self.printer.files(file, path.splitext(file)[0], extension, codec) except FFmpegError as e: self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') self.utils.errors += 1 @@ -86,14 +89,13 @@ class Compress: return out_file else: self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') - return f'{out_dir}/{os.path.splitext(file)[0]}.{extension}' + return f'{out_dir}/{path.splitext(file)[0]}.{extension}' - - def image(self, in_dir, file, out_dir, extension): + def image(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: quality = self.params.image_quality - out_file = self.utils.check_duplicates(in_dir, out_dir, f"{os.path.splitext(file)[0]}.{extension}") + out_file = self.utils.check_duplicates(in_dir, out_dir, f"{path.splitext(file)[0]}.{extension}") try: - image = Image.open(f'{in_dir}/{file}') + image = Image.open(path.join(in_dir, file)) if (extension == "jpg" or extension == "jpeg" or (extension == "webp" and not self.params.webp_rgba)): @@ -115,36 +117,35 @@ class Compress: lossless=self.params.image_lossless, quality=quality, minimize_size=True) - self.printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") + self.printer.files(file, path.splitext(file)[0], extension, f"{quality}%") except Exception as e: - self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') + self.utils.add_unprocessed_file(path.join(in_dir, file), path.join(out_dir, file)) self.utils.errors += 1 if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") return out_file - - def unknown(self, in_dir, filename, out_dir): + def unknown(self, in_dir: str, filename: str, out_dir: str) -> str: if self.params.force_compress: self.printer.unknown_file(filename) out_file = self.utils.check_duplicates(in_dir, out_dir, filename) try: (FFmpeg() - .input(f'{in_dir}/{filename}') + .input(path.join(in_dir, filename)) .output(out_file) .execute() ) except FFmpegError as e: - self.utils.add_unprocessed_file(f'{in_dir}/{filename}', f'{out_dir}/{filename}') + self.utils.add_unprocessed_file(path.join(in_dir, filename), path.join(out_dir, filename)) self.utils.errors += 1 if not self.params.hide_errors: self.printer.error(f"File {filename} can't be processed! Error: {e}") return out_file else: - self.utils.add_unprocessed_file(f'{in_dir}/{filename}', f'{out_dir}/{filename}') - return f'{out_dir}/{filename}' + self.utils.add_unprocessed_file(path.join(in_dir, filename), path.join(out_dir, filename)) + return path.join(out_dir, filename) - def compress(self, _dir, filename, source, output): + def compress(self, _dir: str, filename: str, source: str, output: str): match File.get_type(filename): case "audio": out_file = self.audio(_dir, filename, output, self.params.audio_ext) @@ -156,7 +157,7 @@ class Compress: out_file = self.unknown(_dir, filename, output) if self.params.mimic_mode: - self.utils.mimic_rename(out_file, f'{_dir}/{filename}', source) + self.utils.mimic_rename(out_file, path.join(_dir, filename), source) self.printer.bar.update() self.printer.bar.next() diff --git a/vnrecode/params.py b/vnrecode/params.py index 89fe28f..ea15691 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -1,4 +1,4 @@ -from argparse import ArgumentParser +from argparse import ArgumentParser, Namespace from dataclasses import dataclass from typing import Self import tomllib @@ -68,7 +68,7 @@ class Params: ) @staticmethod - def get_args(): + def get_args() -> Namespace: parser = ArgumentParser(prog="vnrecode", description="Python utility to compress Visual Novel Resources" ) diff --git a/vnrecode/printer.py b/vnrecode/printer.py index 6080caf..30aa496 100644 --- a/vnrecode/printer.py +++ b/vnrecode/printer.py @@ -15,7 +15,7 @@ class Printer: # Fill whole string with spaces for cleaning progress bar @staticmethod - def clean_str(string): + def clean_str(string: str) -> str: return string + " " * (os.get_terminal_size().columns - len(string)) @staticmethod @@ -23,20 +23,20 @@ class Printer: if sys.platform == "win32": colorama.init() - def bar_print(self, string): + def bar_print(self, string: str): print(string) self.bar.update() - def info(self, string): + def info(self, string: str): self.bar_print(self.clean_str(f"\r\033[100m- {string}\033[49m")) - def warning(self, string): + def warning(self, string: str): self.bar_print(self.clean_str(f"\r\033[93m!\033[0m {string}\033[49m")) - def error(self, string): + def error(self, string: str): self.bar_print(self.clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m")) - def files(self, source, dest, dest_ext, comment): + def files(self, source: str, dest: str, dest_ext: str, comment: str): source_ext = os.path.splitext(source)[1] source_name = os.path.splitext(source)[0] diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 8acd4dc..ec5c2c5 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -17,15 +17,15 @@ class Utils: os.system("pause") @staticmethod - def get_size(directory): + def get_size(directory: str) -> int: total_size = 0 for folder, folders, files in os.walk(directory): for file in files: - if not os.path.islink(f"{folder}/{file}"): - total_size += os.path.getsize(f"{folder}/{file}") + if not os.path.islink(os.path.join(folder, file)): + total_size += os.path.getsize(os.path.join(folder, file)) return total_size - def get_compression(self, source, output): + def get_compression(self, source: str, output: str): try: source = self.get_size(source) output = self.get_size(output) @@ -35,50 +35,50 @@ class Utils: except ZeroDivisionError: self.printer.warning("Nothing compressed!") - def get_compression_status(self, orig_folder): - orig_folder_len = 0 - comp_folder_len = 0 + def get_compression_status(self, source: str): + source_len = 0 + output_len = 0 - for folder, folders, files in os.walk(orig_folder): - orig_folder_len += len(files) + for folder, folders, files in os.walk(source): + source_len += len(files) - for folder, folders, files in os.walk(f'{orig_folder}_compressed'): + for folder, folders, files in os.walk(f'{source}_compressed'): for file in files: if not os.path.splitext(file)[1].count("(copy)"): - comp_folder_len += 1 + output_len += 1 if self.errors != 0: self.printer.warning("Some files failed to compress!") - if orig_folder_len == comp_folder_len: + if source_len == output_len: self.printer.info("Success!") - self.get_compression(orig_folder, f"{orig_folder}_compressed") else: self.printer.warning("Original and compressed folders are not identical!") - self.get_compression(orig_folder, f"{orig_folder}_compressed") + self.get_compression(source, f"{source}_compressed") - def add_unprocessed_file(self, orig_folder, new_folder): + def add_unprocessed_file(self, source: str, output: str): if self.params.copy_unprocessed: - filename = orig_folder.split("/").pop() - copyfile(orig_folder, new_folder) + filename = os.path.split(source)[-1] + copyfile(source, output) self.printer.info(f"File {filename} copied to compressed folder.") - def check_duplicates(self, in_dir, out_dir, filename): - duplicates = glob(f"{in_dir}/{os.path.splitext(filename)[0]}.*") + def check_duplicates(self, source: str, output: str, filename: str) -> str: + duplicates = glob(os.path.join(source, os.path.splitext(filename)[0]+".*")) if len(duplicates) > 1: if filename not in self.duplicates: self.duplicates.append(filename) new_name = os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1] - return f"{out_dir}/{new_name}" - return f"{out_dir}/{filename}" + return os.path.join(output, new_name) + return os.path.join(output, filename) def print_duplicates(self): for filename in self.duplicates: self.printer.warning( f'Duplicate file has been found! Check manually this files - "{filename}", ' - f'"{os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1]}"') + f'"{os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1]}"' + ) - def mimic_rename(self, filename, target, source): + def mimic_rename(self, filename: str, target: str, source: str): if filename.count("(vncopy)"): orig_name = filename.replace("(vncopy)", "") index = self.duplicates.index(os.path.split(orig_name)[-1]) From 90a6b4e0c1d0f94c87a2441a2db5ee22a3e4d0c1 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 03:25:43 +0300 Subject: [PATCH 09/10] vnrecode: add re to duplicate check --- vnrecode/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vnrecode/utils.py b/vnrecode/utils.py index ec5c2c5..58cf962 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -2,6 +2,7 @@ from shutil import copyfile from glob import glob import sys import os +import re class Utils: @@ -63,7 +64,10 @@ class Utils: self.printer.info(f"File {filename} copied to compressed folder.") def check_duplicates(self, source: str, output: str, filename: str) -> str: - duplicates = glob(os.path.join(source, os.path.splitext(filename)[0]+".*")) + files = glob(os.path.join(source, os.path.splitext(filename)[0])+".*") + re_pattern = re.compile(os.path.join(source, os.path.splitext(filename)[0])+r".[a-zA-Z0-9]+$") + duplicates = [f for f in files if re_pattern.match(f)] + if len(duplicates) > 1: if filename not in self.duplicates: self.duplicates.append(filename) From 0b43756ef511177158d7d34a2c3271bd91bcea7a Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 04:30:01 +0300 Subject: [PATCH 10/10] vnrecode: fix duplicates check for case insensitive fs --- vnrecode/application.py | 2 +- vnrecode/utils.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/vnrecode/application.py b/vnrecode/application.py index 68fa45f..912c526 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -42,7 +42,7 @@ class Application: with ThreadPoolExecutor(max_workers=self.params.workers) as executor: futures = [ executor.submit(self.compress, folder, file, source, output) - for file in files if os.path.isfile(f'{folder}/{file}') + for file in files if os.path.isfile(os.path.join(folder, file)) ] for future in as_completed(futures): future.result() diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 58cf962..3410eae 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -4,6 +4,9 @@ import sys import os import re +import fnmatch + + class Utils: def __init__(self, params, printer): @@ -64,12 +67,11 @@ class Utils: self.printer.info(f"File {filename} copied to compressed folder.") def check_duplicates(self, source: str, output: str, filename: str) -> str: - files = glob(os.path.join(source, os.path.splitext(filename)[0])+".*") - re_pattern = re.compile(os.path.join(source, os.path.splitext(filename)[0])+r".[a-zA-Z0-9]+$") - duplicates = [f for f in files if re_pattern.match(f)] + re_pattern = re.compile(os.path.splitext(filename)[0]+r".[a-zA-Z0-9]+$", re.IGNORECASE) + duplicates = [name for name in os.listdir(source) if re_pattern.match(name)] if len(duplicates) > 1: - if filename not in self.duplicates: + if filename.lower() not in (duplicate.lower() for duplicate in self.duplicates): self.duplicates.append(filename) new_name = os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1] return os.path.join(output, new_name)