From 7487eb94bd7971b6183f1ca776e8ca5ad232f31e Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Thu, 29 Aug 2024 01:32:36 +0300 Subject: [PATCH 01/24] Basic refactor for packaging --- .gitignore | 3 + FFMpeg-Compressor/__init__.py | 0 FFMpeg-Compressor/__main__.py | 19 ++ FFMpeg-Compressor/application.py | 48 ++++++ FFMpeg-Compressor/compress.py | 163 ++++++++++++++++++ FFMpeg-Compressor/config.py | 27 +++ FFMpeg-Compressor/main.py | 59 ------- FFMpeg-Compressor/modules/compressor.py | 160 ----------------- FFMpeg-Compressor/modules/configloader.py | 11 -- FFMpeg-Compressor/modules/printer.py | 51 ------ FFMpeg-Compressor/modules/utils.py | 79 --------- FFMpeg-Compressor/printer.py | 46 +++++ FFMpeg-Compressor/utils.py | 71 ++++++++ README.md | 25 ++- RenPy-Android-Unpack/__init__.py | 0 RenPy-Android-Unpack/__main__.py | 52 ++++++ RenPy-Android-Unpack/actions.py | 98 +++++++++++ RenPy-Android-Unpack/printer.py | 14 ++ RenPy-Android-Unpack/requirements.txt | 2 - RenPy-Android-Unpack/unpack.py | 108 ------------ VNDS-to-RenPy/__init__.py | 0 VNDS-to-RenPy/{convert.py => __main__.py} | 0 build.bat | 10 +- build.sh | 12 +- .../requirements.txt => requirements.txt | 0 25 files changed, 573 insertions(+), 485 deletions(-) create mode 100644 FFMpeg-Compressor/__init__.py create mode 100644 FFMpeg-Compressor/__main__.py create mode 100755 FFMpeg-Compressor/application.py create mode 100644 FFMpeg-Compressor/compress.py create mode 100644 FFMpeg-Compressor/config.py delete mode 100755 FFMpeg-Compressor/main.py delete mode 100644 FFMpeg-Compressor/modules/compressor.py delete mode 100644 FFMpeg-Compressor/modules/configloader.py delete mode 100644 FFMpeg-Compressor/modules/printer.py delete mode 100644 FFMpeg-Compressor/modules/utils.py create mode 100644 FFMpeg-Compressor/printer.py create mode 100644 FFMpeg-Compressor/utils.py create mode 100644 RenPy-Android-Unpack/__init__.py create mode 100755 RenPy-Android-Unpack/__main__.py create mode 100755 RenPy-Android-Unpack/actions.py create mode 100644 RenPy-Android-Unpack/printer.py delete mode 100644 RenPy-Android-Unpack/requirements.txt delete mode 100755 RenPy-Android-Unpack/unpack.py create mode 100644 VNDS-to-RenPy/__init__.py rename VNDS-to-RenPy/{convert.py => __main__.py} (100%) mode change 100644 => 100755 build.bat rename FFMpeg-Compressor/requirements.txt => requirements.txt (100%) diff --git a/.gitignore b/.gitignore index 16be8f2..abe0419 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ /output/ +/tests/ +/build/ +/VNTools.egg-info/ diff --git a/FFMpeg-Compressor/__init__.py b/FFMpeg-Compressor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/FFMpeg-Compressor/__main__.py b/FFMpeg-Compressor/__main__.py new file mode 100644 index 0000000..aef4d34 --- /dev/null +++ b/FFMpeg-Compressor/__main__.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +from application import Application +from compress import Compress +from printer import Printer +from config import Config +from utils import Utils + + +def init(): + config = Config.setup_config() + printer = Printer(config.args.source) + utils = Utils(config.config, printer) + compress = Compress(config.config, printer, utils) + + Application(config, compress, printer, utils).run() + + +if __name__ == "__main__": + init() \ No newline at end of file diff --git a/FFMpeg-Compressor/application.py b/FFMpeg-Compressor/application.py new file mode 100755 index 0000000..491267a --- /dev/null +++ b/FFMpeg-Compressor/application.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime +import shutil +import os + + +class Application: + + def __init__(self, config, compress, printer, utils): + self.config = config.config + self.args = config.args + self.compress = compress.compress + self.printer = printer + self.utils = utils + + def compress_worker(self, folder, file, source, output): + if os.path.isfile(f'{folder}/{file}'): + self.compress(folder, file, source, output) + + def run(self): + start_time = datetime.now() + self.printer.win_ascii_esc() + + source = os.path.abspath(self.args.source) + + if os.path.exists(f"{source}_compressed"): + shutil.rmtree(f"{source}_compressed") + + self.printer.info("Creating folders...") + for folder, folders, files in os.walk(source): + if not os.path.exists(folder.replace(source, f"{source}_compressed")): + os.mkdir(folder.replace(source, f"{source}_compressed")) + + self.printer.info(f'Compressing "{folder.replace(source, os.path.split(source)[-1])}" folder...') + output = folder.replace(source, f"{source}_compressed") + + with ThreadPoolExecutor(max_workers=self.config["FFMPEG"]["Workers"]) as executor: + futures = [ + executor.submit(self.compress, folder, file, source, output) + for file in files if os.path.isfile(f'{folder}/{file}') + ] + for future in as_completed(futures): + future.result() + + self.utils.get_compression_status(source) + self.utils.sys_pause() + print(f"Time taken: {datetime.now() - start_time}") \ No newline at end of file diff --git a/FFMpeg-Compressor/compress.py b/FFMpeg-Compressor/compress.py new file mode 100644 index 0000000..5c1a465 --- /dev/null +++ b/FFMpeg-Compressor/compress.py @@ -0,0 +1,163 @@ +from ffmpeg import FFmpeg, FFmpegError +from PIL import Image +import pillow_avif +import os + + +class File: + + @staticmethod + def get_type(filename): + audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] + image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'] + video_ext = ['.3gp' '.amv', '.avi', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', + '.webm', '.ogv'] + + if os.path.splitext(filename)[1] in audio_ext: + return "audio" + elif os.path.splitext(filename)[1] in image_ext: + return "image" + elif os.path.splitext(filename)[1] in video_ext: + return "video" + else: + return "unknown" + + @staticmethod + def has_transparency(img: Image) -> bool: + if img.info.get("transparency", None) is not None: + return True + if img.mode == "P": + transparent = img.info.get("transparency", -1) + for _, index in img.getcolors(): + if index == transparent: + return True + elif img.mode == "RGBA": + extrema = img.getextrema() + if extrema[3][0] < 255: + return True + return False + + +class Compress: + + def __init__(self, config, printer, utils): + self.config = config + self.printer = printer + self.utils = utils + + def audio(self, folder, file, target_folder, extension): + bitrate = self.config['AUDIO']['BitRate'] + try: + (FFmpeg() + .input(f'{folder}/{file}') + .option("hide_banner") + .output(self.utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), + {"b:a": bitrate, "loglevel": "error"}) + .execute() + ) + except FFmpegError as e: + self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.errors += 1 + if not self.config['FFMPEG']['HideErrors']: + self.printer.error(f"File {file} can't be processed! Error: {e}") + self.printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}") + return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' + + + def video(self, folder, file, target_folder, extension): + if not self.config['VIDEO']['SkipVideo']: + codec = self.config['VIDEO']['Codec'] + crf = self.config['VIDEO']['CRF'] + + try: + (FFmpeg() + .input(f'{folder}/{file}') + .option("hide_banner") + .option("hwaccel", "auto") + .output(self.utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), + {"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) + .execute() + ) + self.printer.files(file, os.path.splitext(file)[0], extension, codec) + except FFmpegError as e: + self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.errors += 1 + if not self.config['FFMPEG']['HideErrors']: + self.printer.error(f"File {file} can't be processed! Error: {e}") + else: + self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' + + + def image(self, folder, file, target_folder, extension): + quality = self.config['IMAGE']['Quality'] + try: + image = Image.open(f'{folder}/{file}') + + if (extension == "jpg" or extension == "jpeg" or + (extension == "webp" and not self.config['FFMPEG']['WebpRGBA'])): + if File.has_transparency(image): + self.printer.warning(f"{file} has transparency. Changing to fallback...") + extension = self.config['IMAGE']['FallBackExtension'] + + if File.has_transparency(image): + image.convert('RGBA') + + res_downscale = self.config['IMAGE']['ResDownScale'] + if res_downscale != 1: + width, height = image.size + new_size = (int(width / res_downscale), int(height / res_downscale)) + image = image.resize(new_size) + + image.save(self.utils.check_duplicates(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}"), + optimize=True, + lossless=self.config['IMAGE']['Lossless'], + quality=quality, + minimize_size=True) + self.printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") + except Exception as e: + self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.errors += 1 + if not self.config['FFMPEG']['HideErrors']: + self.printer.error(f"File {file} can't be processed! Error: {e}") + return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' + + + def unknown(self, folder, file, target_folder): + if self.config["FFMPEG"]["ForceCompress"]: + self.printer.unknown_file(file) + try: + (FFmpeg() + .input(f'{folder}/{file}') + .output(self.utils.check_duplicates(f'{target_folder}/{file}')) + .execute() + ) + except FFmpegError as e: + self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.errors += 1 + if not self.config['FFMPEG']['HideErrors']: + self.printer.error(f"File {file} can't be processed! Error: {e}") + else: + self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + return f'{target_folder}/{file}' + + + def compress(self, _dir, filename, source, output): + match File.get_type(filename): + case "audio": + out_file = self.audio(_dir, filename, output, self.config['AUDIO']['Extension']) + case "image": + out_file = self.image(_dir, filename, output, self.config['IMAGE']['Extension']) + case "video": + out_file = self.video(_dir, filename, output, self.config['VIDEO']['Extension']) + case "unknown": + out_file = self.unknown(_dir, filename, output) + + if self.config['FFMPEG']['MimicMode']: + try: + os.rename(out_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed")) + except FileNotFoundError: + self.printer.warning(f"File {out_file} failed to copy to out dir") + + self.printer.bar.update() + self.printer.bar.next() diff --git a/FFMpeg-Compressor/config.py b/FFMpeg-Compressor/config.py new file mode 100644 index 0000000..d36e3d4 --- /dev/null +++ b/FFMpeg-Compressor/config.py @@ -0,0 +1,27 @@ +import os.path +from argparse import Namespace, ArgumentParser +from dataclasses import dataclass +from typing import Any +import tomllib + + +@dataclass +class Config: + + config: dict[str, Any] + args: Namespace + + @classmethod + def setup_config(cls): + parser = ArgumentParser(prog="ffmpeg-comp", + description="Python utility to compress Visual Novel Resources" + ) + parser.add_argument("source") + parser.add_argument("-c", "--config", default="ffmpeg-comp.toml") + args = parser.parse_args() + if os.path.isfile(args.config): + with open(args.config, "rb") as cfile: + config = tomllib.load(cfile) + else: + print("Failed to find config. Check `ffmpeg-comp -h` to more info") + return cls(config=config, args=args) diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py deleted file mode 100755 index 7c05dc5..0000000 --- a/FFMpeg-Compressor/main.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 - -from concurrent.futures import ThreadPoolExecutor, as_completed -from modules.configloader import config -from modules import compressor -from modules import printer -from modules import utils -from datetime import datetime -import shutil -import sys -import os - - -def get_args(): - try: - if sys.argv[1][len(sys.argv[1])-1] == "/": - path = sys.argv[1][:len(sys.argv[1])-1] - else: - path = sys.argv[1] - return path - except IndexError: - print(utils.help_message()) - exit() - - -def compress_worker(folder, file, target_folder, req_folder): - if os.path.isfile(f'{folder}/{file}'): - compressor.compress_file(folder, file, target_folder, req_folder) - - -if __name__ == "__main__": - start_time = datetime.now() - printer.win_ascii_esc() - req_folder = os.path.abspath(get_args()) - - printer.bar_init(req_folder) - - if os.path.exists(f"{req_folder}_compressed"): - shutil.rmtree(f"{req_folder}_compressed") - - printer.info("Creating folders...") - for folder, folders, files in os.walk(req_folder): - if not os.path.exists(folder.replace(req_folder, f"{req_folder}_compressed")): - os.mkdir(folder.replace(req_folder, f"{req_folder}_compressed")) - - printer.info(f"Compressing \"{folder.replace(req_folder, req_folder.split('/').pop())}\" folder...") - target_folder = folder.replace(req_folder, f"{req_folder}_compressed") - - with ThreadPoolExecutor(max_workers=config["FFMPEG"]["Workers"]) as executor: - futures = [ - executor.submit(compress_worker, folder, file, target_folder, req_folder) - for file in files - ] - for future in as_completed(futures): - future.result() - - utils.get_compression_status(req_folder) - utils.sys_pause() - print(f"Time taken: {datetime.now() - start_time}") \ No newline at end of file diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py deleted file mode 100644 index 728953c..0000000 --- a/FFMpeg-Compressor/modules/compressor.py +++ /dev/null @@ -1,160 +0,0 @@ -from modules import configloader -from modules import printer -from modules import utils -from PIL import Image -import pillow_avif -from ffmpeg import FFmpeg, FFmpegError -import os - - -def get_file_type(filename): - audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] - image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'] - video_ext = ['.3gp' '.amv', '.avi', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', - '.webm', '.ogv'] - - if os.path.splitext(filename)[1] in audio_ext: - return "audio" - elif os.path.splitext(filename)[1] in image_ext: - return "image" - elif os.path.splitext(filename)[1] in video_ext: - return "video" - else: - return "unknown" - - -def has_transparency(img): - if img.info.get("transparency", None) is not None: - return True - if img.mode == "P": - transparent = img.info.get("transparency", -1) - for _, index in img.getcolors(): - if index == transparent: - return True - elif img.mode == "RGBA": - extrema = img.getextrema() - if extrema[3][0] < 255: - return True - - return False - - -def compress_audio(folder, file, target_folder, extension): - bitrate = configloader.config['AUDIO']['BitRate'] - try: - (FFmpeg() - .input(f'{folder}/{file}') - .option("hide_banner") - .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), - {"b:a": bitrate, "loglevel": "error"}) - .execute() - ) - except FFmpegError as e: - utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - utils.errors_count += 1 - if not configloader.config['FFMPEG']['HideErrors']: - printer.error(f"File {file} can't be processed! Error: {e}") - printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}") - return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' - - -def compress_video(folder, file, target_folder, extension): - if not configloader.config['VIDEO']['SkipVideo']: - codec = configloader.config['VIDEO']['Codec'] - crf = configloader.config['VIDEO']['CRF'] - - try: - (FFmpeg() - .input(f'{folder}/{file}') - .option("hide_banner") - .option("hwaccel", "auto") - .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), - {"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) - .execute() - ) - printer.files(file, os.path.splitext(file)[0], extension, codec) - except FFmpegError as e: - utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - utils.errors_count += 1 - if not configloader.config['FFMPEG']['HideErrors']: - printer.error(f"File {file} can't be processed! Error: {e}") - else: - utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' - - -def compress_image(folder, file, target_folder, extension): - quality = configloader.config['IMAGE']['Quality'] - try: - image = Image.open(f'{folder}/{file}') - - if (extension == "jpg" or extension == "jpeg" or - (extension == "webp" and not configloader.config['FFMPEG']['WebpRGBA'])): - if has_transparency(image): - printer.warning(f"{file} has transparency. Changing to fallback...") - extension = configloader.config['IMAGE']['FallBackExtension'] - - if has_transparency(image): - image.convert('RGBA') - - res_downscale = configloader.config['IMAGE']['ResDownScale'] - if res_downscale != 1: - width, height = image.size - new_size = (int(width / res_downscale), int(height / res_downscale)) - image = image.resize(new_size) - - image.save(utils.check_duplicates(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}"), - optimize=True, - lossless=configloader.config['IMAGE']['Lossless'], - quality=quality, - minimize_size=True) - printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") - except Exception as e: - utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - utils.errors_count += 1 - if not configloader.config['FFMPEG']['HideErrors']: - printer.error(f"File {file} can't be processed! Error: {e}") - return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' - - -def compress(folder, file, target_folder): - if configloader.config["FFMPEG"]["ForceCompress"]: - printer.unknown_file(file) - try: - (FFmpeg() - .input(f'{folder}/{file}') - .output(f'{target_folder}/{file}') - .execute() - ) - except FFmpegError as e: - utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - utils.errors_count += 1 - if not configloader.config['FFMPEG']['HideErrors']: - printer.error(f"File {file} can't be processed! Error: {e}") - else: - utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - return f'{target_folder}/{file}' - - -def compress_file(_dir, filename, target_dir, source): - match get_file_type(filename): - case "audio": - comp_file = compress_audio(_dir, filename, target_dir, - configloader.config['AUDIO']['Extension']) - case "image": - comp_file = compress_image(_dir, filename, target_dir, - configloader.config['IMAGE']['Extension']) - case "video": - comp_file = compress_video(_dir, filename, target_dir, - configloader.config['VIDEO']['Extension']) - case "unknown": - comp_file = compress(_dir, filename, target_dir) - - if configloader.config['FFMPEG']['MimicMode']: - try: - os.rename(comp_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed")) - except FileNotFoundError: - pass - - printer.bar.update() - printer.bar.next() diff --git a/FFMpeg-Compressor/modules/configloader.py b/FFMpeg-Compressor/modules/configloader.py deleted file mode 100644 index 1a4db0b..0000000 --- a/FFMpeg-Compressor/modules/configloader.py +++ /dev/null @@ -1,11 +0,0 @@ -import tomllib -from modules import printer - -try: - config = tomllib.load(open("ffmpeg-comp.toml", "rb")) -except FileNotFoundError: - try: - config = tomllib.load(open("/etc/ffmpeg-comp.toml", "rb")) - except FileNotFoundError: - printer.error("Config file not found. Please put it next to binary or in to /etc folder.") - exit() diff --git a/FFMpeg-Compressor/modules/printer.py b/FFMpeg-Compressor/modules/printer.py deleted file mode 100644 index 0fdce24..0000000 --- a/FFMpeg-Compressor/modules/printer.py +++ /dev/null @@ -1,51 +0,0 @@ -from progress.bar import IncrementalBar -import colorama -import sys -import os - - -def bar_init(folder): - file_count = 0 - for folder, folders, file in os.walk(folder): - file_count += len(file) - global bar - bar = IncrementalBar('Compressing', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)') - bar.update() - - -def bar_print(string): - print(string) - bar.update() - - -# Fill whole string with spaces for cleaning progress bar -def clean_str(string): - return string + " " * (os.get_terminal_size().columns - len(string)) - - -def info(string): - bar_print(clean_str(f"\r\033[100m- {string}\033[49m")) - - -def warning(string): - bar_print(clean_str(f"\r\033[93m!\033[0m {string}\033[49m")) - - -def error(string): - bar_print(clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m")) - - -def files(source, dest, dest_ext, comment): - source_ext = os.path.splitext(source)[1] - source_name = os.path.splitext(source)[0] - - bar_print(clean_str(f"\r\033[0;32m\u2713\033[0m \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m")) - - -def unknown_file(file): - bar_print(clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)")) - - -def win_ascii_esc(): - if sys.platform == "win32": - colorama.init() diff --git a/FFMpeg-Compressor/modules/utils.py b/FFMpeg-Compressor/modules/utils.py deleted file mode 100644 index 907f9b1..0000000 --- a/FFMpeg-Compressor/modules/utils.py +++ /dev/null @@ -1,79 +0,0 @@ -from modules import configloader -from modules import printer -from shutil import copyfile -import sys -import os - -errors_count = 0 - - -def get_dir_size(directory, files): - total_size = 0 - for folder, folders, files in os.walk(directory): - for file in files: - if not os.path.islink(f"{folder}/{file}"): - total_size += os.path.getsize(f"{folder}/{file}") - return total_size - - -def get_compression(orig, comp): - processed_files = [] - for folder, folders, files in os.walk(comp): - for file in files: - processed_files.append(file) - try: - orig = get_dir_size(orig, processed_files) - comp = get_dir_size(comp, processed_files) - - print(f"\nResult: {orig/1024/1024:.2f}MB -> {comp/1024/1024:.2f}MB ({(comp - orig)/1024/1024:.2f}MB)") - except ZeroDivisionError: - printer.warning("Nothing compressed!") - - -def get_compression_status(orig_folder): - orig_folder_len = 0 - comp_folder_len = 0 - - for folder, folders, files in os.walk(orig_folder): - orig_folder_len += len(files) - - for folder, folders, files in os.walk(f'{orig_folder}_compressed'): - for file in files: - if not os.path.splitext(file)[1].count(" (copy)"): - comp_folder_len += 1 - - if errors_count != 0: - printer.warning("Some files failed to compress!") - - if orig_folder_len == comp_folder_len: - printer.info("Success!") - get_compression(orig_folder, f"{orig_folder}_compressed") - else: - printer.warning("Original and compressed folders are not identical!") - get_compression(orig_folder, f"{orig_folder}_compressed") - - -def add_unprocessed_file(orig_folder, new_folder): - if configloader.config['FFMPEG']['CopyUnprocessed']: - filename = orig_folder.split("/").pop() - copyfile(orig_folder, new_folder) - printer.info(f"File {filename} copied to compressed folder.") - - -def check_duplicates(new_folder): - filename = new_folder.split().pop() - if os.path.exists(new_folder): - printer.warning( - f'Duplicate file has been found! Check manually this files - "{filename}", ' - f'"{os.path.splitext(filename)[0] + "(copy)" + os.path.splitext(filename)[1]}"') - return os.path.splitext(new_folder)[0] + "(copy)" + os.path.splitext(new_folder)[1] - return new_folder - - -def sys_pause(): - if sys.platform == "win32": - os.system("pause") - - -def help_message(): - return "Usage: ffmpeg-comp {folder}" diff --git a/FFMpeg-Compressor/printer.py b/FFMpeg-Compressor/printer.py new file mode 100644 index 0000000..6080caf --- /dev/null +++ b/FFMpeg-Compressor/printer.py @@ -0,0 +1,46 @@ +from progress.bar import IncrementalBar +import colorama +import sys +import os + + +class Printer: + + def __init__(self, folder): + file_count = 0 + for folder, folders, file in os.walk(folder): + file_count += len(file) + self.bar = IncrementalBar('Compressing', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)') + self.bar.update() + + # Fill whole string with spaces for cleaning progress bar + @staticmethod + def clean_str(string): + return string + " " * (os.get_terminal_size().columns - len(string)) + + @staticmethod + def win_ascii_esc(): + if sys.platform == "win32": + colorama.init() + + def bar_print(self, string): + print(string) + self.bar.update() + + def info(self, string): + self.bar_print(self.clean_str(f"\r\033[100m- {string}\033[49m")) + + def warning(self, string): + self.bar_print(self.clean_str(f"\r\033[93m!\033[0m {string}\033[49m")) + + def error(self, string): + self.bar_print(self.clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m")) + + def files(self, source, dest, dest_ext, comment): + source_ext = os.path.splitext(source)[1] + source_name = os.path.splitext(source)[0] + + self.bar_print(self.clean_str(f"\r\033[0;32m\u2713\033[0m \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m")) + + def unknown_file(self, file): + self.bar_print(self.clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)")) diff --git a/FFMpeg-Compressor/utils.py b/FFMpeg-Compressor/utils.py new file mode 100644 index 0000000..48df9e9 --- /dev/null +++ b/FFMpeg-Compressor/utils.py @@ -0,0 +1,71 @@ +from shutil import copyfile +import sys +import os + +class Utils: + + def __init__(self, config, printer): + self.errors = 0 + self.config = config + self.printer = printer + + @staticmethod + def sys_pause(): + if sys.platform == "win32": + os.system("pause") + + @staticmethod + def get_size(directory): + total_size = 0 + for folder, folders, files in os.walk(directory): + for file in files: + if not os.path.islink(f"{folder}/{file}"): + total_size += os.path.getsize(f"{folder}/{file}") + return total_size + + def get_compression(self, source, output): + try: + source = self.get_size(source) + output = self.get_size(output) + + print(f"\nResult: {source/1024/1024:.2f}MB -> " + f"{output/1024/1024:.2f}MB ({(output - source)/1024/1024:.2f}MB)") + except ZeroDivisionError: + self.printer.warning("Nothing compressed!") + + def get_compression_status(self, orig_folder): + orig_folder_len = 0 + comp_folder_len = 0 + + for folder, folders, files in os.walk(orig_folder): + orig_folder_len += len(files) + + for folder, folders, files in os.walk(f'{orig_folder}_compressed'): + for file in files: + if not os.path.splitext(file)[1].count("(copy)"): + comp_folder_len += 1 + + if self.errors != 0: + self.printer.warning("Some files failed to compress!") + + if orig_folder_len == comp_folder_len: + self.printer.info("Success!") + self.get_compression(orig_folder, f"{orig_folder}_compressed") + else: + self.printer.warning("Original and compressed folders are not identical!") + self.get_compression(orig_folder, f"{orig_folder}_compressed") + + def add_unprocessed_file(self, orig_folder, new_folder): + if self.config['FFMPEG']['CopyUnprocessed']: + filename = orig_folder.split("/").pop() + copyfile(orig_folder, new_folder) + self.printer.info(f"File {filename} copied to compressed folder.") + + def check_duplicates(self, new_folder): + filename = new_folder.split().pop() + if os.path.exists(new_folder): + self.printer.warning( + f'Duplicate file has been found! Check manually this files - "{filename}", ' + f'"{os.path.splitext(filename)[0] + "(copy)" + os.path.splitext(filename)[1]}"') + return os.path.splitext(new_folder)[0] + "(copy)" + os.path.splitext(new_folder)[1] + return new_folder diff --git a/README.md b/README.md index 36f092f..370d3eb 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,25 @@ ## VNTools -Collection of tools used by administrators from VN Telegram Channel +Collection of tools used by VienDesu! Porting Team ### Tools -* `FFMpeg-Compressor` - Python utility uses ffmpeg to compress Visual Novel Resources -* `RenPy-Android-Unpack` - A simple Python script for unpacking Ren'Py based .apk and .obb files to ready to use Ren'Py SDK's Project +* `FFMpeg-Compressor` - Python utility to compress Visual Novel Resources +* `RenPy-Android-Unpack` - A Python script for extracting game project from Ren'Py based .apk and .obb files * `RenPy-Unpacker` - Simple .rpy script that will make any RenPy game unpack itself -* `VNDS-to-RenPy` - Simple script for converting vnds scripts to rpy +* `VNDS-to-RenPy` - Simple script for converting VNDS engine scripts to .rpy ones + +### Installation +#### Download from releases: + * Windows - `TODO` + * Linux - `TODO` + * MacOS - `TODO` + +#### Build tools as binaries: + * Run `./build.sh` on UNIX + * Run `.\build.bat` for Windows + * Arch Linux - `TODO` + * NixOS - `TODO` + +#### Install as python package: + * Run `pip install -U .` command in project folder + * Arch Linux - `TODO` + * NixOS - `TODO` diff --git a/RenPy-Android-Unpack/__init__.py b/RenPy-Android-Unpack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/RenPy-Android-Unpack/__main__.py b/RenPy-Android-Unpack/__main__.py new file mode 100755 index 0000000..8826079 --- /dev/null +++ b/RenPy-Android-Unpack/__main__.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +import colorama +import argparse +import sys +import os + +from printer import Printer +from actions import Actions + + +def args_init(): + parser = argparse.ArgumentParser( + prog='rendroid-unpack', + description='Extract Ren\'Py .apk and .obb files into Ren\'Py SDK\'s project' + ) + parser.add_argument('path') + parser.add_argument('-o', '--output') + return parser.parse_args() + + +if __name__ == '__main__': + if sys.platform == "win32": + colorama.init() + args = args_init() + if args.output: + output = args.output + else: + output = '' + actions = Actions(output) + printer = Printer() + + filename = args.path + if os.path.splitext(filename)[1] == '.apk' or os.path.splitext(filename)[1] == '.obb': + actions.clean(['assets'], True) + + printer.info(f'Extracting assets from {filename}... ') + actions.extract().assets(filename) + + printer.info('Renaming game assets... ') + actions.rename().files('assets') + actions.rename().dirs('assets') + + printer.info('Removing unneeded files... ') + if os.path.splitext(filename)[1] == '.apk': + actions.clean(['assets/renpy', 'assets/res'], False) + actions.clean(['assets/dexopt'], True) + + printer.info('Renaming directory... ') + actions.clean([os.path.splitext(filename)[0]], True) + os.rename(os.path.join(output, 'assets'), os.path.splitext(filename)[0]) + else: + Printer.err("It's not an .apk or .obb file!") diff --git a/RenPy-Android-Unpack/actions.py b/RenPy-Android-Unpack/actions.py new file mode 100755 index 0000000..e5d42e6 --- /dev/null +++ b/RenPy-Android-Unpack/actions.py @@ -0,0 +1,98 @@ +from zipfile import ZipFile, BadZipFile +from PIL import Image +import shutil +import os + +from printer import Printer + + +class Extract: + + def __init__(self, output: str): + self.output = output + + @staticmethod + def folder(zip_ref: ZipFile, path: str, dest: str): + for content in zip_ref.namelist(): + if content.split('/')[0] == path: + zip_ref.extract(content, dest) + + @staticmethod + def icon(directory: str): + icons = [] + for folder, folders, files in os.walk(directory): + for file in os.listdir(folder): + if os.path.splitext(file)[1] == ".png": + image = Image.open(f"{folder}/{file}") + if image.size[0] == 432 and image.size[1] == 432: + icons.append(f"{folder}/{file}") + if len(icons) == 0: + raise KeyError + return icons + + def assets(self, file: str): + try: + with ZipFile(file, 'r') as zip_ref: + self.folder(zip_ref, 'assets', self.output) + if os.path.splitext(file)[1] == '.apk': + try: + # ~Ren'Py 8, 7 + self.folder(zip_ref, 'res', os.path.join(self.output, 'assets')) + for icon in self.icon(os.path.join(self.output, 'assets/res')): + os.rename(icon, os.path.join(self.output, "assets", os.path.split(icon)[1])) + except KeyError: + try: + # ~Ren'Py 6 + zip_ref.extract('res/drawable/icon.png', os.path.join(self.output, 'assets')) + os.rename(os.path.join(self.output, 'assets/res/drawable/icon.png'), + os.path.join(self.output, 'assets/icon.png')) + except KeyError: + Printer.warn("Icon not found. Maybe it is not supported apk?") + except BadZipFile: + Printer.err("Cant extract .apk file!") + + +class Rename: + + def __init__(self, output): + self.output = output + + def files(self, directory: str): + for dir_ in os.walk(os.path.join(self.output, directory)): + for file in dir_[2]: + path = f'{dir_[0]}/{file}' + folder = '/'.join(path.split('/')[:len(path.split('/')) - 1]) + newname = f'{path.split("/").pop().replace("x-", "")}' + os.rename(path, f'{folder}/{newname}') + + def dirs(self, directory: str): + dirs = [] + for dir_ in os.walk(os.path.join(self.output, directory)): + dirs.append(dir_[0]) + dirs.reverse() + dirs.pop() + for dir__ in dirs: + folder = '/'.join(dir__.split('/')[:len(dir__.split('/')) - 1]) + newname = f'{dir__.split("/").pop().replace("x-", "")}' + os.rename(dir__, f'{folder}/{newname}') + + +class Actions: + + def __init__(self, output: str): + self.output = output + + def extract(self) -> Extract: + return Extract(self.output) + + def rename(self) -> Rename: + return Rename(self.output) + + def clean(self, names: list, ignore: bool): + for name in names: + name = os.path.join(self.output, name) + try: + shutil.rmtree(name) + except FileNotFoundError: + if not ignore: + Printer.warn(f"Path {name} not found!") diff --git a/RenPy-Android-Unpack/printer.py b/RenPy-Android-Unpack/printer.py new file mode 100644 index 0000000..2c3d31b --- /dev/null +++ b/RenPy-Android-Unpack/printer.py @@ -0,0 +1,14 @@ +class Printer: + + @staticmethod + def info(msg: str): + print(f"\033[100m[INFO] {msg}\033[49m") + + @staticmethod + def warn(msg: str): + print(f"\033[93m[WARN]\033[0m {msg}\033[49m") + + @staticmethod + def err(msg: str): + print(f"\033[31m[ERROR]\033[0m {msg} Exiting...\033[49m") + exit() diff --git a/RenPy-Android-Unpack/requirements.txt b/RenPy-Android-Unpack/requirements.txt deleted file mode 100644 index 3a6b560..0000000 --- a/RenPy-Android-Unpack/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -Pillow==9.5.0 -colorama==0.4.6 \ No newline at end of file diff --git a/RenPy-Android-Unpack/unpack.py b/RenPy-Android-Unpack/unpack.py deleted file mode 100755 index 8b74db7..0000000 --- a/RenPy-Android-Unpack/unpack.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3 -from PIL import Image -import colorama -import zipfile -import shutil -import os -import sys - - -def printer(msg, level): - match level: - case "info": - print(f"\033[100m[INFO] {msg}\033[49m") - case "warn": - print(f"\033[93m[WARN]\033[0m {msg}\033[49m") - case "err": - print(f"\033[31m[ERROR]\033[0m {msg} Exiting...\033[49m") - exit() - - -def extract_folder(zip_ref, path, dest): - for content in zip_ref.namelist(): - if content.split('/')[0] == path: - zip_ref.extract(content, dest) - - -def find_modern_icon(directory): - icons = [] - for folder, folders, files in os.walk(directory): - for file in os.listdir(folder): - if os.path.splitext(file)[1] == ".png": - image = Image.open(f"{folder}/{file}") - if image.size[0] == 432 and image.size[1] == 432: - icons.append(f"{folder}/{file}") - if len(icons) == 0: - raise KeyError - return icons - - -def extract_assets(file): - try: - with zipfile.ZipFile(file, 'r') as zip_ref: - extract_folder(zip_ref, 'assets', '') - if os.path.splitext(file)[1] == '.apk': - try: - # ~Ren'Py 8, 7 - extract_folder(zip_ref, 'res', 'assets') - for icon in find_modern_icon('assets/res'): - os.rename(icon, f"assets/{os.path.split(icon)[1]}") - except KeyError: - try: - # ~Ren'Py 6 - zip_ref.extract('res/drawable/icon.png', 'assets') - os.rename('assets/res/drawable/icon.png', 'assets/icon.png') - except KeyError: - printer("Icon not found. Maybe it is not supported apk?", "warn") - except zipfile.BadZipFile: - return printer("Cant extract .apk file!", "err") - - -def rename_files(directory): - for dir_ in os.walk(directory): - for file in dir_[2]: - path = f'{dir_[0]}/{file}' - folder = '/'.join(path.split('/')[:len(path.split('/')) - 1]) - newname = f'{path.split("/").pop().replace("x-", "")}' - os.rename(path, f'{folder}/{newname}') - - -def rename_dirs(directory): - dirs = [] - for dir_ in os.walk(directory): - dirs.append(dir_[0]) - dirs.reverse() - dirs.pop() - for dir__ in dirs: - folder = '/'.join(dir__.split('/')[:len(dir__.split('/')) - 1]) - newname = f'{dir__.split("/").pop().replace("x-", "")}' - os.rename(dir__, f'{folder}/{newname}') - - -def remove_unneeded(names, ignore): - for name in names: - try: - shutil.rmtree(name) - except FileNotFoundError: - if not ignore: - printer(f"Path {name} not found!", "warn") - - -if __name__ == '__main__': - if sys.platform == "win32": - colorama.init() - for filename in os.listdir(os.getcwd()): - if os.path.splitext(filename)[1] == '.apk' or os.path.splitext(filename)[1] == '.obb': - remove_unneeded(['assets'], True) - printer(f'Extracting assets from {filename}... ', "info") - extract_assets(filename) - printer('Renaming game assets... ', "info") - rename_files('assets') - rename_dirs('assets') - printer('Removing unneeded files... ', "info") - if os.path.splitext(filename)[1] == '.apk': - remove_unneeded(['assets/renpy', 'assets/res'], False) - remove_unneeded(['assets/dexopt'], True) - printer('Renaming directory... ', "info") - remove_unneeded([os.path.splitext(filename)[0]], True) - os.rename('assets', os.path.splitext(filename)[0]) diff --git a/VNDS-to-RenPy/__init__.py b/VNDS-to-RenPy/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/VNDS-to-RenPy/convert.py b/VNDS-to-RenPy/__main__.py similarity index 100% rename from VNDS-to-RenPy/convert.py rename to VNDS-to-RenPy/__main__.py diff --git a/build.bat b/build.bat old mode 100644 new mode 100755 index 237c7e3..688d43d --- a/build.bat +++ b/build.bat @@ -3,16 +3,16 @@ if not defined VIRTUAL_ENV goto :venv_error mkdir output mkdir output\bin -python -m pip install -r FFMpeg-Compressor\requirements.txt || goto :exit -python -m pip install -r RenPy-Android-Unpack\requirements.txt || goto :exit +python -m pip install -r requirements.txt || goto :exit python -m pip install Nuitka || goto :exit -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=ffmpeg-comp FFMpeg-Compressor\main.py || goto :exit +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=ffmpeg-comp FFMpeg-Compressor\__main__.py || goto :exit xcopy FFMpeg-Compressor\ffmpeg-comp.toml output\bin /Y move /Y output\ffmpeg-comp.exe output\bin -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=rendroid-unpack RenPy-Android-Unpack\unpack.py || goto :exit +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=rendroid-unpack RenPy-Android-Unpack\__main__.py || goto :exit move /Y output\rendroid-unpack.exe output\bin -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=vnds2renpy VNDS-to-RenPy/convert.py || goto :exit +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=vnds2renpy VNDS-to-RenPy/__main__.py || goto :exit move /Y output\vnds2renpy.exe output\bin +echo "Done! You can get binaries into output\bin directory" :venv_error echo "Please create and activate venv before running this script: python -m venv .\venv && .\venv\Scripts\activate.bat" diff --git a/build.sh b/build.sh index 7e21bbf..90e774e 100755 --- a/build.sh +++ b/build.sh @@ -8,17 +8,17 @@ fi mkdir -p output mkdir -p output/bin -python3 -m pip install -r FFMpeg-Compressor/requirements.txt -python3 -m pip install -r RenPy-Android-Unpack/requirements.txt +python3 -m pip install -r requirements.txt python3 -m pip install Nuitka case "$(uname -s)" in Linux*) jobs="--jobs=$(nproc)";; Darwin*) jobs="--jobs=$(sysctl -n hw.ncpu)";; esac -python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=ffmpeg-comp FFMpeg-Compressor/main.py +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=ffmpeg-comp FFMpeg-Compressor/__main__.py cp FFMpeg-Compressor/ffmpeg-comp.toml output/bin mv output/ffmpeg-comp output/bin -python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=rendroid-unpack RenPy-Android-Unpack/unpack.py +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=rendroid-unpack RenPy-Android-Unpack/__main__.py mv output/rendroid-unpack output/bin -python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=vnds2renpy VNDS-to-RenPy/convert.py -mv output/vnds2renpy output/bin \ No newline at end of file +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=vnds2renpy VNDS-to-RenPy/__main__.py +mv output/vnds2renpy output/bin +echo "Done! You can get binaries into output/bin directory" \ No newline at end of file diff --git a/FFMpeg-Compressor/requirements.txt b/requirements.txt similarity index 100% rename from FFMpeg-Compressor/requirements.txt rename to requirements.txt From 85df574d3c1fff90f9775000f4e0fdd1db22b281 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Thu, 29 Aug 2024 02:34:10 +0300 Subject: [PATCH 02/24] Python packaging --- .gitignore | 1 + .../README.md | 0 .../__init__.py | 0 RenDroidUnpack/__main__.py | 6 ++++ .../actions.py | 2 +- .../application.py | 7 ++--- .../printer.py | 0 {RenPy-Unpacker => RenPyRipper}/README.md | 0 {RenPy-Unpacker => RenPyRipper}/unpack.rpy | 0 {VNDS-to-RenPy => VNDS2RenPy}/README.md | 0 .../__init__.py | 0 {VNDS-to-RenPy => VNDS2RenPy}/__main__.py | 0 {FFMpeg-Compressor => VNRecode}/README.md | 0 {VNDS-to-RenPy => VNRecode}/__init__.py | 0 {FFMpeg-Compressor => VNRecode}/__main__.py | 10 +++--- .../application.py | 0 {FFMpeg-Compressor => VNRecode}/compress.py | 0 {FFMpeg-Compressor => VNRecode}/config.py | 1 + .../ffmpeg-comp.toml | 0 {FFMpeg-Compressor => VNRecode}/printer.py | 0 {FFMpeg-Compressor => VNRecode}/utils.py | 0 pyproject.toml | 31 +++++++++++++++++++ 22 files changed, 48 insertions(+), 10 deletions(-) rename {RenPy-Android-Unpack => RenDroidUnpack}/README.md (100%) rename {FFMpeg-Compressor => RenDroidUnpack}/__init__.py (100%) create mode 100755 RenDroidUnpack/__main__.py rename {RenPy-Android-Unpack => RenDroidUnpack}/actions.py (99%) rename RenPy-Android-Unpack/__main__.py => RenDroidUnpack/application.py (94%) mode change 100755 => 100644 rename {RenPy-Android-Unpack => RenDroidUnpack}/printer.py (100%) rename {RenPy-Unpacker => RenPyRipper}/README.md (100%) rename {RenPy-Unpacker => RenPyRipper}/unpack.rpy (100%) rename {VNDS-to-RenPy => VNDS2RenPy}/README.md (100%) rename {RenPy-Android-Unpack => VNDS2RenPy}/__init__.py (100%) rename {VNDS-to-RenPy => VNDS2RenPy}/__main__.py (100%) rename {FFMpeg-Compressor => VNRecode}/README.md (100%) rename {VNDS-to-RenPy => VNRecode}/__init__.py (100%) rename {FFMpeg-Compressor => VNRecode}/__main__.py (67%) rename {FFMpeg-Compressor => VNRecode}/application.py (100%) rename {FFMpeg-Compressor => VNRecode}/compress.py (100%) rename {FFMpeg-Compressor => VNRecode}/config.py (97%) rename {FFMpeg-Compressor => VNRecode}/ffmpeg-comp.toml (100%) rename {FFMpeg-Compressor => VNRecode}/printer.py (100%) rename {FFMpeg-Compressor => VNRecode}/utils.py (100%) create mode 100644 pyproject.toml diff --git a/.gitignore b/.gitignore index abe0419..d9d86ba 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /tests/ /build/ /VNTools.egg-info/ +/dist/ diff --git a/RenPy-Android-Unpack/README.md b/RenDroidUnpack/README.md similarity index 100% rename from RenPy-Android-Unpack/README.md rename to RenDroidUnpack/README.md diff --git a/FFMpeg-Compressor/__init__.py b/RenDroidUnpack/__init__.py similarity index 100% rename from FFMpeg-Compressor/__init__.py rename to RenDroidUnpack/__init__.py diff --git a/RenDroidUnpack/__main__.py b/RenDroidUnpack/__main__.py new file mode 100755 index 0000000..6e79193 --- /dev/null +++ b/RenDroidUnpack/__main__.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + +from . import application + +if __name__ == '__main__': + application.launch() diff --git a/RenPy-Android-Unpack/actions.py b/RenDroidUnpack/actions.py similarity index 99% rename from RenPy-Android-Unpack/actions.py rename to RenDroidUnpack/actions.py index e5d42e6..489eee9 100755 --- a/RenPy-Android-Unpack/actions.py +++ b/RenDroidUnpack/actions.py @@ -3,7 +3,7 @@ from PIL import Image import shutil import os -from printer import Printer +from .printer import Printer class Extract: diff --git a/RenPy-Android-Unpack/__main__.py b/RenDroidUnpack/application.py old mode 100755 new mode 100644 similarity index 94% rename from RenPy-Android-Unpack/__main__.py rename to RenDroidUnpack/application.py index 8826079..91d1577 --- a/RenPy-Android-Unpack/__main__.py +++ b/RenDroidUnpack/application.py @@ -4,8 +4,8 @@ import argparse import sys import os -from printer import Printer -from actions import Actions +from .printer import Printer +from .actions import Actions def args_init(): @@ -17,8 +17,7 @@ def args_init(): parser.add_argument('-o', '--output') return parser.parse_args() - -if __name__ == '__main__': +def launch(): if sys.platform == "win32": colorama.init() args = args_init() diff --git a/RenPy-Android-Unpack/printer.py b/RenDroidUnpack/printer.py similarity index 100% rename from RenPy-Android-Unpack/printer.py rename to RenDroidUnpack/printer.py diff --git a/RenPy-Unpacker/README.md b/RenPyRipper/README.md similarity index 100% rename from RenPy-Unpacker/README.md rename to RenPyRipper/README.md diff --git a/RenPy-Unpacker/unpack.rpy b/RenPyRipper/unpack.rpy similarity index 100% rename from RenPy-Unpacker/unpack.rpy rename to RenPyRipper/unpack.rpy diff --git a/VNDS-to-RenPy/README.md b/VNDS2RenPy/README.md similarity index 100% rename from VNDS-to-RenPy/README.md rename to VNDS2RenPy/README.md diff --git a/RenPy-Android-Unpack/__init__.py b/VNDS2RenPy/__init__.py similarity index 100% rename from RenPy-Android-Unpack/__init__.py rename to VNDS2RenPy/__init__.py diff --git a/VNDS-to-RenPy/__main__.py b/VNDS2RenPy/__main__.py similarity index 100% rename from VNDS-to-RenPy/__main__.py rename to VNDS2RenPy/__main__.py diff --git a/FFMpeg-Compressor/README.md b/VNRecode/README.md similarity index 100% rename from FFMpeg-Compressor/README.md rename to VNRecode/README.md diff --git a/VNDS-to-RenPy/__init__.py b/VNRecode/__init__.py similarity index 100% rename from VNDS-to-RenPy/__init__.py rename to VNRecode/__init__.py diff --git a/FFMpeg-Compressor/__main__.py b/VNRecode/__main__.py similarity index 67% rename from FFMpeg-Compressor/__main__.py rename to VNRecode/__main__.py index aef4d34..4e7fb68 100644 --- a/FFMpeg-Compressor/__main__.py +++ b/VNRecode/__main__.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -from application import Application -from compress import Compress -from printer import Printer -from config import Config -from utils import Utils +from .application import Application +from .compress import Compress +from .printer import Printer +from .config import Config +from .utils import Utils def init(): diff --git a/FFMpeg-Compressor/application.py b/VNRecode/application.py similarity index 100% rename from FFMpeg-Compressor/application.py rename to VNRecode/application.py diff --git a/FFMpeg-Compressor/compress.py b/VNRecode/compress.py similarity index 100% rename from FFMpeg-Compressor/compress.py rename to VNRecode/compress.py diff --git a/FFMpeg-Compressor/config.py b/VNRecode/config.py similarity index 97% rename from FFMpeg-Compressor/config.py rename to VNRecode/config.py index d36e3d4..a556c5e 100644 --- a/FFMpeg-Compressor/config.py +++ b/VNRecode/config.py @@ -24,4 +24,5 @@ class Config: config = tomllib.load(cfile) else: print("Failed to find config. Check `ffmpeg-comp -h` to more info") + exit(255) return cls(config=config, args=args) diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/VNRecode/ffmpeg-comp.toml similarity index 100% rename from FFMpeg-Compressor/ffmpeg-comp.toml rename to VNRecode/ffmpeg-comp.toml diff --git a/FFMpeg-Compressor/printer.py b/VNRecode/printer.py similarity index 100% rename from FFMpeg-Compressor/printer.py rename to VNRecode/printer.py diff --git a/FFMpeg-Compressor/utils.py b/VNRecode/utils.py similarity index 100% rename from FFMpeg-Compressor/utils.py rename to VNRecode/utils.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..766fb7f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = [ + "setuptools >= 61.0" +] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = ["VNRecode", "RenDroidUnpack", "VNDS2RenPy"] +include-package-data = true + +[tool.setuptools.package-data] +'VNRecode' = ['*.py'] +'VNDS2RenPy' = ['*.py'] +'RenDroidUnpack' = ['*.py'] + +[project.scripts] +vnrecode = "VNRecode.__main__:init" +vnds2renpy = "VNDS2RenPy.__main__:main" +rendroid-unpack = "RenDroidUnpack.application:launch" + +[project] +name = "vntools" +version = "2.0-dev" +requires-python = ">= 3.11" +dependencies = [ + "Pillow==10.3.0", + "pillow-avif-plugin==1.4.3", + "python-ffmpeg==2.0.12", + "progress==1.6", + "colorama==0.4.6" +] \ No newline at end of file From e5fa49ad53cc6b6191ae9ddc0b666e4fa1461841 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Thu, 29 Aug 2024 02:49:44 +0300 Subject: [PATCH 03/24] Fix packaging names --- .gitignore | 2 +- build.bat | 12 ++++++------ build.sh | 12 ++++++------ pyproject.toml | 14 +++++++------- {RenPyRipper => renpy-ripper}/README.md | 0 RenPyRipper/unpack.rpy => renpy-ripper/ripper.rpy | 0 {RenDroidUnpack => unrenapk}/README.md | 0 {RenDroidUnpack => unrenapk}/__init__.py | 0 {RenDroidUnpack => unrenapk}/__main__.py | 0 {RenDroidUnpack => unrenapk}/actions.py | 0 {RenDroidUnpack => unrenapk}/application.py | 2 +- {RenDroidUnpack => unrenapk}/printer.py | 0 {VNDS2RenPy => vnds2renpy}/README.md | 0 {VNDS2RenPy => vnds2renpy}/__init__.py | 0 {VNDS2RenPy => vnds2renpy}/__main__.py | 0 {VNRecode => vnrecode}/README.md | 0 {VNRecode => vnrecode}/__init__.py | 0 {VNRecode => vnrecode}/__main__.py | 0 {VNRecode => vnrecode}/application.py | 0 {VNRecode => vnrecode}/compress.py | 0 {VNRecode => vnrecode}/config.py | 6 +++--- {VNRecode => vnrecode}/printer.py | 0 {VNRecode => vnrecode}/utils.py | 0 .../ffmpeg-comp.toml => vnrecode/vnrecode.toml | 0 24 files changed, 24 insertions(+), 24 deletions(-) rename {RenPyRipper => renpy-ripper}/README.md (100%) rename RenPyRipper/unpack.rpy => renpy-ripper/ripper.rpy (100%) rename {RenDroidUnpack => unrenapk}/README.md (100%) rename {RenDroidUnpack => unrenapk}/__init__.py (100%) rename {RenDroidUnpack => unrenapk}/__main__.py (100%) rename {RenDroidUnpack => unrenapk}/actions.py (100%) rename {RenDroidUnpack => unrenapk}/application.py (97%) rename {RenDroidUnpack => unrenapk}/printer.py (100%) rename {VNDS2RenPy => vnds2renpy}/README.md (100%) rename {VNDS2RenPy => vnds2renpy}/__init__.py (100%) rename {VNDS2RenPy => vnds2renpy}/__main__.py (100%) rename {VNRecode => vnrecode}/README.md (100%) rename {VNRecode => vnrecode}/__init__.py (100%) rename {VNRecode => vnrecode}/__main__.py (100%) rename {VNRecode => vnrecode}/application.py (100%) rename {VNRecode => vnrecode}/compress.py (100%) rename {VNRecode => vnrecode}/config.py (76%) rename {VNRecode => vnrecode}/printer.py (100%) rename {VNRecode => vnrecode}/utils.py (100%) rename VNRecode/ffmpeg-comp.toml => vnrecode/vnrecode.toml (100%) diff --git a/.gitignore b/.gitignore index d9d86ba..e6aa831 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ /output/ /tests/ /build/ -/VNTools.egg-info/ /dist/ +/vntools.egg-info/ diff --git a/build.bat b/build.bat index 688d43d..89343d1 100755 --- a/build.bat +++ b/build.bat @@ -5,12 +5,12 @@ mkdir output mkdir output\bin python -m pip install -r requirements.txt || goto :exit python -m pip install Nuitka || goto :exit -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=ffmpeg-comp FFMpeg-Compressor\__main__.py || goto :exit -xcopy FFMpeg-Compressor\ffmpeg-comp.toml output\bin /Y -move /Y output\ffmpeg-comp.exe output\bin -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=rendroid-unpack RenPy-Android-Unpack\__main__.py || goto :exit -move /Y output\rendroid-unpack.exe output\bin -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=vnds2renpy VNDS-to-RenPy/__main__.py || goto :exit +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=vnrecode vnrecode\__main__.py || goto :exit +xcopy vnrecode\vnrecode.toml output\bin /Y +move /Y output\vnrecode.exe output\bin +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=unrenapk unrenapk\__main__.py || goto :exit +move /Y output\unrenapk.exe output\bin +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=vnds2renpy vnds2renpy/__main__.py || goto :exit move /Y output\vnds2renpy.exe output\bin echo "Done! You can get binaries into output\bin directory" diff --git a/build.sh b/build.sh index 90e774e..a7cf3b3 100755 --- a/build.sh +++ b/build.sh @@ -14,11 +14,11 @@ case "$(uname -s)" in Linux*) jobs="--jobs=$(nproc)";; Darwin*) jobs="--jobs=$(sysctl -n hw.ncpu)";; esac -python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=ffmpeg-comp FFMpeg-Compressor/__main__.py -cp FFMpeg-Compressor/ffmpeg-comp.toml output/bin -mv output/ffmpeg-comp output/bin -python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=rendroid-unpack RenPy-Android-Unpack/__main__.py -mv output/rendroid-unpack output/bin -python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=vnds2renpy VNDS-to-RenPy/__main__.py +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=vnrecode vnrecode/__main__.py +cp vnrecode/vnrecode.toml output/bin +mv output/vnrecode output/bin +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=unrenapk unrenapk/__main__.py +mv output/unrenapk output/bin +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=vnds2renpy vnds2renpy/__main__.py mv output/vnds2renpy output/bin echo "Done! You can get binaries into output/bin directory" \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 766fb7f..be74c59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,18 +5,18 @@ requires = [ build-backend = "setuptools.build_meta" [tool.setuptools] -packages = ["VNRecode", "RenDroidUnpack", "VNDS2RenPy"] +packages = ["vnrecode", "unrenapk", "vnds2renpy"] include-package-data = true [tool.setuptools.package-data] -'VNRecode' = ['*.py'] -'VNDS2RenPy' = ['*.py'] -'RenDroidUnpack' = ['*.py'] +'vnrecode' = ['*.py', '*.toml'] +'vnds2renpy' = ['*.py'] +'unrenapk' = ['*.py'] [project.scripts] -vnrecode = "VNRecode.__main__:init" -vnds2renpy = "VNDS2RenPy.__main__:main" -rendroid-unpack = "RenDroidUnpack.application:launch" +vnrecode = "vnrecode.__main__:init" +vnds2renpy = "vnds2renpy.__main__:main" +unrenapk = "unrenapk.application:launch" [project] name = "vntools" diff --git a/RenPyRipper/README.md b/renpy-ripper/README.md similarity index 100% rename from RenPyRipper/README.md rename to renpy-ripper/README.md diff --git a/RenPyRipper/unpack.rpy b/renpy-ripper/ripper.rpy similarity index 100% rename from RenPyRipper/unpack.rpy rename to renpy-ripper/ripper.rpy diff --git a/RenDroidUnpack/README.md b/unrenapk/README.md similarity index 100% rename from RenDroidUnpack/README.md rename to unrenapk/README.md diff --git a/RenDroidUnpack/__init__.py b/unrenapk/__init__.py similarity index 100% rename from RenDroidUnpack/__init__.py rename to unrenapk/__init__.py diff --git a/RenDroidUnpack/__main__.py b/unrenapk/__main__.py similarity index 100% rename from RenDroidUnpack/__main__.py rename to unrenapk/__main__.py diff --git a/RenDroidUnpack/actions.py b/unrenapk/actions.py similarity index 100% rename from RenDroidUnpack/actions.py rename to unrenapk/actions.py diff --git a/RenDroidUnpack/application.py b/unrenapk/application.py similarity index 97% rename from RenDroidUnpack/application.py rename to unrenapk/application.py index 91d1577..b66d6ba 100644 --- a/RenDroidUnpack/application.py +++ b/unrenapk/application.py @@ -10,7 +10,7 @@ from .actions import Actions def args_init(): parser = argparse.ArgumentParser( - prog='rendroid-unpack', + prog='unrenapk', description='Extract Ren\'Py .apk and .obb files into Ren\'Py SDK\'s project' ) parser.add_argument('path') diff --git a/RenDroidUnpack/printer.py b/unrenapk/printer.py similarity index 100% rename from RenDroidUnpack/printer.py rename to unrenapk/printer.py diff --git a/VNDS2RenPy/README.md b/vnds2renpy/README.md similarity index 100% rename from VNDS2RenPy/README.md rename to vnds2renpy/README.md diff --git a/VNDS2RenPy/__init__.py b/vnds2renpy/__init__.py similarity index 100% rename from VNDS2RenPy/__init__.py rename to vnds2renpy/__init__.py diff --git a/VNDS2RenPy/__main__.py b/vnds2renpy/__main__.py similarity index 100% rename from VNDS2RenPy/__main__.py rename to vnds2renpy/__main__.py diff --git a/VNRecode/README.md b/vnrecode/README.md similarity index 100% rename from VNRecode/README.md rename to vnrecode/README.md diff --git a/VNRecode/__init__.py b/vnrecode/__init__.py similarity index 100% rename from VNRecode/__init__.py rename to vnrecode/__init__.py diff --git a/VNRecode/__main__.py b/vnrecode/__main__.py similarity index 100% rename from VNRecode/__main__.py rename to vnrecode/__main__.py diff --git a/VNRecode/application.py b/vnrecode/application.py similarity index 100% rename from VNRecode/application.py rename to vnrecode/application.py diff --git a/VNRecode/compress.py b/vnrecode/compress.py similarity index 100% rename from VNRecode/compress.py rename to vnrecode/compress.py diff --git a/VNRecode/config.py b/vnrecode/config.py similarity index 76% rename from VNRecode/config.py rename to vnrecode/config.py index a556c5e..a7cf0ed 100644 --- a/VNRecode/config.py +++ b/vnrecode/config.py @@ -13,16 +13,16 @@ class Config: @classmethod def setup_config(cls): - parser = ArgumentParser(prog="ffmpeg-comp", + parser = ArgumentParser(prog="vnrecode", description="Python utility to compress Visual Novel Resources" ) parser.add_argument("source") - parser.add_argument("-c", "--config", default="ffmpeg-comp.toml") + parser.add_argument("-c", "--config", default="vnrecode.toml") args = parser.parse_args() if os.path.isfile(args.config): with open(args.config, "rb") as cfile: config = tomllib.load(cfile) else: - print("Failed to find config. Check `ffmpeg-comp -h` to more info") + print("Failed to find config. Check `vnrecode -h` to more info") exit(255) return cls(config=config, args=args) diff --git a/VNRecode/printer.py b/vnrecode/printer.py similarity index 100% rename from VNRecode/printer.py rename to vnrecode/printer.py diff --git a/VNRecode/utils.py b/vnrecode/utils.py similarity index 100% rename from VNRecode/utils.py rename to vnrecode/utils.py diff --git a/VNRecode/ffmpeg-comp.toml b/vnrecode/vnrecode.toml similarity index 100% rename from VNRecode/ffmpeg-comp.toml rename to vnrecode/vnrecode.toml From f240fdca5f5c2a0ebcd02d9d6f9c373b01ac23d7 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Thu, 29 Aug 2024 04:01:41 +0300 Subject: [PATCH 04/24] vnrecode: add cli parameters for configuration --- vnrecode/__main__.py | 8 +++--- vnrecode/application.py | 8 +++--- vnrecode/compress.py | 40 +++++++++++++++--------------- vnrecode/config.py | 26 +++++++++++++++++--- vnrecode/params.py | 54 +++++++++++++++++++++++++++++++++++++++++ vnrecode/utils.py | 6 ++--- 6 files changed, 109 insertions(+), 33 deletions(-) create mode 100644 vnrecode/params.py diff --git a/vnrecode/__main__.py b/vnrecode/__main__.py index 4e7fb68..39ef2d6 100644 --- a/vnrecode/__main__.py +++ b/vnrecode/__main__.py @@ -2,17 +2,19 @@ from .application import Application from .compress import Compress from .printer import Printer +from .params import Params from .config import Config from .utils import Utils def init(): config = Config.setup_config() + params = Params.setup(config.config, config.args) printer = Printer(config.args.source) - utils = Utils(config.config, printer) - compress = Compress(config.config, printer, utils) + utils = Utils(params, printer) + compress = Compress(params, printer, utils) - Application(config, compress, printer, utils).run() + Application(params, config.args, compress, printer, utils).run() if __name__ == "__main__": diff --git a/vnrecode/application.py b/vnrecode/application.py index 491267a..75e3107 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -7,9 +7,9 @@ import os class Application: - def __init__(self, config, compress, printer, utils): - self.config = config.config - self.args = config.args + def __init__(self, params, args, compress, printer, utils): + self.params = params + self.args = args self.compress = compress.compress self.printer = printer self.utils = utils @@ -35,7 +35,7 @@ class Application: self.printer.info(f'Compressing "{folder.replace(source, os.path.split(source)[-1])}" folder...') output = folder.replace(source, f"{source}_compressed") - with ThreadPoolExecutor(max_workers=self.config["FFMPEG"]["Workers"]) as executor: + with ThreadPoolExecutor(max_workers=self.params.workers) as executor: futures = [ executor.submit(self.compress, folder, file, source, output) for file in files if os.path.isfile(f'{folder}/{file}') diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 5c1a465..7dbc6fc 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -40,13 +40,13 @@ class File: class Compress: - def __init__(self, config, printer, utils): - self.config = config + def __init__(self, params, printer, utils): + self.params = params self.printer = printer self.utils = utils def audio(self, folder, file, target_folder, extension): - bitrate = self.config['AUDIO']['BitRate'] + bitrate = self.params.audio_bitrate try: (FFmpeg() .input(f'{folder}/{file}') @@ -58,16 +58,16 @@ class Compress: except FFmpegError as e: self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') self.utils.errors += 1 - if not self.config['FFMPEG']['HideErrors']: + if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") self.printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}") return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' def video(self, folder, file, target_folder, extension): - if not self.config['VIDEO']['SkipVideo']: - codec = self.config['VIDEO']['Codec'] - crf = self.config['VIDEO']['CRF'] + if not self.params.video_skip: + codec = self.params.video_codec + crf = self.params.video_crf try: (FFmpeg() @@ -82,7 +82,7 @@ class Compress: except FFmpegError as e: self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') self.utils.errors += 1 - if not self.config['FFMPEG']['HideErrors']: + if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") else: self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') @@ -90,20 +90,20 @@ class Compress: def image(self, folder, file, target_folder, extension): - quality = self.config['IMAGE']['Quality'] + quality = self.params.image_quality try: image = Image.open(f'{folder}/{file}') if (extension == "jpg" or extension == "jpeg" or - (extension == "webp" and not self.config['FFMPEG']['WebpRGBA'])): + (extension == "webp" and not self.params.webp_rgba)): if File.has_transparency(image): self.printer.warning(f"{file} has transparency. Changing to fallback...") - extension = self.config['IMAGE']['FallBackExtension'] + extension = self.params.image_fall_ext if File.has_transparency(image): image.convert('RGBA') - res_downscale = self.config['IMAGE']['ResDownScale'] + res_downscale = self.params.image_downscale if res_downscale != 1: width, height = image.size new_size = (int(width / res_downscale), int(height / res_downscale)) @@ -111,20 +111,20 @@ class Compress: image.save(self.utils.check_duplicates(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}"), optimize=True, - lossless=self.config['IMAGE']['Lossless'], + lossless=self.params.image_lossless, quality=quality, minimize_size=True) self.printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") except Exception as e: self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') self.utils.errors += 1 - if not self.config['FFMPEG']['HideErrors']: + if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' def unknown(self, folder, file, target_folder): - if self.config["FFMPEG"]["ForceCompress"]: + if self.params.force_compress: self.printer.unknown_file(file) try: (FFmpeg() @@ -135,7 +135,7 @@ class Compress: except FFmpegError as e: self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') self.utils.errors += 1 - if not self.config['FFMPEG']['HideErrors']: + if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") else: self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') @@ -145,15 +145,15 @@ class Compress: def compress(self, _dir, filename, source, output): match File.get_type(filename): case "audio": - out_file = self.audio(_dir, filename, output, self.config['AUDIO']['Extension']) + out_file = self.audio(_dir, filename, output, self.params.audio_ext) case "image": - out_file = self.image(_dir, filename, output, self.config['IMAGE']['Extension']) + out_file = self.image(_dir, filename, output, self.params.image_ext) case "video": - out_file = self.video(_dir, filename, output, self.config['VIDEO']['Extension']) + out_file = self.video(_dir, filename, output, self.params.video_ext) case "unknown": out_file = self.unknown(_dir, filename, output) - if self.config['FFMPEG']['MimicMode']: + if self.params.mimic_mode: try: os.rename(out_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed")) except FileNotFoundError: diff --git a/vnrecode/config.py b/vnrecode/config.py index a7cf0ed..4d1336c 100644 --- a/vnrecode/config.py +++ b/vnrecode/config.py @@ -1,8 +1,9 @@ -import os.path from argparse import Namespace, ArgumentParser from dataclasses import dataclass +from sysconfig import get_path from typing import Any import tomllib +import os @dataclass @@ -13,11 +14,30 @@ class Config: @classmethod def setup_config(cls): + default_config = os.path.join(get_path('purelib'), "vnrecode", "vnrecode.toml") parser = ArgumentParser(prog="vnrecode", description="Python utility to compress Visual Novel Resources" ) - parser.add_argument("source") - parser.add_argument("-c", "--config", default="vnrecode.toml") + parser.add_argument("source", help="SourceDir") + parser.add_argument("-c", "--config", default=default_config, help="ConfigFile") + parser.add_argument("-u", action="store_true", help="CopyUnprocessed") + parser.add_argument("-f", "--force", action="store_true", help="ForceCompress") + parser.add_argument("-m", "--mimic", action="store_true", help="MimicMode") + parser.add_argument("-s", "--silent", action="store_true", help="HideErrors") + parser.add_argument("--webprgba", action="store_true", help="WebpRGBA") + parser.add_argument("-j", "--jobs", type=int, help="Workers") + parser.add_argument("-ae", "--aext", help="Audio Extension") + parser.add_argument("-ab", "--abit", help="Audio Bitrate") + parser.add_argument("-id", "--idown", type=int, help="Image Downscale") + parser.add_argument("-ie", "--iext", help="Image Extension") + parser.add_argument("-ife", "--ifallext", help="Image Fallback Extension") + parser.add_argument("-il", "--ilossless", action="store_true", help="Image Lossless") + parser.add_argument("-iq", "--iquality", help="Image Quality") + parser.add_argument("--vcrf", help="Video CRF") + parser.add_argument("-vs", "--vskip", help="Video Skip") + parser.add_argument("-ve", "--vext", help="Video Extension") + parser.add_argument("-vc", "--vcodec", help="Video Codec") + args = parser.parse_args() if os.path.isfile(args.config): with open(args.config, "rb") as cfile: diff --git a/vnrecode/params.py b/vnrecode/params.py new file mode 100644 index 0000000..483d047 --- /dev/null +++ b/vnrecode/params.py @@ -0,0 +1,54 @@ +from dataclasses import dataclass +from typing import Self + + +@dataclass +class Params: + + copy_unprocessed: bool + force_compress: bool + mimic_mode: bool + hide_errors: bool + webp_rgba: bool + workers: int + + audio_ext: str + audio_bitrate: str + + image_downscale: int + image_ext: str + image_fall_ext: str + image_lossless: str + image_quality: int + + video_crf: int + video_skip: bool + video_ext: str + video_codec: str + + @classmethod + def setup(cls, config, args) -> Self: + copy_unprocessed = config["FFMPEG"]["CopyUnprocessed"] if not args.u else args.u + force_compress = config["FFMPEG"]["ForceCompress"] if not args.force else args.force + mimic_mode = config["FFMPEG"]["MimicMode"] if not args.mimic else args.mimic + hide_errors = config["FFMPEG"]["HideErrors"] if not args.silent else args.silent + workers = config["FFMPEG"]["Workers"] if args.jobs is None else args.jobs + webp_rgba = config["FFMPEG"]["WebpRGBA"] if not args.webprgba else args.webprgba + audio_ext = config["AUDIO"]["Extension"] if args.aext is None else args.aext + audio_bitrate = config["AUDIO"]["BitRate"] if args.abit is None else args.abit + image_downscale = config["IMAGE"]["ResDownScale"] if args.idown is None else args.idown + image_ext = config["IMAGE"]["Extension"] if args.iext is None else args.iext + image_fall_ext = config["IMAGE"]["FallBackExtension"] if args.ifallext is None else args.ifallext + image_lossless = config["IMAGE"]["Lossless"] if not args.ilossless else args.ilossless + image_quality = config["IMAGE"]["Quality"] if args.iquality is None else args.iquality + video_crf = config["VIDEO"]["CRF"] if args.vcrf is None else args.vcrf + video_skip = config["VIDEO"]["SkipVideo"] if args.vskip is None else args.vskip + video_ext = config["VIDEO"]["Extension"] if args.vext is None else args.vext + video_codec = config["VIDEO"]["Codec"] if args.vcodec is None else args.vcodec + + return cls( + copy_unprocessed, force_compress, mimic_mode, hide_errors, webp_rgba, workers, + audio_ext, audio_bitrate, + image_downscale, image_ext, image_fall_ext, image_lossless, image_quality, + video_crf, video_skip, video_ext, video_codec + ) diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 48df9e9..040da09 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -4,9 +4,9 @@ import os class Utils: - def __init__(self, config, printer): + def __init__(self, params, printer): self.errors = 0 - self.config = config + self.params = params self.printer = printer @staticmethod @@ -56,7 +56,7 @@ class Utils: self.get_compression(orig_folder, f"{orig_folder}_compressed") def add_unprocessed_file(self, orig_folder, new_folder): - if self.config['FFMPEG']['CopyUnprocessed']: + if self.params.copy_unprocessed: filename = orig_folder.split("/").pop() copyfile(orig_folder, new_folder) self.printer.info(f"File {filename} copied to compressed folder.") From 71c5764f26d1c6c5f158c143610b9d18450b55fb Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Tue, 3 Sep 2024 22:44:58 +0300 Subject: [PATCH 05/24] vnrecode: replace config class to params --- pyproject.toml | 2 +- vnrecode/__main__.py | 8 ++--- vnrecode/application.py | 5 ++- vnrecode/config.py | 48 ------------------------- vnrecode/params.py | 77 ++++++++++++++++++++++++++++++----------- 5 files changed, 63 insertions(+), 77 deletions(-) delete mode 100644 vnrecode/config.py diff --git a/pyproject.toml b/pyproject.toml index be74c59..4d8aa85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ packages = ["vnrecode", "unrenapk", "vnds2renpy"] include-package-data = true [tool.setuptools.package-data] -'vnrecode' = ['*.py', '*.toml'] +'vnrecode' = ['*.py'] 'vnds2renpy' = ['*.py'] 'unrenapk' = ['*.py'] diff --git a/vnrecode/__main__.py b/vnrecode/__main__.py index 39ef2d6..faf1839 100644 --- a/vnrecode/__main__.py +++ b/vnrecode/__main__.py @@ -3,18 +3,16 @@ from .application import Application from .compress import Compress from .printer import Printer from .params import Params -from .config import Config from .utils import Utils def init(): - config = Config.setup_config() - params = Params.setup(config.config, config.args) - printer = Printer(config.args.source) + params = Params.setup() + printer = Printer(params.source) utils = Utils(params, printer) compress = Compress(params, printer, utils) - Application(params, config.args, compress, printer, utils).run() + Application(params, compress, printer, utils).run() if __name__ == "__main__": diff --git a/vnrecode/application.py b/vnrecode/application.py index 75e3107..d32a3cf 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -7,9 +7,8 @@ import os class Application: - def __init__(self, params, args, compress, printer, utils): + def __init__(self, params, compress, printer, utils): self.params = params - self.args = args self.compress = compress.compress self.printer = printer self.utils = utils @@ -22,7 +21,7 @@ class Application: start_time = datetime.now() self.printer.win_ascii_esc() - source = os.path.abspath(self.args.source) + source = os.path.abspath(self.params.source) if os.path.exists(f"{source}_compressed"): shutil.rmtree(f"{source}_compressed") diff --git a/vnrecode/config.py b/vnrecode/config.py deleted file mode 100644 index 4d1336c..0000000 --- a/vnrecode/config.py +++ /dev/null @@ -1,48 +0,0 @@ -from argparse import Namespace, ArgumentParser -from dataclasses import dataclass -from sysconfig import get_path -from typing import Any -import tomllib -import os - - -@dataclass -class Config: - - config: dict[str, Any] - args: Namespace - - @classmethod - def setup_config(cls): - default_config = os.path.join(get_path('purelib'), "vnrecode", "vnrecode.toml") - parser = ArgumentParser(prog="vnrecode", - description="Python utility to compress Visual Novel Resources" - ) - parser.add_argument("source", help="SourceDir") - parser.add_argument("-c", "--config", default=default_config, help="ConfigFile") - parser.add_argument("-u", action="store_true", help="CopyUnprocessed") - parser.add_argument("-f", "--force", action="store_true", help="ForceCompress") - parser.add_argument("-m", "--mimic", action="store_true", help="MimicMode") - parser.add_argument("-s", "--silent", action="store_true", help="HideErrors") - parser.add_argument("--webprgba", action="store_true", help="WebpRGBA") - parser.add_argument("-j", "--jobs", type=int, help="Workers") - parser.add_argument("-ae", "--aext", help="Audio Extension") - parser.add_argument("-ab", "--abit", help="Audio Bitrate") - parser.add_argument("-id", "--idown", type=int, help="Image Downscale") - parser.add_argument("-ie", "--iext", help="Image Extension") - parser.add_argument("-ife", "--ifallext", help="Image Fallback Extension") - parser.add_argument("-il", "--ilossless", action="store_true", help="Image Lossless") - parser.add_argument("-iq", "--iquality", help="Image Quality") - parser.add_argument("--vcrf", help="Video CRF") - parser.add_argument("-vs", "--vskip", help="Video Skip") - parser.add_argument("-ve", "--vext", help="Video Extension") - parser.add_argument("-vc", "--vcodec", help="Video Codec") - - args = parser.parse_args() - if os.path.isfile(args.config): - with open(args.config, "rb") as cfile: - config = tomllib.load(cfile) - else: - print("Failed to find config. Check `vnrecode -h` to more info") - exit(255) - return cls(config=config, args=args) diff --git a/vnrecode/params.py b/vnrecode/params.py index 483d047..789cac5 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -1,6 +1,8 @@ +from argparse import ArgumentParser from dataclasses import dataclass from typing import Self - +import tomllib +import os @dataclass class Params: @@ -26,29 +28,64 @@ class Params: video_ext: str video_codec: str + source: str + @classmethod - def setup(cls, config, args) -> Self: - copy_unprocessed = config["FFMPEG"]["CopyUnprocessed"] if not args.u else args.u - force_compress = config["FFMPEG"]["ForceCompress"] if not args.force else args.force - mimic_mode = config["FFMPEG"]["MimicMode"] if not args.mimic else args.mimic - hide_errors = config["FFMPEG"]["HideErrors"] if not args.silent else args.silent - workers = config["FFMPEG"]["Workers"] if args.jobs is None else args.jobs - webp_rgba = config["FFMPEG"]["WebpRGBA"] if not args.webprgba else args.webprgba - audio_ext = config["AUDIO"]["Extension"] if args.aext is None else args.aext - audio_bitrate = config["AUDIO"]["BitRate"] if args.abit is None else args.abit - image_downscale = config["IMAGE"]["ResDownScale"] if args.idown is None else args.idown - image_ext = config["IMAGE"]["Extension"] if args.iext is None else args.iext - image_fall_ext = config["IMAGE"]["FallBackExtension"] if args.ifallext is None else args.ifallext - image_lossless = config["IMAGE"]["Lossless"] if not args.ilossless else args.ilossless - image_quality = config["IMAGE"]["Quality"] if args.iquality is None else args.iquality - video_crf = config["VIDEO"]["CRF"] if args.vcrf is None else args.vcrf - video_skip = config["VIDEO"]["SkipVideo"] if args.vskip is None else args.vskip - video_ext = config["VIDEO"]["Extension"] if args.vext is None else args.vext - video_codec = config["VIDEO"]["Codec"] if args.vcodec is None else args.vcodec + def setup(cls) -> Self: + parser = ArgumentParser(prog="vnrecode", + description="Python utility to compress Visual Novel Resources" + ) + parser.add_argument("source", help="SourceDir") + parser.add_argument("-c", "--config", help="ConfigFile") + parser.add_argument("-u", type=bool, help="CopyUnprocessed", default=True) + parser.add_argument("-f", "--force", type=bool, help="ForceCompress", default=False) + parser.add_argument("-m", "--mimic", type=bool, help="MimicMode", default=True) + parser.add_argument("-s", "--silent", type=bool, help="HideErrors", default=True) + parser.add_argument("--webprgba", type=bool, help="WebpRGBA", default=True) + parser.add_argument("-j", "--jobs", type=int, help="Workers", default=16) + parser.add_argument("-ae", "--aext", help="Audio Extension", default="opus") + parser.add_argument("-ab", "--abit", help="Audio Bitrate", default="128k") + parser.add_argument("-id", "--idown", type=int, help="Image Downscale", default=1) + parser.add_argument("-ie", "--iext", help="Image Extension", default="avif") + parser.add_argument("-ife", "--ifallext", help="Image Fallback Extension", default="webp") + parser.add_argument("-il", "--ilossless", type=bool, help="Image Lossless", default=True) + parser.add_argument("-iq", "--iquality", type=int, help="Image Quality", default=100) + parser.add_argument("--vcrf", help="Video CRF", type=int, default=27) + parser.add_argument("-vs", "--vskip", help="Video Skip", default=False) + parser.add_argument("-ve", "--vext", help="Video Extension", default="webm") + parser.add_argument("-vc", "--vcodec", help="Video Codec", default="libvpx-vp9") + args = parser.parse_args() + + if args.config is not None: + if os.path.isfile(args.config): + with open(args.config, "rb") as cfile: + config = tomllib.load(cfile) + else: + print("Failed to find config. Check `vnrecode -h` to more info") + exit(255) + + copy_unprocessed = config["FFMPEG"]["CopyUnprocessed"] if args.config else args.u + force_compress = config["FFMPEG"]["ForceCompress"] if args.config else args.force + mimic_mode = config["FFMPEG"]["MimicMode"] if args.config else args.mimic + hide_errors = config["FFMPEG"]["HideErrors"] if args.config else args.silent + workers = config["FFMPEG"]["Workers"] if args.config else args.jobs + webp_rgba = config["FFMPEG"]["WebpRGBA"] if args.config else args.webprgba + audio_ext = config["AUDIO"]["Extension"] if args.config else args.aext + audio_bitrate = config["AUDIO"]["BitRate"] if args.config else args.abit + image_downscale = config["IMAGE"]["ResDownScale"] if args.config else args.idown + image_ext = config["IMAGE"]["Extension"] if args.config else args.iext + image_fall_ext = config["IMAGE"]["FallBackExtension"] if args.config else args.ifallext + image_lossless = config["IMAGE"]["Lossless"] if args.config else args.ilossless + image_quality = config["IMAGE"]["Quality"] if args.config else args.iquality + video_crf = config["VIDEO"]["CRF"] if args.config else args.vcrf + video_skip = config["VIDEO"]["SkipVideo"] if args.config else args.vskip + video_ext = config["VIDEO"]["Extension"] if args.config else args.vext + video_codec = config["VIDEO"]["Codec"] if args.config else args.vcodec + source = args.source return cls( copy_unprocessed, force_compress, mimic_mode, hide_errors, webp_rgba, workers, audio_ext, audio_bitrate, image_downscale, image_ext, image_fall_ext, image_lossless, image_quality, - video_crf, video_skip, video_ext, video_codec + video_crf, video_skip, video_ext, video_codec, source ) From a69b17c624af1df1a9256a7ff5b577380d7b245e Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Tue, 3 Sep 2024 22:54:59 +0300 Subject: [PATCH 06/24] vnrecode: improve naming, fix mimic mode errors --- vnrecode/compress.py | 65 ++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 7dbc6fc..25d3177 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -45,54 +45,56 @@ class Compress: self.printer = printer self.utils = utils - def audio(self, folder, file, target_folder, extension): - bitrate = self.params.audio_bitrate + def audio(self, in_dir, file, out_dir, extension): + bit_rate = self.params.audio_bitrate + out_file = self.utils.check_duplicates(f'{out_dir}/{os.path.splitext(file)[0]}.{extension}') try: (FFmpeg() - .input(f'{folder}/{file}') + .input(f'{in_dir}/{file}') .option("hide_banner") - .output(self.utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), - {"b:a": bitrate, "loglevel": "error"}) + .output(out_file,{"b:a": bit_rate, "loglevel": "error"}) .execute() ) except FFmpegError as e: - self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') self.utils.errors += 1 if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") - self.printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}") - return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' + self.printer.files(file, os.path.splitext(file)[0], extension, f"{bit_rate}") + return out_file - def video(self, folder, file, target_folder, extension): + def video(self, in_dir, file, out_dir, extension): if not self.params.video_skip: + out_file = self.utils.check_duplicates(f'{out_dir}/{os.path.splitext(file)[0]}.{extension}') codec = self.params.video_codec crf = self.params.video_crf try: (FFmpeg() - .input(f'{folder}/{file}') + .input(f'{in_dir}/{file}') .option("hide_banner") .option("hwaccel", "auto") - .output(self.utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), - {"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) + .output(out_file,{"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) .execute() ) self.printer.files(file, os.path.splitext(file)[0], extension, codec) except FFmpegError as e: - self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') self.utils.errors += 1 if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") + return out_file else: - self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' + self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') + return f'{out_dir}/{os.path.splitext(file)[0]}.{extension}' - def image(self, folder, file, target_folder, extension): + def image(self, in_dir, file, out_dir, extension): quality = self.params.image_quality + out_file = self.utils.check_duplicates(f"{out_dir}/{os.path.splitext(file)[0]}.{extension}") try: - image = Image.open(f'{folder}/{file}') + image = Image.open(f'{in_dir}/{file}') if (extension == "jpg" or extension == "jpeg" or (extension == "webp" and not self.params.webp_rgba)): @@ -109,37 +111,39 @@ class Compress: new_size = (int(width / res_downscale), int(height / res_downscale)) image = image.resize(new_size) - image.save(self.utils.check_duplicates(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}"), + image.save(out_file, optimize=True, lossless=self.params.image_lossless, quality=quality, minimize_size=True) self.printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") except Exception as e: - self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') self.utils.errors += 1 if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") - return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' + return out_file - def unknown(self, folder, file, target_folder): + def unknown(self, in_dir, filename, out_dir): if self.params.force_compress: - self.printer.unknown_file(file) + self.printer.unknown_file(filename) + out_file = self.utils.check_duplicates(f'{out_dir}/{filename}') try: (FFmpeg() - .input(f'{folder}/{file}') - .output(self.utils.check_duplicates(f'{target_folder}/{file}')) + .input(f'{in_dir}/{filename}') + .output(out_file) .execute() ) except FFmpegError as e: - self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.add_unprocessed_file(f'{in_dir}/{filename}', f'{out_dir}/{filename}') self.utils.errors += 1 if not self.params.hide_errors: - self.printer.error(f"File {file} can't be processed! Error: {e}") + self.printer.error(f"File {filename} can't be processed! Error: {e}") + return out_file else: - self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - return f'{target_folder}/{file}' + self.utils.add_unprocessed_file(f'{in_dir}/{filename}', f'{out_dir}/{filename}') + return f'{out_dir}/{filename}' def compress(self, _dir, filename, source, output): @@ -154,10 +158,7 @@ class Compress: out_file = self.unknown(_dir, filename, output) if self.params.mimic_mode: - try: - os.rename(out_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed")) - except FileNotFoundError: - self.printer.warning(f"File {out_file} failed to copy to out dir") + os.rename(out_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed")) self.printer.bar.update() self.printer.bar.next() From 03647d4b8446402d2a05b65afbdffc512750a449 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Tue, 3 Sep 2024 23:00:19 +0300 Subject: [PATCH 07/24] vnrecode: rewrite get_type method --- vnrecode/compress.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 25d3177..75202ee 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -8,19 +8,18 @@ class File: @staticmethod def get_type(filename): - audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] - image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'] - video_ext = ['.3gp' '.amv', '.avi', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', - '.webm', '.ogv'] - if os.path.splitext(filename)[1] in audio_ext: - return "audio" - elif os.path.splitext(filename)[1] in image_ext: - return "image" - elif os.path.splitext(filename)[1] in video_ext: - return "video" - else: - return "unknown" + extensions = { + "audio": ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'], + "image": ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'], + "video": ['.3gp' '.amv', '.avi', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', + '.webm', '.ogv'] + } + + for file_type in extensions: + if os.path.splitext(filename)[1] in extensions[file_type]: + return file_type + return "unknown" @staticmethod def has_transparency(img: Image) -> bool: From 92474b4aa41bbbf555fcf72cefed768758868cdc Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 01:21:59 +0300 Subject: [PATCH 08/24] vnrecode: improve duplications check for mt --- vnrecode/application.py | 1 + vnrecode/compress.py | 11 +++++------ vnrecode/utils.py | 27 +++++++++++++++++++++------ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/vnrecode/application.py b/vnrecode/application.py index d32a3cf..c7cb7cd 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -42,6 +42,7 @@ class Application: for future in as_completed(futures): future.result() + self.utils.print_duplicates() self.utils.get_compression_status(source) self.utils.sys_pause() print(f"Time taken: {datetime.now() - start_time}") \ No newline at end of file diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 75202ee..249061e 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -46,7 +46,7 @@ class Compress: def audio(self, in_dir, file, out_dir, extension): bit_rate = self.params.audio_bitrate - out_file = self.utils.check_duplicates(f'{out_dir}/{os.path.splitext(file)[0]}.{extension}') + out_file = self.utils.check_duplicates(in_dir, out_dir, f'{os.path.splitext(file)[0]}.{extension}') try: (FFmpeg() .input(f'{in_dir}/{file}') @@ -65,7 +65,7 @@ class Compress: def video(self, in_dir, file, out_dir, extension): if not self.params.video_skip: - out_file = self.utils.check_duplicates(f'{out_dir}/{os.path.splitext(file)[0]}.{extension}') + out_file = self.utils.check_duplicates(in_dir, out_dir, f'{os.path.splitext(file)[0]}.{extension}') codec = self.params.video_codec crf = self.params.video_crf @@ -91,7 +91,7 @@ class Compress: def image(self, in_dir, file, out_dir, extension): quality = self.params.image_quality - out_file = self.utils.check_duplicates(f"{out_dir}/{os.path.splitext(file)[0]}.{extension}") + out_file = self.utils.check_duplicates(in_dir, out_dir, f"{os.path.splitext(file)[0]}.{extension}") try: image = Image.open(f'{in_dir}/{file}') @@ -127,7 +127,7 @@ class Compress: def unknown(self, in_dir, filename, out_dir): if self.params.force_compress: self.printer.unknown_file(filename) - out_file = self.utils.check_duplicates(f'{out_dir}/{filename}') + out_file = self.utils.check_duplicates(in_dir, out_dir, filename) try: (FFmpeg() .input(f'{in_dir}/{filename}') @@ -144,7 +144,6 @@ class Compress: self.utils.add_unprocessed_file(f'{in_dir}/{filename}', f'{out_dir}/{filename}') return f'{out_dir}/{filename}' - def compress(self, _dir, filename, source, output): match File.get_type(filename): case "audio": @@ -157,7 +156,7 @@ class Compress: out_file = self.unknown(_dir, filename, output) if self.params.mimic_mode: - os.rename(out_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed")) + self.utils.mimic_rename(out_file, f'{_dir}/{filename}', source) self.printer.bar.update() self.printer.bar.next() diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 040da09..96b7db9 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -1,4 +1,5 @@ from shutil import copyfile +from glob import glob import sys import os @@ -8,6 +9,7 @@ class Utils: self.errors = 0 self.params = params self.printer = printer + self.duplicates = [] @staticmethod def sys_pause(): @@ -61,11 +63,24 @@ class Utils: copyfile(orig_folder, new_folder) self.printer.info(f"File {filename} copied to compressed folder.") - def check_duplicates(self, new_folder): - filename = new_folder.split().pop() - if os.path.exists(new_folder): + def check_duplicates(self, in_dir, out_dir, filename): + duplicates = glob(f"{in_dir}/{os.path.splitext(filename)[0]}.*") + if len(duplicates) > 1: + if filename in self.duplicates: + new_name = os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1] + return f"{out_dir}/{new_name}" + self.duplicates.append(filename) + return f"{out_dir}/{filename}" + + def print_duplicates(self): + for filename in self.duplicates: self.printer.warning( f'Duplicate file has been found! Check manually this files - "{filename}", ' - f'"{os.path.splitext(filename)[0] + "(copy)" + os.path.splitext(filename)[1]}"') - return os.path.splitext(new_folder)[0] + "(copy)" + os.path.splitext(new_folder)[1] - return new_folder + f'"{os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1]}"') + + @staticmethod + def mimic_rename(filename, target, source): + if filename.count("(vncopy)"): + target = os.path.splitext(target)[0] + "(vncopy)" + os.path.splitext(target)[1] + + os.rename(filename, target.replace(source, f"{source}_compressed")) \ No newline at end of file From 44c12a568885826539efe78dcbca0e5f758e4c4a Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 01:22:42 +0300 Subject: [PATCH 09/24] Update requirements --- pyproject.toml | 3 ++- requirements.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4d8aa85..f2c379f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,5 +27,6 @@ dependencies = [ "pillow-avif-plugin==1.4.3", "python-ffmpeg==2.0.12", "progress==1.6", - "colorama==0.4.6" + "colorama==0.4.6", + "argparse~=1.4.0" ] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index c3c571c..16bd1e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ Pillow==10.3.0 pillow-avif-plugin==1.4.3 python-ffmpeg==2.0.12 progress==1.6 -colorama==0.4.6 \ No newline at end of file +colorama==0.4.6 +argparse~=1.4.0 \ No newline at end of file From a9aeb5250622845a8fb36995a31befba3bdcd41f Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 01:52:18 +0300 Subject: [PATCH 10/24] vnrecode: improve cli parameters --- vnrecode/params.py | 82 +++++++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 37 deletions(-) diff --git a/vnrecode/params.py b/vnrecode/params.py index 789cac5..c8474c2 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -32,30 +32,7 @@ class Params: @classmethod def setup(cls) -> Self: - parser = ArgumentParser(prog="vnrecode", - description="Python utility to compress Visual Novel Resources" - ) - parser.add_argument("source", help="SourceDir") - parser.add_argument("-c", "--config", help="ConfigFile") - parser.add_argument("-u", type=bool, help="CopyUnprocessed", default=True) - parser.add_argument("-f", "--force", type=bool, help="ForceCompress", default=False) - parser.add_argument("-m", "--mimic", type=bool, help="MimicMode", default=True) - parser.add_argument("-s", "--silent", type=bool, help="HideErrors", default=True) - parser.add_argument("--webprgba", type=bool, help="WebpRGBA", default=True) - parser.add_argument("-j", "--jobs", type=int, help="Workers", default=16) - parser.add_argument("-ae", "--aext", help="Audio Extension", default="opus") - parser.add_argument("-ab", "--abit", help="Audio Bitrate", default="128k") - parser.add_argument("-id", "--idown", type=int, help="Image Downscale", default=1) - parser.add_argument("-ie", "--iext", help="Image Extension", default="avif") - parser.add_argument("-ife", "--ifallext", help="Image Fallback Extension", default="webp") - parser.add_argument("-il", "--ilossless", type=bool, help="Image Lossless", default=True) - parser.add_argument("-iq", "--iquality", type=int, help="Image Quality", default=100) - parser.add_argument("--vcrf", help="Video CRF", type=int, default=27) - parser.add_argument("-vs", "--vskip", help="Video Skip", default=False) - parser.add_argument("-ve", "--vext", help="Video Extension", default="webm") - parser.add_argument("-vc", "--vcodec", help="Video Codec", default="libvpx-vp9") - args = parser.parse_args() - + args = cls.get_args() if args.config is not None: if os.path.isfile(args.config): with open(args.config, "rb") as cfile: @@ -67,20 +44,20 @@ class Params: copy_unprocessed = config["FFMPEG"]["CopyUnprocessed"] if args.config else args.u force_compress = config["FFMPEG"]["ForceCompress"] if args.config else args.force mimic_mode = config["FFMPEG"]["MimicMode"] if args.config else args.mimic - hide_errors = config["FFMPEG"]["HideErrors"] if args.config else args.silent + hide_errors = config["FFMPEG"]["HideErrors"] if args.config else args.show_errors workers = config["FFMPEG"]["Workers"] if args.config else args.jobs - webp_rgba = config["FFMPEG"]["WebpRGBA"] if args.config else args.webprgba - audio_ext = config["AUDIO"]["Extension"] if args.config else args.aext - audio_bitrate = config["AUDIO"]["BitRate"] if args.config else args.abit - image_downscale = config["IMAGE"]["ResDownScale"] if args.config else args.idown - image_ext = config["IMAGE"]["Extension"] if args.config else args.iext - image_fall_ext = config["IMAGE"]["FallBackExtension"] if args.config else args.ifallext - image_lossless = config["IMAGE"]["Lossless"] if args.config else args.ilossless - image_quality = config["IMAGE"]["Quality"] if args.config else args.iquality - video_crf = config["VIDEO"]["CRF"] if args.config else args.vcrf - video_skip = config["VIDEO"]["SkipVideo"] if args.config else args.vskip - video_ext = config["VIDEO"]["Extension"] if args.config else args.vext - video_codec = config["VIDEO"]["Codec"] if args.config else args.vcodec + webp_rgba = config["FFMPEG"]["WebpRGBA"] if args.config else args.webp_rgba + audio_ext = config["AUDIO"]["Extension"] if args.config else args.a_ext + audio_bitrate = config["AUDIO"]["BitRate"] if args.config else args.a_bit + image_downscale = config["IMAGE"]["ResDownScale"] if args.config else args.i_down + image_ext = config["IMAGE"]["Extension"] if args.config else args.i_ext + image_fall_ext = config["IMAGE"]["FallBackExtension"] if args.config else args.i_fallext + image_lossless = config["IMAGE"]["Lossless"] if args.config else args.i_lossless + image_quality = config["IMAGE"]["Quality"] if args.config else args.i_quality + video_crf = config["VIDEO"]["CRF"] if args.config else args.v_crf + video_skip = config["VIDEO"]["SkipVideo"] if args.config else args.v_skip + video_ext = config["VIDEO"]["Extension"] if args.config else args.v_ext + video_codec = config["VIDEO"]["Codec"] if args.config else args.v_codec source = args.source return cls( @@ -89,3 +66,34 @@ class Params: image_downscale, image_ext, image_fall_ext, image_lossless, image_quality, video_crf, video_skip, video_ext, video_codec, source ) + + @staticmethod + def get_args(): + parser = ArgumentParser(prog="vnrecode", + description="Python utility to compress Visual Novel Resources" + ) + parser.add_argument("source", help="Directory with game files to recode") + parser.add_argument("-c", "--config", help="Utility config file") + parser.add_argument("-u", action='store_true', help="Copy unprocessed filed", default=True) + parser.add_argument("-nu", dest='u', action='store_false', help="Don't copy unprocessed") + parser.add_argument("-f", "--force", action='store_true', help="Try to recode unknown files") + parser.add_argument("-m", "--mimic", action='store_true', help="Enable mimic mode", default=True) + parser.add_argument("-nm", "--no-mimic", dest='mimic', action='store_false', help="Disable mimic mode") + parser.add_argument("-v", "--show_errors", action='store_false', help="Show recode errors") + parser.add_argument("--webp_rgba", action='store_true', help="Recode .webp with alpha channel", default=True) + parser.add_argument("--webp_rgb", dest='webp_rgba', action='store_false', help="Recode .webp without alpha channel") + parser.add_argument("-j", "--jobs", type=int, help="Number of threads", default=16) + parser.add_argument("-ae", "--a_ext", help="Audio extension", default="opus") + parser.add_argument("-ab", "--a_bit", help="Audio bit rate", default="128k") + parser.add_argument("-id", "--i_down", type=int, help="Image resolution downscale multiplier", default=1) + parser.add_argument("-ie", "--i_ext", help="Image extension", default="avif") + parser.add_argument("-ife", "--i_fallext", help="Image fallback extension", default="webp") + parser.add_argument("-il", "--i_lossless", action='store_true', help="Image lossless compression mode", default=True) + parser.add_argument("-ilo", "--i_losing", dest='ilossless', action='store_false', help="Image losing compression mode") + parser.add_argument("-iq", "--i_quality", type=int, help="Image quality", default=100) + parser.add_argument("--v_crf", help="Video CRF number", type=int, default=27) + parser.add_argument("-vs", "--v_skip", action='store_true', help="Skip video recoding") + parser.add_argument("-ve", "--v_ext", help="Video extension", default="webm") + parser.add_argument("-vc", "--v_codec", help="Video codec name", default="libvpx-vp9") + args = parser.parse_args() + return args \ No newline at end of file From b534214be9e42ef58dce9c2f199b19ac1fdd67c9 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 02:07:17 +0300 Subject: [PATCH 11/24] vnrecode: accurate duplicate messages --- vnrecode/utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 96b7db9..8acd4dc 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -66,10 +66,10 @@ class Utils: def check_duplicates(self, in_dir, out_dir, filename): duplicates = glob(f"{in_dir}/{os.path.splitext(filename)[0]}.*") if len(duplicates) > 1: - if filename in self.duplicates: + if filename not in self.duplicates: + self.duplicates.append(filename) new_name = os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1] return f"{out_dir}/{new_name}" - self.duplicates.append(filename) return f"{out_dir}/{filename}" def print_duplicates(self): @@ -78,9 +78,11 @@ class Utils: f'Duplicate file has been found! Check manually this files - "{filename}", ' f'"{os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1]}"') - @staticmethod - def mimic_rename(filename, target, source): + def mimic_rename(self, filename, target, source): if filename.count("(vncopy)"): + orig_name = filename.replace("(vncopy)", "") + index = self.duplicates.index(os.path.split(orig_name)[-1]) + self.duplicates[index] = os.path.split(target)[-1] target = os.path.splitext(target)[0] + "(vncopy)" + os.path.splitext(target)[1] os.rename(filename, target.replace(source, f"{source}_compressed")) \ No newline at end of file From 8f9db132e65fdff3b99c9e8aa728f8204d6e9ec9 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 02:12:49 +0300 Subject: [PATCH 12/24] vnrecode: remove unneeded cli parameters --- vnrecode/params.py | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/vnrecode/params.py b/vnrecode/params.py index c8474c2..89fe28f 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -41,7 +41,7 @@ class Params: print("Failed to find config. Check `vnrecode -h` to more info") exit(255) - copy_unprocessed = config["FFMPEG"]["CopyUnprocessed"] if args.config else args.u + copy_unprocessed = config["FFMPEG"]["CopyUnprocessed"] if args.config else args.unproc force_compress = config["FFMPEG"]["ForceCompress"] if args.config else args.force mimic_mode = config["FFMPEG"]["MimicMode"] if args.config else args.mimic hide_errors = config["FFMPEG"]["HideErrors"] if args.config else args.show_errors @@ -73,27 +73,23 @@ class Params: description="Python utility to compress Visual Novel Resources" ) parser.add_argument("source", help="Directory with game files to recode") - parser.add_argument("-c", "--config", help="Utility config file") - parser.add_argument("-u", action='store_true', help="Copy unprocessed filed", default=True) - parser.add_argument("-nu", dest='u', action='store_false', help="Don't copy unprocessed") + parser.add_argument("-c", dest='config', help="Utility config file") + parser.add_argument("-nu", dest='unproc', action='store_false', help="Don't copy unprocessed") parser.add_argument("-f", "--force", action='store_true', help="Try to recode unknown files") - parser.add_argument("-m", "--mimic", action='store_true', help="Enable mimic mode", default=True) parser.add_argument("-nm", "--no-mimic", dest='mimic', action='store_false', help="Disable mimic mode") parser.add_argument("-v", "--show_errors", action='store_false', help="Show recode errors") - parser.add_argument("--webp_rgba", action='store_true', help="Recode .webp with alpha channel", default=True) - parser.add_argument("--webp_rgb", dest='webp_rgba', action='store_false', help="Recode .webp without alpha channel") + parser.add_argument("--webp-rgb", dest='webp_rgba', action='store_false', help="Recode .webp without alpha channel") parser.add_argument("-j", "--jobs", type=int, help="Number of threads", default=16) - parser.add_argument("-ae", "--a_ext", help="Audio extension", default="opus") - parser.add_argument("-ab", "--a_bit", help="Audio bit rate", default="128k") - parser.add_argument("-id", "--i_down", type=int, help="Image resolution downscale multiplier", default=1) - parser.add_argument("-ie", "--i_ext", help="Image extension", default="avif") - parser.add_argument("-ife", "--i_fallext", help="Image fallback extension", default="webp") - parser.add_argument("-il", "--i_lossless", action='store_true', help="Image lossless compression mode", default=True) - parser.add_argument("-ilo", "--i_losing", dest='ilossless', action='store_false', help="Image losing compression mode") - parser.add_argument("-iq", "--i_quality", type=int, help="Image quality", default=100) + parser.add_argument("-ae", dest="a_ext", help="Audio extension", default="opus") + parser.add_argument("-ab", dest="a_bit", help="Audio bit rate", default="128k") + parser.add_argument("-id", dest="i_down", type=int, help="Image resolution downscale multiplier", default=1) + parser.add_argument("-ie", dest="i_ext", help="Image extension", default="avif") + parser.add_argument("-ife", dest="i_fallext", help="Image fallback extension", default="webp") + parser.add_argument("-il", dest='i_lossless', action='store_false', help="Image losing compression mode") + parser.add_argument("-iq", dest="i_quality", type=int, help="Image quality", default=100) parser.add_argument("--v_crf", help="Video CRF number", type=int, default=27) - parser.add_argument("-vs", "--v_skip", action='store_true', help="Skip video recoding") - parser.add_argument("-ve", "--v_ext", help="Video extension", default="webm") - parser.add_argument("-vc", "--v_codec", help="Video codec name", default="libvpx-vp9") + parser.add_argument("-vs", dest="v_skip", action='store_true', help="Skip video recoding") + parser.add_argument("-ve", dest="v_ext", help="Video extension", default="webm") + parser.add_argument("-vc", dest="v_codec", help="Video codec name", default="libvpx-vp9") args = parser.parse_args() return args \ No newline at end of file From 7433027cf3c31f9ca52c502ad3ca4bc034c82cf4 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 02:40:47 +0300 Subject: [PATCH 13/24] vnrecode: add type definitions --- vnrecode/application.py | 11 +++++--- vnrecode/compress.py | 59 +++++++++++++++++++++-------------------- vnrecode/params.py | 4 +-- vnrecode/printer.py | 12 ++++----- vnrecode/utils.py | 46 ++++++++++++++++---------------- 5 files changed, 69 insertions(+), 63 deletions(-) diff --git a/vnrecode/application.py b/vnrecode/application.py index c7cb7cd..68fa45f 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -4,17 +4,22 @@ from datetime import datetime import shutil import os +from .compress import Compress +from .printer import Printer +from .params import Params +from .utils import Utils + class Application: - def __init__(self, params, compress, printer, utils): + def __init__(self, params: Params, compress: Compress, printer: Printer, utils: Utils): self.params = params self.compress = compress.compress self.printer = printer self.utils = utils - def compress_worker(self, folder, file, source, output): - if os.path.isfile(f'{folder}/{file}'): + def compress_worker(self, folder: str, file: str, source: str, output: str): + if os.path.isfile(os.path.join(folder, file)): self.compress(folder, file, source, output) def run(self): diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 249061e..18fb20e 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -1,13 +1,17 @@ from ffmpeg import FFmpeg, FFmpegError from PIL import Image +from os import path import pillow_avif -import os + +from .printer import Printer +from .params import Params +from .utils import Utils class File: @staticmethod - def get_type(filename): + def get_type(filename: str) -> str: extensions = { "audio": ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'], @@ -17,7 +21,7 @@ class File: } for file_type in extensions: - if os.path.splitext(filename)[1] in extensions[file_type]: + if path.splitext(filename)[1] in extensions[file_type]: return file_type return "unknown" @@ -39,45 +43,44 @@ class File: class Compress: - def __init__(self, params, printer, utils): + def __init__(self, params: Params, printer: Printer, utils: Utils): self.params = params self.printer = printer self.utils = utils - def audio(self, in_dir, file, out_dir, extension): + def audio(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: bit_rate = self.params.audio_bitrate - out_file = self.utils.check_duplicates(in_dir, out_dir, f'{os.path.splitext(file)[0]}.{extension}') + out_file = self.utils.check_duplicates(in_dir, out_dir, f'{path.splitext(file)[0]}.{extension}') try: (FFmpeg() - .input(f'{in_dir}/{file}') + .input(path.join(in_dir, file)) .option("hide_banner") .output(out_file,{"b:a": bit_rate, "loglevel": "error"}) .execute() ) except FFmpegError as e: - self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') + self.utils.add_unprocessed_file(path.join(in_dir, file), path.join(out_dir, file)) self.utils.errors += 1 if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") - self.printer.files(file, os.path.splitext(file)[0], extension, f"{bit_rate}") + self.printer.files(file, path.splitext(file)[0], extension, f"{bit_rate}") return out_file - - def video(self, in_dir, file, out_dir, extension): + def video(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: if not self.params.video_skip: - out_file = self.utils.check_duplicates(in_dir, out_dir, f'{os.path.splitext(file)[0]}.{extension}') + out_file = self.utils.check_duplicates(in_dir, out_dir, f'{path.splitext(file)[0]}.{extension}') codec = self.params.video_codec crf = self.params.video_crf try: (FFmpeg() - .input(f'{in_dir}/{file}') + .input(path.join(in_dir, file)) .option("hide_banner") .option("hwaccel", "auto") .output(out_file,{"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) .execute() ) - self.printer.files(file, os.path.splitext(file)[0], extension, codec) + self.printer.files(file, path.splitext(file)[0], extension, codec) except FFmpegError as e: self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') self.utils.errors += 1 @@ -86,14 +89,13 @@ class Compress: return out_file else: self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') - return f'{out_dir}/{os.path.splitext(file)[0]}.{extension}' + return f'{out_dir}/{path.splitext(file)[0]}.{extension}' - - def image(self, in_dir, file, out_dir, extension): + def image(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: quality = self.params.image_quality - out_file = self.utils.check_duplicates(in_dir, out_dir, f"{os.path.splitext(file)[0]}.{extension}") + out_file = self.utils.check_duplicates(in_dir, out_dir, f"{path.splitext(file)[0]}.{extension}") try: - image = Image.open(f'{in_dir}/{file}') + image = Image.open(path.join(in_dir, file)) if (extension == "jpg" or extension == "jpeg" or (extension == "webp" and not self.params.webp_rgba)): @@ -115,36 +117,35 @@ class Compress: lossless=self.params.image_lossless, quality=quality, minimize_size=True) - self.printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") + self.printer.files(file, path.splitext(file)[0], extension, f"{quality}%") except Exception as e: - self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') + self.utils.add_unprocessed_file(path.join(in_dir, file), path.join(out_dir, file)) self.utils.errors += 1 if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") return out_file - - def unknown(self, in_dir, filename, out_dir): + def unknown(self, in_dir: str, filename: str, out_dir: str) -> str: if self.params.force_compress: self.printer.unknown_file(filename) out_file = self.utils.check_duplicates(in_dir, out_dir, filename) try: (FFmpeg() - .input(f'{in_dir}/{filename}') + .input(path.join(in_dir, filename)) .output(out_file) .execute() ) except FFmpegError as e: - self.utils.add_unprocessed_file(f'{in_dir}/{filename}', f'{out_dir}/{filename}') + self.utils.add_unprocessed_file(path.join(in_dir, filename), path.join(out_dir, filename)) self.utils.errors += 1 if not self.params.hide_errors: self.printer.error(f"File {filename} can't be processed! Error: {e}") return out_file else: - self.utils.add_unprocessed_file(f'{in_dir}/{filename}', f'{out_dir}/{filename}') - return f'{out_dir}/{filename}' + self.utils.add_unprocessed_file(path.join(in_dir, filename), path.join(out_dir, filename)) + return path.join(out_dir, filename) - def compress(self, _dir, filename, source, output): + def compress(self, _dir: str, filename: str, source: str, output: str): match File.get_type(filename): case "audio": out_file = self.audio(_dir, filename, output, self.params.audio_ext) @@ -156,7 +157,7 @@ class Compress: out_file = self.unknown(_dir, filename, output) if self.params.mimic_mode: - self.utils.mimic_rename(out_file, f'{_dir}/{filename}', source) + self.utils.mimic_rename(out_file, path.join(_dir, filename), source) self.printer.bar.update() self.printer.bar.next() diff --git a/vnrecode/params.py b/vnrecode/params.py index 89fe28f..ea15691 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -1,4 +1,4 @@ -from argparse import ArgumentParser +from argparse import ArgumentParser, Namespace from dataclasses import dataclass from typing import Self import tomllib @@ -68,7 +68,7 @@ class Params: ) @staticmethod - def get_args(): + def get_args() -> Namespace: parser = ArgumentParser(prog="vnrecode", description="Python utility to compress Visual Novel Resources" ) diff --git a/vnrecode/printer.py b/vnrecode/printer.py index 6080caf..30aa496 100644 --- a/vnrecode/printer.py +++ b/vnrecode/printer.py @@ -15,7 +15,7 @@ class Printer: # Fill whole string with spaces for cleaning progress bar @staticmethod - def clean_str(string): + def clean_str(string: str) -> str: return string + " " * (os.get_terminal_size().columns - len(string)) @staticmethod @@ -23,20 +23,20 @@ class Printer: if sys.platform == "win32": colorama.init() - def bar_print(self, string): + def bar_print(self, string: str): print(string) self.bar.update() - def info(self, string): + def info(self, string: str): self.bar_print(self.clean_str(f"\r\033[100m- {string}\033[49m")) - def warning(self, string): + def warning(self, string: str): self.bar_print(self.clean_str(f"\r\033[93m!\033[0m {string}\033[49m")) - def error(self, string): + def error(self, string: str): self.bar_print(self.clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m")) - def files(self, source, dest, dest_ext, comment): + def files(self, source: str, dest: str, dest_ext: str, comment: str): source_ext = os.path.splitext(source)[1] source_name = os.path.splitext(source)[0] diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 8acd4dc..ec5c2c5 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -17,15 +17,15 @@ class Utils: os.system("pause") @staticmethod - def get_size(directory): + def get_size(directory: str) -> int: total_size = 0 for folder, folders, files in os.walk(directory): for file in files: - if not os.path.islink(f"{folder}/{file}"): - total_size += os.path.getsize(f"{folder}/{file}") + if not os.path.islink(os.path.join(folder, file)): + total_size += os.path.getsize(os.path.join(folder, file)) return total_size - def get_compression(self, source, output): + def get_compression(self, source: str, output: str): try: source = self.get_size(source) output = self.get_size(output) @@ -35,50 +35,50 @@ class Utils: except ZeroDivisionError: self.printer.warning("Nothing compressed!") - def get_compression_status(self, orig_folder): - orig_folder_len = 0 - comp_folder_len = 0 + def get_compression_status(self, source: str): + source_len = 0 + output_len = 0 - for folder, folders, files in os.walk(orig_folder): - orig_folder_len += len(files) + for folder, folders, files in os.walk(source): + source_len += len(files) - for folder, folders, files in os.walk(f'{orig_folder}_compressed'): + for folder, folders, files in os.walk(f'{source}_compressed'): for file in files: if not os.path.splitext(file)[1].count("(copy)"): - comp_folder_len += 1 + output_len += 1 if self.errors != 0: self.printer.warning("Some files failed to compress!") - if orig_folder_len == comp_folder_len: + if source_len == output_len: self.printer.info("Success!") - self.get_compression(orig_folder, f"{orig_folder}_compressed") else: self.printer.warning("Original and compressed folders are not identical!") - self.get_compression(orig_folder, f"{orig_folder}_compressed") + self.get_compression(source, f"{source}_compressed") - def add_unprocessed_file(self, orig_folder, new_folder): + def add_unprocessed_file(self, source: str, output: str): if self.params.copy_unprocessed: - filename = orig_folder.split("/").pop() - copyfile(orig_folder, new_folder) + filename = os.path.split(source)[-1] + copyfile(source, output) self.printer.info(f"File {filename} copied to compressed folder.") - def check_duplicates(self, in_dir, out_dir, filename): - duplicates = glob(f"{in_dir}/{os.path.splitext(filename)[0]}.*") + def check_duplicates(self, source: str, output: str, filename: str) -> str: + duplicates = glob(os.path.join(source, os.path.splitext(filename)[0]+".*")) if len(duplicates) > 1: if filename not in self.duplicates: self.duplicates.append(filename) new_name = os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1] - return f"{out_dir}/{new_name}" - return f"{out_dir}/{filename}" + return os.path.join(output, new_name) + return os.path.join(output, filename) def print_duplicates(self): for filename in self.duplicates: self.printer.warning( f'Duplicate file has been found! Check manually this files - "{filename}", ' - f'"{os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1]}"') + f'"{os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1]}"' + ) - def mimic_rename(self, filename, target, source): + def mimic_rename(self, filename: str, target: str, source: str): if filename.count("(vncopy)"): orig_name = filename.replace("(vncopy)", "") index = self.duplicates.index(os.path.split(orig_name)[-1]) From 90a6b4e0c1d0f94c87a2441a2db5ee22a3e4d0c1 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 03:25:43 +0300 Subject: [PATCH 14/24] vnrecode: add re to duplicate check --- vnrecode/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vnrecode/utils.py b/vnrecode/utils.py index ec5c2c5..58cf962 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -2,6 +2,7 @@ from shutil import copyfile from glob import glob import sys import os +import re class Utils: @@ -63,7 +64,10 @@ class Utils: self.printer.info(f"File {filename} copied to compressed folder.") def check_duplicates(self, source: str, output: str, filename: str) -> str: - duplicates = glob(os.path.join(source, os.path.splitext(filename)[0]+".*")) + files = glob(os.path.join(source, os.path.splitext(filename)[0])+".*") + re_pattern = re.compile(os.path.join(source, os.path.splitext(filename)[0])+r".[a-zA-Z0-9]+$") + duplicates = [f for f in files if re_pattern.match(f)] + if len(duplicates) > 1: if filename not in self.duplicates: self.duplicates.append(filename) From 0b43756ef511177158d7d34a2c3271bd91bcea7a Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 04:30:01 +0300 Subject: [PATCH 15/24] vnrecode: fix duplicates check for case insensitive fs --- vnrecode/application.py | 2 +- vnrecode/utils.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/vnrecode/application.py b/vnrecode/application.py index 68fa45f..912c526 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -42,7 +42,7 @@ class Application: with ThreadPoolExecutor(max_workers=self.params.workers) as executor: futures = [ executor.submit(self.compress, folder, file, source, output) - for file in files if os.path.isfile(f'{folder}/{file}') + for file in files if os.path.isfile(os.path.join(folder, file)) ] for future in as_completed(futures): future.result() diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 58cf962..3410eae 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -4,6 +4,9 @@ import sys import os import re +import fnmatch + + class Utils: def __init__(self, params, printer): @@ -64,12 +67,11 @@ class Utils: self.printer.info(f"File {filename} copied to compressed folder.") def check_duplicates(self, source: str, output: str, filename: str) -> str: - files = glob(os.path.join(source, os.path.splitext(filename)[0])+".*") - re_pattern = re.compile(os.path.join(source, os.path.splitext(filename)[0])+r".[a-zA-Z0-9]+$") - duplicates = [f for f in files if re_pattern.match(f)] + re_pattern = re.compile(os.path.splitext(filename)[0]+r".[a-zA-Z0-9]+$", re.IGNORECASE) + duplicates = [name for name in os.listdir(source) if re_pattern.match(name)] if len(duplicates) > 1: - if filename not in self.duplicates: + if filename.lower() not in (duplicate.lower() for duplicate in self.duplicates): self.duplicates.append(filename) new_name = os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1] return os.path.join(output, new_name) From d8e55bac9a8b9cacba3d96e7bad1ff8ed6f44a46 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sun, 13 Oct 2024 22:28:14 +0300 Subject: [PATCH 16/24] Update all README.md files --- README.md | 8 ++++---- unrenapk/README.md | 4 ++-- vnds2renpy/README.md | 4 ++-- vnrecode/README.md | 39 +++++++++++++++++++++++++++++---------- 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 370d3eb..b3b0091 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,10 @@ Collection of tools used by VienDesu! Porting Team ### Tools -* `FFMpeg-Compressor` - Python utility to compress Visual Novel Resources -* `RenPy-Android-Unpack` - A Python script for extracting game project from Ren'Py based .apk and .obb files -* `RenPy-Unpacker` - Simple .rpy script that will make any RenPy game unpack itself -* `VNDS-to-RenPy` - Simple script for converting VNDS engine scripts to .rpy ones +* `vnrecode` - Python utility to compress Visual Novel Resources +* `unrenapk` - A Python script for extracting game project from Ren'Py based .apk and .obb files +* `renpy-ripper` - Simple .rpy script that will make any RenPy game unpack itself +* `vnds2renpy` - Simple script for converting VNDS engine scripts to .rpy ones ### Installation #### Download from releases: diff --git a/unrenapk/README.md b/unrenapk/README.md index f6992e1..cf83a2c 100644 --- a/unrenapk/README.md +++ b/unrenapk/README.md @@ -1,6 +1,6 @@ -## RenPy-Android-Unpack +## unrenapk A simple Python script for unpacking Ren'Py based .apk and .obb files to ready to use Ren'Py SDK's Project ### How to use * Put some .apk & .obb files in folder -* `rendroid-unpack` (It unpacks all .apk and .obb files in the directory where it is located) +* `unrenapk` (It unpacks all .apk and .obb files in the directory where it is located) diff --git a/vnds2renpy/README.md b/vnds2renpy/README.md index c2ba3df..28091ef 100644 --- a/vnds2renpy/README.md +++ b/vnds2renpy/README.md @@ -1,7 +1,7 @@ -## VNDS-to-RenPy +## vnds2renpy Simple script for converting vnds scripts to rpy ### How to use * Extract VNDS visual novel's archive and get `scripts` folder from these (or `script.zip` if scripts folder is empty) -* Launch `convert.py` (It will automatically extract `scripts.zip` archive (if it needed) and converts .scr scripts to .rpy) +* Launch `vnds2renpy` (It will automatically extract `scripts.zip` archive (if it needed) and converts .scr scripts to .rpy) diff --git a/vnrecode/README.md b/vnrecode/README.md index 40b37d8..ba01b1f 100644 --- a/vnrecode/README.md +++ b/vnrecode/README.md @@ -1,13 +1,7 @@ -## FFMpeg-Compressor -Python utility uses ffmpeg to compress Visual Novel Resources +## vnrecode +Python utility uses Pillow and ffmpeg to compress Visual Novel Resources -### How to use -* Download `ffmpeg-comp.toml` and put in next to binary or in to `/etc` folder -* Change the configuration of the utility in `ffmpeg-comp.toml` for yourself -* `ffmpeg-comp {folder}` -* In result, you get `{folder-compressed}` near with original `{folder}` - -### Configuration +### Configuration file #### FFMPEG section * CopyUnprocessed - Copy all files that failed to compress by ffmpeg to destination folder. In can help to recreate original folder, but with compressed files. (default: `true`) * ForceCompress - Force try to compress all files in directory via ffmpeg. (default: `false`) @@ -32,9 +26,34 @@ Python utility uses ffmpeg to compress Visual Novel Resources * Extension - Required image file extension. It supports: `.3gp`, `.amv`, `.avi`, `.gif`, `.m2l`, `.m4v`, `.mkv`, `.mov`, `.mp4`, `.m4v`, `.mpeg`, `.mpv`, `.webm`, `.ogv` * Codec - (Maybe optional in future) Required video codec. (See official ffmpeg documentation for supported codecs) +### CLI Parameters +##### positional arguments: +* source - Directory with game files to recode + +##### options: +* ` -h, --help ` - show this help message and exit +* ` -c CONFIG ` - Utility config file +* ` -nu ` - Don't copy unprocessed +* ` -f, --force ` - Try to recode unknown files +* ` -nm, --no-mimic ` - Disable mimic mode +* ` -v, --show_errors ` - Show recode errors +* ` --webp-rgb ` - Recode .webp without alpha channel +* ` -j JOBS, --jobs JOBS ` - Number of threads +* ` -ae A_EXT ` - Audio extension +* ` -ab A_BIT ` - Audio bit rate +* ` -id I_DOWN ` - Image resolution downscale multiplier +* ` -ie I_EXT ` - Image extension +* ` -ife I_FALLEXT ` - Image fallback extension +* ` -il ` - Image losing compression mode +* ` -iq I_QUALITY ` - Image quality +* ` --v_crf V_CRF ` - Video CRF number +* ` -vs ` - Skip video recoding +* ` -ve V_EXT ` - Video extension +* ` -vc V_CODEC ` - Video codec name + ### TODO (for testing branch) * [x] Recreate whole game directory with compressed files * [x] Cross-platform (Easy Windows usage and binaries, macOS binaries) -* [x] Use ffmpeg python bindings instead of cli commands +* [x] Use ffmpeg python bindings instead of os.system * [x] Multithread * [ ] Reorganize code \ No newline at end of file From 9bb3cdcccbbab5a97160d4ab247f549d441ad6e9 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 18 Oct 2024 20:50:40 +0300 Subject: [PATCH 17/24] vnrecode: make it private! --- vnrecode/application.py | 40 ++++++++--------- vnrecode/compress.py | 98 ++++++++++++++++++++--------------------- vnrecode/params.py | 4 +- vnrecode/utils.py | 80 ++++++++++++++++----------------- 4 files changed, 107 insertions(+), 115 deletions(-) diff --git a/vnrecode/application.py b/vnrecode/application.py index 912c526..239bb05 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -13,41 +13,37 @@ from .utils import Utils class Application: def __init__(self, params: Params, compress: Compress, printer: Printer, utils: Utils): - self.params = params - self.compress = compress.compress - self.printer = printer - self.utils = utils - - def compress_worker(self, folder: str, file: str, source: str, output: str): - if os.path.isfile(os.path.join(folder, file)): - self.compress(folder, file, source, output) + self.__params = params + self.__compress = compress.compress + self.__printer = printer + self.__utils = utils def run(self): start_time = datetime.now() - self.printer.win_ascii_esc() + self.__printer.win_ascii_esc() - source = os.path.abspath(self.params.source) + source = self.__params.source - if os.path.exists(f"{source}_compressed"): - shutil.rmtree(f"{source}_compressed") + if os.path.exists(self.__params.dest): + shutil.rmtree(self.__params.dest) - self.printer.info("Creating folders...") + self.__printer.info("Creating folders...") for folder, folders, files in os.walk(source): - if not os.path.exists(folder.replace(source, f"{source}_compressed")): - os.mkdir(folder.replace(source, f"{source}_compressed")) + if not os.path.exists(folder.replace(source, self.__params.dest)): + os.mkdir(folder.replace(source, self.__params.dest)) - self.printer.info(f'Compressing "{folder.replace(source, os.path.split(source)[-1])}" folder...') - output = folder.replace(source, f"{source}_compressed") + self.__printer.info(f'Compressing "{folder.replace(source, os.path.split(source)[-1])}" folder...') + output = folder.replace(source, self.__params.dest) - with ThreadPoolExecutor(max_workers=self.params.workers) as executor: + with ThreadPoolExecutor(max_workers=self.__params.workers) as executor: futures = [ - executor.submit(self.compress, folder, file, source, output) + executor.submit(self.__compress, folder, file, output) for file in files if os.path.isfile(os.path.join(folder, file)) ] for future in as_completed(futures): future.result() - self.utils.print_duplicates() - self.utils.get_compression_status(source) - self.utils.sys_pause() + self.__utils.print_duplicates() + self.__utils.get_compression_status() + self.__utils.sys_pause() print(f"Time taken: {datetime.now() - start_time}") \ No newline at end of file diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 18fb20e..c484243 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -44,13 +44,13 @@ class File: class Compress: def __init__(self, params: Params, printer: Printer, utils: Utils): - self.params = params - self.printer = printer - self.utils = utils + self.__params = params + self.__printer = printer + self.__utils = utils def audio(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: - bit_rate = self.params.audio_bitrate - out_file = self.utils.check_duplicates(in_dir, out_dir, f'{path.splitext(file)[0]}.{extension}') + bit_rate = self.__params.audio_bitrate + out_file = self.__utils.check_duplicates(in_dir, out_dir, f'{path.splitext(file)[0]}.{extension}') try: (FFmpeg() .input(path.join(in_dir, file)) @@ -59,18 +59,18 @@ class Compress: .execute() ) except FFmpegError as e: - self.utils.add_unprocessed_file(path.join(in_dir, file), path.join(out_dir, file)) - self.utils.errors += 1 - if not self.params.hide_errors: - self.printer.error(f"File {file} can't be processed! Error: {e}") - self.printer.files(file, path.splitext(file)[0], extension, f"{bit_rate}") + self.__utils.add_unprocessed_file(path.join(in_dir, file), path.join(out_dir, file)) + self.__utils.errors += 1 + if not self.__params.hide_errors: + self.__printer.error(f"File {file} can't be processed! Error: {e}") + self.__printer.files(file, path.splitext(file)[0], extension, f"{bit_rate}") return out_file def video(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: - if not self.params.video_skip: - out_file = self.utils.check_duplicates(in_dir, out_dir, f'{path.splitext(file)[0]}.{extension}') - codec = self.params.video_codec - crf = self.params.video_crf + if not self.__params.video_skip: + out_file = self.__utils.check_duplicates(in_dir, out_dir, f'{path.splitext(file)[0]}.{extension}') + codec = self.__params.video_codec + crf = self.__params.video_crf try: (FFmpeg() @@ -80,33 +80,33 @@ class Compress: .output(out_file,{"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) .execute() ) - self.printer.files(file, path.splitext(file)[0], extension, codec) + self.__printer.files(file, path.splitext(file)[0], extension, codec) except FFmpegError as e: - self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') - self.utils.errors += 1 - if not self.params.hide_errors: - self.printer.error(f"File {file} can't be processed! Error: {e}") + self.__utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') + self.__utils.errors += 1 + if not self.__params.hide_errors: + self.__printer.error(f"File {file} can't be processed! Error: {e}") return out_file else: - self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') + self.__utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') return f'{out_dir}/{path.splitext(file)[0]}.{extension}' def image(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: - quality = self.params.image_quality - out_file = self.utils.check_duplicates(in_dir, out_dir, f"{path.splitext(file)[0]}.{extension}") + quality = self.__params.image_quality + out_file = self.__utils.check_duplicates(in_dir, out_dir, f"{path.splitext(file)[0]}.{extension}") try: image = Image.open(path.join(in_dir, file)) if (extension == "jpg" or extension == "jpeg" or - (extension == "webp" and not self.params.webp_rgba)): + (extension == "webp" and not self.__params.webp_rgba)): if File.has_transparency(image): - self.printer.warning(f"{file} has transparency. Changing to fallback...") - extension = self.params.image_fall_ext + self.__printer.warning(f"{file} has transparency. Changing to fallback...") + extension = self.__params.image_fall_ext if File.has_transparency(image): image.convert('RGBA') - res_downscale = self.params.image_downscale + res_downscale = self.__params.image_downscale if res_downscale != 1: width, height = image.size new_size = (int(width / res_downscale), int(height / res_downscale)) @@ -114,21 +114,21 @@ class Compress: image.save(out_file, optimize=True, - lossless=self.params.image_lossless, + lossless=self.__params.image_lossless, quality=quality, minimize_size=True) - self.printer.files(file, path.splitext(file)[0], extension, f"{quality}%") + self.__printer.files(file, path.splitext(file)[0], extension, f"{quality}%") except Exception as e: - self.utils.add_unprocessed_file(path.join(in_dir, file), path.join(out_dir, file)) - self.utils.errors += 1 - if not self.params.hide_errors: - self.printer.error(f"File {file} can't be processed! Error: {e}") + self.__utils.add_unprocessed_file(path.join(in_dir, file), path.join(out_dir, file)) + self.__utils.errors += 1 + if not self.__params.hide_errors: + self.__printer.error(f"File {file} can't be processed! Error: {e}") return out_file def unknown(self, in_dir: str, filename: str, out_dir: str) -> str: - if self.params.force_compress: - self.printer.unknown_file(filename) - out_file = self.utils.check_duplicates(in_dir, out_dir, filename) + if self.__params.force_compress: + self.__printer.unknown_file(filename) + out_file = self.__utils.check_duplicates(in_dir, out_dir, filename) try: (FFmpeg() .input(path.join(in_dir, filename)) @@ -136,28 +136,28 @@ class Compress: .execute() ) except FFmpegError as e: - self.utils.add_unprocessed_file(path.join(in_dir, filename), path.join(out_dir, filename)) - self.utils.errors += 1 - if not self.params.hide_errors: - self.printer.error(f"File {filename} can't be processed! Error: {e}") + self.__utils.add_unprocessed_file(path.join(in_dir, filename), path.join(out_dir, filename)) + self.__utils.errors += 1 + if not self.__params.hide_errors: + self.__printer.error(f"File {filename} can't be processed! Error: {e}") return out_file else: - self.utils.add_unprocessed_file(path.join(in_dir, filename), path.join(out_dir, filename)) + self.__utils.add_unprocessed_file(path.join(in_dir, filename), path.join(out_dir, filename)) return path.join(out_dir, filename) - def compress(self, _dir: str, filename: str, source: str, output: str): + def compress(self, dir_: str, filename: str, output: str): match File.get_type(filename): case "audio": - out_file = self.audio(_dir, filename, output, self.params.audio_ext) + out_file = self.audio(dir_, filename, output, self.__params.audio_ext) case "image": - out_file = self.image(_dir, filename, output, self.params.image_ext) + out_file = self.image(dir_, filename, output, self.__params.image_ext) case "video": - out_file = self.video(_dir, filename, output, self.params.video_ext) + out_file = self.video(dir_, filename, output, self.__params.video_ext) case "unknown": - out_file = self.unknown(_dir, filename, output) + out_file = self.unknown(dir_, filename, output) - if self.params.mimic_mode: - self.utils.mimic_rename(out_file, path.join(_dir, filename), source) + if self.__params.mimic_mode: + self.__utils.mimic_rename(out_file, path.join(dir_, filename)) - self.printer.bar.update() - self.printer.bar.next() + self.__printer.bar.update() + self.__printer.bar.next() diff --git a/vnrecode/params.py b/vnrecode/params.py index ea15691..ad4ec59 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -29,6 +29,7 @@ class Params: video_codec: str source: str + dest: str @classmethod def setup(cls) -> Self: @@ -59,12 +60,13 @@ class Params: video_ext = config["VIDEO"]["Extension"] if args.config else args.v_ext video_codec = config["VIDEO"]["Codec"] if args.config else args.v_codec source = args.source + dest = f"{source}_compressed" return cls( copy_unprocessed, force_compress, mimic_mode, hide_errors, webp_rgba, workers, audio_ext, audio_bitrate, image_downscale, image_ext, image_fall_ext, image_lossless, image_quality, - video_crf, video_skip, video_ext, video_codec, source + video_crf, video_skip, video_ext, video_codec, source, dest ) @staticmethod diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 3410eae..a75e4c3 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -1,19 +1,16 @@ from shutil import copyfile -from glob import glob import sys import os import re -import fnmatch - class Utils: - def __init__(self, params, printer): - self.errors = 0 - self.params = params - self.printer = printer - self.duplicates = [] + def __init__(self, params_inst, printer_inst): + self.__errors = 0 + self.__params = params_inst + self.__printer = printer_inst + self.__duplicates = [] @staticmethod def sys_pause(): @@ -29,66 +26,63 @@ class Utils: total_size += os.path.getsize(os.path.join(folder, file)) return total_size - def get_compression(self, source: str, output: str): + def get_compression_status(self): + source_len = 0 + output_len = 0 + + for folder, folders, files in os.walk(self.__params.source): + source_len += len(files) + + for folder, folders, files in os.walk(self.__params.dest): + for file in files: + if not os.path.splitext(file)[1].count("(vncopy)"): + output_len += 1 + + if self.__errors != 0: + self.__printer.warning("Some files failed to compress!") + + if source_len == output_len: + self.__printer.info("Success!") + else: + self.__printer.warning("Original and compressed folders are not identical!") try: - source = self.get_size(source) - output = self.get_size(output) + source = self.get_size(self.__params.source) + output = self.get_size(self.__params.dest) print(f"\nResult: {source/1024/1024:.2f}MB -> " f"{output/1024/1024:.2f}MB ({(output - source)/1024/1024:.2f}MB)") except ZeroDivisionError: - self.printer.warning("Nothing compressed!") - - def get_compression_status(self, source: str): - source_len = 0 - output_len = 0 - - for folder, folders, files in os.walk(source): - source_len += len(files) - - for folder, folders, files in os.walk(f'{source}_compressed'): - for file in files: - if not os.path.splitext(file)[1].count("(copy)"): - output_len += 1 - - if self.errors != 0: - self.printer.warning("Some files failed to compress!") - - if source_len == output_len: - self.printer.info("Success!") - else: - self.printer.warning("Original and compressed folders are not identical!") - self.get_compression(source, f"{source}_compressed") + self.__printer.warning("Nothing compressed!") def add_unprocessed_file(self, source: str, output: str): - if self.params.copy_unprocessed: + if self.__params.copy_unprocessed: filename = os.path.split(source)[-1] copyfile(source, output) - self.printer.info(f"File {filename} copied to compressed folder.") + self.__printer.info(f"File {filename} copied to compressed folder.") def check_duplicates(self, source: str, output: str, filename: str) -> str: re_pattern = re.compile(os.path.splitext(filename)[0]+r".[a-zA-Z0-9]+$", re.IGNORECASE) duplicates = [name for name in os.listdir(source) if re_pattern.match(name)] if len(duplicates) > 1: - if filename.lower() not in (duplicate.lower() for duplicate in self.duplicates): - self.duplicates.append(filename) + if filename.lower() not in (duplicate.lower() for duplicate in self.__duplicates): + self.__duplicates.append(filename) new_name = os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1] return os.path.join(output, new_name) return os.path.join(output, filename) def print_duplicates(self): - for filename in self.duplicates: - self.printer.warning( + for filename in self.__duplicates: + self.__printer.warning( f'Duplicate file has been found! Check manually this files - "{filename}", ' f'"{os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1]}"' ) - def mimic_rename(self, filename: str, target: str, source: str): + def mimic_rename(self, filename: str, target: str): if filename.count("(vncopy)"): orig_name = filename.replace("(vncopy)", "") - index = self.duplicates.index(os.path.split(orig_name)[-1]) - self.duplicates[index] = os.path.split(target)[-1] + index = self.__duplicates.index(os.path.split(orig_name)[-1]) + self.__duplicates[index] = os.path.split(target)[-1] target = os.path.splitext(target)[0] + "(vncopy)" + os.path.splitext(target)[1] - os.rename(filename, target.replace(source, f"{source}_compressed")) \ No newline at end of file + os.rename(filename, target.replace(self.__params.source, self.__params.dest)) \ No newline at end of file From 4e6fd332c54917ef9bc6a839f3c689191bcbaeba Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 18 Oct 2024 22:54:37 +0300 Subject: [PATCH 18/24] vnrecode: rewrite duplicates processing --- .gitignore | 1 + vnrecode/compress.py | 52 +++++++++++++++++--------------------------- vnrecode/utils.py | 47 ++++++++++++++++++++------------------- 3 files changed, 46 insertions(+), 54 deletions(-) diff --git a/.gitignore b/.gitignore index e6aa831..e3947bb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /output/ /tests/ +/tests_compressed/ /build/ /dist/ /vntools.egg-info/ diff --git a/vnrecode/compress.py b/vnrecode/compress.py index c484243..d1a2919 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -50,7 +50,8 @@ class Compress: def audio(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: bit_rate = self.__params.audio_bitrate - out_file = self.__utils.check_duplicates(in_dir, out_dir, f'{path.splitext(file)[0]}.{extension}') + prefix = self.__utils.get_hash(file) + out_file = path.join(out_dir, f'{prefix}_{path.splitext(file)[0]}.{extension}') try: (FFmpeg() .input(path.join(in_dir, file)) @@ -59,16 +60,14 @@ class Compress: .execute() ) except FFmpegError as e: - self.__utils.add_unprocessed_file(path.join(in_dir, file), path.join(out_dir, file)) - self.__utils.errors += 1 - if not self.__params.hide_errors: - self.__printer.error(f"File {file} can't be processed! Error: {e}") + self.__utils.catch_unprocessed(path.join(in_dir, file), out_file, e) self.__printer.files(file, path.splitext(file)[0], extension, f"{bit_rate}") return out_file def video(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: + prefix = self.__utils.get_hash(file) + out_file = path.join(out_dir, f'{prefix}_{path.splitext(file)[0]}.{extension}') if not self.__params.video_skip: - out_file = self.__utils.check_duplicates(in_dir, out_dir, f'{path.splitext(file)[0]}.{extension}') codec = self.__params.video_codec crf = self.__params.video_crf @@ -82,18 +81,15 @@ class Compress: ) self.__printer.files(file, path.splitext(file)[0], extension, codec) except FFmpegError as e: - self.__utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') - self.__utils.errors += 1 - if not self.__params.hide_errors: - self.__printer.error(f"File {file} can't be processed! Error: {e}") - return out_file + self.__utils.catch_unprocessed(path.join(in_dir, file), out_file, e) else: - self.__utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') - return f'{out_dir}/{path.splitext(file)[0]}.{extension}' + self.__utils.copy_unprocessed(path.join(in_dir, file), out_file) + return out_file def image(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: quality = self.__params.image_quality - out_file = self.__utils.check_duplicates(in_dir, out_dir, f"{path.splitext(file)[0]}.{extension}") + prefix = self.__utils.get_hash(file) + out_file = path.join(out_dir, f"{prefix}_{path.splitext(file)[0]}.{extension}") try: image = Image.open(path.join(in_dir, file)) @@ -119,31 +115,25 @@ class Compress: minimize_size=True) self.__printer.files(file, path.splitext(file)[0], extension, f"{quality}%") except Exception as e: - self.__utils.add_unprocessed_file(path.join(in_dir, file), path.join(out_dir, file)) - self.__utils.errors += 1 - if not self.__params.hide_errors: - self.__printer.error(f"File {file} can't be processed! Error: {e}") + self.__utils.catch_unprocessed(path.join(in_dir, file), out_file, e) return out_file - def unknown(self, in_dir: str, filename: str, out_dir: str) -> str: + def unknown(self, in_dir: str, file: str, out_dir: str) -> str: + prefix = self.__utils.get_hash(file) + out_file = path.join(out_dir, f"{prefix}_{file}") if self.__params.force_compress: - self.__printer.unknown_file(filename) - out_file = self.__utils.check_duplicates(in_dir, out_dir, filename) + self.__printer.unknown_file(file) try: (FFmpeg() - .input(path.join(in_dir, filename)) + .input(path.join(in_dir, file)) .output(out_file) .execute() ) except FFmpegError as e: - self.__utils.add_unprocessed_file(path.join(in_dir, filename), path.join(out_dir, filename)) - self.__utils.errors += 1 - if not self.__params.hide_errors: - self.__printer.error(f"File {filename} can't be processed! Error: {e}") - return out_file + self.__utils.catch_unprocessed(path.join(in_dir, file), out_file, e) else: - self.__utils.add_unprocessed_file(path.join(in_dir, filename), path.join(out_dir, filename)) - return path.join(out_dir, filename) + self.__utils.copy_unprocessed(path.join(in_dir, file), out_file) + return out_file def compress(self, dir_: str, filename: str, output: str): match File.get_type(filename): @@ -156,8 +146,6 @@ class Compress: case "unknown": out_file = self.unknown(dir_, filename, output) - if self.__params.mimic_mode: - self.__utils.mimic_rename(out_file, path.join(dir_, filename)) - + self.__utils.out_rename(out_file, filename) self.__printer.bar.update() self.__printer.bar.next() diff --git a/vnrecode/utils.py b/vnrecode/utils.py index a75e4c3..bf4e198 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -1,7 +1,7 @@ from shutil import copyfile +import hashlib import sys import os -import re class Utils: @@ -26,6 +26,10 @@ class Utils: total_size += os.path.getsize(os.path.join(folder, file)) return total_size + @staticmethod + def get_hash(filename: str) -> str: + return hashlib.md5(filename.encode()).hexdigest()[:8] + def get_compression_status(self): source_len = 0 output_len = 0 @@ -54,22 +58,23 @@ class Utils: except ZeroDivisionError: self.__printer.warning("Nothing compressed!") - def add_unprocessed_file(self, source: str, output: str): + def catch_unprocessed(self, source, output, error): + self.copy_unprocessed(source, error) + self.__errors += 1 + if not self.__params.hide_errors: + self.__printer.error(f"File {os.path.split(source)[-1]} can't be processed! Error: {error}") + + def copy_unprocessed(self, source, output): if self.__params.copy_unprocessed: - filename = os.path.split(source)[-1] copyfile(source, output) - self.__printer.info(f"File {filename} copied to compressed folder.") + self.__printer.info(f"File {os.path.split(source)[-1]} copied to compressed folder.") - def check_duplicates(self, source: str, output: str, filename: str) -> str: - re_pattern = re.compile(os.path.splitext(filename)[0]+r".[a-zA-Z0-9]+$", re.IGNORECASE) - duplicates = [name for name in os.listdir(source) if re_pattern.match(name)] - - if len(duplicates) > 1: - if filename.lower() not in (duplicate.lower() for duplicate in self.__duplicates): - self.__duplicates.append(filename) - new_name = os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1] - return os.path.join(output, new_name) - return os.path.join(output, filename) + def catch_duplicates(self, path: str) -> str: + if os.path.exists(path): + new_path = os.path.splitext(path)[0] + "(vncopy)" + os.path.splitext(path)[1] + self.__duplicates.append(new_path) + return new_path + return path def print_duplicates(self): for filename in self.__duplicates: @@ -78,11 +83,9 @@ class Utils: f'"{os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1]}"' ) - def mimic_rename(self, filename: str, target: str): - if filename.count("(vncopy)"): - orig_name = filename.replace("(vncopy)", "") - index = self.__duplicates.index(os.path.split(orig_name)[-1]) - self.__duplicates[index] = os.path.split(target)[-1] - target = os.path.splitext(target)[0] + "(vncopy)" + os.path.splitext(target)[1] - - os.rename(filename, target.replace(self.__params.source, self.__params.dest)) \ No newline at end of file + def out_rename(self, filename: str, target: str): + if not self.__params.mimic_mode: + dest_name = self.catch_duplicates(os.path.join(os.path.dirname(filename), target)) + os.rename(filename, dest_name) + else: + os.rename(filename, os.path.join(os.path.dirname(filename), target)) From 1c1e8a92921146f7f243e6ba932d09ceee6b5b89 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 19 Oct 2024 00:25:52 +0300 Subject: [PATCH 19/24] vnrecode: pathlib for all paths --- vnrecode/application.py | 25 ++++----- vnrecode/compress.py | 116 ++++++++++++++++++++-------------------- vnrecode/params.py | 12 ++--- vnrecode/printer.py | 14 +++-- vnrecode/utils.py | 45 +++++++++------- 5 files changed, 108 insertions(+), 104 deletions(-) diff --git a/vnrecode/application.py b/vnrecode/application.py index 239bb05..3b6843e 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime +from pathlib import Path import shutil import os @@ -12,11 +13,11 @@ from .utils import Utils class Application: - def __init__(self, params: Params, compress: Compress, printer: Printer, utils: Utils): - self.__params = params - self.__compress = compress.compress - self.__printer = printer - self.__utils = utils + def __init__(self, params_inst: Params, compress_inst: Compress, printer_inst: Printer, utils_inst: Utils): + self.__params = params_inst + self.__compress = compress_inst.compress + self.__printer = printer_inst + self.__utils = utils_inst def run(self): start_time = datetime.now() @@ -24,21 +25,21 @@ class Application: source = self.__params.source - if os.path.exists(self.__params.dest): + if self.__params.dest.exists(): shutil.rmtree(self.__params.dest) self.__printer.info("Creating folders...") for folder, folders, files in os.walk(source): - if not os.path.exists(folder.replace(source, self.__params.dest)): - os.mkdir(folder.replace(source, self.__params.dest)) + output = Path(folder.replace(str(source), str(self.__params.dest))) + if not output.exists(): + os.mkdir(output) - self.__printer.info(f'Compressing "{folder.replace(source, os.path.split(source)[-1])}" folder...') - output = folder.replace(source, self.__params.dest) + self.__printer.info(f'Compressing "{output}" folder...') with ThreadPoolExecutor(max_workers=self.__params.workers) as executor: futures = [ - executor.submit(self.__compress, folder, file, output) - for file in files if os.path.isfile(os.path.join(folder, file)) + executor.submit(self.__compress, Path(folder, file), Path(output)) + for file in files if Path(folder, file).is_file() ] for future in as_completed(futures): future.result() diff --git a/vnrecode/compress.py b/vnrecode/compress.py index d1a2919..7b06988 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -1,6 +1,6 @@ from ffmpeg import FFmpeg, FFmpegError +from pathlib import Path from PIL import Image -from os import path import pillow_avif from .printer import Printer @@ -11,7 +11,7 @@ from .utils import Utils class File: @staticmethod - def get_type(filename: str) -> str: + def get_type(filename: Path) -> str: extensions = { "audio": ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'], @@ -21,7 +21,7 @@ class File: } for file_type in extensions: - if path.splitext(filename)[1] in extensions[file_type]: + if filename.suffix in extensions[file_type]: return file_type return "unknown" @@ -43,61 +43,39 @@ class File: class Compress: - def __init__(self, params: Params, printer: Printer, utils: Utils): - self.__params = params - self.__printer = printer - self.__utils = utils + def __init__(self, params_inst: Params, printer_inst: Printer, utils_inst: Utils): + self.__params = params_inst + self.__printer = printer_inst + self.__utils = utils_inst - def audio(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: + def audio(self, input_path: Path, output_dir: Path, extension: str) -> Path: bit_rate = self.__params.audio_bitrate - prefix = self.__utils.get_hash(file) - out_file = path.join(out_dir, f'{prefix}_{path.splitext(file)[0]}.{extension}') + prefix = self.__utils.get_hash(input_path.name) + out_file = Path(output_dir, f'{prefix}_{input_path.stem}.{extension}') try: (FFmpeg() - .input(path.join(in_dir, file)) + .input(input_path) .option("hide_banner") .output(out_file,{"b:a": bit_rate, "loglevel": "error"}) .execute() ) except FFmpegError as e: - self.__utils.catch_unprocessed(path.join(in_dir, file), out_file, e) - self.__printer.files(file, path.splitext(file)[0], extension, f"{bit_rate}") + self.__utils.catch_unprocessed(input_path, out_file, e) + self.__printer.files(input_path, out_file, f"{bit_rate}") return out_file - def video(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: - prefix = self.__utils.get_hash(file) - out_file = path.join(out_dir, f'{prefix}_{path.splitext(file)[0]}.{extension}') - if not self.__params.video_skip: - codec = self.__params.video_codec - crf = self.__params.video_crf - - try: - (FFmpeg() - .input(path.join(in_dir, file)) - .option("hide_banner") - .option("hwaccel", "auto") - .output(out_file,{"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) - .execute() - ) - self.__printer.files(file, path.splitext(file)[0], extension, codec) - except FFmpegError as e: - self.__utils.catch_unprocessed(path.join(in_dir, file), out_file, e) - else: - self.__utils.copy_unprocessed(path.join(in_dir, file), out_file) - return out_file - - def image(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: + def image(self, input_path: Path, output_dir: Path, extension: str) -> Path: quality = self.__params.image_quality - prefix = self.__utils.get_hash(file) - out_file = path.join(out_dir, f"{prefix}_{path.splitext(file)[0]}.{extension}") + prefix = self.__utils.get_hash(input_path.name) + out_file = Path(output_dir, f"{prefix}_{input_path.stem}.{extension}") try: - image = Image.open(path.join(in_dir, file)) + image = Image.open(input_path) if (extension == "jpg" or extension == "jpeg" or (extension == "webp" and not self.__params.webp_rgba)): if File.has_transparency(image): - self.__printer.warning(f"{file} has transparency. Changing to fallback...") - extension = self.__params.image_fall_ext + self.__printer.warning(f"{input_path.name} has transparency. Changing to fallback...") + out_file = Path(output_dir, f"{prefix}_{input_path.stem}.{self.__params.image_fall_ext}") if File.has_transparency(image): image.convert('RGBA') @@ -113,39 +91,61 @@ class Compress: lossless=self.__params.image_lossless, quality=quality, minimize_size=True) - self.__printer.files(file, path.splitext(file)[0], extension, f"{quality}%") + self.__printer.files(input_path, out_file, f"{quality}%") except Exception as e: - self.__utils.catch_unprocessed(path.join(in_dir, file), out_file, e) + self.__utils.catch_unprocessed(input_path, out_file, e) return out_file - def unknown(self, in_dir: str, file: str, out_dir: str) -> str: - prefix = self.__utils.get_hash(file) - out_file = path.join(out_dir, f"{prefix}_{file}") - if self.__params.force_compress: - self.__printer.unknown_file(file) + def video(self, input_path: Path, output_dir: Path, extension: str) -> Path: + prefix = self.__utils.get_hash(input_path.name) + out_file = Path(output_dir, f'{prefix}_{input_path.stem}.{extension}') + if not self.__params.video_skip: + codec = self.__params.video_codec + crf = self.__params.video_crf + try: (FFmpeg() - .input(path.join(in_dir, file)) + .input(input_path) + .option("hide_banner") + .option("hwaccel", "auto") + .output(out_file,{"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) + .execute() + ) + self.__printer.files(input_path, out_file, codec) + except FFmpegError as e: + self.__utils.catch_unprocessed(input_path, out_file, e) + else: + self.__utils.copy_unprocessed(input_path, out_file) + return out_file + + def unknown(self, input_path: Path, output_dir: Path) -> Path: + prefix = self.__utils.get_hash(input_path.name) + out_file = Path(output_dir, f"{prefix}_{input_path.name}") + if self.__params.force_compress: + self.__printer.unknown_file(input_path.name) + try: + (FFmpeg() + .input(input_path) .output(out_file) .execute() ) except FFmpegError as e: - self.__utils.catch_unprocessed(path.join(in_dir, file), out_file, e) + self.__utils.catch_unprocessed(input_path, out_file, e) else: - self.__utils.copy_unprocessed(path.join(in_dir, file), out_file) + self.__utils.copy_unprocessed(input_path, out_file) return out_file - def compress(self, dir_: str, filename: str, output: str): - match File.get_type(filename): + def compress(self, source: Path, output: Path): + match File.get_type(source): case "audio": - out_file = self.audio(dir_, filename, output, self.__params.audio_ext) + out_file = self.audio(source, output, self.__params.audio_ext) case "image": - out_file = self.image(dir_, filename, output, self.__params.image_ext) + out_file = self.image(source, output, self.__params.image_ext) case "video": - out_file = self.video(dir_, filename, output, self.__params.video_ext) + out_file = self.video(source, output, self.__params.video_ext) case "unknown": - out_file = self.unknown(dir_, filename, output) + out_file = self.unknown(source, output) - self.__utils.out_rename(out_file, filename) + self.__utils.out_rename(out_file, source.name) self.__printer.bar.update() self.__printer.bar.next() diff --git a/vnrecode/params.py b/vnrecode/params.py index ad4ec59..d4db410 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -1,8 +1,8 @@ from argparse import ArgumentParser, Namespace from dataclasses import dataclass +from pathlib import Path from typing import Self import tomllib -import os @dataclass class Params: @@ -28,14 +28,14 @@ class Params: video_ext: str video_codec: str - source: str - dest: str + source: Path + dest: Path @classmethod def setup(cls) -> Self: args = cls.get_args() if args.config is not None: - if os.path.isfile(args.config): + if Path(args.config).is_file(): with open(args.config, "rb") as cfile: config = tomllib.load(cfile) else: @@ -59,8 +59,8 @@ class Params: video_skip = config["VIDEO"]["SkipVideo"] if args.config else args.v_skip video_ext = config["VIDEO"]["Extension"] if args.config else args.v_ext video_codec = config["VIDEO"]["Codec"] if args.config else args.v_codec - source = args.source - dest = f"{source}_compressed" + source = Path(args.source) + dest = Path(f"{args.source}_compressed") return cls( copy_unprocessed, force_compress, mimic_mode, hide_errors, webp_rgba, workers, diff --git a/vnrecode/printer.py b/vnrecode/printer.py index 30aa496..4de4f3c 100644 --- a/vnrecode/printer.py +++ b/vnrecode/printer.py @@ -1,12 +1,12 @@ from progress.bar import IncrementalBar +from pathlib import Path import colorama import sys import os - class Printer: - def __init__(self, folder): + def __init__(self, folder: Path): file_count = 0 for folder, folders, file in os.walk(folder): file_count += len(file) @@ -36,11 +36,9 @@ class Printer: def error(self, string: str): self.bar_print(self.clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m")) - def files(self, source: str, dest: str, dest_ext: str, comment: str): - source_ext = os.path.splitext(source)[1] - source_name = os.path.splitext(source)[0] + def files(self, source_path: Path, output_path: Path, comment: str): + self.bar_print(self.clean_str(f"\r\033[0;32m\u2713\033[0m \033[0;37m{source_path.stem}\033[0m{source_path.suffix}\033[0;37m -> " + f"{source_path.stem}\033[0m{output_path.suffix}\033[0;37m ({comment})\033[0m")) - self.bar_print(self.clean_str(f"\r\033[0;32m\u2713\033[0m \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m")) - - def unknown_file(self, file): + def unknown_file(self, file: str): self.bar_print(self.clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)")) diff --git a/vnrecode/utils.py b/vnrecode/utils.py index bf4e198..2b52f30 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -1,12 +1,16 @@ from shutil import copyfile +from pathlib import Path import hashlib import sys import os +from vnrecode.printer import Printer +from vnrecode.params import Params + class Utils: - def __init__(self, params_inst, printer_inst): + def __init__(self, params_inst: Params, printer_inst: Printer): self.__errors = 0 self.__params = params_inst self.__printer = printer_inst @@ -18,12 +22,13 @@ class Utils: os.system("pause") @staticmethod - def get_size(directory: str) -> int: + def get_size(directory: Path) -> int: total_size = 0 for folder, folders, files in os.walk(directory): for file in files: - if not os.path.islink(os.path.join(folder, file)): - total_size += os.path.getsize(os.path.join(folder, file)) + path = Path(folder, file) + if not path.is_symlink(): + total_size += path.stat().st_size return total_size @staticmethod @@ -39,7 +44,7 @@ class Utils: for folder, folders, files in os.walk(self.__params.dest): for file in files: - if not os.path.splitext(file)[1].count("(vncopy)"): + if not file.count("(vncopy)"): output_len += 1 if self.__errors != 0: @@ -58,20 +63,20 @@ class Utils: except ZeroDivisionError: self.__printer.warning("Nothing compressed!") - def catch_unprocessed(self, source, output, error): - self.copy_unprocessed(source, error) + def catch_unprocessed(self, input_path: Path, output_path: Path, error): + self.copy_unprocessed(input_path, output_path) self.__errors += 1 if not self.__params.hide_errors: - self.__printer.error(f"File {os.path.split(source)[-1]} can't be processed! Error: {error}") + self.__printer.error(f"File {input_path.name} can't be processed! Error: {error}") - def copy_unprocessed(self, source, output): + def copy_unprocessed(self, input_path: Path, output_path: Path): if self.__params.copy_unprocessed: - copyfile(source, output) - self.__printer.info(f"File {os.path.split(source)[-1]} copied to compressed folder.") + copyfile(input_path, output_path) + self.__printer.info(f"File {input_path.name} copied to compressed folder.") - def catch_duplicates(self, path: str) -> str: - if os.path.exists(path): - new_path = os.path.splitext(path)[0] + "(vncopy)" + os.path.splitext(path)[1] + def catch_duplicates(self, path: Path) -> Path: + if path.exists(): + new_path = Path(path.stem + "(vncopy)" + path.suffix) self.__duplicates.append(new_path) return new_path return path @@ -79,13 +84,13 @@ class Utils: def print_duplicates(self): for filename in self.__duplicates: self.__printer.warning( - f'Duplicate file has been found! Check manually this files - "{filename}", ' - f'"{os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1]}"' + f'Duplicate file has been found! Check manually this files - "{filename.name}", ' + f'"{filename.stem + "(vncopy)" + filename.suffix}"' ) - def out_rename(self, filename: str, target: str): + def out_rename(self, out_path: Path, target: str): if not self.__params.mimic_mode: - dest_name = self.catch_duplicates(os.path.join(os.path.dirname(filename), target)) - os.rename(filename, dest_name) + dest_name = self.catch_duplicates(Path(out_path.parent, target)) + os.rename(out_path, dest_name) else: - os.rename(filename, os.path.join(os.path.dirname(filename), target)) + os.rename(out_path, Path(out_path.parent, target)) From df20bd363685700213942e5f5cebf02776b154a0 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 19 Oct 2024 01:45:35 +0300 Subject: [PATCH 20/24] vnrecode: docstrings --- vnrecode/__main__.py | 4 +++ vnrecode/application.py | 11 ++++++- vnrecode/compress.py | 52 +++++++++++++++++++++++++++++++-- vnrecode/params.py | 12 ++++++++ vnrecode/printer.py | 57 ++++++++++++++++++++++++++++++++---- vnrecode/utils.py | 64 ++++++++++++++++++++++++++++++++--------- 6 files changed, 177 insertions(+), 23 deletions(-) diff --git a/vnrecode/__main__.py b/vnrecode/__main__.py index faf1839..fe45306 100644 --- a/vnrecode/__main__.py +++ b/vnrecode/__main__.py @@ -7,6 +7,10 @@ from .utils import Utils def init(): + """ + This function creates all needed class instances and run utility + :return: None + """ params = Params.setup() printer = Printer(params.source) utils = Utils(params, printer) diff --git a/vnrecode/application.py b/vnrecode/application.py index 3b6843e..c43102b 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -12,6 +12,9 @@ from .utils import Utils class Application: + """ + Main class for utility + """ def __init__(self, params_inst: Params, compress_inst: Compress, printer_inst: Printer, utils_inst: Utils): self.__params = params_inst @@ -20,6 +23,12 @@ class Application: self.__utils = utils_inst def run(self): + """ + Method creates a folder in which all the recoded files will be placed, + creates a queue of recoding processes for each file and, when the files are run out in the original folder, + calls functions to display the result + :return: None + """ start_time = datetime.now() self.__printer.win_ascii_esc() @@ -45,6 +54,6 @@ class Application: future.result() self.__utils.print_duplicates() - self.__utils.get_compression_status() + self.__utils.get_recode_status() self.__utils.sys_pause() print(f"Time taken: {datetime.now() - start_time}") \ No newline at end of file diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 7b06988..4050e4c 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -9,10 +9,17 @@ from .utils import Utils class File: + """ + Class contains some methods to work with files + """ @staticmethod - def get_type(filename: Path) -> str: - + def get_type(path: Path) -> str: + """ + Method returns filetype string for file + :param path: Path of file to determine type + :return: filetype string: audio, image, video, unknown + """ extensions = { "audio": ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'], "image": ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'], @@ -21,12 +28,17 @@ class File: } for file_type in extensions: - if filename.suffix in extensions[file_type]: + if path.suffix in extensions[file_type]: return file_type return "unknown" @staticmethod def has_transparency(img: Image) -> bool: + """ + Method checks if image has transparency + :param img: Pillow Image + :return: bool + """ if img.info.get("transparency", None) is not None: return True if img.mode == "P": @@ -49,6 +61,13 @@ class Compress: self.__utils = utils_inst def audio(self, input_path: Path, output_dir: Path, extension: str) -> Path: + """ + Method recodes audio files to another format using ffmpeg utility + :param input_path: Path of the original audio file + :param output_dir: Path of the output (compression) folder + :param extension: Extension of the new audio file + :return: Path of compressed audio file with md5 hash as prefix + """ bit_rate = self.__params.audio_bitrate prefix = self.__utils.get_hash(input_path.name) out_file = Path(output_dir, f'{prefix}_{input_path.stem}.{extension}') @@ -65,6 +84,13 @@ class Compress: return out_file def image(self, input_path: Path, output_dir: Path, extension: str) -> Path: + """ + Method recodes image files to another format using Pillow + :param input_path: Path of the original image file + :param output_dir: Path of the output (compression) folder + :param extension: Extension of the new image file + :return: Path of compressed image file with md5 hash as prefix + """ quality = self.__params.image_quality prefix = self.__utils.get_hash(input_path.name) out_file = Path(output_dir, f"{prefix}_{input_path.stem}.{extension}") @@ -97,6 +123,13 @@ class Compress: return out_file def video(self, input_path: Path, output_dir: Path, extension: str) -> Path: + """ + Method recodes video files to another format using ffmpeg utility + :param input_path: Path of the original video file + :param output_dir: Path of the output (compression) folder + :param extension: Extension of the new video file + :return: Path of compressed video file with md5 hash as prefix + """ prefix = self.__utils.get_hash(input_path.name) out_file = Path(output_dir, f'{prefix}_{input_path.stem}.{extension}') if not self.__params.video_skip: @@ -119,6 +152,13 @@ class Compress: return out_file def unknown(self, input_path: Path, output_dir: Path) -> Path: + """ + Method recodes files with "unknown" file format using ffmpeg, + in the hope that ffmpeg supports this file type and the default settings for it will reduce its size + :param input_path: Path of the original file + :param output_dir: Path of the output (compression) folder + :return: Path of compressed file with md5 hash as prefix + """ prefix = self.__utils.get_hash(input_path.name) out_file = Path(output_dir, f"{prefix}_{input_path.name}") if self.__params.force_compress: @@ -136,6 +176,12 @@ class Compress: return out_file def compress(self, source: Path, output: Path): + """ + It the core method for this program. Method determines file type and call compress function for it + :param source: Path of file to compress + :param output: Path of output file + :return: None + """ match File.get_type(source): case "audio": out_file = self.audio(source, output, self.__params.audio_ext) diff --git a/vnrecode/params.py b/vnrecode/params.py index d4db410..d0388f2 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -7,6 +7,10 @@ import tomllib @dataclass class Params: + """ + This dataclass contains all parameters for utility + """ + copy_unprocessed: bool force_compress: bool mimic_mode: bool @@ -33,6 +37,10 @@ class Params: @classmethod def setup(cls) -> Self: + """ + Method initialize all parameters and returns class instance + :return: Params instance + """ args = cls.get_args() if args.config is not None: if Path(args.config).is_file(): @@ -71,6 +79,10 @@ class Params: @staticmethod def get_args() -> Namespace: + """ + Method gets CLI arguments and returns argparse.Namespace instance + :return: argparse.Namespace of CLI args + """ parser = ArgumentParser(prog="vnrecode", description="Python utility to compress Visual Novel Resources" ) diff --git a/vnrecode/printer.py b/vnrecode/printer.py index 4de4f3c..a6046f9 100644 --- a/vnrecode/printer.py +++ b/vnrecode/printer.py @@ -5,40 +5,87 @@ import sys import os class Printer: + """ + Class implements CLI UI for this utility + """ - def __init__(self, folder: Path): + def __init__(self, source: Path): + """ + :param source: Path of original (compressing) folder to count its files for progress bar + """ file_count = 0 - for folder, folders, file in os.walk(folder): + for folder, folders, file in os.walk(source): file_count += len(file) self.bar = IncrementalBar('Compressing', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)') self.bar.update() - # Fill whole string with spaces for cleaning progress bar @staticmethod def clean_str(string: str) -> str: + """ + Method fills end of string with spaces to remove progress bar garbage from console + :param string: String to "clean" + :return: "Clean" string + """ return string + " " * (os.get_terminal_size().columns - len(string)) @staticmethod def win_ascii_esc(): + """ + Method setups colorama for cmd + :return: None + """ if sys.platform == "win32": colorama.init() def bar_print(self, string: str): + """ + Method prints some string in console and updates progress bar + :param string: String to print + :return: None + """ print(string) self.bar.update() def info(self, string: str): + """ + Method prints string with decor for info messages + :param string: String to print + :return: None + """ self.bar_print(self.clean_str(f"\r\033[100m- {string}\033[49m")) def warning(self, string: str): + """ + Method prints string with decor for warning messages + :param string: String to print + :return: None + """ self.bar_print(self.clean_str(f"\r\033[93m!\033[0m {string}\033[49m")) def error(self, string: str): + """ + Method prints string with decor for error messages + :param string: String to print + :return: None + """ self.bar_print(self.clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m")) def files(self, source_path: Path, output_path: Path, comment: str): + """ + Method prints the result of recoding a file with some decorations in the form: + input file name -> output file name (quality setting) + :param source_path: Input file Path + :param output_path: Output file Path + :param comment: Comment about recode quality setting + :return: None + """ self.bar_print(self.clean_str(f"\r\033[0;32m\u2713\033[0m \033[0;37m{source_path.stem}\033[0m{source_path.suffix}\033[0;37m -> " f"{source_path.stem}\033[0m{output_path.suffix}\033[0;37m ({comment})\033[0m")) - def unknown_file(self, file: str): - self.bar_print(self.clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)")) + def unknown_file(self, filename: str): + """ + Method prints the result of recoding unknown file + :param filename: Name of unknown file + :return: + """ + self.bar_print(self.clean_str(f"\r\u2713 \033[0;33m{filename}\033[0m (File will be force compressed via ffmpeg)")) diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 2b52f30..7da06d3 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -9,6 +9,9 @@ from vnrecode.params import Params class Utils: + """ + Class contains various methods for internal utility use + """ def __init__(self, params_inst: Params, printer_inst: Printer): self.__errors = 0 @@ -18,24 +21,27 @@ class Utils: @staticmethod def sys_pause(): + """ + Method calls pause for Windows cmd shell + :return: None + """ if sys.platform == "win32": os.system("pause") - @staticmethod - def get_size(directory: Path) -> int: - total_size = 0 - for folder, folders, files in os.walk(directory): - for file in files: - path = Path(folder, file) - if not path.is_symlink(): - total_size += path.stat().st_size - return total_size - @staticmethod def get_hash(filename: str) -> str: + """ + Method returns 8 chars of md5 hash for filename + :param filename: File name to get md5 + :return: 8 chars of md5 hash + """ return hashlib.md5(filename.encode()).hexdigest()[:8] - def get_compression_status(self): + def get_recode_status(self): + """ + Method prints recoding results + :return: None + """ source_len = 0 output_len = 0 @@ -55,8 +61,8 @@ class Utils: else: self.__printer.warning("Original and compressed folders are not identical!") try: - source = self.get_size(self.__params.source) - output = self.get_size(self.__params.dest) + source = sum(file.stat().st_size for file in self.__params.source.glob('**/*') if file.is_file()) + output = sum(file.stat().st_size for file in self.__params.dest.glob('**/*') if file.is_file()) print(f"\nResult: {source/1024/1024:.2f}MB -> " f"{output/1024/1024:.2f}MB ({(output - source)/1024/1024:.2f}MB)") @@ -64,24 +70,48 @@ class Utils: self.__printer.warning("Nothing compressed!") def catch_unprocessed(self, input_path: Path, output_path: Path, error): + """ + Method processes files that have not been recoded due to an error and prints error to console + if hide_errors parameter is False + :param input_path: Path of unprocessed file + :param output_path: Destination path of unprocessed file + :param error: Recoding exception + :return: None + """ self.copy_unprocessed(input_path, output_path) self.__errors += 1 if not self.__params.hide_errors: self.__printer.error(f"File {input_path.name} can't be processed! Error: {error}") def copy_unprocessed(self, input_path: Path, output_path: Path): + """ + Method copies an unprocessed file from the source folder to the destination folder + :param input_path: Path of unprocessed file + :param output_path: Destination path of unprocessed file + :return: None + """ if self.__params.copy_unprocessed: copyfile(input_path, output_path) self.__printer.info(f"File {input_path.name} copied to compressed folder.") def catch_duplicates(self, path: Path) -> Path: - if path.exists(): + """ + Method checks if file path exists and returns folder/filename(vncopy).ext path + if duplicate founded + :param path: Some file Path + :return: Duplicate path name with (vncopy) on end + """ + if path.is_file() and path.exists(): new_path = Path(path.stem + "(vncopy)" + path.suffix) self.__duplicates.append(new_path) return new_path return path def print_duplicates(self): + """ + Method prints message about all duplicates generated during recode process + :return: None + """ for filename in self.__duplicates: self.__printer.warning( f'Duplicate file has been found! Check manually this files - "{filename.name}", ' @@ -89,6 +119,12 @@ class Utils: ) def out_rename(self, out_path: Path, target: str): + """ + Method removes md5 hash from file name and changes file extension in dependence of mimic mode + :param out_path: Recoded file Path + :param target: Target filename + :return: None + """ if not self.__params.mimic_mode: dest_name = self.catch_duplicates(Path(out_path.parent, target)) os.rename(out_path, dest_name) From 407ab98000df8b18a9d1bf2593a588e1c9210196 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 19 Oct 2024 02:20:58 +0300 Subject: [PATCH 21/24] vnrecode: duplicates check for more than two files --- vnrecode/compress.py | 2 +- vnrecode/utils.py | 24 ++++++++++++++---------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 4050e4c..b1c2567 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -192,6 +192,6 @@ class Compress: case "unknown": out_file = self.unknown(source, output) - self.__utils.out_rename(out_file, source.name) + self.__utils.out_rename(out_file, source) self.__printer.bar.update() self.__printer.bar.next() diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 7da06d3..c30abe3 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -17,7 +17,7 @@ class Utils: self.__errors = 0 self.__params = params_inst self.__printer = printer_inst - self.__duplicates = [] + self.__duplicates = {} @staticmethod def sys_pause(): @@ -102,9 +102,13 @@ class Utils: :return: Duplicate path name with (vncopy) on end """ if path.is_file() and path.exists(): - new_path = Path(path.stem + "(vncopy)" + path.suffix) - self.__duplicates.append(new_path) - return new_path + orig_name = path.name.replace("(vncopy)", "") + new_path = Path(path.parent, path.stem + "(vncopy)" + path.suffix) + try: self.__duplicates[orig_name] + except KeyError: self.__duplicates[orig_name] = [] + if not new_path.name in self.__duplicates[orig_name]: + self.__duplicates[orig_name].append(new_path.name) + return self.catch_duplicates(new_path) return path def print_duplicates(self): @@ -112,13 +116,13 @@ class Utils: Method prints message about all duplicates generated during recode process :return: None """ - for filename in self.__duplicates: + for filename in self.__duplicates.keys(): self.__printer.warning( - f'Duplicate file has been found! Check manually this files - "{filename.name}", ' - f'"{filename.stem + "(vncopy)" + filename.suffix}"' + f'Duplicate file has been found! Check manually this files - "{filename}", ' + + ', '.join(self.__duplicates[filename]) ) - def out_rename(self, out_path: Path, target: str): + def out_rename(self, out_path: Path, target: Path): """ Method removes md5 hash from file name and changes file extension in dependence of mimic mode :param out_path: Recoded file Path @@ -126,7 +130,7 @@ class Utils: :return: None """ if not self.__params.mimic_mode: - dest_name = self.catch_duplicates(Path(out_path.parent, target)) + dest_name = self.catch_duplicates(Path(out_path.parent, target.stem+out_path.suffix)) os.rename(out_path, dest_name) else: - os.rename(out_path, Path(out_path.parent, target)) + os.rename(out_path, Path(out_path.parent, target.name)) From a75314d2ad3878af69cd90d45decb1360353ab75 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 19 Oct 2024 02:44:51 +0300 Subject: [PATCH 22/24] vnrecode: ignore ansi escapes for string cleaning --- vnrecode/printer.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/vnrecode/printer.py b/vnrecode/printer.py index a6046f9..19650aa 100644 --- a/vnrecode/printer.py +++ b/vnrecode/printer.py @@ -3,6 +3,7 @@ from pathlib import Path import colorama import sys import os +import re class Printer: """ @@ -26,7 +27,8 @@ class Printer: :param string: String to "clean" :return: "Clean" string """ - return string + " " * (os.get_terminal_size().columns - len(string)) + ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') + return string + " " * (os.get_terminal_size().columns - len(ansi_escape.sub('', string))) @staticmethod def win_ascii_esc(): @@ -56,18 +58,18 @@ class Printer: def warning(self, string: str): """ - Method prints string with decor for warning messages - :param string: String to print - :return: None - """ + Method prints string with decor for warning messages + :param string: String to print + :return: None + """ self.bar_print(self.clean_str(f"\r\033[93m!\033[0m {string}\033[49m")) def error(self, string: str): """ - Method prints string with decor for error messages - :param string: String to print - :return: None - """ + Method prints string with decor for error messages + :param string: String to print + :return: None + """ self.bar_print(self.clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m")) def files(self, source_path: Path, output_path: Path, comment: str): From bc84703b73bf1d0e6f6def3368bbce59170ef8c9 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 19 Oct 2024 02:51:57 +0300 Subject: [PATCH 23/24] vnrecode: fix typo in input folder name --- vnrecode/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnrecode/application.py b/vnrecode/application.py index c43102b..2b19681 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -43,7 +43,7 @@ class Application: if not output.exists(): os.mkdir(output) - self.__printer.info(f'Compressing "{output}" folder...') + self.__printer.info(f'Compressing "{folder}" folder...') with ThreadPoolExecutor(max_workers=self.__params.workers) as executor: futures = [ From 626eaae5e2ee78504785c5f399bb6440602fb7bb Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 1 Nov 2024 03:23:50 +0300 Subject: [PATCH 24/24] vnrecode: fix renaming for same name subfolders --- vnrecode/application.py | 2 +- vnrecode/utils.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/vnrecode/application.py b/vnrecode/application.py index 2b19681..216157e 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -39,7 +39,7 @@ class Application: self.__printer.info("Creating folders...") for folder, folders, files in os.walk(source): - output = Path(folder.replace(str(source), str(self.__params.dest))) + output = self.__utils.get_comp_subdir(folder) if not output.exists(): os.mkdir(output) diff --git a/vnrecode/utils.py b/vnrecode/utils.py index c30abe3..af0dd8c 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -37,6 +37,14 @@ class Utils: """ return hashlib.md5(filename.encode()).hexdigest()[:8] + def get_comp_subdir(self, folder: str) -> Path: + """ + Method returns the Path from str, changing the source folder in it to a compressed one + :param folder: source subfolder + :return: Path object with compressed subfolder + """ + return Path(folder.replace(str(self.__params.source), str(self.__params.dest), 1)) + def get_recode_status(self): """ Method prints recoding results