diff --git a/PKGBUILD b/PKGBUILD new file mode 100644 index 0000000..0c6ef83 --- /dev/null +++ b/PKGBUILD @@ -0,0 +1,31 @@ +# Maintainer: D. Can Celasun +# Contributor: Ezekiel Bethel + +_pkgname=VNTools +pkgname=vntools-git +pkgver=2.0.e5bf961 +pkgrel=1 +pkgdesc="Collection of tools used by VienDesu! Porting Team" +arch=("any") +url="https://github.com/VienDesuPorting/VNTools" +depends=("python" "python-pillow" "python-pillow-avif-plugin" "python-python-ffmpeg" "python-progress" "python-colorama") +makedepends=("python-setuptools" "git") +provides=("vntools") +source=("git+${url}.git#branch=testing") +sha256sums=("SKIP") + +pkgver() { + cd "${srcdir}/${_pkgname}" + printf "2.0.%s" "$(git rev-parse --short HEAD)" +} + +build() { + cd "${srcdir}/${_pkgname}" + python -m build --wheel --no-isolation +} + +package() { + cd "${srcdir}/${_pkgname}" + python -m installer --destdir="${pkgdir}" dist/*.whl +} + diff --git a/README.md b/README.md index b3b0091..f9b3c17 100644 --- a/README.md +++ b/README.md @@ -9,17 +9,15 @@ Collection of tools used by VienDesu! Porting Team ### Installation #### Download from releases: - * Windows - `TODO` - * Linux - `TODO` - * MacOS - `TODO` + * Windows - [x64](https://git.viende.su/VienDesuPorting/VNTools/releases/download/2.0.0/vntools-win-x64.zip) + * Linux - [x86_64](https://git.viende.su/VienDesuPorting/VNTools/releases/download/2.0.0/vntools-linux-x86_64.zip) [arm64](https://git.viende.su/VienDesuPorting/VNTools/releases/download/2.0.0/vntools-linux-arm64.zip) + * MacOS - [x86_64](https://git.viende.su/VienDesuPorting/VNTools/releases/download/2.0.0/vntools-darwin-x86_64.zip) [arm64](https://git.viende.su/VienDesuPorting/VNTools/releases/download/2.0.0/vntools-darwin-arm64.zip) #### Build tools as binaries: - * Run `./build.sh` on UNIX + * Run `./build.sh` for UNIX * Run `.\build.bat` for Windows - * Arch Linux - `TODO` - * NixOS - `TODO` #### Install as python package: * Run `pip install -U .` command in project folder - * Arch Linux - `TODO` + * Arch Linux - `paru -Bi .` * NixOS - `TODO` diff --git a/pyproject.toml b/pyproject.toml index f2c379f..01df088 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,13 +20,13 @@ unrenapk = "unrenapk.application:launch" [project] name = "vntools" -version = "2.0-dev" +version = "2.0.0" requires-python = ">= 3.11" dependencies = [ - "Pillow==10.3.0", - "pillow-avif-plugin==1.4.3", - "python-ffmpeg==2.0.12", - "progress==1.6", - "colorama==0.4.6", - "argparse~=1.4.0" + "Pillow>=10.3.0", + "pillow-avif-plugin>=1.4.3", + "python-ffmpeg>=2.0.12", + "progress>=1.6", + "colorama>=0.4.6", + "argparse>=1.4.0" ] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 16bd1e6..c6dd905 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -Pillow==10.3.0 -pillow-avif-plugin==1.4.3 -python-ffmpeg==2.0.12 -progress==1.6 -colorama==0.4.6 -argparse~=1.4.0 \ No newline at end of file +Pillow>=10.3.0 +pillow-avif-plugin>=1.4.3 +python-ffmpeg>=2.0.12 +progress>=1.6 +colorama>=0.4.6 +argparse>=1.4.0 diff --git a/vnrecode/__main__.py b/vnrecode/__main__.py index 28e325a..e44ac79 100644 --- a/vnrecode/__main__.py +++ b/vnrecode/__main__.py @@ -12,7 +12,7 @@ def init(): :return: None """ params = Params.setup() - printer = Printer(params) + printer = Printer(params.source) utils = Utils(params, printer) compress = Compress(params, printer, utils) diff --git a/vnrecode/application.py b/vnrecode/application.py index 91108c1..b8bbc57 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -3,8 +3,6 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime from pathlib import Path import shutil -import psutil -import signal import os from vnrecode.compress import Compress @@ -12,6 +10,7 @@ from vnrecode.printer import Printer from vnrecode.params import Params from vnrecode.utils import Utils + class Application: """ Main class for utility @@ -38,25 +37,23 @@ class Application: if self.__params.dest.exists(): shutil.rmtree(self.__params.dest) + self.__printer.info("Creating folders...") for folder, folders, files in os.walk(source): output = self.__utils.get_comp_subdir(folder) if not output.exists(): os.mkdir(output) - for chunk in range(0, len(files), self.__params.workers): - with ThreadPoolExecutor(max_workers=self.__params.workers) as executor: - self.__printer.workers = [] - #for file in files: - for file in files[chunk:chunk+self.__params.workers]: - if Path(folder, file).is_file(): - work_dict = { - "task": executor.submit(self.__compress, Path(folder, file), Path(output)), - "path": [Path(folder, file), Path(output)] - } - self.__printer.workers.append(work_dict) + self.__printer.info(f'Compressing "{folder}" folder...') + + with ThreadPoolExecutor(max_workers=self.__params.workers) as executor: + futures = [ + executor.submit(self.__compress, Path(folder, file), Path(output)) + for file in files if Path(folder, file).is_file() + ] + for future in as_completed(futures): + future.result() self.__utils.print_duplicates() self.__utils.get_recode_status() - self.__printer.plain(f"Time taken: {datetime.now() - start_time}") - self.__printer.stop() - self.__utils.sys_pause() \ No newline at end of file + self.__utils.sys_pause() + print(f"Time taken: {datetime.now() - start_time}") \ No newline at end of file diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 7435092..bbff31d 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -57,6 +57,7 @@ class Compress: def __init__(self, params_inst: Params, printer_inst: Printer, utils_inst: Utils): self.__params = params_inst + self.__printer = printer_inst self.__utils = utils_inst def audio(self, input_path: Path, output_dir: Path, extension: str) -> Path: @@ -69,7 +70,7 @@ class Compress: """ bit_rate = self.__params.audio_bitrate prefix = self.__utils.get_hash(input_path.name) - out_file = Path(output_dir, f'{prefix}_{input_path.stem}.{extension}') + out_file = Path(output_dir, f'.{prefix}_{input_path.stem}.{extension}') try: (FFmpeg() .input(input_path) @@ -79,6 +80,7 @@ class Compress: ) except FFmpegError as e: self.__utils.catch_unprocessed(input_path, out_file, e) + self.__printer.files(input_path, out_file, f"{bit_rate}") return out_file def image(self, input_path: Path, output_dir: Path, extension: str) -> Path: @@ -91,14 +93,15 @@ class Compress: """ quality = self.__params.image_quality prefix = self.__utils.get_hash(input_path.name) - out_file = Path(output_dir, f"{prefix}_{input_path.stem}.{extension}") + out_file = Path(output_dir, f".{prefix}_{input_path.stem}.{extension}") try: image = Image.open(input_path) if (extension == "jpg" or extension == "jpeg" or (extension == "webp" and not self.__params.webp_rgba)): if File.has_transparency(image): - out_file = Path(output_dir, f"{prefix}_{input_path.stem}.{self.__params.image_fall_ext}") + self.__printer.warning(f"{input_path.name} has transparency. Changing to fallback...") + out_file = Path(output_dir, f".{prefix}_{input_path.stem}.{self.__params.image_fall_ext}") if File.has_transparency(image): image.convert('RGBA') @@ -114,6 +117,7 @@ class Compress: lossless=self.__params.image_lossless, quality=quality, minimize_size=True) + self.__printer.files(input_path, out_file, f"{quality}%") except Exception as e: self.__utils.catch_unprocessed(input_path, out_file, e) return out_file @@ -127,7 +131,7 @@ class Compress: :return: Path of compressed video file with md5 hash as prefix """ prefix = self.__utils.get_hash(input_path.name) - out_file = Path(output_dir, f'{prefix}_{input_path.stem}.{extension}') + out_file = Path(output_dir, f'.{prefix}_{input_path.stem}.{extension}') if not self.__params.video_skip: codec = self.__params.video_codec crf = self.__params.video_crf @@ -140,6 +144,7 @@ class Compress: .output(out_file,{"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) .execute() ) + self.__printer.files(input_path, out_file, codec) except FFmpegError as e: self.__utils.catch_unprocessed(input_path, out_file, e) else: @@ -155,8 +160,9 @@ class Compress: :return: Path of compressed file with md5 hash as prefix """ prefix = self.__utils.get_hash(input_path.name) - out_file = Path(output_dir, f"{prefix}_{input_path.name}") + out_file = Path(output_dir, f".{prefix}_{input_path.name}") if self.__params.force_compress: + self.__printer.unknown_file(input_path.name) try: (FFmpeg() .input(input_path) @@ -187,5 +193,5 @@ class Compress: out_file = self.unknown(source, output) self.__utils.out_rename(out_file, source) - #self.__printer.bar.update() - #self.__printer.bar.next() + self.__printer.bar.update() + self.__printer.bar.next() diff --git a/vnrecode/params.py b/vnrecode/params.py index cf2e9e3..61fd8de 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -68,7 +68,10 @@ class Params: video_ext = config["VIDEO"]["Extension"] if args.config else args.v_ext video_codec = config["VIDEO"]["Codec"] if args.config else args.v_codec source = Path(args.source) - dest = Path(f"{args.source}_compressed") + if not source.exists(): + print("Requested path does not exists. Exiting!") + exit(255) + dest = Path(source.parent, source.name + f"_compressed") return cls( copy_unprocessed, force_compress, mimic_mode, hide_errors, webp_rgba, workers, @@ -93,17 +96,17 @@ class Params: parser.add_argument("-nm", "--no-mimic", dest='mimic', action='store_false', help="Disable mimic mode") parser.add_argument("-v", "--show_errors", action='store_false', help="Show recode errors") parser.add_argument("--webp-rgb", dest='webp_rgba', action='store_false', help="Recode .webp without alpha channel") - parser.add_argument("-j", "--jobs", type=int, help="Number of threads", default=8) - parser.add_argument("-ae", dest="a_ext", help="Audio extension", default="opus") - parser.add_argument("-ab", dest="a_bit", help="Audio bit rate", default="128k") - parser.add_argument("-id", dest="i_down", type=int, help="Image resolution downscale multiplier", default=1) - parser.add_argument("-ie", dest="i_ext", help="Image extension", default="avif") - parser.add_argument("-ife", dest="i_fallext", help="Image fallback extension", default="webp") + parser.add_argument("-j", "--jobs", type=int, help="Number of threads (default: 16)", default=16) + parser.add_argument("-ae", dest="a_ext", help="Audio extension (default: opus)", default="opus") + parser.add_argument("-ab", dest="a_bit", help="Audio bit rate (default: 128k)", default="128k") + parser.add_argument("-id", dest="i_down", type=float, help="Image resolution downscale multiplier (default: 1)", default=1) + parser.add_argument("-ie", dest="i_ext", help="Image extension (default: avif)", default="avif") + parser.add_argument("-ife", dest="i_fallext", help="Image fallback extension (default: webp)", default="webp") parser.add_argument("-il", dest='i_lossless', action='store_false', help="Image losing compression mode") - parser.add_argument("-iq", dest="i_quality", type=int, help="Image quality", default=100) - parser.add_argument("--v_crf", help="Video CRF number", type=int, default=27) + parser.add_argument("-iq", dest="i_quality", type=int, help="Image quality (default: 100)", default=100) + parser.add_argument("--v_crf", help="Video CRF number (default: 27)", type=int, default=27) parser.add_argument("-vs", dest="v_skip", action='store_true', help="Skip video recoding") - parser.add_argument("-ve", dest="v_ext", help="Video extension", default="webm") - parser.add_argument("-vc", dest="v_codec", help="Video codec name", default="libvpx-vp9") + parser.add_argument("-ve", dest="v_ext", help="Video extension (default: webm)", default="webm") + parser.add_argument("-vc", dest="v_codec", help="Video codec name (default: libvpx-vp9)", default="libvpx-vp9") args = parser.parse_args() return args \ No newline at end of file diff --git a/vnrecode/printer.py b/vnrecode/printer.py index 1ec9703..18d5138 100644 --- a/vnrecode/printer.py +++ b/vnrecode/printer.py @@ -1,103 +1,23 @@ -from time import sleep +from progress.bar import IncrementalBar +from pathlib import Path import colorama import sys import os -from vnrecode.params import Params -from concurrent.futures import ThreadPoolExecutor - - class Printer: """ Class implements CLI UI for this utility """ - __anim = ["\u280b", "\u2819", "\u28e0", "\u28c4"] - __ui_size = int - __messages = [] - - def __init__(self, params_inst: Params): + def __init__(self, source: Path): """ - :param params_inst: + :param source: Path of original (compressing) folder to count its files for progress bar """ file_count = 0 - for folder, folders, file in os.walk(params_inst.source): + for folder, folders, file in os.walk(source): file_count += len(file) - self.workers = [] - self.__ui_size = 0 - self.__running = True - self.__ui_updater = ThreadPoolExecutor().submit(self.update) - - def __print_msgs(self): - for msg in self.__messages: - self.__ui_size += 1 - print(msg) - - def __print_bar(self): - from random import randint - print(f"Recoding... [███████████████] {randint(0, 100)}%") - self.__ui_size += 1 - - def __print_folder(self): - if len(self.workers) > 0: - print(f"\x1b[2K\r\033[100m{self.workers[0]['path'][0].parent}\033[49m:") - self.__ui_size += 1 - - def __print_works(self, frame): - for task in self.workers: - if task['task'].__getstate__()['_state'] == "RUNNING": - self.__ui_size += 1 - print( - f"[{self.__anim[frame % len(self.__anim)]}] " - f"\033[0;37m{task['path'][0].stem}\033[0m{task['path'][0].suffix}\033[0;37m -> " - f"{task['path'][0].stem}\033[0m.file") - - def __clear(self): - print("\033[F\x1b[2K" * self.__ui_size, end='') - self.__ui_size = 0 - - def update(self): - frame = 0 - while self.__running: - self.__print_msgs() - self.__print_bar() - self.__print_folder() - self.__print_works(frame) - sleep(0.1) - self.__clear() - frame+=1 - - def stop(self): - self.__running = False - self.__ui_updater.result() - self.__print_msgs() - - def plain(self, string: str): - self.__messages.append(string) - - def info(self, string: str): - """ - Method prints string with decor for info messages - :param string: String to print - :return: None - """ - self.__messages.append(f"\x1b[2K\r\033[100m- {string}\033[49m") - - def warning(self, string: str): - """ - Method prints string with decor for warning messages - :param string: String to print - :return: None - """ - self.__messages.append(f"\x1b[2K\r\033[93m!\033[0m {string}\033[49m") - - def error(self, string: str): - """ - Method prints string with decor for error messages - :param string: String to print - :return: None - """ - self.__messages.append(f"\x1b[2K\r\033[31m\u2715\033[0m {string}\033[49m") + self.bar = IncrementalBar('Recoding', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)') + self.bar.update() @staticmethod def win_ascii_esc(): @@ -106,4 +26,57 @@ class Printer: :return: None """ if sys.platform == "win32": - colorama.init() \ No newline at end of file + colorama.init() + + def bar_print(self, string: str): + """ + Method prints some string in console and updates progress bar + :param string: String to print + :return: None + """ + print(string) + self.bar.update() + + def info(self, string: str): + """ + Method prints string with decor for info messages + :param string: String to print + :return: None + """ + self.bar_print(f"\x1b[2K\r\033[100m- {string}\033[49m") + + def warning(self, string: str): + """ + Method prints string with decor for warning messages + :param string: String to print + :return: None + """ + self.bar_print(f"\x1b[2K\r\033[93m!\033[0m {string}\033[49m") + + def error(self, string: str): + """ + Method prints string with decor for error messages + :param string: String to print + :return: None + """ + self.bar_print(f"\x1b[2K\r\033[31m\u2715\033[0m {string}\033[49m") + + def files(self, source_path: Path, output_path: Path, comment: str): + """ + Method prints the result of recoding a file with some decorations in the form: + input file name -> output file name (quality setting) + :param source_path: Input file Path + :param output_path: Output file Path + :param comment: Comment about recode quality setting + :return: None + """ + self.bar_print(f"\x1b[2K\r\033[0;32m\u2713\033[0m \033[0;37m{source_path.stem}\033[0m{source_path.suffix}\033[0;37m -> " + f"{source_path.stem}\033[0m{output_path.suffix}\033[0;37m ({comment})\033[0m") + + def unknown_file(self, filename: str): + """ + Method prints the result of recoding unknown file + :param filename: Name of unknown file + :return: + """ + self.bar_print(f"\x1b[2K\r\u2713 \033[0;33m{filename}\033[0m (File will be force compressed via ffmpeg)") diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 33352f1..af0dd8c 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -68,14 +68,13 @@ class Utils: self.__printer.info("Success!") else: self.__printer.warning("Original and compressed folders are not identical!") + try: + source = sum(file.stat().st_size for file in self.__params.source.glob('**/*') if file.is_file()) + output = sum(file.stat().st_size for file in self.__params.dest.glob('**/*') if file.is_file()) - source = sum(file.stat().st_size for file in self.__params.source.glob('**/*') if file.is_file()) - output = sum(file.stat().st_size for file in self.__params.dest.glob('**/*') if file.is_file()) - - if (output - source) != 0: - self.__printer.plain(f"Result: {source/1024/1024:.2f}MB -> " - f"{output/1024/1024:.2f}MB ({(output - source)/1024/1024:.2f}MB)") - else: + print(f"\nResult: {source/1024/1024:.2f}MB -> " + f"{output/1024/1024:.2f}MB ({(output - source)/1024/1024:.2f}MB)") + except ZeroDivisionError: self.__printer.warning("Nothing compressed!") def catch_unprocessed(self, input_path: Path, output_path: Path, error): @@ -101,7 +100,7 @@ class Utils: """ if self.__params.copy_unprocessed: copyfile(input_path, output_path) - #self.__printer.info(f"File {input_path.name} copied to compressed folder.") + self.__printer.info(f"File {input_path.name} copied to compressed folder.") def catch_duplicates(self, path: Path) -> Path: """