From 694cf4650fcdcd3fc4844b6c0ee8bf820ec211d5 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 9 Nov 2024 06:59:59 +0300 Subject: [PATCH] vnrecode: draft of multithread UI --- vnrecode/__main__.py | 2 +- vnrecode/application.py | 29 ++++---- vnrecode/compress.py | 10 +-- vnrecode/params.py | 2 +- vnrecode/printer.py | 149 ++++++++++++++++++++++++---------------- vnrecode/utils.py | 15 ++-- 6 files changed, 116 insertions(+), 91 deletions(-) diff --git a/vnrecode/__main__.py b/vnrecode/__main__.py index e44ac79..28e325a 100644 --- a/vnrecode/__main__.py +++ b/vnrecode/__main__.py @@ -12,7 +12,7 @@ def init(): :return: None """ params = Params.setup() - printer = Printer(params.source) + printer = Printer(params) utils = Utils(params, printer) compress = Compress(params, printer, utils) diff --git a/vnrecode/application.py b/vnrecode/application.py index b8bbc57..91108c1 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -3,6 +3,8 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime from pathlib import Path import shutil +import psutil +import signal import os from vnrecode.compress import Compress @@ -10,7 +12,6 @@ from vnrecode.printer import Printer from vnrecode.params import Params from vnrecode.utils import Utils - class Application: """ Main class for utility @@ -37,23 +38,25 @@ class Application: if self.__params.dest.exists(): shutil.rmtree(self.__params.dest) - self.__printer.info("Creating folders...") for folder, folders, files in os.walk(source): output = self.__utils.get_comp_subdir(folder) if not output.exists(): os.mkdir(output) - self.__printer.info(f'Compressing "{folder}" folder...') - - with ThreadPoolExecutor(max_workers=self.__params.workers) as executor: - futures = [ - executor.submit(self.__compress, Path(folder, file), Path(output)) - for file in files if Path(folder, file).is_file() - ] - for future in as_completed(futures): - future.result() + for chunk in range(0, len(files), self.__params.workers): + with ThreadPoolExecutor(max_workers=self.__params.workers) as executor: + self.__printer.workers = [] + #for file in files: + for file in files[chunk:chunk+self.__params.workers]: + if Path(folder, file).is_file(): + work_dict = { + "task": executor.submit(self.__compress, Path(folder, file), Path(output)), + "path": [Path(folder, file), Path(output)] + } + self.__printer.workers.append(work_dict) self.__utils.print_duplicates() self.__utils.get_recode_status() - self.__utils.sys_pause() - print(f"Time taken: {datetime.now() - start_time}") \ No newline at end of file + self.__printer.plain(f"Time taken: {datetime.now() - start_time}") + self.__printer.stop() + self.__utils.sys_pause() \ No newline at end of file diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 680da22..7435092 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -57,7 +57,6 @@ class Compress: def __init__(self, params_inst: Params, printer_inst: Printer, utils_inst: Utils): self.__params = params_inst - self.__printer = printer_inst self.__utils = utils_inst def audio(self, input_path: Path, output_dir: Path, extension: str) -> Path: @@ -80,7 +79,6 @@ class Compress: ) except FFmpegError as e: self.__utils.catch_unprocessed(input_path, out_file, e) - self.__printer.files(input_path, out_file, f"{bit_rate}") return out_file def image(self, input_path: Path, output_dir: Path, extension: str) -> Path: @@ -100,7 +98,6 @@ class Compress: if (extension == "jpg" or extension == "jpeg" or (extension == "webp" and not self.__params.webp_rgba)): if File.has_transparency(image): - self.__printer.warning(f"{input_path.name} has transparency. Changing to fallback...") out_file = Path(output_dir, f"{prefix}_{input_path.stem}.{self.__params.image_fall_ext}") if File.has_transparency(image): @@ -117,7 +114,6 @@ class Compress: lossless=self.__params.image_lossless, quality=quality, minimize_size=True) - self.__printer.files(input_path, out_file, f"{quality}%") except Exception as e: self.__utils.catch_unprocessed(input_path, out_file, e) return out_file @@ -144,7 +140,6 @@ class Compress: .output(out_file,{"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) .execute() ) - self.__printer.files(input_path, out_file, codec) except FFmpegError as e: self.__utils.catch_unprocessed(input_path, out_file, e) else: @@ -162,7 +157,6 @@ class Compress: prefix = self.__utils.get_hash(input_path.name) out_file = Path(output_dir, f"{prefix}_{input_path.name}") if self.__params.force_compress: - self.__printer.unknown_file(input_path.name) try: (FFmpeg() .input(input_path) @@ -193,5 +187,5 @@ class Compress: out_file = self.unknown(source, output) self.__utils.out_rename(out_file, source) - self.__printer.bar.update() - self.__printer.bar.next() + #self.__printer.bar.update() + #self.__printer.bar.next() diff --git a/vnrecode/params.py b/vnrecode/params.py index d0388f2..cf2e9e3 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -93,7 +93,7 @@ class Params: parser.add_argument("-nm", "--no-mimic", dest='mimic', action='store_false', help="Disable mimic mode") parser.add_argument("-v", "--show_errors", action='store_false', help="Show recode errors") parser.add_argument("--webp-rgb", dest='webp_rgba', action='store_false', help="Recode .webp without alpha channel") - parser.add_argument("-j", "--jobs", type=int, help="Number of threads", default=16) + parser.add_argument("-j", "--jobs", type=int, help="Number of threads", default=8) parser.add_argument("-ae", dest="a_ext", help="Audio extension", default="opus") parser.add_argument("-ab", dest="a_bit", help="Audio bit rate", default="128k") parser.add_argument("-id", dest="i_down", type=int, help="Image resolution downscale multiplier", default=1) diff --git a/vnrecode/printer.py b/vnrecode/printer.py index 9207ba7..1ec9703 100644 --- a/vnrecode/printer.py +++ b/vnrecode/printer.py @@ -1,23 +1,103 @@ -from progress.bar import IncrementalBar -from pathlib import Path +from time import sleep import colorama import sys import os +from vnrecode.params import Params +from concurrent.futures import ThreadPoolExecutor + + class Printer: """ Class implements CLI UI for this utility """ - def __init__(self, source: Path): + __anim = ["\u280b", "\u2819", "\u28e0", "\u28c4"] + __ui_size = int + __messages = [] + + def __init__(self, params_inst: Params): """ - :param source: Path of original (compressing) folder to count its files for progress bar + :param params_inst: """ file_count = 0 - for folder, folders, file in os.walk(source): + for folder, folders, file in os.walk(params_inst.source): file_count += len(file) - self.bar = IncrementalBar('Compressing', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)') - self.bar.update() + self.workers = [] + self.__ui_size = 0 + self.__running = True + self.__ui_updater = ThreadPoolExecutor().submit(self.update) + + def __print_msgs(self): + for msg in self.__messages: + self.__ui_size += 1 + print(msg) + + def __print_bar(self): + from random import randint + print(f"Recoding... [███████████████] {randint(0, 100)}%") + self.__ui_size += 1 + + def __print_folder(self): + if len(self.workers) > 0: + print(f"\x1b[2K\r\033[100m{self.workers[0]['path'][0].parent}\033[49m:") + self.__ui_size += 1 + + def __print_works(self, frame): + for task in self.workers: + if task['task'].__getstate__()['_state'] == "RUNNING": + self.__ui_size += 1 + print( + f"[{self.__anim[frame % len(self.__anim)]}] " + f"\033[0;37m{task['path'][0].stem}\033[0m{task['path'][0].suffix}\033[0;37m -> " + f"{task['path'][0].stem}\033[0m.file") + + def __clear(self): + print("\033[F\x1b[2K" * self.__ui_size, end='') + self.__ui_size = 0 + + def update(self): + frame = 0 + while self.__running: + self.__print_msgs() + self.__print_bar() + self.__print_folder() + self.__print_works(frame) + sleep(0.1) + self.__clear() + frame+=1 + + def stop(self): + self.__running = False + self.__ui_updater.result() + self.__print_msgs() + + def plain(self, string: str): + self.__messages.append(string) + + def info(self, string: str): + """ + Method prints string with decor for info messages + :param string: String to print + :return: None + """ + self.__messages.append(f"\x1b[2K\r\033[100m- {string}\033[49m") + + def warning(self, string: str): + """ + Method prints string with decor for warning messages + :param string: String to print + :return: None + """ + self.__messages.append(f"\x1b[2K\r\033[93m!\033[0m {string}\033[49m") + + def error(self, string: str): + """ + Method prints string with decor for error messages + :param string: String to print + :return: None + """ + self.__messages.append(f"\x1b[2K\r\033[31m\u2715\033[0m {string}\033[49m") @staticmethod def win_ascii_esc(): @@ -26,57 +106,4 @@ class Printer: :return: None """ if sys.platform == "win32": - colorama.init() - - def bar_print(self, string: str): - """ - Method prints some string in console and updates progress bar - :param string: String to print - :return: None - """ - print(string) - self.bar.update() - - def info(self, string: str): - """ - Method prints string with decor for info messages - :param string: String to print - :return: None - """ - self.bar_print(f"\x1b[2K\r\033[100m- {string}\033[49m") - - def warning(self, string: str): - """ - Method prints string with decor for warning messages - :param string: String to print - :return: None - """ - self.bar_print(f"\x1b[2K\r\033[93m!\033[0m {string}\033[49m") - - def error(self, string: str): - """ - Method prints string with decor for error messages - :param string: String to print - :return: None - """ - self.bar_print(f"\x1b[2K\r\033[31m\u2715\033[0m {string}\033[49m") - - def files(self, source_path: Path, output_path: Path, comment: str): - """ - Method prints the result of recoding a file with some decorations in the form: - input file name -> output file name (quality setting) - :param source_path: Input file Path - :param output_path: Output file Path - :param comment: Comment about recode quality setting - :return: None - """ - self.bar_print(f"\x1b[2K\r\033[0;32m\u2713\033[0m \033[0;37m{source_path.stem}\033[0m{source_path.suffix}\033[0;37m -> " - f"{source_path.stem}\033[0m{output_path.suffix}\033[0;37m ({comment})\033[0m") - - def unknown_file(self, filename: str): - """ - Method prints the result of recoding unknown file - :param filename: Name of unknown file - :return: - """ - self.bar_print(f"\x1b[2K\r\u2713 \033[0;33m{filename}\033[0m (File will be force compressed via ffmpeg)") + colorama.init() \ No newline at end of file diff --git a/vnrecode/utils.py b/vnrecode/utils.py index af0dd8c..33352f1 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -68,13 +68,14 @@ class Utils: self.__printer.info("Success!") else: self.__printer.warning("Original and compressed folders are not identical!") - try: - source = sum(file.stat().st_size for file in self.__params.source.glob('**/*') if file.is_file()) - output = sum(file.stat().st_size for file in self.__params.dest.glob('**/*') if file.is_file()) - print(f"\nResult: {source/1024/1024:.2f}MB -> " - f"{output/1024/1024:.2f}MB ({(output - source)/1024/1024:.2f}MB)") - except ZeroDivisionError: + source = sum(file.stat().st_size for file in self.__params.source.glob('**/*') if file.is_file()) + output = sum(file.stat().st_size for file in self.__params.dest.glob('**/*') if file.is_file()) + + if (output - source) != 0: + self.__printer.plain(f"Result: {source/1024/1024:.2f}MB -> " + f"{output/1024/1024:.2f}MB ({(output - source)/1024/1024:.2f}MB)") + else: self.__printer.warning("Nothing compressed!") def catch_unprocessed(self, input_path: Path, output_path: Path, error): @@ -100,7 +101,7 @@ class Utils: """ if self.__params.copy_unprocessed: copyfile(input_path, output_path) - self.__printer.info(f"File {input_path.name} copied to compressed folder.") + #self.__printer.info(f"File {input_path.name} copied to compressed folder.") def catch_duplicates(self, path: Path) -> Path: """