vnrecode: draft of multithread UI

PKGBUILD

@@ -1,31 +0,0 @@
-# Maintainer: D. Can Celasun <can[at]dcc[dot]im>
-# Contributor: Ezekiel Bethel <mctinfoilball@gmail.com>
-
-_pkgname=VNTools
-pkgname=vntools-git
-pkgver=2.0.e5bf961
-pkgrel=1
-pkgdesc="Collection of tools used by VienDesu! Porting Team"
-arch=("any")
-url="https://github.com/VienDesuPorting/VNTools"
-depends=("python" "python-pillow" "python-pillow-avif-plugin" "python-python-ffmpeg" "python-progress" "python-colorama")
-makedepends=("python-setuptools" "git")
-provides=("vntools")
-source=("git+${url}.git#branch=testing")
-sha256sums=("SKIP")
-
-pkgver() {
-  cd "${srcdir}/${_pkgname}"
-  printf "2.0.%s" "$(git rev-parse --short HEAD)"
-}
-
-build() {
-  cd "${srcdir}/${_pkgname}"
-  python -m build --wheel --no-isolation
-}
-
-package() {
-  cd "${srcdir}/${_pkgname}"
-  python -m installer --destdir="${pkgdir}" dist/*.whl
-}
-

README.md

@@ -9,15 +9,17 @@ Collection of tools used by VienDesu! Porting Team
 
 ### Installation
 #### Download from releases:
-  * Windows - [x64](https://git.viende.su/VienDesuPorting/VNTools/releases/download/2.0.0/vntools-win-x64.zip)
-  * Linux - [x86_64](https://git.viende.su/VienDesuPorting/VNTools/releases/download/2.0.0/vntools-linux-x86_64.zip) [arm64](https://git.viende.su/VienDesuPorting/VNTools/releases/download/2.0.0/vntools-linux-arm64.zip)
-  * MacOS - [x86_64](https://git.viende.su/VienDesuPorting/VNTools/releases/download/2.0.0/vntools-darwin-x86_64.zip) [arm64](https://git.viende.su/VienDesuPorting/VNTools/releases/download/2.0.0/vntools-darwin-arm64.zip)
+  * Windows - `TODO`
+  * Linux - `TODO`
+  * MacOS - `TODO`
 
 #### Build tools as binaries:
-  * Run `./build.sh` for UNIX 
+  * Run `./build.sh` on UNIX 
   * Run `.\build.bat` for Windows
+  * Arch Linux - `TODO`
+  * NixOS - `TODO`
 
 #### Install as python package:
   * Run `pip install -U .` command in project folder
-  * Arch Linux - `paru -Bi .`
+  * Arch Linux - `TODO`
   * NixOS - `TODO`

pyproject.toml

@@ -20,13 +20,13 @@ unrenapk = "unrenapk.application:launch"
 
 [project]
 name = "vntools"
-version = "2.0.0"
+version = "2.0-dev"
 requires-python = ">= 3.11"
 dependencies = [
-    "Pillow>=10.3.0",
-    "pillow-avif-plugin>=1.4.3",
-    "python-ffmpeg>=2.0.12",
-    "progress>=1.6",
-    "colorama>=0.4.6",
-    "argparse>=1.4.0"
+    "Pillow==10.3.0",
+    "pillow-avif-plugin==1.4.3",
+    "python-ffmpeg==2.0.12",
+    "progress==1.6",
+    "colorama==0.4.6",
+    "argparse~=1.4.0"
 ]

requirements.txt

@@ -1,6 +1,6 @@
-Pillow>=10.3.0
-pillow-avif-plugin>=1.4.3
-python-ffmpeg>=2.0.12
-progress>=1.6
-colorama>=0.4.6
-argparse>=1.4.0
+Pillow==10.3.0
+pillow-avif-plugin==1.4.3
+python-ffmpeg==2.0.12
+progress==1.6
+colorama==0.4.6
+argparse~=1.4.0

vnrecode/__main__.py

@@ -12,7 +12,7 @@ def init():
     :return: None
     """
     params = Params.setup()
-    printer = Printer(params.source)
+    printer = Printer(params)
     utils = Utils(params, printer)
     compress = Compress(params, printer, utils)
 

vnrecode/application.py

@@ -3,6 +3,8 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime
 from pathlib import Path
 import shutil
+import psutil
+import signal
 import os
 
 from vnrecode.compress import Compress
@@ -10,7 +12,6 @@ from vnrecode.printer import Printer
 from vnrecode.params import Params
 from vnrecode.utils import Utils
 
-
 class Application:
     """
     Main class for utility
@@ -37,23 +38,25 @@ class Application:
         if self.__params.dest.exists():
             shutil.rmtree(self.__params.dest)
 
-        self.__printer.info("Creating folders...")
         for folder, folders, files in os.walk(source):
             output = self.__utils.get_comp_subdir(folder)
             if not output.exists():
                 os.mkdir(output)
 
-            self.__printer.info(f'Compressing "{folder}" folder...')
-
-            with ThreadPoolExecutor(max_workers=self.__params.workers) as executor:
-                futures = [
-                    executor.submit(self.__compress, Path(folder, file), Path(output))
-                    for file in files if Path(folder, file).is_file()
-                ]
-                for future in as_completed(futures):
-                    future.result()
+            for chunk in range(0, len(files), self.__params.workers):
+                with ThreadPoolExecutor(max_workers=self.__params.workers) as executor:
+                    self.__printer.workers = []
+                    #for file in files:
+                    for file in files[chunk:chunk+self.__params.workers]:
+                        if Path(folder, file).is_file():
+                            work_dict = {
+                                "task": executor.submit(self.__compress, Path(folder, file), Path(output)),
+                                "path": [Path(folder, file), Path(output)]
+                            }
+                            self.__printer.workers.append(work_dict)
 
         self.__utils.print_duplicates()
         self.__utils.get_recode_status()
+        self.__printer.plain(f"Time taken: {datetime.now() - start_time}")
+        self.__printer.stop()
         self.__utils.sys_pause()
-        print(f"Time taken: {datetime.now() - start_time}")

vnrecode/compress.py

@@ -57,7 +57,6 @@ class Compress:
 
     def __init__(self, params_inst: Params, printer_inst: Printer, utils_inst: Utils):
         self.__params = params_inst
-        self.__printer = printer_inst
         self.__utils = utils_inst
 
     def audio(self, input_path: Path, output_dir: Path, extension: str) -> Path:
@@ -70,7 +69,7 @@ class Compress:
         """
         bit_rate = self.__params.audio_bitrate
         prefix = self.__utils.get_hash(input_path.name)
-        out_file = Path(output_dir, f'.{prefix}_{input_path.stem}.{extension}')
+        out_file = Path(output_dir, f'{prefix}_{input_path.stem}.{extension}')
         try:
             (FFmpeg()
              .input(input_path)
@@ -80,7 +79,6 @@ class Compress:
              )
         except FFmpegError as e:
             self.__utils.catch_unprocessed(input_path, out_file, e)
-        self.__printer.files(input_path, out_file, f"{bit_rate}")
         return out_file
 
     def image(self, input_path: Path, output_dir: Path, extension: str) -> Path:
@@ -93,15 +91,14 @@ class Compress:
         """
         quality = self.__params.image_quality
         prefix = self.__utils.get_hash(input_path.name)
-        out_file = Path(output_dir, f".{prefix}_{input_path.stem}.{extension}")
+        out_file = Path(output_dir, f"{prefix}_{input_path.stem}.{extension}")
         try:
             image = Image.open(input_path)
 
             if (extension == "jpg" or extension == "jpeg" or
                     (extension == "webp" and not self.__params.webp_rgba)):
                 if File.has_transparency(image):
-                    self.__printer.warning(f"{input_path.name} has transparency. Changing to fallback...")
-                    out_file = Path(output_dir, f".{prefix}_{input_path.stem}.{self.__params.image_fall_ext}")
+                    out_file = Path(output_dir, f"{prefix}_{input_path.stem}.{self.__params.image_fall_ext}")
 
             if File.has_transparency(image):
                 image.convert('RGBA')
@@ -117,7 +114,6 @@ class Compress:
                        lossless=self.__params.image_lossless,
                        quality=quality,
                        minimize_size=True)
-            self.__printer.files(input_path, out_file, f"{quality}%")
         except Exception as e:
             self.__utils.catch_unprocessed(input_path, out_file, e)
         return out_file
@@ -131,7 +127,7 @@ class Compress:
         :return: Path of compressed video file with md5 hash as prefix
         """
         prefix = self.__utils.get_hash(input_path.name)
-        out_file = Path(output_dir, f'.{prefix}_{input_path.stem}.{extension}')
+        out_file = Path(output_dir, f'{prefix}_{input_path.stem}.{extension}')
         if not self.__params.video_skip:
             codec = self.__params.video_codec
             crf = self.__params.video_crf
@@ -144,7 +140,6 @@ class Compress:
                  .output(out_file,{"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf)
                  .execute()
                  )
-                self.__printer.files(input_path, out_file, codec)
             except FFmpegError as e:
                 self.__utils.catch_unprocessed(input_path, out_file, e)
         else:
@@ -160,9 +155,8 @@ class Compress:
         :return: Path of compressed file with md5 hash as prefix
         """
         prefix = self.__utils.get_hash(input_path.name)
-        out_file = Path(output_dir, f".{prefix}_{input_path.name}")
+        out_file = Path(output_dir, f"{prefix}_{input_path.name}")
         if self.__params.force_compress:
-            self.__printer.unknown_file(input_path.name)
             try:
                 (FFmpeg()
                  .input(input_path)
@@ -193,5 +187,5 @@ class Compress:
                 out_file = self.unknown(source, output)
 
         self.__utils.out_rename(out_file, source)
-        self.__printer.bar.update()
-        self.__printer.bar.next()
+        #self.__printer.bar.update()
+        #self.__printer.bar.next()

vnrecode/params.py

@@ -68,10 +68,7 @@ class Params:
         video_ext = config["VIDEO"]["Extension"] if args.config else args.v_ext
         video_codec = config["VIDEO"]["Codec"] if args.config else args.v_codec
         source = Path(args.source)
-        if not source.exists():
-            print("Requested path does not exists. Exiting!")
-            exit(255)
-        dest = Path(source.parent, source.name + f"_compressed")
+        dest = Path(f"{args.source}_compressed")
 
         return cls(
             copy_unprocessed, force_compress, mimic_mode, hide_errors, webp_rgba, workers,
@@ -96,17 +93,17 @@ class Params:
         parser.add_argument("-nm", "--no-mimic", dest='mimic', action='store_false', help="Disable mimic mode")
         parser.add_argument("-v", "--show_errors", action='store_false', help="Show recode errors")
         parser.add_argument("--webp-rgb", dest='webp_rgba', action='store_false', help="Recode .webp without alpha channel")
-        parser.add_argument("-j", "--jobs", type=int, help="Number of threads (default: 16)", default=16)
-        parser.add_argument("-ae", dest="a_ext", help="Audio extension (default: opus)", default="opus")
-        parser.add_argument("-ab", dest="a_bit", help="Audio bit rate (default: 128k)", default="128k")
-        parser.add_argument("-id", dest="i_down", type=float, help="Image resolution downscale multiplier (default: 1)", default=1)
-        parser.add_argument("-ie", dest="i_ext", help="Image extension (default: avif)", default="avif")
-        parser.add_argument("-ife", dest="i_fallext", help="Image fallback extension (default: webp)", default="webp")
+        parser.add_argument("-j", "--jobs", type=int, help="Number of threads", default=8)
+        parser.add_argument("-ae", dest="a_ext", help="Audio extension", default="opus")
+        parser.add_argument("-ab", dest="a_bit", help="Audio bit rate", default="128k")
+        parser.add_argument("-id", dest="i_down", type=int, help="Image resolution downscale multiplier", default=1)
+        parser.add_argument("-ie", dest="i_ext", help="Image extension", default="avif")
+        parser.add_argument("-ife", dest="i_fallext", help="Image fallback extension", default="webp")
         parser.add_argument("-il", dest='i_lossless', action='store_false', help="Image losing compression mode")
-        parser.add_argument("-iq", dest="i_quality", type=int, help="Image quality (default: 100)", default=100)
-        parser.add_argument("--v_crf", help="Video CRF number (default: 27)", type=int, default=27)
+        parser.add_argument("-iq", dest="i_quality", type=int, help="Image quality", default=100)
+        parser.add_argument("--v_crf", help="Video CRF number", type=int, default=27)
         parser.add_argument("-vs", dest="v_skip", action='store_true', help="Skip video recoding")
-        parser.add_argument("-ve", dest="v_ext", help="Video extension (default: webm)", default="webm")
-        parser.add_argument("-vc", dest="v_codec", help="Video codec name (default: libvpx-vp9)", default="libvpx-vp9")
+        parser.add_argument("-ve", dest="v_ext", help="Video extension", default="webm")
+        parser.add_argument("-vc", dest="v_codec", help="Video codec name", default="libvpx-vp9")
         args = parser.parse_args()
         return args

vnrecode/printer.py

@@ -1,23 +1,103 @@
-from progress.bar import IncrementalBar
-from pathlib import Path
+from time import sleep
 import colorama
 import sys
 import os
 
+from vnrecode.params import Params
+from concurrent.futures import ThreadPoolExecutor
+
+
 class Printer:
     """
     Class implements CLI UI for this utility
     """
 
-    def __init__(self, source: Path):
+    __anim = ["\u280b", "\u2819", "\u28e0", "\u28c4"]
+    __ui_size = int
+    __messages = []
+
+    def __init__(self, params_inst: Params):
         """
-        :param source: Path of original (compressing) folder to count its files for progress bar
+        :param params_inst:
         """
         file_count = 0
-        for folder, folders, file in os.walk(source):
+        for folder, folders, file in os.walk(params_inst.source):
             file_count += len(file)
-        self.bar = IncrementalBar('Recoding', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)')
-        self.bar.update()
+        self.workers = []
+        self.__ui_size = 0
+        self.__running = True
+        self.__ui_updater = ThreadPoolExecutor().submit(self.update)
+
+    def __print_msgs(self):
+        for msg in self.__messages:
+            self.__ui_size += 1
+            print(msg)
+
+    def __print_bar(self):
+        from random import randint
+        print(f"Recoding... [███████████████] {randint(0, 100)}%")
+        self.__ui_size += 1
+
+    def __print_folder(self):
+        if len(self.workers) > 0:
+            print(f"\x1b[2K\r\033[100m{self.workers[0]['path'][0].parent}\033[49m:")
+            self.__ui_size += 1
+
+    def __print_works(self, frame):
+        for task in self.workers:
+            if task['task'].__getstate__()['_state'] == "RUNNING":
+                self.__ui_size += 1
+                print(
+                    f"[{self.__anim[frame % len(self.__anim)]}] "
+                    f"\033[0;37m{task['path'][0].stem}\033[0m{task['path'][0].suffix}\033[0;37m -> "
+                    f"{task['path'][0].stem}\033[0m.file")
+
+    def __clear(self):
+        print("\033[F\x1b[2K" * self.__ui_size, end='')
+        self.__ui_size = 0
+
+    def update(self):
+        frame = 0
+        while self.__running:
+            self.__print_msgs()
+            self.__print_bar()
+            self.__print_folder()
+            self.__print_works(frame)
+            sleep(0.1)
+            self.__clear()
+            frame+=1
+
+    def stop(self):
+        self.__running = False
+        self.__ui_updater.result()
+        self.__print_msgs()
+
+    def plain(self, string: str):
+        self.__messages.append(string)
+
+    def info(self, string: str):
+        """
+        Method prints string with decor for info messages
+        :param string: String to print
+        :return: None
+        """
+        self.__messages.append(f"\x1b[2K\r\033[100m- {string}\033[49m")
+
+    def warning(self, string: str):
+        """
+        Method prints string with decor for warning messages
+        :param string: String to print
+        :return: None
+        """
+        self.__messages.append(f"\x1b[2K\r\033[93m!\033[0m {string}\033[49m")
+
+    def error(self, string: str):
+        """
+        Method prints string with decor for error messages
+        :param string: String to print
+        :return: None
+        """
+        self.__messages.append(f"\x1b[2K\r\033[31m\u2715\033[0m {string}\033[49m")
 
     @staticmethod
     def win_ascii_esc():
@@ -27,56 +107,3 @@ class Printer:
         """
         if sys.platform == "win32":
             colorama.init()
-
-    def bar_print(self, string: str):
-        """
-        Method prints some string in console and updates progress bar
-        :param string: String to print
-        :return: None
-        """
-        print(string)
-        self.bar.update()
-
-    def info(self, string: str):
-        """
-        Method prints string with decor for info messages
-        :param string: String to print
-        :return: None
-        """
-        self.bar_print(f"\x1b[2K\r\033[100m- {string}\033[49m")
-
-    def warning(self, string: str):
-        """
-        Method prints string with decor for warning messages
-        :param string: String to print
-        :return: None
-        """
-        self.bar_print(f"\x1b[2K\r\033[93m!\033[0m {string}\033[49m")
-
-    def error(self, string: str):
-        """
-        Method prints string with decor for error messages
-        :param string: String to print
-        :return: None
-        """
-        self.bar_print(f"\x1b[2K\r\033[31m\u2715\033[0m {string}\033[49m")
-
-    def files(self, source_path: Path, output_path: Path, comment: str):
-        """
-        Method prints the result of recoding a file with some decorations in the form:
-        input file name -> output file name (quality setting)
-        :param source_path: Input file Path
-        :param output_path: Output file Path
-        :param comment: Comment about recode quality setting
-        :return: None
-        """
-        self.bar_print(f"\x1b[2K\r\033[0;32m\u2713\033[0m \033[0;37m{source_path.stem}\033[0m{source_path.suffix}\033[0;37m -> "
-                                      f"{source_path.stem}\033[0m{output_path.suffix}\033[0;37m ({comment})\033[0m")
-
-    def unknown_file(self, filename: str):
-        """
-        Method prints the result of recoding unknown file
-        :param filename: Name of unknown file
-        :return:
-        """
-        self.bar_print(f"\x1b[2K\r\u2713 \033[0;33m{filename}\033[0m (File will be force compressed via ffmpeg)")

vnrecode/utils.py

@@ -68,13 +68,14 @@ class Utils:
             self.__printer.info("Success!")
         else:
             self.__printer.warning("Original and compressed folders are not identical!")
-        try:
-            source = sum(file.stat().st_size for file in self.__params.source.glob('**/*') if file.is_file())
-            output = sum(file.stat().st_size for file in self.__params.dest.glob('**/*') if file.is_file())
 
-            print(f"\nResult: {source/1024/1024:.2f}MB -> "
-                  f"{output/1024/1024:.2f}MB ({(output - source)/1024/1024:.2f}MB)")
-        except ZeroDivisionError:
+        source = sum(file.stat().st_size for file in self.__params.source.glob('**/*') if file.is_file())
+        output = sum(file.stat().st_size for file in self.__params.dest.glob('**/*') if file.is_file())
+
+        if (output - source) != 0:
+            self.__printer.plain(f"Result: {source/1024/1024:.2f}MB -> "
+                                 f"{output/1024/1024:.2f}MB ({(output - source)/1024/1024:.2f}MB)")
+        else:
             self.__printer.warning("Nothing compressed!")
 
     def catch_unprocessed(self, input_path: Path, output_path: Path, error):
@@ -100,7 +101,7 @@ class Utils:
         """
         if self.__params.copy_unprocessed:
             copyfile(input_path, output_path)
-            self.__printer.info(f"File {input_path.name} copied to compressed folder.")
+            #self.__printer.info(f"File {input_path.name} copied to compressed folder.")
 
     def catch_duplicates(self, path: Path) -> Path:
         """