Fix packaging names

vnrecode/README.md

@@ -0,0 +1,40 @@
+## FFMpeg-Compressor
+Python utility uses ffmpeg to compress Visual Novel Resources
+
+### How to use
+* Download `ffmpeg-comp.toml` and put in next to binary or in to `/etc` folder
+* Change the configuration of the utility in `ffmpeg-comp.toml` for yourself
+* `ffmpeg-comp {folder}`
+* In result, you get `{folder-compressed}` near with original `{folder}`
+
+### Configuration
+#### FFMPEG section
+* CopyUnprocessed - Copy all files that failed to compress by ffmpeg to destination folder. In can help to recreate original folder, but with compressed files. (default: `true`)
+* ForceCompress - Force try to compress all files in directory via ffmpeg. (default: `false`)
+* MimicMode - Rename compressed file to it original name and extension. VN engines determine the file type by its header, so for example PNG file named file.jpg will be loaded as PNG file. (default: `true`)
+* HideErrors - Hide some errors about compression. (default: `true`)
+* WebpRGBA - Alpha channel in webp. If false switches extension to png. (default: `true`)
+
+#### AUDIO section
+* Extension - Required audio file extension. It supports: `.aac`, `.flac`, `.m4a`, `.mp3`, `.ogg`, `.opus`, `.raw`, `.wav`, `.wma`.
+* BitRate - Required audio bitrate. For best quality use `320k` value.
+
+#### IMAGE section
+* ResDownScale - Downscale image resolution count. (default: `1`)
+* Extension - Required image file extension. It supports: `.apng`, `.avif`, `.bmp`, `.tga`, `.tiff`, `.dds`, `.svg`, `.webp`, `.jpg/.jpeg`, `.png`
+* FallBackExtension - Extension if current format does not support RGBA.
+* Lossless - Enables lossless compression for supported formats. With this quality parameter means quality of compression. (default: `false`)
+* Quality - Quality level of images. Values range: `0-100` (100 - best quality, 0 - worst quality)
+
+#### VIDEO section
+* CRF ("Constant Quality") - Video quality parameter for ffmpeg. The CRF value can be from 0 to 63. Lower values mean better quality. Recommended values range from 15 to 35, with 31 being recommended for 1080p HD video. (default: `27`)
+* SkipVideo - Skip processing all video files. (default: `false`)
+* Extension - Required image file extension. It supports: `.3gp`, `.amv`, `.avi`, `.gif`, `.m2l`, `.m4v`, `.mkv`, `.mov`, `.mp4`, `.m4v`, `.mpeg`, `.mpv`, `.webm`, `.ogv`
+* Codec - (Maybe optional in future) Required video codec. (See official ffmpeg documentation for supported codecs) 
+
+### TODO (for testing branch)
+* [x] Recreate whole game directory with compressed files
+* [x] Cross-platform (Easy Windows usage and binaries, macOS binaries)
+* [x] Use ffmpeg python bindings instead of cli commands
+* [x] Multithread
+* [ ] Reorganize code

vnrecode/__main__.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+from .application import Application
+from .compress import Compress
+from .printer import Printer
+from .config import Config
+from .utils import Utils
+
+
+def init():
+    config = Config.setup_config()
+    printer = Printer(config.args.source)
+    utils = Utils(config.config, printer)
+    compress = Compress(config.config, printer, utils)
+
+    Application(config, compress, printer, utils).run()
+
+
+if __name__ == "__main__":
+    init()

vnrecode/application.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime
+import shutil
+import os
+
+
+class Application:
+
+    def __init__(self, config, compress, printer, utils):
+        self.config = config.config
+        self.args = config.args
+        self.compress = compress.compress
+        self.printer = printer
+        self.utils = utils
+
+    def compress_worker(self, folder, file, source, output):
+        if os.path.isfile(f'{folder}/{file}'):
+            self.compress(folder, file, source, output)
+
+    def run(self):
+        start_time = datetime.now()
+        self.printer.win_ascii_esc()
+
+        source = os.path.abspath(self.args.source)
+
+        if os.path.exists(f"{source}_compressed"):
+            shutil.rmtree(f"{source}_compressed")
+
+        self.printer.info("Creating folders...")
+        for folder, folders, files in os.walk(source):
+            if not os.path.exists(folder.replace(source, f"{source}_compressed")):
+                os.mkdir(folder.replace(source, f"{source}_compressed"))
+
+            self.printer.info(f'Compressing "{folder.replace(source, os.path.split(source)[-1])}" folder...')
+            output = folder.replace(source, f"{source}_compressed")
+
+            with ThreadPoolExecutor(max_workers=self.config["FFMPEG"]["Workers"]) as executor:
+                futures = [
+                    executor.submit(self.compress, folder, file, source, output)
+                    for file in files if os.path.isfile(f'{folder}/{file}')
+                ]
+                for future in as_completed(futures):
+                    future.result()
+
+        self.utils.get_compression_status(source)
+        self.utils.sys_pause()
+        print(f"Time taken: {datetime.now() - start_time}")

vnrecode/compress.py

@@ -0,0 +1,163 @@
+from ffmpeg import FFmpeg, FFmpegError
+from PIL import Image
+import pillow_avif
+import os
+
+
+class File:
+
+    @staticmethod
+    def get_type(filename):
+        audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma']
+        image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png']
+        video_ext = ['.3gp' '.amv', '.avi', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv',
+                     '.webm', '.ogv']
+
+        if os.path.splitext(filename)[1] in audio_ext:
+            return "audio"
+        elif os.path.splitext(filename)[1] in image_ext:
+            return "image"
+        elif os.path.splitext(filename)[1] in video_ext:
+            return "video"
+        else:
+            return "unknown"
+
+    @staticmethod
+    def has_transparency(img: Image) -> bool:
+        if img.info.get("transparency", None) is not None:
+            return True
+        if img.mode == "P":
+            transparent = img.info.get("transparency", -1)
+            for _, index in img.getcolors():
+                if index == transparent:
+                    return True
+        elif img.mode == "RGBA":
+            extrema = img.getextrema()
+            if extrema[3][0] < 255:
+                return True
+        return False
+
+
+class Compress:
+
+    def __init__(self, config, printer, utils):
+        self.config = config
+        self.printer = printer
+        self.utils = utils
+
+    def audio(self, folder, file, target_folder, extension):
+        bitrate = self.config['AUDIO']['BitRate']
+        try:
+            (FFmpeg()
+             .input(f'{folder}/{file}')
+             .option("hide_banner")
+             .output(self.utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'),
+                     {"b:a": bitrate, "loglevel": "error"})
+             .execute()
+             )
+        except FFmpegError as e:
+            self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
+            self.utils.errors += 1
+            if not self.config['FFMPEG']['HideErrors']:
+                self.printer.error(f"File {file} can't be processed! Error: {e}")
+        self.printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}")
+        return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'
+
+
+    def video(self, folder, file, target_folder, extension):
+        if not self.config['VIDEO']['SkipVideo']:
+            codec = self.config['VIDEO']['Codec']
+            crf = self.config['VIDEO']['CRF']
+
+            try:
+                (FFmpeg()
+                 .input(f'{folder}/{file}')
+                 .option("hide_banner")
+                 .option("hwaccel", "auto")
+                 .output(self.utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'),
+                         {"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf)
+                 .execute()
+                 )
+                self.printer.files(file, os.path.splitext(file)[0], extension, codec)
+            except FFmpegError as e:
+                self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
+                self.utils.errors += 1
+                if not self.config['FFMPEG']['HideErrors']:
+                    self.printer.error(f"File {file} can't be processed! Error: {e}")
+        else:
+            self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
+        return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'
+
+
+    def image(self, folder, file, target_folder, extension):
+        quality = self.config['IMAGE']['Quality']
+        try:
+            image = Image.open(f'{folder}/{file}')
+
+            if (extension == "jpg" or extension == "jpeg" or
+                    (extension == "webp" and not self.config['FFMPEG']['WebpRGBA'])):
+                if File.has_transparency(image):
+                    self.printer.warning(f"{file} has transparency. Changing to fallback...")
+                    extension = self.config['IMAGE']['FallBackExtension']
+
+            if File.has_transparency(image):
+                image.convert('RGBA')
+
+            res_downscale = self.config['IMAGE']['ResDownScale']
+            if res_downscale != 1:
+                width, height = image.size
+                new_size = (int(width / res_downscale), int(height / res_downscale))
+                image = image.resize(new_size)
+
+            image.save(self.utils.check_duplicates(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}"),
+                       optimize=True,
+                       lossless=self.config['IMAGE']['Lossless'],
+                       quality=quality,
+                       minimize_size=True)
+            self.printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%")
+        except Exception as e:
+            self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
+            self.utils.errors += 1
+            if not self.config['FFMPEG']['HideErrors']:
+                self.printer.error(f"File {file} can't be processed! Error: {e}")
+        return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'
+
+
+    def unknown(self, folder, file, target_folder):
+        if self.config["FFMPEG"]["ForceCompress"]:
+            self.printer.unknown_file(file)
+            try:
+                (FFmpeg()
+                 .input(f'{folder}/{file}')
+                 .output(self.utils.check_duplicates(f'{target_folder}/{file}'))
+                 .execute()
+                 )
+            except FFmpegError as e:
+                self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
+                self.utils.errors += 1
+                if not self.config['FFMPEG']['HideErrors']:
+                    self.printer.error(f"File {file} can't be processed! Error: {e}")
+        else:
+            self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
+        return f'{target_folder}/{file}'
+
+
+    def compress(self, _dir, filename, source, output):
+        match File.get_type(filename):
+            case "audio":
+                out_file = self.audio(_dir, filename, output, self.config['AUDIO']['Extension'])
+            case "image":
+                out_file = self.image(_dir, filename, output, self.config['IMAGE']['Extension'])
+            case "video":
+                out_file = self.video(_dir, filename, output, self.config['VIDEO']['Extension'])
+            case "unknown":
+                out_file = self.unknown(_dir, filename, output)
+
+        if self.config['FFMPEG']['MimicMode']:
+            try:
+                os.rename(out_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed"))
+            except FileNotFoundError:
+                self.printer.warning(f"File {out_file} failed to copy to out dir")
+
+        self.printer.bar.update()
+        self.printer.bar.next()

vnrecode/config.py

@@ -0,0 +1,28 @@
+import os.path
+from argparse import Namespace, ArgumentParser
+from dataclasses import dataclass
+from typing import Any
+import tomllib
+
+
+@dataclass
+class Config:
+
+    config: dict[str, Any]
+    args: Namespace
+
+    @classmethod
+    def setup_config(cls):
+        parser = ArgumentParser(prog="vnrecode",
+                                description="Python utility to compress Visual Novel Resources"
+                                )
+        parser.add_argument("source")
+        parser.add_argument("-c", "--config", default="vnrecode.toml")
+        args = parser.parse_args()
+        if os.path.isfile(args.config):
+            with open(args.config, "rb") as cfile:
+                config = tomllib.load(cfile)
+        else:
+            print("Failed to find config. Check `vnrecode -h` to more info")
+            exit(255)
+        return cls(config=config, args=args)

vnrecode/printer.py

@@ -0,0 +1,46 @@
+from progress.bar import IncrementalBar
+import colorama
+import sys
+import os
+
+
+class Printer:
+
+    def __init__(self, folder):
+        file_count = 0
+        for folder, folders, file in os.walk(folder):
+            file_count += len(file)
+        self.bar = IncrementalBar('Compressing', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)')
+        self.bar.update()
+
+    # Fill whole string with spaces for cleaning progress bar
+    @staticmethod
+    def clean_str(string):
+        return string + " " * (os.get_terminal_size().columns - len(string))
+
+    @staticmethod
+    def win_ascii_esc():
+        if sys.platform == "win32":
+            colorama.init()
+
+    def bar_print(self, string):
+        print(string)
+        self.bar.update()
+
+    def info(self, string):
+        self.bar_print(self.clean_str(f"\r\033[100m- {string}\033[49m"))
+
+    def warning(self, string):
+        self.bar_print(self.clean_str(f"\r\033[93m!\033[0m {string}\033[49m"))
+
+    def error(self, string):
+        self.bar_print(self.clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m"))
+
+    def files(self, source, dest, dest_ext, comment):
+        source_ext = os.path.splitext(source)[1]
+        source_name = os.path.splitext(source)[0]
+
+        self.bar_print(self.clean_str(f"\r\033[0;32m\u2713\033[0m \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m"))
+
+    def unknown_file(self, file):
+        self.bar_print(self.clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)"))

vnrecode/utils.py

@@ -0,0 +1,71 @@
+from shutil import copyfile
+import sys
+import os
+
+class Utils:
+
+    def __init__(self, config, printer):
+        self.errors = 0
+        self.config = config
+        self.printer = printer
+
+    @staticmethod
+    def sys_pause():
+        if sys.platform == "win32":
+            os.system("pause")
+
+    @staticmethod
+    def get_size(directory):
+        total_size = 0
+        for folder, folders, files in os.walk(directory):
+            for file in files:
+                if not os.path.islink(f"{folder}/{file}"):
+                    total_size += os.path.getsize(f"{folder}/{file}")
+        return total_size
+
+    def get_compression(self, source, output):
+        try:
+            source = self.get_size(source)
+            output = self.get_size(output)
+
+            print(f"\nResult: {source/1024/1024:.2f}MB -> "
+                  f"{output/1024/1024:.2f}MB ({(output - source)/1024/1024:.2f}MB)")
+        except ZeroDivisionError:
+            self.printer.warning("Nothing compressed!")
+
+    def get_compression_status(self, orig_folder):
+        orig_folder_len = 0
+        comp_folder_len = 0
+
+        for folder, folders, files in os.walk(orig_folder):
+            orig_folder_len += len(files)
+
+        for folder, folders, files in os.walk(f'{orig_folder}_compressed'):
+            for file in files:
+                if not os.path.splitext(file)[1].count("(copy)"):
+                    comp_folder_len += 1
+
+        if self.errors != 0:
+            self.printer.warning("Some files failed to compress!")
+
+        if orig_folder_len == comp_folder_len:
+            self.printer.info("Success!")
+            self.get_compression(orig_folder, f"{orig_folder}_compressed")
+        else:
+            self.printer.warning("Original and compressed folders are not identical!")
+            self.get_compression(orig_folder, f"{orig_folder}_compressed")
+
+    def add_unprocessed_file(self, orig_folder, new_folder):
+        if self.config['FFMPEG']['CopyUnprocessed']:
+            filename = orig_folder.split("/").pop()
+            copyfile(orig_folder, new_folder)
+            self.printer.info(f"File {filename} copied to compressed folder.")
+
+    def check_duplicates(self, new_folder):
+        filename = new_folder.split().pop()
+        if os.path.exists(new_folder):
+            self.printer.warning(
+                f'Duplicate file has been found! Check manually this files - "{filename}", '
+                f'"{os.path.splitext(filename)[0] + "(copy)" + os.path.splitext(filename)[1]}"')
+            return os.path.splitext(new_folder)[0] + "(copy)" + os.path.splitext(new_folder)[1]
+        return new_folder

vnrecode/vnrecode.toml

@@ -0,0 +1,24 @@
+[FFMPEG]
+CopyUnprocessed = true
+ForceCompress = false
+MimicMode = true
+HideErrors = true
+WebpRGBA = true
+Workers = 16
+
+[AUDIO]
+Extension = "opus"
+BitRate = "128k"
+
+[IMAGE]
+ResDownScale = 1
+Extension = "avif"
+FallBackExtension = "webp"
+Lossless = true
+Quality = 100
+
+[VIDEO]
+CRF = 27
+SkipVideo = false
+Extension = "webm"
+Codec = "libvpx-vp9"