From 7487eb94bd7971b6183f1ca776e8ca5ad232f31e Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Thu, 29 Aug 2024 01:32:36 +0300 Subject: [PATCH] Basic refactor for packaging --- .gitignore | 3 + FFMpeg-Compressor/__init__.py | 0 FFMpeg-Compressor/__main__.py | 19 ++ FFMpeg-Compressor/application.py | 48 ++++++ FFMpeg-Compressor/compress.py | 163 ++++++++++++++++++ FFMpeg-Compressor/config.py | 27 +++ FFMpeg-Compressor/main.py | 59 ------- FFMpeg-Compressor/modules/compressor.py | 160 ----------------- FFMpeg-Compressor/modules/configloader.py | 11 -- FFMpeg-Compressor/modules/printer.py | 51 ------ FFMpeg-Compressor/modules/utils.py | 79 --------- FFMpeg-Compressor/printer.py | 46 +++++ FFMpeg-Compressor/utils.py | 71 ++++++++ README.md | 25 ++- RenPy-Android-Unpack/__init__.py | 0 RenPy-Android-Unpack/__main__.py | 52 ++++++ RenPy-Android-Unpack/actions.py | 98 +++++++++++ RenPy-Android-Unpack/printer.py | 14 ++ RenPy-Android-Unpack/requirements.txt | 2 - RenPy-Android-Unpack/unpack.py | 108 ------------ VNDS-to-RenPy/__init__.py | 0 VNDS-to-RenPy/{convert.py => __main__.py} | 0 build.bat | 10 +- build.sh | 12 +- .../requirements.txt => requirements.txt | 0 25 files changed, 573 insertions(+), 485 deletions(-) create mode 100644 FFMpeg-Compressor/__init__.py create mode 100644 FFMpeg-Compressor/__main__.py create mode 100755 FFMpeg-Compressor/application.py create mode 100644 FFMpeg-Compressor/compress.py create mode 100644 FFMpeg-Compressor/config.py delete mode 100755 FFMpeg-Compressor/main.py delete mode 100644 FFMpeg-Compressor/modules/compressor.py delete mode 100644 FFMpeg-Compressor/modules/configloader.py delete mode 100644 FFMpeg-Compressor/modules/printer.py delete mode 100644 FFMpeg-Compressor/modules/utils.py create mode 100644 FFMpeg-Compressor/printer.py create mode 100644 FFMpeg-Compressor/utils.py create mode 100644 RenPy-Android-Unpack/__init__.py create mode 100755 RenPy-Android-Unpack/__main__.py create mode 100755 RenPy-Android-Unpack/actions.py create mode 100644 RenPy-Android-Unpack/printer.py delete mode 100644 RenPy-Android-Unpack/requirements.txt delete mode 100755 RenPy-Android-Unpack/unpack.py create mode 100644 VNDS-to-RenPy/__init__.py rename VNDS-to-RenPy/{convert.py => __main__.py} (100%) mode change 100644 => 100755 build.bat rename FFMpeg-Compressor/requirements.txt => requirements.txt (100%) diff --git a/.gitignore b/.gitignore index 16be8f2..abe0419 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ /output/ +/tests/ +/build/ +/VNTools.egg-info/ diff --git a/FFMpeg-Compressor/__init__.py b/FFMpeg-Compressor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/FFMpeg-Compressor/__main__.py b/FFMpeg-Compressor/__main__.py new file mode 100644 index 0000000..aef4d34 --- /dev/null +++ b/FFMpeg-Compressor/__main__.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +from application import Application +from compress import Compress +from printer import Printer +from config import Config +from utils import Utils + + +def init(): + config = Config.setup_config() + printer = Printer(config.args.source) + utils = Utils(config.config, printer) + compress = Compress(config.config, printer, utils) + + Application(config, compress, printer, utils).run() + + +if __name__ == "__main__": + init() \ No newline at end of file diff --git a/FFMpeg-Compressor/application.py b/FFMpeg-Compressor/application.py new file mode 100755 index 0000000..491267a --- /dev/null +++ b/FFMpeg-Compressor/application.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime +import shutil +import os + + +class Application: + + def __init__(self, config, compress, printer, utils): + self.config = config.config + self.args = config.args + self.compress = compress.compress + self.printer = printer + self.utils = utils + + def compress_worker(self, folder, file, source, output): + if os.path.isfile(f'{folder}/{file}'): + self.compress(folder, file, source, output) + + def run(self): + start_time = datetime.now() + self.printer.win_ascii_esc() + + source = os.path.abspath(self.args.source) + + if os.path.exists(f"{source}_compressed"): + shutil.rmtree(f"{source}_compressed") + + self.printer.info("Creating folders...") + for folder, folders, files in os.walk(source): + if not os.path.exists(folder.replace(source, f"{source}_compressed")): + os.mkdir(folder.replace(source, f"{source}_compressed")) + + self.printer.info(f'Compressing "{folder.replace(source, os.path.split(source)[-1])}" folder...') + output = folder.replace(source, f"{source}_compressed") + + with ThreadPoolExecutor(max_workers=self.config["FFMPEG"]["Workers"]) as executor: + futures = [ + executor.submit(self.compress, folder, file, source, output) + for file in files if os.path.isfile(f'{folder}/{file}') + ] + for future in as_completed(futures): + future.result() + + self.utils.get_compression_status(source) + self.utils.sys_pause() + print(f"Time taken: {datetime.now() - start_time}") \ No newline at end of file diff --git a/FFMpeg-Compressor/compress.py b/FFMpeg-Compressor/compress.py new file mode 100644 index 0000000..5c1a465 --- /dev/null +++ b/FFMpeg-Compressor/compress.py @@ -0,0 +1,163 @@ +from ffmpeg import FFmpeg, FFmpegError +from PIL import Image +import pillow_avif +import os + + +class File: + + @staticmethod + def get_type(filename): + audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] + image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'] + video_ext = ['.3gp' '.amv', '.avi', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', + '.webm', '.ogv'] + + if os.path.splitext(filename)[1] in audio_ext: + return "audio" + elif os.path.splitext(filename)[1] in image_ext: + return "image" + elif os.path.splitext(filename)[1] in video_ext: + return "video" + else: + return "unknown" + + @staticmethod + def has_transparency(img: Image) -> bool: + if img.info.get("transparency", None) is not None: + return True + if img.mode == "P": + transparent = img.info.get("transparency", -1) + for _, index in img.getcolors(): + if index == transparent: + return True + elif img.mode == "RGBA": + extrema = img.getextrema() + if extrema[3][0] < 255: + return True + return False + + +class Compress: + + def __init__(self, config, printer, utils): + self.config = config + self.printer = printer + self.utils = utils + + def audio(self, folder, file, target_folder, extension): + bitrate = self.config['AUDIO']['BitRate'] + try: + (FFmpeg() + .input(f'{folder}/{file}') + .option("hide_banner") + .output(self.utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), + {"b:a": bitrate, "loglevel": "error"}) + .execute() + ) + except FFmpegError as e: + self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.errors += 1 + if not self.config['FFMPEG']['HideErrors']: + self.printer.error(f"File {file} can't be processed! Error: {e}") + self.printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}") + return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' + + + def video(self, folder, file, target_folder, extension): + if not self.config['VIDEO']['SkipVideo']: + codec = self.config['VIDEO']['Codec'] + crf = self.config['VIDEO']['CRF'] + + try: + (FFmpeg() + .input(f'{folder}/{file}') + .option("hide_banner") + .option("hwaccel", "auto") + .output(self.utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), + {"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) + .execute() + ) + self.printer.files(file, os.path.splitext(file)[0], extension, codec) + except FFmpegError as e: + self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.errors += 1 + if not self.config['FFMPEG']['HideErrors']: + self.printer.error(f"File {file} can't be processed! Error: {e}") + else: + self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' + + + def image(self, folder, file, target_folder, extension): + quality = self.config['IMAGE']['Quality'] + try: + image = Image.open(f'{folder}/{file}') + + if (extension == "jpg" or extension == "jpeg" or + (extension == "webp" and not self.config['FFMPEG']['WebpRGBA'])): + if File.has_transparency(image): + self.printer.warning(f"{file} has transparency. Changing to fallback...") + extension = self.config['IMAGE']['FallBackExtension'] + + if File.has_transparency(image): + image.convert('RGBA') + + res_downscale = self.config['IMAGE']['ResDownScale'] + if res_downscale != 1: + width, height = image.size + new_size = (int(width / res_downscale), int(height / res_downscale)) + image = image.resize(new_size) + + image.save(self.utils.check_duplicates(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}"), + optimize=True, + lossless=self.config['IMAGE']['Lossless'], + quality=quality, + minimize_size=True) + self.printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") + except Exception as e: + self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.errors += 1 + if not self.config['FFMPEG']['HideErrors']: + self.printer.error(f"File {file} can't be processed! Error: {e}") + return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' + + + def unknown(self, folder, file, target_folder): + if self.config["FFMPEG"]["ForceCompress"]: + self.printer.unknown_file(file) + try: + (FFmpeg() + .input(f'{folder}/{file}') + .output(self.utils.check_duplicates(f'{target_folder}/{file}')) + .execute() + ) + except FFmpegError as e: + self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.errors += 1 + if not self.config['FFMPEG']['HideErrors']: + self.printer.error(f"File {file} can't be processed! Error: {e}") + else: + self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + return f'{target_folder}/{file}' + + + def compress(self, _dir, filename, source, output): + match File.get_type(filename): + case "audio": + out_file = self.audio(_dir, filename, output, self.config['AUDIO']['Extension']) + case "image": + out_file = self.image(_dir, filename, output, self.config['IMAGE']['Extension']) + case "video": + out_file = self.video(_dir, filename, output, self.config['VIDEO']['Extension']) + case "unknown": + out_file = self.unknown(_dir, filename, output) + + if self.config['FFMPEG']['MimicMode']: + try: + os.rename(out_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed")) + except FileNotFoundError: + self.printer.warning(f"File {out_file} failed to copy to out dir") + + self.printer.bar.update() + self.printer.bar.next() diff --git a/FFMpeg-Compressor/config.py b/FFMpeg-Compressor/config.py new file mode 100644 index 0000000..d36e3d4 --- /dev/null +++ b/FFMpeg-Compressor/config.py @@ -0,0 +1,27 @@ +import os.path +from argparse import Namespace, ArgumentParser +from dataclasses import dataclass +from typing import Any +import tomllib + + +@dataclass +class Config: + + config: dict[str, Any] + args: Namespace + + @classmethod + def setup_config(cls): + parser = ArgumentParser(prog="ffmpeg-comp", + description="Python utility to compress Visual Novel Resources" + ) + parser.add_argument("source") + parser.add_argument("-c", "--config", default="ffmpeg-comp.toml") + args = parser.parse_args() + if os.path.isfile(args.config): + with open(args.config, "rb") as cfile: + config = tomllib.load(cfile) + else: + print("Failed to find config. Check `ffmpeg-comp -h` to more info") + return cls(config=config, args=args) diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py deleted file mode 100755 index 7c05dc5..0000000 --- a/FFMpeg-Compressor/main.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 - -from concurrent.futures import ThreadPoolExecutor, as_completed -from modules.configloader import config -from modules import compressor -from modules import printer -from modules import utils -from datetime import datetime -import shutil -import sys -import os - - -def get_args(): - try: - if sys.argv[1][len(sys.argv[1])-1] == "/": - path = sys.argv[1][:len(sys.argv[1])-1] - else: - path = sys.argv[1] - return path - except IndexError: - print(utils.help_message()) - exit() - - -def compress_worker(folder, file, target_folder, req_folder): - if os.path.isfile(f'{folder}/{file}'): - compressor.compress_file(folder, file, target_folder, req_folder) - - -if __name__ == "__main__": - start_time = datetime.now() - printer.win_ascii_esc() - req_folder = os.path.abspath(get_args()) - - printer.bar_init(req_folder) - - if os.path.exists(f"{req_folder}_compressed"): - shutil.rmtree(f"{req_folder}_compressed") - - printer.info("Creating folders...") - for folder, folders, files in os.walk(req_folder): - if not os.path.exists(folder.replace(req_folder, f"{req_folder}_compressed")): - os.mkdir(folder.replace(req_folder, f"{req_folder}_compressed")) - - printer.info(f"Compressing \"{folder.replace(req_folder, req_folder.split('/').pop())}\" folder...") - target_folder = folder.replace(req_folder, f"{req_folder}_compressed") - - with ThreadPoolExecutor(max_workers=config["FFMPEG"]["Workers"]) as executor: - futures = [ - executor.submit(compress_worker, folder, file, target_folder, req_folder) - for file in files - ] - for future in as_completed(futures): - future.result() - - utils.get_compression_status(req_folder) - utils.sys_pause() - print(f"Time taken: {datetime.now() - start_time}") \ No newline at end of file diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py deleted file mode 100644 index 728953c..0000000 --- a/FFMpeg-Compressor/modules/compressor.py +++ /dev/null @@ -1,160 +0,0 @@ -from modules import configloader -from modules import printer -from modules import utils -from PIL import Image -import pillow_avif -from ffmpeg import FFmpeg, FFmpegError -import os - - -def get_file_type(filename): - audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] - image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'] - video_ext = ['.3gp' '.amv', '.avi', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', - '.webm', '.ogv'] - - if os.path.splitext(filename)[1] in audio_ext: - return "audio" - elif os.path.splitext(filename)[1] in image_ext: - return "image" - elif os.path.splitext(filename)[1] in video_ext: - return "video" - else: - return "unknown" - - -def has_transparency(img): - if img.info.get("transparency", None) is not None: - return True - if img.mode == "P": - transparent = img.info.get("transparency", -1) - for _, index in img.getcolors(): - if index == transparent: - return True - elif img.mode == "RGBA": - extrema = img.getextrema() - if extrema[3][0] < 255: - return True - - return False - - -def compress_audio(folder, file, target_folder, extension): - bitrate = configloader.config['AUDIO']['BitRate'] - try: - (FFmpeg() - .input(f'{folder}/{file}') - .option("hide_banner") - .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), - {"b:a": bitrate, "loglevel": "error"}) - .execute() - ) - except FFmpegError as e: - utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - utils.errors_count += 1 - if not configloader.config['FFMPEG']['HideErrors']: - printer.error(f"File {file} can't be processed! Error: {e}") - printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}") - return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' - - -def compress_video(folder, file, target_folder, extension): - if not configloader.config['VIDEO']['SkipVideo']: - codec = configloader.config['VIDEO']['Codec'] - crf = configloader.config['VIDEO']['CRF'] - - try: - (FFmpeg() - .input(f'{folder}/{file}') - .option("hide_banner") - .option("hwaccel", "auto") - .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), - {"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) - .execute() - ) - printer.files(file, os.path.splitext(file)[0], extension, codec) - except FFmpegError as e: - utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - utils.errors_count += 1 - if not configloader.config['FFMPEG']['HideErrors']: - printer.error(f"File {file} can't be processed! Error: {e}") - else: - utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' - - -def compress_image(folder, file, target_folder, extension): - quality = configloader.config['IMAGE']['Quality'] - try: - image = Image.open(f'{folder}/{file}') - - if (extension == "jpg" or extension == "jpeg" or - (extension == "webp" and not configloader.config['FFMPEG']['WebpRGBA'])): - if has_transparency(image): - printer.warning(f"{file} has transparency. Changing to fallback...") - extension = configloader.config['IMAGE']['FallBackExtension'] - - if has_transparency(image): - image.convert('RGBA') - - res_downscale = configloader.config['IMAGE']['ResDownScale'] - if res_downscale != 1: - width, height = image.size - new_size = (int(width / res_downscale), int(height / res_downscale)) - image = image.resize(new_size) - - image.save(utils.check_duplicates(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}"), - optimize=True, - lossless=configloader.config['IMAGE']['Lossless'], - quality=quality, - minimize_size=True) - printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") - except Exception as e: - utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - utils.errors_count += 1 - if not configloader.config['FFMPEG']['HideErrors']: - printer.error(f"File {file} can't be processed! Error: {e}") - return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' - - -def compress(folder, file, target_folder): - if configloader.config["FFMPEG"]["ForceCompress"]: - printer.unknown_file(file) - try: - (FFmpeg() - .input(f'{folder}/{file}') - .output(f'{target_folder}/{file}') - .execute() - ) - except FFmpegError as e: - utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - utils.errors_count += 1 - if not configloader.config['FFMPEG']['HideErrors']: - printer.error(f"File {file} can't be processed! Error: {e}") - else: - utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - return f'{target_folder}/{file}' - - -def compress_file(_dir, filename, target_dir, source): - match get_file_type(filename): - case "audio": - comp_file = compress_audio(_dir, filename, target_dir, - configloader.config['AUDIO']['Extension']) - case "image": - comp_file = compress_image(_dir, filename, target_dir, - configloader.config['IMAGE']['Extension']) - case "video": - comp_file = compress_video(_dir, filename, target_dir, - configloader.config['VIDEO']['Extension']) - case "unknown": - comp_file = compress(_dir, filename, target_dir) - - if configloader.config['FFMPEG']['MimicMode']: - try: - os.rename(comp_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed")) - except FileNotFoundError: - pass - - printer.bar.update() - printer.bar.next() diff --git a/FFMpeg-Compressor/modules/configloader.py b/FFMpeg-Compressor/modules/configloader.py deleted file mode 100644 index 1a4db0b..0000000 --- a/FFMpeg-Compressor/modules/configloader.py +++ /dev/null @@ -1,11 +0,0 @@ -import tomllib -from modules import printer - -try: - config = tomllib.load(open("ffmpeg-comp.toml", "rb")) -except FileNotFoundError: - try: - config = tomllib.load(open("/etc/ffmpeg-comp.toml", "rb")) - except FileNotFoundError: - printer.error("Config file not found. Please put it next to binary or in to /etc folder.") - exit() diff --git a/FFMpeg-Compressor/modules/printer.py b/FFMpeg-Compressor/modules/printer.py deleted file mode 100644 index 0fdce24..0000000 --- a/FFMpeg-Compressor/modules/printer.py +++ /dev/null @@ -1,51 +0,0 @@ -from progress.bar import IncrementalBar -import colorama -import sys -import os - - -def bar_init(folder): - file_count = 0 - for folder, folders, file in os.walk(folder): - file_count += len(file) - global bar - bar = IncrementalBar('Compressing', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)') - bar.update() - - -def bar_print(string): - print(string) - bar.update() - - -# Fill whole string with spaces for cleaning progress bar -def clean_str(string): - return string + " " * (os.get_terminal_size().columns - len(string)) - - -def info(string): - bar_print(clean_str(f"\r\033[100m- {string}\033[49m")) - - -def warning(string): - bar_print(clean_str(f"\r\033[93m!\033[0m {string}\033[49m")) - - -def error(string): - bar_print(clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m")) - - -def files(source, dest, dest_ext, comment): - source_ext = os.path.splitext(source)[1] - source_name = os.path.splitext(source)[0] - - bar_print(clean_str(f"\r\033[0;32m\u2713\033[0m \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m")) - - -def unknown_file(file): - bar_print(clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)")) - - -def win_ascii_esc(): - if sys.platform == "win32": - colorama.init() diff --git a/FFMpeg-Compressor/modules/utils.py b/FFMpeg-Compressor/modules/utils.py deleted file mode 100644 index 907f9b1..0000000 --- a/FFMpeg-Compressor/modules/utils.py +++ /dev/null @@ -1,79 +0,0 @@ -from modules import configloader -from modules import printer -from shutil import copyfile -import sys -import os - -errors_count = 0 - - -def get_dir_size(directory, files): - total_size = 0 - for folder, folders, files in os.walk(directory): - for file in files: - if not os.path.islink(f"{folder}/{file}"): - total_size += os.path.getsize(f"{folder}/{file}") - return total_size - - -def get_compression(orig, comp): - processed_files = [] - for folder, folders, files in os.walk(comp): - for file in files: - processed_files.append(file) - try: - orig = get_dir_size(orig, processed_files) - comp = get_dir_size(comp, processed_files) - - print(f"\nResult: {orig/1024/1024:.2f}MB -> {comp/1024/1024:.2f}MB ({(comp - orig)/1024/1024:.2f}MB)") - except ZeroDivisionError: - printer.warning("Nothing compressed!") - - -def get_compression_status(orig_folder): - orig_folder_len = 0 - comp_folder_len = 0 - - for folder, folders, files in os.walk(orig_folder): - orig_folder_len += len(files) - - for folder, folders, files in os.walk(f'{orig_folder}_compressed'): - for file in files: - if not os.path.splitext(file)[1].count(" (copy)"): - comp_folder_len += 1 - - if errors_count != 0: - printer.warning("Some files failed to compress!") - - if orig_folder_len == comp_folder_len: - printer.info("Success!") - get_compression(orig_folder, f"{orig_folder}_compressed") - else: - printer.warning("Original and compressed folders are not identical!") - get_compression(orig_folder, f"{orig_folder}_compressed") - - -def add_unprocessed_file(orig_folder, new_folder): - if configloader.config['FFMPEG']['CopyUnprocessed']: - filename = orig_folder.split("/").pop() - copyfile(orig_folder, new_folder) - printer.info(f"File {filename} copied to compressed folder.") - - -def check_duplicates(new_folder): - filename = new_folder.split().pop() - if os.path.exists(new_folder): - printer.warning( - f'Duplicate file has been found! Check manually this files - "{filename}", ' - f'"{os.path.splitext(filename)[0] + "(copy)" + os.path.splitext(filename)[1]}"') - return os.path.splitext(new_folder)[0] + "(copy)" + os.path.splitext(new_folder)[1] - return new_folder - - -def sys_pause(): - if sys.platform == "win32": - os.system("pause") - - -def help_message(): - return "Usage: ffmpeg-comp {folder}" diff --git a/FFMpeg-Compressor/printer.py b/FFMpeg-Compressor/printer.py new file mode 100644 index 0000000..6080caf --- /dev/null +++ b/FFMpeg-Compressor/printer.py @@ -0,0 +1,46 @@ +from progress.bar import IncrementalBar +import colorama +import sys +import os + + +class Printer: + + def __init__(self, folder): + file_count = 0 + for folder, folders, file in os.walk(folder): + file_count += len(file) + self.bar = IncrementalBar('Compressing', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)') + self.bar.update() + + # Fill whole string with spaces for cleaning progress bar + @staticmethod + def clean_str(string): + return string + " " * (os.get_terminal_size().columns - len(string)) + + @staticmethod + def win_ascii_esc(): + if sys.platform == "win32": + colorama.init() + + def bar_print(self, string): + print(string) + self.bar.update() + + def info(self, string): + self.bar_print(self.clean_str(f"\r\033[100m- {string}\033[49m")) + + def warning(self, string): + self.bar_print(self.clean_str(f"\r\033[93m!\033[0m {string}\033[49m")) + + def error(self, string): + self.bar_print(self.clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m")) + + def files(self, source, dest, dest_ext, comment): + source_ext = os.path.splitext(source)[1] + source_name = os.path.splitext(source)[0] + + self.bar_print(self.clean_str(f"\r\033[0;32m\u2713\033[0m \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m")) + + def unknown_file(self, file): + self.bar_print(self.clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)")) diff --git a/FFMpeg-Compressor/utils.py b/FFMpeg-Compressor/utils.py new file mode 100644 index 0000000..48df9e9 --- /dev/null +++ b/FFMpeg-Compressor/utils.py @@ -0,0 +1,71 @@ +from shutil import copyfile +import sys +import os + +class Utils: + + def __init__(self, config, printer): + self.errors = 0 + self.config = config + self.printer = printer + + @staticmethod + def sys_pause(): + if sys.platform == "win32": + os.system("pause") + + @staticmethod + def get_size(directory): + total_size = 0 + for folder, folders, files in os.walk(directory): + for file in files: + if not os.path.islink(f"{folder}/{file}"): + total_size += os.path.getsize(f"{folder}/{file}") + return total_size + + def get_compression(self, source, output): + try: + source = self.get_size(source) + output = self.get_size(output) + + print(f"\nResult: {source/1024/1024:.2f}MB -> " + f"{output/1024/1024:.2f}MB ({(output - source)/1024/1024:.2f}MB)") + except ZeroDivisionError: + self.printer.warning("Nothing compressed!") + + def get_compression_status(self, orig_folder): + orig_folder_len = 0 + comp_folder_len = 0 + + for folder, folders, files in os.walk(orig_folder): + orig_folder_len += len(files) + + for folder, folders, files in os.walk(f'{orig_folder}_compressed'): + for file in files: + if not os.path.splitext(file)[1].count("(copy)"): + comp_folder_len += 1 + + if self.errors != 0: + self.printer.warning("Some files failed to compress!") + + if orig_folder_len == comp_folder_len: + self.printer.info("Success!") + self.get_compression(orig_folder, f"{orig_folder}_compressed") + else: + self.printer.warning("Original and compressed folders are not identical!") + self.get_compression(orig_folder, f"{orig_folder}_compressed") + + def add_unprocessed_file(self, orig_folder, new_folder): + if self.config['FFMPEG']['CopyUnprocessed']: + filename = orig_folder.split("/").pop() + copyfile(orig_folder, new_folder) + self.printer.info(f"File {filename} copied to compressed folder.") + + def check_duplicates(self, new_folder): + filename = new_folder.split().pop() + if os.path.exists(new_folder): + self.printer.warning( + f'Duplicate file has been found! Check manually this files - "{filename}", ' + f'"{os.path.splitext(filename)[0] + "(copy)" + os.path.splitext(filename)[1]}"') + return os.path.splitext(new_folder)[0] + "(copy)" + os.path.splitext(new_folder)[1] + return new_folder diff --git a/README.md b/README.md index 36f092f..370d3eb 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,25 @@ ## VNTools -Collection of tools used by administrators from VN Telegram Channel +Collection of tools used by VienDesu! Porting Team ### Tools -* `FFMpeg-Compressor` - Python utility uses ffmpeg to compress Visual Novel Resources -* `RenPy-Android-Unpack` - A simple Python script for unpacking Ren'Py based .apk and .obb files to ready to use Ren'Py SDK's Project +* `FFMpeg-Compressor` - Python utility to compress Visual Novel Resources +* `RenPy-Android-Unpack` - A Python script for extracting game project from Ren'Py based .apk and .obb files * `RenPy-Unpacker` - Simple .rpy script that will make any RenPy game unpack itself -* `VNDS-to-RenPy` - Simple script for converting vnds scripts to rpy +* `VNDS-to-RenPy` - Simple script for converting VNDS engine scripts to .rpy ones + +### Installation +#### Download from releases: + * Windows - `TODO` + * Linux - `TODO` + * MacOS - `TODO` + +#### Build tools as binaries: + * Run `./build.sh` on UNIX + * Run `.\build.bat` for Windows + * Arch Linux - `TODO` + * NixOS - `TODO` + +#### Install as python package: + * Run `pip install -U .` command in project folder + * Arch Linux - `TODO` + * NixOS - `TODO` diff --git a/RenPy-Android-Unpack/__init__.py b/RenPy-Android-Unpack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/RenPy-Android-Unpack/__main__.py b/RenPy-Android-Unpack/__main__.py new file mode 100755 index 0000000..8826079 --- /dev/null +++ b/RenPy-Android-Unpack/__main__.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +import colorama +import argparse +import sys +import os + +from printer import Printer +from actions import Actions + + +def args_init(): + parser = argparse.ArgumentParser( + prog='rendroid-unpack', + description='Extract Ren\'Py .apk and .obb files into Ren\'Py SDK\'s project' + ) + parser.add_argument('path') + parser.add_argument('-o', '--output') + return parser.parse_args() + + +if __name__ == '__main__': + if sys.platform == "win32": + colorama.init() + args = args_init() + if args.output: + output = args.output + else: + output = '' + actions = Actions(output) + printer = Printer() + + filename = args.path + if os.path.splitext(filename)[1] == '.apk' or os.path.splitext(filename)[1] == '.obb': + actions.clean(['assets'], True) + + printer.info(f'Extracting assets from {filename}... ') + actions.extract().assets(filename) + + printer.info('Renaming game assets... ') + actions.rename().files('assets') + actions.rename().dirs('assets') + + printer.info('Removing unneeded files... ') + if os.path.splitext(filename)[1] == '.apk': + actions.clean(['assets/renpy', 'assets/res'], False) + actions.clean(['assets/dexopt'], True) + + printer.info('Renaming directory... ') + actions.clean([os.path.splitext(filename)[0]], True) + os.rename(os.path.join(output, 'assets'), os.path.splitext(filename)[0]) + else: + Printer.err("It's not an .apk or .obb file!") diff --git a/RenPy-Android-Unpack/actions.py b/RenPy-Android-Unpack/actions.py new file mode 100755 index 0000000..e5d42e6 --- /dev/null +++ b/RenPy-Android-Unpack/actions.py @@ -0,0 +1,98 @@ +from zipfile import ZipFile, BadZipFile +from PIL import Image +import shutil +import os + +from printer import Printer + + +class Extract: + + def __init__(self, output: str): + self.output = output + + @staticmethod + def folder(zip_ref: ZipFile, path: str, dest: str): + for content in zip_ref.namelist(): + if content.split('/')[0] == path: + zip_ref.extract(content, dest) + + @staticmethod + def icon(directory: str): + icons = [] + for folder, folders, files in os.walk(directory): + for file in os.listdir(folder): + if os.path.splitext(file)[1] == ".png": + image = Image.open(f"{folder}/{file}") + if image.size[0] == 432 and image.size[1] == 432: + icons.append(f"{folder}/{file}") + if len(icons) == 0: + raise KeyError + return icons + + def assets(self, file: str): + try: + with ZipFile(file, 'r') as zip_ref: + self.folder(zip_ref, 'assets', self.output) + if os.path.splitext(file)[1] == '.apk': + try: + # ~Ren'Py 8, 7 + self.folder(zip_ref, 'res', os.path.join(self.output, 'assets')) + for icon in self.icon(os.path.join(self.output, 'assets/res')): + os.rename(icon, os.path.join(self.output, "assets", os.path.split(icon)[1])) + except KeyError: + try: + # ~Ren'Py 6 + zip_ref.extract('res/drawable/icon.png', os.path.join(self.output, 'assets')) + os.rename(os.path.join(self.output, 'assets/res/drawable/icon.png'), + os.path.join(self.output, 'assets/icon.png')) + except KeyError: + Printer.warn("Icon not found. Maybe it is not supported apk?") + except BadZipFile: + Printer.err("Cant extract .apk file!") + + +class Rename: + + def __init__(self, output): + self.output = output + + def files(self, directory: str): + for dir_ in os.walk(os.path.join(self.output, directory)): + for file in dir_[2]: + path = f'{dir_[0]}/{file}' + folder = '/'.join(path.split('/')[:len(path.split('/')) - 1]) + newname = f'{path.split("/").pop().replace("x-", "")}' + os.rename(path, f'{folder}/{newname}') + + def dirs(self, directory: str): + dirs = [] + for dir_ in os.walk(os.path.join(self.output, directory)): + dirs.append(dir_[0]) + dirs.reverse() + dirs.pop() + for dir__ in dirs: + folder = '/'.join(dir__.split('/')[:len(dir__.split('/')) - 1]) + newname = f'{dir__.split("/").pop().replace("x-", "")}' + os.rename(dir__, f'{folder}/{newname}') + + +class Actions: + + def __init__(self, output: str): + self.output = output + + def extract(self) -> Extract: + return Extract(self.output) + + def rename(self) -> Rename: + return Rename(self.output) + + def clean(self, names: list, ignore: bool): + for name in names: + name = os.path.join(self.output, name) + try: + shutil.rmtree(name) + except FileNotFoundError: + if not ignore: + Printer.warn(f"Path {name} not found!") diff --git a/RenPy-Android-Unpack/printer.py b/RenPy-Android-Unpack/printer.py new file mode 100644 index 0000000..2c3d31b --- /dev/null +++ b/RenPy-Android-Unpack/printer.py @@ -0,0 +1,14 @@ +class Printer: + + @staticmethod + def info(msg: str): + print(f"\033[100m[INFO] {msg}\033[49m") + + @staticmethod + def warn(msg: str): + print(f"\033[93m[WARN]\033[0m {msg}\033[49m") + + @staticmethod + def err(msg: str): + print(f"\033[31m[ERROR]\033[0m {msg} Exiting...\033[49m") + exit() diff --git a/RenPy-Android-Unpack/requirements.txt b/RenPy-Android-Unpack/requirements.txt deleted file mode 100644 index 3a6b560..0000000 --- a/RenPy-Android-Unpack/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -Pillow==9.5.0 -colorama==0.4.6 \ No newline at end of file diff --git a/RenPy-Android-Unpack/unpack.py b/RenPy-Android-Unpack/unpack.py deleted file mode 100755 index 8b74db7..0000000 --- a/RenPy-Android-Unpack/unpack.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3 -from PIL import Image -import colorama -import zipfile -import shutil -import os -import sys - - -def printer(msg, level): - match level: - case "info": - print(f"\033[100m[INFO] {msg}\033[49m") - case "warn": - print(f"\033[93m[WARN]\033[0m {msg}\033[49m") - case "err": - print(f"\033[31m[ERROR]\033[0m {msg} Exiting...\033[49m") - exit() - - -def extract_folder(zip_ref, path, dest): - for content in zip_ref.namelist(): - if content.split('/')[0] == path: - zip_ref.extract(content, dest) - - -def find_modern_icon(directory): - icons = [] - for folder, folders, files in os.walk(directory): - for file in os.listdir(folder): - if os.path.splitext(file)[1] == ".png": - image = Image.open(f"{folder}/{file}") - if image.size[0] == 432 and image.size[1] == 432: - icons.append(f"{folder}/{file}") - if len(icons) == 0: - raise KeyError - return icons - - -def extract_assets(file): - try: - with zipfile.ZipFile(file, 'r') as zip_ref: - extract_folder(zip_ref, 'assets', '') - if os.path.splitext(file)[1] == '.apk': - try: - # ~Ren'Py 8, 7 - extract_folder(zip_ref, 'res', 'assets') - for icon in find_modern_icon('assets/res'): - os.rename(icon, f"assets/{os.path.split(icon)[1]}") - except KeyError: - try: - # ~Ren'Py 6 - zip_ref.extract('res/drawable/icon.png', 'assets') - os.rename('assets/res/drawable/icon.png', 'assets/icon.png') - except KeyError: - printer("Icon not found. Maybe it is not supported apk?", "warn") - except zipfile.BadZipFile: - return printer("Cant extract .apk file!", "err") - - -def rename_files(directory): - for dir_ in os.walk(directory): - for file in dir_[2]: - path = f'{dir_[0]}/{file}' - folder = '/'.join(path.split('/')[:len(path.split('/')) - 1]) - newname = f'{path.split("/").pop().replace("x-", "")}' - os.rename(path, f'{folder}/{newname}') - - -def rename_dirs(directory): - dirs = [] - for dir_ in os.walk(directory): - dirs.append(dir_[0]) - dirs.reverse() - dirs.pop() - for dir__ in dirs: - folder = '/'.join(dir__.split('/')[:len(dir__.split('/')) - 1]) - newname = f'{dir__.split("/").pop().replace("x-", "")}' - os.rename(dir__, f'{folder}/{newname}') - - -def remove_unneeded(names, ignore): - for name in names: - try: - shutil.rmtree(name) - except FileNotFoundError: - if not ignore: - printer(f"Path {name} not found!", "warn") - - -if __name__ == '__main__': - if sys.platform == "win32": - colorama.init() - for filename in os.listdir(os.getcwd()): - if os.path.splitext(filename)[1] == '.apk' or os.path.splitext(filename)[1] == '.obb': - remove_unneeded(['assets'], True) - printer(f'Extracting assets from {filename}... ', "info") - extract_assets(filename) - printer('Renaming game assets... ', "info") - rename_files('assets') - rename_dirs('assets') - printer('Removing unneeded files... ', "info") - if os.path.splitext(filename)[1] == '.apk': - remove_unneeded(['assets/renpy', 'assets/res'], False) - remove_unneeded(['assets/dexopt'], True) - printer('Renaming directory... ', "info") - remove_unneeded([os.path.splitext(filename)[0]], True) - os.rename('assets', os.path.splitext(filename)[0]) diff --git a/VNDS-to-RenPy/__init__.py b/VNDS-to-RenPy/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/VNDS-to-RenPy/convert.py b/VNDS-to-RenPy/__main__.py similarity index 100% rename from VNDS-to-RenPy/convert.py rename to VNDS-to-RenPy/__main__.py diff --git a/build.bat b/build.bat old mode 100644 new mode 100755 index 237c7e3..688d43d --- a/build.bat +++ b/build.bat @@ -3,16 +3,16 @@ if not defined VIRTUAL_ENV goto :venv_error mkdir output mkdir output\bin -python -m pip install -r FFMpeg-Compressor\requirements.txt || goto :exit -python -m pip install -r RenPy-Android-Unpack\requirements.txt || goto :exit +python -m pip install -r requirements.txt || goto :exit python -m pip install Nuitka || goto :exit -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=ffmpeg-comp FFMpeg-Compressor\main.py || goto :exit +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=ffmpeg-comp FFMpeg-Compressor\__main__.py || goto :exit xcopy FFMpeg-Compressor\ffmpeg-comp.toml output\bin /Y move /Y output\ffmpeg-comp.exe output\bin -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=rendroid-unpack RenPy-Android-Unpack\unpack.py || goto :exit +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=rendroid-unpack RenPy-Android-Unpack\__main__.py || goto :exit move /Y output\rendroid-unpack.exe output\bin -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=vnds2renpy VNDS-to-RenPy/convert.py || goto :exit +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=vnds2renpy VNDS-to-RenPy/__main__.py || goto :exit move /Y output\vnds2renpy.exe output\bin +echo "Done! You can get binaries into output\bin directory" :venv_error echo "Please create and activate venv before running this script: python -m venv .\venv && .\venv\Scripts\activate.bat" diff --git a/build.sh b/build.sh index 7e21bbf..90e774e 100755 --- a/build.sh +++ b/build.sh @@ -8,17 +8,17 @@ fi mkdir -p output mkdir -p output/bin -python3 -m pip install -r FFMpeg-Compressor/requirements.txt -python3 -m pip install -r RenPy-Android-Unpack/requirements.txt +python3 -m pip install -r requirements.txt python3 -m pip install Nuitka case "$(uname -s)" in Linux*) jobs="--jobs=$(nproc)";; Darwin*) jobs="--jobs=$(sysctl -n hw.ncpu)";; esac -python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=ffmpeg-comp FFMpeg-Compressor/main.py +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=ffmpeg-comp FFMpeg-Compressor/__main__.py cp FFMpeg-Compressor/ffmpeg-comp.toml output/bin mv output/ffmpeg-comp output/bin -python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=rendroid-unpack RenPy-Android-Unpack/unpack.py +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=rendroid-unpack RenPy-Android-Unpack/__main__.py mv output/rendroid-unpack output/bin -python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=vnds2renpy VNDS-to-RenPy/convert.py -mv output/vnds2renpy output/bin \ No newline at end of file +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=vnds2renpy VNDS-to-RenPy/__main__.py +mv output/vnds2renpy output/bin +echo "Done! You can get binaries into output/bin directory" \ No newline at end of file diff --git a/FFMpeg-Compressor/requirements.txt b/requirements.txt similarity index 100% rename from FFMpeg-Compressor/requirements.txt rename to requirements.txt