From 657aa3114884ab8f44e2a6f54962623e337e9f11 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 30 Sep 2023 23:51:53 +0300 Subject: [PATCH 001/105] FFMpeg-Compressor: Rewrite some code --- FFMpeg-Compressor/ffmpeg-comp.toml | 8 +- FFMpeg-Compressor/main.py | 14 ++- FFMpeg-Compressor/modules/compressor.py | 112 ++++++++++++---------- FFMpeg-Compressor/modules/configloader.py | 11 +++ 4 files changed, 92 insertions(+), 53 deletions(-) create mode 100644 FFMpeg-Compressor/modules/configloader.py diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index 9f117bb..800584d 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -2,14 +2,14 @@ FFmpegParams = "-hide_banner -loglevel error" [AUDIO] -Extension = "mp3" +Extension = "original" BitRate = "320k" [IMAGE] -Extension = "jpg" -CompLevel = 100 +Extension = "original" +CompLevel = 20 JpegComp = 3 [VIDEO] -Extension = "webm" +Extension = "original" Codec = "libvpx-vp9" diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py index 3a14022..c410721 100755 --- a/FFMpeg-Compressor/main.py +++ b/FFMpeg-Compressor/main.py @@ -23,11 +23,23 @@ printer.bar_init(orig_folder) if os.path.exists(f"{orig_folder}_compressed"): shutil.rmtree(f"{orig_folder}_compressed") + printer.info("Creating folders...") for folder, folders, files in os.walk(orig_folder): if not os.path.exists(folder.replace(orig_folder, f"{orig_folder}_compressed")): os.mkdir(folder.replace(orig_folder, f"{orig_folder}_compressed")) printer.info(f"Compressing \"{folder.replace(orig_folder, orig_folder.split('/').pop())}\" folder...") - compressor.compress(orig_folder, folder) + target_folder = folder.replace(orig_folder, f"{orig_folder}_compressed") + for file in os.listdir(folder): + if os.path.isfile(f'{folder}/{file}'): + match compressor.get_file_type(file): + case "audio": + compressor.compress_audio(folder, file, target_folder) + case "image": + compressor.compress_image(folder, file, target_folder) + case "video": + compressor.compress_video(folder, file, target_folder) + case "unknown": + compressor.compress(folder, file, target_folder) utils.get_compression_status(orig_folder) diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 5f3d959..1022d97 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -1,25 +1,39 @@ from modules import printer +from modules import configloader from PIL import Image -import tomllib import os -audio_exts = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] -image_exts = ['.apng', '.avif', '.jfif', '.pjpeg', '.pjp', '.svg', '.webp', '.jpg', '.jpeg', '.png', '.raw'] -video_exts = ['.3gp' '.amv', '.avi', '.gif', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', '.webm', '.ogv'] -try: - config = tomllib.load(open("ffmpeg-comp.toml", "rb")) -except FileNotFoundError: - try: - config = tomllib.load(open("/etc/ffmpeg-comp.toml", "rb")) - except FileNotFoundError: - printer.error("Config file not found. Please put it next to binary or in to /etc folder.") - exit() +def get_req_ext(file): + if configloader.config['AUDIO']['Extension'] == "original" or \ + configloader.config['IMAGE']['Extension'] == "original" or \ + configloader.config['VIDEO']['Extension'] == "original": + return os.path.splitext(file)[1][1:] -ffmpeg_params = config['FFMPEG']['FFmpegParams'] -req_audio_ext = config['AUDIO']['Extension'] -req_image_ext = config['IMAGE']['Extension'] -req_video_ext = config['VIDEO']['Extension'] + match get_file_type(file): + case "audio": + return configloader.config['AUDIO']['Extension'] + case "image": + return configloader.config['IMAGE']['Extension'] + case "video": + return configloader.config['VIDEO']['Extension'] + + +def get_file_type(file): + audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] + image_ext = ['.apng', '.avif', '.jfif', '.pjpeg', '.pjp', '.svg', '.webp', '.jpg', '.jpeg', '.png', '.raw'] + video_ext = ['.3gp' '.amv', '.avi', '.gif', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', '.webm', + '.ogv'] + file_extension = os.path.splitext(file)[1] + + if file_extension in audio_ext: + return "audio" + elif file_extension in image_ext: + return "image" + elif file_extension in video_ext: + return "video" + else: + return "unknown" def has_transparency(img): @@ -38,43 +52,45 @@ def has_transparency(img): return False -def compress(root_folder, folder): - target_folder = folder.replace(root_folder, f"{root_folder}_compressed") - for file in os.listdir(folder): - if os.path.isfile(f'{folder}/{file}'): - if os.path.splitext(file)[1] in audio_exts: +def compress_audio(folder, file, target_folder): + ffmpeg_params = configloader.config['FFMPEG']['FFmpegParams'] + bitrate = configloader.config['AUDIO']['BitRate'] - bitrate = config['AUDIO']['BitRate'] - printer.files(file, os.path.splitext(file)[0], req_audio_ext, f"{bitrate}bit/s") - os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} " - f"'{target_folder}/{os.path.splitext(file)[0]}.{req_audio_ext}'") + printer.files(file, os.path.splitext(file)[0], get_req_ext(file), f"{bitrate}") + os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -q:a {bitrate} " + f"'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}'") - elif os.path.splitext(file)[1] in image_exts: - if req_image_ext == "jpg" or req_image_ext == "jpeg": +def compress_video(folder, file, target_folder): + ffmpeg_params = configloader.config['FFMPEG']['FFmpegParams'] + codec = configloader.config['VIDEO']['Codec'] - if not has_transparency(Image.open(f'{folder}/{file}')): - jpg_comp = config['IMAGE']['JpegComp'] - printer.files(file, os.path.splitext(file)[0], req_image_ext, f"level {jpg_comp}") - os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -q {jpg_comp} " - f"'{target_folder}/{os.path.splitext(file)[0]}.{req_image_ext}'") + printer.files(file, os.path.splitext(file)[0], get_req_ext(file), codec) + os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -vcodec {codec} " + f"'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}'") - else: - printer.warning(f"{file} has transparency (.jpg not support it). Skipping...") - else: - comp_level = config['IMAGE']['CompLevel'] - printer.files(file, os.path.splitext(file)[0], req_image_ext, f"{comp_level}%") - os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -compression_level {comp_level} " - f"'{target_folder}/{os.path.splitext(file)[0]}.{req_image_ext}'") +def compress_image(folder, file, target_folder): + ffmpeg_params = configloader.config['FFMPEG']['FFmpegParams'] + comp_level = configloader.config['IMAGE']['CompLevel'] + jpg_comp = configloader.config['IMAGE']['JpegComp'] - elif os.path.splitext(file)[1] in video_exts: - codec = config['VIDEO']['Codec'] - printer.files(file, os.path.splitext(file)[0], req_video_ext, codec) - os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -vcodec {codec} " - f"'{target_folder}/{os.path.splitext(file)[0]}.{req_video_ext}'") + if get_req_ext(file) == "jpg" or get_req_ext(file) == "jpeg": - else: - printer.warning("File extension not recognized. This may affect the quality of the compression.") - printer.unknown_file(file) - os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} '{target_folder}/{file}'") + if not has_transparency(Image.open(f'{folder}/{file}')): + printer.files(file, os.path.splitext(file)[0], get_req_ext(file), f"level {jpg_comp}") + os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -q {jpg_comp} " + f"'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}'") + else: + printer.warning(f"{file} has transparency (.jpg not support it). Skipping...") + else: + printer.files(file, os.path.splitext(file)[0], get_req_ext(file), f"{comp_level}%") + os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -compression_level {comp_level} " + f"'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}'") + + +def compress(folder, file, target_folder): + ffmpeg_params = configloader.config['FFMPEG']['FFmpegParams'] + printer.warning("File extension not recognized. This may affect the quality of the compression.") + printer.unknown_file(file) + os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} '{target_folder}/{file}'") diff --git a/FFMpeg-Compressor/modules/configloader.py b/FFMpeg-Compressor/modules/configloader.py new file mode 100644 index 0000000..1a4db0b --- /dev/null +++ b/FFMpeg-Compressor/modules/configloader.py @@ -0,0 +1,11 @@ +import tomllib +from modules import printer + +try: + config = tomllib.load(open("ffmpeg-comp.toml", "rb")) +except FileNotFoundError: + try: + config = tomllib.load(open("/etc/ffmpeg-comp.toml", "rb")) + except FileNotFoundError: + printer.error("Config file not found. Please put it next to binary or in to /etc folder.") + exit() From d57585f5a261d8c48a374bbb416f1d2b9c331fa7 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sun, 1 Oct 2023 22:45:38 +0300 Subject: [PATCH 002/105] FFMpeg-Compressor: Implement mimic mode --- FFMpeg-Compressor/ffmpeg-comp.toml | 7 ++++--- FFMpeg-Compressor/main.py | 11 +++++++---- FFMpeg-Compressor/modules/compressor.py | 9 ++++----- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index 800584d..2154d35 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -1,15 +1,16 @@ [FFMPEG] FFmpegParams = "-hide_banner -loglevel error" +MimicMode = true [AUDIO] -Extension = "original" +Extension = "mp3" BitRate = "320k" [IMAGE] -Extension = "original" +Extension = "png" CompLevel = 20 JpegComp = 3 [VIDEO] -Extension = "original" +Extension = "mp4" Codec = "libvpx-vp9" diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py index c410721..790adee 100755 --- a/FFMpeg-Compressor/main.py +++ b/FFMpeg-Compressor/main.py @@ -1,5 +1,6 @@ #!/bin/python3 +from modules import configloader from modules import compressor from modules import printer from modules import utils @@ -35,11 +36,13 @@ for folder, folders, files in os.walk(orig_folder): if os.path.isfile(f'{folder}/{file}'): match compressor.get_file_type(file): case "audio": - compressor.compress_audio(folder, file, target_folder) + comp_file = compressor.compress_audio(folder, file, target_folder) case "image": - compressor.compress_image(folder, file, target_folder) + comp_file = compressor.compress_image(folder, file, target_folder) case "video": - compressor.compress_video(folder, file, target_folder) + comp_file = compressor.compress_video(folder, file, target_folder) case "unknown": - compressor.compress(folder, file, target_folder) + comp_file = compressor.compress(folder, file, target_folder) + if configloader.config['FFMPEG']['MimicMode']: + os.rename(comp_file, f'{folder}_compressed/{file}') utils.get_compression_status(orig_folder) diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 1022d97..3455988 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -5,11 +5,6 @@ import os def get_req_ext(file): - if configloader.config['AUDIO']['Extension'] == "original" or \ - configloader.config['IMAGE']['Extension'] == "original" or \ - configloader.config['VIDEO']['Extension'] == "original": - return os.path.splitext(file)[1][1:] - match get_file_type(file): case "audio": return configloader.config['AUDIO']['Extension'] @@ -59,6 +54,7 @@ def compress_audio(folder, file, target_folder): printer.files(file, os.path.splitext(file)[0], get_req_ext(file), f"{bitrate}") os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -q:a {bitrate} " f"'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}'") + return f'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}' def compress_video(folder, file, target_folder): @@ -68,6 +64,7 @@ def compress_video(folder, file, target_folder): printer.files(file, os.path.splitext(file)[0], get_req_ext(file), codec) os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -vcodec {codec} " f"'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}'") + return f'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}' def compress_image(folder, file, target_folder): @@ -87,6 +84,7 @@ def compress_image(folder, file, target_folder): printer.files(file, os.path.splitext(file)[0], get_req_ext(file), f"{comp_level}%") os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -compression_level {comp_level} " f"'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}'") + return f'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}' def compress(folder, file, target_folder): @@ -94,3 +92,4 @@ def compress(folder, file, target_folder): printer.warning("File extension not recognized. This may affect the quality of the compression.") printer.unknown_file(file) os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} '{target_folder}/{file}'") + return f'{target_folder}/{file}' From 4b76a600fb659c5a85d5b097a279db6fb4c7b3ca Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sun, 1 Oct 2023 22:56:28 +0300 Subject: [PATCH 003/105] Add installing Nuitka in build script --- build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sh b/build.sh index 3eec12d..475e1b2 100755 --- a/build.sh +++ b/build.sh @@ -1,6 +1,7 @@ #!/bin/bash mkdir output mkdir output/bin +python3 -m pip install Nuitka nuitka3 --jobs=$(nproc) --output-dir=output --follow-imports --output-filename=output/bin/ffmpeg-comp FFMpeg-Compressor/main.py cp FFMpeg-Compressor/ffmpeg-comp.toml output/bin/ nuitka3 --jobs=$(nproc) --output-dir=output --follow-imports --output-filename=output/bin/rendroid-unpack RenPy-Android-Unpack/unpack.py \ No newline at end of file From 06116b5de14c3fe974808a3d495dd8bb90b61f1f Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sun, 1 Oct 2023 23:29:34 +0300 Subject: [PATCH 004/105] FFMpeg-Compressor: Add Configuration section in README.md --- FFMpeg-Compressor/README.md | 18 ++++++++++++++++++ FFMpeg-Compressor/ffmpeg-comp.toml | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index f30ceed..e3e2ae5 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -6,3 +6,21 @@ Python utility uses ffmpeg to compress Visual Novel Resources * Change the configuration of the utility in `ffmpeg-comp.toml` for yourself * `ffmpeg-comp {folder}` * In result you get `{folder-compressed}` near with original `{folder}` + +### Configuration +#### FFMPEG section +* FFMpegParams - Some parameters & flags for ffmpeg command line interface (default: `"-hide_banner -loglevel error"`) +* MimicMode - Rename compressed file to it original name and extension. VN engines determine the file type by its header, so for example PNG file named file.jpg will be loaded as PNG file. (default: `false`) + +#### AUDIO section +* Extension - Required audio file extension. It supports: `.aac`, `.flac`, `.m4a`, `.mp3`, `.ogg`, `.opus`, `.raw`, `.wav`, `.wma`. +* BitRate - (mp3 only, for now) Required audio bitrate. For best quality use `320k` value, but for worse use `1-9` (9 worst) number range. + +#### IMAGE section +* Extension - Required image file extension. It supports: `.apng`, `.avif`, `.jfif`, `.pjpeg`, `.pjp`, `.svg`, `.webp`, `.jpg/.jpeg`, `.png`, `.raw` +* CompLevel - Compression level for images. Values range: `0-100` (100 - max compression, 0 - min compression) +* JpegComp - (May be deleted in future) Compression level specific for jpeg images. Values range: `0-10` (10 - max compression, 0 - min compression) + +#### VIDEO section +* Extension - Required image file extension. It supports: `.3gp`, `.amv`, `.avi`, `.gif`, `.m4v`, `.mkv`, `.mov`, `.mp4`, `.m4v`, `.mpeg`, `.mpv`, `.webm`, `.ogv` +* Codec - (May be optional in future) Required video codec. (See official ffmpeg documentation for supported codecs) \ No newline at end of file diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index 2154d35..a8ee625 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -1,6 +1,6 @@ [FFMPEG] FFmpegParams = "-hide_banner -loglevel error" -MimicMode = true +MimicMode = false [AUDIO] Extension = "mp3" From 875df526ad8aa464f418d9a41b317b1ab5b5a89c Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sun, 1 Oct 2023 23:36:23 +0300 Subject: [PATCH 005/105] FFMpeg-Compressor: TODO in README.md --- FFMpeg-Compressor/README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index e3e2ae5..b2dc130 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -23,4 +23,10 @@ Python utility uses ffmpeg to compress Visual Novel Resources #### VIDEO section * Extension - Required image file extension. It supports: `.3gp`, `.amv`, `.avi`, `.gif`, `.m4v`, `.mkv`, `.mov`, `.mp4`, `.m4v`, `.mpeg`, `.mpv`, `.webm`, `.ogv` -* Codec - (May be optional in future) Required video codec. (See official ffmpeg documentation for supported codecs) \ No newline at end of file +* Codec - (May be optional in future) Required video codec. (See official ffmpeg documentation for supported codecs) + +### TODO (for testing branch) +* [ ] Recreate whole game directory with compressed files +* [ ] Cross platform (Easy Windows usage and binaries, MacOS binaries) +* [ ] Use ffmpeg python bindings instead of cli commands +* [ ] Reorganize code \ No newline at end of file From 44c2b1a44b74e928bc4cbd065dd9bcb55f560d93 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sun, 1 Oct 2023 23:41:27 +0300 Subject: [PATCH 006/105] RenPy-Android-Unpack: Update README.md --- RenPy-Android-Unpack/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/RenPy-Android-Unpack/README.md b/RenPy-Android-Unpack/README.md index 95beb69..f6992e1 100644 --- a/RenPy-Android-Unpack/README.md +++ b/RenPy-Android-Unpack/README.md @@ -1,6 +1,6 @@ ## RenPy-Android-Unpack -A simple Python script for unpacking Ren'Py based .apk files for later rebuilding in the Ren'Py SDK +A simple Python script for unpacking Ren'Py based .apk and .obb files to ready to use Ren'Py SDK's Project ### How to use -* Put some .apk files in folder -* `python3 unpack.py` +* Put some .apk & .obb files in folder +* `rendroid-unpack` (It unpacks all .apk and .obb files in the directory where it is located) From 73fd5e44d1144e703423f6d09cbdd57244dc0725 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sun, 1 Oct 2023 23:45:43 +0300 Subject: [PATCH 007/105] RenPy-Unpacker: Update README.md --- RenPy-Unpacker/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/RenPy-Unpacker/README.md b/RenPy-Unpacker/README.md index 41a3c75..42a021f 100644 --- a/RenPy-Unpacker/README.md +++ b/RenPy-Unpacker/README.md @@ -2,7 +2,9 @@ Simple .rpy script that will make any RenPy game unpack itself ### How to use -* Put .rpyc from releases page to game's `game` folder -* Open your game and wait until it not be launched -* Unpacked assets will be in `unpack` folder near with game's executable +* Put .rpyc file from releases page to `game` folder +* Open your game and wait until it not be fully loaded +* Extracted assets will be in `unpack` folder near with game's executable * Enjoy! + +It can help with getting assets from encrypted .rpa files with custom encryption. From e4de88142480464c7639f9f5f18e183f78b32e9d Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Mon, 2 Oct 2023 17:39:19 +0300 Subject: [PATCH 008/105] FFMpeg-Compressor: Skip all ffmpeg questions by default --- FFMpeg-Compressor/README.md | 2 +- FFMpeg-Compressor/ffmpeg-comp.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index b2dc130..b3f8024 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -9,7 +9,7 @@ Python utility uses ffmpeg to compress Visual Novel Resources ### Configuration #### FFMPEG section -* FFMpegParams - Some parameters & flags for ffmpeg command line interface (default: `"-hide_banner -loglevel error"`) +* FFMpegParams - Some parameters & flags for ffmpeg command line interface (default: `"-n -hide_banner -loglevel error"`) * MimicMode - Rename compressed file to it original name and extension. VN engines determine the file type by its header, so for example PNG file named file.jpg will be loaded as PNG file. (default: `false`) #### AUDIO section diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index a8ee625..802fb3b 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -1,5 +1,5 @@ [FFMPEG] -FFmpegParams = "-hide_banner -loglevel error" +FFmpegParams = "-n -hide_banner -loglevel error" MimicMode = false [AUDIO] From cefd0bc9babfdb7bcfbb3b353b872e3c3fe16fc3 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Mon, 2 Oct 2023 19:23:11 +0300 Subject: [PATCH 009/105] FFMpeg-Compressor: Copy all unprocessed files to destination folder --- FFMpeg-Compressor/ffmpeg-comp.toml | 3 ++- FFMpeg-Compressor/main.py | 13 ++++++++++-- FFMpeg-Compressor/modules/utils.py | 32 ++++++++++++++++++++++++------ 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index 802fb3b..43199df 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -1,5 +1,6 @@ [FFMPEG] FFmpegParams = "-n -hide_banner -loglevel error" +CopyUnprocessed = false MimicMode = false [AUDIO] @@ -7,7 +8,7 @@ Extension = "mp3" BitRate = "320k" [IMAGE] -Extension = "png" +Extension = "jpg" CompLevel = 20 JpegComp = 3 diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py index 790adee..bd7314a 100755 --- a/FFMpeg-Compressor/main.py +++ b/FFMpeg-Compressor/main.py @@ -1,4 +1,4 @@ -#!/bin/python3 +#!python3 from modules import configloader from modules import compressor @@ -43,6 +43,15 @@ for folder, folders, files in os.walk(orig_folder): comp_file = compressor.compress_video(folder, file, target_folder) case "unknown": comp_file = compressor.compress(folder, file, target_folder) + if configloader.config['FFMPEG']['MimicMode']: - os.rename(comp_file, f'{folder}_compressed/{file}') + try: + os.rename(comp_file, f'{folder}_compressed/{file}') + except FileNotFoundError: + printer.error(f"File {file} can't be processed! Maybe it is ffmpeg error or unsupported file. " + f"You can change -loglevel in ffmpeg parameters to see full error.") + +if configloader.config['FFMPEG']['CopyUnprocessed']: + printer.info("Copying unprocessed files...") + utils.add_unprocessed_files(orig_folder) utils.get_compression_status(orig_folder) diff --git a/FFMpeg-Compressor/modules/utils.py b/FFMpeg-Compressor/modules/utils.py index 66f3b80..3eb4d58 100644 --- a/FFMpeg-Compressor/modules/utils.py +++ b/FFMpeg-Compressor/modules/utils.py @@ -1,4 +1,6 @@ from modules import printer +from shutil import copyfile +from glob import glob import os @@ -32,11 +34,13 @@ def get_compression_status(orig_folder): orig_folder_len = 0 comp_folder_len = 0 - for folder, folders, file in os.walk(orig_folder): - orig_folder_len += len(file) + for folder, folders, files in os.walk(orig_folder): + orig_folder_len += len(files) - for folder, folders, file in os.walk(f'{orig_folder}_compressed'): - comp_folder_len += len(file) + for folder, folders, files in os.walk(f'{orig_folder}_compressed'): + for file in files: + if not os.path.splitext(file)[1].count(" (copy)"): + comp_folder_len += 1 if orig_folder_len == comp_folder_len: printer.info("Success!") @@ -46,6 +50,22 @@ def get_compression_status(orig_folder): get_compression(orig_folder, f"{orig_folder}_compressed") +def add_unprocessed_files(orig_folder): + for folder, folders, files in os.walk(orig_folder): + for file in files: + new_folder = f"{folder}".replace(orig_folder, f"{orig_folder}_compressed") + if len(glob(f"{folder}/{os.path.splitext(file)[0]}*")) != 1: + if len(glob(f"{new_folder}/{file}")): + copyfile(f"{folder}/{file}", f"{new_folder}/{file} (copy)") + printer.warning(f'Duplicate file has been found! Check manually this files - "{file}", "{file} (copy)"') + else: + copyfile(f"{folder}/{file}", f"{new_folder}/{file}") + printer.info(f"File {file} copied to compressed folder.") + else: + if not len(glob(f"{folder}_compressed/{os.path.splitext(file)[0]}*")): + copyfile(f"{folder}/{file}", f"{new_folder}/{file}") + printer.info(f"File {file} copied to compressed folder.") + + def help_message(): - text = "Usage: ffmpeg-comp {folder}" - return text + return "Usage: ffmpeg-comp {folder}" From f15bb3df7e019bcfeb794a1ce7c3fbc1ccb771b6 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Mon, 2 Oct 2023 19:46:12 +0300 Subject: [PATCH 010/105] FFMpeg-Compressor: Hide all ffmpeg warnings and errors --- FFMpeg-Compressor/ffmpeg-comp.toml | 2 +- FFMpeg-Compressor/main.py | 2 ++ FFMpeg-Compressor/modules/compressor.py | 1 - FFMpeg-Compressor/modules/printer.py | 2 +- FFMpeg-Compressor/modules/utils.py | 15 ++++++++++++++- 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index 43199df..2827a2a 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -1,5 +1,5 @@ [FFMPEG] -FFmpegParams = "-n -hide_banner -loglevel error" +FFmpegParams = "-n -hide_banner -loglevel quiet" CopyUnprocessed = false MimicMode = false diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py index bd7314a..f17dae7 100755 --- a/FFMpeg-Compressor/main.py +++ b/FFMpeg-Compressor/main.py @@ -44,6 +44,8 @@ for folder, folders, files in os.walk(orig_folder): case "unknown": comp_file = compressor.compress(folder, file, target_folder) + utils.check_file_existing(folder.replace(orig_folder, f"{orig_folder}_compressed"), file) + if configloader.config['FFMPEG']['MimicMode']: try: os.rename(comp_file, f'{folder}_compressed/{file}') diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 3455988..7c382b8 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -89,7 +89,6 @@ def compress_image(folder, file, target_folder): def compress(folder, file, target_folder): ffmpeg_params = configloader.config['FFMPEG']['FFmpegParams'] - printer.warning("File extension not recognized. This may affect the quality of the compression.") printer.unknown_file(file) os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} '{target_folder}/{file}'") return f'{target_folder}/{file}' diff --git a/FFMpeg-Compressor/modules/printer.py b/FFMpeg-Compressor/modules/printer.py index f735b01..f297a7d 100644 --- a/FFMpeg-Compressor/modules/printer.py +++ b/FFMpeg-Compressor/modules/printer.py @@ -38,5 +38,5 @@ def files(source, dest, dest_ext, comment): def unknown_file(file): - print(clean_str(f"\r[COMP] \033[0;33m{file}\033[0m")) + print(clean_str(f"\r[COMP] \033[0;33m{file}\033[0m (File extension not recognized)")) bar.next() diff --git a/FFMpeg-Compressor/modules/utils.py b/FFMpeg-Compressor/modules/utils.py index 3eb4d58..2828e28 100644 --- a/FFMpeg-Compressor/modules/utils.py +++ b/FFMpeg-Compressor/modules/utils.py @@ -3,6 +3,8 @@ from shutil import copyfile from glob import glob import os +errors_count = 0 + def get_dir_size(directory, files): total_size = 0 @@ -42,11 +44,14 @@ def get_compression_status(orig_folder): if not os.path.splitext(file)[1].count(" (copy)"): comp_folder_len += 1 + if errors_count != 0: + printer.warning("Some files failed to compress!") + if orig_folder_len == comp_folder_len: printer.info("Success!") get_compression(orig_folder, f"{orig_folder}_compressed") else: - printer.warning("Some files failed to compress!") + printer.warning("Original and compressed folders are not identical!") get_compression(orig_folder, f"{orig_folder}_compressed") @@ -67,5 +72,13 @@ def add_unprocessed_files(orig_folder): printer.info(f"File {file} copied to compressed folder.") +def check_file_existing(folder, file): + if not len(glob(f"{folder}/{os.path.splitext(file)[0]}*")): + global errors_count + errors_count += 1 + printer.error(f"{file} not processed. It can be ffmpeg error or file type is unsupported. " + f"You can set '-loglevel error' in ffmpeg config to see full error.") + + def help_message(): return "Usage: ffmpeg-comp {folder}" From e3b21e7d81c4a820dfb57262d46e59b701161bbf Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Mon, 2 Oct 2023 19:48:45 +0300 Subject: [PATCH 011/105] FFMpeg-Compressor: Update README.md --- FFMpeg-Compressor/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index b3f8024..854ec81 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -9,7 +9,8 @@ Python utility uses ffmpeg to compress Visual Novel Resources ### Configuration #### FFMPEG section -* FFMpegParams - Some parameters & flags for ffmpeg command line interface (default: `"-n -hide_banner -loglevel error"`) +* FFMpegParams - Some parameters & flags for ffmpeg command line interface (default: `"-n -hide_banner -loglevel quiet"`) +* CopyUnprocessed - Copy all files that failed to compress by ffmpeg to destination folder. In can helps to recreate original folder, but with compressed files. * MimicMode - Rename compressed file to it original name and extension. VN engines determine the file type by its header, so for example PNG file named file.jpg will be loaded as PNG file. (default: `false`) #### AUDIO section @@ -26,7 +27,7 @@ Python utility uses ffmpeg to compress Visual Novel Resources * Codec - (May be optional in future) Required video codec. (See official ffmpeg documentation for supported codecs) ### TODO (for testing branch) -* [ ] Recreate whole game directory with compressed files +* [x] Recreate whole game directory with compressed files * [ ] Cross platform (Easy Windows usage and binaries, MacOS binaries) * [ ] Use ffmpeg python bindings instead of cli commands * [ ] Reorganize code \ No newline at end of file From c36a403a1c9cfda8286d3e6d06034e2bc2d1e3d8 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Mon, 2 Oct 2023 19:50:36 +0300 Subject: [PATCH 012/105] FFMpeg-Compressor: Add .bmp support --- FFMpeg-Compressor/README.md | 2 +- FFMpeg-Compressor/modules/compressor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index 854ec81..612fd46 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -18,7 +18,7 @@ Python utility uses ffmpeg to compress Visual Novel Resources * BitRate - (mp3 only, for now) Required audio bitrate. For best quality use `320k` value, but for worse use `1-9` (9 worst) number range. #### IMAGE section -* Extension - Required image file extension. It supports: `.apng`, `.avif`, `.jfif`, `.pjpeg`, `.pjp`, `.svg`, `.webp`, `.jpg/.jpeg`, `.png`, `.raw` +* Extension - Required image file extension. It supports: `.apng`, `.avif`, `.bmp`, `.jfif`, `.pjpeg`, `.pjp`, `.svg`, `.webp`, `.jpg/.jpeg`, `.png`, `.raw` * CompLevel - Compression level for images. Values range: `0-100` (100 - max compression, 0 - min compression) * JpegComp - (May be deleted in future) Compression level specific for jpeg images. Values range: `0-10` (10 - max compression, 0 - min compression) diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 7c382b8..53305c3 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -16,7 +16,7 @@ def get_req_ext(file): def get_file_type(file): audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] - image_ext = ['.apng', '.avif', '.jfif', '.pjpeg', '.pjp', '.svg', '.webp', '.jpg', '.jpeg', '.png', '.raw'] + image_ext = ['.apng', '.avif', '.bmp', '.jfif', '.pjpeg', '.pjp', '.svg', '.webp', '.jpg', '.jpeg', '.png', '.raw'] video_ext = ['.3gp' '.amv', '.avi', '.gif', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', '.webm', '.ogv'] file_extension = os.path.splitext(file)[1] From 04cc0159099715786bfcc680859d145bdbad1e55 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Mon, 2 Oct 2023 20:05:11 +0300 Subject: [PATCH 013/105] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 127caae..584d873 100644 --- a/README.md +++ b/README.md @@ -3,5 +3,5 @@ Collection of tools used by administrators from VN Telegram Channel ### Tools * `FFMpeg-Compressor` - Python utility uses ffmpeg to compress Visual Novel Resources -* `RenPy-Android-Unpack` - Simple Python script for unpacking Ren'Py based .apk files for later rebuilding in the Ren'Py SDK +* `RenPy-Android-Unpack` - A simple Python script for unpacking Ren'Py based .apk and .obb files to ready to use Ren'Py SDK's Project * `RenPy-Unpacker` - Simple .rpy script that will make any RenPy game unpack itself From 70e3254395d96fbf579613fafb507d634c57c9fd Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 7 Oct 2023 00:06:51 +0300 Subject: [PATCH 014/105] FFMpeg-Compressor: Fix Mimic mode with subdirs --- FFMpeg-Compressor/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py index f17dae7..3bf6936 100755 --- a/FFMpeg-Compressor/main.py +++ b/FFMpeg-Compressor/main.py @@ -48,7 +48,7 @@ for folder, folders, files in os.walk(orig_folder): if configloader.config['FFMPEG']['MimicMode']: try: - os.rename(comp_file, f'{folder}_compressed/{file}') + os.rename(comp_file, f'{folder}/{file}'.replace(orig_folder, f"{orig_folder}_compressed")) except FileNotFoundError: printer.error(f"File {file} can't be processed! Maybe it is ffmpeg error or unsupported file. " f"You can change -loglevel in ffmpeg parameters to see full error.") From 329196a376290fc57e628af154aa5ca481b0ec76 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 7 Oct 2023 00:52:07 +0300 Subject: [PATCH 015/105] FFMpeg-Compressor: New HideErrors parameter --- FFMpeg-Compressor/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py index 3bf6936..5ad3afb 100755 --- a/FFMpeg-Compressor/main.py +++ b/FFMpeg-Compressor/main.py @@ -50,8 +50,9 @@ for folder, folders, files in os.walk(orig_folder): try: os.rename(comp_file, f'{folder}/{file}'.replace(orig_folder, f"{orig_folder}_compressed")) except FileNotFoundError: - printer.error(f"File {file} can't be processed! Maybe it is ffmpeg error or unsupported file. " - f"You can change -loglevel in ffmpeg parameters to see full error.") + if not configloader.config['FFMPEG']['HideErrors']: + printer.error(f"File {file} can't be processed! Maybe it is ffmpeg error or unsupported file. " + f"You can change -loglevel in ffmpeg parameters to see full error.") if configloader.config['FFMPEG']['CopyUnprocessed']: printer.info("Copying unprocessed files...") From e9777b8779ad6208da8ec0091964760b07073936 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 7 Oct 2023 01:01:11 +0300 Subject: [PATCH 016/105] Determining logic cpu count in Darwin --- build.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 475e1b2..47d5a09 100755 --- a/build.sh +++ b/build.sh @@ -2,6 +2,10 @@ mkdir output mkdir output/bin python3 -m pip install Nuitka -nuitka3 --jobs=$(nproc) --output-dir=output --follow-imports --output-filename=output/bin/ffmpeg-comp FFMpeg-Compressor/main.py +case "$(uname -s)" in + Linux*) jobs="--jobs=$(nproc)";; + Darwin*) jobs="--jobs=$(sysctl -n hw.ncpu)";; +esac +nuitka3 "${jobs}" --output-dir=output --follow-imports --output-filename=output/bin/ffmpeg-comp FFMpeg-Compressor/main.py cp FFMpeg-Compressor/ffmpeg-comp.toml output/bin/ -nuitka3 --jobs=$(nproc) --output-dir=output --follow-imports --output-filename=output/bin/rendroid-unpack RenPy-Android-Unpack/unpack.py \ No newline at end of file +nuitka3 "${jobs}" --output-dir=output --follow-imports --output-filename=output/bin/rendroid-unpack RenPy-Android-Unpack/unpack.py \ No newline at end of file From d3c8b43b4913471e93d60ef4e58a66de9974c57d Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 7 Oct 2023 01:03:48 +0300 Subject: [PATCH 017/105] FFMpeg-Compressor: Add new missing parameter into config --- FFMpeg-Compressor/ffmpeg-comp.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index 2827a2a..629dc39 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -2,6 +2,7 @@ FFmpegParams = "-n -hide_banner -loglevel quiet" CopyUnprocessed = false MimicMode = false +HideErrors = false [AUDIO] Extension = "mp3" From 0a7e62477ae28291d91e25d45a14abc58cb04e75 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 7 Oct 2023 13:27:48 +0300 Subject: [PATCH 018/105] FFMpeg-Compressor: Fix for duplicated file algorithm --- FFMpeg-Compressor/modules/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FFMpeg-Compressor/modules/utils.py b/FFMpeg-Compressor/modules/utils.py index 2828e28..010a71f 100644 --- a/FFMpeg-Compressor/modules/utils.py +++ b/FFMpeg-Compressor/modules/utils.py @@ -59,7 +59,7 @@ def add_unprocessed_files(orig_folder): for folder, folders, files in os.walk(orig_folder): for file in files: new_folder = f"{folder}".replace(orig_folder, f"{orig_folder}_compressed") - if len(glob(f"{folder}/{os.path.splitext(file)[0]}*")) != 1: + if len(glob(f"{folder}/{os.path.splitext(file)[0]}.*")) > 1: if len(glob(f"{new_folder}/{file}")): copyfile(f"{folder}/{file}", f"{new_folder}/{file} (copy)") printer.warning(f'Duplicate file has been found! Check manually this files - "{file}", "{file} (copy)"') From 224359f25171933299a751194d99c75bbe621f26 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sun, 8 Oct 2023 21:05:27 +0300 Subject: [PATCH 019/105] FFMpeg-Compressor: Fix errors hiding --- FFMpeg-Compressor/modules/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/FFMpeg-Compressor/modules/utils.py b/FFMpeg-Compressor/modules/utils.py index 010a71f..cc8e646 100644 --- a/FFMpeg-Compressor/modules/utils.py +++ b/FFMpeg-Compressor/modules/utils.py @@ -1,3 +1,4 @@ +from modules import configloader from modules import printer from shutil import copyfile from glob import glob @@ -76,8 +77,9 @@ def check_file_existing(folder, file): if not len(glob(f"{folder}/{os.path.splitext(file)[0]}*")): global errors_count errors_count += 1 - printer.error(f"{file} not processed. It can be ffmpeg error or file type is unsupported. " - f"You can set '-loglevel error' in ffmpeg config to see full error.") + if not configloader.config['FFMPEG']['HideErrors']: + printer.error(f"{file} not processed. It can be ffmpeg error or file type is unsupported. " + f"You can set '-loglevel error' in ffmpeg config to see full error.") def help_message(): From 7d087fc5b6e1e5879ada73ab0e6c12b105d56d8e Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Mon, 9 Oct 2023 23:40:31 +0300 Subject: [PATCH 020/105] FFMpeg-Compressor: Option to disabling WebP RGBA --- FFMpeg-Compressor/ffmpeg-comp.toml | 3 ++- FFMpeg-Compressor/modules/compressor.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index 629dc39..820219b 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -3,6 +3,7 @@ FFmpegParams = "-n -hide_banner -loglevel quiet" CopyUnprocessed = false MimicMode = false HideErrors = false +WebpRGBA = true [AUDIO] Extension = "mp3" @@ -15,4 +16,4 @@ JpegComp = 3 [VIDEO] Extension = "mp4" -Codec = "libvpx-vp9" +Codec = "libvpx-vp9" \ No newline at end of file diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 53305c3..3ae9ca9 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -80,6 +80,18 @@ def compress_image(folder, file, target_folder): f"'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}'") else: printer.warning(f"{file} has transparency (.jpg not support it). Skipping...") + + elif get_req_ext(file) == "webp" and configloader.config['FFMPEG']['WebpRGBA']: + if not has_transparency(Image.open(f'{folder}/{file}')): + printer.files(file, os.path.splitext(file)[0], get_req_ext(file), f"{comp_level}%") + os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -compression_level {comp_level} " + f"'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}'") + else: + printer.warning(f"{file} has transparency, but WebP RGBA disabled in config. Changing to png...") + printer.files(file, os.path.splitext(file)[0], "png", f"{comp_level}%") + os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -compression_level {comp_level} " + f"'{target_folder}/{os.path.splitext(file)[0]}.png'") + else: printer.files(file, os.path.splitext(file)[0], get_req_ext(file), f"{comp_level}%") os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -compression_level {comp_level} " From ea893fd8d19be85cb11ae6b895f3c47164937021 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Mon, 9 Oct 2023 23:47:16 +0300 Subject: [PATCH 021/105] FFMpeg-Compressor: Delete JpegCompression parameter in config --- FFMpeg-Compressor/README.md | 1 - FFMpeg-Compressor/ffmpeg-comp.toml | 1 - FFMpeg-Compressor/modules/compressor.py | 5 ++--- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index 612fd46..2a9d7fb 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -20,7 +20,6 @@ Python utility uses ffmpeg to compress Visual Novel Resources #### IMAGE section * Extension - Required image file extension. It supports: `.apng`, `.avif`, `.bmp`, `.jfif`, `.pjpeg`, `.pjp`, `.svg`, `.webp`, `.jpg/.jpeg`, `.png`, `.raw` * CompLevel - Compression level for images. Values range: `0-100` (100 - max compression, 0 - min compression) -* JpegComp - (May be deleted in future) Compression level specific for jpeg images. Values range: `0-10` (10 - max compression, 0 - min compression) #### VIDEO section * Extension - Required image file extension. It supports: `.3gp`, `.amv`, `.avi`, `.gif`, `.m4v`, `.mkv`, `.mov`, `.mp4`, `.m4v`, `.mpeg`, `.mpv`, `.webm`, `.ogv` diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index 820219b..1a1794b 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -12,7 +12,6 @@ BitRate = "320k" [IMAGE] Extension = "jpg" CompLevel = 20 -JpegComp = 3 [VIDEO] Extension = "mp4" diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 3ae9ca9..4cd5545 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -70,13 +70,12 @@ def compress_video(folder, file, target_folder): def compress_image(folder, file, target_folder): ffmpeg_params = configloader.config['FFMPEG']['FFmpegParams'] comp_level = configloader.config['IMAGE']['CompLevel'] - jpg_comp = configloader.config['IMAGE']['JpegComp'] if get_req_ext(file) == "jpg" or get_req_ext(file) == "jpeg": if not has_transparency(Image.open(f'{folder}/{file}')): - printer.files(file, os.path.splitext(file)[0], get_req_ext(file), f"level {jpg_comp}") - os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -q {jpg_comp} " + printer.files(file, os.path.splitext(file)[0], get_req_ext(file), f"{comp_level}%") + os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -q {comp_level/10} " f"'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}'") else: printer.warning(f"{file} has transparency (.jpg not support it). Skipping...") From 12b7fced035cf97f1d6e9566415f1f91c3c467ef Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Mon, 9 Oct 2023 23:55:10 +0300 Subject: [PATCH 022/105] FFMpeg-Compressor: Fix typo --- FFMpeg-Compressor/modules/compressor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 4cd5545..39bc6b5 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -80,7 +80,7 @@ def compress_image(folder, file, target_folder): else: printer.warning(f"{file} has transparency (.jpg not support it). Skipping...") - elif get_req_ext(file) == "webp" and configloader.config['FFMPEG']['WebpRGBA']: + elif get_req_ext(file) == "webp" and not configloader.config['FFMPEG']['WebpRGBA']: if not has_transparency(Image.open(f'{folder}/{file}')): printer.files(file, os.path.splitext(file)[0], get_req_ext(file), f"{comp_level}%") os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -compression_level {comp_level} " From 616e7f63ef8a43d79a6f128575c8dffed0703a69 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Mon, 9 Oct 2023 23:56:52 +0300 Subject: [PATCH 023/105] FFMpeg-Compressor: Use ffmpeg hwaccel if available --- FFMpeg-Compressor/README.md | 2 +- FFMpeg-Compressor/ffmpeg-comp.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index 2a9d7fb..698a62d 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -9,7 +9,7 @@ Python utility uses ffmpeg to compress Visual Novel Resources ### Configuration #### FFMPEG section -* FFMpegParams - Some parameters & flags for ffmpeg command line interface (default: `"-n -hide_banner -loglevel quiet"`) +* FFMpegParams - Some parameters & flags for ffmpeg command line interface (default: `"-n -hide_banner -loglevel quiet -hwaccel auto"`) * CopyUnprocessed - Copy all files that failed to compress by ffmpeg to destination folder. In can helps to recreate original folder, but with compressed files. * MimicMode - Rename compressed file to it original name and extension. VN engines determine the file type by its header, so for example PNG file named file.jpg will be loaded as PNG file. (default: `false`) diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index 1a1794b..5f3e02d 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -1,5 +1,5 @@ [FFMPEG] -FFmpegParams = "-n -hide_banner -loglevel quiet" +FFmpegParams = "-n -hide_banner -loglevel quiet -hwaccel auto" CopyUnprocessed = false MimicMode = false HideErrors = false From 603032fe786b5e38c99fdc48dff8f0f064396aac Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Tue, 10 Oct 2023 02:19:37 +0300 Subject: [PATCH 024/105] FFMpeg-Compressor: Fix recursive unprocessed files copying --- FFMpeg-Compressor/modules/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/FFMpeg-Compressor/modules/utils.py b/FFMpeg-Compressor/modules/utils.py index cc8e646..4559158 100644 --- a/FFMpeg-Compressor/modules/utils.py +++ b/FFMpeg-Compressor/modules/utils.py @@ -63,18 +63,19 @@ def add_unprocessed_files(orig_folder): if len(glob(f"{folder}/{os.path.splitext(file)[0]}.*")) > 1: if len(glob(f"{new_folder}/{file}")): copyfile(f"{folder}/{file}", f"{new_folder}/{file} (copy)") - printer.warning(f'Duplicate file has been found! Check manually this files - "{file}", "{file} (copy)"') + printer.warning( + f'Duplicate file has been found! Check manually this files - "{file}", "{file} (copy)"') else: copyfile(f"{folder}/{file}", f"{new_folder}/{file}") printer.info(f"File {file} copied to compressed folder.") else: - if not len(glob(f"{folder}_compressed/{os.path.splitext(file)[0]}*")): + if not len(glob(f"{new_folder}/{os.path.splitext(file)[0]}.*")): # Why it can't find files?!??!??!?!? copyfile(f"{folder}/{file}", f"{new_folder}/{file}") printer.info(f"File {file} copied to compressed folder.") def check_file_existing(folder, file): - if not len(glob(f"{folder}/{os.path.splitext(file)[0]}*")): + if not len(glob(f"{folder}/{os.path.splitext(file)[0]}.*")): global errors_count errors_count += 1 if not configloader.config['FFMPEG']['HideErrors']: From 7dee19ae086bffcdc7f5668d30d9ac2db1002749 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 11 Oct 2023 18:53:35 +0300 Subject: [PATCH 025/105] FFMpeg-Compressor: Revert enabling hwaccel by default --- FFMpeg-Compressor/README.md | 2 +- FFMpeg-Compressor/ffmpeg-comp.toml | 2 +- FFMpeg-Compressor/modules/utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index 698a62d..2a9d7fb 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -9,7 +9,7 @@ Python utility uses ffmpeg to compress Visual Novel Resources ### Configuration #### FFMPEG section -* FFMpegParams - Some parameters & flags for ffmpeg command line interface (default: `"-n -hide_banner -loglevel quiet -hwaccel auto"`) +* FFMpegParams - Some parameters & flags for ffmpeg command line interface (default: `"-n -hide_banner -loglevel quiet"`) * CopyUnprocessed - Copy all files that failed to compress by ffmpeg to destination folder. In can helps to recreate original folder, but with compressed files. * MimicMode - Rename compressed file to it original name and extension. VN engines determine the file type by its header, so for example PNG file named file.jpg will be loaded as PNG file. (default: `false`) diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index 5f3e02d..1a1794b 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -1,5 +1,5 @@ [FFMPEG] -FFmpegParams = "-n -hide_banner -loglevel quiet -hwaccel auto" +FFmpegParams = "-n -hide_banner -loglevel quiet" CopyUnprocessed = false MimicMode = false HideErrors = false diff --git a/FFMpeg-Compressor/modules/utils.py b/FFMpeg-Compressor/modules/utils.py index 4559158..15ef2cc 100644 --- a/FFMpeg-Compressor/modules/utils.py +++ b/FFMpeg-Compressor/modules/utils.py @@ -69,7 +69,7 @@ def add_unprocessed_files(orig_folder): copyfile(f"{folder}/{file}", f"{new_folder}/{file}") printer.info(f"File {file} copied to compressed folder.") else: - if not len(glob(f"{new_folder}/{os.path.splitext(file)[0]}.*")): # Why it can't find files?!??!??!?!? + if not len(glob(f"{new_folder}/{os.path.splitext(file)[0]}.*")): copyfile(f"{folder}/{file}", f"{new_folder}/{file}") printer.info(f"File {file} copied to compressed folder.") From 37e3c9b257393e71bfa203dc2a338bbdd756c00f Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 11 Oct 2023 18:56:00 +0300 Subject: [PATCH 026/105] RenPy-Android-Unpack: Execute python3 from PATH --- RenPy-Android-Unpack/unpack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RenPy-Android-Unpack/unpack.py b/RenPy-Android-Unpack/unpack.py index 07cc074..7a9777a 100755 --- a/RenPy-Android-Unpack/unpack.py +++ b/RenPy-Android-Unpack/unpack.py @@ -1,4 +1,4 @@ -#!/bin/python3 +#!python3 import zipfile import os import shutil From e48b7599d16d6513dc2abd72ecff05ec62a6aeb1 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Mon, 30 Oct 2023 00:56:53 +0300 Subject: [PATCH 027/105] FFMpeg-Compressor: Support .m2t file format --- FFMpeg-Compressor/README.md | 2 +- FFMpeg-Compressor/modules/compressor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index 2a9d7fb..fdb9e83 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -22,7 +22,7 @@ Python utility uses ffmpeg to compress Visual Novel Resources * CompLevel - Compression level for images. Values range: `0-100` (100 - max compression, 0 - min compression) #### VIDEO section -* Extension - Required image file extension. It supports: `.3gp`, `.amv`, `.avi`, `.gif`, `.m4v`, `.mkv`, `.mov`, `.mp4`, `.m4v`, `.mpeg`, `.mpv`, `.webm`, `.ogv` +* Extension - Required image file extension. It supports: `.3gp`, `.amv`, `.avi`, `.gif`, `.m2l`, `.m4v`, `.mkv`, `.mov`, `.mp4`, `.m4v`, `.mpeg`, `.mpv`, `.webm`, `.ogv` * Codec - (May be optional in future) Required video codec. (See official ffmpeg documentation for supported codecs) ### TODO (for testing branch) diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 39bc6b5..d29b4ce 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -17,7 +17,7 @@ def get_req_ext(file): def get_file_type(file): audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] image_ext = ['.apng', '.avif', '.bmp', '.jfif', '.pjpeg', '.pjp', '.svg', '.webp', '.jpg', '.jpeg', '.png', '.raw'] - video_ext = ['.3gp' '.amv', '.avi', '.gif', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', '.webm', + video_ext = ['.3gp' '.amv', '.avi', '.gif', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', '.webm', '.ogv'] file_extension = os.path.splitext(file)[1] From 37ff1f78b326f188b1d1743c2cef5e58c0e13c0d Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Thu, 16 Nov 2023 01:26:23 +0300 Subject: [PATCH 028/105] FFMpeg-Compressor: Use ffmpeg-python and Pillow instead of ffmpeg cli --- FFMpeg-Compressor/README.md | 9 +- FFMpeg-Compressor/ffmpeg-comp.toml | 9 +- FFMpeg-Compressor/main.py | 99 +++++++++++-------- FFMpeg-Compressor/modules/compressor.py | 123 +++++++++++------------- FFMpeg-Compressor/modules/utils.py | 9 -- FFMpeg-Compressor/requirements.txt | 2 + 6 files changed, 126 insertions(+), 125 deletions(-) diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index fdb9e83..de7d334 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -9,17 +9,18 @@ Python utility uses ffmpeg to compress Visual Novel Resources ### Configuration #### FFMPEG section -* FFMpegParams - Some parameters & flags for ffmpeg command line interface (default: `"-n -hide_banner -loglevel quiet"`) * CopyUnprocessed - Copy all files that failed to compress by ffmpeg to destination folder. In can helps to recreate original folder, but with compressed files. * MimicMode - Rename compressed file to it original name and extension. VN engines determine the file type by its header, so for example PNG file named file.jpg will be loaded as PNG file. (default: `false`) +* HideErrors - Hide some errors about compression. (default: `false`) +* WebpRGBA - Alpha channel in webp. If false switches extension to png. (default: `true`) #### AUDIO section * Extension - Required audio file extension. It supports: `.aac`, `.flac`, `.m4a`, `.mp3`, `.ogg`, `.opus`, `.raw`, `.wav`, `.wma`. * BitRate - (mp3 only, for now) Required audio bitrate. For best quality use `320k` value, but for worse use `1-9` (9 worst) number range. #### IMAGE section -* Extension - Required image file extension. It supports: `.apng`, `.avif`, `.bmp`, `.jfif`, `.pjpeg`, `.pjp`, `.svg`, `.webp`, `.jpg/.jpeg`, `.png`, `.raw` -* CompLevel - Compression level for images. Values range: `0-100` (100 - max compression, 0 - min compression) +* Extension - Required image file extension. It supports: `.apng`, `.avif`, `.bmp`, `.tga`, `.tiff`, `.dds`, `.svg`, `.webp`, `.jpg/.jpeg`, `.png` +* Quality - Quality level of images. Values range: `0-100` (100 - best quality, 0 - worst quality) #### VIDEO section * Extension - Required image file extension. It supports: `.3gp`, `.amv`, `.avi`, `.gif`, `.m2l`, `.m4v`, `.mkv`, `.mov`, `.mp4`, `.m4v`, `.mpeg`, `.mpv`, `.webm`, `.ogv` @@ -28,5 +29,5 @@ Python utility uses ffmpeg to compress Visual Novel Resources ### TODO (for testing branch) * [x] Recreate whole game directory with compressed files * [ ] Cross platform (Easy Windows usage and binaries, MacOS binaries) -* [ ] Use ffmpeg python bindings instead of cli commands +* [x] Use ffmpeg python bindings instead of cli commands * [ ] Reorganize code \ No newline at end of file diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index 1a1794b..f615c54 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -1,18 +1,17 @@ [FFMPEG] -FFmpegParams = "-n -hide_banner -loglevel quiet" CopyUnprocessed = false MimicMode = false HideErrors = false WebpRGBA = true [AUDIO] -Extension = "mp3" +Extension = "opus" BitRate = "320k" [IMAGE] -Extension = "jpg" -CompLevel = 20 +Extension = "avif" +Quality = 20 [VIDEO] -Extension = "mp4" +Extension = "webm" Codec = "libvpx-vp9" \ No newline at end of file diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py index 5ad3afb..84d86df 100755 --- a/FFMpeg-Compressor/main.py +++ b/FFMpeg-Compressor/main.py @@ -8,53 +8,70 @@ import shutil import sys import os -try: - if sys.argv[1][len(sys.argv[1])-1] == "/": - arg_path = sys.argv[1][:len(sys.argv[1])-1] + +def get_file_type(filename): + audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] + image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'] + video_ext = ['.3gp' '.amv', '.avi', '.gif', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', + '.webm', '.ogv'] + + if os.path.splitext(filename)[1] in audio_ext: + return "audio" + elif os.path.splitext(filename)[1] in image_ext: + return "image" + elif os.path.splitext(filename)[1] in video_ext: + return "video" else: - arg_path = sys.argv[1] -except IndexError: - print(utils.help_message()) - exit() + return "unknown" -orig_folder = arg_path -printer.orig_folder = arg_path -printer.bar_init(orig_folder) +if __name__ == "__main__": + try: + if sys.argv[1][len(sys.argv[1])-1] == "/": + arg_path = sys.argv[1][:len(sys.argv[1])-1] + else: + arg_path = sys.argv[1] + except IndexError: + print(utils.help_message()) + exit() -if os.path.exists(f"{orig_folder}_compressed"): - shutil.rmtree(f"{orig_folder}_compressed") + orig_folder = arg_path + printer.orig_folder = arg_path -printer.info("Creating folders...") -for folder, folders, files in os.walk(orig_folder): - if not os.path.exists(folder.replace(orig_folder, f"{orig_folder}_compressed")): - os.mkdir(folder.replace(orig_folder, f"{orig_folder}_compressed")) + printer.bar_init(orig_folder) - printer.info(f"Compressing \"{folder.replace(orig_folder, orig_folder.split('/').pop())}\" folder...") - target_folder = folder.replace(orig_folder, f"{orig_folder}_compressed") - for file in os.listdir(folder): - if os.path.isfile(f'{folder}/{file}'): - match compressor.get_file_type(file): - case "audio": - comp_file = compressor.compress_audio(folder, file, target_folder) - case "image": - comp_file = compressor.compress_image(folder, file, target_folder) - case "video": - comp_file = compressor.compress_video(folder, file, target_folder) - case "unknown": - comp_file = compressor.compress(folder, file, target_folder) + if os.path.exists(f"{orig_folder}_compressed"): + shutil.rmtree(f"{orig_folder}_compressed") - utils.check_file_existing(folder.replace(orig_folder, f"{orig_folder}_compressed"), file) + printer.info("Creating folders...") + for folder, folders, files in os.walk(orig_folder): + if not os.path.exists(folder.replace(orig_folder, f"{orig_folder}_compressed")): + os.mkdir(folder.replace(orig_folder, f"{orig_folder}_compressed")) - if configloader.config['FFMPEG']['MimicMode']: - try: - os.rename(comp_file, f'{folder}/{file}'.replace(orig_folder, f"{orig_folder}_compressed")) - except FileNotFoundError: - if not configloader.config['FFMPEG']['HideErrors']: - printer.error(f"File {file} can't be processed! Maybe it is ffmpeg error or unsupported file. " - f"You can change -loglevel in ffmpeg parameters to see full error.") + printer.info(f"Compressing \"{folder.replace(orig_folder, orig_folder.split('/').pop())}\" folder...") + target_folder = folder.replace(orig_folder, f"{orig_folder}_compressed") + for file in os.listdir(folder): + if os.path.isfile(f'{folder}/{file}'): + match get_file_type(file): + case "audio": + comp_file = compressor.compress_audio(folder, file, target_folder, + configloader.config['AUDIO']['Extension']) + case "image": + comp_file = compressor.compress_image(folder, file, target_folder, + configloader.config['IMAGE']['Extension']) + case "video": + comp_file = compressor.compress_video(folder, file, target_folder, + configloader.config['VIDEO']['Extension']) + case "unknown": + comp_file = compressor.compress(folder, file, target_folder) -if configloader.config['FFMPEG']['CopyUnprocessed']: - printer.info("Copying unprocessed files...") - utils.add_unprocessed_files(orig_folder) -utils.get_compression_status(orig_folder) + if configloader.config['FFMPEG']['MimicMode']: + try: + os.rename(comp_file, f'{folder}/{file}'.replace(orig_folder, f"{orig_folder}_compressed")) + except FileNotFoundError: + pass + + if configloader.config['FFMPEG']['CopyUnprocessed']: + printer.info("Copying unprocessed files...") + utils.add_unprocessed_files(orig_folder) + utils.get_compression_status(orig_folder) diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index d29b4ce..c3f833a 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -1,36 +1,12 @@ -from modules import printer from modules import configloader +from modules import printer +from modules import utils from PIL import Image +import pillow_avif +import ffmpeg import os -def get_req_ext(file): - match get_file_type(file): - case "audio": - return configloader.config['AUDIO']['Extension'] - case "image": - return configloader.config['IMAGE']['Extension'] - case "video": - return configloader.config['VIDEO']['Extension'] - - -def get_file_type(file): - audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] - image_ext = ['.apng', '.avif', '.bmp', '.jfif', '.pjpeg', '.pjp', '.svg', '.webp', '.jpg', '.jpeg', '.png', '.raw'] - video_ext = ['.3gp' '.amv', '.avi', '.gif', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', '.webm', - '.ogv'] - file_extension = os.path.splitext(file)[1] - - if file_extension in audio_ext: - return "audio" - elif file_extension in image_ext: - return "image" - elif file_extension in video_ext: - return "video" - else: - return "unknown" - - def has_transparency(img): if img.info.get("transparency", None) is not None: return True @@ -47,59 +23,74 @@ def has_transparency(img): return False -def compress_audio(folder, file, target_folder): - ffmpeg_params = configloader.config['FFMPEG']['FFmpegParams'] +def compress_audio(folder, file, target_folder, extension): bitrate = configloader.config['AUDIO']['BitRate'] - printer.files(file, os.path.splitext(file)[0], get_req_ext(file), f"{bitrate}") - os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -q:a {bitrate} " - f"'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}'") - return f'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}' + printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}") + try: + (ffmpeg + .input(f'{folder}/{file}') + .output(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}', audio_bitrate=bitrate) + .run(quiet=True) + ) + except ffmpeg._run.Error: + utils.errors_count += 1 + if not configloader.config['FFMPEG']['HideErrors']: + printer.error(f"File {file} can't be processed! Maybe it is ffmpeg error or unsupported file.") + return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' -def compress_video(folder, file, target_folder): - ffmpeg_params = configloader.config['FFMPEG']['FFmpegParams'] +def compress_video(folder, file, target_folder, extension): codec = configloader.config['VIDEO']['Codec'] - printer.files(file, os.path.splitext(file)[0], get_req_ext(file), codec) - os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -vcodec {codec} " - f"'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}'") - return f'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}' + printer.files(file, os.path.splitext(file)[0], extension, codec) + try: + (ffmpeg + .input(f'{folder}/{file}') + .output(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}', format=codec) + .run(quiet=True) + ) + except ffmpeg._run.Error: + utils.errors_count += 1 + if not configloader.config['FFMPEG']['HideErrors']: + printer.error(f"File {file} can't be processed! Maybe it is ffmpeg error or unsupported file.") + return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' -def compress_image(folder, file, target_folder): - ffmpeg_params = configloader.config['FFMPEG']['FFmpegParams'] - comp_level = configloader.config['IMAGE']['CompLevel'] +def compress_image(folder, file, target_folder, extension): + quality = configloader.config['IMAGE']['Quality'] - if get_req_ext(file) == "jpg" or get_req_ext(file) == "jpeg": + image = Image.open(f'{folder}/{file}') - if not has_transparency(Image.open(f'{folder}/{file}')): - printer.files(file, os.path.splitext(file)[0], get_req_ext(file), f"{comp_level}%") - os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -q {comp_level/10} " - f"'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}'") - else: - printer.warning(f"{file} has transparency (.jpg not support it). Skipping...") + if (extension == "jpg" or extension == "jpeg" or + (extension == "webp" and not configloader.config['FFMPEG']['WebpRGBA'])): - elif get_req_ext(file) == "webp" and not configloader.config['FFMPEG']['WebpRGBA']: - if not has_transparency(Image.open(f'{folder}/{file}')): - printer.files(file, os.path.splitext(file)[0], get_req_ext(file), f"{comp_level}%") - os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -compression_level {comp_level} " - f"'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}'") - else: - printer.warning(f"{file} has transparency, but WebP RGBA disabled in config. Changing to png...") - printer.files(file, os.path.splitext(file)[0], "png", f"{comp_level}%") - os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -compression_level {comp_level} " - f"'{target_folder}/{os.path.splitext(file)[0]}.png'") + if has_transparency(Image.open(f'{folder}/{file}')): + printer.warning(f"{file} has transparency. Changing to png...") + printer.files(file, os.path.splitext(file)[0], "png", f"{quality}%") + image.save(f"{target_folder}/{os.path.splitext(file)[0]}.png", + optimize=True, + quality=quality) + return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' else: - printer.files(file, os.path.splitext(file)[0], get_req_ext(file), f"{comp_level}%") - os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} -compression_level {comp_level} " - f"'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}'") - return f'{target_folder}/{os.path.splitext(file)[0]}.{get_req_ext(file)}' + printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") + image.save(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}", + optimize=True, + quality=quality) + return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' def compress(folder, file, target_folder): - ffmpeg_params = configloader.config['FFMPEG']['FFmpegParams'] printer.unknown_file(file) - os.system(f"ffmpeg -i '{folder}/{file}' {ffmpeg_params} '{target_folder}/{file}'") + try: + (ffmpeg + .input(f'{folder}/{file}') + .output(f'{target_folder}/{file}') + .run(quiet=True) + ) + except ffmpeg._run.Error: + utils.errors_count += 1 + if not configloader.config['FFMPEG']['HideErrors']: + printer.error(f"File {file} can't be processed! Maybe it is ffmpeg error or unsupported file.") return f'{target_folder}/{file}' diff --git a/FFMpeg-Compressor/modules/utils.py b/FFMpeg-Compressor/modules/utils.py index 15ef2cc..08074e7 100644 --- a/FFMpeg-Compressor/modules/utils.py +++ b/FFMpeg-Compressor/modules/utils.py @@ -74,14 +74,5 @@ def add_unprocessed_files(orig_folder): printer.info(f"File {file} copied to compressed folder.") -def check_file_existing(folder, file): - if not len(glob(f"{folder}/{os.path.splitext(file)[0]}.*")): - global errors_count - errors_count += 1 - if not configloader.config['FFMPEG']['HideErrors']: - printer.error(f"{file} not processed. It can be ffmpeg error or file type is unsupported. " - f"You can set '-loglevel error' in ffmpeg config to see full error.") - - def help_message(): return "Usage: ffmpeg-comp {folder}" diff --git a/FFMpeg-Compressor/requirements.txt b/FFMpeg-Compressor/requirements.txt index 290af37..8d1f56a 100644 --- a/FFMpeg-Compressor/requirements.txt +++ b/FFMpeg-Compressor/requirements.txt @@ -1,2 +1,4 @@ Pillow==9.5.0 +pillow-avif-plugin==1.4.1 +ffmpeg-python==0.2.0 progress==1.6 From 721d33efa72f1e9921d341d6daa185cf8b448d08 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Thu, 16 Nov 2023 02:12:04 +0300 Subject: [PATCH 029/105] FFMpeg-Compressor: Slightly rewrite interface --- FFMpeg-Compressor/modules/printer.py | 14 ++++++-------- FFMpeg-Compressor/modules/utils.py | 11 ++++------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/FFMpeg-Compressor/modules/printer.py b/FFMpeg-Compressor/modules/printer.py index f297a7d..a473dc0 100644 --- a/FFMpeg-Compressor/modules/printer.py +++ b/FFMpeg-Compressor/modules/printer.py @@ -8,15 +8,15 @@ def clean_str(string): def info(string): - print(clean_str(f"\r\033[0;32m[INFO]\033[0m {string}")) + print(clean_str(f"\r\033[100mI {string}\033[49m")) def warning(string): - print(clean_str(f"\r\033[0;33m[WARNING]\033[0m {string}")) + print(clean_str(f"\r\033[93mW\033[0m {string}\033[49m")) def error(string): - print(clean_str(f"\r\033[0;31m[ERROR]\033[0m {string}")) + print(clean_str(f"\r\033[31mE\033[0m {string}\033[49m")) def bar_init(folder): @@ -24,19 +24,17 @@ def bar_init(folder): for folder, folders, file in os.walk(folder): file_count += len(file) global bar - bar = IncrementalBar('Compressing', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%) - ETA: %(eta)ds') + bar = IncrementalBar('Compressing', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)') def files(source, dest, dest_ext, comment): - source_ext = os.path.splitext(source)[1] source_name = os.path.splitext(source)[0] - print(clean_str(f"\r[COMP] \033[0;32m{source_name}\033[0m{source_ext}\033[0;32m -> {dest}\033[0m.{dest_ext}\033[0;32m ({comment})\033[0m")) + print(clean_str(f"\r* \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m")) bar.next() def unknown_file(file): - - print(clean_str(f"\r[COMP] \033[0;33m{file}\033[0m (File extension not recognized)")) + print(clean_str(f"\r* \033[0;33m{file}\033[0m (Not recognized)")) bar.next() diff --git a/FFMpeg-Compressor/modules/utils.py b/FFMpeg-Compressor/modules/utils.py index 08074e7..dd5e80a 100644 --- a/FFMpeg-Compressor/modules/utils.py +++ b/FFMpeg-Compressor/modules/utils.py @@ -21,14 +21,11 @@ def get_compression(orig, comp): for folder, folders, files in os.walk(comp): for file in files: processed_files.append(file) - try: - comp = 100 - int((get_dir_size(comp, processed_files) / get_dir_size(orig, processed_files)) * 100) - if comp < 0: - printer.warning(f'Compression: {comp}%') - printer.warning("The resulting files are larger than the original ones!") - else: - printer.info(f'Compression: {comp}%') + orig = get_dir_size(orig, processed_files) + comp = get_dir_size(comp, processed_files) + + print(f"Result: {orig/1024/1024:.2}MB -> {comp/1024/1024:.2}MB Δ {(orig - comp)/1024/1024:.2}MB") except ZeroDivisionError: printer.warning("Nothing compressed!") From f7ecee568338f30c8180abb8aa0f9da09ca845cb Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Thu, 16 Nov 2023 02:36:06 +0300 Subject: [PATCH 030/105] FFMpeg-Compressor: Update README.md --- FFMpeg-Compressor/README.md | 3 ++- FFMpeg-Compressor/ffmpeg-comp.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index de7d334..63c54dc 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -30,4 +30,5 @@ Python utility uses ffmpeg to compress Visual Novel Resources * [x] Recreate whole game directory with compressed files * [ ] Cross platform (Easy Windows usage and binaries, MacOS binaries) * [x] Use ffmpeg python bindings instead of cli commands -* [ ] Reorganize code \ No newline at end of file +* [ ] Reorganize code +* [ ] Multithread \ No newline at end of file diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index f615c54..6374e20 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -10,7 +10,7 @@ BitRate = "320k" [IMAGE] Extension = "avif" -Quality = 20 +Quality = 80 [VIDEO] Extension = "webm" From 4b050a56590f08712131794f3db9de9e057268e6 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Thu, 16 Nov 2023 14:27:50 +0300 Subject: [PATCH 031/105] FFMpeg-Compressor: Add lossless parameter --- FFMpeg-Compressor/README.md | 1 + FFMpeg-Compressor/ffmpeg-comp.toml | 1 + FFMpeg-Compressor/modules/compressor.py | 1 + 3 files changed, 3 insertions(+) diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index 63c54dc..34129ff 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -20,6 +20,7 @@ Python utility uses ffmpeg to compress Visual Novel Resources #### IMAGE section * Extension - Required image file extension. It supports: `.apng`, `.avif`, `.bmp`, `.tga`, `.tiff`, `.dds`, `.svg`, `.webp`, `.jpg/.jpeg`, `.png` +* Lossless - Enables lossless copression for supported formats. With this quality parameter means quality of compression. (default: `false`) * Quality - Quality level of images. Values range: `0-100` (100 - best quality, 0 - worst quality) #### VIDEO section diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index 6374e20..a8c1df5 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -10,6 +10,7 @@ BitRate = "320k" [IMAGE] Extension = "avif" +Lossless = false Quality = 80 [VIDEO] diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index c3f833a..5fdd2d8 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -77,6 +77,7 @@ def compress_image(folder, file, target_folder, extension): printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") image.save(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}", optimize=True, + lossless=configloader.config['IMAGE']['Lossless'], quality=quality) return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' From e34b06b507221768b83eaf0e279068ced5cf66c4 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Tue, 28 Nov 2023 17:08:17 +0300 Subject: [PATCH 032/105] FFMpeg-Compressor: Fix comp size regression --- FFMpeg-Compressor/modules/compressor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 5fdd2d8..a73c988 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -70,6 +70,7 @@ def compress_image(folder, file, target_folder, extension): printer.files(file, os.path.splitext(file)[0], "png", f"{quality}%") image.save(f"{target_folder}/{os.path.splitext(file)[0]}.png", optimize=True, + minimize_size=True, quality=quality) return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' @@ -78,7 +79,8 @@ def compress_image(folder, file, target_folder, extension): image.save(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}", optimize=True, lossless=configloader.config['IMAGE']['Lossless'], - quality=quality) + quality=quality, + minimize_size=True) return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' From 2f31c22d91579cfdef7b97071ce263b2def6c33b Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Tue, 28 Nov 2023 17:10:22 +0300 Subject: [PATCH 033/105] FFMpeg-Compressor: Fix some logic --- FFMpeg-Compressor/modules/compressor.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index a73c988..0c437e5 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -64,23 +64,16 @@ def compress_image(folder, file, target_folder, extension): if (extension == "jpg" or extension == "jpeg" or (extension == "webp" and not configloader.config['FFMPEG']['WebpRGBA'])): - if has_transparency(Image.open(f'{folder}/{file}')): printer.warning(f"{file} has transparency. Changing to png...") - printer.files(file, os.path.splitext(file)[0], "png", f"{quality}%") - image.save(f"{target_folder}/{os.path.splitext(file)[0]}.png", - optimize=True, - minimize_size=True, - quality=quality) - return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' + extension = ".png" - else: - printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") - image.save(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}", - optimize=True, - lossless=configloader.config['IMAGE']['Lossless'], - quality=quality, - minimize_size=True) + printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") + image.save(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}", + optimize=True, + lossless=configloader.config['IMAGE']['Lossless'], + quality=quality, + minimize_size=True) return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' From e70dd5614296eb58794ea5ad4b49b8369dba48b8 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Tue, 28 Nov 2023 19:07:09 +0300 Subject: [PATCH 034/105] FFMpeg-Compressor: FallBackExtension parameter --- FFMpeg-Compressor/README.md | 1 + FFMpeg-Compressor/ffmpeg-comp.toml | 1 + FFMpeg-Compressor/modules/compressor.py | 6 +++--- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index 34129ff..4ce9e0b 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -20,6 +20,7 @@ Python utility uses ffmpeg to compress Visual Novel Resources #### IMAGE section * Extension - Required image file extension. It supports: `.apng`, `.avif`, `.bmp`, `.tga`, `.tiff`, `.dds`, `.svg`, `.webp`, `.jpg/.jpeg`, `.png` +* FallBackExtension - Extension if current format does not support RGBA. * Lossless - Enables lossless copression for supported formats. With this quality parameter means quality of compression. (default: `false`) * Quality - Quality level of images. Values range: `0-100` (100 - best quality, 0 - worst quality) diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index a8c1df5..3840e5a 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -10,6 +10,7 @@ BitRate = "320k" [IMAGE] Extension = "avif" +FallBackExtension = "webp" Lossless = false Quality = 80 diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 0c437e5..2203081 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -62,11 +62,11 @@ def compress_image(folder, file, target_folder, extension): image = Image.open(f'{folder}/{file}') - if (extension == "jpg" or extension == "jpeg" or + if (extension == "jpg" or extension == "jpeg" or extension == "avif" or (extension == "webp" and not configloader.config['FFMPEG']['WebpRGBA'])): if has_transparency(Image.open(f'{folder}/{file}')): - printer.warning(f"{file} has transparency. Changing to png...") - extension = ".png" + printer.warning(f"{file} has transparency. Changing to fallback...") + extension = configloader.config['IMAGE']['FallBackExtension'] printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") image.save(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}", From fd7b9cd4d9fe3b2c789ba83b2d6e3ca3c12dbdf9 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Tue, 28 Nov 2023 21:59:23 +0300 Subject: [PATCH 035/105] FFMpeg-Compressor: Fix codec in video compression --- FFMpeg-Compressor/modules/compressor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 2203081..71f571c 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -47,7 +47,7 @@ def compress_video(folder, file, target_folder, extension): try: (ffmpeg .input(f'{folder}/{file}') - .output(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}', format=codec) + .output(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}', vcodec=codec) .run(quiet=True) ) except ffmpeg._run.Error: @@ -64,7 +64,7 @@ def compress_image(folder, file, target_folder, extension): if (extension == "jpg" or extension == "jpeg" or extension == "avif" or (extension == "webp" and not configloader.config['FFMPEG']['WebpRGBA'])): - if has_transparency(Image.open(f'{folder}/{file}')): + if has_transparency(image): printer.warning(f"{file} has transparency. Changing to fallback...") extension = configloader.config['IMAGE']['FallBackExtension'] From 6698db5feffacb51ef758d389c845072cd36d705 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Tue, 28 Nov 2023 23:36:01 +0300 Subject: [PATCH 036/105] FFMpeg-Compressor: Fix size displaying --- FFMpeg-Compressor/modules/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FFMpeg-Compressor/modules/utils.py b/FFMpeg-Compressor/modules/utils.py index dd5e80a..ffd6cff 100644 --- a/FFMpeg-Compressor/modules/utils.py +++ b/FFMpeg-Compressor/modules/utils.py @@ -25,7 +25,7 @@ def get_compression(orig, comp): orig = get_dir_size(orig, processed_files) comp = get_dir_size(comp, processed_files) - print(f"Result: {orig/1024/1024:.2}MB -> {comp/1024/1024:.2}MB Δ {(orig - comp)/1024/1024:.2}MB") + print(f"Result: {orig/1024/1024:.2f}MB -> {comp/1024/1024:.2f}MB Δ {(orig - comp)/1024/1024:.2f}MB") except ZeroDivisionError: printer.warning("Nothing compressed!") From 27efc155b963951fdd383f077e9d0e110b3c6d6f Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 6 Jan 2024 14:25:01 +0300 Subject: [PATCH 037/105] FFMpeg-Compressor: Copy unprocessed files while running --- FFMpeg-Compressor/main.py | 3 -- FFMpeg-Compressor/modules/compressor.py | 53 +++++++++++++++---------- FFMpeg-Compressor/modules/utils.py | 31 +++++++-------- 3 files changed, 46 insertions(+), 41 deletions(-) diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py index 84d86df..8e68e42 100755 --- a/FFMpeg-Compressor/main.py +++ b/FFMpeg-Compressor/main.py @@ -71,7 +71,4 @@ if __name__ == "__main__": except FileNotFoundError: pass - if configloader.config['FFMPEG']['CopyUnprocessed']: - printer.info("Copying unprocessed files...") - utils.add_unprocessed_files(orig_folder) utils.get_compression_status(orig_folder) diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 71f571c..b1d22ea 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -30,13 +30,15 @@ def compress_audio(folder, file, target_folder, extension): try: (ffmpeg .input(f'{folder}/{file}') - .output(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}', audio_bitrate=bitrate) + .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), + audio_bitrate=bitrate) .run(quiet=True) ) - except ffmpeg._run.Error: + except ffmpeg._run.Error as e: + utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') utils.errors_count += 1 if not configloader.config['FFMPEG']['HideErrors']: - printer.error(f"File {file} can't be processed! Maybe it is ffmpeg error or unsupported file.") + printer.error(f"File {file} can't be processed! Error: {e}") return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' @@ -47,33 +49,39 @@ def compress_video(folder, file, target_folder, extension): try: (ffmpeg .input(f'{folder}/{file}') - .output(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}', vcodec=codec) + .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), vcodec=codec) .run(quiet=True) ) - except ffmpeg._run.Error: + except ffmpeg._run.Error as e: + utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') utils.errors_count += 1 if not configloader.config['FFMPEG']['HideErrors']: - printer.error(f"File {file} can't be processed! Maybe it is ffmpeg error or unsupported file.") + printer.error(f"File {file} can't be processed! Error: {e}") return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' def compress_image(folder, file, target_folder, extension): quality = configloader.config['IMAGE']['Quality'] - - image = Image.open(f'{folder}/{file}') - - if (extension == "jpg" or extension == "jpeg" or extension == "avif" or - (extension == "webp" and not configloader.config['FFMPEG']['WebpRGBA'])): - if has_transparency(image): - printer.warning(f"{file} has transparency. Changing to fallback...") - extension = configloader.config['IMAGE']['FallBackExtension'] - printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") - image.save(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}", - optimize=True, - lossless=configloader.config['IMAGE']['Lossless'], - quality=quality, - minimize_size=True) + try: + image = Image.open(f'{folder}/{file}') + + if (extension == "jpg" or extension == "jpeg" or extension == "avif" or + (extension == "webp" and not configloader.config['FFMPEG']['WebpRGBA'])): + if has_transparency(image): + printer.warning(f"{file} has transparency. Changing to fallback...") + extension = configloader.config['IMAGE']['FallBackExtension'] + + image.save(utils.check_duplicates(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}"), + optimize=True, + lossless=configloader.config['IMAGE']['Lossless'], + quality=quality, + minimize_size=True) + except Exception as e: + utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + utils.errors_count += 1 + if not configloader.config['FFMPEG']['HideErrors']: + printer.error(f"File {file} can't be processed! Error: {e}") return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' @@ -85,8 +93,9 @@ def compress(folder, file, target_folder): .output(f'{target_folder}/{file}') .run(quiet=True) ) - except ffmpeg._run.Error: + except ffmpeg._run.Error as e: + utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') utils.errors_count += 1 if not configloader.config['FFMPEG']['HideErrors']: - printer.error(f"File {file} can't be processed! Maybe it is ffmpeg error or unsupported file.") + printer.error(f"File {file} can't be processed! Error: {e}") return f'{target_folder}/{file}' diff --git a/FFMpeg-Compressor/modules/utils.py b/FFMpeg-Compressor/modules/utils.py index ffd6cff..226ecb5 100644 --- a/FFMpeg-Compressor/modules/utils.py +++ b/FFMpeg-Compressor/modules/utils.py @@ -53,22 +53,21 @@ def get_compression_status(orig_folder): get_compression(orig_folder, f"{orig_folder}_compressed") -def add_unprocessed_files(orig_folder): - for folder, folders, files in os.walk(orig_folder): - for file in files: - new_folder = f"{folder}".replace(orig_folder, f"{orig_folder}_compressed") - if len(glob(f"{folder}/{os.path.splitext(file)[0]}.*")) > 1: - if len(glob(f"{new_folder}/{file}")): - copyfile(f"{folder}/{file}", f"{new_folder}/{file} (copy)") - printer.warning( - f'Duplicate file has been found! Check manually this files - "{file}", "{file} (copy)"') - else: - copyfile(f"{folder}/{file}", f"{new_folder}/{file}") - printer.info(f"File {file} copied to compressed folder.") - else: - if not len(glob(f"{new_folder}/{os.path.splitext(file)[0]}.*")): - copyfile(f"{folder}/{file}", f"{new_folder}/{file}") - printer.info(f"File {file} copied to compressed folder.") +def add_unprocessed_file(orig_folder, new_folder): + if configloader.config['FFMPEG']['CopyUnprocessed']: + filename = orig_folder.split().pop() + copyfile(orig_folder, new_folder) + printer.info(f"File {filename} copied to compressed folder.") + + +def check_duplicates(new_folder): + filename = new_folder.split().pop() + if os.path.exists(new_folder): + printer.warning( + f'Duplicate file has been found! Check manually this files - "{filename}", ' + f'"{os.path.splitext(filename)[0] + "(copy)" + os.path.splitext(filename)[1]}"') + return os.path.splitext(new_folder)[0] + "(copy)" + os.path.splitext(new_folder)[1] + return new_folder def help_message(): From 7bcb74b70fe6776b47b8920ee94cb73cc424e80c Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sun, 7 Jan 2024 19:43:10 +0300 Subject: [PATCH 038/105] Add VNDS-to-RenPy script --- README.md | 1 + VNDS-to-RenPy/convert.py | 352 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 353 insertions(+) create mode 100644 VNDS-to-RenPy/convert.py diff --git a/README.md b/README.md index 584d873..36f092f 100644 --- a/README.md +++ b/README.md @@ -5,3 +5,4 @@ Collection of tools used by administrators from VN Telegram Channel * `FFMpeg-Compressor` - Python utility uses ffmpeg to compress Visual Novel Resources * `RenPy-Android-Unpack` - A simple Python script for unpacking Ren'Py based .apk and .obb files to ready to use Ren'Py SDK's Project * `RenPy-Unpacker` - Simple .rpy script that will make any RenPy game unpack itself +* `VNDS-to-RenPy` - Simple script for converting vnds scripts to rpy diff --git a/VNDS-to-RenPy/convert.py b/VNDS-to-RenPy/convert.py new file mode 100644 index 0000000..3a7cc54 --- /dev/null +++ b/VNDS-to-RenPy/convert.py @@ -0,0 +1,352 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python +# +# Automatically converts VNDS to Ren'Py. + +import os +import zipfile +from io import open + +# Sets of variables found in the game. +global_variables = set() +game_variables = set() + +def unjp(s): + print(s) + white = "QWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm1234567890" + for a in s: + if a not in white: + s = s.replace(a, str(ord(a))) + if s[0].isdigit(): + s = "var" + s + print(s) + return s + +def scan_gsetvar(l): + l = l.split()[0] + l = unjp(l) + global_variables.add(l) + +def scan_setvar(l): + if l[0] == "~": + return + + l = l.split()[0] + l = unjp(l) + game_variables.add(l) + +with zipfile.ZipFile("script.zip", 'r') as zip_ref: + zip_ref.extractall() + +def scan_script(infn): + inf = open("script/" + infn, encoding="utf-8") + + for l in inf: + l = l.replace("\xef\xbb\xbf", "") + l = l.strip() + + if not l: + continue + + if " " in l: + command, l = l.split(' ', 1) + else: + command = l + l = "" + + if "scan_" + command in globals(): + globals()["scan_" + command](l) + + inf.close() + + +class ConvertState(object): + + def __init__(self, outf, shortfn=""): + self.outf = outf + self.depth = 0 + self.shortfn = shortfn + self.textbuffer = "" + self.empty_block = False + self.imindex = 0 + + def write(self, s, *args): + + + self.outf.write(" " * self.depth + "\n") + + line = " " * self.depth + s % args + "\n" + + if isinstance(line, str): + line = line.encode("utf-8") + + self.outf.write(line.decode("utf-8")) + + self.empty_block = False + + def indent(self): + self.depth += 1 + + def outdent(self): + self.depth -= 1 + +def convert_endscript(cs, l): + cs.write("return") + +def convert_label(cs, l): + l = l.replace("*", "_star_") + l = l.replace("-", "_") + l = cs.shortfn + "_" + l + + cs.outdent() + cs.write("label %s:", l) + cs.indent() + +def convert_goto(cs, l): + l = l.replace("*", "_star_") + l = cs.shortfn + "_" + l + l = l.replace("-", "_") + + cs.write("jump %s", l) + +def convert_choice(cs, l): + choices = l.split("|") + + cs.write("menu:") + cs.indent() + + for i, c in enumerate(choices): + cs.write("%r:", c) + cs.indent() + cs.write("$ selected = %d", i + 1) + cs.outdent() + + cs.outdent() + +def convert_setvar(cs, l): + if l[0] == "~": + return + + print(f"l = {l}") + if l == 'chain = "main.scr start"': + var, op, val = "chain", "=", "main.scr start" + else: + var, op, val = l.split() + var = unjp(var) + + if op == "=": + cs.write("$ %s = %s", var, val) + elif op == "+": + cs.write("$ %s += %s", var, val) + elif op == "-": + cs.write("$ %s -= %s", var, val) + else: + raise Exception("Unknown operation " + op) + +def convert_gsetvar(cs, l): + if l[0] == "~": + return + + var, op, val = l.split() + var = unjp(var) + + if op == "=": + cs.write("$ persistent.%s = %s", var, val) + elif op == "+": + cs.write("$ persistent.%s += %s", var, val) + elif op == "-": + cs.write("$ persistent.%s -= %s", var, val) + else: + raise Exception("Unknown operation " + op) + +def convert_if(cs, l): + var, rest = l.strip().split(' ', 1) + var = unjp(var) + + if var in global_variables: + var = "persistent." + var + + cs.write("if %s %s:", var, rest) + cs.indent() + + cs.empty_block = True + +def convert_fi(cs, l): + if cs.empty_block: + cs.write("pass") + + cs.outdent() + +def convert_sound(cs, l): + if l == "~": + cs.write("stop sound") + else: + l = "sound/" + l + cs.write("play sound \"%s\"", l) + +def convert_setimg(cs, l): + fn, x, y = l.split() + x = int(x) + y = int(y) + fn = "foreground/" + fn + + cs.write("show expression %r as i%d at fgpos(%d, %d)", fn, cs.imindex, x, y) + cs.imindex += 1 + +def convert_delay(cs, l): + try: + t = int(l) / 60.0 + cs.write("pause %f", t) + except: + pass + + + +def convert_bgload(cs, fn): + + if " " in fn: + fn, delay = fn.split(" ", 1) + delay = int(delay) / 60.0 + else: + delay = 0.5 + + assert " " not in fn + + cs.write("nvl clear") + fn = "background/" + fn + cs.write("scene expression %r", fn) + cs.imindex = 0 + + if delay: + cs.write("with Dissolve(%f)", delay) + +def convert_jump(cs, fn): + if " " in fn: + fn, l = fn.split(" ", 1) + l = l.replace("*", "_star_") + l = l.replace("-", "_") + fn = fn.replace(".scr", "").replace("-", "_") + cs.write("jump %s_%s", fn, l) + return + + fn = fn.replace(".scr", "").replace("-", "_") + cs.write("jump %s", fn) + +def convert_music(cs, fn): + if " " in fn: + fn, loops = fn.split(" ", 1) + else: + loops = 0 + + if int(loops) > 0: + noloop = " noloop" + else: + noloop = "" + + if fn == "~": + cs.write("stop music") + else: + fn = "sound/" + fn + cs.write("play music %r%s", fn, noloop) + +def convert_text(cs, text): + + while text and (text[0] == "~" or text[0] == "!"): + text = text[1:] + + if not text: + return + + if text[0] == "@": + cs.textbuffer += text[1:] + "\n" + return + + text = cs.textbuffer + text + text = text.replace("\\", "\\\\") + text = text.replace("\"", "\\\"") + text = text.replace("\n", "\\n") + + cs.write('"%s"', text) + cs.textbuffer = "" + +def convert_script(infn): + dir_rp = "rpy/" + if not os.path.exists(dir_rp): + os.mkdir(dir_rp) + + if os.path.exists(dir_rp): + shortfn = infn.replace(".scr", "") + shortfn = shortfn.replace("-", "_") + inf = open("script/" + infn, encoding="utf-8") + outf = open (dir_rp + shortfn + ".rpy", "w", encoding = "utf-8") + cs = ConvertState(outf, shortfn) + cs.write("label %s:", shortfn) + cs.indent() + + for l in inf: + l = l.replace("\xef\xbb\xbf", "") + l = l.strip() + + if not l: + continue + + if l[0] == "#": + continue + + if " " in l: + command, l = l.split(' ', 1) + else: + command = l + l = "" + + if "convert_" + command in globals(): + globals()["convert_" + command](cs, l) + else: + print("Unknown command", repr(command), repr(l), repr(infn)) + + outf.close() + +def main(): + for i in os.listdir("script"): + if not i.endswith(".scr"): + continue + + scan_script(i) + + for i in os.listdir("script"): + if not i.endswith(".scr"): + continue + + convert_script(i) + + outf = open("rpy/_start.rpy", "w", encoding="utf-8") + cs = ConvertState(outf) + cs.write("init python:") + cs.indent() + + for i in global_variables: + i = unjp(i) + cs.write("if persistent.%s is None:", i) + cs.indent() + cs.write("persistent.%s = 0", i) + cs.outdent() + + # prevent non-empty block. + cs.write("pass") + + cs.outdent() + cs.write("label start:") + cs.indent() + + for i in game_variables: + i = unjp(i) + cs.write("$ %s = 0", i) + + cs.write("window show") + cs.write("jump main") + + cs.outdent() + outf.close() + +if __name__ == "__main__": + main() From e4f028f75b9343b49b9a487d4a6696d7d40ca281 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sun, 7 Jan 2024 20:13:03 +0300 Subject: [PATCH 039/105] VNDS-to-RenPy: Add README.md --- VNDS-to-RenPy/README.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 VNDS-to-RenPy/README.md diff --git a/VNDS-to-RenPy/README.md b/VNDS-to-RenPy/README.md new file mode 100644 index 0000000..c2ba3df --- /dev/null +++ b/VNDS-to-RenPy/README.md @@ -0,0 +1,7 @@ +## VNDS-to-RenPy +Simple script for converting vnds scripts to rpy + +### How to use + +* Extract VNDS visual novel's archive and get `scripts` folder from these (or `script.zip` if scripts folder is empty) +* Launch `convert.py` (It will automatically extract `scripts.zip` archive (if it needed) and converts .scr scripts to .rpy) From 07e9b09b5c2f5daf409b20d182817032111be1df Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 12 Jan 2024 17:38:26 +0300 Subject: [PATCH 040/105] FFMpeg-Compressor: Fix main.py running --- FFMpeg-Compressor/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py index 8e68e42..6402410 100755 --- a/FFMpeg-Compressor/main.py +++ b/FFMpeg-Compressor/main.py @@ -1,4 +1,4 @@ -#!python3 +#!/bin/env python3 from modules import configloader from modules import compressor From ee88780a9f714a0973591f3c748a360203217da6 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 12 Jan 2024 20:56:04 +0300 Subject: [PATCH 041/105] Add Windows build script, improve build scripts --- .gitignore | 1 + FFMpeg-Compressor/main.py | 2 +- build.bat | 11 +++++++++++ build.sh | 9 ++++++--- 4 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 .gitignore create mode 100644 build.bat diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..16be8f2 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/output/ diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py index 6402410..54f6c1d 100755 --- a/FFMpeg-Compressor/main.py +++ b/FFMpeg-Compressor/main.py @@ -1,4 +1,4 @@ -#!/bin/env python3 +#!env python3 from modules import configloader from modules import compressor diff --git a/build.bat b/build.bat new file mode 100644 index 0000000..f3dd3f2 --- /dev/null +++ b/build.bat @@ -0,0 +1,11 @@ +@Echo off +mkdir output +mkdir output\bin +python -m pip install -r FFMpeg-Compressor\requirements.txt +python -m pip install Nuitka +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=ffmpeg-comp FFMpeg-Compressor\main.py +xcopy FFMpeg-Compressor\ffmpeg-comp.toml output\bin +move output\ffmpeg-comp.exe output\bin +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=rendroid-unpack RenPy-Android-Unpack\unpack.py +move output\rendroid-unpack.exe output\bin +pause \ No newline at end of file diff --git a/build.sh b/build.sh index 47d5a09..6f5726e 100755 --- a/build.sh +++ b/build.sh @@ -1,11 +1,14 @@ #!/bin/bash mkdir output mkdir output/bin +python3 -m pip install -r FFMpeg-Compressor/requirements.txt python3 -m pip install Nuitka case "$(uname -s)" in Linux*) jobs="--jobs=$(nproc)";; Darwin*) jobs="--jobs=$(sysctl -n hw.ncpu)";; esac -nuitka3 "${jobs}" --output-dir=output --follow-imports --output-filename=output/bin/ffmpeg-comp FFMpeg-Compressor/main.py -cp FFMpeg-Compressor/ffmpeg-comp.toml output/bin/ -nuitka3 "${jobs}" --output-dir=output --follow-imports --output-filename=output/bin/rendroid-unpack RenPy-Android-Unpack/unpack.py \ No newline at end of file +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=ffmpeg-comp FFMpeg-Compressor/main.py +cp FFMpeg-Compressor/ffmpeg-comp.toml output/bin +mv output/ffmpeg-comp output/bin +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=rendroid-unpack RenPy-Android-Unpack/unpack.py +mv output/rendroid-unpack output/bin \ No newline at end of file From 9b98fa72f9e238950775a4714df6654654ad8343 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 12 Jan 2024 21:12:03 +0300 Subject: [PATCH 042/105] FFMpeg-Compressor: Update README.md --- FFMpeg-Compressor/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index 4ce9e0b..8a7295e 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -30,7 +30,7 @@ Python utility uses ffmpeg to compress Visual Novel Resources ### TODO (for testing branch) * [x] Recreate whole game directory with compressed files -* [ ] Cross platform (Easy Windows usage and binaries, MacOS binaries) +* [x] Cross platform (Easy Windows usage and binaries, MacOS binaries) * [x] Use ffmpeg python bindings instead of cli commands * [ ] Reorganize code * [ ] Multithread \ No newline at end of file From b9c7b512de95888f7752efe80f59856f5ee432df Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 12 Jan 2024 21:58:34 +0300 Subject: [PATCH 043/105] Exit on build error --- build.bat | 14 +++++++++----- build.sh | 5 +++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/build.bat b/build.bat index f3dd3f2..5727098 100644 --- a/build.bat +++ b/build.bat @@ -1,11 +1,15 @@ @Echo off mkdir output mkdir output\bin -python -m pip install -r FFMpeg-Compressor\requirements.txt -python -m pip install Nuitka -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=ffmpeg-comp FFMpeg-Compressor\main.py +python -m pip install -r FFMpeg-Compressor\requirements.txt || goto :exit +python -m pip install Nuitka || goto :exit +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=ffmpeg-comp FFMpeg-Compressor\main.py || goto :exit xcopy FFMpeg-Compressor\ffmpeg-comp.toml output\bin move output\ffmpeg-comp.exe output\bin -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=rendroid-unpack RenPy-Android-Unpack\unpack.py +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=rendroid-unpack RenPy-Android-Unpack\unpack.py || goto :exit move output\rendroid-unpack.exe output\bin -pause \ No newline at end of file +pause + +:exit +pause +exit /b %exitlevel% \ No newline at end of file diff --git a/build.sh b/build.sh index 6f5726e..213cb1b 100755 --- a/build.sh +++ b/build.sh @@ -1,6 +1,7 @@ #!/bin/bash -mkdir output -mkdir output/bin +sed -e +mkdir -p output +mkdir -p output/bin python3 -m pip install -r FFMpeg-Compressor/requirements.txt python3 -m pip install Nuitka case "$(uname -s)" in From dc858b4ebbce1fbcdb6c995f1a7b303435036c25 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 12 Jan 2024 23:12:19 +0300 Subject: [PATCH 044/105] Overwrite binaries in output folder --- build.bat | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/build.bat b/build.bat index 5727098..a0b952c 100644 --- a/build.bat +++ b/build.bat @@ -4,11 +4,10 @@ mkdir output\bin python -m pip install -r FFMpeg-Compressor\requirements.txt || goto :exit python -m pip install Nuitka || goto :exit python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=ffmpeg-comp FFMpeg-Compressor\main.py || goto :exit -xcopy FFMpeg-Compressor\ffmpeg-comp.toml output\bin -move output\ffmpeg-comp.exe output\bin +xcopy FFMpeg-Compressor\ffmpeg-comp.toml output\bin /Y +move /Y output\ffmpeg-comp.exe output\bin python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=rendroid-unpack RenPy-Android-Unpack\unpack.py || goto :exit -move output\rendroid-unpack.exe output\bin -pause +move /Y output\rendroid-unpack.exe output\bin :exit pause From 7ed04ffd22dfbc9485bc05bbbbf12ab545e82399 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 12 Jan 2024 23:38:07 +0300 Subject: [PATCH 045/105] FFMpeg-Compressor: Windows cmd support --- FFMpeg-Compressor/main.py | 6 ++++-- FFMpeg-Compressor/modules/printer.py | 9 ++++++++- FFMpeg-Compressor/modules/utils.py | 7 ++++++- FFMpeg-Compressor/requirements.txt | 1 + 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py index 54f6c1d..ca91c75 100755 --- a/FFMpeg-Compressor/main.py +++ b/FFMpeg-Compressor/main.py @@ -26,6 +26,7 @@ def get_file_type(filename): if __name__ == "__main__": + utils.win_ascii_esc() try: if sys.argv[1][len(sys.argv[1])-1] == "/": arg_path = sys.argv[1][:len(sys.argv[1])-1] @@ -35,8 +36,8 @@ if __name__ == "__main__": print(utils.help_message()) exit() - orig_folder = arg_path - printer.orig_folder = arg_path + orig_folder = os.path.abspath(arg_path) + printer.orig_folder = os.path.abspath(arg_path) printer.bar_init(orig_folder) @@ -72,3 +73,4 @@ if __name__ == "__main__": pass utils.get_compression_status(orig_folder) + utils.sys_pause() diff --git a/FFMpeg-Compressor/modules/printer.py b/FFMpeg-Compressor/modules/printer.py index a473dc0..f82b2e2 100644 --- a/FFMpeg-Compressor/modules/printer.py +++ b/FFMpeg-Compressor/modules/printer.py @@ -1,5 +1,7 @@ -import os from progress.bar import IncrementalBar +import colorama +import sys +import os # Fill whole string with spaces for cleaning progress bar @@ -38,3 +40,8 @@ def files(source, dest, dest_ext, comment): def unknown_file(file): print(clean_str(f"\r* \033[0;33m{file}\033[0m (Not recognized)")) bar.next() + + +def win_ascii_esc(): + if sys.platform == "win32": + colorama.init() diff --git a/FFMpeg-Compressor/modules/utils.py b/FFMpeg-Compressor/modules/utils.py index 226ecb5..f1a3256 100644 --- a/FFMpeg-Compressor/modules/utils.py +++ b/FFMpeg-Compressor/modules/utils.py @@ -1,7 +1,7 @@ from modules import configloader from modules import printer from shutil import copyfile -from glob import glob +import sys import os errors_count = 0 @@ -70,5 +70,10 @@ def check_duplicates(new_folder): return new_folder +def sys_pause(): + if sys.platform == "win32": + os.system("pause") + + def help_message(): return "Usage: ffmpeg-comp {folder}" diff --git a/FFMpeg-Compressor/requirements.txt b/FFMpeg-Compressor/requirements.txt index 8d1f56a..88b8dd7 100644 --- a/FFMpeg-Compressor/requirements.txt +++ b/FFMpeg-Compressor/requirements.txt @@ -2,3 +2,4 @@ Pillow==9.5.0 pillow-avif-plugin==1.4.1 ffmpeg-python==0.2.0 progress==1.6 +colorama==0.4.6 \ No newline at end of file From 775b5c539f5d6c34b35cea8a266c603dd71101f0 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 12 Jan 2024 23:58:58 +0300 Subject: [PATCH 046/105] Compile VNDS-to-RenPy binary --- build.bat | 2 ++ build.sh | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/build.bat b/build.bat index a0b952c..8145028 100644 --- a/build.bat +++ b/build.bat @@ -8,6 +8,8 @@ xcopy FFMpeg-Compressor\ffmpeg-comp.toml output\bin /Y move /Y output\ffmpeg-comp.exe output\bin python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=rendroid-unpack RenPy-Android-Unpack\unpack.py || goto :exit move /Y output\rendroid-unpack.exe output\bin +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=vnds2renpy VNDS-to-RenPy/convert.py || goto :exit +move /Y output\vnds2renpy.exe output\bin :exit pause diff --git a/build.sh b/build.sh index 213cb1b..df02940 100755 --- a/build.sh +++ b/build.sh @@ -1,5 +1,5 @@ #!/bin/bash -sed -e +set -e mkdir -p output mkdir -p output/bin python3 -m pip install -r FFMpeg-Compressor/requirements.txt @@ -12,4 +12,6 @@ python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --out cp FFMpeg-Compressor/ffmpeg-comp.toml output/bin mv output/ffmpeg-comp output/bin python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=rendroid-unpack RenPy-Android-Unpack/unpack.py -mv output/rendroid-unpack output/bin \ No newline at end of file +mv output/rendroid-unpack output/bin +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=vnds2renpy VNDS-to-RenPy/convert.py +mv output/vnds2renpy output/bin \ No newline at end of file From c1b57bddf2c40931a1311282ec02efa3353e85a7 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Mon, 15 Jan 2024 20:59:38 +0300 Subject: [PATCH 047/105] Fix typo --- FFMpeg-Compressor/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py index ca91c75..09e2741 100755 --- a/FFMpeg-Compressor/main.py +++ b/FFMpeg-Compressor/main.py @@ -26,7 +26,7 @@ def get_file_type(filename): if __name__ == "__main__": - utils.win_ascii_esc() + printer.win_ascii_esc() try: if sys.argv[1][len(sys.argv[1])-1] == "/": arg_path = sys.argv[1][:len(sys.argv[1])-1] From 248f08c7d9bd28c28194f214c788c3ae5d8c6077 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Mon, 15 Jan 2024 22:20:23 +0300 Subject: [PATCH 048/105] FFMpeg-Compressor: Add ForceCompress parameter --- FFMpeg-Compressor/README.md | 1 + FFMpeg-Compressor/ffmpeg-comp.toml | 1 + FFMpeg-Compressor/modules/compressor.py | 25 ++++++++++++++----------- FFMpeg-Compressor/modules/printer.py | 2 +- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index 8a7295e..e1f025c 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -10,6 +10,7 @@ Python utility uses ffmpeg to compress Visual Novel Resources ### Configuration #### FFMPEG section * CopyUnprocessed - Copy all files that failed to compress by ffmpeg to destination folder. In can helps to recreate original folder, but with compressed files. +* ForceCompress - Force try to compress all files in directory via ffmpeg. (default: `false`) * MimicMode - Rename compressed file to it original name and extension. VN engines determine the file type by its header, so for example PNG file named file.jpg will be loaded as PNG file. (default: `false`) * HideErrors - Hide some errors about compression. (default: `false`) * WebpRGBA - Alpha channel in webp. If false switches extension to png. (default: `true`) diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index 3840e5a..2f14653 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -1,5 +1,6 @@ [FFMPEG] CopyUnprocessed = false +ForceCompress = false MimicMode = false HideErrors = false WebpRGBA = true diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index b1d22ea..3fc67af 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -86,16 +86,19 @@ def compress_image(folder, file, target_folder, extension): def compress(folder, file, target_folder): - printer.unknown_file(file) - try: - (ffmpeg - .input(f'{folder}/{file}') - .output(f'{target_folder}/{file}') - .run(quiet=True) - ) - except ffmpeg._run.Error as e: + if configloader.config["FFMPEG"]["ForceCompress"]: + printer.unknown_file(file) + try: + (ffmpeg + .input(f'{folder}/{file}') + .output(f'{target_folder}/{file}') + .run(quiet=True) + ) + except ffmpeg._run.Error as e: + utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + utils.errors_count += 1 + if not configloader.config['FFMPEG']['HideErrors']: + printer.error(f"File {file} can't be processed! Error: {e}") + else: utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - utils.errors_count += 1 - if not configloader.config['FFMPEG']['HideErrors']: - printer.error(f"File {file} can't be processed! Error: {e}") return f'{target_folder}/{file}' diff --git a/FFMpeg-Compressor/modules/printer.py b/FFMpeg-Compressor/modules/printer.py index f82b2e2..4c99ced 100644 --- a/FFMpeg-Compressor/modules/printer.py +++ b/FFMpeg-Compressor/modules/printer.py @@ -38,7 +38,7 @@ def files(source, dest, dest_ext, comment): def unknown_file(file): - print(clean_str(f"\r* \033[0;33m{file}\033[0m (Not recognized)")) + print(clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed wia ffmpeg)")) bar.next() From 20f86f7c0e87bb0b20ab1c95b3c5b7e146347e35 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 3 Feb 2024 22:45:47 +0300 Subject: [PATCH 049/105] RenPy-Android-Unpack: Some improvements --- RenPy-Android-Unpack/unpack.py | 83 +++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 31 deletions(-) diff --git a/RenPy-Android-Unpack/unpack.py b/RenPy-Android-Unpack/unpack.py index 7a9777a..b8d6d05 100755 --- a/RenPy-Android-Unpack/unpack.py +++ b/RenPy-Android-Unpack/unpack.py @@ -1,23 +1,42 @@ #!python3 +import colorama import zipfile -import os import shutil +import os +import sys + + +def printer(msg, level): + match level: + case "info": + print(f"\033[100m[INFO] {msg}\033[49m") + case "warn": + print(f"\033[93m[WARN]\033[0m {msg}\033[49m") + case "err": + print(f"\033[31m[ERROR]\033[0m {msg} Exiting...\033[49m") + exit() def extract_assets(file): - with zipfile.ZipFile(file, 'r') as zip_ref: - for content in zip_ref.namelist(): - if content.split('/')[0] == 'assets': - zip_ref.extract(content) - if os.path.splitext(file)[1] == '.apk': - try: - zip_ref.extract('res/mipmap-xxxhdpi-v4/icon_background.png', 'assets') - zip_ref.extract('res/mipmap-xxxhdpi-v4/icon_foreground.png', 'assets') - os.rename('assets/res/mipmap-xxxhdpi-v4/icon_background.png', 'assets/android-icon_background.png') - os.rename('assets/res/mipmap-xxxhdpi-v4/icon_foreground.png', 'assets/android-icon_foreground.png') - except KeyError: - zip_ref.extract('res/drawable/icon.png', 'assets') - os.rename('assets/res/drawable/icon.png', 'assets/icon.png') + try: + with zipfile.ZipFile(file, 'r') as zip_ref: + for content in zip_ref.namelist(): + if content.split('/')[0] == 'assets': + zip_ref.extract(content) + if os.path.splitext(file)[1] == '.apk': + try: + zip_ref.extract('res/mipmap-xxxhdpi-v4/icon_background.png', 'assets') + zip_ref.extract('res/mipmap-xxxhdpi-v4/icon_foreground.png', 'assets') + os.rename('assets/res/mipmap-xxxhdpi-v4/icon_background.png', 'assets/android-icon_background.png') + os.rename('assets/res/mipmap-xxxhdpi-v4/icon_foreground.png', 'assets/android-icon_foreground.png') + except KeyError: + try: + zip_ref.extract('res/drawable/icon.png', 'assets') + os.rename('assets/res/drawable/icon.png', 'assets/icon.png') + except KeyError: + printer("Icon not found. Maybe it is not supported apk?", "warn") + except zipfile.BadZipFile: + return printer("Cant extract .apk file!", "err") def rename_files(directory): @@ -41,27 +60,29 @@ def rename_dirs(directory): os.rename(dir__, f'{folder}/{newname}') +def remove_unneeded(names, ignore): + for name in names: + try: + shutil.rmtree(name) + except FileNotFoundError: + if not ignore: + printer(f"Path {name} not found!", "warn") + + if __name__ == '__main__': + if sys.platform == "win32": + colorama.init() for filename in os.listdir(os.getcwd()): - renpy_warn = 0 if os.path.splitext(filename)[1] == '.apk' or os.path.splitext(filename)[1] == '.obb': - print(f'[INFO] Extracting assets from {filename}... ', end='') + remove_unneeded(['assets'], True) + printer(f'Extracting assets from {filename}... ', "info") extract_assets(filename) - print('Done') - print('[INFO] Renaming game assets... ', end='') + printer('Renaming game assets... ', "info") rename_files('assets') rename_dirs('assets') - print('Done') - print('[INFO] Removing unneeded files... ', end='') - try: - shutil.rmtree('assets/renpy') - except FileNotFoundError: - renpy_warn = 1 + printer('Removing unneeded files... ', "info") if os.path.splitext(filename)[1] == '.apk': - shutil.rmtree('assets/res') - print('Done') - if renpy_warn: - print("[WARN] File does not contain renpy folder!") - print('[INFO] Renaming directory... ', end='') - os.rename('assets', f'{os.path.splitext(filename)[0]}') - print('Done') + remove_unneeded(['assets/renpy', 'assets/res', 'assets/dexopt'], False) + printer('Renaming directory... ', "info") + remove_unneeded([os.path.splitext(filename)[0]], True) + os.rename('assets', os.path.splitext(filename)[0]) From dbf627d15ebfeb5643df9c495aefbb9dab1fe2e8 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 3 Feb 2024 23:39:00 +0300 Subject: [PATCH 050/105] RenPy-Android-Unpack: Newest Ren'Py support --- RenPy-Android-Unpack/requirements.txt | 2 ++ RenPy-Android-Unpack/unpack.py | 36 +++++++++++++++++++++------ 2 files changed, 30 insertions(+), 8 deletions(-) create mode 100644 RenPy-Android-Unpack/requirements.txt diff --git a/RenPy-Android-Unpack/requirements.txt b/RenPy-Android-Unpack/requirements.txt new file mode 100644 index 0000000..3a6b560 --- /dev/null +++ b/RenPy-Android-Unpack/requirements.txt @@ -0,0 +1,2 @@ +Pillow==9.5.0 +colorama==0.4.6 \ No newline at end of file diff --git a/RenPy-Android-Unpack/unpack.py b/RenPy-Android-Unpack/unpack.py index b8d6d05..2dfaecd 100755 --- a/RenPy-Android-Unpack/unpack.py +++ b/RenPy-Android-Unpack/unpack.py @@ -1,4 +1,5 @@ #!python3 +from PIL import Image import colorama import zipfile import shutil @@ -17,20 +18,38 @@ def printer(msg, level): exit() +def extract_folder(zip_ref, path, dest): + for content in zip_ref.namelist(): + if content.split('/')[0] == path: + zip_ref.extract(content, dest) + + +def find_modern_icon(directory): + icons = [] + for folder, folders, files in os.walk(directory): + for file in os.listdir(folder): + if os.path.splitext(file)[1] == ".png": + image = Image.open(f"{folder}/{file}") + if image.size[0] == 432 and image.size[1] == 432: + icons.append(f"{folder}/{file}") + if len(icons) == 0: + raise KeyError + return icons + + def extract_assets(file): try: with zipfile.ZipFile(file, 'r') as zip_ref: - for content in zip_ref.namelist(): - if content.split('/')[0] == 'assets': - zip_ref.extract(content) + extract_folder(zip_ref, 'assets', '') if os.path.splitext(file)[1] == '.apk': try: - zip_ref.extract('res/mipmap-xxxhdpi-v4/icon_background.png', 'assets') - zip_ref.extract('res/mipmap-xxxhdpi-v4/icon_foreground.png', 'assets') - os.rename('assets/res/mipmap-xxxhdpi-v4/icon_background.png', 'assets/android-icon_background.png') - os.rename('assets/res/mipmap-xxxhdpi-v4/icon_foreground.png', 'assets/android-icon_foreground.png') + # ~Ren'Py 8, 7 + extract_folder(zip_ref, 'res', 'assets') + for icon in find_modern_icon('assets/res'): + os.rename(icon, f"assets/{os.path.split(icon)[1]}") except KeyError: try: + # ~Ren'Py 6 zip_ref.extract('res/drawable/icon.png', 'assets') os.rename('assets/res/drawable/icon.png', 'assets/icon.png') except KeyError: @@ -82,7 +101,8 @@ if __name__ == '__main__': rename_dirs('assets') printer('Removing unneeded files... ', "info") if os.path.splitext(filename)[1] == '.apk': - remove_unneeded(['assets/renpy', 'assets/res', 'assets/dexopt'], False) + remove_unneeded(['assets/renpy', 'assets/res'], False) + remove_unneeded(['assets/dexopt'], True) printer('Renaming directory... ', "info") remove_unneeded([os.path.splitext(filename)[0]], True) os.rename('assets', os.path.splitext(filename)[0]) From 01d99b818fb21eb1e11311a9fafed67ab0de5fdd Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 3 Feb 2024 23:54:50 +0300 Subject: [PATCH 051/105] Update build script --- build.bat | 1 + build.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/build.bat b/build.bat index 8145028..def90d6 100644 --- a/build.bat +++ b/build.bat @@ -2,6 +2,7 @@ mkdir output mkdir output\bin python -m pip install -r FFMpeg-Compressor\requirements.txt || goto :exit +python -m pip install -r RenPy-Android-Unpack\requirements.txt || goto :exit python -m pip install Nuitka || goto :exit python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=ffmpeg-comp FFMpeg-Compressor\main.py || goto :exit xcopy FFMpeg-Compressor\ffmpeg-comp.toml output\bin /Y diff --git a/build.sh b/build.sh index df02940..53caada 100755 --- a/build.sh +++ b/build.sh @@ -3,6 +3,7 @@ set -e mkdir -p output mkdir -p output/bin python3 -m pip install -r FFMpeg-Compressor/requirements.txt +python3 -m pip install -r RenPy-Android-Unpack/requirements.txt python3 -m pip install Nuitka case "$(uname -s)" in Linux*) jobs="--jobs=$(nproc)";; From f39c28168bb5e0bf1ed1c1f59a747a95f02f4280 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 3 Feb 2024 23:58:40 +0300 Subject: [PATCH 052/105] Multiplatform shebangs --- FFMpeg-Compressor/main.py | 2 +- RenPy-Android-Unpack/unpack.py | 2 +- VNDS-to-RenPy/convert.py | 23 +++++++++++------------ 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py index 09e2741..85f2322 100755 --- a/FFMpeg-Compressor/main.py +++ b/FFMpeg-Compressor/main.py @@ -1,4 +1,4 @@ -#!env python3 +#!/usr/bin/env python3 from modules import configloader from modules import compressor diff --git a/RenPy-Android-Unpack/unpack.py b/RenPy-Android-Unpack/unpack.py index 2dfaecd..8b74db7 100755 --- a/RenPy-Android-Unpack/unpack.py +++ b/RenPy-Android-Unpack/unpack.py @@ -1,4 +1,4 @@ -#!python3 +#!/usr/bin/env python3 from PIL import Image import colorama import zipfile diff --git a/VNDS-to-RenPy/convert.py b/VNDS-to-RenPy/convert.py index 3a7cc54..07a58f7 100644 --- a/VNDS-to-RenPy/convert.py +++ b/VNDS-to-RenPy/convert.py @@ -1,5 +1,4 @@ -# -*- coding: utf-8 -*- -#!/usr/bin/env python +#!/usr/bin/env python3 # # Automatically converts VNDS to Ren'Py. @@ -37,7 +36,7 @@ def scan_setvar(l): with zipfile.ZipFile("script.zip", 'r') as zip_ref: zip_ref.extractall() - + def scan_script(infn): inf = open("script/" + infn, encoding="utf-8") @@ -108,7 +107,7 @@ def convert_goto(cs, l): l = l.replace("-", "_") cs.write("jump %s", l) - + def convert_choice(cs, l): choices = l.split("|") @@ -158,7 +157,7 @@ def convert_gsetvar(cs, l): cs.write("$ persistent.%s -= %s", var, val) else: raise Exception("Unknown operation " + op) - + def convert_if(cs, l): var, rest = l.strip().split(' ', 1) var = unjp(var) @@ -170,7 +169,7 @@ def convert_if(cs, l): cs.indent() cs.empty_block = True - + def convert_fi(cs, l): if cs.empty_block: cs.write("pass") @@ -192,7 +191,7 @@ def convert_setimg(cs, l): cs.write("show expression %r as i%d at fgpos(%d, %d)", fn, cs.imindex, x, y) cs.imindex += 1 - + def convert_delay(cs, l): try: t = int(l) / 60.0 @@ -200,8 +199,8 @@ def convert_delay(cs, l): except: pass - - + + def convert_bgload(cs, fn): if " " in fn: @@ -219,7 +218,7 @@ def convert_bgload(cs, fn): if delay: cs.write("with Dissolve(%f)", delay) - + def convert_jump(cs, fn): if " " in fn: fn, l = fn.split(" ", 1) @@ -231,7 +230,7 @@ def convert_jump(cs, fn): fn = fn.replace(".scr", "").replace("-", "_") cs.write("jump %s", fn) - + def convert_music(cs, fn): if " " in fn: fn, loops = fn.split(" ", 1) @@ -248,7 +247,7 @@ def convert_music(cs, fn): else: fn = "sound/" + fn cs.write("play music %r%s", fn, noloop) - + def convert_text(cs, text): while text and (text[0] == "~" or text[0] == "!"): From 7c355e38f7868d5417581b5b3deb8b14cf8fd78b Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 23 Feb 2024 00:23:16 +0300 Subject: [PATCH 053/105] FFMpeg-Compressor: New config default parameters --- FFMpeg-Compressor/README.md | 8 ++++---- FFMpeg-Compressor/ffmpeg-comp.toml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index e1f025c..56b8ff4 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -9,15 +9,15 @@ Python utility uses ffmpeg to compress Visual Novel Resources ### Configuration #### FFMPEG section -* CopyUnprocessed - Copy all files that failed to compress by ffmpeg to destination folder. In can helps to recreate original folder, but with compressed files. +* CopyUnprocessed - Copy all files that failed to compress by ffmpeg to destination folder. In can helps to recreate original folder, but with compressed files. (default: `true`) * ForceCompress - Force try to compress all files in directory via ffmpeg. (default: `false`) -* MimicMode - Rename compressed file to it original name and extension. VN engines determine the file type by its header, so for example PNG file named file.jpg will be loaded as PNG file. (default: `false`) -* HideErrors - Hide some errors about compression. (default: `false`) +* MimicMode - Rename compressed file to it original name and extension. VN engines determine the file type by its header, so for example PNG file named file.jpg will be loaded as PNG file. (default: `true`) +* HideErrors - Hide some errors about compression. (default: `true`) * WebpRGBA - Alpha channel in webp. If false switches extension to png. (default: `true`) #### AUDIO section * Extension - Required audio file extension. It supports: `.aac`, `.flac`, `.m4a`, `.mp3`, `.ogg`, `.opus`, `.raw`, `.wav`, `.wma`. -* BitRate - (mp3 only, for now) Required audio bitrate. For best quality use `320k` value, but for worse use `1-9` (9 worst) number range. +* BitRate - Required audio bitrate. For best quality use `320k` value. #### IMAGE section * Extension - Required image file extension. It supports: `.apng`, `.avif`, `.bmp`, `.tga`, `.tiff`, `.dds`, `.svg`, `.webp`, `.jpg/.jpeg`, `.png` diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index 2f14653..f8c0551 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -1,13 +1,13 @@ [FFMPEG] -CopyUnprocessed = false +CopyUnprocessed = true ForceCompress = false -MimicMode = false -HideErrors = false +MimicMode = true +HideErrors = true WebpRGBA = true [AUDIO] Extension = "opus" -BitRate = "320k" +BitRate = "128k" [IMAGE] Extension = "avif" From f9f6afe95a46d4d3e4ce10162c5a29a041255688 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 12 Apr 2024 17:19:53 +0300 Subject: [PATCH 054/105] FFMpeg-Compressor: Add resolution scaledown --- FFMpeg-Compressor/ffmpeg-comp.toml | 5 +++-- FFMpeg-Compressor/modules/compressor.py | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index f8c0551..51a4866 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -10,10 +10,11 @@ Extension = "opus" BitRate = "128k" [IMAGE] +ResDownScale = 1 Extension = "avif" FallBackExtension = "webp" -Lossless = false -Quality = 80 +Lossless = true +Quality = 100 [VIDEO] Extension = "webm" diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 3fc67af..48b4d92 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -66,12 +66,17 @@ def compress_image(folder, file, target_folder, extension): try: image = Image.open(f'{folder}/{file}') + width, height = image.size + res_downscale = configloader.config['IMAGE']['ResDownScale'] + new_size = (int(width / res_downscale), int(height / res_downscale)) + if (extension == "jpg" or extension == "jpeg" or extension == "avif" or (extension == "webp" and not configloader.config['FFMPEG']['WebpRGBA'])): if has_transparency(image): printer.warning(f"{file} has transparency. Changing to fallback...") extension = configloader.config['IMAGE']['FallBackExtension'] + image = image.resize(new_size) image.save(utils.check_duplicates(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}"), optimize=True, lossless=configloader.config['IMAGE']['Lossless'], From a973fe79e81a48f9dd0fe6e1544e02ba9a48ef68 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Mon, 22 Apr 2024 23:23:06 +0300 Subject: [PATCH 055/105] FFMpeg-Compressor: Add SkipVideo parameter --- FFMpeg-Compressor/README.md | 12 ++++++----- FFMpeg-Compressor/ffmpeg-comp.toml | 1 + FFMpeg-Compressor/modules/compressor.py | 27 ++++++++++++++----------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index 56b8ff4..4d04f2f 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -5,11 +5,11 @@ Python utility uses ffmpeg to compress Visual Novel Resources * Download `ffmpeg-comp.toml` and put in next to binary or in to `/etc` folder * Change the configuration of the utility in `ffmpeg-comp.toml` for yourself * `ffmpeg-comp {folder}` -* In result you get `{folder-compressed}` near with original `{folder}` +* In result, you get `{folder-compressed}` near with original `{folder}` ### Configuration #### FFMPEG section -* CopyUnprocessed - Copy all files that failed to compress by ffmpeg to destination folder. In can helps to recreate original folder, but with compressed files. (default: `true`) +* CopyUnprocessed - Copy all files that failed to compress by ffmpeg to destination folder. In can help to recreate original folder, but with compressed files. (default: `true`) * ForceCompress - Force try to compress all files in directory via ffmpeg. (default: `false`) * MimicMode - Rename compressed file to it original name and extension. VN engines determine the file type by its header, so for example PNG file named file.jpg will be loaded as PNG file. (default: `true`) * HideErrors - Hide some errors about compression. (default: `true`) @@ -20,18 +20,20 @@ Python utility uses ffmpeg to compress Visual Novel Resources * BitRate - Required audio bitrate. For best quality use `320k` value. #### IMAGE section +* ResDownScale - Downscale image resolution count. (default: `1`) * Extension - Required image file extension. It supports: `.apng`, `.avif`, `.bmp`, `.tga`, `.tiff`, `.dds`, `.svg`, `.webp`, `.jpg/.jpeg`, `.png` * FallBackExtension - Extension if current format does not support RGBA. -* Lossless - Enables lossless copression for supported formats. With this quality parameter means quality of compression. (default: `false`) +* Lossless - Enables lossless compression for supported formats. With this quality parameter means quality of compression. (default: `false`) * Quality - Quality level of images. Values range: `0-100` (100 - best quality, 0 - worst quality) #### VIDEO section +* SkipVideo - Skip processing all video files. (default: `false`) * Extension - Required image file extension. It supports: `.3gp`, `.amv`, `.avi`, `.gif`, `.m2l`, `.m4v`, `.mkv`, `.mov`, `.mp4`, `.m4v`, `.mpeg`, `.mpv`, `.webm`, `.ogv` -* Codec - (May be optional in future) Required video codec. (See official ffmpeg documentation for supported codecs) +* Codec - (Maybe optional in future) Required video codec. (See official ffmpeg documentation for supported codecs) ### TODO (for testing branch) * [x] Recreate whole game directory with compressed files -* [x] Cross platform (Easy Windows usage and binaries, MacOS binaries) +* [x] Cross-platform (Easy Windows usage and binaries, macOS binaries) * [x] Use ffmpeg python bindings instead of cli commands * [ ] Reorganize code * [ ] Multithread \ No newline at end of file diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index 51a4866..fbbe2d0 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -17,5 +17,6 @@ Lossless = true Quality = 100 [VIDEO] +SkipVideo = true Extension = "webm" Codec = "libvpx-vp9" \ No newline at end of file diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 48b4d92..769a85e 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -43,20 +43,23 @@ def compress_audio(folder, file, target_folder, extension): def compress_video(folder, file, target_folder, extension): - codec = configloader.config['VIDEO']['Codec'] + if not configloader.config['VIDEO']['SkipVideo']: + codec = configloader.config['VIDEO']['Codec'] - printer.files(file, os.path.splitext(file)[0], extension, codec) - try: - (ffmpeg - .input(f'{folder}/{file}') - .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), vcodec=codec) - .run(quiet=True) - ) - except ffmpeg._run.Error as e: + printer.files(file, os.path.splitext(file)[0], extension, codec) + try: + (ffmpeg + .input(f'{folder}/{file}') + .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), vcodec=codec) + .run(quiet=True) + ) + except ffmpeg._run.Error as e: + utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + utils.errors_count += 1 + if not configloader.config['FFMPEG']['HideErrors']: + printer.error(f"File {file} can't be processed! Error: {e}") + else: utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - utils.errors_count += 1 - if not configloader.config['FFMPEG']['HideErrors']: - printer.error(f"File {file} can't be processed! Error: {e}") return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' From 0ad60b5b947f1799e1ebe5d088f1f063cb2e8f2e Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 26 Apr 2024 01:33:51 +0300 Subject: [PATCH 056/105] FFMpeg-Compressor: Switch to other ffmpeg lib, add CRF parameter for video --- FFMpeg-Compressor/README.md | 1 + FFMpeg-Compressor/ffmpeg-comp.toml | 3 ++- FFMpeg-Compressor/modules/compressor.py | 26 +++++++++++++------------ FFMpeg-Compressor/modules/printer.py | 2 +- FFMpeg-Compressor/requirements.txt | 2 +- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index 4d04f2f..496d47c 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -27,6 +27,7 @@ Python utility uses ffmpeg to compress Visual Novel Resources * Quality - Quality level of images. Values range: `0-100` (100 - best quality, 0 - worst quality) #### VIDEO section +* CRF ("Constant Quality") - Video quality parameter for ffmpeg. The CRF value can be from 0 to 63. Lower values mean better quality. Recommended values range from 15 to 35, with 31 being recommended for 1080p HD video. (default: `27`) * SkipVideo - Skip processing all video files. (default: `false`) * Extension - Required image file extension. It supports: `.3gp`, `.amv`, `.avi`, `.gif`, `.m2l`, `.m4v`, `.mkv`, `.mov`, `.mp4`, `.m4v`, `.mpeg`, `.mpv`, `.webm`, `.ogv` * Codec - (Maybe optional in future) Required video codec. (See official ffmpeg documentation for supported codecs) diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index fbbe2d0..7005024 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -17,6 +17,7 @@ Lossless = true Quality = 100 [VIDEO] -SkipVideo = true +CRF = 27 +SkipVideo = false Extension = "webm" Codec = "libvpx-vp9" \ No newline at end of file diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 769a85e..8c3b737 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -3,7 +3,7 @@ from modules import printer from modules import utils from PIL import Image import pillow_avif -import ffmpeg +from ffmpeg import FFmpeg, FFmpegError import os @@ -28,13 +28,13 @@ def compress_audio(folder, file, target_folder, extension): printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}") try: - (ffmpeg + (FFmpeg() .input(f'{folder}/{file}') .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), - audio_bitrate=bitrate) - .run(quiet=True) + {"b:a": bitrate}) + .execute() ) - except ffmpeg._run.Error as e: + except FFmpegError as e: utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') utils.errors_count += 1 if not configloader.config['FFMPEG']['HideErrors']: @@ -45,15 +45,17 @@ def compress_audio(folder, file, target_folder, extension): def compress_video(folder, file, target_folder, extension): if not configloader.config['VIDEO']['SkipVideo']: codec = configloader.config['VIDEO']['Codec'] + crf = configloader.config['VIDEO']['CRF'] printer.files(file, os.path.splitext(file)[0], extension, codec) try: - (ffmpeg + (FFmpeg() .input(f'{folder}/{file}') - .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), vcodec=codec) - .run(quiet=True) + .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), + {"codec:v": codec, "v:b": 0}, crf=crf) + .execute() ) - except ffmpeg._run.Error as e: + except FFmpegError as e: utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') utils.errors_count += 1 if not configloader.config['FFMPEG']['HideErrors']: @@ -97,12 +99,12 @@ def compress(folder, file, target_folder): if configloader.config["FFMPEG"]["ForceCompress"]: printer.unknown_file(file) try: - (ffmpeg + (FFmpeg() .input(f'{folder}/{file}') .output(f'{target_folder}/{file}') - .run(quiet=True) + .execute() ) - except ffmpeg._run.Error as e: + except FFmpegError as e: utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') utils.errors_count += 1 if not configloader.config['FFMPEG']['HideErrors']: diff --git a/FFMpeg-Compressor/modules/printer.py b/FFMpeg-Compressor/modules/printer.py index 4c99ced..5b63a56 100644 --- a/FFMpeg-Compressor/modules/printer.py +++ b/FFMpeg-Compressor/modules/printer.py @@ -38,7 +38,7 @@ def files(source, dest, dest_ext, comment): def unknown_file(file): - print(clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed wia ffmpeg)")) + print(clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)")) bar.next() diff --git a/FFMpeg-Compressor/requirements.txt b/FFMpeg-Compressor/requirements.txt index 88b8dd7..5201a85 100644 --- a/FFMpeg-Compressor/requirements.txt +++ b/FFMpeg-Compressor/requirements.txt @@ -1,5 +1,5 @@ Pillow==9.5.0 pillow-avif-plugin==1.4.1 -ffmpeg-python==0.2.0 +python-ffmpeg==2.0.12 progress==1.6 colorama==0.4.6 \ No newline at end of file From bf7c97d125bbbc3d189e697db9610659c1cb5333 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 4 May 2024 23:51:40 +0300 Subject: [PATCH 057/105] Add venv creation message --- build.bat | 6 ++++++ build.sh | 8 +++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/build.bat b/build.bat index def90d6..237c7e3 100644 --- a/build.bat +++ b/build.bat @@ -1,4 +1,6 @@ @Echo off +if not defined VIRTUAL_ENV goto :venv_error + mkdir output mkdir output\bin python -m pip install -r FFMpeg-Compressor\requirements.txt || goto :exit @@ -12,6 +14,10 @@ move /Y output\rendroid-unpack.exe output\bin python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=vnds2renpy VNDS-to-RenPy/convert.py || goto :exit move /Y output\vnds2renpy.exe output\bin +:venv_error +echo "Please create and activate venv before running this script: python -m venv .\venv && .\venv\Scripts\activate.bat" +goto :exit + :exit pause exit /b %exitlevel% \ No newline at end of file diff --git a/build.sh b/build.sh index 53caada..7e21bbf 100755 --- a/build.sh +++ b/build.sh @@ -1,5 +1,11 @@ -#!/bin/bash +#!/usr/bin/env bash set -e +if [[ "$VIRTUAL_ENV" == "" ]] +then + echo -e "Please create and activate venv before running this script: \033[100mpython3 -m venv venv && source ./venv/bin/activate\033[49m" + exit +fi + mkdir -p output mkdir -p output/bin python3 -m pip install -r FFMpeg-Compressor/requirements.txt From adb8c8384063dcb67297333627553bb6ec8fc67c Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sun, 5 May 2024 01:05:01 +0300 Subject: [PATCH 058/105] FFMpeg-Compressor: Don't use resize if downscale is 1 --- FFMpeg-Compressor/modules/compressor.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 8c3b737..254a312 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -71,17 +71,18 @@ def compress_image(folder, file, target_folder, extension): try: image = Image.open(f'{folder}/{file}') - width, height = image.size - res_downscale = configloader.config['IMAGE']['ResDownScale'] - new_size = (int(width / res_downscale), int(height / res_downscale)) - if (extension == "jpg" or extension == "jpeg" or extension == "avif" or (extension == "webp" and not configloader.config['FFMPEG']['WebpRGBA'])): if has_transparency(image): printer.warning(f"{file} has transparency. Changing to fallback...") extension = configloader.config['IMAGE']['FallBackExtension'] - image = image.resize(new_size) + res_downscale = configloader.config['IMAGE']['ResDownScale'] + if res_downscale != 1: + width, height = image.size + new_size = (int(width / res_downscale), int(height / res_downscale)) + image = image.resize(new_size) + image.save(utils.check_duplicates(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}"), optimize=True, lossless=configloader.config['IMAGE']['Lossless'], From 40c2683132af8c9482400f326f354735f7945e0a Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sun, 5 May 2024 01:11:58 +0300 Subject: [PATCH 059/105] FFMpeg-Compressor: Add time taken message --- FFMpeg-Compressor/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py index 85f2322..2721629 100755 --- a/FFMpeg-Compressor/main.py +++ b/FFMpeg-Compressor/main.py @@ -4,6 +4,7 @@ from modules import configloader from modules import compressor from modules import printer from modules import utils +from datetime import datetime import shutil import sys import os @@ -26,6 +27,7 @@ def get_file_type(filename): if __name__ == "__main__": + start_time = datetime.now() printer.win_ascii_esc() try: if sys.argv[1][len(sys.argv[1])-1] == "/": @@ -74,3 +76,4 @@ if __name__ == "__main__": utils.get_compression_status(orig_folder) utils.sys_pause() + print(f"Time taken: {datetime.now() - start_time}") From 9612d30d06e344b2ca7b58794335f0b2d10ae72c Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sun, 5 May 2024 01:12:29 +0300 Subject: [PATCH 060/105] FFMpeg-Compressor: Add avif transparency support --- FFMpeg-Compressor/modules/compressor.py | 2 +- FFMpeg-Compressor/requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 254a312..7260787 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -71,7 +71,7 @@ def compress_image(folder, file, target_folder, extension): try: image = Image.open(f'{folder}/{file}') - if (extension == "jpg" or extension == "jpeg" or extension == "avif" or + if (extension == "jpg" or extension == "jpeg" or (extension == "webp" and not configloader.config['FFMPEG']['WebpRGBA'])): if has_transparency(image): printer.warning(f"{file} has transparency. Changing to fallback...") diff --git a/FFMpeg-Compressor/requirements.txt b/FFMpeg-Compressor/requirements.txt index 5201a85..c3c571c 100644 --- a/FFMpeg-Compressor/requirements.txt +++ b/FFMpeg-Compressor/requirements.txt @@ -1,5 +1,5 @@ -Pillow==9.5.0 -pillow-avif-plugin==1.4.1 +Pillow==10.3.0 +pillow-avif-plugin==1.4.3 python-ffmpeg==2.0.12 progress==1.6 colorama==0.4.6 \ No newline at end of file From e71cad4d5f4616477dca2579c69cc05f4e4d1031 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sun, 5 May 2024 03:14:27 +0300 Subject: [PATCH 061/105] FFMpeg-Compressor: Refactor and bar fix --- FFMpeg-Compressor/main.py | 74 +++++++------------------ FFMpeg-Compressor/modules/compressor.py | 40 +++++++++++++ FFMpeg-Compressor/modules/printer.py | 46 ++++++++------- FFMpeg-Compressor/modules/utils.py | 2 +- 4 files changed, 87 insertions(+), 75 deletions(-) diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py index 2721629..39bf58d 100755 --- a/FFMpeg-Compressor/main.py +++ b/FFMpeg-Compressor/main.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -from modules import configloader from modules import compressor from modules import printer from modules import utils @@ -10,70 +9,39 @@ import sys import os -def get_file_type(filename): - audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] - image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'] - video_ext = ['.3gp' '.amv', '.avi', '.gif', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', - '.webm', '.ogv'] - - if os.path.splitext(filename)[1] in audio_ext: - return "audio" - elif os.path.splitext(filename)[1] in image_ext: - return "image" - elif os.path.splitext(filename)[1] in video_ext: - return "video" - else: - return "unknown" +def get_args(): + try: + if sys.argv[1][len(sys.argv[1])-1] == "/": + path = sys.argv[1][:len(sys.argv[1])-1] + else: + path = sys.argv[1] + return path + except IndexError: + print(utils.help_message()) + exit() if __name__ == "__main__": start_time = datetime.now() printer.win_ascii_esc() - try: - if sys.argv[1][len(sys.argv[1])-1] == "/": - arg_path = sys.argv[1][:len(sys.argv[1])-1] - else: - arg_path = sys.argv[1] - except IndexError: - print(utils.help_message()) - exit() + req_folder = os.path.abspath(get_args()) - orig_folder = os.path.abspath(arg_path) - printer.orig_folder = os.path.abspath(arg_path) + printer.bar_init(req_folder) - printer.bar_init(orig_folder) - - if os.path.exists(f"{orig_folder}_compressed"): - shutil.rmtree(f"{orig_folder}_compressed") + if os.path.exists(f"{req_folder}_compressed"): + shutil.rmtree(f"{req_folder}_compressed") printer.info("Creating folders...") - for folder, folders, files in os.walk(orig_folder): - if not os.path.exists(folder.replace(orig_folder, f"{orig_folder}_compressed")): - os.mkdir(folder.replace(orig_folder, f"{orig_folder}_compressed")) + for folder, folders, files in os.walk(req_folder): + if not os.path.exists(folder.replace(req_folder, f"{req_folder}_compressed")): + os.mkdir(folder.replace(req_folder, f"{req_folder}_compressed")) - printer.info(f"Compressing \"{folder.replace(orig_folder, orig_folder.split('/').pop())}\" folder...") - target_folder = folder.replace(orig_folder, f"{orig_folder}_compressed") + printer.info(f"Compressing \"{folder.replace(req_folder, req_folder.split('/').pop())}\" folder...") + target_folder = folder.replace(req_folder, f"{req_folder}_compressed") for file in os.listdir(folder): if os.path.isfile(f'{folder}/{file}'): - match get_file_type(file): - case "audio": - comp_file = compressor.compress_audio(folder, file, target_folder, - configloader.config['AUDIO']['Extension']) - case "image": - comp_file = compressor.compress_image(folder, file, target_folder, - configloader.config['IMAGE']['Extension']) - case "video": - comp_file = compressor.compress_video(folder, file, target_folder, - configloader.config['VIDEO']['Extension']) - case "unknown": - comp_file = compressor.compress(folder, file, target_folder) + compressor.compress_file(folder, file, target_folder, req_folder) - if configloader.config['FFMPEG']['MimicMode']: - try: - os.rename(comp_file, f'{folder}/{file}'.replace(orig_folder, f"{orig_folder}_compressed")) - except FileNotFoundError: - pass - - utils.get_compression_status(orig_folder) + utils.get_compression_status(req_folder) utils.sys_pause() print(f"Time taken: {datetime.now() - start_time}") diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 7260787..dd1c0ce 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -7,6 +7,22 @@ from ffmpeg import FFmpeg, FFmpegError import os +def get_file_type(filename): + audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] + image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'] + video_ext = ['.3gp' '.amv', '.avi', '.gif', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', + '.webm', '.ogv'] + + if os.path.splitext(filename)[1] in audio_ext: + return "audio" + elif os.path.splitext(filename)[1] in image_ext: + return "image" + elif os.path.splitext(filename)[1] in video_ext: + return "video" + else: + return "unknown" + + def has_transparency(img): if img.info.get("transparency", None) is not None: return True @@ -113,3 +129,27 @@ def compress(folder, file, target_folder): else: utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') return f'{target_folder}/{file}' + + +def compress_file(_dir, filename, target_dir, source): + match get_file_type(filename): + case "audio": + comp_file = compress_audio(_dir, filename, target_dir, + configloader.config['AUDIO']['Extension']) + case "image": + comp_file = compress_image(_dir, filename, target_dir, + configloader.config['IMAGE']['Extension']) + case "video": + comp_file = compress_video(_dir, filename, target_dir, + configloader.config['VIDEO']['Extension']) + case "unknown": + comp_file = compress(_dir, filename, target_dir) + + if configloader.config['FFMPEG']['MimicMode']: + try: + os.rename(comp_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed")) + except FileNotFoundError: + pass + + printer.bar.update() + printer.bar.next() diff --git a/FFMpeg-Compressor/modules/printer.py b/FFMpeg-Compressor/modules/printer.py index 5b63a56..a1a3452 100644 --- a/FFMpeg-Compressor/modules/printer.py +++ b/FFMpeg-Compressor/modules/printer.py @@ -4,42 +4,46 @@ import sys import os -# Fill whole string with spaces for cleaning progress bar -def clean_str(string): - return string + " " * (os.get_terminal_size().columns - len(string)) - - -def info(string): - print(clean_str(f"\r\033[100mI {string}\033[49m")) - - -def warning(string): - print(clean_str(f"\r\033[93mW\033[0m {string}\033[49m")) - - -def error(string): - print(clean_str(f"\r\033[31mE\033[0m {string}\033[49m")) - - def bar_init(folder): file_count = 0 for folder, folders, file in os.walk(folder): file_count += len(file) global bar bar = IncrementalBar('Compressing', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)') + bar.update() + + +def bar_print(string): + print(string) + bar.update() + + +# Fill whole string with spaces for cleaning progress bar +def clean_str(string): + return string + " " * (os.get_terminal_size().columns - len(string)) + + +def info(string): + bar_print(clean_str(f"\r\033[100mI {string}\033[49m")) + + +def warning(string): + bar_print(clean_str(f"\r\033[93mW\033[0m {string}\033[49m")) + + +def error(string): + bar_print(clean_str(f"\r\033[31mE\033[0m {string}\033[49m")) def files(source, dest, dest_ext, comment): source_ext = os.path.splitext(source)[1] source_name = os.path.splitext(source)[0] - print(clean_str(f"\r* \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m")) - bar.next() + bar_print(clean_str(f"\r* \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m ...")) def unknown_file(file): - print(clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)")) - bar.next() + bar_print(clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)")) def win_ascii_esc(): diff --git a/FFMpeg-Compressor/modules/utils.py b/FFMpeg-Compressor/modules/utils.py index f1a3256..db3e548 100644 --- a/FFMpeg-Compressor/modules/utils.py +++ b/FFMpeg-Compressor/modules/utils.py @@ -25,7 +25,7 @@ def get_compression(orig, comp): orig = get_dir_size(orig, processed_files) comp = get_dir_size(comp, processed_files) - print(f"Result: {orig/1024/1024:.2f}MB -> {comp/1024/1024:.2f}MB Δ {(orig - comp)/1024/1024:.2f}MB") + print(f"\nResult: {orig/1024/1024:.2f}MB -> {comp/1024/1024:.2f}MB Δ {(orig - comp)/1024/1024:.2f}MB") except ZeroDivisionError: printer.warning("Nothing compressed!") From ad6ea5aa68f0edcbd5eedad196a789641ef438e2 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Mon, 6 May 2024 23:13:17 +0300 Subject: [PATCH 062/105] FFMpeg-Compressor: Fix unicode data decoding in metadata --- FFMpeg-Compressor/modules/compressor.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index dd1c0ce..8e956fe 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -46,8 +46,9 @@ def compress_audio(folder, file, target_folder, extension): try: (FFmpeg() .input(f'{folder}/{file}') + .option("hide_banner") .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), - {"b:a": bitrate}) + {"b:a": bitrate, "loglevel": "error"}) .execute() ) except FFmpegError as e: @@ -67,8 +68,9 @@ def compress_video(folder, file, target_folder, extension): try: (FFmpeg() .input(f'{folder}/{file}') + .option("hide_banner") .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), - {"codec:v": codec, "v:b": 0}, crf=crf) + {"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) .execute() ) except FFmpegError as e: From df1122bcd0270e6a4cd6e160fdd50cde9f931eef Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Thu, 9 May 2024 00:18:57 +0300 Subject: [PATCH 063/105] RenPy-Unpacker: Add MacOS support --- RenPy-Unpacker/unpack.rpy | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/RenPy-Unpacker/unpack.rpy b/RenPy-Unpacker/unpack.rpy index 005e855..905102f 100644 --- a/RenPy-Unpacker/unpack.rpy +++ b/RenPy-Unpacker/unpack.rpy @@ -2,8 +2,13 @@ init 4 python: import os for asset in renpy.list_files(): - if os.path.splitext(asset)[1] != ".rpa" and asset != "unpack.rpyc": - output = "unpack/game/" + asset + if os.path.splitext(asset)[1] != ".rpa" and not asset.count("unpack.rpy"): # Ignore .rpa and script itself + if renpy.macintosh: + game_path = os.path.expanduser('~') + "/" + config.name # Unpack assets to home folder (on mac you cant get cwd) + output = game_path + "/game/" + asset + else: + output = "unpack/game/" + asset # Unpack assets to game folder + if not os.path.exists(os.path.dirname(output)): os.makedirs(os.path.dirname(output)) From 82ac0a1301c60454aa01a47aeb12450e26716771 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Thu, 16 May 2024 07:54:20 +0300 Subject: [PATCH 064/105] FFMpeg-Compressor: Fix transparency decoding --- FFMpeg-Compressor/modules/compressor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 8e956fe..8d46559 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -87,7 +87,7 @@ def compress_image(folder, file, target_folder, extension): quality = configloader.config['IMAGE']['Quality'] printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") try: - image = Image.open(f'{folder}/{file}') + image = Image.open(f'{folder}/{file}').convert('RGBA') if (extension == "jpg" or extension == "jpeg" or (extension == "webp" and not configloader.config['FFMPEG']['WebpRGBA'])): From da64641bedcbe1f5563ae8bbeac2b532441b40d1 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 15 Jun 2024 02:56:27 +0300 Subject: [PATCH 065/105] FFMpeg-Compressor: Do not convert not transparent to RGBA --- FFMpeg-Compressor/modules/compressor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index 8d46559..d7c4e77 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -87,7 +87,7 @@ def compress_image(folder, file, target_folder, extension): quality = configloader.config['IMAGE']['Quality'] printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") try: - image = Image.open(f'{folder}/{file}').convert('RGBA') + image = Image.open(f'{folder}/{file}') if (extension == "jpg" or extension == "jpeg" or (extension == "webp" and not configloader.config['FFMPEG']['WebpRGBA'])): @@ -95,6 +95,9 @@ def compress_image(folder, file, target_folder, extension): printer.warning(f"{file} has transparency. Changing to fallback...") extension = configloader.config['IMAGE']['FallBackExtension'] + if has_transparency(image): + image.convert('RGBA') + res_downscale = configloader.config['IMAGE']['ResDownScale'] if res_downscale != 1: width, height = image.size From 54820279d151ba7eb85fb0555f249f91f67b505a Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 15 Jun 2024 03:34:19 +0300 Subject: [PATCH 066/105] FFMpeg-Compressor: Enable hardware video decoding --- FFMpeg-Compressor/main.py | 2 +- FFMpeg-Compressor/modules/compressor.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py index 39bf58d..d8bdd21 100755 --- a/FFMpeg-Compressor/main.py +++ b/FFMpeg-Compressor/main.py @@ -38,7 +38,7 @@ if __name__ == "__main__": printer.info(f"Compressing \"{folder.replace(req_folder, req_folder.split('/').pop())}\" folder...") target_folder = folder.replace(req_folder, f"{req_folder}_compressed") - for file in os.listdir(folder): + for file in files: if os.path.isfile(f'{folder}/{file}'): compressor.compress_file(folder, file, target_folder, req_folder) diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index d7c4e77..b8fa04a 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -69,6 +69,7 @@ def compress_video(folder, file, target_folder, extension): (FFmpeg() .input(f'{folder}/{file}') .option("hide_banner") + .option("hwaccel", "auto") .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), {"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) .execute() From 23f7e5ec670d24682d934e4e598e5a855acdc237 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 19 Jul 2024 02:17:04 +0300 Subject: [PATCH 067/105] FFMpeg-Compressor: Implement multiple FIFO workers --- FFMpeg-Compressor/README.md | 4 ++-- FFMpeg-Compressor/ffmpeg-comp.toml | 1 + FFMpeg-Compressor/main.py | 20 ++++++++++++++++---- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/FFMpeg-Compressor/README.md b/FFMpeg-Compressor/README.md index 496d47c..40b37d8 100644 --- a/FFMpeg-Compressor/README.md +++ b/FFMpeg-Compressor/README.md @@ -36,5 +36,5 @@ Python utility uses ffmpeg to compress Visual Novel Resources * [x] Recreate whole game directory with compressed files * [x] Cross-platform (Easy Windows usage and binaries, macOS binaries) * [x] Use ffmpeg python bindings instead of cli commands -* [ ] Reorganize code -* [ ] Multithread \ No newline at end of file +* [x] Multithread +* [ ] Reorganize code \ No newline at end of file diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/FFMpeg-Compressor/ffmpeg-comp.toml index 7005024..15551da 100644 --- a/FFMpeg-Compressor/ffmpeg-comp.toml +++ b/FFMpeg-Compressor/ffmpeg-comp.toml @@ -4,6 +4,7 @@ ForceCompress = false MimicMode = true HideErrors = true WebpRGBA = true +Workers = 16 [AUDIO] Extension = "opus" diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py index d8bdd21..7c05dc5 100755 --- a/FFMpeg-Compressor/main.py +++ b/FFMpeg-Compressor/main.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor, as_completed +from modules.configloader import config from modules import compressor from modules import printer from modules import utils @@ -21,6 +23,11 @@ def get_args(): exit() +def compress_worker(folder, file, target_folder, req_folder): + if os.path.isfile(f'{folder}/{file}'): + compressor.compress_file(folder, file, target_folder, req_folder) + + if __name__ == "__main__": start_time = datetime.now() printer.win_ascii_esc() @@ -38,10 +45,15 @@ if __name__ == "__main__": printer.info(f"Compressing \"{folder.replace(req_folder, req_folder.split('/').pop())}\" folder...") target_folder = folder.replace(req_folder, f"{req_folder}_compressed") - for file in files: - if os.path.isfile(f'{folder}/{file}'): - compressor.compress_file(folder, file, target_folder, req_folder) + + with ThreadPoolExecutor(max_workers=config["FFMPEG"]["Workers"]) as executor: + futures = [ + executor.submit(compress_worker, folder, file, target_folder, req_folder) + for file in files + ] + for future in as_completed(futures): + future.result() utils.get_compression_status(req_folder) utils.sys_pause() - print(f"Time taken: {datetime.now() - start_time}") + print(f"Time taken: {datetime.now() - start_time}") \ No newline at end of file From dff5bd12f1bf4281f4da33384c93024fcbf77d66 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 19 Jul 2024 02:22:38 +0300 Subject: [PATCH 068/105] FFMpeg-Compressor: Remove .gif from video section --- FFMpeg-Compressor/modules/compressor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index b8fa04a..db9080e 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -10,7 +10,7 @@ import os def get_file_type(filename): audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'] - video_ext = ['.3gp' '.amv', '.avi', '.gif', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', + video_ext = ['.3gp' '.amv', '.avi', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', '.webm', '.ogv'] if os.path.splitext(filename)[1] in audio_ext: From 31e82b59b331e46ac6ec68ad5c1cf1b204431e2c Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 19 Jul 2024 02:36:00 +0300 Subject: [PATCH 069/105] FFMpeg-Compressor: Edit result message --- FFMpeg-Compressor/modules/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FFMpeg-Compressor/modules/utils.py b/FFMpeg-Compressor/modules/utils.py index db3e548..301923c 100644 --- a/FFMpeg-Compressor/modules/utils.py +++ b/FFMpeg-Compressor/modules/utils.py @@ -25,7 +25,7 @@ def get_compression(orig, comp): orig = get_dir_size(orig, processed_files) comp = get_dir_size(comp, processed_files) - print(f"\nResult: {orig/1024/1024:.2f}MB -> {comp/1024/1024:.2f}MB Δ {(orig - comp)/1024/1024:.2f}MB") + print(f"\nResult: {orig/1024/1024:.2f}MB -> {comp/1024/1024:.2f}MB ({(comp - orig)/1024/1024:.2f}MB)") except ZeroDivisionError: printer.warning("Nothing compressed!") From 0a9114ff64dbf9c88dbe4661d422985918027b0c Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 19 Jul 2024 03:02:38 +0300 Subject: [PATCH 070/105] FFMpeg-Compressor: Show only completed tasks --- FFMpeg-Compressor/modules/compressor.py | 7 +++---- FFMpeg-Compressor/modules/printer.py | 8 ++++---- FFMpeg-Compressor/modules/utils.py | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py index db9080e..728953c 100644 --- a/FFMpeg-Compressor/modules/compressor.py +++ b/FFMpeg-Compressor/modules/compressor.py @@ -41,8 +41,6 @@ def has_transparency(img): def compress_audio(folder, file, target_folder, extension): bitrate = configloader.config['AUDIO']['BitRate'] - - printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}") try: (FFmpeg() .input(f'{folder}/{file}') @@ -56,6 +54,7 @@ def compress_audio(folder, file, target_folder, extension): utils.errors_count += 1 if not configloader.config['FFMPEG']['HideErrors']: printer.error(f"File {file} can't be processed! Error: {e}") + printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}") return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' @@ -64,7 +63,6 @@ def compress_video(folder, file, target_folder, extension): codec = configloader.config['VIDEO']['Codec'] crf = configloader.config['VIDEO']['CRF'] - printer.files(file, os.path.splitext(file)[0], extension, codec) try: (FFmpeg() .input(f'{folder}/{file}') @@ -74,6 +72,7 @@ def compress_video(folder, file, target_folder, extension): {"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) .execute() ) + printer.files(file, os.path.splitext(file)[0], extension, codec) except FFmpegError as e: utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') utils.errors_count += 1 @@ -86,7 +85,6 @@ def compress_video(folder, file, target_folder, extension): def compress_image(folder, file, target_folder, extension): quality = configloader.config['IMAGE']['Quality'] - printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") try: image = Image.open(f'{folder}/{file}') @@ -110,6 +108,7 @@ def compress_image(folder, file, target_folder, extension): lossless=configloader.config['IMAGE']['Lossless'], quality=quality, minimize_size=True) + printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") except Exception as e: utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') utils.errors_count += 1 diff --git a/FFMpeg-Compressor/modules/printer.py b/FFMpeg-Compressor/modules/printer.py index a1a3452..0fdce24 100644 --- a/FFMpeg-Compressor/modules/printer.py +++ b/FFMpeg-Compressor/modules/printer.py @@ -24,22 +24,22 @@ def clean_str(string): def info(string): - bar_print(clean_str(f"\r\033[100mI {string}\033[49m")) + bar_print(clean_str(f"\r\033[100m- {string}\033[49m")) def warning(string): - bar_print(clean_str(f"\r\033[93mW\033[0m {string}\033[49m")) + bar_print(clean_str(f"\r\033[93m!\033[0m {string}\033[49m")) def error(string): - bar_print(clean_str(f"\r\033[31mE\033[0m {string}\033[49m")) + bar_print(clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m")) def files(source, dest, dest_ext, comment): source_ext = os.path.splitext(source)[1] source_name = os.path.splitext(source)[0] - bar_print(clean_str(f"\r* \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m ...")) + bar_print(clean_str(f"\r\033[0;32m\u2713\033[0m \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m")) def unknown_file(file): diff --git a/FFMpeg-Compressor/modules/utils.py b/FFMpeg-Compressor/modules/utils.py index 301923c..907f9b1 100644 --- a/FFMpeg-Compressor/modules/utils.py +++ b/FFMpeg-Compressor/modules/utils.py @@ -55,7 +55,7 @@ def get_compression_status(orig_folder): def add_unprocessed_file(orig_folder, new_folder): if configloader.config['FFMPEG']['CopyUnprocessed']: - filename = orig_folder.split().pop() + filename = orig_folder.split("/").pop() copyfile(orig_folder, new_folder) printer.info(f"File {filename} copied to compressed folder.") From 7487eb94bd7971b6183f1ca776e8ca5ad232f31e Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Thu, 29 Aug 2024 01:32:36 +0300 Subject: [PATCH 071/105] Basic refactor for packaging --- .gitignore | 3 + FFMpeg-Compressor/__init__.py | 0 FFMpeg-Compressor/__main__.py | 19 ++ FFMpeg-Compressor/application.py | 48 ++++++ FFMpeg-Compressor/compress.py | 163 ++++++++++++++++++ FFMpeg-Compressor/config.py | 27 +++ FFMpeg-Compressor/main.py | 59 ------- FFMpeg-Compressor/modules/compressor.py | 160 ----------------- FFMpeg-Compressor/modules/configloader.py | 11 -- FFMpeg-Compressor/modules/printer.py | 51 ------ FFMpeg-Compressor/modules/utils.py | 79 --------- FFMpeg-Compressor/printer.py | 46 +++++ FFMpeg-Compressor/utils.py | 71 ++++++++ README.md | 25 ++- RenPy-Android-Unpack/__init__.py | 0 RenPy-Android-Unpack/__main__.py | 52 ++++++ RenPy-Android-Unpack/actions.py | 98 +++++++++++ RenPy-Android-Unpack/printer.py | 14 ++ RenPy-Android-Unpack/requirements.txt | 2 - RenPy-Android-Unpack/unpack.py | 108 ------------ VNDS-to-RenPy/__init__.py | 0 VNDS-to-RenPy/{convert.py => __main__.py} | 0 build.bat | 10 +- build.sh | 12 +- .../requirements.txt => requirements.txt | 0 25 files changed, 573 insertions(+), 485 deletions(-) create mode 100644 FFMpeg-Compressor/__init__.py create mode 100644 FFMpeg-Compressor/__main__.py create mode 100755 FFMpeg-Compressor/application.py create mode 100644 FFMpeg-Compressor/compress.py create mode 100644 FFMpeg-Compressor/config.py delete mode 100755 FFMpeg-Compressor/main.py delete mode 100644 FFMpeg-Compressor/modules/compressor.py delete mode 100644 FFMpeg-Compressor/modules/configloader.py delete mode 100644 FFMpeg-Compressor/modules/printer.py delete mode 100644 FFMpeg-Compressor/modules/utils.py create mode 100644 FFMpeg-Compressor/printer.py create mode 100644 FFMpeg-Compressor/utils.py create mode 100644 RenPy-Android-Unpack/__init__.py create mode 100755 RenPy-Android-Unpack/__main__.py create mode 100755 RenPy-Android-Unpack/actions.py create mode 100644 RenPy-Android-Unpack/printer.py delete mode 100644 RenPy-Android-Unpack/requirements.txt delete mode 100755 RenPy-Android-Unpack/unpack.py create mode 100644 VNDS-to-RenPy/__init__.py rename VNDS-to-RenPy/{convert.py => __main__.py} (100%) mode change 100644 => 100755 build.bat rename FFMpeg-Compressor/requirements.txt => requirements.txt (100%) diff --git a/.gitignore b/.gitignore index 16be8f2..abe0419 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ /output/ +/tests/ +/build/ +/VNTools.egg-info/ diff --git a/FFMpeg-Compressor/__init__.py b/FFMpeg-Compressor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/FFMpeg-Compressor/__main__.py b/FFMpeg-Compressor/__main__.py new file mode 100644 index 0000000..aef4d34 --- /dev/null +++ b/FFMpeg-Compressor/__main__.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +from application import Application +from compress import Compress +from printer import Printer +from config import Config +from utils import Utils + + +def init(): + config = Config.setup_config() + printer = Printer(config.args.source) + utils = Utils(config.config, printer) + compress = Compress(config.config, printer, utils) + + Application(config, compress, printer, utils).run() + + +if __name__ == "__main__": + init() \ No newline at end of file diff --git a/FFMpeg-Compressor/application.py b/FFMpeg-Compressor/application.py new file mode 100755 index 0000000..491267a --- /dev/null +++ b/FFMpeg-Compressor/application.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime +import shutil +import os + + +class Application: + + def __init__(self, config, compress, printer, utils): + self.config = config.config + self.args = config.args + self.compress = compress.compress + self.printer = printer + self.utils = utils + + def compress_worker(self, folder, file, source, output): + if os.path.isfile(f'{folder}/{file}'): + self.compress(folder, file, source, output) + + def run(self): + start_time = datetime.now() + self.printer.win_ascii_esc() + + source = os.path.abspath(self.args.source) + + if os.path.exists(f"{source}_compressed"): + shutil.rmtree(f"{source}_compressed") + + self.printer.info("Creating folders...") + for folder, folders, files in os.walk(source): + if not os.path.exists(folder.replace(source, f"{source}_compressed")): + os.mkdir(folder.replace(source, f"{source}_compressed")) + + self.printer.info(f'Compressing "{folder.replace(source, os.path.split(source)[-1])}" folder...') + output = folder.replace(source, f"{source}_compressed") + + with ThreadPoolExecutor(max_workers=self.config["FFMPEG"]["Workers"]) as executor: + futures = [ + executor.submit(self.compress, folder, file, source, output) + for file in files if os.path.isfile(f'{folder}/{file}') + ] + for future in as_completed(futures): + future.result() + + self.utils.get_compression_status(source) + self.utils.sys_pause() + print(f"Time taken: {datetime.now() - start_time}") \ No newline at end of file diff --git a/FFMpeg-Compressor/compress.py b/FFMpeg-Compressor/compress.py new file mode 100644 index 0000000..5c1a465 --- /dev/null +++ b/FFMpeg-Compressor/compress.py @@ -0,0 +1,163 @@ +from ffmpeg import FFmpeg, FFmpegError +from PIL import Image +import pillow_avif +import os + + +class File: + + @staticmethod + def get_type(filename): + audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] + image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'] + video_ext = ['.3gp' '.amv', '.avi', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', + '.webm', '.ogv'] + + if os.path.splitext(filename)[1] in audio_ext: + return "audio" + elif os.path.splitext(filename)[1] in image_ext: + return "image" + elif os.path.splitext(filename)[1] in video_ext: + return "video" + else: + return "unknown" + + @staticmethod + def has_transparency(img: Image) -> bool: + if img.info.get("transparency", None) is not None: + return True + if img.mode == "P": + transparent = img.info.get("transparency", -1) + for _, index in img.getcolors(): + if index == transparent: + return True + elif img.mode == "RGBA": + extrema = img.getextrema() + if extrema[3][0] < 255: + return True + return False + + +class Compress: + + def __init__(self, config, printer, utils): + self.config = config + self.printer = printer + self.utils = utils + + def audio(self, folder, file, target_folder, extension): + bitrate = self.config['AUDIO']['BitRate'] + try: + (FFmpeg() + .input(f'{folder}/{file}') + .option("hide_banner") + .output(self.utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), + {"b:a": bitrate, "loglevel": "error"}) + .execute() + ) + except FFmpegError as e: + self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.errors += 1 + if not self.config['FFMPEG']['HideErrors']: + self.printer.error(f"File {file} can't be processed! Error: {e}") + self.printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}") + return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' + + + def video(self, folder, file, target_folder, extension): + if not self.config['VIDEO']['SkipVideo']: + codec = self.config['VIDEO']['Codec'] + crf = self.config['VIDEO']['CRF'] + + try: + (FFmpeg() + .input(f'{folder}/{file}') + .option("hide_banner") + .option("hwaccel", "auto") + .output(self.utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), + {"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) + .execute() + ) + self.printer.files(file, os.path.splitext(file)[0], extension, codec) + except FFmpegError as e: + self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.errors += 1 + if not self.config['FFMPEG']['HideErrors']: + self.printer.error(f"File {file} can't be processed! Error: {e}") + else: + self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' + + + def image(self, folder, file, target_folder, extension): + quality = self.config['IMAGE']['Quality'] + try: + image = Image.open(f'{folder}/{file}') + + if (extension == "jpg" or extension == "jpeg" or + (extension == "webp" and not self.config['FFMPEG']['WebpRGBA'])): + if File.has_transparency(image): + self.printer.warning(f"{file} has transparency. Changing to fallback...") + extension = self.config['IMAGE']['FallBackExtension'] + + if File.has_transparency(image): + image.convert('RGBA') + + res_downscale = self.config['IMAGE']['ResDownScale'] + if res_downscale != 1: + width, height = image.size + new_size = (int(width / res_downscale), int(height / res_downscale)) + image = image.resize(new_size) + + image.save(self.utils.check_duplicates(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}"), + optimize=True, + lossless=self.config['IMAGE']['Lossless'], + quality=quality, + minimize_size=True) + self.printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") + except Exception as e: + self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.errors += 1 + if not self.config['FFMPEG']['HideErrors']: + self.printer.error(f"File {file} can't be processed! Error: {e}") + return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' + + + def unknown(self, folder, file, target_folder): + if self.config["FFMPEG"]["ForceCompress"]: + self.printer.unknown_file(file) + try: + (FFmpeg() + .input(f'{folder}/{file}') + .output(self.utils.check_duplicates(f'{target_folder}/{file}')) + .execute() + ) + except FFmpegError as e: + self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.errors += 1 + if not self.config['FFMPEG']['HideErrors']: + self.printer.error(f"File {file} can't be processed! Error: {e}") + else: + self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + return f'{target_folder}/{file}' + + + def compress(self, _dir, filename, source, output): + match File.get_type(filename): + case "audio": + out_file = self.audio(_dir, filename, output, self.config['AUDIO']['Extension']) + case "image": + out_file = self.image(_dir, filename, output, self.config['IMAGE']['Extension']) + case "video": + out_file = self.video(_dir, filename, output, self.config['VIDEO']['Extension']) + case "unknown": + out_file = self.unknown(_dir, filename, output) + + if self.config['FFMPEG']['MimicMode']: + try: + os.rename(out_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed")) + except FileNotFoundError: + self.printer.warning(f"File {out_file} failed to copy to out dir") + + self.printer.bar.update() + self.printer.bar.next() diff --git a/FFMpeg-Compressor/config.py b/FFMpeg-Compressor/config.py new file mode 100644 index 0000000..d36e3d4 --- /dev/null +++ b/FFMpeg-Compressor/config.py @@ -0,0 +1,27 @@ +import os.path +from argparse import Namespace, ArgumentParser +from dataclasses import dataclass +from typing import Any +import tomllib + + +@dataclass +class Config: + + config: dict[str, Any] + args: Namespace + + @classmethod + def setup_config(cls): + parser = ArgumentParser(prog="ffmpeg-comp", + description="Python utility to compress Visual Novel Resources" + ) + parser.add_argument("source") + parser.add_argument("-c", "--config", default="ffmpeg-comp.toml") + args = parser.parse_args() + if os.path.isfile(args.config): + with open(args.config, "rb") as cfile: + config = tomllib.load(cfile) + else: + print("Failed to find config. Check `ffmpeg-comp -h` to more info") + return cls(config=config, args=args) diff --git a/FFMpeg-Compressor/main.py b/FFMpeg-Compressor/main.py deleted file mode 100755 index 7c05dc5..0000000 --- a/FFMpeg-Compressor/main.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 - -from concurrent.futures import ThreadPoolExecutor, as_completed -from modules.configloader import config -from modules import compressor -from modules import printer -from modules import utils -from datetime import datetime -import shutil -import sys -import os - - -def get_args(): - try: - if sys.argv[1][len(sys.argv[1])-1] == "/": - path = sys.argv[1][:len(sys.argv[1])-1] - else: - path = sys.argv[1] - return path - except IndexError: - print(utils.help_message()) - exit() - - -def compress_worker(folder, file, target_folder, req_folder): - if os.path.isfile(f'{folder}/{file}'): - compressor.compress_file(folder, file, target_folder, req_folder) - - -if __name__ == "__main__": - start_time = datetime.now() - printer.win_ascii_esc() - req_folder = os.path.abspath(get_args()) - - printer.bar_init(req_folder) - - if os.path.exists(f"{req_folder}_compressed"): - shutil.rmtree(f"{req_folder}_compressed") - - printer.info("Creating folders...") - for folder, folders, files in os.walk(req_folder): - if not os.path.exists(folder.replace(req_folder, f"{req_folder}_compressed")): - os.mkdir(folder.replace(req_folder, f"{req_folder}_compressed")) - - printer.info(f"Compressing \"{folder.replace(req_folder, req_folder.split('/').pop())}\" folder...") - target_folder = folder.replace(req_folder, f"{req_folder}_compressed") - - with ThreadPoolExecutor(max_workers=config["FFMPEG"]["Workers"]) as executor: - futures = [ - executor.submit(compress_worker, folder, file, target_folder, req_folder) - for file in files - ] - for future in as_completed(futures): - future.result() - - utils.get_compression_status(req_folder) - utils.sys_pause() - print(f"Time taken: {datetime.now() - start_time}") \ No newline at end of file diff --git a/FFMpeg-Compressor/modules/compressor.py b/FFMpeg-Compressor/modules/compressor.py deleted file mode 100644 index 728953c..0000000 --- a/FFMpeg-Compressor/modules/compressor.py +++ /dev/null @@ -1,160 +0,0 @@ -from modules import configloader -from modules import printer -from modules import utils -from PIL import Image -import pillow_avif -from ffmpeg import FFmpeg, FFmpegError -import os - - -def get_file_type(filename): - audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] - image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'] - video_ext = ['.3gp' '.amv', '.avi', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', - '.webm', '.ogv'] - - if os.path.splitext(filename)[1] in audio_ext: - return "audio" - elif os.path.splitext(filename)[1] in image_ext: - return "image" - elif os.path.splitext(filename)[1] in video_ext: - return "video" - else: - return "unknown" - - -def has_transparency(img): - if img.info.get("transparency", None) is not None: - return True - if img.mode == "P": - transparent = img.info.get("transparency", -1) - for _, index in img.getcolors(): - if index == transparent: - return True - elif img.mode == "RGBA": - extrema = img.getextrema() - if extrema[3][0] < 255: - return True - - return False - - -def compress_audio(folder, file, target_folder, extension): - bitrate = configloader.config['AUDIO']['BitRate'] - try: - (FFmpeg() - .input(f'{folder}/{file}') - .option("hide_banner") - .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), - {"b:a": bitrate, "loglevel": "error"}) - .execute() - ) - except FFmpegError as e: - utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - utils.errors_count += 1 - if not configloader.config['FFMPEG']['HideErrors']: - printer.error(f"File {file} can't be processed! Error: {e}") - printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}") - return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' - - -def compress_video(folder, file, target_folder, extension): - if not configloader.config['VIDEO']['SkipVideo']: - codec = configloader.config['VIDEO']['Codec'] - crf = configloader.config['VIDEO']['CRF'] - - try: - (FFmpeg() - .input(f'{folder}/{file}') - .option("hide_banner") - .option("hwaccel", "auto") - .output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), - {"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) - .execute() - ) - printer.files(file, os.path.splitext(file)[0], extension, codec) - except FFmpegError as e: - utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - utils.errors_count += 1 - if not configloader.config['FFMPEG']['HideErrors']: - printer.error(f"File {file} can't be processed! Error: {e}") - else: - utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' - - -def compress_image(folder, file, target_folder, extension): - quality = configloader.config['IMAGE']['Quality'] - try: - image = Image.open(f'{folder}/{file}') - - if (extension == "jpg" or extension == "jpeg" or - (extension == "webp" and not configloader.config['FFMPEG']['WebpRGBA'])): - if has_transparency(image): - printer.warning(f"{file} has transparency. Changing to fallback...") - extension = configloader.config['IMAGE']['FallBackExtension'] - - if has_transparency(image): - image.convert('RGBA') - - res_downscale = configloader.config['IMAGE']['ResDownScale'] - if res_downscale != 1: - width, height = image.size - new_size = (int(width / res_downscale), int(height / res_downscale)) - image = image.resize(new_size) - - image.save(utils.check_duplicates(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}"), - optimize=True, - lossless=configloader.config['IMAGE']['Lossless'], - quality=quality, - minimize_size=True) - printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") - except Exception as e: - utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - utils.errors_count += 1 - if not configloader.config['FFMPEG']['HideErrors']: - printer.error(f"File {file} can't be processed! Error: {e}") - return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' - - -def compress(folder, file, target_folder): - if configloader.config["FFMPEG"]["ForceCompress"]: - printer.unknown_file(file) - try: - (FFmpeg() - .input(f'{folder}/{file}') - .output(f'{target_folder}/{file}') - .execute() - ) - except FFmpegError as e: - utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - utils.errors_count += 1 - if not configloader.config['FFMPEG']['HideErrors']: - printer.error(f"File {file} can't be processed! Error: {e}") - else: - utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - return f'{target_folder}/{file}' - - -def compress_file(_dir, filename, target_dir, source): - match get_file_type(filename): - case "audio": - comp_file = compress_audio(_dir, filename, target_dir, - configloader.config['AUDIO']['Extension']) - case "image": - comp_file = compress_image(_dir, filename, target_dir, - configloader.config['IMAGE']['Extension']) - case "video": - comp_file = compress_video(_dir, filename, target_dir, - configloader.config['VIDEO']['Extension']) - case "unknown": - comp_file = compress(_dir, filename, target_dir) - - if configloader.config['FFMPEG']['MimicMode']: - try: - os.rename(comp_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed")) - except FileNotFoundError: - pass - - printer.bar.update() - printer.bar.next() diff --git a/FFMpeg-Compressor/modules/configloader.py b/FFMpeg-Compressor/modules/configloader.py deleted file mode 100644 index 1a4db0b..0000000 --- a/FFMpeg-Compressor/modules/configloader.py +++ /dev/null @@ -1,11 +0,0 @@ -import tomllib -from modules import printer - -try: - config = tomllib.load(open("ffmpeg-comp.toml", "rb")) -except FileNotFoundError: - try: - config = tomllib.load(open("/etc/ffmpeg-comp.toml", "rb")) - except FileNotFoundError: - printer.error("Config file not found. Please put it next to binary or in to /etc folder.") - exit() diff --git a/FFMpeg-Compressor/modules/printer.py b/FFMpeg-Compressor/modules/printer.py deleted file mode 100644 index 0fdce24..0000000 --- a/FFMpeg-Compressor/modules/printer.py +++ /dev/null @@ -1,51 +0,0 @@ -from progress.bar import IncrementalBar -import colorama -import sys -import os - - -def bar_init(folder): - file_count = 0 - for folder, folders, file in os.walk(folder): - file_count += len(file) - global bar - bar = IncrementalBar('Compressing', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)') - bar.update() - - -def bar_print(string): - print(string) - bar.update() - - -# Fill whole string with spaces for cleaning progress bar -def clean_str(string): - return string + " " * (os.get_terminal_size().columns - len(string)) - - -def info(string): - bar_print(clean_str(f"\r\033[100m- {string}\033[49m")) - - -def warning(string): - bar_print(clean_str(f"\r\033[93m!\033[0m {string}\033[49m")) - - -def error(string): - bar_print(clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m")) - - -def files(source, dest, dest_ext, comment): - source_ext = os.path.splitext(source)[1] - source_name = os.path.splitext(source)[0] - - bar_print(clean_str(f"\r\033[0;32m\u2713\033[0m \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m")) - - -def unknown_file(file): - bar_print(clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)")) - - -def win_ascii_esc(): - if sys.platform == "win32": - colorama.init() diff --git a/FFMpeg-Compressor/modules/utils.py b/FFMpeg-Compressor/modules/utils.py deleted file mode 100644 index 907f9b1..0000000 --- a/FFMpeg-Compressor/modules/utils.py +++ /dev/null @@ -1,79 +0,0 @@ -from modules import configloader -from modules import printer -from shutil import copyfile -import sys -import os - -errors_count = 0 - - -def get_dir_size(directory, files): - total_size = 0 - for folder, folders, files in os.walk(directory): - for file in files: - if not os.path.islink(f"{folder}/{file}"): - total_size += os.path.getsize(f"{folder}/{file}") - return total_size - - -def get_compression(orig, comp): - processed_files = [] - for folder, folders, files in os.walk(comp): - for file in files: - processed_files.append(file) - try: - orig = get_dir_size(orig, processed_files) - comp = get_dir_size(comp, processed_files) - - print(f"\nResult: {orig/1024/1024:.2f}MB -> {comp/1024/1024:.2f}MB ({(comp - orig)/1024/1024:.2f}MB)") - except ZeroDivisionError: - printer.warning("Nothing compressed!") - - -def get_compression_status(orig_folder): - orig_folder_len = 0 - comp_folder_len = 0 - - for folder, folders, files in os.walk(orig_folder): - orig_folder_len += len(files) - - for folder, folders, files in os.walk(f'{orig_folder}_compressed'): - for file in files: - if not os.path.splitext(file)[1].count(" (copy)"): - comp_folder_len += 1 - - if errors_count != 0: - printer.warning("Some files failed to compress!") - - if orig_folder_len == comp_folder_len: - printer.info("Success!") - get_compression(orig_folder, f"{orig_folder}_compressed") - else: - printer.warning("Original and compressed folders are not identical!") - get_compression(orig_folder, f"{orig_folder}_compressed") - - -def add_unprocessed_file(orig_folder, new_folder): - if configloader.config['FFMPEG']['CopyUnprocessed']: - filename = orig_folder.split("/").pop() - copyfile(orig_folder, new_folder) - printer.info(f"File {filename} copied to compressed folder.") - - -def check_duplicates(new_folder): - filename = new_folder.split().pop() - if os.path.exists(new_folder): - printer.warning( - f'Duplicate file has been found! Check manually this files - "{filename}", ' - f'"{os.path.splitext(filename)[0] + "(copy)" + os.path.splitext(filename)[1]}"') - return os.path.splitext(new_folder)[0] + "(copy)" + os.path.splitext(new_folder)[1] - return new_folder - - -def sys_pause(): - if sys.platform == "win32": - os.system("pause") - - -def help_message(): - return "Usage: ffmpeg-comp {folder}" diff --git a/FFMpeg-Compressor/printer.py b/FFMpeg-Compressor/printer.py new file mode 100644 index 0000000..6080caf --- /dev/null +++ b/FFMpeg-Compressor/printer.py @@ -0,0 +1,46 @@ +from progress.bar import IncrementalBar +import colorama +import sys +import os + + +class Printer: + + def __init__(self, folder): + file_count = 0 + for folder, folders, file in os.walk(folder): + file_count += len(file) + self.bar = IncrementalBar('Compressing', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)') + self.bar.update() + + # Fill whole string with spaces for cleaning progress bar + @staticmethod + def clean_str(string): + return string + " " * (os.get_terminal_size().columns - len(string)) + + @staticmethod + def win_ascii_esc(): + if sys.platform == "win32": + colorama.init() + + def bar_print(self, string): + print(string) + self.bar.update() + + def info(self, string): + self.bar_print(self.clean_str(f"\r\033[100m- {string}\033[49m")) + + def warning(self, string): + self.bar_print(self.clean_str(f"\r\033[93m!\033[0m {string}\033[49m")) + + def error(self, string): + self.bar_print(self.clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m")) + + def files(self, source, dest, dest_ext, comment): + source_ext = os.path.splitext(source)[1] + source_name = os.path.splitext(source)[0] + + self.bar_print(self.clean_str(f"\r\033[0;32m\u2713\033[0m \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m")) + + def unknown_file(self, file): + self.bar_print(self.clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)")) diff --git a/FFMpeg-Compressor/utils.py b/FFMpeg-Compressor/utils.py new file mode 100644 index 0000000..48df9e9 --- /dev/null +++ b/FFMpeg-Compressor/utils.py @@ -0,0 +1,71 @@ +from shutil import copyfile +import sys +import os + +class Utils: + + def __init__(self, config, printer): + self.errors = 0 + self.config = config + self.printer = printer + + @staticmethod + def sys_pause(): + if sys.platform == "win32": + os.system("pause") + + @staticmethod + def get_size(directory): + total_size = 0 + for folder, folders, files in os.walk(directory): + for file in files: + if not os.path.islink(f"{folder}/{file}"): + total_size += os.path.getsize(f"{folder}/{file}") + return total_size + + def get_compression(self, source, output): + try: + source = self.get_size(source) + output = self.get_size(output) + + print(f"\nResult: {source/1024/1024:.2f}MB -> " + f"{output/1024/1024:.2f}MB ({(output - source)/1024/1024:.2f}MB)") + except ZeroDivisionError: + self.printer.warning("Nothing compressed!") + + def get_compression_status(self, orig_folder): + orig_folder_len = 0 + comp_folder_len = 0 + + for folder, folders, files in os.walk(orig_folder): + orig_folder_len += len(files) + + for folder, folders, files in os.walk(f'{orig_folder}_compressed'): + for file in files: + if not os.path.splitext(file)[1].count("(copy)"): + comp_folder_len += 1 + + if self.errors != 0: + self.printer.warning("Some files failed to compress!") + + if orig_folder_len == comp_folder_len: + self.printer.info("Success!") + self.get_compression(orig_folder, f"{orig_folder}_compressed") + else: + self.printer.warning("Original and compressed folders are not identical!") + self.get_compression(orig_folder, f"{orig_folder}_compressed") + + def add_unprocessed_file(self, orig_folder, new_folder): + if self.config['FFMPEG']['CopyUnprocessed']: + filename = orig_folder.split("/").pop() + copyfile(orig_folder, new_folder) + self.printer.info(f"File {filename} copied to compressed folder.") + + def check_duplicates(self, new_folder): + filename = new_folder.split().pop() + if os.path.exists(new_folder): + self.printer.warning( + f'Duplicate file has been found! Check manually this files - "{filename}", ' + f'"{os.path.splitext(filename)[0] + "(copy)" + os.path.splitext(filename)[1]}"') + return os.path.splitext(new_folder)[0] + "(copy)" + os.path.splitext(new_folder)[1] + return new_folder diff --git a/README.md b/README.md index 36f092f..370d3eb 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,25 @@ ## VNTools -Collection of tools used by administrators from VN Telegram Channel +Collection of tools used by VienDesu! Porting Team ### Tools -* `FFMpeg-Compressor` - Python utility uses ffmpeg to compress Visual Novel Resources -* `RenPy-Android-Unpack` - A simple Python script for unpacking Ren'Py based .apk and .obb files to ready to use Ren'Py SDK's Project +* `FFMpeg-Compressor` - Python utility to compress Visual Novel Resources +* `RenPy-Android-Unpack` - A Python script for extracting game project from Ren'Py based .apk and .obb files * `RenPy-Unpacker` - Simple .rpy script that will make any RenPy game unpack itself -* `VNDS-to-RenPy` - Simple script for converting vnds scripts to rpy +* `VNDS-to-RenPy` - Simple script for converting VNDS engine scripts to .rpy ones + +### Installation +#### Download from releases: + * Windows - `TODO` + * Linux - `TODO` + * MacOS - `TODO` + +#### Build tools as binaries: + * Run `./build.sh` on UNIX + * Run `.\build.bat` for Windows + * Arch Linux - `TODO` + * NixOS - `TODO` + +#### Install as python package: + * Run `pip install -U .` command in project folder + * Arch Linux - `TODO` + * NixOS - `TODO` diff --git a/RenPy-Android-Unpack/__init__.py b/RenPy-Android-Unpack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/RenPy-Android-Unpack/__main__.py b/RenPy-Android-Unpack/__main__.py new file mode 100755 index 0000000..8826079 --- /dev/null +++ b/RenPy-Android-Unpack/__main__.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +import colorama +import argparse +import sys +import os + +from printer import Printer +from actions import Actions + + +def args_init(): + parser = argparse.ArgumentParser( + prog='rendroid-unpack', + description='Extract Ren\'Py .apk and .obb files into Ren\'Py SDK\'s project' + ) + parser.add_argument('path') + parser.add_argument('-o', '--output') + return parser.parse_args() + + +if __name__ == '__main__': + if sys.platform == "win32": + colorama.init() + args = args_init() + if args.output: + output = args.output + else: + output = '' + actions = Actions(output) + printer = Printer() + + filename = args.path + if os.path.splitext(filename)[1] == '.apk' or os.path.splitext(filename)[1] == '.obb': + actions.clean(['assets'], True) + + printer.info(f'Extracting assets from {filename}... ') + actions.extract().assets(filename) + + printer.info('Renaming game assets... ') + actions.rename().files('assets') + actions.rename().dirs('assets') + + printer.info('Removing unneeded files... ') + if os.path.splitext(filename)[1] == '.apk': + actions.clean(['assets/renpy', 'assets/res'], False) + actions.clean(['assets/dexopt'], True) + + printer.info('Renaming directory... ') + actions.clean([os.path.splitext(filename)[0]], True) + os.rename(os.path.join(output, 'assets'), os.path.splitext(filename)[0]) + else: + Printer.err("It's not an .apk or .obb file!") diff --git a/RenPy-Android-Unpack/actions.py b/RenPy-Android-Unpack/actions.py new file mode 100755 index 0000000..e5d42e6 --- /dev/null +++ b/RenPy-Android-Unpack/actions.py @@ -0,0 +1,98 @@ +from zipfile import ZipFile, BadZipFile +from PIL import Image +import shutil +import os + +from printer import Printer + + +class Extract: + + def __init__(self, output: str): + self.output = output + + @staticmethod + def folder(zip_ref: ZipFile, path: str, dest: str): + for content in zip_ref.namelist(): + if content.split('/')[0] == path: + zip_ref.extract(content, dest) + + @staticmethod + def icon(directory: str): + icons = [] + for folder, folders, files in os.walk(directory): + for file in os.listdir(folder): + if os.path.splitext(file)[1] == ".png": + image = Image.open(f"{folder}/{file}") + if image.size[0] == 432 and image.size[1] == 432: + icons.append(f"{folder}/{file}") + if len(icons) == 0: + raise KeyError + return icons + + def assets(self, file: str): + try: + with ZipFile(file, 'r') as zip_ref: + self.folder(zip_ref, 'assets', self.output) + if os.path.splitext(file)[1] == '.apk': + try: + # ~Ren'Py 8, 7 + self.folder(zip_ref, 'res', os.path.join(self.output, 'assets')) + for icon in self.icon(os.path.join(self.output, 'assets/res')): + os.rename(icon, os.path.join(self.output, "assets", os.path.split(icon)[1])) + except KeyError: + try: + # ~Ren'Py 6 + zip_ref.extract('res/drawable/icon.png', os.path.join(self.output, 'assets')) + os.rename(os.path.join(self.output, 'assets/res/drawable/icon.png'), + os.path.join(self.output, 'assets/icon.png')) + except KeyError: + Printer.warn("Icon not found. Maybe it is not supported apk?") + except BadZipFile: + Printer.err("Cant extract .apk file!") + + +class Rename: + + def __init__(self, output): + self.output = output + + def files(self, directory: str): + for dir_ in os.walk(os.path.join(self.output, directory)): + for file in dir_[2]: + path = f'{dir_[0]}/{file}' + folder = '/'.join(path.split('/')[:len(path.split('/')) - 1]) + newname = f'{path.split("/").pop().replace("x-", "")}' + os.rename(path, f'{folder}/{newname}') + + def dirs(self, directory: str): + dirs = [] + for dir_ in os.walk(os.path.join(self.output, directory)): + dirs.append(dir_[0]) + dirs.reverse() + dirs.pop() + for dir__ in dirs: + folder = '/'.join(dir__.split('/')[:len(dir__.split('/')) - 1]) + newname = f'{dir__.split("/").pop().replace("x-", "")}' + os.rename(dir__, f'{folder}/{newname}') + + +class Actions: + + def __init__(self, output: str): + self.output = output + + def extract(self) -> Extract: + return Extract(self.output) + + def rename(self) -> Rename: + return Rename(self.output) + + def clean(self, names: list, ignore: bool): + for name in names: + name = os.path.join(self.output, name) + try: + shutil.rmtree(name) + except FileNotFoundError: + if not ignore: + Printer.warn(f"Path {name} not found!") diff --git a/RenPy-Android-Unpack/printer.py b/RenPy-Android-Unpack/printer.py new file mode 100644 index 0000000..2c3d31b --- /dev/null +++ b/RenPy-Android-Unpack/printer.py @@ -0,0 +1,14 @@ +class Printer: + + @staticmethod + def info(msg: str): + print(f"\033[100m[INFO] {msg}\033[49m") + + @staticmethod + def warn(msg: str): + print(f"\033[93m[WARN]\033[0m {msg}\033[49m") + + @staticmethod + def err(msg: str): + print(f"\033[31m[ERROR]\033[0m {msg} Exiting...\033[49m") + exit() diff --git a/RenPy-Android-Unpack/requirements.txt b/RenPy-Android-Unpack/requirements.txt deleted file mode 100644 index 3a6b560..0000000 --- a/RenPy-Android-Unpack/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -Pillow==9.5.0 -colorama==0.4.6 \ No newline at end of file diff --git a/RenPy-Android-Unpack/unpack.py b/RenPy-Android-Unpack/unpack.py deleted file mode 100755 index 8b74db7..0000000 --- a/RenPy-Android-Unpack/unpack.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3 -from PIL import Image -import colorama -import zipfile -import shutil -import os -import sys - - -def printer(msg, level): - match level: - case "info": - print(f"\033[100m[INFO] {msg}\033[49m") - case "warn": - print(f"\033[93m[WARN]\033[0m {msg}\033[49m") - case "err": - print(f"\033[31m[ERROR]\033[0m {msg} Exiting...\033[49m") - exit() - - -def extract_folder(zip_ref, path, dest): - for content in zip_ref.namelist(): - if content.split('/')[0] == path: - zip_ref.extract(content, dest) - - -def find_modern_icon(directory): - icons = [] - for folder, folders, files in os.walk(directory): - for file in os.listdir(folder): - if os.path.splitext(file)[1] == ".png": - image = Image.open(f"{folder}/{file}") - if image.size[0] == 432 and image.size[1] == 432: - icons.append(f"{folder}/{file}") - if len(icons) == 0: - raise KeyError - return icons - - -def extract_assets(file): - try: - with zipfile.ZipFile(file, 'r') as zip_ref: - extract_folder(zip_ref, 'assets', '') - if os.path.splitext(file)[1] == '.apk': - try: - # ~Ren'Py 8, 7 - extract_folder(zip_ref, 'res', 'assets') - for icon in find_modern_icon('assets/res'): - os.rename(icon, f"assets/{os.path.split(icon)[1]}") - except KeyError: - try: - # ~Ren'Py 6 - zip_ref.extract('res/drawable/icon.png', 'assets') - os.rename('assets/res/drawable/icon.png', 'assets/icon.png') - except KeyError: - printer("Icon not found. Maybe it is not supported apk?", "warn") - except zipfile.BadZipFile: - return printer("Cant extract .apk file!", "err") - - -def rename_files(directory): - for dir_ in os.walk(directory): - for file in dir_[2]: - path = f'{dir_[0]}/{file}' - folder = '/'.join(path.split('/')[:len(path.split('/')) - 1]) - newname = f'{path.split("/").pop().replace("x-", "")}' - os.rename(path, f'{folder}/{newname}') - - -def rename_dirs(directory): - dirs = [] - for dir_ in os.walk(directory): - dirs.append(dir_[0]) - dirs.reverse() - dirs.pop() - for dir__ in dirs: - folder = '/'.join(dir__.split('/')[:len(dir__.split('/')) - 1]) - newname = f'{dir__.split("/").pop().replace("x-", "")}' - os.rename(dir__, f'{folder}/{newname}') - - -def remove_unneeded(names, ignore): - for name in names: - try: - shutil.rmtree(name) - except FileNotFoundError: - if not ignore: - printer(f"Path {name} not found!", "warn") - - -if __name__ == '__main__': - if sys.platform == "win32": - colorama.init() - for filename in os.listdir(os.getcwd()): - if os.path.splitext(filename)[1] == '.apk' or os.path.splitext(filename)[1] == '.obb': - remove_unneeded(['assets'], True) - printer(f'Extracting assets from {filename}... ', "info") - extract_assets(filename) - printer('Renaming game assets... ', "info") - rename_files('assets') - rename_dirs('assets') - printer('Removing unneeded files... ', "info") - if os.path.splitext(filename)[1] == '.apk': - remove_unneeded(['assets/renpy', 'assets/res'], False) - remove_unneeded(['assets/dexopt'], True) - printer('Renaming directory... ', "info") - remove_unneeded([os.path.splitext(filename)[0]], True) - os.rename('assets', os.path.splitext(filename)[0]) diff --git a/VNDS-to-RenPy/__init__.py b/VNDS-to-RenPy/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/VNDS-to-RenPy/convert.py b/VNDS-to-RenPy/__main__.py similarity index 100% rename from VNDS-to-RenPy/convert.py rename to VNDS-to-RenPy/__main__.py diff --git a/build.bat b/build.bat old mode 100644 new mode 100755 index 237c7e3..688d43d --- a/build.bat +++ b/build.bat @@ -3,16 +3,16 @@ if not defined VIRTUAL_ENV goto :venv_error mkdir output mkdir output\bin -python -m pip install -r FFMpeg-Compressor\requirements.txt || goto :exit -python -m pip install -r RenPy-Android-Unpack\requirements.txt || goto :exit +python -m pip install -r requirements.txt || goto :exit python -m pip install Nuitka || goto :exit -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=ffmpeg-comp FFMpeg-Compressor\main.py || goto :exit +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=ffmpeg-comp FFMpeg-Compressor\__main__.py || goto :exit xcopy FFMpeg-Compressor\ffmpeg-comp.toml output\bin /Y move /Y output\ffmpeg-comp.exe output\bin -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=rendroid-unpack RenPy-Android-Unpack\unpack.py || goto :exit +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=rendroid-unpack RenPy-Android-Unpack\__main__.py || goto :exit move /Y output\rendroid-unpack.exe output\bin -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=vnds2renpy VNDS-to-RenPy/convert.py || goto :exit +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=vnds2renpy VNDS-to-RenPy/__main__.py || goto :exit move /Y output\vnds2renpy.exe output\bin +echo "Done! You can get binaries into output\bin directory" :venv_error echo "Please create and activate venv before running this script: python -m venv .\venv && .\venv\Scripts\activate.bat" diff --git a/build.sh b/build.sh index 7e21bbf..90e774e 100755 --- a/build.sh +++ b/build.sh @@ -8,17 +8,17 @@ fi mkdir -p output mkdir -p output/bin -python3 -m pip install -r FFMpeg-Compressor/requirements.txt -python3 -m pip install -r RenPy-Android-Unpack/requirements.txt +python3 -m pip install -r requirements.txt python3 -m pip install Nuitka case "$(uname -s)" in Linux*) jobs="--jobs=$(nproc)";; Darwin*) jobs="--jobs=$(sysctl -n hw.ncpu)";; esac -python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=ffmpeg-comp FFMpeg-Compressor/main.py +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=ffmpeg-comp FFMpeg-Compressor/__main__.py cp FFMpeg-Compressor/ffmpeg-comp.toml output/bin mv output/ffmpeg-comp output/bin -python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=rendroid-unpack RenPy-Android-Unpack/unpack.py +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=rendroid-unpack RenPy-Android-Unpack/__main__.py mv output/rendroid-unpack output/bin -python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=vnds2renpy VNDS-to-RenPy/convert.py -mv output/vnds2renpy output/bin \ No newline at end of file +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=vnds2renpy VNDS-to-RenPy/__main__.py +mv output/vnds2renpy output/bin +echo "Done! You can get binaries into output/bin directory" \ No newline at end of file diff --git a/FFMpeg-Compressor/requirements.txt b/requirements.txt similarity index 100% rename from FFMpeg-Compressor/requirements.txt rename to requirements.txt From 85df574d3c1fff90f9775000f4e0fdd1db22b281 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Thu, 29 Aug 2024 02:34:10 +0300 Subject: [PATCH 072/105] Python packaging --- .gitignore | 1 + .../README.md | 0 .../__init__.py | 0 RenDroidUnpack/__main__.py | 6 ++++ .../actions.py | 2 +- .../application.py | 7 ++--- .../printer.py | 0 {RenPy-Unpacker => RenPyRipper}/README.md | 0 {RenPy-Unpacker => RenPyRipper}/unpack.rpy | 0 {VNDS-to-RenPy => VNDS2RenPy}/README.md | 0 .../__init__.py | 0 {VNDS-to-RenPy => VNDS2RenPy}/__main__.py | 0 {FFMpeg-Compressor => VNRecode}/README.md | 0 {VNDS-to-RenPy => VNRecode}/__init__.py | 0 {FFMpeg-Compressor => VNRecode}/__main__.py | 10 +++--- .../application.py | 0 {FFMpeg-Compressor => VNRecode}/compress.py | 0 {FFMpeg-Compressor => VNRecode}/config.py | 1 + .../ffmpeg-comp.toml | 0 {FFMpeg-Compressor => VNRecode}/printer.py | 0 {FFMpeg-Compressor => VNRecode}/utils.py | 0 pyproject.toml | 31 +++++++++++++++++++ 22 files changed, 48 insertions(+), 10 deletions(-) rename {RenPy-Android-Unpack => RenDroidUnpack}/README.md (100%) rename {FFMpeg-Compressor => RenDroidUnpack}/__init__.py (100%) create mode 100755 RenDroidUnpack/__main__.py rename {RenPy-Android-Unpack => RenDroidUnpack}/actions.py (99%) rename RenPy-Android-Unpack/__main__.py => RenDroidUnpack/application.py (94%) mode change 100755 => 100644 rename {RenPy-Android-Unpack => RenDroidUnpack}/printer.py (100%) rename {RenPy-Unpacker => RenPyRipper}/README.md (100%) rename {RenPy-Unpacker => RenPyRipper}/unpack.rpy (100%) rename {VNDS-to-RenPy => VNDS2RenPy}/README.md (100%) rename {RenPy-Android-Unpack => VNDS2RenPy}/__init__.py (100%) rename {VNDS-to-RenPy => VNDS2RenPy}/__main__.py (100%) rename {FFMpeg-Compressor => VNRecode}/README.md (100%) rename {VNDS-to-RenPy => VNRecode}/__init__.py (100%) rename {FFMpeg-Compressor => VNRecode}/__main__.py (67%) rename {FFMpeg-Compressor => VNRecode}/application.py (100%) rename {FFMpeg-Compressor => VNRecode}/compress.py (100%) rename {FFMpeg-Compressor => VNRecode}/config.py (97%) rename {FFMpeg-Compressor => VNRecode}/ffmpeg-comp.toml (100%) rename {FFMpeg-Compressor => VNRecode}/printer.py (100%) rename {FFMpeg-Compressor => VNRecode}/utils.py (100%) create mode 100644 pyproject.toml diff --git a/.gitignore b/.gitignore index abe0419..d9d86ba 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /tests/ /build/ /VNTools.egg-info/ +/dist/ diff --git a/RenPy-Android-Unpack/README.md b/RenDroidUnpack/README.md similarity index 100% rename from RenPy-Android-Unpack/README.md rename to RenDroidUnpack/README.md diff --git a/FFMpeg-Compressor/__init__.py b/RenDroidUnpack/__init__.py similarity index 100% rename from FFMpeg-Compressor/__init__.py rename to RenDroidUnpack/__init__.py diff --git a/RenDroidUnpack/__main__.py b/RenDroidUnpack/__main__.py new file mode 100755 index 0000000..6e79193 --- /dev/null +++ b/RenDroidUnpack/__main__.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + +from . import application + +if __name__ == '__main__': + application.launch() diff --git a/RenPy-Android-Unpack/actions.py b/RenDroidUnpack/actions.py similarity index 99% rename from RenPy-Android-Unpack/actions.py rename to RenDroidUnpack/actions.py index e5d42e6..489eee9 100755 --- a/RenPy-Android-Unpack/actions.py +++ b/RenDroidUnpack/actions.py @@ -3,7 +3,7 @@ from PIL import Image import shutil import os -from printer import Printer +from .printer import Printer class Extract: diff --git a/RenPy-Android-Unpack/__main__.py b/RenDroidUnpack/application.py old mode 100755 new mode 100644 similarity index 94% rename from RenPy-Android-Unpack/__main__.py rename to RenDroidUnpack/application.py index 8826079..91d1577 --- a/RenPy-Android-Unpack/__main__.py +++ b/RenDroidUnpack/application.py @@ -4,8 +4,8 @@ import argparse import sys import os -from printer import Printer -from actions import Actions +from .printer import Printer +from .actions import Actions def args_init(): @@ -17,8 +17,7 @@ def args_init(): parser.add_argument('-o', '--output') return parser.parse_args() - -if __name__ == '__main__': +def launch(): if sys.platform == "win32": colorama.init() args = args_init() diff --git a/RenPy-Android-Unpack/printer.py b/RenDroidUnpack/printer.py similarity index 100% rename from RenPy-Android-Unpack/printer.py rename to RenDroidUnpack/printer.py diff --git a/RenPy-Unpacker/README.md b/RenPyRipper/README.md similarity index 100% rename from RenPy-Unpacker/README.md rename to RenPyRipper/README.md diff --git a/RenPy-Unpacker/unpack.rpy b/RenPyRipper/unpack.rpy similarity index 100% rename from RenPy-Unpacker/unpack.rpy rename to RenPyRipper/unpack.rpy diff --git a/VNDS-to-RenPy/README.md b/VNDS2RenPy/README.md similarity index 100% rename from VNDS-to-RenPy/README.md rename to VNDS2RenPy/README.md diff --git a/RenPy-Android-Unpack/__init__.py b/VNDS2RenPy/__init__.py similarity index 100% rename from RenPy-Android-Unpack/__init__.py rename to VNDS2RenPy/__init__.py diff --git a/VNDS-to-RenPy/__main__.py b/VNDS2RenPy/__main__.py similarity index 100% rename from VNDS-to-RenPy/__main__.py rename to VNDS2RenPy/__main__.py diff --git a/FFMpeg-Compressor/README.md b/VNRecode/README.md similarity index 100% rename from FFMpeg-Compressor/README.md rename to VNRecode/README.md diff --git a/VNDS-to-RenPy/__init__.py b/VNRecode/__init__.py similarity index 100% rename from VNDS-to-RenPy/__init__.py rename to VNRecode/__init__.py diff --git a/FFMpeg-Compressor/__main__.py b/VNRecode/__main__.py similarity index 67% rename from FFMpeg-Compressor/__main__.py rename to VNRecode/__main__.py index aef4d34..4e7fb68 100644 --- a/FFMpeg-Compressor/__main__.py +++ b/VNRecode/__main__.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -from application import Application -from compress import Compress -from printer import Printer -from config import Config -from utils import Utils +from .application import Application +from .compress import Compress +from .printer import Printer +from .config import Config +from .utils import Utils def init(): diff --git a/FFMpeg-Compressor/application.py b/VNRecode/application.py similarity index 100% rename from FFMpeg-Compressor/application.py rename to VNRecode/application.py diff --git a/FFMpeg-Compressor/compress.py b/VNRecode/compress.py similarity index 100% rename from FFMpeg-Compressor/compress.py rename to VNRecode/compress.py diff --git a/FFMpeg-Compressor/config.py b/VNRecode/config.py similarity index 97% rename from FFMpeg-Compressor/config.py rename to VNRecode/config.py index d36e3d4..a556c5e 100644 --- a/FFMpeg-Compressor/config.py +++ b/VNRecode/config.py @@ -24,4 +24,5 @@ class Config: config = tomllib.load(cfile) else: print("Failed to find config. Check `ffmpeg-comp -h` to more info") + exit(255) return cls(config=config, args=args) diff --git a/FFMpeg-Compressor/ffmpeg-comp.toml b/VNRecode/ffmpeg-comp.toml similarity index 100% rename from FFMpeg-Compressor/ffmpeg-comp.toml rename to VNRecode/ffmpeg-comp.toml diff --git a/FFMpeg-Compressor/printer.py b/VNRecode/printer.py similarity index 100% rename from FFMpeg-Compressor/printer.py rename to VNRecode/printer.py diff --git a/FFMpeg-Compressor/utils.py b/VNRecode/utils.py similarity index 100% rename from FFMpeg-Compressor/utils.py rename to VNRecode/utils.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..766fb7f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = [ + "setuptools >= 61.0" +] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = ["VNRecode", "RenDroidUnpack", "VNDS2RenPy"] +include-package-data = true + +[tool.setuptools.package-data] +'VNRecode' = ['*.py'] +'VNDS2RenPy' = ['*.py'] +'RenDroidUnpack' = ['*.py'] + +[project.scripts] +vnrecode = "VNRecode.__main__:init" +vnds2renpy = "VNDS2RenPy.__main__:main" +rendroid-unpack = "RenDroidUnpack.application:launch" + +[project] +name = "vntools" +version = "2.0-dev" +requires-python = ">= 3.11" +dependencies = [ + "Pillow==10.3.0", + "pillow-avif-plugin==1.4.3", + "python-ffmpeg==2.0.12", + "progress==1.6", + "colorama==0.4.6" +] \ No newline at end of file From e5fa49ad53cc6b6191ae9ddc0b666e4fa1461841 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Thu, 29 Aug 2024 02:49:44 +0300 Subject: [PATCH 073/105] Fix packaging names --- .gitignore | 2 +- build.bat | 12 ++++++------ build.sh | 12 ++++++------ pyproject.toml | 14 +++++++------- {RenPyRipper => renpy-ripper}/README.md | 0 RenPyRipper/unpack.rpy => renpy-ripper/ripper.rpy | 0 {RenDroidUnpack => unrenapk}/README.md | 0 {RenDroidUnpack => unrenapk}/__init__.py | 0 {RenDroidUnpack => unrenapk}/__main__.py | 0 {RenDroidUnpack => unrenapk}/actions.py | 0 {RenDroidUnpack => unrenapk}/application.py | 2 +- {RenDroidUnpack => unrenapk}/printer.py | 0 {VNDS2RenPy => vnds2renpy}/README.md | 0 {VNDS2RenPy => vnds2renpy}/__init__.py | 0 {VNDS2RenPy => vnds2renpy}/__main__.py | 0 {VNRecode => vnrecode}/README.md | 0 {VNRecode => vnrecode}/__init__.py | 0 {VNRecode => vnrecode}/__main__.py | 0 {VNRecode => vnrecode}/application.py | 0 {VNRecode => vnrecode}/compress.py | 0 {VNRecode => vnrecode}/config.py | 6 +++--- {VNRecode => vnrecode}/printer.py | 0 {VNRecode => vnrecode}/utils.py | 0 .../ffmpeg-comp.toml => vnrecode/vnrecode.toml | 0 24 files changed, 24 insertions(+), 24 deletions(-) rename {RenPyRipper => renpy-ripper}/README.md (100%) rename RenPyRipper/unpack.rpy => renpy-ripper/ripper.rpy (100%) rename {RenDroidUnpack => unrenapk}/README.md (100%) rename {RenDroidUnpack => unrenapk}/__init__.py (100%) rename {RenDroidUnpack => unrenapk}/__main__.py (100%) rename {RenDroidUnpack => unrenapk}/actions.py (100%) rename {RenDroidUnpack => unrenapk}/application.py (97%) rename {RenDroidUnpack => unrenapk}/printer.py (100%) rename {VNDS2RenPy => vnds2renpy}/README.md (100%) rename {VNDS2RenPy => vnds2renpy}/__init__.py (100%) rename {VNDS2RenPy => vnds2renpy}/__main__.py (100%) rename {VNRecode => vnrecode}/README.md (100%) rename {VNRecode => vnrecode}/__init__.py (100%) rename {VNRecode => vnrecode}/__main__.py (100%) rename {VNRecode => vnrecode}/application.py (100%) rename {VNRecode => vnrecode}/compress.py (100%) rename {VNRecode => vnrecode}/config.py (76%) rename {VNRecode => vnrecode}/printer.py (100%) rename {VNRecode => vnrecode}/utils.py (100%) rename VNRecode/ffmpeg-comp.toml => vnrecode/vnrecode.toml (100%) diff --git a/.gitignore b/.gitignore index d9d86ba..e6aa831 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ /output/ /tests/ /build/ -/VNTools.egg-info/ /dist/ +/vntools.egg-info/ diff --git a/build.bat b/build.bat index 688d43d..89343d1 100755 --- a/build.bat +++ b/build.bat @@ -5,12 +5,12 @@ mkdir output mkdir output\bin python -m pip install -r requirements.txt || goto :exit python -m pip install Nuitka || goto :exit -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=ffmpeg-comp FFMpeg-Compressor\__main__.py || goto :exit -xcopy FFMpeg-Compressor\ffmpeg-comp.toml output\bin /Y -move /Y output\ffmpeg-comp.exe output\bin -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=rendroid-unpack RenPy-Android-Unpack\__main__.py || goto :exit -move /Y output\rendroid-unpack.exe output\bin -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=vnds2renpy VNDS-to-RenPy/__main__.py || goto :exit +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=vnrecode vnrecode\__main__.py || goto :exit +xcopy vnrecode\vnrecode.toml output\bin /Y +move /Y output\vnrecode.exe output\bin +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=unrenapk unrenapk\__main__.py || goto :exit +move /Y output\unrenapk.exe output\bin +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=vnds2renpy vnds2renpy/__main__.py || goto :exit move /Y output\vnds2renpy.exe output\bin echo "Done! You can get binaries into output\bin directory" diff --git a/build.sh b/build.sh index 90e774e..a7cf3b3 100755 --- a/build.sh +++ b/build.sh @@ -14,11 +14,11 @@ case "$(uname -s)" in Linux*) jobs="--jobs=$(nproc)";; Darwin*) jobs="--jobs=$(sysctl -n hw.ncpu)";; esac -python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=ffmpeg-comp FFMpeg-Compressor/__main__.py -cp FFMpeg-Compressor/ffmpeg-comp.toml output/bin -mv output/ffmpeg-comp output/bin -python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=rendroid-unpack RenPy-Android-Unpack/__main__.py -mv output/rendroid-unpack output/bin -python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=vnds2renpy VNDS-to-RenPy/__main__.py +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=vnrecode vnrecode/__main__.py +cp vnrecode/vnrecode.toml output/bin +mv output/vnrecode output/bin +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=unrenapk unrenapk/__main__.py +mv output/unrenapk output/bin +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=vnds2renpy vnds2renpy/__main__.py mv output/vnds2renpy output/bin echo "Done! You can get binaries into output/bin directory" \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 766fb7f..be74c59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,18 +5,18 @@ requires = [ build-backend = "setuptools.build_meta" [tool.setuptools] -packages = ["VNRecode", "RenDroidUnpack", "VNDS2RenPy"] +packages = ["vnrecode", "unrenapk", "vnds2renpy"] include-package-data = true [tool.setuptools.package-data] -'VNRecode' = ['*.py'] -'VNDS2RenPy' = ['*.py'] -'RenDroidUnpack' = ['*.py'] +'vnrecode' = ['*.py', '*.toml'] +'vnds2renpy' = ['*.py'] +'unrenapk' = ['*.py'] [project.scripts] -vnrecode = "VNRecode.__main__:init" -vnds2renpy = "VNDS2RenPy.__main__:main" -rendroid-unpack = "RenDroidUnpack.application:launch" +vnrecode = "vnrecode.__main__:init" +vnds2renpy = "vnds2renpy.__main__:main" +unrenapk = "unrenapk.application:launch" [project] name = "vntools" diff --git a/RenPyRipper/README.md b/renpy-ripper/README.md similarity index 100% rename from RenPyRipper/README.md rename to renpy-ripper/README.md diff --git a/RenPyRipper/unpack.rpy b/renpy-ripper/ripper.rpy similarity index 100% rename from RenPyRipper/unpack.rpy rename to renpy-ripper/ripper.rpy diff --git a/RenDroidUnpack/README.md b/unrenapk/README.md similarity index 100% rename from RenDroidUnpack/README.md rename to unrenapk/README.md diff --git a/RenDroidUnpack/__init__.py b/unrenapk/__init__.py similarity index 100% rename from RenDroidUnpack/__init__.py rename to unrenapk/__init__.py diff --git a/RenDroidUnpack/__main__.py b/unrenapk/__main__.py similarity index 100% rename from RenDroidUnpack/__main__.py rename to unrenapk/__main__.py diff --git a/RenDroidUnpack/actions.py b/unrenapk/actions.py similarity index 100% rename from RenDroidUnpack/actions.py rename to unrenapk/actions.py diff --git a/RenDroidUnpack/application.py b/unrenapk/application.py similarity index 97% rename from RenDroidUnpack/application.py rename to unrenapk/application.py index 91d1577..b66d6ba 100644 --- a/RenDroidUnpack/application.py +++ b/unrenapk/application.py @@ -10,7 +10,7 @@ from .actions import Actions def args_init(): parser = argparse.ArgumentParser( - prog='rendroid-unpack', + prog='unrenapk', description='Extract Ren\'Py .apk and .obb files into Ren\'Py SDK\'s project' ) parser.add_argument('path') diff --git a/RenDroidUnpack/printer.py b/unrenapk/printer.py similarity index 100% rename from RenDroidUnpack/printer.py rename to unrenapk/printer.py diff --git a/VNDS2RenPy/README.md b/vnds2renpy/README.md similarity index 100% rename from VNDS2RenPy/README.md rename to vnds2renpy/README.md diff --git a/VNDS2RenPy/__init__.py b/vnds2renpy/__init__.py similarity index 100% rename from VNDS2RenPy/__init__.py rename to vnds2renpy/__init__.py diff --git a/VNDS2RenPy/__main__.py b/vnds2renpy/__main__.py similarity index 100% rename from VNDS2RenPy/__main__.py rename to vnds2renpy/__main__.py diff --git a/VNRecode/README.md b/vnrecode/README.md similarity index 100% rename from VNRecode/README.md rename to vnrecode/README.md diff --git a/VNRecode/__init__.py b/vnrecode/__init__.py similarity index 100% rename from VNRecode/__init__.py rename to vnrecode/__init__.py diff --git a/VNRecode/__main__.py b/vnrecode/__main__.py similarity index 100% rename from VNRecode/__main__.py rename to vnrecode/__main__.py diff --git a/VNRecode/application.py b/vnrecode/application.py similarity index 100% rename from VNRecode/application.py rename to vnrecode/application.py diff --git a/VNRecode/compress.py b/vnrecode/compress.py similarity index 100% rename from VNRecode/compress.py rename to vnrecode/compress.py diff --git a/VNRecode/config.py b/vnrecode/config.py similarity index 76% rename from VNRecode/config.py rename to vnrecode/config.py index a556c5e..a7cf0ed 100644 --- a/VNRecode/config.py +++ b/vnrecode/config.py @@ -13,16 +13,16 @@ class Config: @classmethod def setup_config(cls): - parser = ArgumentParser(prog="ffmpeg-comp", + parser = ArgumentParser(prog="vnrecode", description="Python utility to compress Visual Novel Resources" ) parser.add_argument("source") - parser.add_argument("-c", "--config", default="ffmpeg-comp.toml") + parser.add_argument("-c", "--config", default="vnrecode.toml") args = parser.parse_args() if os.path.isfile(args.config): with open(args.config, "rb") as cfile: config = tomllib.load(cfile) else: - print("Failed to find config. Check `ffmpeg-comp -h` to more info") + print("Failed to find config. Check `vnrecode -h` to more info") exit(255) return cls(config=config, args=args) diff --git a/VNRecode/printer.py b/vnrecode/printer.py similarity index 100% rename from VNRecode/printer.py rename to vnrecode/printer.py diff --git a/VNRecode/utils.py b/vnrecode/utils.py similarity index 100% rename from VNRecode/utils.py rename to vnrecode/utils.py diff --git a/VNRecode/ffmpeg-comp.toml b/vnrecode/vnrecode.toml similarity index 100% rename from VNRecode/ffmpeg-comp.toml rename to vnrecode/vnrecode.toml From f240fdca5f5c2a0ebcd02d9d6f9c373b01ac23d7 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Thu, 29 Aug 2024 04:01:41 +0300 Subject: [PATCH 074/105] vnrecode: add cli parameters for configuration --- vnrecode/__main__.py | 8 +++--- vnrecode/application.py | 8 +++--- vnrecode/compress.py | 40 +++++++++++++++--------------- vnrecode/config.py | 26 +++++++++++++++++--- vnrecode/params.py | 54 +++++++++++++++++++++++++++++++++++++++++ vnrecode/utils.py | 6 ++--- 6 files changed, 109 insertions(+), 33 deletions(-) create mode 100644 vnrecode/params.py diff --git a/vnrecode/__main__.py b/vnrecode/__main__.py index 4e7fb68..39ef2d6 100644 --- a/vnrecode/__main__.py +++ b/vnrecode/__main__.py @@ -2,17 +2,19 @@ from .application import Application from .compress import Compress from .printer import Printer +from .params import Params from .config import Config from .utils import Utils def init(): config = Config.setup_config() + params = Params.setup(config.config, config.args) printer = Printer(config.args.source) - utils = Utils(config.config, printer) - compress = Compress(config.config, printer, utils) + utils = Utils(params, printer) + compress = Compress(params, printer, utils) - Application(config, compress, printer, utils).run() + Application(params, config.args, compress, printer, utils).run() if __name__ == "__main__": diff --git a/vnrecode/application.py b/vnrecode/application.py index 491267a..75e3107 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -7,9 +7,9 @@ import os class Application: - def __init__(self, config, compress, printer, utils): - self.config = config.config - self.args = config.args + def __init__(self, params, args, compress, printer, utils): + self.params = params + self.args = args self.compress = compress.compress self.printer = printer self.utils = utils @@ -35,7 +35,7 @@ class Application: self.printer.info(f'Compressing "{folder.replace(source, os.path.split(source)[-1])}" folder...') output = folder.replace(source, f"{source}_compressed") - with ThreadPoolExecutor(max_workers=self.config["FFMPEG"]["Workers"]) as executor: + with ThreadPoolExecutor(max_workers=self.params.workers) as executor: futures = [ executor.submit(self.compress, folder, file, source, output) for file in files if os.path.isfile(f'{folder}/{file}') diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 5c1a465..7dbc6fc 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -40,13 +40,13 @@ class File: class Compress: - def __init__(self, config, printer, utils): - self.config = config + def __init__(self, params, printer, utils): + self.params = params self.printer = printer self.utils = utils def audio(self, folder, file, target_folder, extension): - bitrate = self.config['AUDIO']['BitRate'] + bitrate = self.params.audio_bitrate try: (FFmpeg() .input(f'{folder}/{file}') @@ -58,16 +58,16 @@ class Compress: except FFmpegError as e: self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') self.utils.errors += 1 - if not self.config['FFMPEG']['HideErrors']: + if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") self.printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}") return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' def video(self, folder, file, target_folder, extension): - if not self.config['VIDEO']['SkipVideo']: - codec = self.config['VIDEO']['Codec'] - crf = self.config['VIDEO']['CRF'] + if not self.params.video_skip: + codec = self.params.video_codec + crf = self.params.video_crf try: (FFmpeg() @@ -82,7 +82,7 @@ class Compress: except FFmpegError as e: self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') self.utils.errors += 1 - if not self.config['FFMPEG']['HideErrors']: + if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") else: self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') @@ -90,20 +90,20 @@ class Compress: def image(self, folder, file, target_folder, extension): - quality = self.config['IMAGE']['Quality'] + quality = self.params.image_quality try: image = Image.open(f'{folder}/{file}') if (extension == "jpg" or extension == "jpeg" or - (extension == "webp" and not self.config['FFMPEG']['WebpRGBA'])): + (extension == "webp" and not self.params.webp_rgba)): if File.has_transparency(image): self.printer.warning(f"{file} has transparency. Changing to fallback...") - extension = self.config['IMAGE']['FallBackExtension'] + extension = self.params.image_fall_ext if File.has_transparency(image): image.convert('RGBA') - res_downscale = self.config['IMAGE']['ResDownScale'] + res_downscale = self.params.image_downscale if res_downscale != 1: width, height = image.size new_size = (int(width / res_downscale), int(height / res_downscale)) @@ -111,20 +111,20 @@ class Compress: image.save(self.utils.check_duplicates(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}"), optimize=True, - lossless=self.config['IMAGE']['Lossless'], + lossless=self.params.image_lossless, quality=quality, minimize_size=True) self.printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") except Exception as e: self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') self.utils.errors += 1 - if not self.config['FFMPEG']['HideErrors']: + if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' def unknown(self, folder, file, target_folder): - if self.config["FFMPEG"]["ForceCompress"]: + if self.params.force_compress: self.printer.unknown_file(file) try: (FFmpeg() @@ -135,7 +135,7 @@ class Compress: except FFmpegError as e: self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') self.utils.errors += 1 - if not self.config['FFMPEG']['HideErrors']: + if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") else: self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') @@ -145,15 +145,15 @@ class Compress: def compress(self, _dir, filename, source, output): match File.get_type(filename): case "audio": - out_file = self.audio(_dir, filename, output, self.config['AUDIO']['Extension']) + out_file = self.audio(_dir, filename, output, self.params.audio_ext) case "image": - out_file = self.image(_dir, filename, output, self.config['IMAGE']['Extension']) + out_file = self.image(_dir, filename, output, self.params.image_ext) case "video": - out_file = self.video(_dir, filename, output, self.config['VIDEO']['Extension']) + out_file = self.video(_dir, filename, output, self.params.video_ext) case "unknown": out_file = self.unknown(_dir, filename, output) - if self.config['FFMPEG']['MimicMode']: + if self.params.mimic_mode: try: os.rename(out_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed")) except FileNotFoundError: diff --git a/vnrecode/config.py b/vnrecode/config.py index a7cf0ed..4d1336c 100644 --- a/vnrecode/config.py +++ b/vnrecode/config.py @@ -1,8 +1,9 @@ -import os.path from argparse import Namespace, ArgumentParser from dataclasses import dataclass +from sysconfig import get_path from typing import Any import tomllib +import os @dataclass @@ -13,11 +14,30 @@ class Config: @classmethod def setup_config(cls): + default_config = os.path.join(get_path('purelib'), "vnrecode", "vnrecode.toml") parser = ArgumentParser(prog="vnrecode", description="Python utility to compress Visual Novel Resources" ) - parser.add_argument("source") - parser.add_argument("-c", "--config", default="vnrecode.toml") + parser.add_argument("source", help="SourceDir") + parser.add_argument("-c", "--config", default=default_config, help="ConfigFile") + parser.add_argument("-u", action="store_true", help="CopyUnprocessed") + parser.add_argument("-f", "--force", action="store_true", help="ForceCompress") + parser.add_argument("-m", "--mimic", action="store_true", help="MimicMode") + parser.add_argument("-s", "--silent", action="store_true", help="HideErrors") + parser.add_argument("--webprgba", action="store_true", help="WebpRGBA") + parser.add_argument("-j", "--jobs", type=int, help="Workers") + parser.add_argument("-ae", "--aext", help="Audio Extension") + parser.add_argument("-ab", "--abit", help="Audio Bitrate") + parser.add_argument("-id", "--idown", type=int, help="Image Downscale") + parser.add_argument("-ie", "--iext", help="Image Extension") + parser.add_argument("-ife", "--ifallext", help="Image Fallback Extension") + parser.add_argument("-il", "--ilossless", action="store_true", help="Image Lossless") + parser.add_argument("-iq", "--iquality", help="Image Quality") + parser.add_argument("--vcrf", help="Video CRF") + parser.add_argument("-vs", "--vskip", help="Video Skip") + parser.add_argument("-ve", "--vext", help="Video Extension") + parser.add_argument("-vc", "--vcodec", help="Video Codec") + args = parser.parse_args() if os.path.isfile(args.config): with open(args.config, "rb") as cfile: diff --git a/vnrecode/params.py b/vnrecode/params.py new file mode 100644 index 0000000..483d047 --- /dev/null +++ b/vnrecode/params.py @@ -0,0 +1,54 @@ +from dataclasses import dataclass +from typing import Self + + +@dataclass +class Params: + + copy_unprocessed: bool + force_compress: bool + mimic_mode: bool + hide_errors: bool + webp_rgba: bool + workers: int + + audio_ext: str + audio_bitrate: str + + image_downscale: int + image_ext: str + image_fall_ext: str + image_lossless: str + image_quality: int + + video_crf: int + video_skip: bool + video_ext: str + video_codec: str + + @classmethod + def setup(cls, config, args) -> Self: + copy_unprocessed = config["FFMPEG"]["CopyUnprocessed"] if not args.u else args.u + force_compress = config["FFMPEG"]["ForceCompress"] if not args.force else args.force + mimic_mode = config["FFMPEG"]["MimicMode"] if not args.mimic else args.mimic + hide_errors = config["FFMPEG"]["HideErrors"] if not args.silent else args.silent + workers = config["FFMPEG"]["Workers"] if args.jobs is None else args.jobs + webp_rgba = config["FFMPEG"]["WebpRGBA"] if not args.webprgba else args.webprgba + audio_ext = config["AUDIO"]["Extension"] if args.aext is None else args.aext + audio_bitrate = config["AUDIO"]["BitRate"] if args.abit is None else args.abit + image_downscale = config["IMAGE"]["ResDownScale"] if args.idown is None else args.idown + image_ext = config["IMAGE"]["Extension"] if args.iext is None else args.iext + image_fall_ext = config["IMAGE"]["FallBackExtension"] if args.ifallext is None else args.ifallext + image_lossless = config["IMAGE"]["Lossless"] if not args.ilossless else args.ilossless + image_quality = config["IMAGE"]["Quality"] if args.iquality is None else args.iquality + video_crf = config["VIDEO"]["CRF"] if args.vcrf is None else args.vcrf + video_skip = config["VIDEO"]["SkipVideo"] if args.vskip is None else args.vskip + video_ext = config["VIDEO"]["Extension"] if args.vext is None else args.vext + video_codec = config["VIDEO"]["Codec"] if args.vcodec is None else args.vcodec + + return cls( + copy_unprocessed, force_compress, mimic_mode, hide_errors, webp_rgba, workers, + audio_ext, audio_bitrate, + image_downscale, image_ext, image_fall_ext, image_lossless, image_quality, + video_crf, video_skip, video_ext, video_codec + ) diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 48df9e9..040da09 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -4,9 +4,9 @@ import os class Utils: - def __init__(self, config, printer): + def __init__(self, params, printer): self.errors = 0 - self.config = config + self.params = params self.printer = printer @staticmethod @@ -56,7 +56,7 @@ class Utils: self.get_compression(orig_folder, f"{orig_folder}_compressed") def add_unprocessed_file(self, orig_folder, new_folder): - if self.config['FFMPEG']['CopyUnprocessed']: + if self.params.copy_unprocessed: filename = orig_folder.split("/").pop() copyfile(orig_folder, new_folder) self.printer.info(f"File {filename} copied to compressed folder.") From 71c5764f26d1c6c5f158c143610b9d18450b55fb Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Tue, 3 Sep 2024 22:44:58 +0300 Subject: [PATCH 075/105] vnrecode: replace config class to params --- pyproject.toml | 2 +- vnrecode/__main__.py | 8 ++--- vnrecode/application.py | 5 ++- vnrecode/config.py | 48 ------------------------- vnrecode/params.py | 77 ++++++++++++++++++++++++++++++----------- 5 files changed, 63 insertions(+), 77 deletions(-) delete mode 100644 vnrecode/config.py diff --git a/pyproject.toml b/pyproject.toml index be74c59..4d8aa85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ packages = ["vnrecode", "unrenapk", "vnds2renpy"] include-package-data = true [tool.setuptools.package-data] -'vnrecode' = ['*.py', '*.toml'] +'vnrecode' = ['*.py'] 'vnds2renpy' = ['*.py'] 'unrenapk' = ['*.py'] diff --git a/vnrecode/__main__.py b/vnrecode/__main__.py index 39ef2d6..faf1839 100644 --- a/vnrecode/__main__.py +++ b/vnrecode/__main__.py @@ -3,18 +3,16 @@ from .application import Application from .compress import Compress from .printer import Printer from .params import Params -from .config import Config from .utils import Utils def init(): - config = Config.setup_config() - params = Params.setup(config.config, config.args) - printer = Printer(config.args.source) + params = Params.setup() + printer = Printer(params.source) utils = Utils(params, printer) compress = Compress(params, printer, utils) - Application(params, config.args, compress, printer, utils).run() + Application(params, compress, printer, utils).run() if __name__ == "__main__": diff --git a/vnrecode/application.py b/vnrecode/application.py index 75e3107..d32a3cf 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -7,9 +7,8 @@ import os class Application: - def __init__(self, params, args, compress, printer, utils): + def __init__(self, params, compress, printer, utils): self.params = params - self.args = args self.compress = compress.compress self.printer = printer self.utils = utils @@ -22,7 +21,7 @@ class Application: start_time = datetime.now() self.printer.win_ascii_esc() - source = os.path.abspath(self.args.source) + source = os.path.abspath(self.params.source) if os.path.exists(f"{source}_compressed"): shutil.rmtree(f"{source}_compressed") diff --git a/vnrecode/config.py b/vnrecode/config.py deleted file mode 100644 index 4d1336c..0000000 --- a/vnrecode/config.py +++ /dev/null @@ -1,48 +0,0 @@ -from argparse import Namespace, ArgumentParser -from dataclasses import dataclass -from sysconfig import get_path -from typing import Any -import tomllib -import os - - -@dataclass -class Config: - - config: dict[str, Any] - args: Namespace - - @classmethod - def setup_config(cls): - default_config = os.path.join(get_path('purelib'), "vnrecode", "vnrecode.toml") - parser = ArgumentParser(prog="vnrecode", - description="Python utility to compress Visual Novel Resources" - ) - parser.add_argument("source", help="SourceDir") - parser.add_argument("-c", "--config", default=default_config, help="ConfigFile") - parser.add_argument("-u", action="store_true", help="CopyUnprocessed") - parser.add_argument("-f", "--force", action="store_true", help="ForceCompress") - parser.add_argument("-m", "--mimic", action="store_true", help="MimicMode") - parser.add_argument("-s", "--silent", action="store_true", help="HideErrors") - parser.add_argument("--webprgba", action="store_true", help="WebpRGBA") - parser.add_argument("-j", "--jobs", type=int, help="Workers") - parser.add_argument("-ae", "--aext", help="Audio Extension") - parser.add_argument("-ab", "--abit", help="Audio Bitrate") - parser.add_argument("-id", "--idown", type=int, help="Image Downscale") - parser.add_argument("-ie", "--iext", help="Image Extension") - parser.add_argument("-ife", "--ifallext", help="Image Fallback Extension") - parser.add_argument("-il", "--ilossless", action="store_true", help="Image Lossless") - parser.add_argument("-iq", "--iquality", help="Image Quality") - parser.add_argument("--vcrf", help="Video CRF") - parser.add_argument("-vs", "--vskip", help="Video Skip") - parser.add_argument("-ve", "--vext", help="Video Extension") - parser.add_argument("-vc", "--vcodec", help="Video Codec") - - args = parser.parse_args() - if os.path.isfile(args.config): - with open(args.config, "rb") as cfile: - config = tomllib.load(cfile) - else: - print("Failed to find config. Check `vnrecode -h` to more info") - exit(255) - return cls(config=config, args=args) diff --git a/vnrecode/params.py b/vnrecode/params.py index 483d047..789cac5 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -1,6 +1,8 @@ +from argparse import ArgumentParser from dataclasses import dataclass from typing import Self - +import tomllib +import os @dataclass class Params: @@ -26,29 +28,64 @@ class Params: video_ext: str video_codec: str + source: str + @classmethod - def setup(cls, config, args) -> Self: - copy_unprocessed = config["FFMPEG"]["CopyUnprocessed"] if not args.u else args.u - force_compress = config["FFMPEG"]["ForceCompress"] if not args.force else args.force - mimic_mode = config["FFMPEG"]["MimicMode"] if not args.mimic else args.mimic - hide_errors = config["FFMPEG"]["HideErrors"] if not args.silent else args.silent - workers = config["FFMPEG"]["Workers"] if args.jobs is None else args.jobs - webp_rgba = config["FFMPEG"]["WebpRGBA"] if not args.webprgba else args.webprgba - audio_ext = config["AUDIO"]["Extension"] if args.aext is None else args.aext - audio_bitrate = config["AUDIO"]["BitRate"] if args.abit is None else args.abit - image_downscale = config["IMAGE"]["ResDownScale"] if args.idown is None else args.idown - image_ext = config["IMAGE"]["Extension"] if args.iext is None else args.iext - image_fall_ext = config["IMAGE"]["FallBackExtension"] if args.ifallext is None else args.ifallext - image_lossless = config["IMAGE"]["Lossless"] if not args.ilossless else args.ilossless - image_quality = config["IMAGE"]["Quality"] if args.iquality is None else args.iquality - video_crf = config["VIDEO"]["CRF"] if args.vcrf is None else args.vcrf - video_skip = config["VIDEO"]["SkipVideo"] if args.vskip is None else args.vskip - video_ext = config["VIDEO"]["Extension"] if args.vext is None else args.vext - video_codec = config["VIDEO"]["Codec"] if args.vcodec is None else args.vcodec + def setup(cls) -> Self: + parser = ArgumentParser(prog="vnrecode", + description="Python utility to compress Visual Novel Resources" + ) + parser.add_argument("source", help="SourceDir") + parser.add_argument("-c", "--config", help="ConfigFile") + parser.add_argument("-u", type=bool, help="CopyUnprocessed", default=True) + parser.add_argument("-f", "--force", type=bool, help="ForceCompress", default=False) + parser.add_argument("-m", "--mimic", type=bool, help="MimicMode", default=True) + parser.add_argument("-s", "--silent", type=bool, help="HideErrors", default=True) + parser.add_argument("--webprgba", type=bool, help="WebpRGBA", default=True) + parser.add_argument("-j", "--jobs", type=int, help="Workers", default=16) + parser.add_argument("-ae", "--aext", help="Audio Extension", default="opus") + parser.add_argument("-ab", "--abit", help="Audio Bitrate", default="128k") + parser.add_argument("-id", "--idown", type=int, help="Image Downscale", default=1) + parser.add_argument("-ie", "--iext", help="Image Extension", default="avif") + parser.add_argument("-ife", "--ifallext", help="Image Fallback Extension", default="webp") + parser.add_argument("-il", "--ilossless", type=bool, help="Image Lossless", default=True) + parser.add_argument("-iq", "--iquality", type=int, help="Image Quality", default=100) + parser.add_argument("--vcrf", help="Video CRF", type=int, default=27) + parser.add_argument("-vs", "--vskip", help="Video Skip", default=False) + parser.add_argument("-ve", "--vext", help="Video Extension", default="webm") + parser.add_argument("-vc", "--vcodec", help="Video Codec", default="libvpx-vp9") + args = parser.parse_args() + + if args.config is not None: + if os.path.isfile(args.config): + with open(args.config, "rb") as cfile: + config = tomllib.load(cfile) + else: + print("Failed to find config. Check `vnrecode -h` to more info") + exit(255) + + copy_unprocessed = config["FFMPEG"]["CopyUnprocessed"] if args.config else args.u + force_compress = config["FFMPEG"]["ForceCompress"] if args.config else args.force + mimic_mode = config["FFMPEG"]["MimicMode"] if args.config else args.mimic + hide_errors = config["FFMPEG"]["HideErrors"] if args.config else args.silent + workers = config["FFMPEG"]["Workers"] if args.config else args.jobs + webp_rgba = config["FFMPEG"]["WebpRGBA"] if args.config else args.webprgba + audio_ext = config["AUDIO"]["Extension"] if args.config else args.aext + audio_bitrate = config["AUDIO"]["BitRate"] if args.config else args.abit + image_downscale = config["IMAGE"]["ResDownScale"] if args.config else args.idown + image_ext = config["IMAGE"]["Extension"] if args.config else args.iext + image_fall_ext = config["IMAGE"]["FallBackExtension"] if args.config else args.ifallext + image_lossless = config["IMAGE"]["Lossless"] if args.config else args.ilossless + image_quality = config["IMAGE"]["Quality"] if args.config else args.iquality + video_crf = config["VIDEO"]["CRF"] if args.config else args.vcrf + video_skip = config["VIDEO"]["SkipVideo"] if args.config else args.vskip + video_ext = config["VIDEO"]["Extension"] if args.config else args.vext + video_codec = config["VIDEO"]["Codec"] if args.config else args.vcodec + source = args.source return cls( copy_unprocessed, force_compress, mimic_mode, hide_errors, webp_rgba, workers, audio_ext, audio_bitrate, image_downscale, image_ext, image_fall_ext, image_lossless, image_quality, - video_crf, video_skip, video_ext, video_codec + video_crf, video_skip, video_ext, video_codec, source ) From a69b17c624af1df1a9256a7ff5b577380d7b245e Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Tue, 3 Sep 2024 22:54:59 +0300 Subject: [PATCH 076/105] vnrecode: improve naming, fix mimic mode errors --- vnrecode/compress.py | 65 ++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 7dbc6fc..25d3177 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -45,54 +45,56 @@ class Compress: self.printer = printer self.utils = utils - def audio(self, folder, file, target_folder, extension): - bitrate = self.params.audio_bitrate + def audio(self, in_dir, file, out_dir, extension): + bit_rate = self.params.audio_bitrate + out_file = self.utils.check_duplicates(f'{out_dir}/{os.path.splitext(file)[0]}.{extension}') try: (FFmpeg() - .input(f'{folder}/{file}') + .input(f'{in_dir}/{file}') .option("hide_banner") - .output(self.utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), - {"b:a": bitrate, "loglevel": "error"}) + .output(out_file,{"b:a": bit_rate, "loglevel": "error"}) .execute() ) except FFmpegError as e: - self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') self.utils.errors += 1 if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") - self.printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}") - return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' + self.printer.files(file, os.path.splitext(file)[0], extension, f"{bit_rate}") + return out_file - def video(self, folder, file, target_folder, extension): + def video(self, in_dir, file, out_dir, extension): if not self.params.video_skip: + out_file = self.utils.check_duplicates(f'{out_dir}/{os.path.splitext(file)[0]}.{extension}') codec = self.params.video_codec crf = self.params.video_crf try: (FFmpeg() - .input(f'{folder}/{file}') + .input(f'{in_dir}/{file}') .option("hide_banner") .option("hwaccel", "auto") - .output(self.utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'), - {"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) + .output(out_file,{"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) .execute() ) self.printer.files(file, os.path.splitext(file)[0], extension, codec) except FFmpegError as e: - self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') self.utils.errors += 1 if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") + return out_file else: - self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' + self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') + return f'{out_dir}/{os.path.splitext(file)[0]}.{extension}' - def image(self, folder, file, target_folder, extension): + def image(self, in_dir, file, out_dir, extension): quality = self.params.image_quality + out_file = self.utils.check_duplicates(f"{out_dir}/{os.path.splitext(file)[0]}.{extension}") try: - image = Image.open(f'{folder}/{file}') + image = Image.open(f'{in_dir}/{file}') if (extension == "jpg" or extension == "jpeg" or (extension == "webp" and not self.params.webp_rgba)): @@ -109,37 +111,39 @@ class Compress: new_size = (int(width / res_downscale), int(height / res_downscale)) image = image.resize(new_size) - image.save(self.utils.check_duplicates(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}"), + image.save(out_file, optimize=True, lossless=self.params.image_lossless, quality=quality, minimize_size=True) self.printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") except Exception as e: - self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') self.utils.errors += 1 if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") - return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}' + return out_file - def unknown(self, folder, file, target_folder): + def unknown(self, in_dir, filename, out_dir): if self.params.force_compress: - self.printer.unknown_file(file) + self.printer.unknown_file(filename) + out_file = self.utils.check_duplicates(f'{out_dir}/{filename}') try: (FFmpeg() - .input(f'{folder}/{file}') - .output(self.utils.check_duplicates(f'{target_folder}/{file}')) + .input(f'{in_dir}/{filename}') + .output(out_file) .execute() ) except FFmpegError as e: - self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') + self.utils.add_unprocessed_file(f'{in_dir}/{filename}', f'{out_dir}/{filename}') self.utils.errors += 1 if not self.params.hide_errors: - self.printer.error(f"File {file} can't be processed! Error: {e}") + self.printer.error(f"File {filename} can't be processed! Error: {e}") + return out_file else: - self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}') - return f'{target_folder}/{file}' + self.utils.add_unprocessed_file(f'{in_dir}/{filename}', f'{out_dir}/{filename}') + return f'{out_dir}/{filename}' def compress(self, _dir, filename, source, output): @@ -154,10 +158,7 @@ class Compress: out_file = self.unknown(_dir, filename, output) if self.params.mimic_mode: - try: - os.rename(out_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed")) - except FileNotFoundError: - self.printer.warning(f"File {out_file} failed to copy to out dir") + os.rename(out_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed")) self.printer.bar.update() self.printer.bar.next() From 03647d4b8446402d2a05b65afbdffc512750a449 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Tue, 3 Sep 2024 23:00:19 +0300 Subject: [PATCH 077/105] vnrecode: rewrite get_type method --- vnrecode/compress.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 25d3177..75202ee 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -8,19 +8,18 @@ class File: @staticmethod def get_type(filename): - audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'] - image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'] - video_ext = ['.3gp' '.amv', '.avi', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', - '.webm', '.ogv'] - if os.path.splitext(filename)[1] in audio_ext: - return "audio" - elif os.path.splitext(filename)[1] in image_ext: - return "image" - elif os.path.splitext(filename)[1] in video_ext: - return "video" - else: - return "unknown" + extensions = { + "audio": ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'], + "image": ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'], + "video": ['.3gp' '.amv', '.avi', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv', + '.webm', '.ogv'] + } + + for file_type in extensions: + if os.path.splitext(filename)[1] in extensions[file_type]: + return file_type + return "unknown" @staticmethod def has_transparency(img: Image) -> bool: From 92474b4aa41bbbf555fcf72cefed768758868cdc Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 01:21:59 +0300 Subject: [PATCH 078/105] vnrecode: improve duplications check for mt --- vnrecode/application.py | 1 + vnrecode/compress.py | 11 +++++------ vnrecode/utils.py | 27 +++++++++++++++++++++------ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/vnrecode/application.py b/vnrecode/application.py index d32a3cf..c7cb7cd 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -42,6 +42,7 @@ class Application: for future in as_completed(futures): future.result() + self.utils.print_duplicates() self.utils.get_compression_status(source) self.utils.sys_pause() print(f"Time taken: {datetime.now() - start_time}") \ No newline at end of file diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 75202ee..249061e 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -46,7 +46,7 @@ class Compress: def audio(self, in_dir, file, out_dir, extension): bit_rate = self.params.audio_bitrate - out_file = self.utils.check_duplicates(f'{out_dir}/{os.path.splitext(file)[0]}.{extension}') + out_file = self.utils.check_duplicates(in_dir, out_dir, f'{os.path.splitext(file)[0]}.{extension}') try: (FFmpeg() .input(f'{in_dir}/{file}') @@ -65,7 +65,7 @@ class Compress: def video(self, in_dir, file, out_dir, extension): if not self.params.video_skip: - out_file = self.utils.check_duplicates(f'{out_dir}/{os.path.splitext(file)[0]}.{extension}') + out_file = self.utils.check_duplicates(in_dir, out_dir, f'{os.path.splitext(file)[0]}.{extension}') codec = self.params.video_codec crf = self.params.video_crf @@ -91,7 +91,7 @@ class Compress: def image(self, in_dir, file, out_dir, extension): quality = self.params.image_quality - out_file = self.utils.check_duplicates(f"{out_dir}/{os.path.splitext(file)[0]}.{extension}") + out_file = self.utils.check_duplicates(in_dir, out_dir, f"{os.path.splitext(file)[0]}.{extension}") try: image = Image.open(f'{in_dir}/{file}') @@ -127,7 +127,7 @@ class Compress: def unknown(self, in_dir, filename, out_dir): if self.params.force_compress: self.printer.unknown_file(filename) - out_file = self.utils.check_duplicates(f'{out_dir}/{filename}') + out_file = self.utils.check_duplicates(in_dir, out_dir, filename) try: (FFmpeg() .input(f'{in_dir}/{filename}') @@ -144,7 +144,6 @@ class Compress: self.utils.add_unprocessed_file(f'{in_dir}/{filename}', f'{out_dir}/{filename}') return f'{out_dir}/{filename}' - def compress(self, _dir, filename, source, output): match File.get_type(filename): case "audio": @@ -157,7 +156,7 @@ class Compress: out_file = self.unknown(_dir, filename, output) if self.params.mimic_mode: - os.rename(out_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed")) + self.utils.mimic_rename(out_file, f'{_dir}/{filename}', source) self.printer.bar.update() self.printer.bar.next() diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 040da09..96b7db9 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -1,4 +1,5 @@ from shutil import copyfile +from glob import glob import sys import os @@ -8,6 +9,7 @@ class Utils: self.errors = 0 self.params = params self.printer = printer + self.duplicates = [] @staticmethod def sys_pause(): @@ -61,11 +63,24 @@ class Utils: copyfile(orig_folder, new_folder) self.printer.info(f"File {filename} copied to compressed folder.") - def check_duplicates(self, new_folder): - filename = new_folder.split().pop() - if os.path.exists(new_folder): + def check_duplicates(self, in_dir, out_dir, filename): + duplicates = glob(f"{in_dir}/{os.path.splitext(filename)[0]}.*") + if len(duplicates) > 1: + if filename in self.duplicates: + new_name = os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1] + return f"{out_dir}/{new_name}" + self.duplicates.append(filename) + return f"{out_dir}/{filename}" + + def print_duplicates(self): + for filename in self.duplicates: self.printer.warning( f'Duplicate file has been found! Check manually this files - "{filename}", ' - f'"{os.path.splitext(filename)[0] + "(copy)" + os.path.splitext(filename)[1]}"') - return os.path.splitext(new_folder)[0] + "(copy)" + os.path.splitext(new_folder)[1] - return new_folder + f'"{os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1]}"') + + @staticmethod + def mimic_rename(filename, target, source): + if filename.count("(vncopy)"): + target = os.path.splitext(target)[0] + "(vncopy)" + os.path.splitext(target)[1] + + os.rename(filename, target.replace(source, f"{source}_compressed")) \ No newline at end of file From 44c12a568885826539efe78dcbca0e5f758e4c4a Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 01:22:42 +0300 Subject: [PATCH 079/105] Update requirements --- pyproject.toml | 3 ++- requirements.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4d8aa85..f2c379f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,5 +27,6 @@ dependencies = [ "pillow-avif-plugin==1.4.3", "python-ffmpeg==2.0.12", "progress==1.6", - "colorama==0.4.6" + "colorama==0.4.6", + "argparse~=1.4.0" ] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index c3c571c..16bd1e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ Pillow==10.3.0 pillow-avif-plugin==1.4.3 python-ffmpeg==2.0.12 progress==1.6 -colorama==0.4.6 \ No newline at end of file +colorama==0.4.6 +argparse~=1.4.0 \ No newline at end of file From a9aeb5250622845a8fb36995a31befba3bdcd41f Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 01:52:18 +0300 Subject: [PATCH 080/105] vnrecode: improve cli parameters --- vnrecode/params.py | 82 +++++++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 37 deletions(-) diff --git a/vnrecode/params.py b/vnrecode/params.py index 789cac5..c8474c2 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -32,30 +32,7 @@ class Params: @classmethod def setup(cls) -> Self: - parser = ArgumentParser(prog="vnrecode", - description="Python utility to compress Visual Novel Resources" - ) - parser.add_argument("source", help="SourceDir") - parser.add_argument("-c", "--config", help="ConfigFile") - parser.add_argument("-u", type=bool, help="CopyUnprocessed", default=True) - parser.add_argument("-f", "--force", type=bool, help="ForceCompress", default=False) - parser.add_argument("-m", "--mimic", type=bool, help="MimicMode", default=True) - parser.add_argument("-s", "--silent", type=bool, help="HideErrors", default=True) - parser.add_argument("--webprgba", type=bool, help="WebpRGBA", default=True) - parser.add_argument("-j", "--jobs", type=int, help="Workers", default=16) - parser.add_argument("-ae", "--aext", help="Audio Extension", default="opus") - parser.add_argument("-ab", "--abit", help="Audio Bitrate", default="128k") - parser.add_argument("-id", "--idown", type=int, help="Image Downscale", default=1) - parser.add_argument("-ie", "--iext", help="Image Extension", default="avif") - parser.add_argument("-ife", "--ifallext", help="Image Fallback Extension", default="webp") - parser.add_argument("-il", "--ilossless", type=bool, help="Image Lossless", default=True) - parser.add_argument("-iq", "--iquality", type=int, help="Image Quality", default=100) - parser.add_argument("--vcrf", help="Video CRF", type=int, default=27) - parser.add_argument("-vs", "--vskip", help="Video Skip", default=False) - parser.add_argument("-ve", "--vext", help="Video Extension", default="webm") - parser.add_argument("-vc", "--vcodec", help="Video Codec", default="libvpx-vp9") - args = parser.parse_args() - + args = cls.get_args() if args.config is not None: if os.path.isfile(args.config): with open(args.config, "rb") as cfile: @@ -67,20 +44,20 @@ class Params: copy_unprocessed = config["FFMPEG"]["CopyUnprocessed"] if args.config else args.u force_compress = config["FFMPEG"]["ForceCompress"] if args.config else args.force mimic_mode = config["FFMPEG"]["MimicMode"] if args.config else args.mimic - hide_errors = config["FFMPEG"]["HideErrors"] if args.config else args.silent + hide_errors = config["FFMPEG"]["HideErrors"] if args.config else args.show_errors workers = config["FFMPEG"]["Workers"] if args.config else args.jobs - webp_rgba = config["FFMPEG"]["WebpRGBA"] if args.config else args.webprgba - audio_ext = config["AUDIO"]["Extension"] if args.config else args.aext - audio_bitrate = config["AUDIO"]["BitRate"] if args.config else args.abit - image_downscale = config["IMAGE"]["ResDownScale"] if args.config else args.idown - image_ext = config["IMAGE"]["Extension"] if args.config else args.iext - image_fall_ext = config["IMAGE"]["FallBackExtension"] if args.config else args.ifallext - image_lossless = config["IMAGE"]["Lossless"] if args.config else args.ilossless - image_quality = config["IMAGE"]["Quality"] if args.config else args.iquality - video_crf = config["VIDEO"]["CRF"] if args.config else args.vcrf - video_skip = config["VIDEO"]["SkipVideo"] if args.config else args.vskip - video_ext = config["VIDEO"]["Extension"] if args.config else args.vext - video_codec = config["VIDEO"]["Codec"] if args.config else args.vcodec + webp_rgba = config["FFMPEG"]["WebpRGBA"] if args.config else args.webp_rgba + audio_ext = config["AUDIO"]["Extension"] if args.config else args.a_ext + audio_bitrate = config["AUDIO"]["BitRate"] if args.config else args.a_bit + image_downscale = config["IMAGE"]["ResDownScale"] if args.config else args.i_down + image_ext = config["IMAGE"]["Extension"] if args.config else args.i_ext + image_fall_ext = config["IMAGE"]["FallBackExtension"] if args.config else args.i_fallext + image_lossless = config["IMAGE"]["Lossless"] if args.config else args.i_lossless + image_quality = config["IMAGE"]["Quality"] if args.config else args.i_quality + video_crf = config["VIDEO"]["CRF"] if args.config else args.v_crf + video_skip = config["VIDEO"]["SkipVideo"] if args.config else args.v_skip + video_ext = config["VIDEO"]["Extension"] if args.config else args.v_ext + video_codec = config["VIDEO"]["Codec"] if args.config else args.v_codec source = args.source return cls( @@ -89,3 +66,34 @@ class Params: image_downscale, image_ext, image_fall_ext, image_lossless, image_quality, video_crf, video_skip, video_ext, video_codec, source ) + + @staticmethod + def get_args(): + parser = ArgumentParser(prog="vnrecode", + description="Python utility to compress Visual Novel Resources" + ) + parser.add_argument("source", help="Directory with game files to recode") + parser.add_argument("-c", "--config", help="Utility config file") + parser.add_argument("-u", action='store_true', help="Copy unprocessed filed", default=True) + parser.add_argument("-nu", dest='u', action='store_false', help="Don't copy unprocessed") + parser.add_argument("-f", "--force", action='store_true', help="Try to recode unknown files") + parser.add_argument("-m", "--mimic", action='store_true', help="Enable mimic mode", default=True) + parser.add_argument("-nm", "--no-mimic", dest='mimic', action='store_false', help="Disable mimic mode") + parser.add_argument("-v", "--show_errors", action='store_false', help="Show recode errors") + parser.add_argument("--webp_rgba", action='store_true', help="Recode .webp with alpha channel", default=True) + parser.add_argument("--webp_rgb", dest='webp_rgba', action='store_false', help="Recode .webp without alpha channel") + parser.add_argument("-j", "--jobs", type=int, help="Number of threads", default=16) + parser.add_argument("-ae", "--a_ext", help="Audio extension", default="opus") + parser.add_argument("-ab", "--a_bit", help="Audio bit rate", default="128k") + parser.add_argument("-id", "--i_down", type=int, help="Image resolution downscale multiplier", default=1) + parser.add_argument("-ie", "--i_ext", help="Image extension", default="avif") + parser.add_argument("-ife", "--i_fallext", help="Image fallback extension", default="webp") + parser.add_argument("-il", "--i_lossless", action='store_true', help="Image lossless compression mode", default=True) + parser.add_argument("-ilo", "--i_losing", dest='ilossless', action='store_false', help="Image losing compression mode") + parser.add_argument("-iq", "--i_quality", type=int, help="Image quality", default=100) + parser.add_argument("--v_crf", help="Video CRF number", type=int, default=27) + parser.add_argument("-vs", "--v_skip", action='store_true', help="Skip video recoding") + parser.add_argument("-ve", "--v_ext", help="Video extension", default="webm") + parser.add_argument("-vc", "--v_codec", help="Video codec name", default="libvpx-vp9") + args = parser.parse_args() + return args \ No newline at end of file From b534214be9e42ef58dce9c2f199b19ac1fdd67c9 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 02:07:17 +0300 Subject: [PATCH 081/105] vnrecode: accurate duplicate messages --- vnrecode/utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 96b7db9..8acd4dc 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -66,10 +66,10 @@ class Utils: def check_duplicates(self, in_dir, out_dir, filename): duplicates = glob(f"{in_dir}/{os.path.splitext(filename)[0]}.*") if len(duplicates) > 1: - if filename in self.duplicates: + if filename not in self.duplicates: + self.duplicates.append(filename) new_name = os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1] return f"{out_dir}/{new_name}" - self.duplicates.append(filename) return f"{out_dir}/{filename}" def print_duplicates(self): @@ -78,9 +78,11 @@ class Utils: f'Duplicate file has been found! Check manually this files - "{filename}", ' f'"{os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1]}"') - @staticmethod - def mimic_rename(filename, target, source): + def mimic_rename(self, filename, target, source): if filename.count("(vncopy)"): + orig_name = filename.replace("(vncopy)", "") + index = self.duplicates.index(os.path.split(orig_name)[-1]) + self.duplicates[index] = os.path.split(target)[-1] target = os.path.splitext(target)[0] + "(vncopy)" + os.path.splitext(target)[1] os.rename(filename, target.replace(source, f"{source}_compressed")) \ No newline at end of file From 8f9db132e65fdff3b99c9e8aa728f8204d6e9ec9 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 02:12:49 +0300 Subject: [PATCH 082/105] vnrecode: remove unneeded cli parameters --- vnrecode/params.py | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/vnrecode/params.py b/vnrecode/params.py index c8474c2..89fe28f 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -41,7 +41,7 @@ class Params: print("Failed to find config. Check `vnrecode -h` to more info") exit(255) - copy_unprocessed = config["FFMPEG"]["CopyUnprocessed"] if args.config else args.u + copy_unprocessed = config["FFMPEG"]["CopyUnprocessed"] if args.config else args.unproc force_compress = config["FFMPEG"]["ForceCompress"] if args.config else args.force mimic_mode = config["FFMPEG"]["MimicMode"] if args.config else args.mimic hide_errors = config["FFMPEG"]["HideErrors"] if args.config else args.show_errors @@ -73,27 +73,23 @@ class Params: description="Python utility to compress Visual Novel Resources" ) parser.add_argument("source", help="Directory with game files to recode") - parser.add_argument("-c", "--config", help="Utility config file") - parser.add_argument("-u", action='store_true', help="Copy unprocessed filed", default=True) - parser.add_argument("-nu", dest='u', action='store_false', help="Don't copy unprocessed") + parser.add_argument("-c", dest='config', help="Utility config file") + parser.add_argument("-nu", dest='unproc', action='store_false', help="Don't copy unprocessed") parser.add_argument("-f", "--force", action='store_true', help="Try to recode unknown files") - parser.add_argument("-m", "--mimic", action='store_true', help="Enable mimic mode", default=True) parser.add_argument("-nm", "--no-mimic", dest='mimic', action='store_false', help="Disable mimic mode") parser.add_argument("-v", "--show_errors", action='store_false', help="Show recode errors") - parser.add_argument("--webp_rgba", action='store_true', help="Recode .webp with alpha channel", default=True) - parser.add_argument("--webp_rgb", dest='webp_rgba', action='store_false', help="Recode .webp without alpha channel") + parser.add_argument("--webp-rgb", dest='webp_rgba', action='store_false', help="Recode .webp without alpha channel") parser.add_argument("-j", "--jobs", type=int, help="Number of threads", default=16) - parser.add_argument("-ae", "--a_ext", help="Audio extension", default="opus") - parser.add_argument("-ab", "--a_bit", help="Audio bit rate", default="128k") - parser.add_argument("-id", "--i_down", type=int, help="Image resolution downscale multiplier", default=1) - parser.add_argument("-ie", "--i_ext", help="Image extension", default="avif") - parser.add_argument("-ife", "--i_fallext", help="Image fallback extension", default="webp") - parser.add_argument("-il", "--i_lossless", action='store_true', help="Image lossless compression mode", default=True) - parser.add_argument("-ilo", "--i_losing", dest='ilossless', action='store_false', help="Image losing compression mode") - parser.add_argument("-iq", "--i_quality", type=int, help="Image quality", default=100) + parser.add_argument("-ae", dest="a_ext", help="Audio extension", default="opus") + parser.add_argument("-ab", dest="a_bit", help="Audio bit rate", default="128k") + parser.add_argument("-id", dest="i_down", type=int, help="Image resolution downscale multiplier", default=1) + parser.add_argument("-ie", dest="i_ext", help="Image extension", default="avif") + parser.add_argument("-ife", dest="i_fallext", help="Image fallback extension", default="webp") + parser.add_argument("-il", dest='i_lossless', action='store_false', help="Image losing compression mode") + parser.add_argument("-iq", dest="i_quality", type=int, help="Image quality", default=100) parser.add_argument("--v_crf", help="Video CRF number", type=int, default=27) - parser.add_argument("-vs", "--v_skip", action='store_true', help="Skip video recoding") - parser.add_argument("-ve", "--v_ext", help="Video extension", default="webm") - parser.add_argument("-vc", "--v_codec", help="Video codec name", default="libvpx-vp9") + parser.add_argument("-vs", dest="v_skip", action='store_true', help="Skip video recoding") + parser.add_argument("-ve", dest="v_ext", help="Video extension", default="webm") + parser.add_argument("-vc", dest="v_codec", help="Video codec name", default="libvpx-vp9") args = parser.parse_args() return args \ No newline at end of file From 7433027cf3c31f9ca52c502ad3ca4bc034c82cf4 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 02:40:47 +0300 Subject: [PATCH 083/105] vnrecode: add type definitions --- vnrecode/application.py | 11 +++++--- vnrecode/compress.py | 59 +++++++++++++++++++++-------------------- vnrecode/params.py | 4 +-- vnrecode/printer.py | 12 ++++----- vnrecode/utils.py | 46 ++++++++++++++++---------------- 5 files changed, 69 insertions(+), 63 deletions(-) diff --git a/vnrecode/application.py b/vnrecode/application.py index c7cb7cd..68fa45f 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -4,17 +4,22 @@ from datetime import datetime import shutil import os +from .compress import Compress +from .printer import Printer +from .params import Params +from .utils import Utils + class Application: - def __init__(self, params, compress, printer, utils): + def __init__(self, params: Params, compress: Compress, printer: Printer, utils: Utils): self.params = params self.compress = compress.compress self.printer = printer self.utils = utils - def compress_worker(self, folder, file, source, output): - if os.path.isfile(f'{folder}/{file}'): + def compress_worker(self, folder: str, file: str, source: str, output: str): + if os.path.isfile(os.path.join(folder, file)): self.compress(folder, file, source, output) def run(self): diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 249061e..18fb20e 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -1,13 +1,17 @@ from ffmpeg import FFmpeg, FFmpegError from PIL import Image +from os import path import pillow_avif -import os + +from .printer import Printer +from .params import Params +from .utils import Utils class File: @staticmethod - def get_type(filename): + def get_type(filename: str) -> str: extensions = { "audio": ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'], @@ -17,7 +21,7 @@ class File: } for file_type in extensions: - if os.path.splitext(filename)[1] in extensions[file_type]: + if path.splitext(filename)[1] in extensions[file_type]: return file_type return "unknown" @@ -39,45 +43,44 @@ class File: class Compress: - def __init__(self, params, printer, utils): + def __init__(self, params: Params, printer: Printer, utils: Utils): self.params = params self.printer = printer self.utils = utils - def audio(self, in_dir, file, out_dir, extension): + def audio(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: bit_rate = self.params.audio_bitrate - out_file = self.utils.check_duplicates(in_dir, out_dir, f'{os.path.splitext(file)[0]}.{extension}') + out_file = self.utils.check_duplicates(in_dir, out_dir, f'{path.splitext(file)[0]}.{extension}') try: (FFmpeg() - .input(f'{in_dir}/{file}') + .input(path.join(in_dir, file)) .option("hide_banner") .output(out_file,{"b:a": bit_rate, "loglevel": "error"}) .execute() ) except FFmpegError as e: - self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') + self.utils.add_unprocessed_file(path.join(in_dir, file), path.join(out_dir, file)) self.utils.errors += 1 if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") - self.printer.files(file, os.path.splitext(file)[0], extension, f"{bit_rate}") + self.printer.files(file, path.splitext(file)[0], extension, f"{bit_rate}") return out_file - - def video(self, in_dir, file, out_dir, extension): + def video(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: if not self.params.video_skip: - out_file = self.utils.check_duplicates(in_dir, out_dir, f'{os.path.splitext(file)[0]}.{extension}') + out_file = self.utils.check_duplicates(in_dir, out_dir, f'{path.splitext(file)[0]}.{extension}') codec = self.params.video_codec crf = self.params.video_crf try: (FFmpeg() - .input(f'{in_dir}/{file}') + .input(path.join(in_dir, file)) .option("hide_banner") .option("hwaccel", "auto") .output(out_file,{"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) .execute() ) - self.printer.files(file, os.path.splitext(file)[0], extension, codec) + self.printer.files(file, path.splitext(file)[0], extension, codec) except FFmpegError as e: self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') self.utils.errors += 1 @@ -86,14 +89,13 @@ class Compress: return out_file else: self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') - return f'{out_dir}/{os.path.splitext(file)[0]}.{extension}' + return f'{out_dir}/{path.splitext(file)[0]}.{extension}' - - def image(self, in_dir, file, out_dir, extension): + def image(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: quality = self.params.image_quality - out_file = self.utils.check_duplicates(in_dir, out_dir, f"{os.path.splitext(file)[0]}.{extension}") + out_file = self.utils.check_duplicates(in_dir, out_dir, f"{path.splitext(file)[0]}.{extension}") try: - image = Image.open(f'{in_dir}/{file}') + image = Image.open(path.join(in_dir, file)) if (extension == "jpg" or extension == "jpeg" or (extension == "webp" and not self.params.webp_rgba)): @@ -115,36 +117,35 @@ class Compress: lossless=self.params.image_lossless, quality=quality, minimize_size=True) - self.printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%") + self.printer.files(file, path.splitext(file)[0], extension, f"{quality}%") except Exception as e: - self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') + self.utils.add_unprocessed_file(path.join(in_dir, file), path.join(out_dir, file)) self.utils.errors += 1 if not self.params.hide_errors: self.printer.error(f"File {file} can't be processed! Error: {e}") return out_file - - def unknown(self, in_dir, filename, out_dir): + def unknown(self, in_dir: str, filename: str, out_dir: str) -> str: if self.params.force_compress: self.printer.unknown_file(filename) out_file = self.utils.check_duplicates(in_dir, out_dir, filename) try: (FFmpeg() - .input(f'{in_dir}/{filename}') + .input(path.join(in_dir, filename)) .output(out_file) .execute() ) except FFmpegError as e: - self.utils.add_unprocessed_file(f'{in_dir}/{filename}', f'{out_dir}/{filename}') + self.utils.add_unprocessed_file(path.join(in_dir, filename), path.join(out_dir, filename)) self.utils.errors += 1 if not self.params.hide_errors: self.printer.error(f"File {filename} can't be processed! Error: {e}") return out_file else: - self.utils.add_unprocessed_file(f'{in_dir}/{filename}', f'{out_dir}/{filename}') - return f'{out_dir}/{filename}' + self.utils.add_unprocessed_file(path.join(in_dir, filename), path.join(out_dir, filename)) + return path.join(out_dir, filename) - def compress(self, _dir, filename, source, output): + def compress(self, _dir: str, filename: str, source: str, output: str): match File.get_type(filename): case "audio": out_file = self.audio(_dir, filename, output, self.params.audio_ext) @@ -156,7 +157,7 @@ class Compress: out_file = self.unknown(_dir, filename, output) if self.params.mimic_mode: - self.utils.mimic_rename(out_file, f'{_dir}/{filename}', source) + self.utils.mimic_rename(out_file, path.join(_dir, filename), source) self.printer.bar.update() self.printer.bar.next() diff --git a/vnrecode/params.py b/vnrecode/params.py index 89fe28f..ea15691 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -1,4 +1,4 @@ -from argparse import ArgumentParser +from argparse import ArgumentParser, Namespace from dataclasses import dataclass from typing import Self import tomllib @@ -68,7 +68,7 @@ class Params: ) @staticmethod - def get_args(): + def get_args() -> Namespace: parser = ArgumentParser(prog="vnrecode", description="Python utility to compress Visual Novel Resources" ) diff --git a/vnrecode/printer.py b/vnrecode/printer.py index 6080caf..30aa496 100644 --- a/vnrecode/printer.py +++ b/vnrecode/printer.py @@ -15,7 +15,7 @@ class Printer: # Fill whole string with spaces for cleaning progress bar @staticmethod - def clean_str(string): + def clean_str(string: str) -> str: return string + " " * (os.get_terminal_size().columns - len(string)) @staticmethod @@ -23,20 +23,20 @@ class Printer: if sys.platform == "win32": colorama.init() - def bar_print(self, string): + def bar_print(self, string: str): print(string) self.bar.update() - def info(self, string): + def info(self, string: str): self.bar_print(self.clean_str(f"\r\033[100m- {string}\033[49m")) - def warning(self, string): + def warning(self, string: str): self.bar_print(self.clean_str(f"\r\033[93m!\033[0m {string}\033[49m")) - def error(self, string): + def error(self, string: str): self.bar_print(self.clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m")) - def files(self, source, dest, dest_ext, comment): + def files(self, source: str, dest: str, dest_ext: str, comment: str): source_ext = os.path.splitext(source)[1] source_name = os.path.splitext(source)[0] diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 8acd4dc..ec5c2c5 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -17,15 +17,15 @@ class Utils: os.system("pause") @staticmethod - def get_size(directory): + def get_size(directory: str) -> int: total_size = 0 for folder, folders, files in os.walk(directory): for file in files: - if not os.path.islink(f"{folder}/{file}"): - total_size += os.path.getsize(f"{folder}/{file}") + if not os.path.islink(os.path.join(folder, file)): + total_size += os.path.getsize(os.path.join(folder, file)) return total_size - def get_compression(self, source, output): + def get_compression(self, source: str, output: str): try: source = self.get_size(source) output = self.get_size(output) @@ -35,50 +35,50 @@ class Utils: except ZeroDivisionError: self.printer.warning("Nothing compressed!") - def get_compression_status(self, orig_folder): - orig_folder_len = 0 - comp_folder_len = 0 + def get_compression_status(self, source: str): + source_len = 0 + output_len = 0 - for folder, folders, files in os.walk(orig_folder): - orig_folder_len += len(files) + for folder, folders, files in os.walk(source): + source_len += len(files) - for folder, folders, files in os.walk(f'{orig_folder}_compressed'): + for folder, folders, files in os.walk(f'{source}_compressed'): for file in files: if not os.path.splitext(file)[1].count("(copy)"): - comp_folder_len += 1 + output_len += 1 if self.errors != 0: self.printer.warning("Some files failed to compress!") - if orig_folder_len == comp_folder_len: + if source_len == output_len: self.printer.info("Success!") - self.get_compression(orig_folder, f"{orig_folder}_compressed") else: self.printer.warning("Original and compressed folders are not identical!") - self.get_compression(orig_folder, f"{orig_folder}_compressed") + self.get_compression(source, f"{source}_compressed") - def add_unprocessed_file(self, orig_folder, new_folder): + def add_unprocessed_file(self, source: str, output: str): if self.params.copy_unprocessed: - filename = orig_folder.split("/").pop() - copyfile(orig_folder, new_folder) + filename = os.path.split(source)[-1] + copyfile(source, output) self.printer.info(f"File {filename} copied to compressed folder.") - def check_duplicates(self, in_dir, out_dir, filename): - duplicates = glob(f"{in_dir}/{os.path.splitext(filename)[0]}.*") + def check_duplicates(self, source: str, output: str, filename: str) -> str: + duplicates = glob(os.path.join(source, os.path.splitext(filename)[0]+".*")) if len(duplicates) > 1: if filename not in self.duplicates: self.duplicates.append(filename) new_name = os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1] - return f"{out_dir}/{new_name}" - return f"{out_dir}/{filename}" + return os.path.join(output, new_name) + return os.path.join(output, filename) def print_duplicates(self): for filename in self.duplicates: self.printer.warning( f'Duplicate file has been found! Check manually this files - "{filename}", ' - f'"{os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1]}"') + f'"{os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1]}"' + ) - def mimic_rename(self, filename, target, source): + def mimic_rename(self, filename: str, target: str, source: str): if filename.count("(vncopy)"): orig_name = filename.replace("(vncopy)", "") index = self.duplicates.index(os.path.split(orig_name)[-1]) From 90a6b4e0c1d0f94c87a2441a2db5ee22a3e4d0c1 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 03:25:43 +0300 Subject: [PATCH 084/105] vnrecode: add re to duplicate check --- vnrecode/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vnrecode/utils.py b/vnrecode/utils.py index ec5c2c5..58cf962 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -2,6 +2,7 @@ from shutil import copyfile from glob import glob import sys import os +import re class Utils: @@ -63,7 +64,10 @@ class Utils: self.printer.info(f"File {filename} copied to compressed folder.") def check_duplicates(self, source: str, output: str, filename: str) -> str: - duplicates = glob(os.path.join(source, os.path.splitext(filename)[0]+".*")) + files = glob(os.path.join(source, os.path.splitext(filename)[0])+".*") + re_pattern = re.compile(os.path.join(source, os.path.splitext(filename)[0])+r".[a-zA-Z0-9]+$") + duplicates = [f for f in files if re_pattern.match(f)] + if len(duplicates) > 1: if filename not in self.duplicates: self.duplicates.append(filename) From 0b43756ef511177158d7d34a2c3271bd91bcea7a Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Wed, 4 Sep 2024 04:30:01 +0300 Subject: [PATCH 085/105] vnrecode: fix duplicates check for case insensitive fs --- vnrecode/application.py | 2 +- vnrecode/utils.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/vnrecode/application.py b/vnrecode/application.py index 68fa45f..912c526 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -42,7 +42,7 @@ class Application: with ThreadPoolExecutor(max_workers=self.params.workers) as executor: futures = [ executor.submit(self.compress, folder, file, source, output) - for file in files if os.path.isfile(f'{folder}/{file}') + for file in files if os.path.isfile(os.path.join(folder, file)) ] for future in as_completed(futures): future.result() diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 58cf962..3410eae 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -4,6 +4,9 @@ import sys import os import re +import fnmatch + + class Utils: def __init__(self, params, printer): @@ -64,12 +67,11 @@ class Utils: self.printer.info(f"File {filename} copied to compressed folder.") def check_duplicates(self, source: str, output: str, filename: str) -> str: - files = glob(os.path.join(source, os.path.splitext(filename)[0])+".*") - re_pattern = re.compile(os.path.join(source, os.path.splitext(filename)[0])+r".[a-zA-Z0-9]+$") - duplicates = [f for f in files if re_pattern.match(f)] + re_pattern = re.compile(os.path.splitext(filename)[0]+r".[a-zA-Z0-9]+$", re.IGNORECASE) + duplicates = [name for name in os.listdir(source) if re_pattern.match(name)] if len(duplicates) > 1: - if filename not in self.duplicates: + if filename.lower() not in (duplicate.lower() for duplicate in self.duplicates): self.duplicates.append(filename) new_name = os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1] return os.path.join(output, new_name) From d8e55bac9a8b9cacba3d96e7bad1ff8ed6f44a46 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sun, 13 Oct 2024 22:28:14 +0300 Subject: [PATCH 086/105] Update all README.md files --- README.md | 8 ++++---- unrenapk/README.md | 4 ++-- vnds2renpy/README.md | 4 ++-- vnrecode/README.md | 39 +++++++++++++++++++++++++++++---------- 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 370d3eb..b3b0091 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,10 @@ Collection of tools used by VienDesu! Porting Team ### Tools -* `FFMpeg-Compressor` - Python utility to compress Visual Novel Resources -* `RenPy-Android-Unpack` - A Python script for extracting game project from Ren'Py based .apk and .obb files -* `RenPy-Unpacker` - Simple .rpy script that will make any RenPy game unpack itself -* `VNDS-to-RenPy` - Simple script for converting VNDS engine scripts to .rpy ones +* `vnrecode` - Python utility to compress Visual Novel Resources +* `unrenapk` - A Python script for extracting game project from Ren'Py based .apk and .obb files +* `renpy-ripper` - Simple .rpy script that will make any RenPy game unpack itself +* `vnds2renpy` - Simple script for converting VNDS engine scripts to .rpy ones ### Installation #### Download from releases: diff --git a/unrenapk/README.md b/unrenapk/README.md index f6992e1..cf83a2c 100644 --- a/unrenapk/README.md +++ b/unrenapk/README.md @@ -1,6 +1,6 @@ -## RenPy-Android-Unpack +## unrenapk A simple Python script for unpacking Ren'Py based .apk and .obb files to ready to use Ren'Py SDK's Project ### How to use * Put some .apk & .obb files in folder -* `rendroid-unpack` (It unpacks all .apk and .obb files in the directory where it is located) +* `unrenapk` (It unpacks all .apk and .obb files in the directory where it is located) diff --git a/vnds2renpy/README.md b/vnds2renpy/README.md index c2ba3df..28091ef 100644 --- a/vnds2renpy/README.md +++ b/vnds2renpy/README.md @@ -1,7 +1,7 @@ -## VNDS-to-RenPy +## vnds2renpy Simple script for converting vnds scripts to rpy ### How to use * Extract VNDS visual novel's archive and get `scripts` folder from these (or `script.zip` if scripts folder is empty) -* Launch `convert.py` (It will automatically extract `scripts.zip` archive (if it needed) and converts .scr scripts to .rpy) +* Launch `vnds2renpy` (It will automatically extract `scripts.zip` archive (if it needed) and converts .scr scripts to .rpy) diff --git a/vnrecode/README.md b/vnrecode/README.md index 40b37d8..ba01b1f 100644 --- a/vnrecode/README.md +++ b/vnrecode/README.md @@ -1,13 +1,7 @@ -## FFMpeg-Compressor -Python utility uses ffmpeg to compress Visual Novel Resources +## vnrecode +Python utility uses Pillow and ffmpeg to compress Visual Novel Resources -### How to use -* Download `ffmpeg-comp.toml` and put in next to binary or in to `/etc` folder -* Change the configuration of the utility in `ffmpeg-comp.toml` for yourself -* `ffmpeg-comp {folder}` -* In result, you get `{folder-compressed}` near with original `{folder}` - -### Configuration +### Configuration file #### FFMPEG section * CopyUnprocessed - Copy all files that failed to compress by ffmpeg to destination folder. In can help to recreate original folder, but with compressed files. (default: `true`) * ForceCompress - Force try to compress all files in directory via ffmpeg. (default: `false`) @@ -32,9 +26,34 @@ Python utility uses ffmpeg to compress Visual Novel Resources * Extension - Required image file extension. It supports: `.3gp`, `.amv`, `.avi`, `.gif`, `.m2l`, `.m4v`, `.mkv`, `.mov`, `.mp4`, `.m4v`, `.mpeg`, `.mpv`, `.webm`, `.ogv` * Codec - (Maybe optional in future) Required video codec. (See official ffmpeg documentation for supported codecs) +### CLI Parameters +##### positional arguments: +* source - Directory with game files to recode + +##### options: +* ` -h, --help ` - show this help message and exit +* ` -c CONFIG ` - Utility config file +* ` -nu ` - Don't copy unprocessed +* ` -f, --force ` - Try to recode unknown files +* ` -nm, --no-mimic ` - Disable mimic mode +* ` -v, --show_errors ` - Show recode errors +* ` --webp-rgb ` - Recode .webp without alpha channel +* ` -j JOBS, --jobs JOBS ` - Number of threads +* ` -ae A_EXT ` - Audio extension +* ` -ab A_BIT ` - Audio bit rate +* ` -id I_DOWN ` - Image resolution downscale multiplier +* ` -ie I_EXT ` - Image extension +* ` -ife I_FALLEXT ` - Image fallback extension +* ` -il ` - Image losing compression mode +* ` -iq I_QUALITY ` - Image quality +* ` --v_crf V_CRF ` - Video CRF number +* ` -vs ` - Skip video recoding +* ` -ve V_EXT ` - Video extension +* ` -vc V_CODEC ` - Video codec name + ### TODO (for testing branch) * [x] Recreate whole game directory with compressed files * [x] Cross-platform (Easy Windows usage and binaries, macOS binaries) -* [x] Use ffmpeg python bindings instead of cli commands +* [x] Use ffmpeg python bindings instead of os.system * [x] Multithread * [ ] Reorganize code \ No newline at end of file From 9bb3cdcccbbab5a97160d4ab247f549d441ad6e9 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 18 Oct 2024 20:50:40 +0300 Subject: [PATCH 087/105] vnrecode: make it private! --- vnrecode/application.py | 40 ++++++++--------- vnrecode/compress.py | 98 ++++++++++++++++++++--------------------- vnrecode/params.py | 4 +- vnrecode/utils.py | 80 ++++++++++++++++----------------- 4 files changed, 107 insertions(+), 115 deletions(-) diff --git a/vnrecode/application.py b/vnrecode/application.py index 912c526..239bb05 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -13,41 +13,37 @@ from .utils import Utils class Application: def __init__(self, params: Params, compress: Compress, printer: Printer, utils: Utils): - self.params = params - self.compress = compress.compress - self.printer = printer - self.utils = utils - - def compress_worker(self, folder: str, file: str, source: str, output: str): - if os.path.isfile(os.path.join(folder, file)): - self.compress(folder, file, source, output) + self.__params = params + self.__compress = compress.compress + self.__printer = printer + self.__utils = utils def run(self): start_time = datetime.now() - self.printer.win_ascii_esc() + self.__printer.win_ascii_esc() - source = os.path.abspath(self.params.source) + source = self.__params.source - if os.path.exists(f"{source}_compressed"): - shutil.rmtree(f"{source}_compressed") + if os.path.exists(self.__params.dest): + shutil.rmtree(self.__params.dest) - self.printer.info("Creating folders...") + self.__printer.info("Creating folders...") for folder, folders, files in os.walk(source): - if not os.path.exists(folder.replace(source, f"{source}_compressed")): - os.mkdir(folder.replace(source, f"{source}_compressed")) + if not os.path.exists(folder.replace(source, self.__params.dest)): + os.mkdir(folder.replace(source, self.__params.dest)) - self.printer.info(f'Compressing "{folder.replace(source, os.path.split(source)[-1])}" folder...') - output = folder.replace(source, f"{source}_compressed") + self.__printer.info(f'Compressing "{folder.replace(source, os.path.split(source)[-1])}" folder...') + output = folder.replace(source, self.__params.dest) - with ThreadPoolExecutor(max_workers=self.params.workers) as executor: + with ThreadPoolExecutor(max_workers=self.__params.workers) as executor: futures = [ - executor.submit(self.compress, folder, file, source, output) + executor.submit(self.__compress, folder, file, output) for file in files if os.path.isfile(os.path.join(folder, file)) ] for future in as_completed(futures): future.result() - self.utils.print_duplicates() - self.utils.get_compression_status(source) - self.utils.sys_pause() + self.__utils.print_duplicates() + self.__utils.get_compression_status() + self.__utils.sys_pause() print(f"Time taken: {datetime.now() - start_time}") \ No newline at end of file diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 18fb20e..c484243 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -44,13 +44,13 @@ class File: class Compress: def __init__(self, params: Params, printer: Printer, utils: Utils): - self.params = params - self.printer = printer - self.utils = utils + self.__params = params + self.__printer = printer + self.__utils = utils def audio(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: - bit_rate = self.params.audio_bitrate - out_file = self.utils.check_duplicates(in_dir, out_dir, f'{path.splitext(file)[0]}.{extension}') + bit_rate = self.__params.audio_bitrate + out_file = self.__utils.check_duplicates(in_dir, out_dir, f'{path.splitext(file)[0]}.{extension}') try: (FFmpeg() .input(path.join(in_dir, file)) @@ -59,18 +59,18 @@ class Compress: .execute() ) except FFmpegError as e: - self.utils.add_unprocessed_file(path.join(in_dir, file), path.join(out_dir, file)) - self.utils.errors += 1 - if not self.params.hide_errors: - self.printer.error(f"File {file} can't be processed! Error: {e}") - self.printer.files(file, path.splitext(file)[0], extension, f"{bit_rate}") + self.__utils.add_unprocessed_file(path.join(in_dir, file), path.join(out_dir, file)) + self.__utils.errors += 1 + if not self.__params.hide_errors: + self.__printer.error(f"File {file} can't be processed! Error: {e}") + self.__printer.files(file, path.splitext(file)[0], extension, f"{bit_rate}") return out_file def video(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: - if not self.params.video_skip: - out_file = self.utils.check_duplicates(in_dir, out_dir, f'{path.splitext(file)[0]}.{extension}') - codec = self.params.video_codec - crf = self.params.video_crf + if not self.__params.video_skip: + out_file = self.__utils.check_duplicates(in_dir, out_dir, f'{path.splitext(file)[0]}.{extension}') + codec = self.__params.video_codec + crf = self.__params.video_crf try: (FFmpeg() @@ -80,33 +80,33 @@ class Compress: .output(out_file,{"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) .execute() ) - self.printer.files(file, path.splitext(file)[0], extension, codec) + self.__printer.files(file, path.splitext(file)[0], extension, codec) except FFmpegError as e: - self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') - self.utils.errors += 1 - if not self.params.hide_errors: - self.printer.error(f"File {file} can't be processed! Error: {e}") + self.__utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') + self.__utils.errors += 1 + if not self.__params.hide_errors: + self.__printer.error(f"File {file} can't be processed! Error: {e}") return out_file else: - self.utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') + self.__utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') return f'{out_dir}/{path.splitext(file)[0]}.{extension}' def image(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: - quality = self.params.image_quality - out_file = self.utils.check_duplicates(in_dir, out_dir, f"{path.splitext(file)[0]}.{extension}") + quality = self.__params.image_quality + out_file = self.__utils.check_duplicates(in_dir, out_dir, f"{path.splitext(file)[0]}.{extension}") try: image = Image.open(path.join(in_dir, file)) if (extension == "jpg" or extension == "jpeg" or - (extension == "webp" and not self.params.webp_rgba)): + (extension == "webp" and not self.__params.webp_rgba)): if File.has_transparency(image): - self.printer.warning(f"{file} has transparency. Changing to fallback...") - extension = self.params.image_fall_ext + self.__printer.warning(f"{file} has transparency. Changing to fallback...") + extension = self.__params.image_fall_ext if File.has_transparency(image): image.convert('RGBA') - res_downscale = self.params.image_downscale + res_downscale = self.__params.image_downscale if res_downscale != 1: width, height = image.size new_size = (int(width / res_downscale), int(height / res_downscale)) @@ -114,21 +114,21 @@ class Compress: image.save(out_file, optimize=True, - lossless=self.params.image_lossless, + lossless=self.__params.image_lossless, quality=quality, minimize_size=True) - self.printer.files(file, path.splitext(file)[0], extension, f"{quality}%") + self.__printer.files(file, path.splitext(file)[0], extension, f"{quality}%") except Exception as e: - self.utils.add_unprocessed_file(path.join(in_dir, file), path.join(out_dir, file)) - self.utils.errors += 1 - if not self.params.hide_errors: - self.printer.error(f"File {file} can't be processed! Error: {e}") + self.__utils.add_unprocessed_file(path.join(in_dir, file), path.join(out_dir, file)) + self.__utils.errors += 1 + if not self.__params.hide_errors: + self.__printer.error(f"File {file} can't be processed! Error: {e}") return out_file def unknown(self, in_dir: str, filename: str, out_dir: str) -> str: - if self.params.force_compress: - self.printer.unknown_file(filename) - out_file = self.utils.check_duplicates(in_dir, out_dir, filename) + if self.__params.force_compress: + self.__printer.unknown_file(filename) + out_file = self.__utils.check_duplicates(in_dir, out_dir, filename) try: (FFmpeg() .input(path.join(in_dir, filename)) @@ -136,28 +136,28 @@ class Compress: .execute() ) except FFmpegError as e: - self.utils.add_unprocessed_file(path.join(in_dir, filename), path.join(out_dir, filename)) - self.utils.errors += 1 - if not self.params.hide_errors: - self.printer.error(f"File {filename} can't be processed! Error: {e}") + self.__utils.add_unprocessed_file(path.join(in_dir, filename), path.join(out_dir, filename)) + self.__utils.errors += 1 + if not self.__params.hide_errors: + self.__printer.error(f"File {filename} can't be processed! Error: {e}") return out_file else: - self.utils.add_unprocessed_file(path.join(in_dir, filename), path.join(out_dir, filename)) + self.__utils.add_unprocessed_file(path.join(in_dir, filename), path.join(out_dir, filename)) return path.join(out_dir, filename) - def compress(self, _dir: str, filename: str, source: str, output: str): + def compress(self, dir_: str, filename: str, output: str): match File.get_type(filename): case "audio": - out_file = self.audio(_dir, filename, output, self.params.audio_ext) + out_file = self.audio(dir_, filename, output, self.__params.audio_ext) case "image": - out_file = self.image(_dir, filename, output, self.params.image_ext) + out_file = self.image(dir_, filename, output, self.__params.image_ext) case "video": - out_file = self.video(_dir, filename, output, self.params.video_ext) + out_file = self.video(dir_, filename, output, self.__params.video_ext) case "unknown": - out_file = self.unknown(_dir, filename, output) + out_file = self.unknown(dir_, filename, output) - if self.params.mimic_mode: - self.utils.mimic_rename(out_file, path.join(_dir, filename), source) + if self.__params.mimic_mode: + self.__utils.mimic_rename(out_file, path.join(dir_, filename)) - self.printer.bar.update() - self.printer.bar.next() + self.__printer.bar.update() + self.__printer.bar.next() diff --git a/vnrecode/params.py b/vnrecode/params.py index ea15691..ad4ec59 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -29,6 +29,7 @@ class Params: video_codec: str source: str + dest: str @classmethod def setup(cls) -> Self: @@ -59,12 +60,13 @@ class Params: video_ext = config["VIDEO"]["Extension"] if args.config else args.v_ext video_codec = config["VIDEO"]["Codec"] if args.config else args.v_codec source = args.source + dest = f"{source}_compressed" return cls( copy_unprocessed, force_compress, mimic_mode, hide_errors, webp_rgba, workers, audio_ext, audio_bitrate, image_downscale, image_ext, image_fall_ext, image_lossless, image_quality, - video_crf, video_skip, video_ext, video_codec, source + video_crf, video_skip, video_ext, video_codec, source, dest ) @staticmethod diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 3410eae..a75e4c3 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -1,19 +1,16 @@ from shutil import copyfile -from glob import glob import sys import os import re -import fnmatch - class Utils: - def __init__(self, params, printer): - self.errors = 0 - self.params = params - self.printer = printer - self.duplicates = [] + def __init__(self, params_inst, printer_inst): + self.__errors = 0 + self.__params = params_inst + self.__printer = printer_inst + self.__duplicates = [] @staticmethod def sys_pause(): @@ -29,66 +26,63 @@ class Utils: total_size += os.path.getsize(os.path.join(folder, file)) return total_size - def get_compression(self, source: str, output: str): + def get_compression_status(self): + source_len = 0 + output_len = 0 + + for folder, folders, files in os.walk(self.__params.source): + source_len += len(files) + + for folder, folders, files in os.walk(self.__params.dest): + for file in files: + if not os.path.splitext(file)[1].count("(vncopy)"): + output_len += 1 + + if self.__errors != 0: + self.__printer.warning("Some files failed to compress!") + + if source_len == output_len: + self.__printer.info("Success!") + else: + self.__printer.warning("Original and compressed folders are not identical!") try: - source = self.get_size(source) - output = self.get_size(output) + source = self.get_size(self.__params.source) + output = self.get_size(self.__params.dest) print(f"\nResult: {source/1024/1024:.2f}MB -> " f"{output/1024/1024:.2f}MB ({(output - source)/1024/1024:.2f}MB)") except ZeroDivisionError: - self.printer.warning("Nothing compressed!") - - def get_compression_status(self, source: str): - source_len = 0 - output_len = 0 - - for folder, folders, files in os.walk(source): - source_len += len(files) - - for folder, folders, files in os.walk(f'{source}_compressed'): - for file in files: - if not os.path.splitext(file)[1].count("(copy)"): - output_len += 1 - - if self.errors != 0: - self.printer.warning("Some files failed to compress!") - - if source_len == output_len: - self.printer.info("Success!") - else: - self.printer.warning("Original and compressed folders are not identical!") - self.get_compression(source, f"{source}_compressed") + self.__printer.warning("Nothing compressed!") def add_unprocessed_file(self, source: str, output: str): - if self.params.copy_unprocessed: + if self.__params.copy_unprocessed: filename = os.path.split(source)[-1] copyfile(source, output) - self.printer.info(f"File {filename} copied to compressed folder.") + self.__printer.info(f"File {filename} copied to compressed folder.") def check_duplicates(self, source: str, output: str, filename: str) -> str: re_pattern = re.compile(os.path.splitext(filename)[0]+r".[a-zA-Z0-9]+$", re.IGNORECASE) duplicates = [name for name in os.listdir(source) if re_pattern.match(name)] if len(duplicates) > 1: - if filename.lower() not in (duplicate.lower() for duplicate in self.duplicates): - self.duplicates.append(filename) + if filename.lower() not in (duplicate.lower() for duplicate in self.__duplicates): + self.__duplicates.append(filename) new_name = os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1] return os.path.join(output, new_name) return os.path.join(output, filename) def print_duplicates(self): - for filename in self.duplicates: - self.printer.warning( + for filename in self.__duplicates: + self.__printer.warning( f'Duplicate file has been found! Check manually this files - "{filename}", ' f'"{os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1]}"' ) - def mimic_rename(self, filename: str, target: str, source: str): + def mimic_rename(self, filename: str, target: str): if filename.count("(vncopy)"): orig_name = filename.replace("(vncopy)", "") - index = self.duplicates.index(os.path.split(orig_name)[-1]) - self.duplicates[index] = os.path.split(target)[-1] + index = self.__duplicates.index(os.path.split(orig_name)[-1]) + self.__duplicates[index] = os.path.split(target)[-1] target = os.path.splitext(target)[0] + "(vncopy)" + os.path.splitext(target)[1] - os.rename(filename, target.replace(source, f"{source}_compressed")) \ No newline at end of file + os.rename(filename, target.replace(self.__params.source, self.__params.dest)) \ No newline at end of file From 4e6fd332c54917ef9bc6a839f3c689191bcbaeba Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 18 Oct 2024 22:54:37 +0300 Subject: [PATCH 088/105] vnrecode: rewrite duplicates processing --- .gitignore | 1 + vnrecode/compress.py | 52 +++++++++++++++++--------------------------- vnrecode/utils.py | 47 ++++++++++++++++++++------------------- 3 files changed, 46 insertions(+), 54 deletions(-) diff --git a/.gitignore b/.gitignore index e6aa831..e3947bb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /output/ /tests/ +/tests_compressed/ /build/ /dist/ /vntools.egg-info/ diff --git a/vnrecode/compress.py b/vnrecode/compress.py index c484243..d1a2919 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -50,7 +50,8 @@ class Compress: def audio(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: bit_rate = self.__params.audio_bitrate - out_file = self.__utils.check_duplicates(in_dir, out_dir, f'{path.splitext(file)[0]}.{extension}') + prefix = self.__utils.get_hash(file) + out_file = path.join(out_dir, f'{prefix}_{path.splitext(file)[0]}.{extension}') try: (FFmpeg() .input(path.join(in_dir, file)) @@ -59,16 +60,14 @@ class Compress: .execute() ) except FFmpegError as e: - self.__utils.add_unprocessed_file(path.join(in_dir, file), path.join(out_dir, file)) - self.__utils.errors += 1 - if not self.__params.hide_errors: - self.__printer.error(f"File {file} can't be processed! Error: {e}") + self.__utils.catch_unprocessed(path.join(in_dir, file), out_file, e) self.__printer.files(file, path.splitext(file)[0], extension, f"{bit_rate}") return out_file def video(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: + prefix = self.__utils.get_hash(file) + out_file = path.join(out_dir, f'{prefix}_{path.splitext(file)[0]}.{extension}') if not self.__params.video_skip: - out_file = self.__utils.check_duplicates(in_dir, out_dir, f'{path.splitext(file)[0]}.{extension}') codec = self.__params.video_codec crf = self.__params.video_crf @@ -82,18 +81,15 @@ class Compress: ) self.__printer.files(file, path.splitext(file)[0], extension, codec) except FFmpegError as e: - self.__utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') - self.__utils.errors += 1 - if not self.__params.hide_errors: - self.__printer.error(f"File {file} can't be processed! Error: {e}") - return out_file + self.__utils.catch_unprocessed(path.join(in_dir, file), out_file, e) else: - self.__utils.add_unprocessed_file(f'{in_dir}/{file}', f'{out_dir}/{file}') - return f'{out_dir}/{path.splitext(file)[0]}.{extension}' + self.__utils.copy_unprocessed(path.join(in_dir, file), out_file) + return out_file def image(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: quality = self.__params.image_quality - out_file = self.__utils.check_duplicates(in_dir, out_dir, f"{path.splitext(file)[0]}.{extension}") + prefix = self.__utils.get_hash(file) + out_file = path.join(out_dir, f"{prefix}_{path.splitext(file)[0]}.{extension}") try: image = Image.open(path.join(in_dir, file)) @@ -119,31 +115,25 @@ class Compress: minimize_size=True) self.__printer.files(file, path.splitext(file)[0], extension, f"{quality}%") except Exception as e: - self.__utils.add_unprocessed_file(path.join(in_dir, file), path.join(out_dir, file)) - self.__utils.errors += 1 - if not self.__params.hide_errors: - self.__printer.error(f"File {file} can't be processed! Error: {e}") + self.__utils.catch_unprocessed(path.join(in_dir, file), out_file, e) return out_file - def unknown(self, in_dir: str, filename: str, out_dir: str) -> str: + def unknown(self, in_dir: str, file: str, out_dir: str) -> str: + prefix = self.__utils.get_hash(file) + out_file = path.join(out_dir, f"{prefix}_{file}") if self.__params.force_compress: - self.__printer.unknown_file(filename) - out_file = self.__utils.check_duplicates(in_dir, out_dir, filename) + self.__printer.unknown_file(file) try: (FFmpeg() - .input(path.join(in_dir, filename)) + .input(path.join(in_dir, file)) .output(out_file) .execute() ) except FFmpegError as e: - self.__utils.add_unprocessed_file(path.join(in_dir, filename), path.join(out_dir, filename)) - self.__utils.errors += 1 - if not self.__params.hide_errors: - self.__printer.error(f"File {filename} can't be processed! Error: {e}") - return out_file + self.__utils.catch_unprocessed(path.join(in_dir, file), out_file, e) else: - self.__utils.add_unprocessed_file(path.join(in_dir, filename), path.join(out_dir, filename)) - return path.join(out_dir, filename) + self.__utils.copy_unprocessed(path.join(in_dir, file), out_file) + return out_file def compress(self, dir_: str, filename: str, output: str): match File.get_type(filename): @@ -156,8 +146,6 @@ class Compress: case "unknown": out_file = self.unknown(dir_, filename, output) - if self.__params.mimic_mode: - self.__utils.mimic_rename(out_file, path.join(dir_, filename)) - + self.__utils.out_rename(out_file, filename) self.__printer.bar.update() self.__printer.bar.next() diff --git a/vnrecode/utils.py b/vnrecode/utils.py index a75e4c3..bf4e198 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -1,7 +1,7 @@ from shutil import copyfile +import hashlib import sys import os -import re class Utils: @@ -26,6 +26,10 @@ class Utils: total_size += os.path.getsize(os.path.join(folder, file)) return total_size + @staticmethod + def get_hash(filename: str) -> str: + return hashlib.md5(filename.encode()).hexdigest()[:8] + def get_compression_status(self): source_len = 0 output_len = 0 @@ -54,22 +58,23 @@ class Utils: except ZeroDivisionError: self.__printer.warning("Nothing compressed!") - def add_unprocessed_file(self, source: str, output: str): + def catch_unprocessed(self, source, output, error): + self.copy_unprocessed(source, error) + self.__errors += 1 + if not self.__params.hide_errors: + self.__printer.error(f"File {os.path.split(source)[-1]} can't be processed! Error: {error}") + + def copy_unprocessed(self, source, output): if self.__params.copy_unprocessed: - filename = os.path.split(source)[-1] copyfile(source, output) - self.__printer.info(f"File {filename} copied to compressed folder.") + self.__printer.info(f"File {os.path.split(source)[-1]} copied to compressed folder.") - def check_duplicates(self, source: str, output: str, filename: str) -> str: - re_pattern = re.compile(os.path.splitext(filename)[0]+r".[a-zA-Z0-9]+$", re.IGNORECASE) - duplicates = [name for name in os.listdir(source) if re_pattern.match(name)] - - if len(duplicates) > 1: - if filename.lower() not in (duplicate.lower() for duplicate in self.__duplicates): - self.__duplicates.append(filename) - new_name = os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1] - return os.path.join(output, new_name) - return os.path.join(output, filename) + def catch_duplicates(self, path: str) -> str: + if os.path.exists(path): + new_path = os.path.splitext(path)[0] + "(vncopy)" + os.path.splitext(path)[1] + self.__duplicates.append(new_path) + return new_path + return path def print_duplicates(self): for filename in self.__duplicates: @@ -78,11 +83,9 @@ class Utils: f'"{os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1]}"' ) - def mimic_rename(self, filename: str, target: str): - if filename.count("(vncopy)"): - orig_name = filename.replace("(vncopy)", "") - index = self.__duplicates.index(os.path.split(orig_name)[-1]) - self.__duplicates[index] = os.path.split(target)[-1] - target = os.path.splitext(target)[0] + "(vncopy)" + os.path.splitext(target)[1] - - os.rename(filename, target.replace(self.__params.source, self.__params.dest)) \ No newline at end of file + def out_rename(self, filename: str, target: str): + if not self.__params.mimic_mode: + dest_name = self.catch_duplicates(os.path.join(os.path.dirname(filename), target)) + os.rename(filename, dest_name) + else: + os.rename(filename, os.path.join(os.path.dirname(filename), target)) From 1c1e8a92921146f7f243e6ba932d09ceee6b5b89 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 19 Oct 2024 00:25:52 +0300 Subject: [PATCH 089/105] vnrecode: pathlib for all paths --- vnrecode/application.py | 25 ++++----- vnrecode/compress.py | 116 ++++++++++++++++++++-------------------- vnrecode/params.py | 12 ++--- vnrecode/printer.py | 14 +++-- vnrecode/utils.py | 45 +++++++++------- 5 files changed, 108 insertions(+), 104 deletions(-) diff --git a/vnrecode/application.py b/vnrecode/application.py index 239bb05..3b6843e 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime +from pathlib import Path import shutil import os @@ -12,11 +13,11 @@ from .utils import Utils class Application: - def __init__(self, params: Params, compress: Compress, printer: Printer, utils: Utils): - self.__params = params - self.__compress = compress.compress - self.__printer = printer - self.__utils = utils + def __init__(self, params_inst: Params, compress_inst: Compress, printer_inst: Printer, utils_inst: Utils): + self.__params = params_inst + self.__compress = compress_inst.compress + self.__printer = printer_inst + self.__utils = utils_inst def run(self): start_time = datetime.now() @@ -24,21 +25,21 @@ class Application: source = self.__params.source - if os.path.exists(self.__params.dest): + if self.__params.dest.exists(): shutil.rmtree(self.__params.dest) self.__printer.info("Creating folders...") for folder, folders, files in os.walk(source): - if not os.path.exists(folder.replace(source, self.__params.dest)): - os.mkdir(folder.replace(source, self.__params.dest)) + output = Path(folder.replace(str(source), str(self.__params.dest))) + if not output.exists(): + os.mkdir(output) - self.__printer.info(f'Compressing "{folder.replace(source, os.path.split(source)[-1])}" folder...') - output = folder.replace(source, self.__params.dest) + self.__printer.info(f'Compressing "{output}" folder...') with ThreadPoolExecutor(max_workers=self.__params.workers) as executor: futures = [ - executor.submit(self.__compress, folder, file, output) - for file in files if os.path.isfile(os.path.join(folder, file)) + executor.submit(self.__compress, Path(folder, file), Path(output)) + for file in files if Path(folder, file).is_file() ] for future in as_completed(futures): future.result() diff --git a/vnrecode/compress.py b/vnrecode/compress.py index d1a2919..7b06988 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -1,6 +1,6 @@ from ffmpeg import FFmpeg, FFmpegError +from pathlib import Path from PIL import Image -from os import path import pillow_avif from .printer import Printer @@ -11,7 +11,7 @@ from .utils import Utils class File: @staticmethod - def get_type(filename: str) -> str: + def get_type(filename: Path) -> str: extensions = { "audio": ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'], @@ -21,7 +21,7 @@ class File: } for file_type in extensions: - if path.splitext(filename)[1] in extensions[file_type]: + if filename.suffix in extensions[file_type]: return file_type return "unknown" @@ -43,61 +43,39 @@ class File: class Compress: - def __init__(self, params: Params, printer: Printer, utils: Utils): - self.__params = params - self.__printer = printer - self.__utils = utils + def __init__(self, params_inst: Params, printer_inst: Printer, utils_inst: Utils): + self.__params = params_inst + self.__printer = printer_inst + self.__utils = utils_inst - def audio(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: + def audio(self, input_path: Path, output_dir: Path, extension: str) -> Path: bit_rate = self.__params.audio_bitrate - prefix = self.__utils.get_hash(file) - out_file = path.join(out_dir, f'{prefix}_{path.splitext(file)[0]}.{extension}') + prefix = self.__utils.get_hash(input_path.name) + out_file = Path(output_dir, f'{prefix}_{input_path.stem}.{extension}') try: (FFmpeg() - .input(path.join(in_dir, file)) + .input(input_path) .option("hide_banner") .output(out_file,{"b:a": bit_rate, "loglevel": "error"}) .execute() ) except FFmpegError as e: - self.__utils.catch_unprocessed(path.join(in_dir, file), out_file, e) - self.__printer.files(file, path.splitext(file)[0], extension, f"{bit_rate}") + self.__utils.catch_unprocessed(input_path, out_file, e) + self.__printer.files(input_path, out_file, f"{bit_rate}") return out_file - def video(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: - prefix = self.__utils.get_hash(file) - out_file = path.join(out_dir, f'{prefix}_{path.splitext(file)[0]}.{extension}') - if not self.__params.video_skip: - codec = self.__params.video_codec - crf = self.__params.video_crf - - try: - (FFmpeg() - .input(path.join(in_dir, file)) - .option("hide_banner") - .option("hwaccel", "auto") - .output(out_file,{"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) - .execute() - ) - self.__printer.files(file, path.splitext(file)[0], extension, codec) - except FFmpegError as e: - self.__utils.catch_unprocessed(path.join(in_dir, file), out_file, e) - else: - self.__utils.copy_unprocessed(path.join(in_dir, file), out_file) - return out_file - - def image(self, in_dir: str, file: str, out_dir: str, extension: str) -> str: + def image(self, input_path: Path, output_dir: Path, extension: str) -> Path: quality = self.__params.image_quality - prefix = self.__utils.get_hash(file) - out_file = path.join(out_dir, f"{prefix}_{path.splitext(file)[0]}.{extension}") + prefix = self.__utils.get_hash(input_path.name) + out_file = Path(output_dir, f"{prefix}_{input_path.stem}.{extension}") try: - image = Image.open(path.join(in_dir, file)) + image = Image.open(input_path) if (extension == "jpg" or extension == "jpeg" or (extension == "webp" and not self.__params.webp_rgba)): if File.has_transparency(image): - self.__printer.warning(f"{file} has transparency. Changing to fallback...") - extension = self.__params.image_fall_ext + self.__printer.warning(f"{input_path.name} has transparency. Changing to fallback...") + out_file = Path(output_dir, f"{prefix}_{input_path.stem}.{self.__params.image_fall_ext}") if File.has_transparency(image): image.convert('RGBA') @@ -113,39 +91,61 @@ class Compress: lossless=self.__params.image_lossless, quality=quality, minimize_size=True) - self.__printer.files(file, path.splitext(file)[0], extension, f"{quality}%") + self.__printer.files(input_path, out_file, f"{quality}%") except Exception as e: - self.__utils.catch_unprocessed(path.join(in_dir, file), out_file, e) + self.__utils.catch_unprocessed(input_path, out_file, e) return out_file - def unknown(self, in_dir: str, file: str, out_dir: str) -> str: - prefix = self.__utils.get_hash(file) - out_file = path.join(out_dir, f"{prefix}_{file}") - if self.__params.force_compress: - self.__printer.unknown_file(file) + def video(self, input_path: Path, output_dir: Path, extension: str) -> Path: + prefix = self.__utils.get_hash(input_path.name) + out_file = Path(output_dir, f'{prefix}_{input_path.stem}.{extension}') + if not self.__params.video_skip: + codec = self.__params.video_codec + crf = self.__params.video_crf + try: (FFmpeg() - .input(path.join(in_dir, file)) + .input(input_path) + .option("hide_banner") + .option("hwaccel", "auto") + .output(out_file,{"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf) + .execute() + ) + self.__printer.files(input_path, out_file, codec) + except FFmpegError as e: + self.__utils.catch_unprocessed(input_path, out_file, e) + else: + self.__utils.copy_unprocessed(input_path, out_file) + return out_file + + def unknown(self, input_path: Path, output_dir: Path) -> Path: + prefix = self.__utils.get_hash(input_path.name) + out_file = Path(output_dir, f"{prefix}_{input_path.name}") + if self.__params.force_compress: + self.__printer.unknown_file(input_path.name) + try: + (FFmpeg() + .input(input_path) .output(out_file) .execute() ) except FFmpegError as e: - self.__utils.catch_unprocessed(path.join(in_dir, file), out_file, e) + self.__utils.catch_unprocessed(input_path, out_file, e) else: - self.__utils.copy_unprocessed(path.join(in_dir, file), out_file) + self.__utils.copy_unprocessed(input_path, out_file) return out_file - def compress(self, dir_: str, filename: str, output: str): - match File.get_type(filename): + def compress(self, source: Path, output: Path): + match File.get_type(source): case "audio": - out_file = self.audio(dir_, filename, output, self.__params.audio_ext) + out_file = self.audio(source, output, self.__params.audio_ext) case "image": - out_file = self.image(dir_, filename, output, self.__params.image_ext) + out_file = self.image(source, output, self.__params.image_ext) case "video": - out_file = self.video(dir_, filename, output, self.__params.video_ext) + out_file = self.video(source, output, self.__params.video_ext) case "unknown": - out_file = self.unknown(dir_, filename, output) + out_file = self.unknown(source, output) - self.__utils.out_rename(out_file, filename) + self.__utils.out_rename(out_file, source.name) self.__printer.bar.update() self.__printer.bar.next() diff --git a/vnrecode/params.py b/vnrecode/params.py index ad4ec59..d4db410 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -1,8 +1,8 @@ from argparse import ArgumentParser, Namespace from dataclasses import dataclass +from pathlib import Path from typing import Self import tomllib -import os @dataclass class Params: @@ -28,14 +28,14 @@ class Params: video_ext: str video_codec: str - source: str - dest: str + source: Path + dest: Path @classmethod def setup(cls) -> Self: args = cls.get_args() if args.config is not None: - if os.path.isfile(args.config): + if Path(args.config).is_file(): with open(args.config, "rb") as cfile: config = tomllib.load(cfile) else: @@ -59,8 +59,8 @@ class Params: video_skip = config["VIDEO"]["SkipVideo"] if args.config else args.v_skip video_ext = config["VIDEO"]["Extension"] if args.config else args.v_ext video_codec = config["VIDEO"]["Codec"] if args.config else args.v_codec - source = args.source - dest = f"{source}_compressed" + source = Path(args.source) + dest = Path(f"{args.source}_compressed") return cls( copy_unprocessed, force_compress, mimic_mode, hide_errors, webp_rgba, workers, diff --git a/vnrecode/printer.py b/vnrecode/printer.py index 30aa496..4de4f3c 100644 --- a/vnrecode/printer.py +++ b/vnrecode/printer.py @@ -1,12 +1,12 @@ from progress.bar import IncrementalBar +from pathlib import Path import colorama import sys import os - class Printer: - def __init__(self, folder): + def __init__(self, folder: Path): file_count = 0 for folder, folders, file in os.walk(folder): file_count += len(file) @@ -36,11 +36,9 @@ class Printer: def error(self, string: str): self.bar_print(self.clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m")) - def files(self, source: str, dest: str, dest_ext: str, comment: str): - source_ext = os.path.splitext(source)[1] - source_name = os.path.splitext(source)[0] + def files(self, source_path: Path, output_path: Path, comment: str): + self.bar_print(self.clean_str(f"\r\033[0;32m\u2713\033[0m \033[0;37m{source_path.stem}\033[0m{source_path.suffix}\033[0;37m -> " + f"{source_path.stem}\033[0m{output_path.suffix}\033[0;37m ({comment})\033[0m")) - self.bar_print(self.clean_str(f"\r\033[0;32m\u2713\033[0m \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m")) - - def unknown_file(self, file): + def unknown_file(self, file: str): self.bar_print(self.clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)")) diff --git a/vnrecode/utils.py b/vnrecode/utils.py index bf4e198..2b52f30 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -1,12 +1,16 @@ from shutil import copyfile +from pathlib import Path import hashlib import sys import os +from vnrecode.printer import Printer +from vnrecode.params import Params + class Utils: - def __init__(self, params_inst, printer_inst): + def __init__(self, params_inst: Params, printer_inst: Printer): self.__errors = 0 self.__params = params_inst self.__printer = printer_inst @@ -18,12 +22,13 @@ class Utils: os.system("pause") @staticmethod - def get_size(directory: str) -> int: + def get_size(directory: Path) -> int: total_size = 0 for folder, folders, files in os.walk(directory): for file in files: - if not os.path.islink(os.path.join(folder, file)): - total_size += os.path.getsize(os.path.join(folder, file)) + path = Path(folder, file) + if not path.is_symlink(): + total_size += path.stat().st_size return total_size @staticmethod @@ -39,7 +44,7 @@ class Utils: for folder, folders, files in os.walk(self.__params.dest): for file in files: - if not os.path.splitext(file)[1].count("(vncopy)"): + if not file.count("(vncopy)"): output_len += 1 if self.__errors != 0: @@ -58,20 +63,20 @@ class Utils: except ZeroDivisionError: self.__printer.warning("Nothing compressed!") - def catch_unprocessed(self, source, output, error): - self.copy_unprocessed(source, error) + def catch_unprocessed(self, input_path: Path, output_path: Path, error): + self.copy_unprocessed(input_path, output_path) self.__errors += 1 if not self.__params.hide_errors: - self.__printer.error(f"File {os.path.split(source)[-1]} can't be processed! Error: {error}") + self.__printer.error(f"File {input_path.name} can't be processed! Error: {error}") - def copy_unprocessed(self, source, output): + def copy_unprocessed(self, input_path: Path, output_path: Path): if self.__params.copy_unprocessed: - copyfile(source, output) - self.__printer.info(f"File {os.path.split(source)[-1]} copied to compressed folder.") + copyfile(input_path, output_path) + self.__printer.info(f"File {input_path.name} copied to compressed folder.") - def catch_duplicates(self, path: str) -> str: - if os.path.exists(path): - new_path = os.path.splitext(path)[0] + "(vncopy)" + os.path.splitext(path)[1] + def catch_duplicates(self, path: Path) -> Path: + if path.exists(): + new_path = Path(path.stem + "(vncopy)" + path.suffix) self.__duplicates.append(new_path) return new_path return path @@ -79,13 +84,13 @@ class Utils: def print_duplicates(self): for filename in self.__duplicates: self.__printer.warning( - f'Duplicate file has been found! Check manually this files - "{filename}", ' - f'"{os.path.splitext(filename)[0] + "(vncopy)" + os.path.splitext(filename)[1]}"' + f'Duplicate file has been found! Check manually this files - "{filename.name}", ' + f'"{filename.stem + "(vncopy)" + filename.suffix}"' ) - def out_rename(self, filename: str, target: str): + def out_rename(self, out_path: Path, target: str): if not self.__params.mimic_mode: - dest_name = self.catch_duplicates(os.path.join(os.path.dirname(filename), target)) - os.rename(filename, dest_name) + dest_name = self.catch_duplicates(Path(out_path.parent, target)) + os.rename(out_path, dest_name) else: - os.rename(filename, os.path.join(os.path.dirname(filename), target)) + os.rename(out_path, Path(out_path.parent, target)) From df20bd363685700213942e5f5cebf02776b154a0 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 19 Oct 2024 01:45:35 +0300 Subject: [PATCH 090/105] vnrecode: docstrings --- vnrecode/__main__.py | 4 +++ vnrecode/application.py | 11 ++++++- vnrecode/compress.py | 52 +++++++++++++++++++++++++++++++-- vnrecode/params.py | 12 ++++++++ vnrecode/printer.py | 57 ++++++++++++++++++++++++++++++++---- vnrecode/utils.py | 64 ++++++++++++++++++++++++++++++++--------- 6 files changed, 177 insertions(+), 23 deletions(-) diff --git a/vnrecode/__main__.py b/vnrecode/__main__.py index faf1839..fe45306 100644 --- a/vnrecode/__main__.py +++ b/vnrecode/__main__.py @@ -7,6 +7,10 @@ from .utils import Utils def init(): + """ + This function creates all needed class instances and run utility + :return: None + """ params = Params.setup() printer = Printer(params.source) utils = Utils(params, printer) diff --git a/vnrecode/application.py b/vnrecode/application.py index 3b6843e..c43102b 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -12,6 +12,9 @@ from .utils import Utils class Application: + """ + Main class for utility + """ def __init__(self, params_inst: Params, compress_inst: Compress, printer_inst: Printer, utils_inst: Utils): self.__params = params_inst @@ -20,6 +23,12 @@ class Application: self.__utils = utils_inst def run(self): + """ + Method creates a folder in which all the recoded files will be placed, + creates a queue of recoding processes for each file and, when the files are run out in the original folder, + calls functions to display the result + :return: None + """ start_time = datetime.now() self.__printer.win_ascii_esc() @@ -45,6 +54,6 @@ class Application: future.result() self.__utils.print_duplicates() - self.__utils.get_compression_status() + self.__utils.get_recode_status() self.__utils.sys_pause() print(f"Time taken: {datetime.now() - start_time}") \ No newline at end of file diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 7b06988..4050e4c 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -9,10 +9,17 @@ from .utils import Utils class File: + """ + Class contains some methods to work with files + """ @staticmethod - def get_type(filename: Path) -> str: - + def get_type(path: Path) -> str: + """ + Method returns filetype string for file + :param path: Path of file to determine type + :return: filetype string: audio, image, video, unknown + """ extensions = { "audio": ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma'], "image": ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png'], @@ -21,12 +28,17 @@ class File: } for file_type in extensions: - if filename.suffix in extensions[file_type]: + if path.suffix in extensions[file_type]: return file_type return "unknown" @staticmethod def has_transparency(img: Image) -> bool: + """ + Method checks if image has transparency + :param img: Pillow Image + :return: bool + """ if img.info.get("transparency", None) is not None: return True if img.mode == "P": @@ -49,6 +61,13 @@ class Compress: self.__utils = utils_inst def audio(self, input_path: Path, output_dir: Path, extension: str) -> Path: + """ + Method recodes audio files to another format using ffmpeg utility + :param input_path: Path of the original audio file + :param output_dir: Path of the output (compression) folder + :param extension: Extension of the new audio file + :return: Path of compressed audio file with md5 hash as prefix + """ bit_rate = self.__params.audio_bitrate prefix = self.__utils.get_hash(input_path.name) out_file = Path(output_dir, f'{prefix}_{input_path.stem}.{extension}') @@ -65,6 +84,13 @@ class Compress: return out_file def image(self, input_path: Path, output_dir: Path, extension: str) -> Path: + """ + Method recodes image files to another format using Pillow + :param input_path: Path of the original image file + :param output_dir: Path of the output (compression) folder + :param extension: Extension of the new image file + :return: Path of compressed image file with md5 hash as prefix + """ quality = self.__params.image_quality prefix = self.__utils.get_hash(input_path.name) out_file = Path(output_dir, f"{prefix}_{input_path.stem}.{extension}") @@ -97,6 +123,13 @@ class Compress: return out_file def video(self, input_path: Path, output_dir: Path, extension: str) -> Path: + """ + Method recodes video files to another format using ffmpeg utility + :param input_path: Path of the original video file + :param output_dir: Path of the output (compression) folder + :param extension: Extension of the new video file + :return: Path of compressed video file with md5 hash as prefix + """ prefix = self.__utils.get_hash(input_path.name) out_file = Path(output_dir, f'{prefix}_{input_path.stem}.{extension}') if not self.__params.video_skip: @@ -119,6 +152,13 @@ class Compress: return out_file def unknown(self, input_path: Path, output_dir: Path) -> Path: + """ + Method recodes files with "unknown" file format using ffmpeg, + in the hope that ffmpeg supports this file type and the default settings for it will reduce its size + :param input_path: Path of the original file + :param output_dir: Path of the output (compression) folder + :return: Path of compressed file with md5 hash as prefix + """ prefix = self.__utils.get_hash(input_path.name) out_file = Path(output_dir, f"{prefix}_{input_path.name}") if self.__params.force_compress: @@ -136,6 +176,12 @@ class Compress: return out_file def compress(self, source: Path, output: Path): + """ + It the core method for this program. Method determines file type and call compress function for it + :param source: Path of file to compress + :param output: Path of output file + :return: None + """ match File.get_type(source): case "audio": out_file = self.audio(source, output, self.__params.audio_ext) diff --git a/vnrecode/params.py b/vnrecode/params.py index d4db410..d0388f2 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -7,6 +7,10 @@ import tomllib @dataclass class Params: + """ + This dataclass contains all parameters for utility + """ + copy_unprocessed: bool force_compress: bool mimic_mode: bool @@ -33,6 +37,10 @@ class Params: @classmethod def setup(cls) -> Self: + """ + Method initialize all parameters and returns class instance + :return: Params instance + """ args = cls.get_args() if args.config is not None: if Path(args.config).is_file(): @@ -71,6 +79,10 @@ class Params: @staticmethod def get_args() -> Namespace: + """ + Method gets CLI arguments and returns argparse.Namespace instance + :return: argparse.Namespace of CLI args + """ parser = ArgumentParser(prog="vnrecode", description="Python utility to compress Visual Novel Resources" ) diff --git a/vnrecode/printer.py b/vnrecode/printer.py index 4de4f3c..a6046f9 100644 --- a/vnrecode/printer.py +++ b/vnrecode/printer.py @@ -5,40 +5,87 @@ import sys import os class Printer: + """ + Class implements CLI UI for this utility + """ - def __init__(self, folder: Path): + def __init__(self, source: Path): + """ + :param source: Path of original (compressing) folder to count its files for progress bar + """ file_count = 0 - for folder, folders, file in os.walk(folder): + for folder, folders, file in os.walk(source): file_count += len(file) self.bar = IncrementalBar('Compressing', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)') self.bar.update() - # Fill whole string with spaces for cleaning progress bar @staticmethod def clean_str(string: str) -> str: + """ + Method fills end of string with spaces to remove progress bar garbage from console + :param string: String to "clean" + :return: "Clean" string + """ return string + " " * (os.get_terminal_size().columns - len(string)) @staticmethod def win_ascii_esc(): + """ + Method setups colorama for cmd + :return: None + """ if sys.platform == "win32": colorama.init() def bar_print(self, string: str): + """ + Method prints some string in console and updates progress bar + :param string: String to print + :return: None + """ print(string) self.bar.update() def info(self, string: str): + """ + Method prints string with decor for info messages + :param string: String to print + :return: None + """ self.bar_print(self.clean_str(f"\r\033[100m- {string}\033[49m")) def warning(self, string: str): + """ + Method prints string with decor for warning messages + :param string: String to print + :return: None + """ self.bar_print(self.clean_str(f"\r\033[93m!\033[0m {string}\033[49m")) def error(self, string: str): + """ + Method prints string with decor for error messages + :param string: String to print + :return: None + """ self.bar_print(self.clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m")) def files(self, source_path: Path, output_path: Path, comment: str): + """ + Method prints the result of recoding a file with some decorations in the form: + input file name -> output file name (quality setting) + :param source_path: Input file Path + :param output_path: Output file Path + :param comment: Comment about recode quality setting + :return: None + """ self.bar_print(self.clean_str(f"\r\033[0;32m\u2713\033[0m \033[0;37m{source_path.stem}\033[0m{source_path.suffix}\033[0;37m -> " f"{source_path.stem}\033[0m{output_path.suffix}\033[0;37m ({comment})\033[0m")) - def unknown_file(self, file: str): - self.bar_print(self.clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)")) + def unknown_file(self, filename: str): + """ + Method prints the result of recoding unknown file + :param filename: Name of unknown file + :return: + """ + self.bar_print(self.clean_str(f"\r\u2713 \033[0;33m{filename}\033[0m (File will be force compressed via ffmpeg)")) diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 2b52f30..7da06d3 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -9,6 +9,9 @@ from vnrecode.params import Params class Utils: + """ + Class contains various methods for internal utility use + """ def __init__(self, params_inst: Params, printer_inst: Printer): self.__errors = 0 @@ -18,24 +21,27 @@ class Utils: @staticmethod def sys_pause(): + """ + Method calls pause for Windows cmd shell + :return: None + """ if sys.platform == "win32": os.system("pause") - @staticmethod - def get_size(directory: Path) -> int: - total_size = 0 - for folder, folders, files in os.walk(directory): - for file in files: - path = Path(folder, file) - if not path.is_symlink(): - total_size += path.stat().st_size - return total_size - @staticmethod def get_hash(filename: str) -> str: + """ + Method returns 8 chars of md5 hash for filename + :param filename: File name to get md5 + :return: 8 chars of md5 hash + """ return hashlib.md5(filename.encode()).hexdigest()[:8] - def get_compression_status(self): + def get_recode_status(self): + """ + Method prints recoding results + :return: None + """ source_len = 0 output_len = 0 @@ -55,8 +61,8 @@ class Utils: else: self.__printer.warning("Original and compressed folders are not identical!") try: - source = self.get_size(self.__params.source) - output = self.get_size(self.__params.dest) + source = sum(file.stat().st_size for file in self.__params.source.glob('**/*') if file.is_file()) + output = sum(file.stat().st_size for file in self.__params.dest.glob('**/*') if file.is_file()) print(f"\nResult: {source/1024/1024:.2f}MB -> " f"{output/1024/1024:.2f}MB ({(output - source)/1024/1024:.2f}MB)") @@ -64,24 +70,48 @@ class Utils: self.__printer.warning("Nothing compressed!") def catch_unprocessed(self, input_path: Path, output_path: Path, error): + """ + Method processes files that have not been recoded due to an error and prints error to console + if hide_errors parameter is False + :param input_path: Path of unprocessed file + :param output_path: Destination path of unprocessed file + :param error: Recoding exception + :return: None + """ self.copy_unprocessed(input_path, output_path) self.__errors += 1 if not self.__params.hide_errors: self.__printer.error(f"File {input_path.name} can't be processed! Error: {error}") def copy_unprocessed(self, input_path: Path, output_path: Path): + """ + Method copies an unprocessed file from the source folder to the destination folder + :param input_path: Path of unprocessed file + :param output_path: Destination path of unprocessed file + :return: None + """ if self.__params.copy_unprocessed: copyfile(input_path, output_path) self.__printer.info(f"File {input_path.name} copied to compressed folder.") def catch_duplicates(self, path: Path) -> Path: - if path.exists(): + """ + Method checks if file path exists and returns folder/filename(vncopy).ext path + if duplicate founded + :param path: Some file Path + :return: Duplicate path name with (vncopy) on end + """ + if path.is_file() and path.exists(): new_path = Path(path.stem + "(vncopy)" + path.suffix) self.__duplicates.append(new_path) return new_path return path def print_duplicates(self): + """ + Method prints message about all duplicates generated during recode process + :return: None + """ for filename in self.__duplicates: self.__printer.warning( f'Duplicate file has been found! Check manually this files - "{filename.name}", ' @@ -89,6 +119,12 @@ class Utils: ) def out_rename(self, out_path: Path, target: str): + """ + Method removes md5 hash from file name and changes file extension in dependence of mimic mode + :param out_path: Recoded file Path + :param target: Target filename + :return: None + """ if not self.__params.mimic_mode: dest_name = self.catch_duplicates(Path(out_path.parent, target)) os.rename(out_path, dest_name) From 407ab98000df8b18a9d1bf2593a588e1c9210196 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 19 Oct 2024 02:20:58 +0300 Subject: [PATCH 091/105] vnrecode: duplicates check for more than two files --- vnrecode/compress.py | 2 +- vnrecode/utils.py | 24 ++++++++++++++---------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 4050e4c..b1c2567 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -192,6 +192,6 @@ class Compress: case "unknown": out_file = self.unknown(source, output) - self.__utils.out_rename(out_file, source.name) + self.__utils.out_rename(out_file, source) self.__printer.bar.update() self.__printer.bar.next() diff --git a/vnrecode/utils.py b/vnrecode/utils.py index 7da06d3..c30abe3 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -17,7 +17,7 @@ class Utils: self.__errors = 0 self.__params = params_inst self.__printer = printer_inst - self.__duplicates = [] + self.__duplicates = {} @staticmethod def sys_pause(): @@ -102,9 +102,13 @@ class Utils: :return: Duplicate path name with (vncopy) on end """ if path.is_file() and path.exists(): - new_path = Path(path.stem + "(vncopy)" + path.suffix) - self.__duplicates.append(new_path) - return new_path + orig_name = path.name.replace("(vncopy)", "") + new_path = Path(path.parent, path.stem + "(vncopy)" + path.suffix) + try: self.__duplicates[orig_name] + except KeyError: self.__duplicates[orig_name] = [] + if not new_path.name in self.__duplicates[orig_name]: + self.__duplicates[orig_name].append(new_path.name) + return self.catch_duplicates(new_path) return path def print_duplicates(self): @@ -112,13 +116,13 @@ class Utils: Method prints message about all duplicates generated during recode process :return: None """ - for filename in self.__duplicates: + for filename in self.__duplicates.keys(): self.__printer.warning( - f'Duplicate file has been found! Check manually this files - "{filename.name}", ' - f'"{filename.stem + "(vncopy)" + filename.suffix}"' + f'Duplicate file has been found! Check manually this files - "{filename}", ' + + ', '.join(self.__duplicates[filename]) ) - def out_rename(self, out_path: Path, target: str): + def out_rename(self, out_path: Path, target: Path): """ Method removes md5 hash from file name and changes file extension in dependence of mimic mode :param out_path: Recoded file Path @@ -126,7 +130,7 @@ class Utils: :return: None """ if not self.__params.mimic_mode: - dest_name = self.catch_duplicates(Path(out_path.parent, target)) + dest_name = self.catch_duplicates(Path(out_path.parent, target.stem+out_path.suffix)) os.rename(out_path, dest_name) else: - os.rename(out_path, Path(out_path.parent, target)) + os.rename(out_path, Path(out_path.parent, target.name)) From a75314d2ad3878af69cd90d45decb1360353ab75 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 19 Oct 2024 02:44:51 +0300 Subject: [PATCH 092/105] vnrecode: ignore ansi escapes for string cleaning --- vnrecode/printer.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/vnrecode/printer.py b/vnrecode/printer.py index a6046f9..19650aa 100644 --- a/vnrecode/printer.py +++ b/vnrecode/printer.py @@ -3,6 +3,7 @@ from pathlib import Path import colorama import sys import os +import re class Printer: """ @@ -26,7 +27,8 @@ class Printer: :param string: String to "clean" :return: "Clean" string """ - return string + " " * (os.get_terminal_size().columns - len(string)) + ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') + return string + " " * (os.get_terminal_size().columns - len(ansi_escape.sub('', string))) @staticmethod def win_ascii_esc(): @@ -56,18 +58,18 @@ class Printer: def warning(self, string: str): """ - Method prints string with decor for warning messages - :param string: String to print - :return: None - """ + Method prints string with decor for warning messages + :param string: String to print + :return: None + """ self.bar_print(self.clean_str(f"\r\033[93m!\033[0m {string}\033[49m")) def error(self, string: str): """ - Method prints string with decor for error messages - :param string: String to print - :return: None - """ + Method prints string with decor for error messages + :param string: String to print + :return: None + """ self.bar_print(self.clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m")) def files(self, source_path: Path, output_path: Path, comment: str): From bc84703b73bf1d0e6f6def3368bbce59170ef8c9 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 19 Oct 2024 02:51:57 +0300 Subject: [PATCH 093/105] vnrecode: fix typo in input folder name --- vnrecode/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnrecode/application.py b/vnrecode/application.py index c43102b..2b19681 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -43,7 +43,7 @@ class Application: if not output.exists(): os.mkdir(output) - self.__printer.info(f'Compressing "{output}" folder...') + self.__printer.info(f'Compressing "{folder}" folder...') with ThreadPoolExecutor(max_workers=self.__params.workers) as executor: futures = [ From 626eaae5e2ee78504785c5f399bb6440602fb7bb Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 1 Nov 2024 03:23:50 +0300 Subject: [PATCH 094/105] vnrecode: fix renaming for same name subfolders --- vnrecode/application.py | 2 +- vnrecode/utils.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/vnrecode/application.py b/vnrecode/application.py index 2b19681..216157e 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -39,7 +39,7 @@ class Application: self.__printer.info("Creating folders...") for folder, folders, files in os.walk(source): - output = Path(folder.replace(str(source), str(self.__params.dest))) + output = self.__utils.get_comp_subdir(folder) if not output.exists(): os.mkdir(output) diff --git a/vnrecode/utils.py b/vnrecode/utils.py index c30abe3..af0dd8c 100644 --- a/vnrecode/utils.py +++ b/vnrecode/utils.py @@ -37,6 +37,14 @@ class Utils: """ return hashlib.md5(filename.encode()).hexdigest()[:8] + def get_comp_subdir(self, folder: str) -> Path: + """ + Method returns the Path from str, changing the source folder in it to a compressed one + :param folder: source subfolder + :return: Path object with compressed subfolder + """ + return Path(folder.replace(str(self.__params.source), str(self.__params.dest), 1)) + def get_recode_status(self): """ Method prints recoding results From d4092b46df529787b65bba31a3a8a7e329fa3d47 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 8 Nov 2024 04:33:05 +0300 Subject: [PATCH 095/105] vnrecode: replace funky method with ASCII symbol --- vnrecode/printer.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/vnrecode/printer.py b/vnrecode/printer.py index 19650aa..9207ba7 100644 --- a/vnrecode/printer.py +++ b/vnrecode/printer.py @@ -3,7 +3,6 @@ from pathlib import Path import colorama import sys import os -import re class Printer: """ @@ -20,16 +19,6 @@ class Printer: self.bar = IncrementalBar('Compressing', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)') self.bar.update() - @staticmethod - def clean_str(string: str) -> str: - """ - Method fills end of string with spaces to remove progress bar garbage from console - :param string: String to "clean" - :return: "Clean" string - """ - ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') - return string + " " * (os.get_terminal_size().columns - len(ansi_escape.sub('', string))) - @staticmethod def win_ascii_esc(): """ @@ -54,7 +43,7 @@ class Printer: :param string: String to print :return: None """ - self.bar_print(self.clean_str(f"\r\033[100m- {string}\033[49m")) + self.bar_print(f"\x1b[2K\r\033[100m- {string}\033[49m") def warning(self, string: str): """ @@ -62,7 +51,7 @@ class Printer: :param string: String to print :return: None """ - self.bar_print(self.clean_str(f"\r\033[93m!\033[0m {string}\033[49m")) + self.bar_print(f"\x1b[2K\r\033[93m!\033[0m {string}\033[49m") def error(self, string: str): """ @@ -70,7 +59,7 @@ class Printer: :param string: String to print :return: None """ - self.bar_print(self.clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m")) + self.bar_print(f"\x1b[2K\r\033[31m\u2715\033[0m {string}\033[49m") def files(self, source_path: Path, output_path: Path, comment: str): """ @@ -81,8 +70,8 @@ class Printer: :param comment: Comment about recode quality setting :return: None """ - self.bar_print(self.clean_str(f"\r\033[0;32m\u2713\033[0m \033[0;37m{source_path.stem}\033[0m{source_path.suffix}\033[0;37m -> " - f"{source_path.stem}\033[0m{output_path.suffix}\033[0;37m ({comment})\033[0m")) + self.bar_print(f"\x1b[2K\r\033[0;32m\u2713\033[0m \033[0;37m{source_path.stem}\033[0m{source_path.suffix}\033[0;37m -> " + f"{source_path.stem}\033[0m{output_path.suffix}\033[0;37m ({comment})\033[0m") def unknown_file(self, filename: str): """ @@ -90,4 +79,4 @@ class Printer: :param filename: Name of unknown file :return: """ - self.bar_print(self.clean_str(f"\r\u2713 \033[0;33m{filename}\033[0m (File will be force compressed via ffmpeg)")) + self.bar_print(f"\x1b[2K\r\u2713 \033[0;33m{filename}\033[0m (File will be force compressed via ffmpeg)") From 85f1c3776f15b83653927af560268d6248f1d0dc Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 8 Nov 2024 05:10:23 +0300 Subject: [PATCH 096/105] Fix compiled binaries launch --- build.bat | 6 +++--- build.sh | 6 +++--- unrenapk/__main__.py | 2 +- unrenapk/actions.py | 2 +- unrenapk/application.py | 4 ++-- vnrecode/__main__.py | 10 +++++----- vnrecode/application.py | 8 ++++---- vnrecode/compress.py | 6 +++--- 8 files changed, 22 insertions(+), 22 deletions(-) diff --git a/build.bat b/build.bat index 89343d1..821ed6c 100755 --- a/build.bat +++ b/build.bat @@ -5,12 +5,12 @@ mkdir output mkdir output\bin python -m pip install -r requirements.txt || goto :exit python -m pip install Nuitka || goto :exit -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=vnrecode vnrecode\__main__.py || goto :exit +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=vnrecode vnrecode || goto :exit xcopy vnrecode\vnrecode.toml output\bin /Y move /Y output\vnrecode.exe output\bin -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=unrenapk unrenapk\__main__.py || goto :exit +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=unrenapk unrenapk || goto :exit move /Y output\unrenapk.exe output\bin -python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=vnds2renpy vnds2renpy/__main__.py || goto :exit +python -m nuitka --jobs=%NUMBER_OF_PROCESSORS% --output-dir=output --follow-imports --onefile --output-filename=vnds2renpy vnds2renpy || goto :exit move /Y output\vnds2renpy.exe output\bin echo "Done! You can get binaries into output\bin directory" diff --git a/build.sh b/build.sh index a7cf3b3..ffbd2b4 100755 --- a/build.sh +++ b/build.sh @@ -14,11 +14,11 @@ case "$(uname -s)" in Linux*) jobs="--jobs=$(nproc)";; Darwin*) jobs="--jobs=$(sysctl -n hw.ncpu)";; esac -python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=vnrecode vnrecode/__main__.py +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=vnrecode vnrecode cp vnrecode/vnrecode.toml output/bin mv output/vnrecode output/bin -python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=unrenapk unrenapk/__main__.py +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=unrenapk unrenapk mv output/unrenapk output/bin -python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=vnds2renpy vnds2renpy/__main__.py +python3 -m nuitka "${jobs}" --output-dir=output --onefile --follow-imports --output-filename=vnds2renpy vnds2renpy mv output/vnds2renpy output/bin echo "Done! You can get binaries into output/bin directory" \ No newline at end of file diff --git a/unrenapk/__main__.py b/unrenapk/__main__.py index 6e79193..c33b4bc 100755 --- a/unrenapk/__main__.py +++ b/unrenapk/__main__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -from . import application +from unrenapk import application if __name__ == '__main__': application.launch() diff --git a/unrenapk/actions.py b/unrenapk/actions.py index 489eee9..a55f576 100755 --- a/unrenapk/actions.py +++ b/unrenapk/actions.py @@ -3,7 +3,7 @@ from PIL import Image import shutil import os -from .printer import Printer +from unrenapk.printer import Printer class Extract: diff --git a/unrenapk/application.py b/unrenapk/application.py index b66d6ba..1c3df6b 100644 --- a/unrenapk/application.py +++ b/unrenapk/application.py @@ -4,8 +4,8 @@ import argparse import sys import os -from .printer import Printer -from .actions import Actions +from unrenapk.printer import Printer +from unrenapk.actions import Actions def args_init(): diff --git a/vnrecode/__main__.py b/vnrecode/__main__.py index fe45306..e44ac79 100644 --- a/vnrecode/__main__.py +++ b/vnrecode/__main__.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -from .application import Application -from .compress import Compress -from .printer import Printer -from .params import Params -from .utils import Utils +from vnrecode.application import Application +from vnrecode.compress import Compress +from vnrecode.printer import Printer +from vnrecode.params import Params +from vnrecode.utils import Utils def init(): diff --git a/vnrecode/application.py b/vnrecode/application.py index 216157e..b8bbc57 100755 --- a/vnrecode/application.py +++ b/vnrecode/application.py @@ -5,10 +5,10 @@ from pathlib import Path import shutil import os -from .compress import Compress -from .printer import Printer -from .params import Params -from .utils import Utils +from vnrecode.compress import Compress +from vnrecode.printer import Printer +from vnrecode.params import Params +from vnrecode.utils import Utils class Application: diff --git a/vnrecode/compress.py b/vnrecode/compress.py index b1c2567..680da22 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -3,9 +3,9 @@ from pathlib import Path from PIL import Image import pillow_avif -from .printer import Printer -from .params import Params -from .utils import Utils +from vnrecode.printer import Printer +from vnrecode.params import Params +from vnrecode.utils import Utils class File: From c1f152d8d0c77bff19659c7e062cd189be22f690 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 28 Dec 2024 02:09:56 +0300 Subject: [PATCH 097/105] Remove strict dependencies versioning --- pyproject.toml | 12 ++++++------ requirements.txt | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f2c379f..a195225 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,10 +23,10 @@ name = "vntools" version = "2.0-dev" requires-python = ">= 3.11" dependencies = [ - "Pillow==10.3.0", - "pillow-avif-plugin==1.4.3", - "python-ffmpeg==2.0.12", - "progress==1.6", - "colorama==0.4.6", - "argparse~=1.4.0" + "Pillow>=10.3.0", + "pillow-avif-plugin>=1.4.3", + "python-ffmpeg>=2.0.12", + "progress>=1.6", + "colorama>=0.4.6", + "argparse>=1.4.0" ] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 16bd1e6..c6dd905 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -Pillow==10.3.0 -pillow-avif-plugin==1.4.3 -python-ffmpeg==2.0.12 -progress==1.6 -colorama==0.4.6 -argparse~=1.4.0 \ No newline at end of file +Pillow>=10.3.0 +pillow-avif-plugin>=1.4.3 +python-ffmpeg>=2.0.12 +progress>=1.6 +colorama>=0.4.6 +argparse>=1.4.0 From 9c367e5249b0dc5ccacea0701fe74b286f7afabd Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 28 Dec 2024 03:12:45 +0300 Subject: [PATCH 098/105] vnrecode: fix dest path name --- vnrecode/params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnrecode/params.py b/vnrecode/params.py index d0388f2..c102482 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -68,7 +68,7 @@ class Params: video_ext = config["VIDEO"]["Extension"] if args.config else args.v_ext video_codec = config["VIDEO"]["Codec"] if args.config else args.v_codec source = Path(args.source) - dest = Path(f"{args.source}_compressed") + dest = Path(source.name + f"_compressed") return cls( copy_unprocessed, force_compress, mimic_mode, hide_errors, webp_rgba, workers, From 3802c52ba054817728cb8cada6d5e3a4ab35f130 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 24 Jan 2025 02:30:55 +0300 Subject: [PATCH 099/105] vnrecode: check if source path exist (lol) --- vnrecode/params.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vnrecode/params.py b/vnrecode/params.py index c102482..552036c 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -68,6 +68,9 @@ class Params: video_ext = config["VIDEO"]["Extension"] if args.config else args.v_ext video_codec = config["VIDEO"]["Codec"] if args.config else args.v_codec source = Path(args.source) + if not source.exists(): + print("Requested path does not exists. Exiting!") + exit(255) dest = Path(source.name + f"_compressed") return cls( From ce62218cdaf4ff35784e25622290f0690ce18370 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 24 Jan 2025 02:31:26 +0300 Subject: [PATCH 100/105] vnrecode: change progress bar message --- vnrecode/printer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnrecode/printer.py b/vnrecode/printer.py index 9207ba7..18d5138 100644 --- a/vnrecode/printer.py +++ b/vnrecode/printer.py @@ -16,7 +16,7 @@ class Printer: file_count = 0 for folder, folders, file in os.walk(source): file_count += len(file) - self.bar = IncrementalBar('Compressing', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)') + self.bar = IncrementalBar('Recoding', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)') self.bar.update() @staticmethod From debc1755bbd094b4626a9cb568849106cf8d69c4 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 24 Jan 2025 03:22:27 +0300 Subject: [PATCH 101/105] vnrecode: add info about default values --- vnrecode/params.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/vnrecode/params.py b/vnrecode/params.py index 552036c..a20e9ce 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -96,17 +96,17 @@ class Params: parser.add_argument("-nm", "--no-mimic", dest='mimic', action='store_false', help="Disable mimic mode") parser.add_argument("-v", "--show_errors", action='store_false', help="Show recode errors") parser.add_argument("--webp-rgb", dest='webp_rgba', action='store_false', help="Recode .webp without alpha channel") - parser.add_argument("-j", "--jobs", type=int, help="Number of threads", default=16) - parser.add_argument("-ae", dest="a_ext", help="Audio extension", default="opus") - parser.add_argument("-ab", dest="a_bit", help="Audio bit rate", default="128k") - parser.add_argument("-id", dest="i_down", type=int, help="Image resolution downscale multiplier", default=1) - parser.add_argument("-ie", dest="i_ext", help="Image extension", default="avif") - parser.add_argument("-ife", dest="i_fallext", help="Image fallback extension", default="webp") + parser.add_argument("-j", "--jobs", type=int, help="Number of threads (default: 16)", default=16) + parser.add_argument("-ae", dest="a_ext", help="Audio extension (default: opus)", default="opus") + parser.add_argument("-ab", dest="a_bit", help="Audio bit rate (default: 128k)", default="128k") + parser.add_argument("-id", dest="i_down", type=int, help="Image resolution downscale multiplier (default: 1)", default=1) + parser.add_argument("-ie", dest="i_ext", help="Image extension (default: avif)", default="avif") + parser.add_argument("-ife", dest="i_fallext", help="Image fallback extension (default: webp)", default="webp") parser.add_argument("-il", dest='i_lossless', action='store_false', help="Image losing compression mode") - parser.add_argument("-iq", dest="i_quality", type=int, help="Image quality", default=100) - parser.add_argument("--v_crf", help="Video CRF number", type=int, default=27) + parser.add_argument("-iq", dest="i_quality", type=int, help="Image quality (default: 100)", default=100) + parser.add_argument("--v_crf", help="Video CRF number (default: 27)", type=int, default=27) parser.add_argument("-vs", dest="v_skip", action='store_true', help="Skip video recoding") - parser.add_argument("-ve", dest="v_ext", help="Video extension", default="webm") - parser.add_argument("-vc", dest="v_codec", help="Video codec name", default="libvpx-vp9") + parser.add_argument("-ve", dest="v_ext", help="Video extension (default: webm)", default="webm") + parser.add_argument("-vc", dest="v_codec", help="Video codec name (default: libvpx-vp9)", default="libvpx-vp9") args = parser.parse_args() return args \ No newline at end of file From e5bf961ddbf22df9a899eb8ed5110975a690e1f5 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Tue, 28 Jan 2025 21:17:57 +0300 Subject: [PATCH 102/105] vnrecode: fix dest path relativity --- vnrecode/params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnrecode/params.py b/vnrecode/params.py index a20e9ce..18f43a6 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -71,7 +71,7 @@ class Params: if not source.exists(): print("Requested path does not exists. Exiting!") exit(255) - dest = Path(source.name + f"_compressed") + dest = Path(source.parent, source.name + f"_compressed") return cls( copy_unprocessed, force_compress, mimic_mode, hide_errors, webp_rgba, workers, From 555ea0ebbb17cc227721086857944a34dcf3040e Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Sat, 1 Feb 2025 15:45:14 +0300 Subject: [PATCH 103/105] Initial Arch Linux packaging --- PKGBUILD | 31 +++++++++++++++++++++++++++++++ README.md | 4 +--- 2 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 PKGBUILD diff --git a/PKGBUILD b/PKGBUILD new file mode 100644 index 0000000..0c6ef83 --- /dev/null +++ b/PKGBUILD @@ -0,0 +1,31 @@ +# Maintainer: D. Can Celasun +# Contributor: Ezekiel Bethel + +_pkgname=VNTools +pkgname=vntools-git +pkgver=2.0.e5bf961 +pkgrel=1 +pkgdesc="Collection of tools used by VienDesu! Porting Team" +arch=("any") +url="https://github.com/VienDesuPorting/VNTools" +depends=("python" "python-pillow" "python-pillow-avif-plugin" "python-python-ffmpeg" "python-progress" "python-colorama") +makedepends=("python-setuptools" "git") +provides=("vntools") +source=("git+${url}.git#branch=testing") +sha256sums=("SKIP") + +pkgver() { + cd "${srcdir}/${_pkgname}" + printf "2.0.%s" "$(git rev-parse --short HEAD)" +} + +build() { + cd "${srcdir}/${_pkgname}" + python -m build --wheel --no-isolation +} + +package() { + cd "${srcdir}/${_pkgname}" + python -m installer --destdir="${pkgdir}" dist/*.whl +} + diff --git a/README.md b/README.md index b3b0091..eb026d5 100644 --- a/README.md +++ b/README.md @@ -16,10 +16,8 @@ Collection of tools used by VienDesu! Porting Team #### Build tools as binaries: * Run `./build.sh` on UNIX * Run `.\build.bat` for Windows - * Arch Linux - `TODO` - * NixOS - `TODO` #### Install as python package: * Run `pip install -U .` command in project folder - * Arch Linux - `TODO` + * Arch Linux - `paru -Bi .` * NixOS - `TODO` From 00925a4908ca346faafe37f73f1484028c40c617 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 7 Feb 2025 02:19:03 +0300 Subject: [PATCH 104/105] vnrecode: float image downscale multiplier --- vnrecode/params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnrecode/params.py b/vnrecode/params.py index 18f43a6..61fd8de 100644 --- a/vnrecode/params.py +++ b/vnrecode/params.py @@ -99,7 +99,7 @@ class Params: parser.add_argument("-j", "--jobs", type=int, help="Number of threads (default: 16)", default=16) parser.add_argument("-ae", dest="a_ext", help="Audio extension (default: opus)", default="opus") parser.add_argument("-ab", dest="a_bit", help="Audio bit rate (default: 128k)", default="128k") - parser.add_argument("-id", dest="i_down", type=int, help="Image resolution downscale multiplier (default: 1)", default=1) + parser.add_argument("-id", dest="i_down", type=float, help="Image resolution downscale multiplier (default: 1)", default=1) parser.add_argument("-ie", dest="i_ext", help="Image extension (default: avif)", default="avif") parser.add_argument("-ife", dest="i_fallext", help="Image fallback extension (default: webp)", default="webp") parser.add_argument("-il", dest='i_lossless', action='store_false', help="Image losing compression mode") From 3f3de59844f231743c8119c214bfaefe39113bd1 Mon Sep 17 00:00:00 2001 From: OleSTEEP Date: Fri, 7 Feb 2025 02:24:34 +0300 Subject: [PATCH 105/105] vnrecode: make temp files hidden --- vnrecode/compress.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vnrecode/compress.py b/vnrecode/compress.py index 680da22..bbff31d 100644 --- a/vnrecode/compress.py +++ b/vnrecode/compress.py @@ -70,7 +70,7 @@ class Compress: """ bit_rate = self.__params.audio_bitrate prefix = self.__utils.get_hash(input_path.name) - out_file = Path(output_dir, f'{prefix}_{input_path.stem}.{extension}') + out_file = Path(output_dir, f'.{prefix}_{input_path.stem}.{extension}') try: (FFmpeg() .input(input_path) @@ -93,7 +93,7 @@ class Compress: """ quality = self.__params.image_quality prefix = self.__utils.get_hash(input_path.name) - out_file = Path(output_dir, f"{prefix}_{input_path.stem}.{extension}") + out_file = Path(output_dir, f".{prefix}_{input_path.stem}.{extension}") try: image = Image.open(input_path) @@ -101,7 +101,7 @@ class Compress: (extension == "webp" and not self.__params.webp_rgba)): if File.has_transparency(image): self.__printer.warning(f"{input_path.name} has transparency. Changing to fallback...") - out_file = Path(output_dir, f"{prefix}_{input_path.stem}.{self.__params.image_fall_ext}") + out_file = Path(output_dir, f".{prefix}_{input_path.stem}.{self.__params.image_fall_ext}") if File.has_transparency(image): image.convert('RGBA') @@ -131,7 +131,7 @@ class Compress: :return: Path of compressed video file with md5 hash as prefix """ prefix = self.__utils.get_hash(input_path.name) - out_file = Path(output_dir, f'{prefix}_{input_path.stem}.{extension}') + out_file = Path(output_dir, f'.{prefix}_{input_path.stem}.{extension}') if not self.__params.video_skip: codec = self.__params.video_codec crf = self.__params.video_crf @@ -160,7 +160,7 @@ class Compress: :return: Path of compressed file with md5 hash as prefix """ prefix = self.__utils.get_hash(input_path.name) - out_file = Path(output_dir, f"{prefix}_{input_path.name}") + out_file = Path(output_dir, f".{prefix}_{input_path.name}") if self.__params.force_compress: self.__printer.unknown_file(input_path.name) try: