Basic refactor for packaging

This commit is contained in:
OleSTEEP 2024-08-29 01:32:36 +03:00
parent 0a9114ff64
commit 7487eb94bd
25 changed files with 573 additions and 485 deletions

View file

View file

@ -0,0 +1,19 @@
#!/usr/bin/env python3
from application import Application
from compress import Compress
from printer import Printer
from config import Config
from utils import Utils
def init():
config = Config.setup_config()
printer = Printer(config.args.source)
utils = Utils(config.config, printer)
compress = Compress(config.config, printer, utils)
Application(config, compress, printer, utils).run()
if __name__ == "__main__":
init()

View file

@ -0,0 +1,48 @@
#!/usr/bin/env python3
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime
import shutil
import os
class Application:
def __init__(self, config, compress, printer, utils):
self.config = config.config
self.args = config.args
self.compress = compress.compress
self.printer = printer
self.utils = utils
def compress_worker(self, folder, file, source, output):
if os.path.isfile(f'{folder}/{file}'):
self.compress(folder, file, source, output)
def run(self):
start_time = datetime.now()
self.printer.win_ascii_esc()
source = os.path.abspath(self.args.source)
if os.path.exists(f"{source}_compressed"):
shutil.rmtree(f"{source}_compressed")
self.printer.info("Creating folders...")
for folder, folders, files in os.walk(source):
if not os.path.exists(folder.replace(source, f"{source}_compressed")):
os.mkdir(folder.replace(source, f"{source}_compressed"))
self.printer.info(f'Compressing "{folder.replace(source, os.path.split(source)[-1])}" folder...')
output = folder.replace(source, f"{source}_compressed")
with ThreadPoolExecutor(max_workers=self.config["FFMPEG"]["Workers"]) as executor:
futures = [
executor.submit(self.compress, folder, file, source, output)
for file in files if os.path.isfile(f'{folder}/{file}')
]
for future in as_completed(futures):
future.result()
self.utils.get_compression_status(source)
self.utils.sys_pause()
print(f"Time taken: {datetime.now() - start_time}")

View file

@ -0,0 +1,163 @@
from ffmpeg import FFmpeg, FFmpegError
from PIL import Image
import pillow_avif
import os
class File:
@staticmethod
def get_type(filename):
audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma']
image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png']
video_ext = ['.3gp' '.amv', '.avi', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv',
'.webm', '.ogv']
if os.path.splitext(filename)[1] in audio_ext:
return "audio"
elif os.path.splitext(filename)[1] in image_ext:
return "image"
elif os.path.splitext(filename)[1] in video_ext:
return "video"
else:
return "unknown"
@staticmethod
def has_transparency(img: Image) -> bool:
if img.info.get("transparency", None) is not None:
return True
if img.mode == "P":
transparent = img.info.get("transparency", -1)
for _, index in img.getcolors():
if index == transparent:
return True
elif img.mode == "RGBA":
extrema = img.getextrema()
if extrema[3][0] < 255:
return True
return False
class Compress:
def __init__(self, config, printer, utils):
self.config = config
self.printer = printer
self.utils = utils
def audio(self, folder, file, target_folder, extension):
bitrate = self.config['AUDIO']['BitRate']
try:
(FFmpeg()
.input(f'{folder}/{file}')
.option("hide_banner")
.output(self.utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'),
{"b:a": bitrate, "loglevel": "error"})
.execute()
)
except FFmpegError as e:
self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
self.utils.errors += 1
if not self.config['FFMPEG']['HideErrors']:
self.printer.error(f"File {file} can't be processed! Error: {e}")
self.printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}")
return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'
def video(self, folder, file, target_folder, extension):
if not self.config['VIDEO']['SkipVideo']:
codec = self.config['VIDEO']['Codec']
crf = self.config['VIDEO']['CRF']
try:
(FFmpeg()
.input(f'{folder}/{file}')
.option("hide_banner")
.option("hwaccel", "auto")
.output(self.utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'),
{"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf)
.execute()
)
self.printer.files(file, os.path.splitext(file)[0], extension, codec)
except FFmpegError as e:
self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
self.utils.errors += 1
if not self.config['FFMPEG']['HideErrors']:
self.printer.error(f"File {file} can't be processed! Error: {e}")
else:
self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'
def image(self, folder, file, target_folder, extension):
quality = self.config['IMAGE']['Quality']
try:
image = Image.open(f'{folder}/{file}')
if (extension == "jpg" or extension == "jpeg" or
(extension == "webp" and not self.config['FFMPEG']['WebpRGBA'])):
if File.has_transparency(image):
self.printer.warning(f"{file} has transparency. Changing to fallback...")
extension = self.config['IMAGE']['FallBackExtension']
if File.has_transparency(image):
image.convert('RGBA')
res_downscale = self.config['IMAGE']['ResDownScale']
if res_downscale != 1:
width, height = image.size
new_size = (int(width / res_downscale), int(height / res_downscale))
image = image.resize(new_size)
image.save(self.utils.check_duplicates(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}"),
optimize=True,
lossless=self.config['IMAGE']['Lossless'],
quality=quality,
minimize_size=True)
self.printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%")
except Exception as e:
self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
self.utils.errors += 1
if not self.config['FFMPEG']['HideErrors']:
self.printer.error(f"File {file} can't be processed! Error: {e}")
return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'
def unknown(self, folder, file, target_folder):
if self.config["FFMPEG"]["ForceCompress"]:
self.printer.unknown_file(file)
try:
(FFmpeg()
.input(f'{folder}/{file}')
.output(self.utils.check_duplicates(f'{target_folder}/{file}'))
.execute()
)
except FFmpegError as e:
self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
self.utils.errors += 1
if not self.config['FFMPEG']['HideErrors']:
self.printer.error(f"File {file} can't be processed! Error: {e}")
else:
self.utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
return f'{target_folder}/{file}'
def compress(self, _dir, filename, source, output):
match File.get_type(filename):
case "audio":
out_file = self.audio(_dir, filename, output, self.config['AUDIO']['Extension'])
case "image":
out_file = self.image(_dir, filename, output, self.config['IMAGE']['Extension'])
case "video":
out_file = self.video(_dir, filename, output, self.config['VIDEO']['Extension'])
case "unknown":
out_file = self.unknown(_dir, filename, output)
if self.config['FFMPEG']['MimicMode']:
try:
os.rename(out_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed"))
except FileNotFoundError:
self.printer.warning(f"File {out_file} failed to copy to out dir")
self.printer.bar.update()
self.printer.bar.next()

View file

@ -0,0 +1,27 @@
import os.path
from argparse import Namespace, ArgumentParser
from dataclasses import dataclass
from typing import Any
import tomllib
@dataclass
class Config:
config: dict[str, Any]
args: Namespace
@classmethod
def setup_config(cls):
parser = ArgumentParser(prog="ffmpeg-comp",
description="Python utility to compress Visual Novel Resources"
)
parser.add_argument("source")
parser.add_argument("-c", "--config", default="ffmpeg-comp.toml")
args = parser.parse_args()
if os.path.isfile(args.config):
with open(args.config, "rb") as cfile:
config = tomllib.load(cfile)
else:
print("Failed to find config. Check `ffmpeg-comp -h` to more info")
return cls(config=config, args=args)

View file

@ -1,59 +0,0 @@
#!/usr/bin/env python3
from concurrent.futures import ThreadPoolExecutor, as_completed
from modules.configloader import config
from modules import compressor
from modules import printer
from modules import utils
from datetime import datetime
import shutil
import sys
import os
def get_args():
try:
if sys.argv[1][len(sys.argv[1])-1] == "/":
path = sys.argv[1][:len(sys.argv[1])-1]
else:
path = sys.argv[1]
return path
except IndexError:
print(utils.help_message())
exit()
def compress_worker(folder, file, target_folder, req_folder):
if os.path.isfile(f'{folder}/{file}'):
compressor.compress_file(folder, file, target_folder, req_folder)
if __name__ == "__main__":
start_time = datetime.now()
printer.win_ascii_esc()
req_folder = os.path.abspath(get_args())
printer.bar_init(req_folder)
if os.path.exists(f"{req_folder}_compressed"):
shutil.rmtree(f"{req_folder}_compressed")
printer.info("Creating folders...")
for folder, folders, files in os.walk(req_folder):
if not os.path.exists(folder.replace(req_folder, f"{req_folder}_compressed")):
os.mkdir(folder.replace(req_folder, f"{req_folder}_compressed"))
printer.info(f"Compressing \"{folder.replace(req_folder, req_folder.split('/').pop())}\" folder...")
target_folder = folder.replace(req_folder, f"{req_folder}_compressed")
with ThreadPoolExecutor(max_workers=config["FFMPEG"]["Workers"]) as executor:
futures = [
executor.submit(compress_worker, folder, file, target_folder, req_folder)
for file in files
]
for future in as_completed(futures):
future.result()
utils.get_compression_status(req_folder)
utils.sys_pause()
print(f"Time taken: {datetime.now() - start_time}")

View file

@ -1,160 +0,0 @@
from modules import configloader
from modules import printer
from modules import utils
from PIL import Image
import pillow_avif
from ffmpeg import FFmpeg, FFmpegError
import os
def get_file_type(filename):
audio_ext = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.opus', '.raw', '.wav', '.wma']
image_ext = ['.apng', '.avif', '.bmp', '.tga', '.tiff', '.dds', '.svg', '.webp', '.jpg', '.jpeg', '.png']
video_ext = ['.3gp' '.amv', '.avi', '.m2t', '.m4v', '.mkv', '.mov', '.mp4', '.m4v', '.mpeg', '.mpv',
'.webm', '.ogv']
if os.path.splitext(filename)[1] in audio_ext:
return "audio"
elif os.path.splitext(filename)[1] in image_ext:
return "image"
elif os.path.splitext(filename)[1] in video_ext:
return "video"
else:
return "unknown"
def has_transparency(img):
if img.info.get("transparency", None) is not None:
return True
if img.mode == "P":
transparent = img.info.get("transparency", -1)
for _, index in img.getcolors():
if index == transparent:
return True
elif img.mode == "RGBA":
extrema = img.getextrema()
if extrema[3][0] < 255:
return True
return False
def compress_audio(folder, file, target_folder, extension):
bitrate = configloader.config['AUDIO']['BitRate']
try:
(FFmpeg()
.input(f'{folder}/{file}')
.option("hide_banner")
.output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'),
{"b:a": bitrate, "loglevel": "error"})
.execute()
)
except FFmpegError as e:
utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
utils.errors_count += 1
if not configloader.config['FFMPEG']['HideErrors']:
printer.error(f"File {file} can't be processed! Error: {e}")
printer.files(file, os.path.splitext(file)[0], extension, f"{bitrate}")
return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'
def compress_video(folder, file, target_folder, extension):
if not configloader.config['VIDEO']['SkipVideo']:
codec = configloader.config['VIDEO']['Codec']
crf = configloader.config['VIDEO']['CRF']
try:
(FFmpeg()
.input(f'{folder}/{file}')
.option("hide_banner")
.option("hwaccel", "auto")
.output(utils.check_duplicates(f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'),
{"codec:v": codec, "v:b": 0, "loglevel": "error"}, crf=crf)
.execute()
)
printer.files(file, os.path.splitext(file)[0], extension, codec)
except FFmpegError as e:
utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
utils.errors_count += 1
if not configloader.config['FFMPEG']['HideErrors']:
printer.error(f"File {file} can't be processed! Error: {e}")
else:
utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'
def compress_image(folder, file, target_folder, extension):
quality = configloader.config['IMAGE']['Quality']
try:
image = Image.open(f'{folder}/{file}')
if (extension == "jpg" or extension == "jpeg" or
(extension == "webp" and not configloader.config['FFMPEG']['WebpRGBA'])):
if has_transparency(image):
printer.warning(f"{file} has transparency. Changing to fallback...")
extension = configloader.config['IMAGE']['FallBackExtension']
if has_transparency(image):
image.convert('RGBA')
res_downscale = configloader.config['IMAGE']['ResDownScale']
if res_downscale != 1:
width, height = image.size
new_size = (int(width / res_downscale), int(height / res_downscale))
image = image.resize(new_size)
image.save(utils.check_duplicates(f"{target_folder}/{os.path.splitext(file)[0]}.{extension}"),
optimize=True,
lossless=configloader.config['IMAGE']['Lossless'],
quality=quality,
minimize_size=True)
printer.files(file, os.path.splitext(file)[0], extension, f"{quality}%")
except Exception as e:
utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
utils.errors_count += 1
if not configloader.config['FFMPEG']['HideErrors']:
printer.error(f"File {file} can't be processed! Error: {e}")
return f'{target_folder}/{os.path.splitext(file)[0]}.{extension}'
def compress(folder, file, target_folder):
if configloader.config["FFMPEG"]["ForceCompress"]:
printer.unknown_file(file)
try:
(FFmpeg()
.input(f'{folder}/{file}')
.output(f'{target_folder}/{file}')
.execute()
)
except FFmpegError as e:
utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
utils.errors_count += 1
if not configloader.config['FFMPEG']['HideErrors']:
printer.error(f"File {file} can't be processed! Error: {e}")
else:
utils.add_unprocessed_file(f'{folder}/{file}', f'{target_folder}/{file}')
return f'{target_folder}/{file}'
def compress_file(_dir, filename, target_dir, source):
match get_file_type(filename):
case "audio":
comp_file = compress_audio(_dir, filename, target_dir,
configloader.config['AUDIO']['Extension'])
case "image":
comp_file = compress_image(_dir, filename, target_dir,
configloader.config['IMAGE']['Extension'])
case "video":
comp_file = compress_video(_dir, filename, target_dir,
configloader.config['VIDEO']['Extension'])
case "unknown":
comp_file = compress(_dir, filename, target_dir)
if configloader.config['FFMPEG']['MimicMode']:
try:
os.rename(comp_file, f'{_dir}/{filename}'.replace(source, f"{source}_compressed"))
except FileNotFoundError:
pass
printer.bar.update()
printer.bar.next()

View file

@ -1,11 +0,0 @@
import tomllib
from modules import printer
try:
config = tomllib.load(open("ffmpeg-comp.toml", "rb"))
except FileNotFoundError:
try:
config = tomllib.load(open("/etc/ffmpeg-comp.toml", "rb"))
except FileNotFoundError:
printer.error("Config file not found. Please put it next to binary or in to /etc folder.")
exit()

View file

@ -1,51 +0,0 @@
from progress.bar import IncrementalBar
import colorama
import sys
import os
def bar_init(folder):
file_count = 0
for folder, folders, file in os.walk(folder):
file_count += len(file)
global bar
bar = IncrementalBar('Compressing', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)')
bar.update()
def bar_print(string):
print(string)
bar.update()
# Fill whole string with spaces for cleaning progress bar
def clean_str(string):
return string + " " * (os.get_terminal_size().columns - len(string))
def info(string):
bar_print(clean_str(f"\r\033[100m- {string}\033[49m"))
def warning(string):
bar_print(clean_str(f"\r\033[93m!\033[0m {string}\033[49m"))
def error(string):
bar_print(clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m"))
def files(source, dest, dest_ext, comment):
source_ext = os.path.splitext(source)[1]
source_name = os.path.splitext(source)[0]
bar_print(clean_str(f"\r\033[0;32m\u2713\033[0m \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m"))
def unknown_file(file):
bar_print(clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)"))
def win_ascii_esc():
if sys.platform == "win32":
colorama.init()

View file

@ -1,79 +0,0 @@
from modules import configloader
from modules import printer
from shutil import copyfile
import sys
import os
errors_count = 0
def get_dir_size(directory, files):
total_size = 0
for folder, folders, files in os.walk(directory):
for file in files:
if not os.path.islink(f"{folder}/{file}"):
total_size += os.path.getsize(f"{folder}/{file}")
return total_size
def get_compression(orig, comp):
processed_files = []
for folder, folders, files in os.walk(comp):
for file in files:
processed_files.append(file)
try:
orig = get_dir_size(orig, processed_files)
comp = get_dir_size(comp, processed_files)
print(f"\nResult: {orig/1024/1024:.2f}MB -> {comp/1024/1024:.2f}MB ({(comp - orig)/1024/1024:.2f}MB)")
except ZeroDivisionError:
printer.warning("Nothing compressed!")
def get_compression_status(orig_folder):
orig_folder_len = 0
comp_folder_len = 0
for folder, folders, files in os.walk(orig_folder):
orig_folder_len += len(files)
for folder, folders, files in os.walk(f'{orig_folder}_compressed'):
for file in files:
if not os.path.splitext(file)[1].count(" (copy)"):
comp_folder_len += 1
if errors_count != 0:
printer.warning("Some files failed to compress!")
if orig_folder_len == comp_folder_len:
printer.info("Success!")
get_compression(orig_folder, f"{orig_folder}_compressed")
else:
printer.warning("Original and compressed folders are not identical!")
get_compression(orig_folder, f"{orig_folder}_compressed")
def add_unprocessed_file(orig_folder, new_folder):
if configloader.config['FFMPEG']['CopyUnprocessed']:
filename = orig_folder.split("/").pop()
copyfile(orig_folder, new_folder)
printer.info(f"File {filename} copied to compressed folder.")
def check_duplicates(new_folder):
filename = new_folder.split().pop()
if os.path.exists(new_folder):
printer.warning(
f'Duplicate file has been found! Check manually this files - "{filename}", '
f'"{os.path.splitext(filename)[0] + "(copy)" + os.path.splitext(filename)[1]}"')
return os.path.splitext(new_folder)[0] + "(copy)" + os.path.splitext(new_folder)[1]
return new_folder
def sys_pause():
if sys.platform == "win32":
os.system("pause")
def help_message():
return "Usage: ffmpeg-comp {folder}"

View file

@ -0,0 +1,46 @@
from progress.bar import IncrementalBar
import colorama
import sys
import os
class Printer:
def __init__(self, folder):
file_count = 0
for folder, folders, file in os.walk(folder):
file_count += len(file)
self.bar = IncrementalBar('Compressing', max=file_count, suffix='[%(index)d/%(max)d] (%(percent).1f%%)')
self.bar.update()
# Fill whole string with spaces for cleaning progress bar
@staticmethod
def clean_str(string):
return string + " " * (os.get_terminal_size().columns - len(string))
@staticmethod
def win_ascii_esc():
if sys.platform == "win32":
colorama.init()
def bar_print(self, string):
print(string)
self.bar.update()
def info(self, string):
self.bar_print(self.clean_str(f"\r\033[100m- {string}\033[49m"))
def warning(self, string):
self.bar_print(self.clean_str(f"\r\033[93m!\033[0m {string}\033[49m"))
def error(self, string):
self.bar_print(self.clean_str(f"\r\033[31m\u2715\033[0m {string}\033[49m"))
def files(self, source, dest, dest_ext, comment):
source_ext = os.path.splitext(source)[1]
source_name = os.path.splitext(source)[0]
self.bar_print(self.clean_str(f"\r\033[0;32m\u2713\033[0m \033[0;37m{source_name}\033[0m{source_ext}\033[0;37m -> {dest}\033[0m.{dest_ext}\033[0;37m ({comment})\033[0m"))
def unknown_file(self, file):
self.bar_print(self.clean_str(f"\r* \033[0;33m{file}\033[0m (File will be force compressed via ffmpeg)"))

View file

@ -1,5 +0,0 @@
Pillow==10.3.0
pillow-avif-plugin==1.4.3
python-ffmpeg==2.0.12
progress==1.6
colorama==0.4.6

View file

@ -0,0 +1,71 @@
from shutil import copyfile
import sys
import os
class Utils:
def __init__(self, config, printer):
self.errors = 0
self.config = config
self.printer = printer
@staticmethod
def sys_pause():
if sys.platform == "win32":
os.system("pause")
@staticmethod
def get_size(directory):
total_size = 0
for folder, folders, files in os.walk(directory):
for file in files:
if not os.path.islink(f"{folder}/{file}"):
total_size += os.path.getsize(f"{folder}/{file}")
return total_size
def get_compression(self, source, output):
try:
source = self.get_size(source)
output = self.get_size(output)
print(f"\nResult: {source/1024/1024:.2f}MB -> "
f"{output/1024/1024:.2f}MB ({(output - source)/1024/1024:.2f}MB)")
except ZeroDivisionError:
self.printer.warning("Nothing compressed!")
def get_compression_status(self, orig_folder):
orig_folder_len = 0
comp_folder_len = 0
for folder, folders, files in os.walk(orig_folder):
orig_folder_len += len(files)
for folder, folders, files in os.walk(f'{orig_folder}_compressed'):
for file in files:
if not os.path.splitext(file)[1].count("(copy)"):
comp_folder_len += 1
if self.errors != 0:
self.printer.warning("Some files failed to compress!")
if orig_folder_len == comp_folder_len:
self.printer.info("Success!")
self.get_compression(orig_folder, f"{orig_folder}_compressed")
else:
self.printer.warning("Original and compressed folders are not identical!")
self.get_compression(orig_folder, f"{orig_folder}_compressed")
def add_unprocessed_file(self, orig_folder, new_folder):
if self.config['FFMPEG']['CopyUnprocessed']:
filename = orig_folder.split("/").pop()
copyfile(orig_folder, new_folder)
self.printer.info(f"File {filename} copied to compressed folder.")
def check_duplicates(self, new_folder):
filename = new_folder.split().pop()
if os.path.exists(new_folder):
self.printer.warning(
f'Duplicate file has been found! Check manually this files - "{filename}", '
f'"{os.path.splitext(filename)[0] + "(copy)" + os.path.splitext(filename)[1]}"')
return os.path.splitext(new_folder)[0] + "(copy)" + os.path.splitext(new_folder)[1]
return new_folder