bump to 8.0.0

* Improved color detection

* use pure python

---------

Co-authored-by: Alex Xu <alexkurosakimh3@gmail.com>

.github/workflows/package-linux.yml

@@ -34,7 +34,7 @@ jobs:
     - name: Install python dependencies
       run: |
         sudo apt-get update 
-        sudo apt-get install -y libpng-dev libjpeg-dev p7zip-full p7zip-rar python3-pip squashfs-tools libfuse2 libxcb-cursor0
+        sudo apt-get install -y libpng-dev libjpeg-dev 7zip python3-pip squashfs-tools libfuse2 libxcb-cursor0
         python -m pip install --upgrade pip setuptools wheel certifi pyinstaller --no-binary pyinstaller 
         python -m pip install -r requirements.txt
     - name: build binary

Dockerfile-base

@@ -8,7 +8,7 @@ RUN echo "I'm building for $TARGETOS/$TARGETARCH/$TARGETVARIANT"
 COPY requirements.txt /opt/kcc/
 ENV PATH="/opt/venv/bin:$PATH"
 RUN DEBIAN_FRONTEND=noninteractive apt-get update -y && apt-get -yq upgrade && \
-    apt-get install -y libpng-dev libjpeg-dev p7zip-full unrar-free libgl1 && \
+    apt-get install -y libpng-dev libjpeg-dev 7zip unrar-free libgl1 && \
     python -m pip install --upgrade pip && \
     python -m venv /opt/venv && \
     python -m pip install -r /opt/kcc/requirements.txt
@@ -55,7 +55,7 @@ RUN set -x && \
     KEPT_PACKAGES+=(locales-all) && \
     KEPT_PACKAGES+=(libfreetype6) && \
     KEPT_PACKAGES+=(libfontconfig1) && \
-    KEPT_PACKAGES+=(p7zip-full) && \
+    KEPT_PACKAGES+=(7zip) && \
     KEPT_PACKAGES+=(python3) && \
     KEPT_PACKAGES+=(python3-pip) && \
     KEPT_PACKAGES+=(unrar-free) && \
@@ -113,7 +113,7 @@ RUN set -x && \
     KEPT_PACKAGES+=(locales-all) && \
     KEPT_PACKAGES+=(libfreetype6) && \
     KEPT_PACKAGES+=(libfontconfig1) && \
-    KEPT_PACKAGES+=(p7zip-full) && \
+    KEPT_PACKAGES+=(7zip) && \
     KEPT_PACKAGES+=(python3) && \
     KEPT_PACKAGES+=(python3-pip) && \
     KEPT_PACKAGES+=(unrar-free) && \
@@ -158,7 +158,7 @@ LABEL org.opencontainers.image.title="Kindle Comic Converter"
 ENV PATH="/opt/venv/bin:$PATH"
 WORKDIR /app
 RUN DEBIAN_FRONTEND=noninteractive apt-get update -y && apt-get -yq upgrade && \
-    apt-get install -y p7zip-full unrar-free  && \
+    apt-get install -y 7zip unrar-free  && \
     ln -s /app/kindlegen /bin/kindlegen && \
     echo docker-base-20241116 > /IMAGE_VERSION
 

README.md

@@ -27,7 +27,7 @@ on underpowered ereaders with small storage capacities.
 KCC avoids many common formatting issues (some of which occur [even on the Kindle Store](https://github.com/ciromattia/kcc/wiki/Kindle-Store-bad-formatting)), such as:
 1) faded black levels causing unneccessarily low contrast, which is hard to see and can cause eyestrain.
 2) unneccessary margins at the bottom of the screen
-3) Not utilizing the full 1860x2480 resolution of the 10" Kindle Scribe (feature in progress)
+3) Not utilizing the full 1860x2480 resolution of the 10" Kindle Scribe
 4) incorrect page turn direction for manga that's read right to left
 5) unaligned two page spreads in landscape, where pages are shifted over by 1
 
@@ -155,7 +155,7 @@ Please check [our wiki](https://github.com/ciromattia/kcc/wiki/) for more detail
 CLI version of **KCC** is intended for power users. It allows using options that might not be compatible and decrease the quality of output.
 CLI version has reduced dependencies, on Debian based distributions this commands should install all needed dependencies:
 ```
-sudo apt-get install python3 p7zip-full python3-pil python3-psutil python3-slugify
+sudo apt-get install python3 7zip python3-pil python3-psutil python3-slugify
 ```
 
 ### Profiles:

kindlecomicconverter/KCC_gui.py

@@ -40,10 +40,10 @@ from packaging.version import Version
 from raven import Client
 from tempfile import gettempdir
 
-from .shared import HTMLStripper, available_archive_tools, sanitizeTrace, walkLevel, subprocess_run
+from .shared import HTMLStripper, sanitizeTrace, walkLevel, subprocess_run
+from .comicarchive import SEVENZIP, available_archive_tools
 from . import __version__
 from . import comic2ebook
-from . import image
 from . import metadata
 from . import kindle
 from . import KCC_ui
@@ -1166,7 +1166,7 @@ class KCCGUI(KCC_ui.Ui_mainWindow):
                             'info')
         
         self.tar = 'tar' in available_archive_tools()
-        self.sevenzip = '7z' in available_archive_tools()
+        self.sevenzip = SEVENZIP in available_archive_tools()
         if not any([self.tar, self.sevenzip]):
             self.addMessage('<a href="https://github.com/ciromattia/kcc#7-zip">Install 7z (link)</a>'
                             ' to enable CBZ/CBR/ZIP/etc processing.', 'warning')

kindlecomicconverter/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '7.6.0'
+__version__ = '8.0.0'
 __license__ = 'ISC'
 __copyright__ = '2012-2022, Ciro Mattia Gonano <ciromattia@gmail.com>, Pawel Jastrzebski <pawelj@iosphe.re>, darodi'
 __docformat__ = 'restructuredtext en'

kindlecomicconverter/comic2ebook.py

@@ -42,7 +42,8 @@ from subprocess import STDOUT, PIPE, CalledProcessError
 from psutil import virtual_memory, disk_usage
 from html import escape as hescape
 
-from .shared import available_archive_tools, getImageFileName, walkSort, walkLevel, sanitizeTrace, subprocess_run
+from .shared import getImageFileName, walkSort, walkLevel, sanitizeTrace, subprocess_run
+from .comicarchive import SEVENZIP, available_archive_tools
 from . import comic2panel
 from . import image
 from . import comicarchive
@@ -647,7 +648,7 @@ def imgFileProcessing(work):
             img.autocontrastImage()
             img.resizeImage()
             img.optimizeForDisplay(opt.reducerainbow)
-            if opt.forcecolor and workImg.color:
+            if opt.forcecolor and img.color:
                 pass
             elif opt.forcepng:
                 img.quantizeImage()
@@ -822,6 +823,19 @@ def getPanelViewSize(deviceres, size):
     return str(int(x)), str(int(y))
 
 
+def removeNonImages(filetree):
+    for root, dirs, files in os.walk(filetree):
+        for name in files:
+            _, ext = getImageFileName(name)
+            if ext not in ('.png', '.jpg', '.jpeg', '.gif', '.webp', '.jp2'):
+                if os.path.exists(os.path.join(root, name)):
+                    os.remove(os.path.join(root, name))
+    # remove empty nested folders
+    for root, dirs, files in os.walk(filetree, False):
+        if not files and not dirs:
+            os.rmdir(root)
+
+
 def sanitizeTree(filetree):
     chapterNames = {}
     page = 1
@@ -830,13 +844,13 @@ def sanitizeTree(filetree):
         dirs.sort(key=OS_SORT_KEY)
         files.sort(key=OS_SORT_KEY)
         for name in files:
-            splitname = os.path.splitext(name)
+            _, ext = getImageFileName(name)
 
             # 9999 page limit
             slugified = f'kcc-{page:04}'
             page += 1
 
-            newKey = os.path.join(root, slugified + splitname[1])
+            newKey = os.path.join(root, slugified + ext)
             key = os.path.join(root, name)
             if key != newKey:
                 os.replace(key, newKey)
@@ -990,10 +1004,6 @@ def detectSuboptimalProcessing(tmppath, orgpath):
                         os.remove(os.path.join(root, name))
                 except OSError as e:
                     raise RuntimeError(f"{name}: {e}")
-    # remove empty nested folders
-    for root, dirs, files in os.walk(tmppath, False):
-        if not files and not dirs:
-            os.rmdir(root)
     if alreadyProcessed:
         print("WARNING: Source files are probably created by KCC. The second conversion will decrease quality.")
         if GUI:
@@ -1027,12 +1037,12 @@ def slugify(value):
 def makeZIP(zipfilename, basedir, isepub=False):
     start = perf_counter()
     zipfilename = os.path.abspath(zipfilename) + '.zip'
-    if '7z' in available_archive_tools():
+    if SEVENZIP in available_archive_tools():
         if isepub:
             mimetypeFile = open(os.path.join(basedir, 'mimetype'), 'w')
             mimetypeFile.write('application/epub+zip')
             mimetypeFile.close()
-        subprocess_run(['7z', 'a', '-tzip', zipfilename, os.path.join(basedir, "*")], capture_output=True, check=True)
+        subprocess_run([SEVENZIP, 'a', '-tzip', zipfilename, os.path.join(basedir, "*")], capture_output=True, check=True)
     else:
         zipOutput = ZipFile(zipfilename, 'w', ZIP_DEFLATED)
         if isepub:
@@ -1232,7 +1242,7 @@ def checkTools(source):
     source = source.upper()
     if source.endswith('.CB7') or source.endswith('.7Z') or source.endswith('.RAR') or source.endswith('.CBR') or \
             source.endswith('.ZIP') or source.endswith('.CBZ'):
-        if '7z' not in available_archive_tools():
+        if SEVENZIP not in available_archive_tools():
             print('ERROR: 7z is missing!')
             sys.exit(1)
     if options.format == 'MOBI':
@@ -1309,6 +1319,7 @@ def makeBook(source, qtgui=None):
     path = getWorkFolder(source)
     print("Checking images...")
     getComicInfo(os.path.join(path, "OEBPS", "Images"), source)
+    removeNonImages(os.path.join(path, "OEBPS", "Images"))
     detectSuboptimalProcessing(os.path.join(path, "OEBPS", "Images"), source)
     chapterNames, cover_path = sanitizeTree(os.path.join(path, 'OEBPS', 'Images'))
     cover = image.Cover(cover_path, options)

kindlecomicconverter/comicarchive.py

@@ -18,7 +18,7 @@
 # PERFORMANCE OF THIS SOFTWARE.
 #
 
-from functools import cached_property
+from functools import cached_property, lru_cache
 import os
 import platform
 import distro
@@ -28,6 +28,7 @@ from xml.parsers.expat import ExpatError
 from .shared import subprocess_run
 
 EXTRACTION_ERROR = 'Failed to extract archive. Try extracting file outside of KCC.'
+SEVENZIP = '7z' if os.name == 'nt' else '7zz'
 
 
 class ComicArchive:
@@ -39,7 +40,7 @@ class ComicArchive:
     @cached_property
     def type(self):    
         extraction_commands = [
-            ['7z', 'l', '-y', '-p1', self.filepath],
+            [SEVENZIP, 'l', '-y', '-p1', self.filepath],
         ]
 
         if distro.id() == 'fedora' or distro.like() == 'fedora':
@@ -68,7 +69,7 @@ class ComicArchive:
 
         extraction_commands = [
             ['tar', '--exclude', '__MACOSX', '--exclude', '.DS_Store', '--exclude', 'thumbs.db', '--exclude', 'Thumbs.db', '-xf', self.filepath, '-C', targetdir],
-            ['7z', 'x', '-y', '-xr!__MACOSX', '-xr!.DS_Store', '-xr!thumbs.db', '-xr!Thumbs.db', '-o' + targetdir, self.filepath],
+            [SEVENZIP, 'x', '-y', '-xr!__MACOSX', '-xr!.DS_Store', '-xr!thumbs.db', '-xr!Thumbs.db', '-o' + targetdir, self.filepath],
         ]
 
         if platform.system() == 'Darwin':
@@ -100,13 +101,13 @@ class ComicArchive:
     def addFile(self, sourcefile):
         if self.type in ['RAR', 'RAR5']:
             raise NotImplementedError
-        process = subprocess_run(['7z', 'a', '-y', self.filepath, sourcefile],
+        process = subprocess_run([SEVENZIP, 'a', '-y', self.filepath, sourcefile],
                         stdout=PIPE, stderr=STDOUT)
         if process.returncode != 0:
             raise OSError('Failed to add the file.')
 
     def extractMetadata(self):
-        process = subprocess_run(['7z', 'x', '-y', '-so', self.filepath, 'ComicInfo.xml'],
+        process = subprocess_run([SEVENZIP, 'x', '-y', '-so', self.filepath, 'ComicInfo.xml'],
                         stdout=PIPE, stderr=STDOUT)
         if process.returncode != 0:
             raise OSError(EXTRACTION_ERROR)
@@ -114,3 +115,16 @@ class ComicArchive:
             return parseString(process.stdout)
         except ExpatError:
             return None
+
+@lru_cache
+def available_archive_tools():
+    available = []
+
+    for tool in ['tar', SEVENZIP, 'unar', 'unrar']:
+        try:
+            subprocess_run([tool], stdout=PIPE, stderr=STDOUT)
+            available.append(tool)
+        except (FileNotFoundError, CalledProcessError):
+            pass
+    
+    return available

kindlecomicconverter/image.py

@@ -20,7 +20,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import io
 import os
+import numpy as np
 from pathlib import Path
+from functools import cached_property
 import mozjpeg_lossless_optimization
 from PIL import Image, ImageOps, ImageStat, ImageChops, ImageFilter, ImageDraw
 from .page_number_crop_alg import get_bbox_crop_margin_page_number, get_bbox_crop_margin
@@ -146,11 +148,9 @@ class ComicPageParser:
 
         # Detect corruption in source image, let caller catch any exceptions triggered.
         srcImgPath = os.path.join(source[0], source[1])
+        Image.open(srcImgPath).verify()
         self.image = Image.open(srcImgPath)
-        self.image.verify()
-        self.image = Image.open(srcImgPath).convert('RGB')
 
-        self.color = self.colorCheck()
         self.fill = self.fillCheck()
         # backwards compatibility for Pillow >9.1.0
         if not hasattr(Image, 'Resampling'):
@@ -181,13 +181,13 @@ class ComicPageParser:
             new_image = Image.new("RGB", (int(width / 2), int(height*2)))
             new_image.paste(pageone, (0, 0))
             new_image.paste(pagetwo, (0, height))
-            self.payload.append(['N', self.source, new_image, self.color, self.fill])
+            self.payload.append(['N', self.source, new_image, self.fill])
         elif (width > height) != (dstwidth > dstheight) and width <= dstheight and height <= dstwidth \
                 and not self.opt.webtoon and self.opt.splitter == 1:
             spread = self.image
             if not self.opt.norotate:
                 spread = spread.rotate(90, Image.Resampling.BICUBIC, True)
-            self.payload.append(['R', self.source, spread, self.color, self.fill])
+            self.payload.append(['R', self.source, spread, self.fill])
         elif (width > height) != (dstwidth > dstheight) and not self.opt.webtoon:
             if self.opt.splitter != 1:
                 if width > height:
@@ -202,38 +202,15 @@ class ComicPageParser:
                 else:
                     pageone = self.image.crop(leftbox)
                     pagetwo = self.image.crop(rightbox)
-                self.payload.append(['S1', self.source, pageone, self.color, self.fill])
-                self.payload.append(['S2', self.source, pagetwo, self.color, self.fill])
+                self.payload.append(['S1', self.source, pageone, self.fill])
+                self.payload.append(['S2', self.source, pagetwo, self.fill])
             if self.opt.splitter > 0:
                 spread = self.image
                 if not self.opt.norotate:
                     spread = spread.rotate(90, Image.Resampling.BICUBIC, True)
-                self.payload.append(['R', self.source, spread,
-                                    self.color, self.fill])
+                self.payload.append(['R', self.source, spread, self.fill])
         else:
-            self.payload.append(['N', self.source, self.image, self.color, self.fill])
-
-    def colorCheck(self):
-        if self.opt.webtoon:
-            return True
-        else:
-            img = self.image.copy()
-            bands = img.getbands()
-            if bands == ('R', 'G', 'B') or bands == ('R', 'G', 'B', 'A'):
-                thumb = img.resize((40, 40))
-                SSE, bias = 0, [0, 0, 0]
-                bias = ImageStat.Stat(thumb).mean[:3]
-                bias = [b - sum(bias) / 3 for b in bias]
-                for pixel in thumb.getdata():
-                    mu = sum(pixel) / 3
-                    SSE += sum((pixel[i] - mu - bias[i]) * (pixel[i] - mu - bias[i]) for i in [0, 1, 2])
-                MSE = float(SSE) / (40 * 40)
-                if MSE > 22:
-                    return True
-                else:
-                    return False
-            else:
-                return False
+            self.payload.append(['N', self.source, self.image, self.fill])
 
     def fillCheck(self):
         if self.opt.bordersColor:
@@ -275,14 +252,14 @@ class ComicPageParser:
 
 
 class ComicPage:
-    def __init__(self, options, mode, path, image, color, fill):
+    def __init__(self, options, mode, path, image, fill):
         self.opt = options
         _, self.size, self.palette, self.gamma = self.opt.profileData
         if self.opt.hq:
             self.size = (int(self.size[0] * 1.5), int(self.size[1] * 1.5))
         self.kindle_scribe_azw3 = (options.profile == 'KS') and (options.format in ('MOBI', 'EPUB'))
-        self.image = image
-        self.color = color
+        self.original_color_mode = image.mode
+        self.image = image.convert("RGB")
         self.fill = fill
         self.rotated = False
         self.orgPath = os.path.join(path[0], path[1])
@@ -301,6 +278,26 @@ class ComicPage:
         if not hasattr(Image, 'Resampling'):
             Image.Resampling = Image
 
+    @cached_property
+    def color(self):
+        if self.original_color_mode in ("L", "1"):
+            return False
+        img = self.image.convert("YCbCr")
+        _, cb, cr = img.split()
+
+        cb_hist = cb.histogram()
+        cr_hist = cr.histogram()
+        cb_nonzero = [i for i, e in enumerate(cb_hist) if e]
+        cr_nonzero = [i for i, e in enumerate(cr_hist) if e]
+        cb_spread = cb_nonzero[-1] - cb_nonzero[0] if len(cb_nonzero) else 0
+        cr_spread = cr_nonzero[-1] - cr_nonzero[0] if len(cr_nonzero) else 0
+
+        SPREAD_THRESHOLD=20
+        if cb_spread < SPREAD_THRESHOLD and cr_spread < SPREAD_THRESHOLD:
+            return False
+        else:
+            return True
+        
     def saveToDir(self):
         try:
             flags = []

kindlecomicconverter/shared.py

@@ -18,7 +18,6 @@
 # PERFORMANCE OF THIS SOFTWARE.
 #
 
-from functools import lru_cache
 import os
 from html.parser import HTMLParser
 import subprocess
@@ -49,12 +48,6 @@ class HTMLStripper(HTMLParser):
 def getImageFileName(imgfile):
     name, ext = os.path.splitext(imgfile)
     ext = ext.lower()
-    if (name.startswith('.') and len(name) == 1):
-        return None
-    if name.startswith('._'):
-        return None
-    if ext not in ['.png', '.jpg', '.jpeg', '.gif', '.webp', '.jp2', '.j2k', '.jpx']:
-        return None
     return [name, ext]
 
 
@@ -131,19 +124,6 @@ def dependencyCheck(level):
         print('ERROR: ' + ', '.join(missing) + ' is not installed!')
         sys.exit(1)
 
-@lru_cache
-def available_archive_tools():
-    available = []
-
-    for tool in ['tar', '7z', 'unar', 'unrar']:
-        try:
-            subprocess_run([tool], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-            available.append(tool)
-        except (FileNotFoundError, subprocess.CalledProcessError):
-            pass
-    
-    return available
-
 def subprocess_run(command, **kwargs):
     if (os.name == 'nt'):
         kwargs.setdefault('creationflags', subprocess.CREATE_NO_WINDOW)

requirements.txt

@@ -5,7 +5,7 @@ requests>=2.31.0
 python-slugify>=1.2.1
 raven>=6.0.0
 packaging>=23.2
-mozjpeg-lossless-optimization==1.2.0
+mozjpeg-lossless-optimization>=1.2.0
 natsort>=8.4.0
 distro>=1.8.0
 numpy>=1.22.4