bump 8.0.1

* only slugify if sort matters

* add comments

* make conditions more granular

* fix

* shorten

.github/FUNDING.yml

@@ -0,0 +1,15 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: eink_dude
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
+polar: # Replace with a single Polar username
+buy_me_a_coffee: # Replace with a single Buy Me a Coffee username
+thanks_dev: # Replace with a single thanks.dev username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

README.md

@@ -27,7 +27,7 @@ on underpowered ereaders with small storage capacities.
 KCC avoids many common formatting issues (some of which occur [even on the Kindle Store](https://github.com/ciromattia/kcc/wiki/Kindle-Store-bad-formatting)), such as:
 1) faded black levels causing unneccessarily low contrast, which is hard to see and can cause eyestrain.
 2) unneccessary margins at the bottom of the screen
-3) Not utilizing the full 1860x2480 resolution of the 10" Kindle Scribe (feature in progress)
+3) Not utilizing the full 1860x2480 resolution of the 10" Kindle Scribe
 4) incorrect page turn direction for manga that's read right to left
 5) unaligned two page spreads in landscape, where pages are shifted over by 1
 

kindlecomicconverter/KCC_gui.py

@@ -40,10 +40,10 @@ from packaging.version import Version
 from raven import Client
 from tempfile import gettempdir
 
-from .shared import HTMLStripper, available_archive_tools, sanitizeTrace, walkLevel, subprocess_run
+from .shared import HTMLStripper, sanitizeTrace, walkLevel, subprocess_run
+from .comicarchive import SEVENZIP, available_archive_tools
 from . import __version__
 from . import comic2ebook
-from . import image
 from . import metadata
 from . import kindle
 from . import KCC_ui
@@ -1166,7 +1166,7 @@ class KCCGUI(KCC_ui.Ui_mainWindow):
                             'info')
         
         self.tar = 'tar' in available_archive_tools()
-        self.sevenzip = '7z' in available_archive_tools()
+        self.sevenzip = SEVENZIP in available_archive_tools()
         if not any([self.tar, self.sevenzip]):
             self.addMessage('<a href="https://github.com/ciromattia/kcc#7-zip">Install 7z (link)</a>'
                             ' to enable CBZ/CBR/ZIP/etc processing.', 'warning')

kindlecomicconverter/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '7.6.0'
+__version__ = '8.0.1'
 __license__ = 'ISC'
 __copyright__ = '2012-2022, Ciro Mattia Gonano <ciromattia@gmail.com>, Pawel Jastrzebski <pawelj@iosphe.re>, darodi'
 __docformat__ = 'restructuredtext en'

kindlecomicconverter/comic2ebook.py

@@ -34,7 +34,7 @@ from tempfile import mkdtemp, gettempdir, TemporaryFile
 from shutil import move, copytree, rmtree, copyfile
 from multiprocessing import Pool
 from uuid import uuid4
-from natsort import os_sort_keygen
+from natsort import os_sort_keygen, os_sorted
 from slugify import slugify as slugify_ext
 from PIL import Image, ImageFile
 from pathlib import Path
@@ -42,7 +42,8 @@ from subprocess import STDOUT, PIPE, CalledProcessError
 from psutil import virtual_memory, disk_usage
 from html import escape as hescape
 
-from .shared import available_archive_tools, getImageFileName, walkSort, walkLevel, sanitizeTrace, subprocess_run
+from .shared import getImageFileName, walkSort, walkLevel, sanitizeTrace, subprocess_run
+from .comicarchive import SEVENZIP, available_archive_tools
 from . import comic2panel
 from . import image
 from . import comicarchive
@@ -647,7 +648,7 @@ def imgFileProcessing(work):
             img.autocontrastImage()
             img.resizeImage()
             img.optimizeForDisplay(opt.reducerainbow)
-            if opt.forcecolor and workImg.color:
+            if opt.forcecolor and img.color:
                 pass
             elif opt.forcepng:
                 img.quantizeImage()
@@ -690,16 +691,6 @@ def getWorkFolder(afile):
                 cbx = comicarchive.ComicArchive(afile)
                 path = cbx.extract(workdir)
                 sanitizePermissions(path)
-                tdir = os.listdir(workdir)
-                if len(tdir) == 2 and 'ComicInfo.xml' in tdir:
-                    tdir.remove('ComicInfo.xml')
-                    if os.path.isdir(os.path.join(workdir, tdir[0])):
-                        os.replace(
-                            os.path.join(workdir, 'ComicInfo.xml'),
-                            os.path.join(workdir, tdir[0], 'ComicInfo.xml')
-                        )
-                if len(tdir) == 1 and os.path.isdir(os.path.join(workdir, tdir[0])):
-                    path = os.path.join(workdir, tdir[0])           
             except OSError as e:
                 rmtree(workdir, True)
                 raise UserWarning(e)
@@ -822,29 +813,48 @@ def getPanelViewSize(deviceres, size):
     return str(int(x)), str(int(y))
 
 
+def removeNonImages(filetree):
+    # clean dot from original file
+    dot_clean(filetree)
+
+    for root, dirs, files in os.walk(filetree):
+        for name in files:
+            _, ext = getImageFileName(name)
+            if ext not in ('.png', '.jpg', '.jpeg', '.gif', '.webp', '.jp2'):
+                if os.path.exists(os.path.join(root, name)):
+                    os.remove(os.path.join(root, name))
+    # remove empty nested folders
+    for root, dirs, files in os.walk(filetree, False):
+        if not files and not dirs:
+            os.rmdir(root)
+
+
 def sanitizeTree(filetree):
     chapterNames = {}
     page = 1
     cover_path = None
     for root, dirs, files in os.walk(filetree):
-        dirs.sort(key=OS_SORT_KEY)
         files.sort(key=OS_SORT_KEY)
         for name in files:
-            splitname = os.path.splitext(name)
+            _, ext = getImageFileName(name)
 
             # 9999 page limit
-            slugified = f'kcc-{page:04}'
+            unique_name = f'kcc-{page:04}'
             page += 1
 
-            newKey = os.path.join(root, slugified + splitname[1])
+            newKey = os.path.join(root, unique_name + ext)
             key = os.path.join(root, name)
             if key != newKey:
                 os.replace(key, newKey)
             if not cover_path:
                 cover_path = newKey
+        is_natural_sorted = False
+        if os_sorted(dirs) == sorted(dirs):
+            is_natural_sorted = True
+        dirs.sort(key=OS_SORT_KEY)
         for i, name in enumerate(dirs):
             tmpName = name
-            slugified = slugify(name)
+            slugified = slugify(name, is_natural_sorted)
             while os.path.exists(os.path.join(root, slugified)) and name.upper() != slugified.upper():
                 slugified += "A"
             chapterNames[slugified] = tmpName
@@ -870,8 +880,7 @@ def sanitizePermissions(filetree):
             os.chmod(os.path.join(root, name), S_IWRITE | S_IREAD)
         for name in dirs:
             os.chmod(os.path.join(root, name), S_IWRITE | S_IREAD | S_IEXEC)
-    # clean dot from original file
+
-    dot_clean(filetree)
 
 def dot_clean(filetree):
     for root, _, files in os.walk(filetree, topdown=False):
@@ -990,10 +999,6 @@ def detectSuboptimalProcessing(tmppath, orgpath):
                         os.remove(os.path.join(root, name))
                 except OSError as e:
                     raise RuntimeError(f"{name}: {e}")
-    # remove empty nested folders
-    for root, dirs, files in os.walk(tmppath, False):
-        if not files and not dirs:
-            os.rmdir(root)
     if alreadyProcessed:
         print("WARNING: Source files are probably created by KCC. The second conversion will decrease quality.")
         if GUI:
@@ -1016,23 +1021,27 @@ def createNewTome(parent):
     return tomePath, tomePathRoot
 
 
-def slugify(value):
+def slugify(value, is_natural_sorted):
-    if options.format == 'CBZ':
+    if options.format == 'CBZ' and is_natural_sorted:
         return value
-    value = slugify_ext(value, regex_pattern=r'[^-a-z0-9_\.]+').strip('.')
+    if options.format != 'CBZ':
-    value = sub(r'0*([0-9]{4,})', r'\1', sub(r'([0-9]+)', r'0000\1', value, count=2))
+        # convert all unicode to ascii via slugify
+        value = slugify_ext(value, regex_pattern=r'[^-a-z0-9_\.]+').strip('.')
+    if not is_natural_sorted:
+        # pad zeros to numbers
+        value = sub(r'0*([0-9]{4,})', r'\1', sub(r'([0-9]+)', r'0000\1', value, count=2))
     return value
 
 
 def makeZIP(zipfilename, basedir, isepub=False):
     start = perf_counter()
     zipfilename = os.path.abspath(zipfilename) + '.zip'
-    if '7z' in available_archive_tools():
+    if SEVENZIP in available_archive_tools():
         if isepub:
             mimetypeFile = open(os.path.join(basedir, 'mimetype'), 'w')
             mimetypeFile.write('application/epub+zip')
             mimetypeFile.close()
-        subprocess_run(['7z', 'a', '-tzip', zipfilename, os.path.join(basedir, "*")], capture_output=True, check=True)
+        subprocess_run([SEVENZIP, 'a', '-tzip', zipfilename, os.path.join(basedir, "*")], capture_output=True, check=True)
     else:
         zipOutput = ZipFile(zipfilename, 'w', ZIP_DEFLATED)
         if isepub:
@@ -1232,7 +1241,7 @@ def checkTools(source):
     source = source.upper()
     if source.endswith('.CB7') or source.endswith('.7Z') or source.endswith('.RAR') or source.endswith('.CBR') or \
             source.endswith('.ZIP') or source.endswith('.CBZ'):
-        if '7z' not in available_archive_tools():
+        if SEVENZIP not in available_archive_tools():
             print('ERROR: 7z is missing!')
             sys.exit(1)
     if options.format == 'MOBI':
@@ -1309,6 +1318,7 @@ def makeBook(source, qtgui=None):
     path = getWorkFolder(source)
     print("Checking images...")
     getComicInfo(os.path.join(path, "OEBPS", "Images"), source)
+    removeNonImages(os.path.join(path, "OEBPS", "Images"))
     detectSuboptimalProcessing(os.path.join(path, "OEBPS", "Images"), source)
     chapterNames, cover_path = sanitizeTree(os.path.join(path, 'OEBPS', 'Images'))
     cover = image.Cover(cover_path, options)

kindlecomicconverter/comicarchive.py

@@ -18,7 +18,7 @@
 # PERFORMANCE OF THIS SOFTWARE.
 #
 
-from functools import cached_property
+from functools import cached_property, lru_cache
 import os
 import platform
 import distro
@@ -28,6 +28,7 @@ from xml.parsers.expat import ExpatError
 from .shared import subprocess_run
 
 EXTRACTION_ERROR = 'Failed to extract archive. Try extracting file outside of KCC.'
+SEVENZIP = '7zz' if platform.system() == 'Darwin' else '7z'
 
 
 class ComicArchive:
@@ -39,7 +40,7 @@ class ComicArchive:
     @cached_property
     def type(self):    
         extraction_commands = [
-            ['7z', 'l', '-y', '-p1', self.filepath],
+            [SEVENZIP, 'l', '-y', '-p1', self.filepath],
         ]
 
         if distro.id() == 'fedora' or distro.like() == 'fedora':
@@ -68,7 +69,7 @@ class ComicArchive:
 
         extraction_commands = [
             ['tar', '--exclude', '__MACOSX', '--exclude', '.DS_Store', '--exclude', 'thumbs.db', '--exclude', 'Thumbs.db', '-xf', self.filepath, '-C', targetdir],
-            ['7z', 'x', '-y', '-xr!__MACOSX', '-xr!.DS_Store', '-xr!thumbs.db', '-xr!Thumbs.db', '-o' + targetdir, self.filepath],
+            [SEVENZIP, 'x', '-y', '-xr!__MACOSX', '-xr!.DS_Store', '-xr!thumbs.db', '-xr!Thumbs.db', '-o' + targetdir, self.filepath],
         ]
 
         if platform.system() == 'Darwin':
@@ -100,13 +101,13 @@ class ComicArchive:
     def addFile(self, sourcefile):
         if self.type in ['RAR', 'RAR5']:
             raise NotImplementedError
-        process = subprocess_run(['7z', 'a', '-y', self.filepath, sourcefile],
+        process = subprocess_run([SEVENZIP, 'a', '-y', self.filepath, sourcefile],
                         stdout=PIPE, stderr=STDOUT)
         if process.returncode != 0:
             raise OSError('Failed to add the file.')
 
     def extractMetadata(self):
-        process = subprocess_run(['7z', 'x', '-y', '-so', self.filepath, 'ComicInfo.xml'],
+        process = subprocess_run([SEVENZIP, 'x', '-y', '-so', self.filepath, 'ComicInfo.xml'],
                         stdout=PIPE, stderr=STDOUT)
         if process.returncode != 0:
             raise OSError(EXTRACTION_ERROR)
@@ -114,3 +115,16 @@ class ComicArchive:
             return parseString(process.stdout)
         except ExpatError:
             return None
+
+@lru_cache
+def available_archive_tools():
+    available = []
+
+    for tool in ['tar', SEVENZIP, 'unar', 'unrar']:
+        try:
+            subprocess_run([tool], stdout=PIPE, stderr=STDOUT)
+            available.append(tool)
+        except (FileNotFoundError, CalledProcessError):
+            pass
+    
+    return available

kindlecomicconverter/image.py

@@ -20,7 +20,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import io
 import os
+import numpy as np
 from pathlib import Path
+from functools import cached_property
 import mozjpeg_lossless_optimization
 from PIL import Image, ImageOps, ImageStat, ImageChops, ImageFilter, ImageDraw
 from .page_number_crop_alg import get_bbox_crop_margin_page_number, get_bbox_crop_margin
@@ -146,11 +148,9 @@ class ComicPageParser:
 
         # Detect corruption in source image, let caller catch any exceptions triggered.
         srcImgPath = os.path.join(source[0], source[1])
+        Image.open(srcImgPath).verify()
         self.image = Image.open(srcImgPath)
-        self.image.verify()
-        self.image = Image.open(srcImgPath).convert('RGB')
 
-        self.color = self.colorCheck()
         self.fill = self.fillCheck()
         # backwards compatibility for Pillow >9.1.0
         if not hasattr(Image, 'Resampling'):
@@ -181,13 +181,13 @@ class ComicPageParser:
             new_image = Image.new("RGB", (int(width / 2), int(height*2)))
             new_image.paste(pageone, (0, 0))
             new_image.paste(pagetwo, (0, height))
-            self.payload.append(['N', self.source, new_image, self.color, self.fill])
+            self.payload.append(['N', self.source, new_image, self.fill])
         elif (width > height) != (dstwidth > dstheight) and width <= dstheight and height <= dstwidth \
                 and not self.opt.webtoon and self.opt.splitter == 1:
             spread = self.image
             if not self.opt.norotate:
                 spread = spread.rotate(90, Image.Resampling.BICUBIC, True)
-            self.payload.append(['R', self.source, spread, self.color, self.fill])
+            self.payload.append(['R', self.source, spread, self.fill])
         elif (width > height) != (dstwidth > dstheight) and not self.opt.webtoon:
             if self.opt.splitter != 1:
                 if width > height:
@@ -202,38 +202,15 @@ class ComicPageParser:
                 else:
                     pageone = self.image.crop(leftbox)
                     pagetwo = self.image.crop(rightbox)
-                self.payload.append(['S1', self.source, pageone, self.color, self.fill])
+                self.payload.append(['S1', self.source, pageone, self.fill])
-                self.payload.append(['S2', self.source, pagetwo, self.color, self.fill])
+                self.payload.append(['S2', self.source, pagetwo, self.fill])
             if self.opt.splitter > 0:
                 spread = self.image
                 if not self.opt.norotate:
                     spread = spread.rotate(90, Image.Resampling.BICUBIC, True)
-                self.payload.append(['R', self.source, spread,
+                self.payload.append(['R', self.source, spread, self.fill])
-                                    self.color, self.fill])
         else:
-            self.payload.append(['N', self.source, self.image, self.color, self.fill])
+            self.payload.append(['N', self.source, self.image, self.fill])
-
-    def colorCheck(self):
-        if self.opt.webtoon:
-            return True
-        else:
-            img = self.image.copy()
-            bands = img.getbands()
-            if bands == ('R', 'G', 'B') or bands == ('R', 'G', 'B', 'A'):
-                thumb = img.resize((40, 40))
-                SSE, bias = 0, [0, 0, 0]
-                bias = ImageStat.Stat(thumb).mean[:3]
-                bias = [b - sum(bias) / 3 for b in bias]
-                for pixel in thumb.getdata():
-                    mu = sum(pixel) / 3
-                    SSE += sum((pixel[i] - mu - bias[i]) * (pixel[i] - mu - bias[i]) for i in [0, 1, 2])
-                MSE = float(SSE) / (40 * 40)
-                if MSE > 22:
-                    return True
-                else:
-                    return False
-            else:
-                return False
 
     def fillCheck(self):
         if self.opt.bordersColor:
@@ -275,14 +252,14 @@ class ComicPageParser:
 
 
 class ComicPage:
-    def __init__(self, options, mode, path, image, color, fill):
+    def __init__(self, options, mode, path, image, fill):
         self.opt = options
         _, self.size, self.palette, self.gamma = self.opt.profileData
         if self.opt.hq:
             self.size = (int(self.size[0] * 1.5), int(self.size[1] * 1.5))
         self.kindle_scribe_azw3 = (options.profile == 'KS') and (options.format in ('MOBI', 'EPUB'))
-        self.image = image
+        self.original_color_mode = image.mode
-        self.color = color
+        self.image = image.convert("RGB")
         self.fill = fill
         self.rotated = False
         self.orgPath = os.path.join(path[0], path[1])
@@ -301,6 +278,26 @@ class ComicPage:
         if not hasattr(Image, 'Resampling'):
             Image.Resampling = Image
 
+    @cached_property
+    def color(self):
+        if self.original_color_mode in ("L", "1"):
+            return False
+        img = self.image.convert("YCbCr")
+        _, cb, cr = img.split()
+
+        cb_hist = cb.histogram()
+        cr_hist = cr.histogram()
+        cb_nonzero = [i for i, e in enumerate(cb_hist) if e]
+        cr_nonzero = [i for i, e in enumerate(cr_hist) if e]
+        cb_spread = cb_nonzero[-1] - cb_nonzero[0] if len(cb_nonzero) else 0
+        cr_spread = cr_nonzero[-1] - cr_nonzero[0] if len(cr_nonzero) else 0
+
+        SPREAD_THRESHOLD=20
+        if cb_spread < SPREAD_THRESHOLD and cr_spread < SPREAD_THRESHOLD:
+            return False
+        else:
+            return True
+        
     def saveToDir(self):
         try:
             flags = []

kindlecomicconverter/shared.py

@@ -18,7 +18,6 @@
 # PERFORMANCE OF THIS SOFTWARE.
 #
 
-from functools import lru_cache
 import os
 from html.parser import HTMLParser
 import subprocess
@@ -49,12 +48,6 @@ class HTMLStripper(HTMLParser):
 def getImageFileName(imgfile):
     name, ext = os.path.splitext(imgfile)
     ext = ext.lower()
-    if (name.startswith('.') and len(name) == 1):
-        return None
-    if name.startswith('._'):
-        return None
-    if ext not in ['.png', '.jpg', '.jpeg', '.gif', '.webp', '.jp2', '.j2k', '.jpx']:
-        return None
     return [name, ext]
 
 
@@ -131,19 +124,6 @@ def dependencyCheck(level):
         print('ERROR: ' + ', '.join(missing) + ' is not installed!')
         sys.exit(1)
 
-@lru_cache
-def available_archive_tools():
-    available = []
-
-    for tool in ['tar', '7z', 'unar', 'unrar']:
-        try:
-            subprocess_run([tool], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-            available.append(tool)
-        except (FileNotFoundError, subprocess.CalledProcessError):
-            pass
-    
-    return available
-
 def subprocess_run(command, **kwargs):
     if (os.name == 'nt'):
         kwargs.setdefault('creationflags', subprocess.CREATE_NO_WINDOW)

requirements.txt

@@ -5,7 +5,7 @@ requests>=2.31.0
 python-slugify>=1.2.1
 raven>=6.0.0
 packaging>=23.2
-mozjpeg-lossless-optimization==1.2.0
+mozjpeg-lossless-optimization>=1.2.0
 natsort>=8.4.0
 distro>=1.8.0
 numpy>=1.22.4