No size limit anymore. KCC now split EPUB files before handling them to KindleGen.

kcc/KCC_gui.py

@@ -111,6 +111,7 @@ class WorkerThread(QtCore.QThread):
         self.emit(QtCore.SIGNAL("addMessage"), '<b>Conversion interrupted.</b>', 'error')
         self.emit(QtCore.SIGNAL("modeConvert"), True)
 
+    # noinspection PyUnboundLocalVariable
     def run(self):
         self.emit(QtCore.SIGNAL("modeConvert"), False)
         profile = ProfileData.ProfileLabels[str(GUI.DeviceBox.currentText())]
@@ -141,6 +142,8 @@ class WorkerThread(QtCore.QThread):
                 argv.append("--gamma=" + self.parent.GammaValue)
             if str(GUI.FormatBox.currentText()) == 'CBZ':
                 argv.append("--cbz-output")
+            if str(GUI.FormatBox.currentText()) == 'MOBI':
+                argv.append("--batchsplit")
         if self.parent.currentMode > 2:
             argv.append("--customwidth=" + str(GUI.customWidth.text()))
             argv.append("--customheight=" + str(GUI.customHeight.text()))
@@ -180,7 +183,8 @@ class WorkerThread(QtCore.QThread):
                                                        % (jobargv[-1], str(err), traceback.format_tb(traceback_)))
                 self.emit(QtCore.SIGNAL("addMessage"), 'KCC failed to create EPUB!', 'error')
             if not self.conversionAlive:
-                os.remove(outputPath)
+                for item in outputPath:
+                    os.remove(item)
                 self.clean()
                 return
             if not self.errors:
@@ -189,64 +193,79 @@ class WorkerThread(QtCore.QThread):
                 else:
                     self.emit(QtCore.SIGNAL("addMessage"), 'Creating EPUB file... Done!', 'info', True)
                 if str(GUI.FormatBox.currentText()) == 'MOBI':
-                    self.emit(QtCore.SIGNAL("addMessage"), 'Creating MOBI file...', 'info')
+                    tomeNumber = 0
-                    self.emit(QtCore.SIGNAL("progressBarTick"), 1)
+                    for item in outputPath:
-                    try:
+                        tomeNumber += 1
-                        self.kindlegenErrorCode = 0
+                        if len(outputPath) > 1:
-                        if os.path.getsize(outputPath) < 367001600:
+                            self.emit(QtCore.SIGNAL("addMessage"), 'Creating MOBI file (' + str(tomeNumber)
-                            output = Popen('kindlegen "' + outputPath + '"', stdout=PIPE, stderr=STDOUT, shell=True)
+                                                                   + '/' + str(len(outputPath)) + ')...', 'info')
-                            for line in output.stdout:
-                                # ERROR: Generic error
-                                if "Error(" in line:
-                                    self.kindlegenErrorCode = 1
-                                    self.kindlegenError = line
-                                # ERROR: EPUB too big
-                                if ":E23026:" in line:
-                                    self.kindlegenErrorCode = 23026
                         else:
-                            # ERROR: EPUB too big
+                            self.emit(QtCore.SIGNAL("addMessage"), 'Creating MOBI file...', 'info')
-                            self.kindlegenErrorCode = 23026
+                        self.emit(QtCore.SIGNAL("progressBarTick"), 1)
-                    except:
-                        # ERROR: Unknown generic error
-                        self.kindlegenErrorCode = 1
-                        continue
-                    if not self.conversionAlive:
-                        os.remove(outputPath)
-                        os.remove(outputPath.replace('.epub', '.mobi'))
-                        self.clean()
-                        return
-                    if self.kindlegenErrorCode == 0:
-                        self.emit(QtCore.SIGNAL("addMessage"), 'Creating MOBI file... Done!', 'info', True)
-                        self.emit(QtCore.SIGNAL("addMessage"), 'Removing SRCS header...', 'info')
-                        os.remove(outputPath)
-                        mobiPath = outputPath.replace('.epub', '.mobi')
-                        shutil.move(mobiPath, mobiPath + '_tostrip')
                         try:
-                            kindlestrip.main((mobiPath + '_tostrip', mobiPath))
+                            self.kindlegenErrorCode = 0
-                        except Exception:
+                            if os.path.getsize(item) < 367001600:
-                            self.errors = True
+                                output = Popen('kindlegen "' + item + '"', stdout=PIPE, stderr=STDOUT, shell=True)
-                        if not self.errors:
+                                for line in output.stdout:
-                            os.remove(mobiPath + '_tostrip')
+                                    # ERROR: Generic error
-                            self.emit(QtCore.SIGNAL("addMessage"), 'Removing SRCS header... Done!', 'info', True)
+                                    if "Error(" in line:
+                                        self.kindlegenErrorCode = 1
+                                        self.kindlegenError = line
+                                    # ERROR: EPUB too big
+                                    if ":E23026:" in line:
+                                        self.kindlegenErrorCode = 23026
+                                    if self.kindlegenErrorCode > 0:
+                                        break
+                            else:
+                                # ERROR: EPUB too big
+                                self.kindlegenErrorCode = 23026
+                        except:
+                            # ERROR: Unknown generic error
+                            self.kindlegenErrorCode = 1
+                            continue
+                        if not self.conversionAlive:
+                            for item in outputPath:
+                                os.remove(item)
+                                os.remove(item.replace('.epub', '.mobi'))
+                            self.clean()
+                            return
+                        if self.kindlegenErrorCode == 0:
+                            if len(outputPath) > 1:
+                                self.emit(QtCore.SIGNAL("addMessage"), 'Creating MOBI file (' + str(tomeNumber) + '/'
+                                                                       + str(len(outputPath)) + ')... Done!', 'info',
+                                                                       True)
+                            else:
+                                self.emit(QtCore.SIGNAL("addMessage"), 'Creating MOBI file... Done!', 'info', True)
+                            self.emit(QtCore.SIGNAL("addMessage"), 'Removing SRCS header...', 'info')
+                            os.remove(item)
+                            mobiPath = item.replace('.epub', '.mobi')
+                            shutil.move(mobiPath, mobiPath + '_tostrip')
+                            try:
+                                kindlestrip.main((mobiPath + '_tostrip', mobiPath))
+                            except Exception:
+                                self.errors = True
+                            if not self.errors:
+                                os.remove(mobiPath + '_tostrip')
+                                self.emit(QtCore.SIGNAL("addMessage"), 'Removing SRCS header... Done!', 'info', True)
+                            else:
+                                shutil.move(mobiPath + '_tostrip', mobiPath)
+                                self.emit(QtCore.SIGNAL("addMessage"),
+                                          'KindleStrip failed to remove SRCS header!', 'warning')
+                                self.emit(QtCore.SIGNAL("addMessage"),
+                                          'MOBI file will work correctly but it will be highly oversized.', 'warning')
                         else:
-                            shutil.move(mobiPath + '_tostrip', mobiPath)
+                            epubSize = (os.path.getsize(item))/1024/1024
-                            self.emit(QtCore.SIGNAL("addMessage"),
+                            os.remove(item)
-                                      'KindleStrip failed to remove SRCS header!', 'warning')
+                            if os.path.exists(item.replace('.epub', '.mobi')):
-                            self.emit(QtCore.SIGNAL("addMessage"),
+                                os.remove(item.replace('.epub', '.mobi'))
-                                      'MOBI file will work correctly but it will be highly oversized.', 'warning')
+                            self.emit(QtCore.SIGNAL("addMessage"), 'KindleGen failed to create MOBI!', 'error')
-                    else:
+                            if self.kindlegenErrorCode == 1 and self.kindlegenError:
-                        epubSize = (os.path.getsize(outputPath))/1024/1024
+                                self.emit(QtCore.SIGNAL("showDialog"), "KindleGen error:\n\n" + self.kindlegenError)
-                        os.remove(outputPath)
+                            if self.kindlegenErrorCode == 23026:
-                        if os.path.exists(outputPath.replace('.epub', '.mobi')):
+                                self.emit(QtCore.SIGNAL("addMessage"), 'Created EPUB file was too big.',
-                            os.remove(outputPath.replace('.epub', '.mobi'))
+                                          'error')
-                        self.emit(QtCore.SIGNAL("addMessage"), 'KindleGen failed to create MOBI!', 'error')
+                                self.emit(QtCore.SIGNAL("addMessage"), 'EPUB file: ' + str(epubSize) + 'MB.'
-                        if self.kindlegenErrorCode == 1 and self.kindlegenError:
+                                                                       ' Supported size: ~300MB.', 'error')
-                            self.emit(QtCore.SIGNAL("showDialog"), "KindleGen error:\n\n" + self.kindlegenError)
-                        if self.kindlegenErrorCode == 23026:
-                            self.emit(QtCore.SIGNAL("addMessage"), 'Created EPUB file was too big.',
-                                      'error')
-                            self.emit(QtCore.SIGNAL("addMessage"), 'EPUB file: ' + str(epubSize) + 'MB.'
-                                                                   ' Supported size: ~300MB.', 'error')
         self.emit(QtCore.SIGNAL("hideProgressBar"))
         self.parent.needClean = True
         self.emit(QtCore.SIGNAL("addMessage"), '<b>All jobs completed.</b>', 'info')

kcc/comic2ebook.py

@@ -451,7 +451,6 @@ def genEpubStruct(path):
     chapterlist = []
     cover = None
     _, deviceres, _, _, panelviewsize = options.profileData
-    sanitizeTree(os.path.join(path, 'OEBPS', 'Images'))
     os.mkdir(os.path.join(path, 'OEBPS', 'Text'))
     f = open(os.path.join(path, 'OEBPS', 'Text', 'style.css'), 'w')
     # DON'T COMPRESS CSS. KINDLE WILL FAIL TO PARSE IT.
@@ -673,6 +672,140 @@ def sanitizeTreeBeforeConversion(filetree):
             os.chmod(os.path.join(root, name), stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC)
 
 
+def getDirectorySize(start_path='.'):
+    total_size = 0
+    for dirpath, dirnames, filenames in os.walk(start_path):
+        for f in filenames:
+            fp = os.path.join(dirpath, f)
+            total_size += os.path.getsize(fp)
+    return total_size
+
+
+def createNewTome():
+    tomePathRoot = tempfile.mkdtemp()
+    tomePath = os.path.join(tomePathRoot, 'OEBPS', 'Images')
+    os.makedirs(tomePath)
+    return tomePath, tomePathRoot
+
+
+def walkLevel(some_dir, level=1):
+    some_dir = some_dir.rstrip(os.path.sep)
+    assert os.path.isdir(some_dir)
+    num_sep = some_dir.count(os.path.sep)
+    for root, dirs, files in os.walk(some_dir):
+        yield root, dirs, files
+        num_sep_this = root.count(os.path.sep)
+        if num_sep + level <= num_sep_this:
+            del dirs[:]
+
+
+def splitDirectory(path, mode):
+    output = []
+    currentSize = 0
+    currentTarget = path
+    if mode == 0:
+        for root, dirs, files in walkLevel(path, 0):
+            for name in files:
+                size = os.path.getsize(os.path.join(root, name))
+                if currentSize + size > 262144000:
+                    currentTarget, pathRoot = createNewTome()
+                    output.append(pathRoot)
+                    currentSize = size
+                else:
+                    currentSize += size
+                if path != currentTarget:
+                    move(os.path.join(root, name), os.path.join(currentTarget, name))
+    elif mode == 1:
+        for root, dirs, files in walkLevel(path, 0):
+            for name in dirs:
+                size = getDirectorySize(os.path.join(root, name))
+                if currentSize + size > 262144000:
+                    currentTarget, pathRoot = createNewTome()
+                    output.append(pathRoot)
+                    currentSize = size
+                else:
+                    currentSize += size
+                if path != currentTarget:
+                    move(os.path.join(root, name), os.path.join(currentTarget, name))
+    elif mode == 2:
+        for root, dirs, files in walkLevel(path, 1):
+            for name in dirs:
+                if root != path:
+                    size = getDirectorySize(os.path.join(root, name))
+                    if currentSize + size > 262144000:
+                        currentTarget, pathRoot = createNewTome()
+                        output.append(pathRoot)
+                        currentSize = size
+                    else:
+                        currentSize += size
+                    if path != currentTarget:
+                        move(os.path.join(root, name), os.path.join(currentTarget, name))
+    return output
+
+
+# noinspection PyUnboundLocalVariable
+def preSplitDirectory(path):
+    if getDirectorySize(os.path.join(path, 'OEBPS', 'Images')) > 262144000:
+        # Detect directory stucture
+        for root, dirs, files in walkLevel(os.path.join(path, 'OEBPS', 'Images'), 0):
+            subdirectoryNumber = len(dirs)
+            filesNumber = len(files)
+        if subdirectoryNumber == 0:
+            # No subdirectories
+            mode = 0
+        else:
+            if filesNumber > 0:
+                print '\nWARNING: Automatic output splitting failed.'
+                if GUI:
+                    GUI.emit(QtCore.SIGNAL("addMessage"), 'Automatic output splitting failed. <a href='
+                                                          '"https://github.com/ciromattia/kcc/wiki'
+                                                          '/Automatic-output-splitting">'
+                                                          'Click for more details.</a>', 'warning')
+                    GUI.emit(QtCore.SIGNAL("addMessage"), '')
+                return [path]
+            detectedSubSubdirectories = False
+            detectedFilesInSubdirectories = False
+            for root, dirs, files in walkLevel(os.path.join(path, 'OEBPS', 'Images'), 1):
+                if root != os.path.join(path, 'OEBPS', 'Images'):
+                    if len(dirs) != 0:
+                        detectedSubSubdirectories = True
+                    elif len(dirs) == 0 and detectedSubSubdirectories:
+                        print '\nWARNING: Automatic output splitting failed.'
+                        if GUI:
+                            GUI.emit(QtCore.SIGNAL("addMessage"), 'Automatic output splitting failed. <a href='
+                                                                  '"https://github.com/ciromattia/kcc/wiki'
+                                                                  '/Automatic-output-splitting">'
+                                                                  'Click for more details.</a>', 'warning')
+                            GUI.emit(QtCore.SIGNAL("addMessage"), '')
+                        return [path]
+                    if len(files) != 0:
+                        detectedFilesInSubdirectories = True
+            if detectedSubSubdirectories:
+                # Two levels of subdirectories
+                mode = 2
+            else:
+                # One level of subdirectories
+                mode = 1
+            if detectedFilesInSubdirectories and detectedSubSubdirectories:
+                print '\nWARNING: Automatic output splitting failed.'
+                if GUI:
+                    GUI.emit(QtCore.SIGNAL("addMessage"), 'Automatic output splitting failed. <a href='
+                                                          '"https://github.com/ciromattia/kcc/wiki'
+                                                          '/Automatic-output-splitting">'
+                                                          'Click for more details.</a>', 'warning')
+                    GUI.emit(QtCore.SIGNAL("addMessage"), '')
+                return [path]
+        # Split directories
+        split = splitDirectory(os.path.join(path, 'OEBPS', 'Images'), mode)
+        path = [path]
+        for tome in split:
+            path.append(tome)
+        return path
+    else:
+        # No splitting is necessary
+        return [path]
+
+
 def Copyright():
     print ('comic2ebook v%(__version__)s. '
            'Written 2013 by Ciro Mattia Gonano and Pawel Jastrzebski.' % globals())
@@ -699,7 +832,7 @@ def main(argv=None, qtGUI=None):
     parser.add_option("-c", "--cbz-output", action="store_true", dest="cbzoutput", default=False,
                       help="Outputs a CBZ archive and does not generate EPUB")
     parser.add_option("--noprocessing", action="store_false", dest="imgproc", default=True,
-                      help="Do not apply image preprocessing (Page splitting and optimizations) [Default=True]")
+                      help="Don't apply image preprocessing (Page splitting and optimizations) [Default=True]")
     parser.add_option("--forcepng", action="store_true", dest="forcepng", default=False,
                       help="Create PNG files instead JPEG (For non-Kindle devices) [Default=False]")
     parser.add_option("--gamma", type="float", dest="gamma", default="0.0",
@@ -716,15 +849,17 @@ def main(argv=None, qtGUI=None):
     parser.add_option("--nosplitrotate", action="store_true", dest="nosplitrotate", default=False,
                       help="Disable splitting and rotation [Default=False]")
     parser.add_option("--nocutpagenumbers", action="store_false", dest="cutpagenumbers", default=True,
-                      help="Do not try to cut page numbering on images [Default=True]")
+                      help="Don't try to cut page numbering on images [Default=True]")
-    parser.add_option("-o", "--output", action="store", dest="output", default=None,
-                      help="Output generated file (EPUB or CBZ) to specified directory or file")
     parser.add_option("--forcecolor", action="store_true", dest="forcecolor", default=False,
-                      help="Do not convert images to grayscale [Default=False]")
+                      help="Don't convert images to grayscale [Default=False]")
+    parser.add_option("--batchsplit", action="store_true", dest="batchsplit", default=False,
+                      help="Split output into multiple files [Default=False]"),
     parser.add_option("--customwidth", type="int", dest="customwidth", default=0,
                       help="Replace screen width provided by device profile [Default=0]")
     parser.add_option("--customheight", type="int", dest="customheight", default=0,
                       help="Replace screen height provided by device profile [Default=0]")
+    parser.add_option("-o", "--output", action="store", dest="output", default=None,
+                      help="Output generated file (EPUB or CBZ) to specified directory or file")
     parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
                       help="Verbose output [Default=False]")
     options, args = parser.parse_args(argv)
@@ -738,8 +873,6 @@ def main(argv=None, qtGUI=None):
         parser.print_help()
         return
     path = getWorkFolder(args[0])
-    if options.title == 'defaulttitle':
-        options.title = os.path.splitext(os.path.basename(args[0]))[0]
     splitCount = 0
     if options.imgproc:
         print "Processing images..."
@@ -748,23 +881,42 @@ def main(argv=None, qtGUI=None):
         dirImgProcess(path + "/OEBPS/Images/")
     if GUI:
         GUI.emit(QtCore.SIGNAL("progressBarTick"), 1)
-    if options.cbzoutput:
+    sanitizeTree(os.path.join(path, 'OEBPS', 'Images'))
-        # if CBZ output wanted, compress all images and return filepath
+    if options.batchsplit:
-        print "\nCreating CBZ file..."
+        tomes = preSplitDirectory(path)
-        filepath = getOutputFilename(args[0], options.output, '.cbz')
-        make_archive(path + '_comic', 'zip', path + '/OEBPS/Images')
     else:
-        print "\nCreating EPUB structure..."
+        tomes = [path]
-        genEpubStruct(path)
+    filepath = []
-        # actually zip the ePub
+    tomeNumber = 0
-        filepath = getOutputFilename(args[0], options.output, '.epub')
+    for tome in tomes:
-        make_archive(path + '_comic', 'zip', path)
+        if len(tomes) > 1:
-    move(path + '_comic.zip', filepath)
+            tomeNumber += 1
-    rmtree(path)
+            options.title = os.path.splitext(os.path.basename(args[0]))[0] + ' ' + str(tomeNumber)
+        elif options.title == 'defaulttitle':
+            options.title = os.path.splitext(os.path.basename(args[0]))[0]
+        if options.cbzoutput:
+            # if CBZ output wanted, compress all images and return filepath
+            print "\nCreating CBZ file..."
+            if len(tomes) > 1:
+                filepath.append(getOutputFilename(args[0], options.output, '.cbz', ' ' + str(tomeNumber)))
+            else:
+                filepath.append(getOutputFilename(args[0], options.output, '.cbz', ''))
+            make_archive(tome + '_comic', 'zip', tome + '/OEBPS/Images')
+        else:
+            print "\nCreating EPUB structure..."
+            genEpubStruct(tome)
+            # actually zip the ePub
+            if len(tomes) > 1:
+                filepath.append(getOutputFilename(args[0], options.output, '.epub', ' ' + str(tomeNumber)))
+            else:
+                filepath.append(getOutputFilename(args[0], options.output, '.epub', ''))
+            make_archive(tome + '_comic', 'zip', tome)
+        move(tome + '_comic.zip', filepath[-1])
+        rmtree(tome)
     return filepath
 
 
-def getOutputFilename(srcpath, wantedname, ext):
+def getOutputFilename(srcpath, wantedname, ext, tomeNumber):
     if not ext.startswith('.'):
         ext = '.' + ext
     if wantedname is not None:
@@ -773,14 +925,13 @@ def getOutputFilename(srcpath, wantedname, ext):
         elif os.path.isdir(srcpath):
             filename = os.path.abspath(options.output) + "/" + os.path.basename(srcpath) + ext
         else:
-            filename = os.path.abspath(options.output) + "/" \
+            filename = os.path.abspath(options.output) + "/" + os.path.basename(os.path.splitext(srcpath)[0]) + ext
-                       + os.path.basename(os.path.splitext(srcpath)[0]) + ext
     elif os.path.isdir(srcpath):
-        filename = srcpath + ext
+        filename = srcpath + tomeNumber + ext
     else:
-        filename = os.path.splitext(srcpath)[0] + ext
+        filename = os.path.splitext(srcpath)[0] + tomeNumber + ext
     if os.path.isfile(filename):
-        filename = os.path.splitext(filename)[0] + '_kcc' + ext
+        filename = os.path.splitext(filename)[0] + '_kcc' + tomeNumber + ext
     return filename