Refactored multiprocessing support

kcc/comic2ebook.py

@@ -31,7 +31,7 @@ import stat
 import string
 from shutil import move, copyfile, copytree, rmtree, make_archive
 from optparse import OptionParser, OptionGroup
-from multiprocessing import Pool, Queue, freeze_support
+from multiprocessing import Pool, freeze_support
 try:
     from PyQt4 import QtCore
 except ImportError:
@@ -308,105 +308,91 @@ def applyImgOptimization(img, opt, overrideQuality=5):
 
 
 def dirImgProcess(path):
+    global workerPool, workerOutput
+    workerPool = Pool()
+    workerOutput = []
     work = []
     pagenumber = 0
-    pagenumbermodifier = 0
-    queue = Queue()
-    pool = Pool(None, fileImgProcess_init, [queue, options])
     for (dirpath, dirnames, filenames) in os.walk(path):
         for afile in filenames:
             if getImageFileName(afile) is not None:
                 pagenumber += 1
-                work.append([afile, dirpath, pagenumber])
+                work.append([afile, dirpath, options])
     if GUI:
         GUI.emit(QtCore.SIGNAL("progressBarTick"), pagenumber)
     if len(work) > 0:
-        splitpages = pool.map_async(func=fileImgProcess, iterable=work)
+        for i in work:
-        pool.close()
+            workerPool.apply_async(func=fileImgProcess, args=(i, ), callback=fileImgProcess_tick)
-        if GUI:
+        workerPool.close()
-            while not splitpages.ready():
+        workerPool.join()
-                # noinspection PyBroadException
+        if GUI and not GUI.conversionAlive:
-                try:
-                    queue.get(True, 5)
-                except:
-                    pass
-                if not GUI.conversionAlive:
-                    pool.terminate()
-                    rmtree(os.path.join(path, '..', '..'), True)
-                    raise UserWarning("Conversion interrupted.")
-                GUI.emit(QtCore.SIGNAL("progressBarTick"))
-        pool.join()
-        queue.close()
-        try:
-            splitpages = splitpages.get()
-        except:
             rmtree(os.path.join(path, '..', '..'), True)
-            raise RuntimeError("One of workers crashed. Cause: " + str(sys.exc_info()[1]))
+            raise UserWarning("Conversion interrupted.")
-        splitpages = filter(None, splitpages)
+        if len(workerOutput) > 0:
-        splitpages.sort()
+            rmtree(os.path.join(path, '..', '..'), True)
-        for page in splitpages:
+            raise RuntimeError("One of workers crashed. Cause: " + workerOutput[0])
-            if (page + pagenumbermodifier) % 2 == 0:
-                pagenumbermodifier += 1
-            pagenumbermodifier += 1
     else:
         rmtree(os.path.join(path, '..', '..'), True)
         raise UserWarning("Source directory is empty.")
 
 
-def fileImgProcess_init(queue, opt):
+def fileImgProcess_tick(output):
-    fileImgProcess.queue = queue
+    if output:
-    fileImgProcess.options = opt
+        workerOutput.append(output)
+        workerPool.terminate()
+    if GUI:
+        GUI.emit(QtCore.SIGNAL("progressBarTick"))
+        if not GUI.conversionAlive:
+            workerPool.terminate()
 
 
-# noinspection PyUnresolvedReferences
 def fileImgProcess(work):
-    afile = work[0]
+    #noinspection PyBroadException
-    dirpath = work[1]
+    try:
-    pagenumber = work[2]
+        afile = work[0]
-    opt = fileImgProcess.options
+        dirpath = work[1]
-    output = None
+        opt = work[2]
-    if opt.verbose:
-        print "Optimizing " + afile + " for " + opt.profile
-    else:
-        print ".",
-    fileImgProcess.queue.put(".")
-    img = image.ComicPage(os.path.join(dirpath, afile), opt.profileData)
-    if opt.quality == 2:
-        wipe = False
-    else:
-        wipe = True
-    if opt.nosplitrotate:
-        split = None
-    else:
-        split = img.splitPage(dirpath, opt.righttoleft, opt.rotate)
-    if split is not None:
         if opt.verbose:
-            print "Splitted " + afile
+            print "Optimizing " + afile + " for " + opt.profile
-        output = pagenumber
+        else:
-        img0 = image.ComicPage(split[0], opt.profileData)
+            print ".",
-        applyImgOptimization(img0, opt)
+        img = image.ComicPage(os.path.join(dirpath, afile), opt.profileData)
-        img0.saveToDir(dirpath, opt.forcepng, opt.forcecolor, wipe)
-        img1 = image.ComicPage(split[1], opt.profileData)
-        applyImgOptimization(img1, opt)
-        img1.saveToDir(dirpath, opt.forcepng, opt.forcecolor, wipe)
         if opt.quality == 2:
-            img3 = image.ComicPage(split[0], opt.profileData)
+            wipe = False
-            applyImgOptimization(img3, opt, 0)
+        else:
-            img3.saveToDir(dirpath, opt.forcepng, opt.forcecolor, True)
+            wipe = True
-            img4 = image.ComicPage(split[1], opt.profileData)
+        if opt.nosplitrotate:
-            applyImgOptimization(img4, opt, 0)
+            split = None
-            img4.saveToDir(dirpath, opt.forcepng, opt.forcecolor, True)
+        else:
-    else:
+            split = img.splitPage(dirpath, opt.righttoleft, opt.rotate)
-        applyImgOptimization(img, opt)
+        if split is not None:
-        img.saveToDir(dirpath, opt.forcepng, opt.forcecolor, wipe)
+            if opt.verbose:
-        if opt.quality == 2:
+                print "Splitted " + afile
-            img2 = image.ComicPage(os.path.join(dirpath, afile), opt.profileData)
+            img0 = image.ComicPage(split[0], opt.profileData)
-            if img.rotated:
+            applyImgOptimization(img0, opt)
-                img2.image = img2.image.rotate(90)
+            img0.saveToDir(dirpath, opt.forcepng, opt.forcecolor, wipe)
-                img2.rotated = True
+            img1 = image.ComicPage(split[1], opt.profileData)
-            applyImgOptimization(img2, opt, 0)
+            applyImgOptimization(img1, opt)
-            img2.saveToDir(dirpath, opt.forcepng, opt.forcecolor, True)
+            img1.saveToDir(dirpath, opt.forcepng, opt.forcecolor, wipe)
-    return output
+            if opt.quality == 2:
+                img3 = image.ComicPage(split[0], opt.profileData)
+                applyImgOptimization(img3, opt, 0)
+                img3.saveToDir(dirpath, opt.forcepng, opt.forcecolor, True)
+                img4 = image.ComicPage(split[1], opt.profileData)
+                applyImgOptimization(img4, opt, 0)
+                img4.saveToDir(dirpath, opt.forcepng, opt.forcecolor, True)
+        else:
+            applyImgOptimization(img, opt)
+            img.saveToDir(dirpath, opt.forcepng, opt.forcecolor, wipe)
+            if opt.quality == 2:
+                img2 = image.ComicPage(os.path.join(dirpath, afile), opt.profileData)
+                if img.rotated:
+                    img2.image = img2.image.rotate(90)
+                    img2.rotated = True
+                applyImgOptimization(img2, opt, 0)
+                img2.saveToDir(dirpath, opt.forcepng, opt.forcecolor, True)
+    except:
+        return str(sys.exc_info()[1])
 
 
 def genEpubStruct(path):