diff options
Diffstat (limited to 'module/plugins')
| -rw-r--r-- | module/plugins/hooks/ExtractArchive.py | 161 | ||||
| -rw-r--r-- | module/plugins/internal/Extractor.py | 30 | ||||
| -rw-r--r-- | module/plugins/internal/SevenZip.py | 38 | ||||
| -rw-r--r-- | module/plugins/internal/UnRar.py | 71 | ||||
| -rw-r--r-- | module/plugins/internal/UnZip.py | 8 |
5 files changed, 170 insertions, 138 deletions
diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 3ea8839dc..a1e85ba57 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -104,13 +104,14 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.29" + __version__ = "1.30" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), ("overwrite" , "bool" , "Overwrite files" , False ), ("keepbroken" , "bool" , "Try to extract broken archives" , False ), - ("repair" , "bool" , "Repair broken archives" , True ), + ("repair" , "bool" , "Repair broken archives (rar required)" , False ), + ("test" , "bool" , "Test archive before extracting" , False ), ("usepasswordfile" , "bool" , "Use password file" , True ), ("passwordfile" , "file" , "Password file" , "archive_password.txt" ), ("delete" , "bool" , "Delete archive when successfully extracted", False ), @@ -128,35 +129,32 @@ class ExtractArchive(Hook): ("Immenz" , "immenz@gmx.net" )] - event_list = ["allDownloadsProcessed"] + event_list = ["allDownloadsProcessed","packageDeleted"] NAME_REPLACEMENTS = [(r'\.part\d+\.rar$', ".part.rar")] - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass - - def setup(self): self.queue = ArchiveQueue(self, "Queue") self.failed = ArchiveQueue(self, "Failed") - self.interval = 60 - self.extracting = False - self.extractors = [] - self.passwords = [] + self.interval = 60 + self.extracting = False + self.lastPackage = False + self.extractors = [] + self.passwords = [] + self.repair = False def coreReady(self): - # self.extracting = False - for p in ("UnRar", "SevenZip", "UnZip"): try: module = self.core.pluginManager.loadModule("internal", p) klass = getattr(module, p) if klass.isUsable(): self.extractors.append(klass) + if klass.REPAIR: + self.repair = self.getConfig("repair") except OSError, e: if e.errno == 2: @@ -173,37 +171,49 @@ class ExtractArchive(Hook): if self.extractors: self.logInfo(_("Activated") + " " + "|".join("%s %s" % (Extractor.__name__,Extractor.VERSION) for Extractor in self.extractors)) - - if self.getConfig("waitall"): - self.extractPackage(*self.queue.get()) #: Resume unfinished extractions - else: - super(ExtractArchive, self).initPeriodical() - + self.extractQueued() #: Resume unfinished extractions else: self.logInfo(_("No Extract plugins activated")) + @threaded + def extractQueued(self,thread): + packages = self.queue.get() + while packages: + if self.lastPackage: # called from allDownloadsProcessed + self.lastPackage = False + if self.extract(packages, thread): #@NOTE: check only if all gone fine, no failed reporting for now + self.manager.dispatchEvent("all_archives_extracted") + self.manager.dispatchEvent("all_archives_processed") + else: + if self.extract(packages, thread): #@NOTE: check only if all gone fine, no failed reporting for now + pass - def periodical(self): - if not self.extracting: - self.extractPackage(*self.queue.get()) + packages = self.queue.get() # check for packages added during extraction @Expose def extractPackage(self, *ids): """ Extract packages with given id""" - self.manager.startThread(self.extract, ids) + for id in ids: + self.queue.add(id) + if not self.getConfig("waitall") and not self.extracting: + self.extractQueued() + + + def packageDeleted(self, pid): + self.queue.remove(pid) def packageFinished(self, pypack): self.queue.add(pypack.id) + if not self.getConfig("waitall") and not self.extracting: + self.extractQueued() - @threaded - def allDownloadsProcessed(self, thread): - if self.extract(self.queue.get(), thread): #@NOTE: check only if all gone fine, no failed reporting for now - self.manager.dispatchEvent("all_archives_extracted") - - self.manager.dispatchEvent("all_archives_processed") + def allDownloadsProcessed(self): + self.lastPackage = True + if not self.extracting: + self.extractQueued() def extract(self, ids, thread=None): @@ -315,13 +325,17 @@ class ExtractArchive(Hook): if recursive and os.path.isfile(file): new_files_ids.append((filename, fid, os.path.dirname(filename))) # append as new target - + + pyfile = self.core.files.getFile(fid) + self.manager.dispatchEvent("archive_extracted", pyfile, archive.out, archive.filename, new_files) + files_ids = new_files_ids # also check extracted files if matched: if success: extracted.append(pid) self.manager.dispatchEvent("package_extracted", pypack) + else: failed.append(pid) self.manager.dispatchEvent("package_extract_failed", pypack) @@ -352,50 +366,65 @@ class ExtractArchive(Hook): encrypted = False try: - try: - archive.check() - - except CRCError, e: - self.logDebug(name, e) - self.logInfo(name, _("Header protected")) - - if self.getConfig("repair"): - self.logWarning(name, _("Repairing...")) - - pyfile.setCustomStatus(_("repairing")) - pyfile.setProgress(0) - - repaired = archive.repair() - - pyfile.setProgress(100) - - if not repaired and not self.getConfig("keepbroken"): - raise CRCError("Archive damaged") - - except PasswordError: - self.logInfo(name, _("Password protected")) - encrypted = True - - except ArchiveError, e: - raise ArchiveError(e) - - self.logDebug("Password: %s" % (password or "No provided")) + self.logDebug("Password: %s" % (password or "None provided")) + passwords = uniqify([password] + self.getPasswords(False)) if self.getConfig("usepasswordfile") else [password] + for pw in passwords: + try: + if self.getConfig("test") or self.repair: + pyfile.setCustomStatus(_("testing")) + if pw: + self.logDebug("Testing with password: %s" % pw) + pyfile.setProgress(0) + archive.test(pw) + pyfile.setProgress(100) + else: + archive.check(pw) + + self.addPassword(pw) + break + + except PasswordError: + if not encrypted: + self.logInfo(name, _("Password protected")) + encrypted = True + + except CRCError, e: + self.logDebug(name, e) + self.logInfo(name, _("CRC Error")) + + if self.repair: + self.logWarning(name, _("Repairing...")) + + pyfile.setCustomStatus(_("repairing")) + pyfile.setProgress(0) + repaired = archive.repair() + pyfile.setProgress(100) + + if not repaired and not self.getConfig("keepbroken"): + raise CRCError("Archive damaged") + + self.addPassword(pw) + break + + raise CRCError("Archive damaged") + + except ArchiveError, e: + raise ArchiveError(e) pyfile.setCustomStatus(_("extracting")) pyfile.setProgress(0) if not encrypted or not self.getConfig("usepasswordfile"): + self.logDebug("Extracting using password: %s" % (password or "None")) archive.extract(password) else: for pw in filter(None, uniqify([password] + self.getPasswords(False))): try: - self.logDebug("Try password: %s" % pw) + self.logDebug("Extracting using password: %s" % pw) - ispw = archive.isPassword(pw) - if ispw or ispw is None: - archive.extract(pw) - self.addPassword(pw) - break + archive.extract(pw) + self.addPassword(pw) + break except PasswordError: self.logDebug("Password was wrong") @@ -419,9 +448,7 @@ class ExtractArchive(Hook): self.logDebug("%s does not exists" % f) self.logInfo(name, _("Extracting finished")) - extracted_files = archive.files or archive.list() - self.manager.dispatchEvent("archive_extracted", pyfile, archive.out, archive.filename, extracted_files) return extracted_files diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index b445f1497..8bf1875cf 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -19,7 +19,7 @@ class PasswordError(Exception): class Extractor: __name__ = "Extractor" - __version__ = "0.20" + __version__ = "0.21" __description__ = """Base extractor plugin""" __license__ = "GPLv3" @@ -30,6 +30,7 @@ class Extractor: EXTENSIONS = [] VERSION = "" + REPAIR = False @classmethod @@ -90,23 +91,24 @@ class Extractor: def check(self): - """Check if password if needed. Raise ArchiveError if integrity is - questionable. + """Quick Check by listing content of archive. + Raises error if password is needed, integrity is questionable or else. - :return: boolean + :raises PasswordError + :raises CRCError :raises ArchiveError """ - raise PasswordError - - - def isPassword(self, password): - """ Check if the given password is/might be correct. - If it can not be decided at this point return true. - - :param password: - :return: boolean + raise NotImplementedError + + def test(self): + """Testing with Extractors buildt-in method + Raises error if password is needed, integrity is questionable or else. + + :raises PasswordError + :raises CRCError + :raises ArchiveError """ - return None + raise NotImplementedError def repair(self): diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index 7ad6b0d7a..bfa7f3943 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -11,7 +11,7 @@ from module.utils import fs_encode, save_join class SevenZip(UnRar): __name__ = "SevenZip" - __version__ = "0.08" + __version__ = "0.09" __description__ = """7-Zip extractor plugin""" __license__ = "GPLv3" @@ -32,9 +32,9 @@ class SevenZip(UnRar): #@NOTE: there are some more uncovered 7z formats re_filelist = re.compile(r'([\d\:]+)\s+([\d\:]+)\s+([\w\.]+)\s+(\d+)\s+(\d+)\s+(.+)') - re_wrongpwd = re.compile(r'(Can not open encrypted archive|Wrong password)', re.I) - re_wrongcrc = re.compile(r'Encrypted\s+\=\s+\+', re.I) - re_version = re.compile(r'7-Zip\s(?:\[64\]\s)?(\d+\.\d+)', re.I) + re_wrongpwd = re.compile(r'(Can not open encrypted archive|Wrong password|Encrypted\s+\=\s+\+)', re.I) + re_wrongcrc = re.compile(r'CRC Failed|Can not open file', re.I) + re_version = re.compile(r'7-Zip\s(?:\[64\]\s)?(\d+\.\d+)', re.I) @classmethod @@ -52,36 +52,38 @@ class SevenZip(UnRar): return True - def check(self): + def test(self, password): file = fs_encode(self.filename) - p = self.call_cmd("t", file) + # 7z can't distinguish crc and pw error in test + p = self.call_cmd("l", "-slt", file) out, err = p.communicate() - if p.returncode > 1: + if self.re_wrongpwd.search(out): + raise PasswordError + + if self.re_wrongpwd.search(err): + raise PasswordError + + if self.re_wrongcrc.search(err): raise CRCError(err) + + + def check(self, password): + file = fs_encode(self.filename) + p = self.call_cmd("l", "-slt", file) out, err = p.communicate() - if p.returncode > 1: - raise ArchiveError(_("Process return code: %d") % p.returncode) - # check if output or error macthes the 'wrong password'-Regexp if self.re_wrongpwd.search(out): raise PasswordError - # check if output matches 'Encrypted = +' if self.re_wrongcrc.search(out): raise CRCError(_("Header protected")) - def isPassword(self, password): - p = self.call_cmd("l", fs_encode(self.filename), password=password) - p.communicate() - return p.returncode == 0 - - def repair(self): return False @@ -142,7 +144,7 @@ class SevenZip(UnRar): #set a password if "password" in kwargs and kwargs["password"]: - args.append("-p'%s'" % kwargs["password"]) + args.append("-p%s" % kwargs["password"]) else: args.append("-p-") diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 54d64c430..ca8fdd326 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.13" + __version__ = "1.14" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" @@ -33,34 +33,40 @@ class UnRar(Extractor): CMD = "unrar" VERSION = "" - EXTENSIONS = [".rar"] - re_multipart = re.compile(r'\.(part|r)(\d+)(?:\.rar)?',re.I) + re_multipart = re.compile(r'\.(part|r)(\d+)(?:\.rar)?(\.rev|\.bad)?',re.I) re_filefixed = re.compile(r'Building (.+)') re_filelist = re.compile(r'^(.)(\s*[\w\.\-]+)\s+(\d+\s+)+(?:\d+\%\s+)?[\d\-]{8}\s+[\d\:]{5}', re.M|re.I) re_wrongpwd = re.compile(r'password', re.I) - re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I) + re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error|corrupt', re.I) - re_version = re.compile(r'UNRAR\s(\d+\.\d+)', re.I) + re_version = re.compile(r'(?:UN)?RAR\s(\d+\.\d+)', re.I) @classmethod def isUsable(cls): if os.name == "nt": - cls.CMD = os.path.join(pypath, "UnRAR.exe") - p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) - out, err = p.communicate() - else: try: + cls.CMD = os.path.join(pypath, "RAR.exe") p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) out, err = p.communicate() - - except OSError: #: fallback to rar - cls.CMD = "rar" + cls.__name__ = "RAR" + cls.REPAIR = True + except OSError: + cls.CMD = os.path.join(pypath, "UnRAR.exe") + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) + out, err = p.communicate() + else: + try: + p = Popen(["rar"], stdout=PIPE, stderr=PIPE) + out, err = p.communicate() + cls.__name__ = "RAR" + cls.REPAIR = True + except OSError: #: fallback to unrar p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) out, err = p.communicate() @@ -74,13 +80,25 @@ class UnRar(Extractor): multipart = cls.re_multipart.search(filename) if multipart: # First Multipart file (part1.rar for *.part1-9.rar format or *.rar for .r1-9 format) handled as normal Archive - return False if (multipart.group(1) == "part" and int(multipart.group(2)) == 1) else True + return False if (multipart.group(1) == "part" and int(multipart.group(2)) == 1 and not multipart.group(3)) else True return False - def check(self): - p = self.call_cmd("l", "-v", fs_encode(self.filename)) + def test(self, password): + p = self.call_cmd("t", "-v", fs_encode(self.filename), password=password) + self._progress(p) + err = p.stderr.read().strip() + + if self.re_wrongpwd.search(err): + raise PasswordError + + if self.re_wrongcrc.search(err): + raise CRCError(err) + + + def check(self, password): + p = self.call_cmd("l", "-v", fs_encode(self.filename), password=password) out, err = p.communicate() if self.re_wrongpwd.search(err): @@ -95,35 +113,14 @@ class UnRar(Extractor): raise PasswordError - def isPassword(self, password): - # at this point we can only verify header protected files - p = self.call_cmd("l", "-v", fs_encode(self.filename), password=password) - out, err = p.communicate() - return False if self.re_wrongpwd.search(err) else True - - def repair(self): p = self.call_cmd("rc", fs_encode(self.filename)) # communicate and retrieve stderr self._progress(p) err = p.stderr.read().strip() - if err or p.returncode: - p = self.call_cmd("r", fs_encode(self.filename)) - - # communicate and retrieve stderr - self._progress(p) - err = p.stderr.read().strip() - - if err or p.returncode: - return False - else: - dir = os.path.dirname(filename) - name = re_filefixed.search(out).group(1) - - self.filename = os.path.join(dir, name) - + return False return True diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index 704b49ca0..d95afbc70 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -12,7 +12,7 @@ from module.utils import fs_encode class UnZip(Extractor): __name__ = "UnZip" - __version__ = "1.10" + __version__ = "1.11" __description__ = """Zip extractor plugin""" __license__ = "GPLv3" @@ -34,7 +34,11 @@ class UnZip(Extractor): return z.namelist() - def check(self): + def check(self, password): + pass + + + def test(self): with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z: badfile = z.testzip() |