From 8bb189cf5495b293cf574e5049c34a64ae36ffe1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 16 Dec 2014 23:04:15 +0100 Subject: Spare code cosmetics --- module/plugins/internal/UnRar.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index c15a4c96e..716a62613 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(AbtractExtractor): __name__ = "UnRar" - __version__ = "0.19" + __version__ = "0.20" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" @@ -32,7 +32,7 @@ class UnRar(AbtractExtractor): CMD = "unrar" # there are some more uncovered rar formats - re_version = re.compile(r'UNRAR ([\w .]+?) freeware') + re_version = re.compile(r'UNRAR ([\w .]+?)') re_splitfile = re.compile(r'(.*)\.part(\d+)\.rar$', re.I) re_partfiles = re.compile(r'.*\.(rar|r\d+)', re.I) re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+') @@ -91,7 +91,7 @@ class UnRar(AbtractExtractor): out, err = p.communicate() if self.re_wrongpwd.search(err): self.passwordProtected = True - self.headerProtected = True + self.headerProtected = True return True # output only used to check if passworded files are present @@ -124,7 +124,7 @@ class UnRar(AbtractExtractor): return True - def extract(self, progress, password=None): + def extract(self, progress, password=""): command = "x" if self.fullpath else "e" p = self.call_unrar(command, self.file, self.out, password=password) @@ -156,6 +156,7 @@ class UnRar(AbtractExtractor): raise CRCError elif "CRC failed" in err: raise WrongPassword + if err.strip(): #: raise error if anything is on stderr raise ArchiveError(err.strip()) if p.returncode: -- cgit v1.2.3 From da9e6c949243613f4d5e100cac6ff192449b4718 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 18 Dec 2014 18:45:17 +0100 Subject: Update extractor plugins --- module/plugins/internal/UnRar.py | 54 ++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 30 deletions(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 716a62613..4bbd2042c 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,53 +22,52 @@ def renice(pid, value): class UnRar(AbtractExtractor): __name__ = "UnRar" - __version__ = "0.20" + __version__ = "0.21" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org")] + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] CMD = "unrar" - # there are some more uncovered rar formats - re_version = re.compile(r'UNRAR ([\w .]+?)') + #@NOTE: there are some more uncovered rar formats re_splitfile = re.compile(r'(.*)\.part(\d+)\.rar$', re.I) re_partfiles = re.compile(r'.*\.(rar|r\d+)', re.I) - re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+') - re_filelist5 = re.compile(r'(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)') + re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+|(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)') re_wrongpwd = re.compile(r'(Corrupt file or wrong password|password incorrect)', re.I) - @staticmethod - def checkDeps(): + @classmethod + def checkDeps(cls): if os.name == "nt": - UnRar.CMD = join(pypath, "UnRAR.exe") - p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) + cls.CMD = join(pypath, "UnRAR.exe") + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) p.communicate() else: try: - p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) p.communicate() - except OSError: + except OSError: # fallback to rar - UnRar.CMD = "rar" - p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) + cls.CMD = "rar" + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) p.communicate() return True - @staticmethod - def getTargets(files_ids): + @classmethod + def getTargets(cls, files_ids): result = [] for file, id in files_ids: if not file.endswith(".rar"): continue - match = UnRar.re_splitfile.findall(file) + match = cls.re_splitfile.findall(file) if match: # only add first parts if int(match[0][1]) == 1: @@ -81,9 +80,8 @@ class UnRar(AbtractExtractor): def init(self): self.passwordProtected = False - self.headerProtected = False #: list files will not work without password - self.smallestFile = None #: small file to test passwords - self.password = "" #: save the correct password + self.headerProtected = False #: list files will not work without password + self.password = "" #: save the correct password def checkArchive(self): @@ -95,16 +93,10 @@ class UnRar(AbtractExtractor): return True # output only used to check if passworded files are present - if self.re_version.search(out): - for attr, size, name in self.re_filelist5.findall(out): - if attr.startswith("*"): - self.passwordProtected = True - return True - else: - for name, size, packed in self.re_filelist.findall(out): - if name.startswith("*"): - self.passwordProtected = True - return True + for attr in self.re_filelist.findall(out): + if attr[0].startswith("*"): + self.passwordProtected = True + return True self.listContent() if not self.files: @@ -131,6 +123,7 @@ class UnRar(AbtractExtractor): renice(p.pid, self.renice) progress(0) + progressstring = "" while True: c = p.stdout.read(1) @@ -147,6 +140,7 @@ class UnRar(AbtractExtractor): # add digit to progressstring else: progressstring = progressstring + c + progress(100) # retrieve stderr -- cgit v1.2.3 From 29df1397bbbe80eced4674b6fa39e16540c80901 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 24 Dec 2014 01:11:44 +0100 Subject: Extractor rewritten --- module/plugins/internal/UnRar.py | 179 +++++++++++++++++++++++---------------- 1 file changed, 108 insertions(+), 71 deletions(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 4bbd2042c..5633b31f7 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -4,11 +4,11 @@ import os import re from glob import glob -from os.path import basename, join +from os.path import basename, dirname, join from string import digits from subprocess import Popen, PIPE -from module.plugins.internal.AbstractExtractor import AbtractExtractor, WrongPassword, ArchiveError, CRCError +from module.plugins.internal.AbstractExtractor import AbtractExtractor, PasswordError, ArchiveError, CRCError from module.utils import save_join, decode @@ -22,21 +22,25 @@ def renice(pid, value): class UnRar(AbtractExtractor): __name__ = "UnRar" - __version__ = "0.21" + __version__ = "1.00" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org"), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] CMD = "unrar" + EXTENSIONS = ["rar", "zip", "cab", "arj", "lzh", "tar", "gz", "bz2", "ace", "uue", "jar", "iso", "7z", "xz", "z"] + + #@NOTE: there are some more uncovered rar formats - re_splitfile = re.compile(r'(.*)\.part(\d+)\.rar$', re.I) - re_partfiles = re.compile(r'.*\.(rar|r\d+)', re.I) + re_rarpart = re.compile(r'(.*)\.part(\d+)\.rar$', re.I) + re_rarfile = re.compile(r'.*\.(rar|r\d+)$', re.I) + re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+|(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)') - re_wrongpwd = re.compile(r'(Corrupt file or wrong password|password incorrect)', re.I) + re_wrongpwd = re.compile(r'password', re.I) + re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I) @classmethod @@ -59,70 +63,100 @@ class UnRar(AbtractExtractor): return True + @classmethod + def isArchive(cls, file): + f = basename(file).lower() + return any(f.endswith('.%s' % ext) for ext in cls.EXTENSIONS) + + @classmethod def getTargets(cls, files_ids): - result = [] + targets = [] for file, id in files_ids: - if not file.endswith(".rar"): + if not cls.isArchive(file): continue - match = cls.re_splitfile.findall(file) - if match: + m = cls.re_rarpart.findall(file) + if m: # only add first parts - if int(match[0][1]) == 1: - result.append((file, id)) + if int(m[0][1]) == 1: + targets.append((file, id)) else: - result.append((file, id)) + targets.append((file, id)) - return result + return targets - def init(self): - self.passwordProtected = False - self.headerProtected = False #: list files will not work without password - self.password = "" #: save the correct password + def check(self, out="", err=""): + if not out or not err: + return + if err.strip(): + if self.re_wrongpwd.search(err): + raise PasswordError - def checkArchive(self): - p = self.call_unrar("l", "-v", self.file) - out, err = p.communicate() - if self.re_wrongpwd.search(err): - self.passwordProtected = True - self.headerProtected = True - return True + elif self.re_wrongcrc.search(err): + raise CRCError + + else: #: raise error if anything is on stderr + raise ArchiveError(err.strip()) # output only used to check if passworded files are present for attr in self.re_filelist.findall(out): if attr[0].startswith("*"): - self.passwordProtected = True - return True + raise PasswordError + - self.listContent() - if not self.files: - raise ArchiveError("Empty Archive") + def verify(self): + p = self.call_cmd("l", "-v", self.file, password=self.password) + + self.check(*p.communicate()) + + if p and p.returncode: + raise ArchiveError("Process terminated") + + if not self.list(): + raise ArchiveError("Empty archive") + + + def isPassword(self, password): + if isinstance(password, basestring): + p = self.call_cmd("l", "-v", self.file, password=password) + out, err = p.communicate() + + if not self.re_wrongpwd.search(err): + return True return False - def checkPassword(self, password): - # at this point we can only verify header protected files - if self.headerProtected: - p = self.call_unrar("l", "-v", self.file, password=password) + def repair(self): + p = self.call_cmd("rc", self.file) + out, err = p.communicate() + + if p.returncode or err.strip(): + p = self.call_cmd("r", self.file) out, err = p.communicate() - if self.re_wrongpwd.search(err): + + if p.returncode or err.strip(): return False + else: + self.file = join(dirname(self.file), re.search(r'(fixed|rebuild)\.%s' % basename(self.file), out).group(0)) return True - def extract(self, progress, password=""): + def extract(self, progress=lambda x: None): + self.verify() + + progress(0) + command = "x" if self.fullpath else "e" - p = self.call_unrar(command, self.file, self.out, password=password) - renice(p.pid, self.renice) + p = self.call_cmd(command, self.file, self.out, password=self.password) - progress(0) + renice(p.pid, self.renice) progressstring = "" while True: @@ -131,7 +165,7 @@ class UnRar(AbtractExtractor): if not c: break # reading a percentage sign -> set progress and restart - if c == '%': + if c is '%': progress(int(progressstring)) progressstring = "" # not reading a digit -> therefore restart @@ -139,46 +173,43 @@ class UnRar(AbtractExtractor): progressstring = "" # add digit to progressstring else: - progressstring = progressstring + c + progressstring += c progress(100) - # retrieve stderr - err = p.stderr.read() + self.files = self.list() - if "CRC failed" in err and not password and not self.passwordProtected: - raise CRCError - elif "CRC failed" in err: - raise WrongPassword + # retrieve stderr + self.check(err=p.stderr.read()) - if err.strip(): #: raise error if anything is on stderr - raise ArchiveError(err.strip()) if p.returncode: raise ArchiveError("Process terminated") - if not self.files: - self.password = password - self.listContent() - def getDeleteFiles(self): if ".part" in basename(self.file): return glob(re.sub("(?<=\.part)([01]+)", "*", self.file, re.I)) + # get files which matches .r* and filter unsuited files out parts = glob(re.sub(r"(?<=\.r)ar$", "*", self.file, re.I)) - return filter(lambda x: self.re_partfiles.match(x), parts) + return filter(lambda x: self.re_rarfile.match(x), parts) - def listContent(self): + + def list(self): command = "vb" if self.fullpath else "lb" - p = self.call_unrar(command, "-v", self.file, password=self.password) + + p = self.call_cmd(command, "-v", self.file, password=self.password) out, err = p.communicate() - if "Cannot open" in err: - raise ArchiveError("Cannot open file") + if err.strip(): + self.m.logError(err) + if "Cannot open" in err: + return list() - if err.strip(): #: only log error at this point - self.m.logError(err.strip()) + if p.returncode: + self.m.logError("Process terminated") + return list() result = set() @@ -186,17 +217,22 @@ class UnRar(AbtractExtractor): f = f.strip() result.add(save_join(self.out, f)) - self.files = result + return list(result) - def call_unrar(self, command, *xargs, **kwargs): + def call_cmd(self, command, *xargs, **kwargs): args = [] + # overwrite flag - args.append("-o+") if self.overwrite else args.append("-o-") + if self.overwrite: + args.append("-o+") + else: + args.append("-o-") + if self.delete: + args.append("-or") - if self.excludefiles: - for word in self.excludefiles.split(';'): - args.append("-x%s" % word) + for word in self.excludefiles: + args.append("-x%s" % word.strip()) # assume yes on all queries args.append("-y") @@ -207,10 +243,11 @@ class UnRar(AbtractExtractor): else: args.append("-p-") + if self.keepbroken: + args.append("-kb") + # NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue call = [self.CMD, command] + args + list(xargs) self.m.logDebug(" ".join(call)) - p = Popen(call, stdout=PIPE, stderr=PIPE) - - return p + return Popen(call, stdout=PIPE, stderr=PIPE) -- cgit v1.2.3 From f71c1ef70a199e42e8a519364d9924e138ffd37c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 24 Dec 2014 15:48:08 +0100 Subject: [ExtractArchive] Remove empty directory --- module/plugins/internal/UnRar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 5633b31f7..b70bf3257 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -8,7 +8,7 @@ from os.path import basename, dirname, join from string import digits from subprocess import Popen, PIPE -from module.plugins.internal.AbstractExtractor import AbtractExtractor, PasswordError, ArchiveError, CRCError +from module.plugins.internal.AbstractExtractor import AbtractExtractor, ArchiveError, CRCError, PasswordError from module.utils import save_join, decode -- cgit v1.2.3 From 4c63928557398891c30d3e2b7c962a07b3483315 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 26 Dec 2014 04:18:41 +0100 Subject: Rename AbstractExtractor to Extractor --- module/plugins/internal/UnRar.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index b70bf3257..572fe95b9 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -8,7 +8,7 @@ from os.path import basename, dirname, join from string import digits from subprocess import Popen, PIPE -from module.plugins.internal.AbstractExtractor import AbtractExtractor, ArchiveError, CRCError, PasswordError +from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError from module.utils import save_join, decode @@ -20,9 +20,9 @@ def renice(pid, value): print "Renice failed" -class UnRar(AbtractExtractor): +class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.00" + __version__ = "1.01" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" -- cgit v1.2.3 From e1baccf1ec914563d3b2b845906cce024e7cd3b1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 13 Jan 2015 23:14:50 +0100 Subject: Replace 'except' with 'except Exception' --- module/plugins/internal/UnRar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 572fe95b9..296405101 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -16,7 +16,7 @@ def renice(pid, value): if os.name != "nt" and value: try: Popen(["renice", str(value), str(pid)], stdout=PIPE, stderr=PIPE, bufsize=-1) - except: + except Exception: print "Renice failed" -- cgit v1.2.3 From 7fc3362307737cd7c565b710ec83c5bdc4d3e8a9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 25 Jan 2015 03:05:00 +0100 Subject: Revert Extractor to the old one (temp) --- module/plugins/internal/UnRar.py | 216 +++++++++++++++++---------------------- 1 file changed, 92 insertions(+), 124 deletions(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 296405101..a1b438e47 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -4,11 +4,11 @@ import os import re from glob import glob -from os.path import basename, dirname, join +from os.path import basename, join from string import digits from subprocess import Popen, PIPE -from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError +from module.plugins.internal.Extractor import Extractor, WrongPassword, ArchiveError, CRCError from module.utils import save_join, decode @@ -22,142 +22,115 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.01" + __version__ = "1.02" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("RaNaN", "RaNaN@pyload.org")] CMD = "unrar" - EXTENSIONS = ["rar", "zip", "cab", "arj", "lzh", "tar", "gz", "bz2", "ace", "uue", "jar", "iso", "7z", "xz", "z"] + # there are some more uncovered rar formats + re_version = re.compile(r"(UNRAR 5[\d.]+(.*?)freeware)") + re_splitfile = re.compile(r"(.*)\.part(\d+)\.rar$", re.I) + re_partfiles = re.compile(r".*\.(rar|r\d+)", re.I) + re_filelist = re.compile(r"(.+)\s+(\d+)\s+(\d+)\s+") + re_filelist5 = re.compile(r"(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)") + re_wrongpwd = re.compile("(Corrupt file or wrong password|password incorrect)", re.I) - #@NOTE: there are some more uncovered rar formats - re_rarpart = re.compile(r'(.*)\.part(\d+)\.rar$', re.I) - re_rarfile = re.compile(r'.*\.(rar|r\d+)$', re.I) - - re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+|(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)') - re_wrongpwd = re.compile(r'password', re.I) - re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I) - - - @classmethod - def checkDeps(cls): + @staticmethod + def checkDeps(): if os.name == "nt": - cls.CMD = join(pypath, "UnRAR.exe") - p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) + UnRar.CMD = join(pypath, "UnRAR.exe") + p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) p.communicate() else: try: - p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) + p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) p.communicate() - except OSError: + # fallback to rar - cls.CMD = "rar" - p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) + UnRar.CMD = "rar" + p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) p.communicate() return True - @classmethod - def isArchive(cls, file): - f = basename(file).lower() - return any(f.endswith('.%s' % ext) for ext in cls.EXTENSIONS) - - - @classmethod - def getTargets(cls, files_ids): - targets = [] + @staticmethod + def getTargets(files_ids): + result = [] for file, id in files_ids: - if not cls.isArchive(file): + if not file.endswith(".rar"): continue - m = cls.re_rarpart.findall(file) - if m: + match = UnRar.re_splitfile.findall(file) + if match: # only add first parts - if int(m[0][1]) == 1: - targets.append((file, id)) + if int(match[0][1]) == 1: + result.append((file, id)) else: - targets.append((file, id)) - - return targets + result.append((file, id)) + return result - def check(self, out="", err=""): - if not out or not err: - return - if err.strip(): - if self.re_wrongpwd.search(err): - raise PasswordError + def init(self): + self.passwordProtected = False + self.headerProtected = False #: list files will not work without password + self.smallestFile = None #: small file to test passwords + self.password = "" #: save the correct password - elif self.re_wrongcrc.search(err): - raise CRCError - else: #: raise error if anything is on stderr - raise ArchiveError(err.strip()) + def checkArchive(self): + p = self.call_unrar("l", "-v", self.file) + out, err = p.communicate() + if self.re_wrongpwd.search(err): + self.passwordProtected = True + self.headerProtected = True + return True # output only used to check if passworded files are present - for attr in self.re_filelist.findall(out): - if attr[0].startswith("*"): - raise PasswordError - - - def verify(self): - p = self.call_cmd("l", "-v", self.file, password=self.password) - - self.check(*p.communicate()) - - if p and p.returncode: - raise ArchiveError("Process terminated") - - if not self.list(): - raise ArchiveError("Empty archive") - - - def isPassword(self, password): - if isinstance(password, basestring): - p = self.call_cmd("l", "-v", self.file, password=password) - out, err = p.communicate() + if self.re_version.search(out): + for attr, size, name in self.re_filelist5.findall(out): + if attr.startswith("*"): + self.passwordProtected = True + return True + else: + for name, size, packed in self.re_filelist.findall(out): + if name.startswith("*"): + self.passwordProtected = True + return True - if not self.re_wrongpwd.search(err): - return True + self.listContent() + if not self.files: + raise ArchiveError("Empty Archive") return False - def repair(self): - p = self.call_cmd("rc", self.file) - out, err = p.communicate() - - if p.returncode or err.strip(): - p = self.call_cmd("r", self.file) + def checkPassword(self, password): + # at this point we can only verify header protected files + if self.headerProtected: + p = self.call_unrar("l", "-v", self.file, password=password) out, err = p.communicate() - - if p.returncode or err.strip(): + if self.re_wrongpwd.search(err): return False - else: - self.file = join(dirname(self.file), re.search(r'(fixed|rebuild)\.%s' % basename(self.file), out).group(0)) return True - def extract(self, progress=lambda x: None): - self.verify() - - progress(0) - + def extract(self, progress, password=None): command = "x" if self.fullpath else "e" - p = self.call_cmd(command, self.file, self.out, password=self.password) - + p = self.call_unrar(command, self.file, self.out, password=password) renice(p.pid, self.renice) + progress(0) progressstring = "" while True: c = p.stdout.read(1) @@ -165,7 +138,7 @@ class UnRar(Extractor): if not c: break # reading a percentage sign -> set progress and restart - if c is '%': + if c == '%': progress(int(progressstring)) progressstring = "" # not reading a digit -> therefore restart @@ -173,43 +146,44 @@ class UnRar(Extractor): progressstring = "" # add digit to progressstring else: - progressstring += c - + progressstring = progressstring + c progress(100) - self.files = self.list() - # retrieve stderr - self.check(err=p.stderr.read()) - + err = p.stderr.read() + + if "CRC failed" in err and not password and not self.passwordProtected: + raise CRCError + elif "CRC failed" in err: + raise WrongPassword + if err.strip(): #: raise error if anything is on stderr + raise ArchiveError(err.strip()) if p.returncode: raise ArchiveError("Process terminated") + if not self.files: + self.password = password + self.listContent() + def getDeleteFiles(self): if ".part" in basename(self.file): return glob(re.sub("(?<=\.part)([01]+)", "*", self.file, re.I)) - # get files which matches .r* and filter unsuited files out parts = glob(re.sub(r"(?<=\.r)ar$", "*", self.file, re.I)) + return filter(lambda x: self.re_partfiles.match(x), parts) - return filter(lambda x: self.re_rarfile.match(x), parts) - - def list(self): + def listContent(self): command = "vb" if self.fullpath else "lb" - - p = self.call_cmd(command, "-v", self.file, password=self.password) + p = self.call_unrar(command, "-v", self.file, password=self.password) out, err = p.communicate() - if err.strip(): - self.m.logError(err) - if "Cannot open" in err: - return list() + if "Cannot open" in err: + raise ArchiveError("Cannot open file") - if p.returncode: - self.m.logError("Process terminated") - return list() + if err.strip(): #: only log error at this point + self.m.logError(err.strip()) result = set() @@ -217,22 +191,17 @@ class UnRar(Extractor): f = f.strip() result.add(save_join(self.out, f)) - return list(result) + self.files = result - def call_cmd(self, command, *xargs, **kwargs): + def call_unrar(self, command, *xargs, **kwargs): args = [] - # overwrite flag - if self.overwrite: - args.append("-o+") - else: - args.append("-o-") - if self.delete: - args.append("-or") + args.append("-o+") if self.overwrite else args.append("-o-") - for word in self.excludefiles: - args.append("-x%s" % word.strip()) + if self.excludefiles: + for word in self.excludefiles.split(';'): + args.append("-x%s" % word) # assume yes on all queries args.append("-y") @@ -243,11 +212,10 @@ class UnRar(Extractor): else: args.append("-p-") - if self.keepbroken: - args.append("-kb") - # NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue call = [self.CMD, command] + args + list(xargs) self.m.logDebug(" ".join(call)) - return Popen(call, stdout=PIPE, stderr=PIPE) + p = Popen(call, stdout=PIPE, stderr=PIPE) + + return p -- cgit v1.2.3 From 99ed44b0d919a33e3a559472893163b347cd1c37 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 25 Jan 2015 19:09:21 +0100 Subject: Fix reverted Extractor --- module/plugins/internal/UnRar.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index a1b438e47..43592c3de 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.02" + __version__ = "1.03" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" @@ -32,12 +32,13 @@ class UnRar(Extractor): CMD = "unrar" # there are some more uncovered rar formats - re_version = re.compile(r"(UNRAR 5[\d.]+(.*?)freeware)") - re_splitfile = re.compile(r"(.*)\.part(\d+)\.rar$", re.I) - re_partfiles = re.compile(r".*\.(rar|r\d+)", re.I) - re_filelist = re.compile(r"(.+)\s+(\d+)\s+(\d+)\s+") - re_filelist5 = re.compile(r"(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)") - re_wrongpwd = re.compile("(Corrupt file or wrong password|password incorrect)", re.I) + re_version = re.compile(r'UNRAR 5[\d.]+') + re_splitfile = re.compile(r'(.*)\.part(\d+)\.rar$', re.I) + re_partfiles = re.compile(r'.*\.(rar|r\d+)$', re.I) + re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+') + re_filelist5 = re.compile(r'(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)') + re_wrongpwd = re.compile(r'Corrupt file or wrong password|password incorrect', re.I) + re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I) @staticmethod @@ -81,9 +82,9 @@ class UnRar(Extractor): def init(self): self.passwordProtected = False - self.headerProtected = False #: list files will not work without password - self.smallestFile = None #: small file to test passwords - self.password = "" #: save the correct password + self.headerProtected = False #: list files will not work without password + self.smallestFile = None #: small file to test passwords + self.password = "" #: save the correct password def checkArchive(self): @@ -152,12 +153,15 @@ class UnRar(Extractor): # retrieve stderr err = p.stderr.read() - if "CRC failed" in err and not password and not self.passwordProtected: - raise CRCError - elif "CRC failed" in err: + if self.re_wrongpwd.search(err): raise WrongPassword - if err.strip(): #: raise error if anything is on stderr + + elif self.re_wrongcrc.search(err): + raise CRCError + + elif err.strip(): #: raise error if anything is on stderr raise ArchiveError(err.strip()) + if p.returncode: raise ArchiveError("Process terminated") -- cgit v1.2.3 From 8848a359a43316fb346b728d1d79d7b72d27e5a0 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 27 Jan 2015 17:48:25 +0100 Subject: Update Extractor (again) --- module/plugins/internal/UnRar.py | 175 +++++++++++++++++++-------------------- 1 file changed, 85 insertions(+), 90 deletions(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 43592c3de..7f1b08caf 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -4,110 +4,92 @@ import os import re from glob import glob -from os.path import basename, join from string import digits from subprocess import Popen, PIPE -from module.plugins.internal.Extractor import Extractor, WrongPassword, ArchiveError, CRCError +from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError from module.utils import save_join, decode def renice(pid, value): - if os.name != "nt" and value: + if value and os.name != "nt": try: Popen(["renice", str(value), str(pid)], stdout=PIPE, stderr=PIPE, bufsize=-1) except Exception: - print "Renice failed" + pass class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.03" + __version__ = "1.04" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org")] + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] CMD = "unrar" - # there are some more uncovered rar formats - re_version = re.compile(r'UNRAR 5[\d.]+') - re_splitfile = re.compile(r'(.*)\.part(\d+)\.rar$', re.I) - re_partfiles = re.compile(r'.*\.(rar|r\d+)$', re.I) - re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+') - re_filelist5 = re.compile(r'(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)') - re_wrongpwd = re.compile(r'Corrupt file or wrong password|password incorrect', re.I) + EXTENSIONS = [".rar", ".zip", ".cab", ".arj", ".lzh", ".tar", ".gz", ".bz2", + ".ace", ".uue", ".jar", ".iso", ".7z", ".xz", ".z"] + + #@NOTE: there are some more uncovered rar formats + re_rarpart1 = re.compile(r'\.part(\d+)\.rar$', re.I) + re_rarpart2 = re.compile(r'\.r(\d+)$', re.I) + + re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+|(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)') + re_wrongpwd = re.compile(r'password', re.I) re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I) - @staticmethod - def checkDeps(): + @classmethod + def checkDeps(cls): if os.name == "nt": - UnRar.CMD = join(pypath, "UnRAR.exe") - p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) + cls.CMD = os.path.join(pypath, "UnRAR.exe") + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) p.communicate() else: try: - p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) p.communicate() - except OSError: - # fallback to rar - UnRar.CMD = "rar" - p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) + except OSError: #: fallback to rar + cls.CMD = "rar" + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) p.communicate() return True - @staticmethod - def getTargets(files_ids): - result = [] + @classmethod + def getTargets(cls, files_ids): + targets = [] - for file, id in files_ids: - if not file.endswith(".rar"): + for filename, id in files_ids: + if not cls.isArchive(filename): continue - match = UnRar.re_splitfile.findall(file) - if match: - # only add first parts - if int(match[0][1]) == 1: - result.append((file, id)) - else: - result.append((file, id)) - - return result - + m = cls.re_rarpart1.match(filename) + if not m or int(m.group(1)) is 1: #@NOTE: only add first part file + targets.append((filename, id)) - def init(self): - self.passwordProtected = False - self.headerProtected = False #: list files will not work without password - self.smallestFile = None #: small file to test passwords - self.password = "" #: save the correct password + return targets def checkArchive(self): - p = self.call_unrar("l", "-v", self.file) + p = self.call_cmd("l", "-v", self.target) out, err = p.communicate() + if self.re_wrongpwd.search(err): - self.passwordProtected = True - self.headerProtected = True return True # output only used to check if passworded files are present - if self.re_version.search(out): - for attr, size, name in self.re_filelist5.findall(out): - if attr.startswith("*"): - self.passwordProtected = True - return True - else: - for name, size, packed in self.re_filelist.findall(out): - if name.startswith("*"): - self.passwordProtected = True - return True + for attr in self.re_filelist.findall(out): + if attr[0].startswith("*"): + return True - self.listContent() + self.files = self.list() if not self.files: raise ArchiveError("Empty Archive") @@ -116,22 +98,18 @@ class UnRar(Extractor): def checkPassword(self, password): # at this point we can only verify header protected files - if self.headerProtected: - p = self.call_unrar("l", "-v", self.file, password=password) - out, err = p.communicate() - if self.re_wrongpwd.search(err): - return False - - return True + p = self.call_cmd("l", "-v", self.target, password=password) + out, err = p.communicate() + return False if self.re_wrongpwd.search(err) else True - def extract(self, progress, password=None): + def extract(self, password=None): command = "x" if self.fullpath else "e" - p = self.call_unrar(command, self.file, self.out, password=password) + p = self.call_cmd(command, self.target, self.out, password=password) + renice(p.pid, self.renice) - progress(0) progressstring = "" while True: c = p.stdout.read(1) @@ -140,21 +118,20 @@ class UnRar(Extractor): break # reading a percentage sign -> set progress and restart if c == '%': - progress(int(progressstring)) + self.notifyProgress(int(progressstring)) progressstring = "" # not reading a digit -> therefore restart elif c not in digits: progressstring = "" # add digit to progressstring else: - progressstring = progressstring + c - progress(100) + progressstring += c # retrieve stderr err = p.stderr.read() if self.re_wrongpwd.search(err): - raise WrongPassword + raise PasswordError elif self.re_wrongcrc.search(err): raise CRCError @@ -162,50 +139,65 @@ class UnRar(Extractor): elif err.strip(): #: raise error if anything is on stderr raise ArchiveError(err.strip()) - if p.returncode: + if p.returncode != 0: raise ArchiveError("Process terminated") if not self.files: - self.password = password - self.listContent() + self.files = self.list(password) def getDeleteFiles(self): - if ".part" in basename(self.file): - return glob(re.sub("(?<=\.part)([01]+)", "*", self.file, re.I)) - # get files which matches .r* and filter unsuited files out - parts = glob(re.sub(r"(?<=\.r)ar$", "*", self.file, re.I)) - return filter(lambda x: self.re_partfiles.match(x), parts) + files = [] + + for i in [1, 2]: + try: + dir, name = os.path.split(self.target) + part = self.getattr(self, "re_rarpart%d" % i).match(name).group(1) + filename = os.path.join(dir, name.replace(part, '*', 1)) + files.extend(glob(filename)) + + except Exception: + continue + if self.target not in files: + files.insert(0, self.target) - def listContent(self): + return files + + + def list(self, password=None): command = "vb" if self.fullpath else "lb" - p = self.call_unrar(command, "-v", self.file, password=self.password) + + p = self.call_cmd(command, "-v", self.target, password=password) out, err = p.communicate() if "Cannot open" in err: raise ArchiveError("Cannot open file") if err.strip(): #: only log error at this point - self.m.logError(err.strip()) + self.manager.logError(err.strip()) result = set() - for f in decode(out).splitlines(): f = f.strip() result.add(save_join(self.out, f)) - self.files = result + return list(result) - def call_unrar(self, command, *xargs, **kwargs): + def call_cmd(self, command, *xargs, **kwargs): args = [] + # overwrite flag - args.append("-o+") if self.overwrite else args.append("-o-") + if self.overwrite: + args.append("-o+") + else: + args.append("-o-") + if self.delete: + args.append("-or") - if self.excludefiles: - for word in self.excludefiles.split(';'): - args.append("-x%s" % word) + for word in self.excludefiles: + args.append("-x%s" % word.strip()) # assume yes on all queries args.append("-y") @@ -216,10 +208,13 @@ class UnRar(Extractor): else: args.append("-p-") + if self.keepbroken: + args.append("-kb") + # NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue call = [self.CMD, command] + args + list(xargs) - self.m.logDebug(" ".join(call)) - p = Popen(call, stdout=PIPE, stderr=PIPE) + self.manager.logDebug(" ".join(call)) + p = Popen(call, stdout=PIPE, stderr=PIPE) return p -- cgit v1.2.3 From 788a06132882300a22f6db3aa7ac3a6009d4d762 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 29 Jan 2015 23:13:54 +0100 Subject: Update Extractor (2) --- module/plugins/internal/UnRar.py | 64 +++++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 20 deletions(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 7f1b08caf..b8e2c3606 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -8,20 +8,21 @@ from string import digits from subprocess import Popen, PIPE from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError -from module.utils import save_join, decode +from module.utils import decode, fs_encode, save_join, uniqify def renice(pid, value): if value and os.name != "nt": try: Popen(["renice", str(value), str(pid)], stdout=PIPE, stderr=PIPE, bufsize=-1) + except Exception: pass class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.04" + __version__ = "1.05" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" @@ -71,42 +72,65 @@ class UnRar(Extractor): continue m = cls.re_rarpart1.match(filename) - if not m or int(m.group(1)) is 1: #@NOTE: only add first part file + if not m or int(m.group(1)) == 1: #@NOTE: only add first part file targets.append((filename, id)) return targets - def checkArchive(self): - p = self.call_cmd("l", "-v", self.target) + def check(self): + p = self.call_cmd("l", "-v", fs_encode(self.filename)) out, err = p.communicate() if self.re_wrongpwd.search(err): - return True + raise PasswordError + + if self.re_wrongcrc.search(err): + raise CRCError # output only used to check if passworded files are present for attr in self.re_filelist.findall(out): if attr[0].startswith("*"): - return True + raise PasswordError self.files = self.list() if not self.files: raise ArchiveError("Empty Archive") - return False - - def checkPassword(self, password): + def isPassword(self, password): # at this point we can only verify header protected files - p = self.call_cmd("l", "-v", self.target, password=password) + p = self.call_cmd("l", "-v", fs_encode(self.filename), password=password) out, err = p.communicate() return False if self.re_wrongpwd.search(err) else True + def repair(self): + p = self.call_cmd("rc", fs_encode(self.filename)) + out, err = p.communicate() + + if p.returncode or err.strip(): + p = self.call_cmd("r", fs_encode(self.filename)) + out, err = p.communicate() + + if p.returncode or err.strip(): + return False + else: + dir, name = os.path.split(filename) + + if 'fixed' in out: + self.filename = os.path.join(dir, 'fixed.' + name) + + elif 'rebuild' in out: + self.filename = os.path.join(dir, 'rebuild.' + name) + + return True + + def extract(self, password=None): command = "x" if self.fullpath else "e" - p = self.call_cmd(command, self.target, self.out, password=password) + p = self.call_cmd(command, fs_encode(self.filename), self.out, password=password) renice(p.pid, self.renice) @@ -139,7 +163,7 @@ class UnRar(Extractor): elif err.strip(): #: raise error if anything is on stderr raise ArchiveError(err.strip()) - if p.returncode != 0: + if p.returncode: raise ArchiveError("Process terminated") if not self.files: @@ -149,18 +173,18 @@ class UnRar(Extractor): def getDeleteFiles(self): files = [] - for i in [1, 2]: + for i in (1, 2): try: - dir, name = os.path.split(self.target) + dir, name = os.path.split(self.filename) part = self.getattr(self, "re_rarpart%d" % i).match(name).group(1) - filename = os.path.join(dir, name.replace(part, '*', 1)) - files.extend(glob(filename)) + file = fs_encode(os.path.join(dir, name.replace(part, '*', 1))) + files.extend(glob(file)) except Exception: continue - if self.target not in files: - files.insert(0, self.target) + if self.filename not in files: + files.insert(0, self.filename) return files @@ -168,7 +192,7 @@ class UnRar(Extractor): def list(self, password=None): command = "vb" if self.fullpath else "lb" - p = self.call_cmd(command, "-v", self.target, password=password) + p = self.call_cmd(command, "-v", fs_encode(self.filename), password=password) out, err = p.communicate() if "Cannot open" in err: -- cgit v1.2.3 From 32b3cb9d9fa6851de35f52d01f78c26012043baa Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 29 Jan 2015 23:30:56 +0100 Subject: [SevenZip] Repair not supported --- module/plugins/internal/UnRar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index b8e2c3606..0738aa63b 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -8,7 +8,7 @@ from string import digits from subprocess import Popen, PIPE from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError -from module.utils import decode, fs_encode, save_join, uniqify +from module.utils import decode, fs_encode, save_join def renice(pid, value): -- cgit v1.2.3 From 79725268402043906f619f7c09e848e02ab8a17b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 31 Jan 2015 22:00:59 +0100 Subject: Spare code cosmetics --- module/plugins/internal/UnRar.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 0738aa63b..f92a2ddd4 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.05" + __version__ = "1.06" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" @@ -45,7 +45,7 @@ class UnRar(Extractor): @classmethod - def checkDeps(cls): + def isUsable(cls): if os.name == "nt": cls.CMD = os.path.join(pypath, "UnRAR.exe") p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) @@ -67,13 +67,13 @@ class UnRar(Extractor): def getTargets(cls, files_ids): targets = [] - for filename, id in files_ids: - if not cls.isArchive(filename): + for fname, id in files_ids: + if not cls.isArchive(fname): continue - m = cls.re_rarpart1.match(filename) + m = cls.re_rarpart1.match(fname) if not m or int(m.group(1)) == 1: #@NOTE: only add first part file - targets.append((filename, id)) + targets.append((fname, id)) return targets -- cgit v1.2.3 From 7368881d2ba95cca3f47afbb0f4ef5861f2774df Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 31 Jan 2015 23:19:30 +0100 Subject: Extractor final fixup --- module/plugins/internal/UnRar.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index f92a2ddd4..fbb1c6a3e 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -71,7 +71,7 @@ class UnRar(Extractor): if not cls.isArchive(fname): continue - m = cls.re_rarpart1.match(fname) + m = cls.re_rarpart1.search(fname) if not m or int(m.group(1)) == 1: #@NOTE: only add first part file targets.append((fname, id)) @@ -86,17 +86,13 @@ class UnRar(Extractor): raise PasswordError if self.re_wrongcrc.search(err): - raise CRCError + raise CRCError(err) # output only used to check if passworded files are present for attr in self.re_filelist.findall(out): if attr[0].startswith("*"): raise PasswordError - self.files = self.list() - if not self.files: - raise ArchiveError("Empty Archive") - def isPassword(self, password): # at this point we can only verify header protected files @@ -158,13 +154,13 @@ class UnRar(Extractor): raise PasswordError elif self.re_wrongcrc.search(err): - raise CRCError + raise CRCError(err) elif err.strip(): #: raise error if anything is on stderr - raise ArchiveError(err.strip()) + raise ArchiveError(err) if p.returncode: - raise ArchiveError("Process terminated") + raise ArchiveError(_("Process return code: %d") % p.returncode) if not self.files: self.files = self.list(password) @@ -176,8 +172,11 @@ class UnRar(Extractor): for i in (1, 2): try: dir, name = os.path.split(self.filename) - part = self.getattr(self, "re_rarpart%d" % i).match(name).group(1) - file = fs_encode(os.path.join(dir, name.replace(part, '*', 1))) + + part = self.getattr(self, "re_rarpart%d" % i).search(name).group(1) + new_name = name[::-1].replace((".part%s.rar" % part)[::-1], ".part*.rar"[::-1], 1)[::-1] + file = fs_encode(os.path.join(dir, new_name) + files.extend(glob(file)) except Exception: @@ -196,7 +195,7 @@ class UnRar(Extractor): out, err = p.communicate() if "Cannot open" in err: - raise ArchiveError("Cannot open file") + raise ArchiveError(_("Cannot open file")) if err.strip(): #: only log error at this point self.manager.logError(err.strip()) -- cgit v1.2.3 From ddca2529cf0ff79ea9e1721a15493407c23a57d9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 31 Jan 2015 23:40:36 +0100 Subject: Fix https://github.com/pyload/pyload/issues/1120 --- module/plugins/internal/UnRar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index fbb1c6a3e..1b6816a9f 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.06" + __version__ = "1.07" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" @@ -175,7 +175,7 @@ class UnRar(Extractor): part = self.getattr(self, "re_rarpart%d" % i).search(name).group(1) new_name = name[::-1].replace((".part%s.rar" % part)[::-1], ".part*.rar"[::-1], 1)[::-1] - file = fs_encode(os.path.join(dir, new_name) + file = fs_encode(os.path.join(dir, new_name)) files.extend(glob(file)) -- cgit v1.2.3 From 8dfb7adc0fc3c858c0ddf9371c2f4580bb8be3c7 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 3 Feb 2015 00:10:11 +0100 Subject: Update Extractor (3) --- module/plugins/internal/UnRar.py | 91 +++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 49 deletions(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 1b6816a9f..d378bf167 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.07" + __version__ = "1.08" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" @@ -39,7 +39,9 @@ class UnRar(Extractor): re_rarpart1 = re.compile(r'\.part(\d+)\.rar$', re.I) re_rarpart2 = re.compile(r'\.r(\d+)$', re.I) + re_filefixed = re.compile(r'Building (.+)') re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+|(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)') + re_wrongpwd = re.compile(r'password', re.I) re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I) @@ -63,21 +65,6 @@ class UnRar(Extractor): return True - @classmethod - def getTargets(cls, files_ids): - targets = [] - - for fname, id in files_ids: - if not cls.isArchive(fname): - continue - - m = cls.re_rarpart1.search(fname) - if not m or int(m.group(1)) == 1: #@NOTE: only add first part file - targets.append((fname, id)) - - return targets - - def check(self): p = self.call_cmd("l", "-v", fs_encode(self.filename)) out, err = p.communicate() @@ -103,67 +90,73 @@ class UnRar(Extractor): def repair(self): p = self.call_cmd("rc", fs_encode(self.filename)) - out, err = p.communicate() - if p.returncode or err.strip(): + # communicate and retrieve stderr + self._progress(p) + err = p.stderr.read().strip() + + if err or p.returncode: p = self.call_cmd("r", fs_encode(self.filename)) - out, err = p.communicate() - if p.returncode or err.strip(): + # communicate and retrieve stderr + self._progress(p) + err = p.stderr.read().strip() + + if err or p.returncode: return False else: - dir, name = os.path.split(filename) - - if 'fixed' in out: - self.filename = os.path.join(dir, 'fixed.' + name) + dir = os.path.dirname(filename) + name = re_filefixed.search(out).group(1) - elif 'rebuild' in out: - self.filename = os.path.join(dir, 'rebuild.' + name) + self.filename = os.path.join(dir, name) return True - def extract(self, password=None): - command = "x" if self.fullpath else "e" - - p = self.call_cmd(command, fs_encode(self.filename), self.out, password=password) - - renice(p.pid, self.renice) - - progressstring = "" + def _progress(self, process): + s = "" while True: - c = p.stdout.read(1) + c = process.stdout.read(1) # quit loop on eof if not c: break # reading a percentage sign -> set progress and restart if c == '%': - self.notifyProgress(int(progressstring)) - progressstring = "" + self.notifyProgress(int(s)) + s = "" # not reading a digit -> therefore restart elif c not in digits: - progressstring = "" + s = "" # add digit to progressstring else: - progressstring += c + s += c - # retrieve stderr - err = p.stderr.read() - if self.re_wrongpwd.search(err): - raise PasswordError + def extract(self, password=None): + command = "x" if self.fullpath else "e" - elif self.re_wrongcrc.search(err): - raise CRCError(err) + p = self.call_cmd(command, fs_encode(self.filename), self.out, password=password) + + renice(p.pid, self.renice) + + # communicate and retrieve stderr + self._progress(p) + err = p.stderr.read().strip() + + if err: + if self.re_wrongpwd.search(err): + raise PasswordError + + elif self.re_wrongcrc.search(err): + raise CRCError(err) - elif err.strip(): #: raise error if anything is on stderr - raise ArchiveError(err) + else: #: raise error if anything is on stderr + raise ArchiveError(err) if p.returncode: raise ArchiveError(_("Process return code: %d") % p.returncode) - if not self.files: - self.files = self.list(password) + self.files = self.list(password) def getDeleteFiles(self): -- cgit v1.2.3 From 8d8cfd57fa44cb84eb9709871bf14a3b4d13d994 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 3 Feb 2015 01:10:04 +0100 Subject: Update Extractor (4) --- module/plugins/internal/UnRar.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index d378bf167..c1cc0fa31 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.08" + __version__ = "1.09" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" @@ -213,14 +213,14 @@ class UnRar(Extractor): args.append("-or") for word in self.excludefiles: - args.append("-x%s" % word.strip()) + args.append("-x'%s'" % word.strip()) # assume yes on all queries args.append("-y") # set a password if "password" in kwargs and kwargs['password']: - args.append("-p%s" % kwargs['password']) + args.append("-p'%s'" % kwargs['password']) else: args.append("-p-") -- cgit v1.2.3 From 896f1f1437442bb9a93a0664f3d447bbc0f444cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Wed, 4 Feb 2015 00:02:27 +0100 Subject: Addressed some Extractor Issues --- module/plugins/internal/UnRar.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index c1cc0fa31..7126c960b 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.09" + __version__ = "1.10" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" @@ -32,7 +32,8 @@ class UnRar(Extractor): CMD = "unrar" - EXTENSIONS = [".rar", ".zip", ".cab", ".arj", ".lzh", ".tar", ".gz", ".bz2", + # TODO: Find out what Filetypes Unrar supports exactly + EXTENSIONS = [".rar", ".cab", ".arj", ".lzh", ".tar", ".gz", ".bz2", ".ace", ".uue", ".jar", ".iso", ".7z", ".xz", ".z"] #@NOTE: there are some more uncovered rar formats @@ -40,7 +41,7 @@ class UnRar(Extractor): re_rarpart2 = re.compile(r'\.r(\d+)$', re.I) re_filefixed = re.compile(r'Building (.+)') - re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+|(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)') + re_filelist = re.compile(r'(.+)\s+(\D+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)') re_wrongpwd = re.compile(r'password', re.I) re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I) @@ -220,7 +221,7 @@ class UnRar(Extractor): # set a password if "password" in kwargs and kwargs['password']: - args.append("-p'%s'" % kwargs['password']) + args.append("-p%s" % kwargs['password']) else: args.append("-p-") -- cgit v1.2.3 From 0df373e5da9d33e4dea3e709521d4883ef7f9b45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Wed, 4 Feb 2015 20:41:03 +0100 Subject: [Unrar] Fix: Deleting All Files after Extraction --- module/plugins/internal/UnRar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 7126c960b..81cfb38a7 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -167,7 +167,7 @@ class UnRar(Extractor): try: dir, name = os.path.split(self.filename) - part = self.getattr(self, "re_rarpart%d" % i).search(name).group(1) + part = getattr(self, "re_rarpart%d" % i).search(name).group(1) new_name = name[::-1].replace((".part%s.rar" % part)[::-1], ".part*.rar"[::-1], 1)[::-1] file = fs_encode(os.path.join(dir, new_name)) -- cgit v1.2.3 From 2a809297f288a585d96af0d8afd894c2a2f695fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Sun, 8 Feb 2015 13:47:59 +0100 Subject: [ExtractArchive] correct fullpath behavior, bugfix --- module/plugins/internal/UnRar.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 81cfb38a7..0ba990006 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.10" + __version__ = "1.11" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" @@ -37,15 +37,16 @@ class UnRar(Extractor): ".ace", ".uue", ".jar", ".iso", ".7z", ".xz", ".z"] #@NOTE: there are some more uncovered rar formats - re_rarpart1 = re.compile(r'\.part(\d+)\.rar$', re.I) - re_rarpart2 = re.compile(r'\.r(\d+)$', re.I) + re_rarpart1 = re.compile(r'\.part(\d+)\.rar$', re.I) + re_rarpart2 = re.compile(r'\.r(\d+)$', re.I) re_filefixed = re.compile(r'Building (.+)') - re_filelist = re.compile(r'(.+)\s+(\D+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)') + re_filelist = re.compile(r'(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)') re_wrongpwd = re.compile(r'password', re.I) re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I) + re_version = re.compile(r'UNRAR\s(\d+)\.\d+', re.I) @classmethod def isUsable(cls): @@ -185,6 +186,10 @@ class UnRar(Extractor): def list(self, password=None): command = "vb" if self.fullpath else "lb" + p = self.call_cmd("", "", fs_encode(self.filename)) + out, err = p.communicate() + version = self.re_version.search(out).group(1) + p = self.call_cmd(command, "-v", fs_encode(self.filename), password=password) out, err = p.communicate() @@ -195,9 +200,16 @@ class UnRar(Extractor): self.manager.logError(err.strip()) result = set() - for f in decode(out).splitlines(): - f = f.strip() - result.add(save_join(self.out, f)) + if not self.fullpath and version =='5': + # NOTE: Unrar 5 always list full path + for f in decode(out).splitlines(): + f = save_join(self.out, os.path.basename(f.strip())) + if os.path.isfile(f): + result.add(save_join(self.out, os.path.basename(f))) + else: + for f in decode(out).splitlines(): + f = f.strip() + result.add(save_join(self.out, f)) return list(result) -- cgit v1.2.3 From 2dc3536e36956eab99fa5f7945dcf60073b5fd57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Mon, 9 Feb 2015 23:36:10 +0100 Subject: [ExtractArchive] better Multipart behavior, new version output --- module/plugins/internal/UnRar.py | 62 +++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 32 deletions(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 0ba990006..f6ca5a2eb 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.11" + __version__ = "1.12" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" @@ -31,42 +31,53 @@ class UnRar(Extractor): CMD = "unrar" + VERSION = "" - # TODO: Find out what Filetypes Unrar supports exactly - EXTENSIONS = [".rar", ".cab", ".arj", ".lzh", ".tar", ".gz", ".bz2", - ".ace", ".uue", ".jar", ".iso", ".7z", ".xz", ".z"] + EXTENSIONS = [".rar"] - #@NOTE: there are some more uncovered rar formats - re_rarpart1 = re.compile(r'\.part(\d+)\.rar$', re.I) - re_rarpart2 = re.compile(r'\.r(\d+)$', re.I) + + re_multipart = re.compile(r'\.(part|r)(\d+)(?:\.rar)?',re.I) re_filefixed = re.compile(r'Building (.+)') - re_filelist = re.compile(r'(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)') + re_filelist = re.compile(r'^(.)(\s*[\w\.\-]+)\s+(\d+\s+)+(?:\d+\%\s+)?[\d\-]{8}\s+[\d\:]{5}', re.M|re.I) re_wrongpwd = re.compile(r'password', re.I) re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I) - re_version = re.compile(r'UNRAR\s(\d+)\.\d+', re.I) + re_version = re.compile(r'UNRAR\s(\d+\.\d+)', re.I) + @classmethod def isUsable(cls): if os.name == "nt": cls.CMD = os.path.join(pypath, "UnRAR.exe") p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) - p.communicate() + out, err = p.communicate() else: try: p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) - p.communicate() + out, err = p.communicate() except OSError: #: fallback to rar cls.CMD = "rar" p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) - p.communicate() + out, err = p.communicate() + + cls.VERSION = cls.re_version.search(out).group(1) return True + @classmethod + def isMultipart(cls,filename): + multipart = cls.re_multipart.search(filename) + if multipart: + # First Multipart file (part1.rar for *.part1-9.rar format or *.rar for .r1-9 format) handled as normal Archive + return False if (multipart.group(1) == "part" and int(multipart.group(2)) == 1) else True + + return False + + def check(self): p = self.call_cmd("l", "-v", fs_encode(self.filename)) out, err = p.communicate() @@ -162,23 +173,14 @@ class UnRar(Extractor): def getDeleteFiles(self): - files = [] + dir, name = os.path.split(self.filename) - for i in (1, 2): - try: - dir, name = os.path.split(self.filename) - - part = getattr(self, "re_rarpart%d" % i).search(name).group(1) - new_name = name[::-1].replace((".part%s.rar" % part)[::-1], ".part*.rar"[::-1], 1)[::-1] - file = fs_encode(os.path.join(dir, new_name)) - - files.extend(glob(file)) + # actually extracted file + files = [self.filename] - except Exception: - continue - - if self.filename not in files: - files.insert(0, self.filename) + # eventually Multipart Files + files.extend(save_join(self.out, os.path.basename(file)) for file in filter(self.isMultipart, os.listdir(dir)) + if re.sub(self.re_multipart,".rar",name) == re.sub(self.re_multipart,".rar",file)) return files @@ -186,10 +188,6 @@ class UnRar(Extractor): def list(self, password=None): command = "vb" if self.fullpath else "lb" - p = self.call_cmd("", "", fs_encode(self.filename)) - out, err = p.communicate() - version = self.re_version.search(out).group(1) - p = self.call_cmd(command, "-v", fs_encode(self.filename), password=password) out, err = p.communicate() @@ -200,7 +198,7 @@ class UnRar(Extractor): self.manager.logError(err.strip()) result = set() - if not self.fullpath and version =='5': + if not self.fullpath and self.rarversion.startswith('5'): # NOTE: Unrar 5 always list full path for f in decode(out).splitlines(): f = save_join(self.out, os.path.basename(f.strip())) -- cgit v1.2.3 From f2ac32085922f739343bac3cf396e703833323f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Wed, 11 Feb 2015 16:32:55 +0100 Subject: [UnRar] bugfixes --- module/plugins/internal/UnRar.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal/UnRar.py') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index f6ca5a2eb..54d64c430 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,12 +22,13 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.12" + __version__ = "1.13" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" __authors__ = [("RaNaN", "RaNaN@pyload.org"), - ("Walter Purcaro", "vuolter@gmail.com")] + ("Walter Purcaro", "vuolter@gmail.com"), + ("Immenz", "immenz@gmx.net"),] CMD = "unrar" @@ -179,7 +180,7 @@ class UnRar(Extractor): files = [self.filename] # eventually Multipart Files - files.extend(save_join(self.out, os.path.basename(file)) for file in filter(self.isMultipart, os.listdir(dir)) + files.extend(save_join(dir, os.path.basename(file)) for file in filter(self.isMultipart, os.listdir(dir)) if re.sub(self.re_multipart,".rar",name) == re.sub(self.re_multipart,".rar",file)) return files @@ -198,7 +199,7 @@ class UnRar(Extractor): self.manager.logError(err.strip()) result = set() - if not self.fullpath and self.rarversion.startswith('5'): + if not self.fullpath and self.VERSION.startswith('5'): # NOTE: Unrar 5 always list full path for f in decode(out).splitlines(): f = save_join(self.out, os.path.basename(f.strip())) -- cgit v1.2.3