From f71c1ef70a199e42e8a519364d9924e138ffd37c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 24 Dec 2014 15:48:08 +0100 Subject: [ExtractArchive] Remove empty directory --- module/plugins/hooks/ExtractArchive.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 16942bef0..8b3b4e310 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -58,7 +58,7 @@ from module.utils import save_join, uniqify class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.00" + __version__ = "1.01" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract full path" , True ), @@ -135,7 +135,7 @@ class ExtractArchive(Hook): self.logInfo(_("Package %s queued for later extracting") % pypack.name) self.queue.append(pid) else: - self.manager.startThread(self.extract, [pid]) + self.extractPackage(pid) @threaded @@ -239,6 +239,7 @@ class ExtractArchive(Hook): new_files = None if new_files is None: + self.logWarning(basename(target), _("No files extracted")) success = False continue @@ -264,6 +265,12 @@ class ExtractArchive(Hook): else: self.logInfo(_("No files found to extract")) + if not matched or not success and subfolder: + try: + os.rmdir(out) + except OSError: + pass + return True if not failed else False -- cgit v1.2.3 From 4c63928557398891c30d3e2b7c962a07b3483315 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 26 Dec 2014 04:18:41 +0100 Subject: Rename AbstractExtractor to Extractor --- module/plugins/hooks/ExtractArchive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 8b3b4e310..af78ffc93 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -51,14 +51,14 @@ if os.name != "nt": from pwd import getpwnam from module.plugins.Hook import Hook, threaded, Expose -from module.plugins.internal.AbstractExtractor import ArchiveError, CRCError, PasswordError +from module.plugins.internal.Extractor import ArchiveError, CRCError, PasswordError from module.utils import save_join, uniqify class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.01" + __version__ = "1.02" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract full path" , True ), -- cgit v1.2.3 From d5ad6ab1c1b8b088e7a4083b1fb7bab72d5679c3 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 31 Dec 2014 18:44:38 +0100 Subject: [ExtractArchive] Fix https://github.com/pyload/pyload/issues/778 --- module/plugins/hooks/ExtractArchive.py | 50 ++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 15 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index af78ffc93..73d8bdb0b 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -58,7 +58,7 @@ from module.utils import save_join, uniqify class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.02" + __version__ = "1.03" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract full path" , True ), @@ -72,7 +72,7 @@ class ExtractArchive(Hook): ("extensions" , "str" , "Extract the following extensions" , "7z,bz2,bzip2,gz,gzip,lha,lzh,lzma,rar,tar,taz,tbz,tbz2,tgz,xar,xz,z,zip"), ("excludefiles" , "str" , "Don't extract the following files" , "*.nfo,*.DS_Store,index.dat,thumb.db" ), ("recursive" , "bool" , "Extract archives in archives" , True ), - ("queue" , "bool" , "Wait for all downloads to be finished" , True ), + ("queue" , "bool" , "Wait for all downloads to be finished" , False ), ("renice" , "int" , "CPU Priority" , 0 )] __description__ = """Extract different kind of archives""" @@ -88,6 +88,10 @@ class ExtractArchive(Hook): pass + def coreReady(self): + self.extracting = False + + def setup(self): self.plugins = [] self.passwords = [] @@ -123,34 +127,50 @@ class ExtractArchive(Hook): self.queue = [] + def periodical(self): + if not self.queue or self.extracting: + return + + local = copy(self.queue) + self.queue[:] = [] + + self.extractPackages(*local) + + @Expose def extractPackage(self, id): - """ Extract package with given id""" - self.manager.startThread(self.extract, [id]) + """ Extract package wrapper""" + self.extractPackages(id) + + + @Expose + def extractPackages(self, *ids): + """ Extract packages with given id""" + self.manager.startThread(self.extract, ids) def packageFinished(self, pypack): - pid = pypack.id - if self.getConfig("queue"): + if self.getConfig("queue") or self.extracting: self.logInfo(_("Package %s queued for later extracting") % pypack.name) - self.queue.append(pid) + self.queue.append(pypack.id) else: - self.extractPackage(pid) + self.extractPackage(pypack.id) @threaded def allDownloadsProcessed(self, thread): local = copy(self.queue) + self.queue[:] = [] - del self.queue[:] - - if self.extract(local, thread): #: check only if all gone fine, no failed reporting for now + if self.extract(local): #: check only if all gone fine, no failed reporting for now self.manager.dispatchEvent("all_archives_extracted") self.manager.dispatchEvent("all_archives_processed") - def extract(self, ids, thread=None): + def extract(self, ids): + self.extracting = True + processed = [] extracted = [] failed = [] @@ -232,7 +252,7 @@ class ExtractArchive(Hook): keepbroken) klass.init() - new_files = self._extract(klass, fid, thread) + new_files = self._extract(klass, fid) except Exception, e: self.logError(basename(target), e) @@ -271,14 +291,14 @@ class ExtractArchive(Hook): except OSError: pass + self.extracting = False return True if not failed else False - def _extract(self, plugin, fid, thread): + def _extract(self, plugin, fid): pyfile = self.core.files.getFile(fid) pyfile.setCustomStatus(_("extracting")) - thread.addActive(pyfile) # keep this file until everything is done try: progress = lambda x: pyfile.setProgress(x) -- cgit v1.2.3 From 60e9c46f32d97d01d728c8515985b58ba33fdafd Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 1 Jan 2015 01:48:37 +0100 Subject: [SimpleHoster] Fix https://github.com/pyload/pyload/issues/997 --- module/plugins/hooks/ExtractArchive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 73d8bdb0b..2e9efa2b0 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -189,7 +189,7 @@ class ExtractArchive(Hook): keepbroken = self.getConfig("keepbroken") if extensions: - self.logDebug("Extensions allowed: %s" % "|.".join(extensions)) + self.logDebug("Extensions: %s" % "|.".join(extensions)) # reload from txt file self.reloadPasswords() -- cgit v1.2.3 From e2d8d605aeb4fd3477a8d681e16cd0b17500a648 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 16 Jan 2015 01:36:04 +0100 Subject: Fix allDownloadsFinished and allDownloadsProcessed defs --- module/plugins/hooks/ExtractArchive.py | 42 +++++++++++++++++----------------- 1 file changed, 21 insertions(+), 21 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 2e9efa2b0..bdbaf64af 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -58,7 +58,7 @@ from module.utils import save_join, uniqify class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.03" + __version__ = "1.04" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract full path" , True ), @@ -158,7 +158,7 @@ class ExtractArchive(Hook): @threaded - def allDownloadsProcessed(self, thread): + def allDownloadsProcessed(self): local = copy(self.queue) self.queue[:] = [] @@ -302,36 +302,36 @@ class ExtractArchive(Hook): try: progress = lambda x: pyfile.setProgress(x) - encrypted = False + encrypted = True #@TODO: set to `False` passwords = self.getPasswords() - try: - self.logInfo(basename(plugin.file), "Verifying...") + # try: + # self.logInfo(basename(plugin.file), "Verifying...") - tmp_password = plugin.password - plugin.password = "" #: Force verifying without password + # tmp_password = plugin.password + # plugin.password = "" #: Force verifying without password - plugin.verify() + # plugin.verify() - except PasswordError: - encrypted = True + # except PasswordError: + # encrypted = True - except CRCError: - self.logWarning(basename(plugin.file), _("Archive damaged")) + # except CRCError: + # self.logWarning(basename(plugin.file), _("Archive damaged")) - if not self.getConfig("repair"): - raise CRCError + # if not self.getConfig("repair"): + # raise CRCError - elif plugin.repair(): - self.logInfo(basename(plugin.file), _("Successfully repaired")) + # elif plugin.repair(): + # self.logInfo(basename(plugin.file), _("Successfully repaired")) - elif not self.getConfig("keepbroken"): - raise ArchiveError(_("Broken archive")) + # elif not self.getConfig("keepbroken"): + # raise ArchiveError(_("Broken archive")) - else: - self.logInfo(basename(plugin.file), _("All OK")) + # else: + # self.logInfo(basename(plugin.file), _("All OK")) - plugin.password = tmp_password + # plugin.password = tmp_password if not encrypted: plugin.extract(progress) -- cgit v1.2.3 From cf4ded052964047de88d676045329b8fa4fca2dc Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 22 Jan 2015 21:31:19 +0100 Subject: Update plugins after CaptchaService changes --- module/plugins/hooks/ExtractArchive.py | 38 +++++++++++++++++----------------- 1 file changed, 19 insertions(+), 19 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index bdbaf64af..f2f4b5207 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -58,7 +58,7 @@ from module.utils import save_join, uniqify class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.04" + __version__ = "1.05" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract full path" , True ), @@ -305,33 +305,33 @@ class ExtractArchive(Hook): encrypted = True #@TODO: set to `False` passwords = self.getPasswords() - # try: - # self.logInfo(basename(plugin.file), "Verifying...") + try: + self.logInfo(basename(plugin.file), "Verifying...") - # tmp_password = plugin.password - # plugin.password = "" #: Force verifying without password + tmp_password = plugin.password + plugin.password = "" #: Force verifying without password - # plugin.verify() + plugin.verify() - # except PasswordError: - # encrypted = True + except PasswordError: + encrypted = True - # except CRCError: - # self.logWarning(basename(plugin.file), _("Archive damaged")) + except CRCError: + self.logWarning(basename(plugin.file), _("Archive damaged")) - # if not self.getConfig("repair"): - # raise CRCError + if not self.getConfig("repair"): + raise CRCError - # elif plugin.repair(): - # self.logInfo(basename(plugin.file), _("Successfully repaired")) + elif plugin.repair(): + self.logInfo(basename(plugin.file), _("Successfully repaired")) - # elif not self.getConfig("keepbroken"): - # raise ArchiveError(_("Broken archive")) + elif not self.getConfig("keepbroken"): + raise ArchiveError(_("Broken archive")) - # else: - # self.logInfo(basename(plugin.file), _("All OK")) + else: + self.logInfo(basename(plugin.file), _("All OK")) - # plugin.password = tmp_password + plugin.password = tmp_password if not encrypted: plugin.extract(progress) -- cgit v1.2.3 From d6fc06740679154f67a0f95aba5463dda1ecf3fb Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 24 Jan 2015 01:52:04 +0100 Subject: [ExtractArchive] Fix encrypted flag init --- module/plugins/hooks/ExtractArchive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index f2f4b5207..38459e5f1 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -58,7 +58,7 @@ from module.utils import save_join, uniqify class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.05" + __version__ = "1.06" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract full path" , True ), @@ -302,7 +302,7 @@ class ExtractArchive(Hook): try: progress = lambda x: pyfile.setProgress(x) - encrypted = True #@TODO: set to `False` + encrypted = False passwords = self.getPasswords() try: -- cgit v1.2.3 From 7fc3362307737cd7c565b710ec83c5bdc4d3e8a9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 25 Jan 2015 03:05:00 +0100 Subject: Revert Extractor to the old one (temp) --- module/plugins/hooks/ExtractArchive.py | 219 ++++++++++----------------------- 1 file changed, 65 insertions(+), 154 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 38459e5f1..9e530ce8f 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -51,33 +51,32 @@ if os.name != "nt": from pwd import getpwnam from module.plugins.Hook import Hook, threaded, Expose -from module.plugins.internal.Extractor import ArchiveError, CRCError, PasswordError -from module.utils import save_join, uniqify +from module.plugins.internal.Extractor import ArchiveError, CRCError, WrongPassword +from module.utils import save_join, fs_encode class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.06" - - __config__ = [("activated" , "bool" , "Activated" , True ), - ("fullpath" , "bool" , "Extract full path" , True ), - ("overwrite" , "bool" , "Overwrite files" , False ), - ("keepbroken" , "bool" , "Extract broken archives" , False ), - ("repair" , "bool" , "Repair broken archives" , True ), - ("passwordfile" , "file" , "Store passwords in file" , "archive_password.txt" ), - ("delete" , "bool" , "Delete archive when successfully extracted", False ), - ("subfolder" , "bool" , "Create subfolder for each package" , False ), - ("destination" , "folder", "Extract files to" , "" ), - ("extensions" , "str" , "Extract the following extensions" , "7z,bz2,bzip2,gz,gzip,lha,lzh,lzma,rar,tar,taz,tbz,tbz2,tgz,xar,xz,z,zip"), - ("excludefiles" , "str" , "Don't extract the following files" , "*.nfo,*.DS_Store,index.dat,thumb.db" ), - ("recursive" , "bool" , "Extract archives in archives" , True ), - ("queue" , "bool" , "Wait for all downloads to be finished" , False ), - ("renice" , "int" , "CPU Priority" , 0 )] + __version__ = "1.07" + + __config__ = [("activated", "bool", "Activated", True), + ("fullpath", "bool", "Extract full path", True), + ("overwrite", "bool", "Overwrite files", True), + ("passwordfile", "file", "password file", "archive_password.txt"), + ("deletearchive", "bool", "Delete archives when done", False), + ("subfolder", "bool", "Create subfolder for each package", False), + ("destination", "folder", "Extract files to", ""), + ("excludefiles", "str", "Exclude files from unpacking (seperated by ;)", ""), + ("recursive", "bool", "Extract archives in archvies", True), + ("queue", "bool", "Wait for all downloads to be finished", True), + ("renice", "int", "CPU Priority", 0)] __description__ = """Extract different kind of archives""" __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("RaNaN", "ranan@pyload.org"), + ("AndroKev", None), + ("Walter Purcaro", "vuolter@gmail.com")] event_list = ["allDownloadsProcessed"] @@ -88,16 +87,12 @@ class ExtractArchive(Hook): pass - def coreReady(self): - self.extracting = False - - def setup(self): self.plugins = [] self.passwords = [] names = [] - for p in ("UnRar", "SevenZip", "UnZip"): + for p in ("UnRar", "UnZip"): try: module = self.core.pluginManager.loadModule("internal", p) klass = getattr(module, p) @@ -127,69 +122,45 @@ class ExtractArchive(Hook): self.queue = [] - def periodical(self): - if not self.queue or self.extracting: - return - - local = copy(self.queue) - self.queue[:] = [] - - self.extractPackages(*local) - - @Expose def extractPackage(self, id): - """ Extract package wrapper""" - self.extractPackages(id) - - - @Expose - def extractPackages(self, *ids): - """ Extract packages with given id""" - self.manager.startThread(self.extract, ids) + """ Extract package with given id""" + self.manager.startThread(self.extract, [id]) def packageFinished(self, pypack): - if self.getConfig("queue") or self.extracting: + pid = pypack.id + if self.getConfig("queue"): self.logInfo(_("Package %s queued for later extracting") % pypack.name) - self.queue.append(pypack.id) + self.queue.append(pid) else: - self.extractPackage(pypack.id) + self.manager.startThread(self.extract, [pid]) @threaded - def allDownloadsProcessed(self): + def allDownloadsProcessed(self, thread): local = copy(self.queue) - self.queue[:] = [] - if self.extract(local): #: check only if all gone fine, no failed reporting for now + del self.queue[:] + + if self.extract(local, thread): #: check only if all gone fine, no failed reporting for now self.manager.dispatchEvent("all_archives_extracted") self.manager.dispatchEvent("all_archives_processed") - def extract(self, ids): - self.extracting = True - + def extract(self, ids, thread=None): processed = [] extracted = [] failed = [] - clearlist = lambda string: [x.lstrip('.') for x in string.replace(' ', '').replace(',', '|').replace(';', '|').split('|')] - destination = self.getConfig("destination") subfolder = self.getConfig("subfolder") fullpath = self.getConfig("fullpath") overwrite = self.getConfig("overwrite") - extensions = clearlist(self.getConfig("extensions")) - excludefiles = clearlist(self.getConfig("excludefiles")) + excludefiles = self.getConfig("excludefiles") renice = self.getConfig("renice") recursive = self.getConfig("recursive") - delete = self.getConfig("delete") - keepbroken = self.getConfig("keepbroken") - - if extensions: - self.logDebug("Extensions: %s" % "|.".join(extensions)) # reload from txt file self.reloadPasswords() @@ -200,7 +171,7 @@ class ExtractArchive(Hook): #iterate packages -> plugins -> targets for pid in ids: p = self.core.files.getPackage(pid) - self.logInfo(_("Check package: %s") % p.name) + self.logInfo(_("Check package %s") % p.name) if not p: continue @@ -208,25 +179,21 @@ class ExtractArchive(Hook): out = save_join(dl, p.folder, destination, "") #: force trailing slash if subfolder: - out = save_join(out, p.folder) + out = save_join(out, fs_encode(p.folder)) if not exists(out): makedirs(out) files_ids = [(save_join(dl, p.folder, x['name']), x['id']) for x in p.getChildren().itervalues()] - matched = False - success = True + matched = False + success = True # check as long there are unseen files while files_ids: new_files_ids = [] - if extensions: - files_ids = [(file, id) for file, id in files_ids if filter(lambda ext: file.endswith(ext), extensions)] - for plugin in self.plugins: targets = plugin.getTargets(files_ids) - if targets: self.logDebug("Targets for %s: %s" % (plugin.__name__, targets)) matched = True @@ -238,32 +205,19 @@ class ExtractArchive(Hook): processed.append(target) # prevent extracting same file twice - self.logInfo(basename(target), _("Extract to: %s") % out) + self.logInfo(basename(target), _("Extract to %s") % out) try: - klass = plugin(self, - target, - out, - p.password, - fullpath, - overwrite, - excludefiles, - renice, - delete, - keepbroken) + klass = plugin(self, target, out, fullpath, overwrite, excludefiles, renice) klass.init() - new_files = self._extract(klass, fid) + new_files = self._extract(klass, fid, [p.password.strip()], thread) except Exception, e: self.logError(basename(target), e) - new_files = None - - if new_files is None: - self.logWarning(basename(target), _("No files extracted")) success = False continue - self.logDebug("Extracted files: %s" % new_files) + self.logDebug("Extracted", new_files) self.setPermissions(new_files) for file in new_files: @@ -285,87 +239,46 @@ class ExtractArchive(Hook): else: self.logInfo(_("No files found to extract")) - if not matched or not success and subfolder: - try: - os.rmdir(out) - except OSError: - pass - - self.extracting = False return True if not failed else False - def _extract(self, plugin, fid): + def _extract(self, plugin, fid, passwords, thread): pyfile = self.core.files.getFile(fid) + deletearchive = self.getConfig("deletearchive") pyfile.setCustomStatus(_("extracting")) + thread.addActive(pyfile) # keep this file until everything is done try: - progress = lambda x: pyfile.setProgress(x) - encrypted = False - passwords = self.getPasswords() - - try: - self.logInfo(basename(plugin.file), "Verifying...") - - tmp_password = plugin.password - plugin.password = "" #: Force verifying without password - - plugin.verify() - - except PasswordError: - encrypted = True - - except CRCError: - self.logWarning(basename(plugin.file), _("Archive damaged")) - - if not self.getConfig("repair"): - raise CRCError - - elif plugin.repair(): - self.logInfo(basename(plugin.file), _("Successfully repaired")) - - elif not self.getConfig("keepbroken"): - raise ArchiveError(_("Broken archive")) - - else: - self.logInfo(basename(plugin.file), _("All OK")) - - plugin.password = tmp_password - - if not encrypted: - plugin.extract(progress) + progress = lambda x: pyfile.setProgress(x) + success = False + if not plugin.checkArchive(): + plugin.extract(progress, pw) + success = True else: self.logInfo(basename(plugin.file), _("Password protected")) + self.logDebug("Passwords: %s" % passwords if passwords else "No password provided") - if plugin.password: - passwords.insert(0, plugin.password) - passwords = uniqify(self.passwords) - self.logDebug("Password: %s" % plugin.password) - else: - self.logDebug("No package password provided") - - for pw in passwords: + for pw in set(passwords) | set(self.getPasswords()): try: self.logDebug("Try password: %s" % pw) - - if plugin.setPassword(pw): - plugin.extract(progress) + if plugin.checkPassword(pw): + plugin.extract(progress, pw) self.addPassword(pw) + success = True break - else: - raise PasswordError - except PasswordError: + except WrongPassword: self.logDebug("Password was wrong") - else: - raise PasswordError + + if not success: + raise Exception(_("Wrong password")) if self.core.debug: - self.logDebug("Would delete: %s" % ", ".join(plugin.getDeleteFiles())) + self.logDebug("Would delete", ", ".join(plugin.getDeleteFiles())) - if self.getConfig("delete"): + if deletearchive: files = plugin.getDeleteFiles() self.logInfo(_("Deleting %s files") % len(files)) for f in files: @@ -381,16 +294,12 @@ class ExtractArchive(Hook): return extracted_files - except PasswordError: - self.logError(basename(plugin.file), _("Wrong password" if passwords else "No password found")) - plugin.password = "" + except ArchiveError, e: + self.logError(basename(plugin.file), _("Archive Error"), e) except CRCError: self.logError(basename(plugin.file), _("CRC Mismatch")) - except ArchiveError, e: - self.logError(basename(plugin.file), _("Archive Error"), e) - except Exception, e: if self.core.debug: print_exc() @@ -398,7 +307,7 @@ class ExtractArchive(Hook): self.manager.dispatchEvent("archive_extract_failed", pyfile) - self.logError(basename(plugin.file), _("Extract failed")) + raise Exception(_("Extract failed")) @Expose @@ -428,13 +337,15 @@ class ExtractArchive(Hook): """ Adds a password to saved list""" passwordfile = self.getConfig("passwordfile") + if pw in self.passwords: + self.passwords.remove(pw) + self.passwords.insert(0, pw) - self.passwords = uniqify(self.passwords) try: with open(passwordfile, "wb") as f: for pw in self.passwords: - f.write(pw + '\n') + f.write(pw + "\n") except IOError, e: self.logError(e) -- cgit v1.2.3 From 99ed44b0d919a33e3a559472893163b347cd1c37 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 25 Jan 2015 19:09:21 +0100 Subject: Fix reverted Extractor --- module/plugins/hooks/ExtractArchive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 9e530ce8f..20e585046 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -58,7 +58,7 @@ from module.utils import save_join, fs_encode class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.07" + __version__ = "1.08" __config__ = [("activated", "bool", "Activated", True), ("fullpath", "bool", "Extract full path", True), @@ -254,7 +254,7 @@ class ExtractArchive(Hook): success = False if not plugin.checkArchive(): - plugin.extract(progress, pw) + plugin.extract(progress, self.getPasswords()) success = True else: self.logInfo(basename(plugin.file), _("Password protected")) -- cgit v1.2.3 From 8848a359a43316fb346b728d1d79d7b72d27e5a0 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 27 Jan 2015 17:48:25 +0100 Subject: Update Extractor (again) --- module/plugins/hooks/ExtractArchive.py | 273 +++++++++++++++++++-------------- 1 file changed, 161 insertions(+), 112 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 20e585046..11764759f 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -6,8 +6,6 @@ import os import sys from copy import copy -from os import remove, chmod, makedirs -from os.path import exists, basename, isfile, isdir from traceback import print_exc # monkey patch bug in python 2.6 and lower @@ -47,35 +45,37 @@ if sys.version_info < (2, 7) and os.name != "nt": if os.name != "nt": from grp import getgrnam - from os import chown from pwd import getpwnam from module.plugins.Hook import Hook, threaded, Expose -from module.plugins.internal.Extractor import ArchiveError, CRCError, WrongPassword -from module.utils import save_join, fs_encode +from module.plugins.internal.Extractor import ArchiveError, CRCError, PasswordError +from module.utils import fs_decode, save_join, uniqify class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.08" - - __config__ = [("activated", "bool", "Activated", True), - ("fullpath", "bool", "Extract full path", True), - ("overwrite", "bool", "Overwrite files", True), - ("passwordfile", "file", "password file", "archive_password.txt"), - ("deletearchive", "bool", "Delete archives when done", False), - ("subfolder", "bool", "Create subfolder for each package", False), - ("destination", "folder", "Extract files to", ""), - ("excludefiles", "str", "Exclude files from unpacking (seperated by ;)", ""), - ("recursive", "bool", "Extract archives in archvies", True), - ("queue", "bool", "Wait for all downloads to be finished", True), - ("renice", "int", "CPU Priority", 0)] + __version__ = "1.09" + + __config__ = [("activated" , "bool" , "Activated" , True ), + ("fullpath" , "bool" , "Extract full path" , True ), + ("overwrite" , "bool" , "Overwrite files" , False ), + ("keepbroken" , "bool" , "Extract broken archives" , False ), + # ("repair" , "bool" , "Repair broken archives" , True ), + ("passwordfile" , "file" , "Store passwords in file" , "archive_password.txt" ), + ("delete" , "bool" , "Delete archive when successfully extracted", False ), + ("subfolder" , "bool" , "Create subfolder for each package" , False ), + ("destination" , "folder", "Extract files to" , "" ), + ("extensions" , "str" , "Extract the following extensions" , "7z,bz2,bzip2,gz,gzip,lha,lzh,lzma,rar,tar,taz,tbz,tbz2,tgz,xar,xz,z,zip"), + ("excludefiles" , "str" , "Don't extract the following files" , "*.nfo,*.DS_Store,index.dat,thumb.db" ), + ("recursive" , "bool" , "Extract archives in archives" , True ), + ("queue" , "bool" , "Wait for all downloads to be finished" , False ), + ("renice" , "int" , "CPU Priority" , 0 )] __description__ = """Extract different kind of archives""" __license__ = "GPLv3" __authors__ = [("RaNaN", "ranan@pyload.org"), - ("AndroKev", None), + ("AndroKev", ""), ("Walter Purcaro", "vuolter@gmail.com")] @@ -87,18 +87,23 @@ class ExtractArchive(Hook): pass + def coreReady(self): + self.extracting = False + + def setup(self): - self.plugins = [] - self.passwords = [] - names = [] + self.interval = 300 + self.extractors = [] + self.passwords = [] - for p in ("UnRar", "UnZip"): + names = [] + for p in ("UnRar", "SevenZip", "UnZip"): try: module = self.core.pluginManager.loadModule("internal", p) klass = getattr(module, p) if klass.checkDeps(): names.append(p) - self.plugins.append(klass) + self.extractors.append(klass) except OSError, e: if e.errno == 2: @@ -118,49 +123,55 @@ class ExtractArchive(Hook): else: self.logInfo(_("No Extract plugins activated")) - # queue with package ids - self.queue = [] + + def periodical(self): + if not self.extracting: + self.extractPackage(*self.getQueue()) @Expose - def extractPackage(self, id): - """ Extract package with given id""" - self.manager.startThread(self.extract, [id]) + def extractPackage(self, *ids): + """ Extract packages with given id""" + self.manager.startThread(self.extract, ids) def packageFinished(self, pypack): - pid = pypack.id - if self.getConfig("queue"): + if self.extracting or self.getConfig("queue"): self.logInfo(_("Package %s queued for later extracting") % pypack.name) - self.queue.append(pid) + self.addToQueue(pypack.id) else: - self.manager.startThread(self.extract, [pid]) + self.extractPackage(pypack.id) @threaded - def allDownloadsProcessed(self, thread): - local = copy(self.queue) - - del self.queue[:] - - if self.extract(local, thread): #: check only if all gone fine, no failed reporting for now + def allDownloadsProcessed(self): + if self.extract(self.getQueue()): #@NOTE: check only if all gone fine, no failed reporting for now self.manager.dispatchEvent("all_archives_extracted") self.manager.dispatchEvent("all_archives_processed") - def extract(self, ids, thread=None): + def extract(self, ids): processed = [] extracted = [] failed = [] + clearList = lambda string: [x.lstrip('.') for x in string.replace(' ', '').replace(',', '|').replace(';', '|').split('|')] + destination = self.getConfig("destination") subfolder = self.getConfig("subfolder") fullpath = self.getConfig("fullpath") overwrite = self.getConfig("overwrite") + extensions = clearList(self.getConfig("extensions")) + excludefiles = clearList(self.getConfig("excludefiles")) excludefiles = self.getConfig("excludefiles") renice = self.getConfig("renice") recursive = self.getConfig("recursive") + delete = self.getConfig("delete") + keepbroken = self.getConfig("keepbroken") + + if extensions: + self.logDebug("Extensions: %s" % "|.".join(extensions)) # reload from txt file self.reloadPasswords() @@ -168,63 +179,81 @@ class ExtractArchive(Hook): # dl folder dl = self.config['general']['download_folder'] - #iterate packages -> plugins -> targets + #iterate packages -> extractors -> targets for pid in ids: - p = self.core.files.getPackage(pid) - self.logInfo(_("Check package %s") % p.name) - if not p: + pypack = self.core.files.getPackage(pid) + + self.logInfo(_("Check package: %s") % pypack.name) + + if not pypack: continue # determine output folder - out = save_join(dl, p.folder, destination, "") #: force trailing slash + out = save_join(dl, pypack.folder, destination, "") #: force trailing slash if subfolder: - out = save_join(out, fs_encode(p.folder)) + out = save_join(out, pypack.folder) - if not exists(out): - makedirs(out) + if not os.path.exists(out): + os.makedirs(out) - files_ids = [(save_join(dl, p.folder, x['name']), x['id']) for x in p.getChildren().itervalues()] - matched = False - success = True + matched = False + success = True + files_ids = [(save_join(dl, pypack.folder, x['name']), x['id']) for x in pypack.getChildren().itervalues()] # check as long there are unseen files while files_ids: new_files_ids = [] - for plugin in self.plugins: - targets = plugin.getTargets(files_ids) + if extensions: + files_ids = [(file, id) for file, id in files_ids if filter(lambda ext: file.endswith(ext), extensions)] + + for Extractor in self.extractors: + targets = Extractor.getTargets(files_ids) if targets: - self.logDebug("Targets for %s: %s" % (plugin.__name__, targets)) + self.logDebug("Targets for %s: %s" % (Extractor.__name__, targets)) matched = True - for target, fid in targets: - if target in processed: - self.logDebug(basename(target), "skipped") + for filename, fid in targets: + fname = os.path.basename(filename) + + if filename in processed: + self.logDebug(fname, "Skipped") continue - processed.append(target) # prevent extracting same file twice + processed.append(filename) # prevent extracting same file twice - self.logInfo(basename(target), _("Extract to %s") % out) + self.logInfo(fname, _("Extract to: %s") % out) try: - klass = plugin(self, target, out, fullpath, overwrite, excludefiles, renice) + self.extracting = True + + klass = Extractor(self, + filename, + out, + fullpath, + overwrite, + excludefiles, + renice, + delete, + keepbroken, + fid) klass.init() - new_files = self._extract(klass, fid, [p.password.strip()], thread) + new_files = self._extract(klass, fid, pypack.password) except Exception, e: - self.logError(basename(target), e) + self.logError(fname, e) success = False continue - self.logDebug("Extracted", new_files) + self.logDebug("Extracted files: %s" % new_files) self.setPermissions(new_files) for file in new_files: - if not exists(file): + if not os.path.exists(file): self.logDebug("New file %s does not exists" % file) continue - if recursive and isfile(file): + if recursive and os.path.isfile(file): new_files_ids.append((file, fid)) # append as new target files_ids = new_files_ids # also check extracted files @@ -232,96 +261,120 @@ class ExtractArchive(Hook): if matched: if success: extracted.append(pid) - self.manager.dispatchEvent("package_extracted", p) + self.manager.dispatchEvent("package_extracted", pypack) else: failed.append(pid) - self.manager.dispatchEvent("package_extract_failed", p) + self.manager.dispatchEvent("package_extract_failed", pypack) else: self.logInfo(_("No files found to extract")) + if not matched or not success and subfolder: + try: + os.rmdir(out) + except OSError: + pass + + self.extracting = False return True if not failed else False - def _extract(self, plugin, fid, passwords, thread): + def _extract(self, archive, fid, password): pyfile = self.core.files.getFile(fid) - deletearchive = self.getConfig("deletearchive") + fname = os.path.basename(fs_decode(archive.target)) pyfile.setCustomStatus(_("extracting")) - thread.addActive(pyfile) # keep this file until everything is done + pyfile.setProgress(0) try: - progress = lambda x: pyfile.setProgress(x) success = False - if not plugin.checkArchive(): - plugin.extract(progress, self.getPasswords()) + if not archive.checkArchive(): + archive.extract(password) success = True else: - self.logInfo(basename(plugin.file), _("Password protected")) - self.logDebug("Passwords: %s" % passwords if passwords else "No password provided") + self.logInfo(fname, _("Password protected")) + self.logDebug("Password: %s" % (password or "No provided")) - for pw in set(passwords) | set(self.getPasswords()): + for pw in set(self.getPasswords(False) + [password]): try: self.logDebug("Try password: %s" % pw) - if plugin.checkPassword(pw): - plugin.extract(progress, pw) + if archive.checkPassword(pw): + archive.extract(pw) self.addPassword(pw) success = True break - except WrongPassword: + except PasswordError: self.logDebug("Password was wrong") + else: + raise PasswordError - if not success: - raise Exception(_("Wrong password")) + pyfile.setProgress(100) + pyfile.setStatus("processing") if self.core.debug: - self.logDebug("Would delete", ", ".join(plugin.getDeleteFiles())) + self.logDebug("Would delete: %s" % ", ".join(plugin.getDeleteFiles())) - if deletearchive: - files = plugin.getDeleteFiles() + if self.getConfig("delete"): + files = archive.getDeleteFiles() self.logInfo(_("Deleting %s files") % len(files)) for f in files: - if exists(f): - remove(f) + if os.path.exists(f): + os.remove(f) else: self.logDebug("%s does not exists" % f) - self.logInfo(basename(plugin.file), _("Extracting finished")) + self.logInfo(fname, _("Extracting finished")) - extracted_files = plugin.getExtractedFiles() - self.manager.dispatchEvent("archive_extracted", pyfile, plugin.out, plugin.file, extracted_files) + extracted_files = archive.getExtractedFiles() + self.manager.dispatchEvent("archive_extracted", pyfile, archive.out, archive.file, extracted_files) return extracted_files - except ArchiveError, e: - self.logError(basename(plugin.file), _("Archive Error"), e) + except PasswordError: + self.logError(fname, _("Wrong password" if password else "No password found")) except CRCError: - self.logError(basename(plugin.file), _("CRC Mismatch")) + self.logError(fname, _("CRC Mismatch")) + + except ArchiveError, e: + self.logError(fname, _("Archive Error"), e) except Exception, e: if self.core.debug: print_exc() - self.logError(basename(plugin.file), _("Unknown Error"), e) + self.logError(fname, _("Unknown Error"), e) + + finally: + pyfile.finishIfDone() self.manager.dispatchEvent("archive_extract_failed", pyfile) raise Exception(_("Extract failed")) + def getQueue(self): + return self.getStorage("ExtractArchive", []) + + + def addToQueue(self, item): + queue = self.getQueue() + return self.setStorage("ExtractArchive", queue + [item] if item not in queue else queue) + + @Expose - def getPasswords(self): + def getPasswords(self, reload=True): """ List of saved passwords """ + if reload: + self.reloadPasswords() + return self.passwords def reloadPasswords(self): - passwordfile = self.getConfig("passwordfile") - try: passwords = [] - with open(passwordfile, "a+") as f: + with open(self.getConfig("passwordfile")) as f: for pw in f.read().splitlines(): passwords.append(pw) @@ -335,17 +388,12 @@ class ExtractArchive(Hook): @Expose def addPassword(self, pw): """ Adds a password to saved list""" - passwordfile = self.getConfig("passwordfile") - - if pw in self.passwords: - self.passwords.remove(pw) - - self.passwords.insert(0, pw) - try: - with open(passwordfile, "wb") as f: + self.passwords = uniqify([pw] + self.passwords) + + with open(self.getConfig("passwordfile"), "wb") as f: for pw in self.passwords: - f.write(pw + "\n") + f.write(pw + '\n') except IOError, e: self.logError(e) @@ -353,20 +401,21 @@ class ExtractArchive(Hook): def setPermissions(self, files): for f in files: - if not exists(f): + if not os.path.exists(f): continue try: if self.config['permission']['change_file']: - if isfile(f): - chmod(f, int(self.config['permission']['file'], 8)) - elif isdir(f): - chmod(f, int(self.config['permission']['folder'], 8)) + if os.path.isfile(f): + os.chmod(f, int(self.config['permission']['file'], 8)) + + elif os.path.isdir(f): + os.chmod(f, int(self.config['permission']['folder'], 8)) if self.config['permission']['change_dl'] and os.name != "nt": uid = getpwnam(self.config['permission']['user'])[2] gid = getgrnam(self.config['permission']['group'])[2] - chown(f, uid, gid) + os.chown(f, uid, gid) except Exception, e: self.logWarning(_("Setting User and Group failed"), e) -- cgit v1.2.3 From 6cdce819e11b9bd1f3048513e7a1e0e5cf70a441 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 28 Jan 2015 01:16:37 +0100 Subject: [ExtractArchive] Fix typo --- module/plugins/hooks/ExtractArchive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 11764759f..5c4e9bc82 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -55,13 +55,13 @@ from module.utils import fs_decode, save_join, uniqify class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.09" + __version__ = "1.10" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract full path" , True ), ("overwrite" , "bool" , "Overwrite files" , False ), ("keepbroken" , "bool" , "Extract broken archives" , False ), - # ("repair" , "bool" , "Repair broken archives" , True ), + ("repair" , "bool" , "Repair broken archives" , True ), ("passwordfile" , "file" , "Store passwords in file" , "archive_password.txt" ), ("delete" , "bool" , "Delete archive when successfully extracted", False ), ("subfolder" , "bool" , "Create subfolder for each package" , False ), -- cgit v1.2.3 From 52acc62381d165b615ebbb4414d846c7aee462d5 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 29 Jan 2015 13:34:33 +0100 Subject: [ExtractArchive] Fix typo (thx SelmaUrban) --- module/plugins/hooks/ExtractArchive.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 5c4e9bc82..c24173964 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -55,7 +55,7 @@ from module.utils import fs_decode, save_join, uniqify class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.10" + __version__ = "1.11" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract full path" , True ), @@ -74,9 +74,7 @@ class ExtractArchive(Hook): __description__ = """Extract different kind of archives""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "ranan@pyload.org"), - ("AndroKev", ""), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] event_list = ["allDownloadsProcessed"] @@ -100,7 +98,7 @@ class ExtractArchive(Hook): for p in ("UnRar", "SevenZip", "UnZip"): try: module = self.core.pluginManager.loadModule("internal", p) - klass = getattr(module, p) + klass = getattr(module, p) if klass.checkDeps(): names.append(p) self.extractors.append(klass) @@ -164,7 +162,6 @@ class ExtractArchive(Hook): overwrite = self.getConfig("overwrite") extensions = clearList(self.getConfig("extensions")) excludefiles = clearList(self.getConfig("excludefiles")) - excludefiles = self.getConfig("excludefiles") renice = self.getConfig("renice") recursive = self.getConfig("recursive") delete = self.getConfig("delete") @@ -386,10 +383,10 @@ class ExtractArchive(Hook): @Expose - def addPassword(self, pw): + def addPassword(self, password): """ Adds a password to saved list""" try: - self.passwords = uniqify([pw] + self.passwords) + self.passwords = uniqify([password] + self.passwords) with open(self.getConfig("passwordfile"), "wb") as f: for pw in self.passwords: -- cgit v1.2.3 From cb9e67a5437ddfafd6a93f5a208b9faf3f2d5575 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 29 Jan 2015 15:56:57 +0100 Subject: Some file encoding fixup + optimizations --- module/plugins/hooks/ExtractArchive.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index c24173964..11427109b 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -49,7 +49,7 @@ if os.name != "nt": from module.plugins.Hook import Hook, threaded, Expose from module.plugins.internal.Extractor import ArchiveError, CRCError, PasswordError -from module.utils import fs_decode, save_join, uniqify +from module.utils import fs_decode, fs_encode, save_join, uniqify class ExtractArchive(Hook): @@ -371,7 +371,9 @@ class ExtractArchive(Hook): def reloadPasswords(self): try: passwords = [] - with open(self.getConfig("passwordfile")) as f: + + file = fs_encode(self.getConfig("passwordfile")) + with open(file) as f: for pw in f.read().splitlines(): passwords.append(pw) @@ -388,7 +390,8 @@ class ExtractArchive(Hook): try: self.passwords = uniqify([password] + self.passwords) - with open(self.getConfig("passwordfile"), "wb") as f: + file = fs_encode(self.getConfig("passwordfile")) + with open(file, "wb") as f: for pw in self.passwords: f.write(pw + '\n') -- cgit v1.2.3 From 788a06132882300a22f6db3aa7ac3a6009d4d762 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 29 Jan 2015 23:13:54 +0100 Subject: Update Extractor (2) --- module/plugins/hooks/ExtractArchive.py | 159 ++++++++++++++++++++++----------- 1 file changed, 105 insertions(+), 54 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 11427109b..f2bc11ec2 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -12,12 +12,14 @@ from traceback import print_exc # http://bugs.python.org/issue6122 , http://bugs.python.org/issue1236 , http://bugs.python.org/issue1731717 if sys.version_info < (2, 7) and os.name != "nt": import errno + from subprocess import Popen def _eintr_retry_call(func, *args): while True: try: return func(*args) + except OSError, e: if e.errno == errno.EINTR: continue @@ -49,28 +51,64 @@ if os.name != "nt": from module.plugins.Hook import Hook, threaded, Expose from module.plugins.internal.Extractor import ArchiveError, CRCError, PasswordError -from module.utils import fs_decode, fs_encode, save_join, uniqify +from module.utils import fs_encode, save_join, uniqify + + +class ArchiveQueue(object): + + def __init__(self, plugin, storage): + self.plugin = plugin + self.storage = storage + + + def get(self): + return self.plugin.getStorage("ExtractArchive:%s" % storage, []) + + + def set(self, value): + return self.plugin.setStorage("ExtractArchive:%s" % storage, value) + + + def clean(self): + return self.set([]) + + + def add(self, item): + queue = self.get() + if item not in queue: + return self.set(queue + [item]) + else: + return True + + + def remove(self, item): + queue = self.get() + queue.pop(item, None) + return self.set(queue) + class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.11" - - __config__ = [("activated" , "bool" , "Activated" , True ), - ("fullpath" , "bool" , "Extract full path" , True ), - ("overwrite" , "bool" , "Overwrite files" , False ), - ("keepbroken" , "bool" , "Extract broken archives" , False ), - ("repair" , "bool" , "Repair broken archives" , True ), - ("passwordfile" , "file" , "Store passwords in file" , "archive_password.txt" ), - ("delete" , "bool" , "Delete archive when successfully extracted", False ), - ("subfolder" , "bool" , "Create subfolder for each package" , False ), - ("destination" , "folder", "Extract files to" , "" ), - ("extensions" , "str" , "Extract the following extensions" , "7z,bz2,bzip2,gz,gzip,lha,lzh,lzma,rar,tar,taz,tbz,tbz2,tgz,xar,xz,z,zip"), - ("excludefiles" , "str" , "Don't extract the following files" , "*.nfo,*.DS_Store,index.dat,thumb.db" ), - ("recursive" , "bool" , "Extract archives in archives" , True ), - ("queue" , "bool" , "Wait for all downloads to be finished" , False ), - ("renice" , "int" , "CPU Priority" , 0 )] + __version__ = "1.12" + + __config__ = [("activated" , "bool" , "Activated" , True ), + ("fullpath" , "bool" , "Extract full path" , True ), + ("overwrite" , "bool" , "Overwrite files" , False ), + ("keepbroken" , "bool" , "Try to extract broken archives" , False ), + ("repair" , "bool" , "Repair broken archives" , False ), + ("extractempty" , "bool" , "Extract empty archives" , True ), + ("usepasswordfile" , "bool" , "Use password file" , True ), + ("passwordfile" , "file" , "Password file" , "archive_password.txt" ), + ("delete" , "bool" , "Delete archive when successfully extracted", False ), + ("subfolder" , "bool" , "Create subfolder for each package" , False ), + ("destination" , "folder", "Extract files to folder" , "" ), + ("extensions" , "str" , "Extract the following extensions" , "7z,bz2,bzip2,gz,gzip,lha,lzh,lzma,rar,tar,taz,tbz,tbz2,tgz,xar,xz,z,zip"), + ("excludefiles" , "str" , "Don't extract the following files" , "*.nfo,*.DS_Store,index.dat,thumb.db" ), + ("recursive" , "bool" , "Extract archives in archives" , True ), + ("queue" , "bool" , "Wait for all downloads to be finished" , False ), + ("renice" , "int" , "CPU priority" , 0 )] __description__ = """Extract different kind of archives""" __license__ = "GPLv3" @@ -90,6 +128,9 @@ class ExtractArchive(Hook): def setup(self): + self.queue = ArchiveQueue(self, "Queue") + self.failed = ArchiveQueue(self, "Failed") + self.interval = 300 self.extractors = [] self.passwords = [] @@ -124,7 +165,7 @@ class ExtractArchive(Hook): def periodical(self): if not self.extracting: - self.extractPackage(*self.getQueue()) + self.extractPackage(*self.queue.get()) @Expose @@ -136,14 +177,14 @@ class ExtractArchive(Hook): def packageFinished(self, pypack): if self.extracting or self.getConfig("queue"): self.logInfo(_("Package %s queued for later extracting") % pypack.name) - self.addToQueue(pypack.id) + self.queue.add(pypack.id) else: self.extractPackage(pypack.id) @threaded def allDownloadsProcessed(self): - if self.extract(self.getQueue()): #@NOTE: check only if all gone fine, no failed reporting for now + if self.extract(self.queue.get()): #@NOTE: check only if all gone fine, no failed reporting for now self.manager.dispatchEvent("all_archives_extracted") self.manager.dispatchEvent("all_archives_processed") @@ -180,11 +221,11 @@ class ExtractArchive(Hook): for pid in ids: pypack = self.core.files.getPackage(pid) - self.logInfo(_("Check package: %s") % pypack.name) - if not pypack: continue + self.logInfo(_("Check package: %s") % pypack.name) + # determine output folder out = save_join(dl, pypack.folder, destination, "") #: force trailing slash @@ -224,19 +265,19 @@ class ExtractArchive(Hook): try: self.extracting = True - klass = Extractor(self, - filename, - out, - fullpath, - overwrite, - excludefiles, - renice, - delete, - keepbroken, - fid) - klass.init() + archive = Extractor(self, + filename, + out, + fullpath, + overwrite, + excludefiles, + renice, + delete, + keepbroken, + fid) + archive.init() - new_files = self._extract(klass, fid, pypack.password) + new_files = self._extract(archive, fid, pypack.password) except Exception, e: self.logError(fname, e) @@ -250,6 +291,7 @@ class ExtractArchive(Hook): if not os.path.exists(file): self.logDebug("New file %s does not exists" % file) continue + if recursive and os.path.isfile(file): new_files_ids.append((file, fid)) # append as new target @@ -262,43 +304,60 @@ class ExtractArchive(Hook): else: failed.append(pid) self.manager.dispatchEvent("package_extract_failed", pypack) + + self.failed.add(pid) else: self.logInfo(_("No files found to extract")) if not matched or not success and subfolder: try: os.rmdir(out) + except OSError: pass + self.queue.remove(pid) + self.extracting = False return True if not failed else False def _extract(self, archive, fid, password): pyfile = self.core.files.getFile(fid) - fname = os.path.basename(fs_decode(archive.target)) + fname = os.path.basename(archive.filename) pyfile.setCustomStatus(_("extracting")) pyfile.setProgress(0) try: - success = False + try: + archive.check() - if not archive.checkArchive(): - archive.extract(password) - success = True - else: + except CRCError: + self.logInfo(fname, _("Header protected")) + + if self.getConfig("repair"): + self.logWarning(fname, "Repairing...") + archive.repair() + + except PasswordError): self.logInfo(fname, _("Password protected")) - self.logDebug("Password: %s" % (password or "No provided")) + except ArchiveError, e: + if e != "Empty Archive" or not self.getConfig("extractempty"): + raise ArchiveError(e) + + self.logDebug("Password: %s" % (password or "No provided")) + + if not self.getConfig("usepasswordfile"): + archive.extract(password) + else: for pw in set(self.getPasswords(False) + [password]): try: self.logDebug("Try password: %s" % pw) - if archive.checkPassword(pw): + if archive.isPassword(pw): archive.extract(pw) self.addPassword(pw) - success = True break except PasswordError: @@ -316,8 +375,9 @@ class ExtractArchive(Hook): files = archive.getDeleteFiles() self.logInfo(_("Deleting %s files") % len(files)) for f in files: - if os.path.exists(f): - os.remove(f) + file = fs_encode(f) + if os.path.exists(file): + os.remove(file) else: self.logDebug("%s does not exists" % f) @@ -350,15 +410,6 @@ class ExtractArchive(Hook): raise Exception(_("Extract failed")) - def getQueue(self): - return self.getStorage("ExtractArchive", []) - - - def addToQueue(self, item): - queue = self.getQueue() - return self.setStorage("ExtractArchive", queue + [item] if item not in queue else queue) - - @Expose def getPasswords(self, reload=True): """ List of saved passwords """ -- cgit v1.2.3 From 096e457a5a44e1ade87504147468fda045e07a88 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 29 Jan 2015 23:19:39 +0100 Subject: [ExtractArchive] Fix password recognition --- module/plugins/hooks/ExtractArchive.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index f2bc11ec2..8c5af1bb5 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -329,6 +329,8 @@ class ExtractArchive(Hook): pyfile.setCustomStatus(_("extracting")) pyfile.setProgress(0) + encrypted = False + try: try: archive.check() @@ -337,11 +339,12 @@ class ExtractArchive(Hook): self.logInfo(fname, _("Header protected")) if self.getConfig("repair"): - self.logWarning(fname, "Repairing...") + self.logWarning(fname, _("Repairing...")) archive.repair() except PasswordError): self.logInfo(fname, _("Password protected")) + encrypted = True except ArchiveError, e: if e != "Empty Archive" or not self.getConfig("extractempty"): @@ -349,7 +352,7 @@ class ExtractArchive(Hook): self.logDebug("Password: %s" % (password or "No provided")) - if not self.getConfig("usepasswordfile"): + if not encrypted or not self.getConfig("usepasswordfile"): archive.extract(password) else: for pw in set(self.getPasswords(False) + [password]): -- cgit v1.2.3 From 9f5aa7aec672ee184a640910fcda2d818e8f19fe Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 29 Jan 2015 23:55:40 +0100 Subject: [ExtractArchive] Fix https://github.com/pyload/pyload/issues/1100 --- module/plugins/hooks/ExtractArchive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 8c5af1bb5..c91228850 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -91,7 +91,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.12" + __version__ = "1.13" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract full path" , True ), @@ -342,7 +342,7 @@ class ExtractArchive(Hook): self.logWarning(fname, _("Repairing...")) archive.repair() - except PasswordError): + except PasswordError: self.logInfo(fname, _("Password protected")) encrypted = True -- cgit v1.2.3 From ddccf7b36d7a941e4ab5539ff0c35b1367fe5337 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 31 Jan 2015 03:45:19 +0100 Subject: [ExtractArchive] Fix typo Fix https://github.com/pyload/pyload/issues/1011#issuecomment-72268195, https://github.com/pyload/pyload/issues/1113, https://github.com/pyload/pyload/issues/1115 --- module/plugins/hooks/ExtractArchive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index c91228850..f1b73bf5f 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -62,11 +62,11 @@ class ArchiveQueue(object): def get(self): - return self.plugin.getStorage("ExtractArchive:%s" % storage, []) + return self.plugin.getStorage("ExtractArchive:%s" % self.storage, []) def set(self, value): - return self.plugin.setStorage("ExtractArchive:%s" % storage, value) + return self.plugin.setStorage("ExtractArchive:%s" % self.storage, value) def clean(self): @@ -91,7 +91,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.13" + __version__ = "1.14" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract full path" , True ), -- cgit v1.2.3 From 4eb3ecd44244734463d9fdec1a68cf25dcb0b625 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 31 Jan 2015 11:19:45 +0100 Subject: [ExtractArchive] Fix https://github.com/pyload/pyload/issues/1109 and https://github.com/pyload/pyload/issues/1116 --- module/plugins/hooks/ExtractArchive.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index f1b73bf5f..a09e96d1f 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -62,17 +62,25 @@ class ArchiveQueue(object): def get(self): - return self.plugin.getStorage("ExtractArchive:%s" % self.storage, []) + return self.plugin.getStorage("ExtractArchive:%s" % self.storage, "").decode('base64').split() def set(self, value): - return self.plugin.setStorage("ExtractArchive:%s" % self.storage, value) + if isinstance(value, list): + item = str(value)[1:-1].replace(' ', '').replace(',', ' ') + else: + item = str(value).strip() + return self.plugin.setStorage("ExtractArchive:%s" % self.storage, item.encode('base64')[:-1]) def clean(self): return self.set([]) + def delete(self): + return self.plugin.delStorage("ExtractArchive:%s" % self.storage) + + def add(self, item): queue = self.get() if item not in queue: @@ -91,7 +99,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.14" + __version__ = "1.15" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract full path" , True ), @@ -387,7 +395,7 @@ class ExtractArchive(Hook): self.logInfo(fname, _("Extracting finished")) extracted_files = archive.getExtractedFiles() - self.manager.dispatchEvent("archive_extracted", pyfile, archive.out, archive.file, extracted_files) + self.manager.dispatchEvent("archive_extracted", pyfile, archive.out, archive.filename, extracted_files) return extracted_files -- cgit v1.2.3 From 9bcc44f285360334980d375601f4e69ac5e0d2af Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 31 Jan 2015 12:02:03 +0100 Subject: [ExtractArchive] Fix typo (thx VinceGitHub) --- module/plugins/hooks/ExtractArchive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index a09e96d1f..494381124 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -99,7 +99,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.15" + __version__ = "1.16" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract full path" , True ), @@ -380,7 +380,7 @@ class ExtractArchive(Hook): pyfile.setStatus("processing") if self.core.debug: - self.logDebug("Would delete: %s" % ", ".join(plugin.getDeleteFiles())) + self.logDebug("Would delete: %s" % ", ".join(archive.getDeleteFiles())) if self.getConfig("delete"): files = archive.getDeleteFiles() -- cgit v1.2.3 From cfdc545c0e0f85450d142fcc64af5bd37ff6da13 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 31 Jan 2015 23:21:17 +0100 Subject: [ExtractArchive] Pre-final fixup --- module/plugins/hooks/ExtractArchive.py | 79 ++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 37 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 494381124..f1641d639 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -99,14 +99,13 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.16" + __version__ = "1.20" __config__ = [("activated" , "bool" , "Activated" , True ), - ("fullpath" , "bool" , "Extract full path" , True ), + ("fullpath" , "bool" , "Extract with full paths" , True ), ("overwrite" , "bool" , "Overwrite files" , False ), ("keepbroken" , "bool" , "Try to extract broken archives" , False ), - ("repair" , "bool" , "Repair broken archives" , False ), - ("extractempty" , "bool" , "Extract empty archives" , True ), + ("repair" , "bool" , "Repair broken archives" , True ), ("usepasswordfile" , "bool" , "Use password file" , True ), ("passwordfile" , "file" , "Password file" , "archive_password.txt" ), ("delete" , "bool" , "Delete archive when successfully extracted", False ), @@ -115,7 +114,7 @@ class ExtractArchive(Hook): ("extensions" , "str" , "Extract the following extensions" , "7z,bz2,bzip2,gz,gzip,lha,lzh,lzma,rar,tar,taz,tbz,tbz2,tgz,xar,xz,z,zip"), ("excludefiles" , "str" , "Don't extract the following files" , "*.nfo,*.DS_Store,index.dat,thumb.db" ), ("recursive" , "bool" , "Extract archives in archives" , True ), - ("queue" , "bool" , "Wait for all downloads to be finished" , False ), + ("waitall" , "bool" , "Wait for all downloads to be finished" , False ), ("renice" , "int" , "CPU priority" , 0 )] __description__ = """Extract different kind of archives""" @@ -148,7 +147,7 @@ class ExtractArchive(Hook): try: module = self.core.pluginManager.loadModule("internal", p) klass = getattr(module, p) - if klass.checkDeps(): + if klass.isUsable(): names.append(p) self.extractors.append(klass) @@ -183,7 +182,7 @@ class ExtractArchive(Hook): def packageFinished(self, pypack): - if self.extracting or self.getConfig("queue"): + if self.extracting or self.getConfig("waitall"): self.logInfo(_("Package %s queued for later extracting") % pypack.name) self.queue.add(pypack.id) else: @@ -203,19 +202,20 @@ class ExtractArchive(Hook): extracted = [] failed = [] - clearList = lambda string: [x.lstrip('.') for x in string.replace(' ', '').replace(',', '|').replace(';', '|').split('|')] + toList = lambda string: string.replace(' ', '').replace(',', '|').replace(';', '|').split('|') destination = self.getConfig("destination") subfolder = self.getConfig("subfolder") fullpath = self.getConfig("fullpath") overwrite = self.getConfig("overwrite") - extensions = clearList(self.getConfig("extensions")) - excludefiles = clearList(self.getConfig("excludefiles")) renice = self.getConfig("renice") recursive = self.getConfig("recursive") delete = self.getConfig("delete") keepbroken = self.getConfig("keepbroken") + extensions = [x.lstrip('.').lower() for x in toList(self.getConfig("extensions"))] + excludefiles = toList(self.getConfig("excludefiles")) + if extensions: self.logDebug("Extensions: %s" % "|.".join(extensions)) @@ -245,14 +245,15 @@ class ExtractArchive(Hook): matched = False success = True - files_ids = [(save_join(dl, pypack.folder, x['name']), x['id']) for x in pypack.getChildren().itervalues()] + files_ids = [(save_join(dl, pypack.folder, pylink['name']), pylink['id']) for pylink in pypack.getChildren().itervalues()] # check as long there are unseen files while files_ids: new_files_ids = [] if extensions: - files_ids = [(file, id) for file, id in files_ids if filter(lambda ext: file.endswith(ext), extensions)] + files_ids = [(fname, fid) for fname, id in files_ids \ + if filter(lambda ext: fname.lower().endswith(ext), extensions)] for Extractor in self.extractors: targets = Extractor.getTargets(files_ids) @@ -260,21 +261,21 @@ class ExtractArchive(Hook): self.logDebug("Targets for %s: %s" % (Extractor.__name__, targets)) matched = True - for filename, fid in targets: - fname = os.path.basename(filename) + for fname, fid in targets: + name = os.path.basename(fname) - if filename in processed: - self.logDebug(fname, "Skipped") + if fname in processed: + self.logDebug(name, "Skipped") continue - processed.append(filename) # prevent extracting same file twice + processed.append(fname) # prevent extracting same file twice - self.logInfo(fname, _("Extract to: %s") % out) + self.logInfo(name, _("Extract to: %s") % out) try: self.extracting = True archive = Extractor(self, - filename, + fname, out, fullpath, overwrite, @@ -288,20 +289,21 @@ class ExtractArchive(Hook): new_files = self._extract(archive, fid, pypack.password) except Exception, e: - self.logError(fname, e) + self.logError(name, e) success = False continue self.logDebug("Extracted files: %s" % new_files) self.setPermissions(new_files) - for file in new_files: + for filename in new_files: + file = fs_encode(filename) if not os.path.exists(file): - self.logDebug("New file %s does not exists" % file) + self.logDebug("New file %s does not exists" % filename) continue if recursive and os.path.isfile(file): - new_files_ids.append((file, fid)) # append as new target + new_files_ids.append((filename, fid)) # append as new target files_ids = new_files_ids # also check extracted files @@ -332,7 +334,7 @@ class ExtractArchive(Hook): def _extract(self, archive, fid, password): pyfile = self.core.files.getFile(fid) - fname = os.path.basename(archive.filename) + name = os.path.basename(archive.filename) pyfile.setCustomStatus(_("extracting")) pyfile.setProgress(0) @@ -343,20 +345,23 @@ class ExtractArchive(Hook): try: archive.check() - except CRCError: - self.logInfo(fname, _("Header protected")) + except CRCError, e: + self.logDebug(name, e) + self.logInfo(name, _("Header protected")) if self.getConfig("repair"): - self.logWarning(fname, _("Repairing...")) - archive.repair() + self.logWarning(name, _("Repairing...")) + repaired = archive.repair() + + if not repaired and not self.getConfig("keepbroken"): + raise CRCError("Archive damaged") except PasswordError: - self.logInfo(fname, _("Password protected")) + self.logInfo(name, _("Password protected")) encrypted = True except ArchiveError, e: - if e != "Empty Archive" or not self.getConfig("extractempty"): - raise ArchiveError(e) + raise ArchiveError(e) self.logDebug("Password: %s" % (password or "No provided")) @@ -392,7 +397,7 @@ class ExtractArchive(Hook): else: self.logDebug("%s does not exists" % f) - self.logInfo(fname, _("Extracting finished")) + self.logInfo(name, _("Extracting finished")) extracted_files = archive.getExtractedFiles() self.manager.dispatchEvent("archive_extracted", pyfile, archive.out, archive.filename, extracted_files) @@ -400,18 +405,18 @@ class ExtractArchive(Hook): return extracted_files except PasswordError: - self.logError(fname, _("Wrong password" if password else "No password found")) + self.logError(name, _("Wrong password" if password else "No password found")) - except CRCError: - self.logError(fname, _("CRC Mismatch")) + except CRCError, e: + self.logError(name, _("CRC mismatch"), e) except ArchiveError, e: - self.logError(fname, _("Archive Error"), e) + self.logError(name, _("Archive error"), e) except Exception, e: + self.logError(name, _("Unknown error"), e) if self.core.debug: print_exc() - self.logError(fname, _("Unknown Error"), e) finally: pyfile.finishIfDone() -- cgit v1.2.3 From b4515ea17dd64df7647f7d7d91ebc0c8a46d5433 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 31 Jan 2015 23:52:43 +0100 Subject: Fix https://github.com/pyload/pyload/issues/1120 (2) --- module/plugins/hooks/ExtractArchive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index f1641d639..dde1a5ddc 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -99,7 +99,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.20" + __version__ = "1.21" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), @@ -252,7 +252,7 @@ class ExtractArchive(Hook): new_files_ids = [] if extensions: - files_ids = [(fname, fid) for fname, id in files_ids \ + files_ids = [(fname, fid) for fname, fid in files_ids \ if filter(lambda ext: fname.lower().endswith(ext), extensions)] for Extractor in self.extractors: -- cgit v1.2.3 From 8bda0302c7a9452a1828ec9c9e2f60901d25e3c5 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 1 Feb 2015 00:08:07 +0100 Subject: [ExtractArchive] Fix http://forum.pyload.org/viewtopic.php?f=10&t=3990 --- module/plugins/hooks/ExtractArchive.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index dde1a5ddc..dd1a82a82 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -91,7 +91,10 @@ class ArchiveQueue(object): def remove(self, item): queue = self.get() - queue.pop(item, None) + try: + queue.remove(item) + except ValueError: + pass return self.set(queue) @@ -99,7 +102,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.21" + __version__ = "1.22" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), -- cgit v1.2.3 From 8dfb7adc0fc3c858c0ddf9371c2f4580bb8be3c7 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 3 Feb 2015 00:10:11 +0100 Subject: Update Extractor (3) --- module/plugins/hooks/ExtractArchive.py | 74 +++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 28 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index dd1a82a82..74dded9b7 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -51,6 +51,7 @@ if os.name != "nt": from module.plugins.Hook import Hook, threaded, Expose from module.plugins.internal.Extractor import ArchiveError, CRCError, PasswordError +from module.plugins.internal.SimpleHoster import replace_patterns from module.utils import fs_encode, save_join, uniqify @@ -94,7 +95,7 @@ class ArchiveQueue(object): try: queue.remove(item) except ValueError: - pass + pass return self.set(queue) @@ -102,7 +103,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.22" + __version__ = "1.23" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), @@ -127,48 +128,55 @@ class ExtractArchive(Hook): event_list = ["allDownloadsProcessed"] + NAME_REPLACEMENTS = [(r'\.part\d+\.rar$', ".part.rar")] + #@TODO: Remove in 0.4.10 def initPeriodical(self): pass - def coreReady(self): - self.extracting = False - - def setup(self): self.queue = ArchiveQueue(self, "Queue") self.failed = ArchiveQueue(self, "Failed") self.interval = 300 + self.extracting = False self.extractors = [] self.passwords = [] - names = [] + + def coreReady(self): + # self.extracting = False + for p in ("UnRar", "SevenZip", "UnZip"): try: module = self.core.pluginManager.loadModule("internal", p) klass = getattr(module, p) if klass.isUsable(): - names.append(p) self.extractors.append(klass) except OSError, e: if e.errno == 2: self.logInfo(_("No %s installed") % p) else: - self.logWarning(_("Could not activate %s") % p, e) + self.logWarning(_("Could not activate: %s") % p, e) if self.core.debug: print_exc() except Exception, e: - self.logWarning(_("Could not activate %s") % p, e) + self.logWarning(_("Could not activate: %s") % p, e) if self.core.debug: print_exc() - if names: - self.logInfo(_("Activated") + " " + " ".join(names)) + if self.extractors: + self.logInfo(_("Activated") + " " + " ".join(Extractor.__name__ for Extractor in self.extractors)) + + if self.getConfig("waitall"): + self.extractPackage(*self.queue.get()) #: Resume unfinished extractions + else: + super(ExtractArchive, self).initPeriodical() + else: self.logInfo(_("No Extract plugins activated")) @@ -185,11 +193,7 @@ class ExtractArchive(Hook): def packageFinished(self, pypack): - if self.extracting or self.getConfig("waitall"): - self.logInfo(_("Package %s queued for later extracting") % pypack.name) - self.queue.add(pypack.id) - else: - self.extractPackage(pypack.id) + self.queue.add(pypack.id) @threaded @@ -201,6 +205,8 @@ class ExtractArchive(Hook): def extract(self, ids): + self.extracting = True + processed = [] extracted = [] failed = [] @@ -267,16 +273,19 @@ class ExtractArchive(Hook): for fname, fid in targets: name = os.path.basename(fname) - if fname in processed: - self.logDebug(name, "Skipped") + if not os.path.exists(fname): + self.logDebug(name, "File not found") continue - processed.append(fname) # prevent extracting same file twice + pname = replace_patterns(fname, self.NAME_REPLACEMENTS) + if pname not in processed: + processed.append(pname) #: prevent extracting same file twice + else: + self.logDebug(name, "Skipped") + continue self.logInfo(name, _("Extract to: %s") % out) try: - self.extracting = True - archive = Extractor(self, fname, out, @@ -339,11 +348,9 @@ class ExtractArchive(Hook): pyfile = self.core.files.getFile(fid) name = os.path.basename(archive.filename) - pyfile.setCustomStatus(_("extracting")) - pyfile.setProgress(0) + pyfile.setStatus("processing") encrypted = False - try: try: archive.check() @@ -354,8 +361,14 @@ class ExtractArchive(Hook): if self.getConfig("repair"): self.logWarning(name, _("Repairing...")) + + pyfile.setCustomStatus(_("repairing")) + pyfile.setProgress(0) + repaired = archive.repair() + pyfile.setProgress(100) + if not repaired and not self.getConfig("keepbroken"): raise CRCError("Archive damaged") @@ -368,13 +381,18 @@ class ExtractArchive(Hook): self.logDebug("Password: %s" % (password or "No provided")) + pyfile.setCustomStatus(_("extracting")) + pyfile.setProgress(0) + if not encrypted or not self.getConfig("usepasswordfile"): archive.extract(password) else: for pw in set(self.getPasswords(False) + [password]): try: self.logDebug("Try password: %s" % pw) - if archive.isPassword(pw): + + ispw = archive.isPassword(pw) + if ispw or ispw is None: archive.extract(pw) self.addPassword(pw) break @@ -385,7 +403,7 @@ class ExtractArchive(Hook): raise PasswordError pyfile.setProgress(100) - pyfile.setStatus("processing") + pyfile.setCustomStatus(_("finalizing")) if self.core.debug: self.logDebug("Would delete: %s" % ", ".join(archive.getDeleteFiles())) @@ -402,7 +420,7 @@ class ExtractArchive(Hook): self.logInfo(name, _("Extracting finished")) - extracted_files = archive.getExtractedFiles() + extracted_files = archive.files or archive.list() self.manager.dispatchEvent("archive_extracted", pyfile, archive.out, archive.filename, extracted_files) return extracted_files -- cgit v1.2.3 From 8d8cfd57fa44cb84eb9709871bf14a3b4d13d994 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 3 Feb 2015 01:10:04 +0100 Subject: Update Extractor (4) --- module/plugins/hooks/ExtractArchive.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 74dded9b7..47325608d 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -103,7 +103,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.23" + __version__ = "1.24" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), @@ -205,6 +205,9 @@ class ExtractArchive(Hook): def extract(self, ids): + if not ids: + return False + self.extracting = True processed = [] @@ -226,7 +229,7 @@ class ExtractArchive(Hook): excludefiles = toList(self.getConfig("excludefiles")) if extensions: - self.logDebug("Extensions: %s" % "|.".join(extensions)) + self.logDebug("Use for extensions: %s" % "|.".join(extensions)) # reload from txt file self.reloadPasswords() @@ -462,7 +465,7 @@ class ExtractArchive(Hook): file = fs_encode(self.getConfig("passwordfile")) with open(file) as f: - for pw in f.read().splitlines(): + for pw in f.read().splitlines()[:-1]: passwords.append(pw) except IOError, e: -- cgit v1.2.3 From 896f1f1437442bb9a93a0664f3d447bbc0f444cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Wed, 4 Feb 2015 00:02:27 +0100 Subject: Addressed some Extractor Issues --- module/plugins/hooks/ExtractArchive.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 47325608d..e1c05878c 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -103,7 +103,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.24" + __version__ = "1.25" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), @@ -312,7 +312,7 @@ class ExtractArchive(Hook): self.setPermissions(new_files) for filename in new_files: - file = fs_encode(filename) + file = fs_encode(save_join(filename, os.path.dirname(archive.filename))) if not os.path.exists(file): self.logDebug("New file %s does not exists" % filename) continue @@ -390,7 +390,7 @@ class ExtractArchive(Hook): if not encrypted or not self.getConfig("usepasswordfile"): archive.extract(password) else: - for pw in set(self.getPasswords(False) + [password]): + for pw in uniqify([password] + self.getPasswords(False)): try: self.logDebug("Try password: %s" % pw) @@ -465,7 +465,7 @@ class ExtractArchive(Hook): file = fs_encode(self.getConfig("passwordfile")) with open(file) as f: - for pw in f.read().splitlines()[:-1]: + for pw in f.read().splitlines(): passwords.append(pw) except IOError, e: -- cgit v1.2.3 From 0df373e5da9d33e4dea3e709521d4883ef7f9b45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Wed, 4 Feb 2015 20:41:03 +0100 Subject: [Unrar] Fix: Deleting All Files after Extraction --- module/plugins/hooks/ExtractArchive.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index e1c05878c..e1d6dff59 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -276,16 +276,16 @@ class ExtractArchive(Hook): for fname, fid in targets: name = os.path.basename(fname) - if not os.path.exists(fname): - self.logDebug(name, "File not found") - continue - pname = replace_patterns(fname, self.NAME_REPLACEMENTS) if pname not in processed: processed.append(pname) #: prevent extracting same file twice else: self.logDebug(name, "Skipped") continue + + if not os.path.exists(fname): + self.logDebug(name, "File not found") + continue self.logInfo(name, _("Extract to: %s") % out) try: -- cgit v1.2.3 From c1268bbbe35ebd8d5308051c169160257ad14c6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Wed, 4 Feb 2015 23:14:12 +0100 Subject: [ExtractArchive] fixed Queue Get --- module/plugins/hooks/ExtractArchive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index e1d6dff59..6e6a6862c 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -63,7 +63,7 @@ class ArchiveQueue(object): def get(self): - return self.plugin.getStorage("ExtractArchive:%s" % self.storage, "").decode('base64').split() + return [int(pid) for pid in self.plugin.getStorage("ExtractArchive:%s" % self.storage, "").decode('base64').split()] def set(self, value): -- cgit v1.2.3 From 39e87e03709b663086c96309bd6ebb3e2dc16afc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Thu, 5 Feb 2015 00:24:08 +0100 Subject: set interval smaller --- module/plugins/hooks/ExtractArchive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 6e6a6862c..b52fa026b 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -140,7 +140,7 @@ class ExtractArchive(Hook): self.queue = ArchiveQueue(self, "Queue") self.failed = ArchiveQueue(self, "Failed") - self.interval = 300 + self.interval = 60 self.extracting = False self.extractors = [] self.passwords = [] -- cgit v1.2.3 From 09b692852ae556cb3f40ffb3f607106faecd5758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Thu, 5 Feb 2015 18:41:59 +0100 Subject: [ExtractArchive] Error Handling when broken ExtractArchive was used --- module/plugins/hooks/ExtractArchive.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index b52fa026b..4756d359c 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -63,7 +63,10 @@ class ArchiveQueue(object): def get(self): - return [int(pid) for pid in self.plugin.getStorage("ExtractArchive:%s" % self.storage, "").decode('base64').split()] + try: + return [int(pid) for pid in self.plugin.getStorage("ExtractArchive:%s" % self.storage, "").decode('base64').split()] + except Exception: + return [] def set(self, value): -- cgit v1.2.3 From 767cf1ccb4420e7cb1805bfe302b8909def79679 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Fri, 6 Feb 2015 21:52:10 +0100 Subject: [ExtractArchive] Fixed: Recursive Extraction --- module/plugins/hooks/ExtractArchive.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 4756d359c..e27c544d6 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -77,10 +77,6 @@ class ArchiveQueue(object): return self.plugin.setStorage("ExtractArchive:%s" % self.storage, item.encode('base64')[:-1]) - def clean(self): - return self.set([]) - - def delete(self): return self.plugin.delStorage("ExtractArchive:%s" % self.storage) @@ -99,6 +95,8 @@ class ArchiveQueue(object): queue.remove(item) except ValueError: pass + if queue == []: + return self.delete() return self.set(queue) @@ -315,7 +313,7 @@ class ExtractArchive(Hook): self.setPermissions(new_files) for filename in new_files: - file = fs_encode(save_join(filename, os.path.dirname(archive.filename))) + file = fs_encode(save_join(os.path.dirname(archive.filename), filename)) if not os.path.exists(file): self.logDebug("New file %s does not exists" % filename) continue -- cgit v1.2.3 From d08c34cb6289ef55c6412d6129e343380f7f20d1 Mon Sep 17 00:00:00 2001 From: stickell Date: Sat, 7 Feb 2015 00:03:28 +0100 Subject: [ExtractArchive] bump version number after #1152 --- module/plugins/hooks/ExtractArchive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index e27c544d6..88036da39 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -104,7 +104,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.25" + __version__ = "1.26" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), -- cgit v1.2.3 From c5daf5fa18490ff5977f06416b14bf0474822974 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Sat, 7 Feb 2015 03:08:28 +0100 Subject: [ExtractArchive] Fix: Show Status/Progressbar --- module/plugins/hooks/ExtractArchive.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 88036da39..81655b108 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -198,14 +198,14 @@ class ExtractArchive(Hook): @threaded - def allDownloadsProcessed(self): - if self.extract(self.queue.get()): #@NOTE: check only if all gone fine, no failed reporting for now + def allDownloadsProcessed(self, thread): + if self.extract(self.queue.get(), thread): #@NOTE: check only if all gone fine, no failed reporting for now self.manager.dispatchEvent("all_archives_extracted") self.manager.dispatchEvent("all_archives_processed") - def extract(self, ids): + def extract(self, ids, thread=None): if not ids: return False @@ -302,7 +302,7 @@ class ExtractArchive(Hook): fid) archive.init() - new_files = self._extract(archive, fid, pypack.password) + new_files = self._extract(archive, fid, pypack.password, thread) except Exception, e: self.logError(name, e) @@ -348,10 +348,11 @@ class ExtractArchive(Hook): return True if not failed else False - def _extract(self, archive, fid, password): + def _extract(self, archive, fid, password, thread): pyfile = self.core.files.getFile(fid) name = os.path.basename(archive.filename) + thread.addActive(pyfile) pyfile.setStatus("processing") encrypted = False -- cgit v1.2.3 From 2a809297f288a585d96af0d8afd894c2a2f695fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Sun, 8 Feb 2015 13:47:59 +0100 Subject: [ExtractArchive] correct fullpath behavior, bugfix --- module/plugins/hooks/ExtractArchive.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 81655b108..6c0177ae9 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -104,7 +104,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.26" + __version__ = "1.27" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), @@ -258,14 +258,14 @@ class ExtractArchive(Hook): matched = False success = True - files_ids = [(save_join(dl, pypack.folder, pylink['name']), pylink['id']) for pylink in pypack.getChildren().itervalues()] + files_ids = [(save_join(dl, pypack.folder, pylink['name']), pylink['id'], out) for pylink in pypack.getChildren().itervalues()] # check as long there are unseen files while files_ids: new_files_ids = [] if extensions: - files_ids = [(fname, fid) for fname, fid in files_ids \ + files_ids = [(fname, fid, fout) for fname, fid, fout in files_ids \ if filter(lambda ext: fname.lower().endswith(ext), extensions)] for Extractor in self.extractors: @@ -274,7 +274,7 @@ class ExtractArchive(Hook): self.logDebug("Targets for %s: %s" % (Extractor.__name__, targets)) matched = True - for fname, fid in targets: + for fname, fid, fout in targets: name = os.path.basename(fname) pname = replace_patterns(fname, self.NAME_REPLACEMENTS) @@ -283,16 +283,16 @@ class ExtractArchive(Hook): else: self.logDebug(name, "Skipped") continue - + if not os.path.exists(fname): self.logDebug(name, "File not found") continue - self.logInfo(name, _("Extract to: %s") % out) + self.logInfo(name, _("Extract to: %s") % fout) try: archive = Extractor(self, fname, - out, + fout, fullpath, overwrite, excludefiles, @@ -319,7 +319,7 @@ class ExtractArchive(Hook): continue if recursive and os.path.isfile(file): - new_files_ids.append((filename, fid)) # append as new target + new_files_ids.append((filename, fid, os.path.dirname(filename))) # append as new target files_ids = new_files_ids # also check extracted files -- cgit v1.2.3 From 25fc80b4e973049ff85122fefac9fbf8c2bf4c45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Sun, 8 Feb 2015 17:03:27 +0100 Subject: [ExtractArchive] don't try '' as password --- module/plugins/hooks/ExtractArchive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 6c0177ae9..a0c0ca366 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -104,7 +104,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.27" + __version__ = "1.28" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), @@ -392,7 +392,7 @@ class ExtractArchive(Hook): if not encrypted or not self.getConfig("usepasswordfile"): archive.extract(password) else: - for pw in uniqify([password] + self.getPasswords(False)): + for pw in filter(None, uniqify([password] + self.getPasswords(False))): try: self.logDebug("Try password: %s" % pw) -- cgit v1.2.3 From 2dc3536e36956eab99fa5f7945dcf60073b5fd57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Mon, 9 Feb 2015 23:36:10 +0100 Subject: [ExtractArchive] better Multipart behavior, new version output --- module/plugins/hooks/ExtractArchive.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index a0c0ca366..3ea8839dc 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -104,7 +104,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.28" + __version__ = "1.29" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), @@ -124,7 +124,8 @@ class ExtractArchive(Hook): __description__ = """Extract different kind of archives""" __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com"), + ("Immenz" , "immenz@gmx.net" )] event_list = ["allDownloadsProcessed"] @@ -171,7 +172,7 @@ class ExtractArchive(Hook): print_exc() if self.extractors: - self.logInfo(_("Activated") + " " + " ".join(Extractor.__name__ for Extractor in self.extractors)) + self.logInfo(_("Activated") + " " + "|".join("%s %s" % (Extractor.__name__,Extractor.VERSION) for Extractor in self.extractors)) if self.getConfig("waitall"): self.extractPackage(*self.queue.get()) #: Resume unfinished extractions @@ -277,13 +278,6 @@ class ExtractArchive(Hook): for fname, fid, fout in targets: name = os.path.basename(fname) - pname = replace_patterns(fname, self.NAME_REPLACEMENTS) - if pname not in processed: - processed.append(pname) #: prevent extracting same file twice - else: - self.logDebug(name, "Skipped") - continue - if not os.path.exists(fname): self.logDebug(name, "File not found") continue @@ -309,6 +303,7 @@ class ExtractArchive(Hook): success = False continue + files_ids.remove((fname, fid, fout)) # don't let other extractors spam log self.logDebug("Extracted files: %s" % new_files) self.setPermissions(new_files) -- cgit v1.2.3 From 56caf611ab8f733ac8a7ea09d2a3d492bd485762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Sun, 22 Feb 2015 03:09:40 +0100 Subject: Some more changes --- module/plugins/hooks/ExtractArchive.py | 161 +++++++++++++++++++-------------- 1 file changed, 94 insertions(+), 67 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 3ea8839dc..a1e85ba57 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -104,13 +104,14 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.29" + __version__ = "1.30" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), ("overwrite" , "bool" , "Overwrite files" , False ), ("keepbroken" , "bool" , "Try to extract broken archives" , False ), - ("repair" , "bool" , "Repair broken archives" , True ), + ("repair" , "bool" , "Repair broken archives (rar required)" , False ), + ("test" , "bool" , "Test archive before extracting" , False ), ("usepasswordfile" , "bool" , "Use password file" , True ), ("passwordfile" , "file" , "Password file" , "archive_password.txt" ), ("delete" , "bool" , "Delete archive when successfully extracted", False ), @@ -128,35 +129,32 @@ class ExtractArchive(Hook): ("Immenz" , "immenz@gmx.net" )] - event_list = ["allDownloadsProcessed"] + event_list = ["allDownloadsProcessed","packageDeleted"] NAME_REPLACEMENTS = [(r'\.part\d+\.rar$', ".part.rar")] - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass - - def setup(self): self.queue = ArchiveQueue(self, "Queue") self.failed = ArchiveQueue(self, "Failed") - self.interval = 60 - self.extracting = False - self.extractors = [] - self.passwords = [] + self.interval = 60 + self.extracting = False + self.lastPackage = False + self.extractors = [] + self.passwords = [] + self.repair = False def coreReady(self): - # self.extracting = False - for p in ("UnRar", "SevenZip", "UnZip"): try: module = self.core.pluginManager.loadModule("internal", p) klass = getattr(module, p) if klass.isUsable(): self.extractors.append(klass) + if klass.REPAIR: + self.repair = self.getConfig("repair") except OSError, e: if e.errno == 2: @@ -173,37 +171,49 @@ class ExtractArchive(Hook): if self.extractors: self.logInfo(_("Activated") + " " + "|".join("%s %s" % (Extractor.__name__,Extractor.VERSION) for Extractor in self.extractors)) - - if self.getConfig("waitall"): - self.extractPackage(*self.queue.get()) #: Resume unfinished extractions - else: - super(ExtractArchive, self).initPeriodical() - + self.extractQueued() #: Resume unfinished extractions else: self.logInfo(_("No Extract plugins activated")) + @threaded + def extractQueued(self,thread): + packages = self.queue.get() + while packages: + if self.lastPackage: # called from allDownloadsProcessed + self.lastPackage = False + if self.extract(packages, thread): #@NOTE: check only if all gone fine, no failed reporting for now + self.manager.dispatchEvent("all_archives_extracted") + self.manager.dispatchEvent("all_archives_processed") + else: + if self.extract(packages, thread): #@NOTE: check only if all gone fine, no failed reporting for now + pass - def periodical(self): - if not self.extracting: - self.extractPackage(*self.queue.get()) + packages = self.queue.get() # check for packages added during extraction @Expose def extractPackage(self, *ids): """ Extract packages with given id""" - self.manager.startThread(self.extract, ids) + for id in ids: + self.queue.add(id) + if not self.getConfig("waitall") and not self.extracting: + self.extractQueued() + + + def packageDeleted(self, pid): + self.queue.remove(pid) def packageFinished(self, pypack): self.queue.add(pypack.id) + if not self.getConfig("waitall") and not self.extracting: + self.extractQueued() - @threaded - def allDownloadsProcessed(self, thread): - if self.extract(self.queue.get(), thread): #@NOTE: check only if all gone fine, no failed reporting for now - self.manager.dispatchEvent("all_archives_extracted") - - self.manager.dispatchEvent("all_archives_processed") + def allDownloadsProcessed(self): + self.lastPackage = True + if not self.extracting: + self.extractQueued() def extract(self, ids, thread=None): @@ -315,13 +325,17 @@ class ExtractArchive(Hook): if recursive and os.path.isfile(file): new_files_ids.append((filename, fid, os.path.dirname(filename))) # append as new target - + + pyfile = self.core.files.getFile(fid) + self.manager.dispatchEvent("archive_extracted", pyfile, archive.out, archive.filename, new_files) + files_ids = new_files_ids # also check extracted files if matched: if success: extracted.append(pid) self.manager.dispatchEvent("package_extracted", pypack) + else: failed.append(pid) self.manager.dispatchEvent("package_extract_failed", pypack) @@ -352,50 +366,65 @@ class ExtractArchive(Hook): encrypted = False try: - try: - archive.check() - - except CRCError, e: - self.logDebug(name, e) - self.logInfo(name, _("Header protected")) - - if self.getConfig("repair"): - self.logWarning(name, _("Repairing...")) - - pyfile.setCustomStatus(_("repairing")) - pyfile.setProgress(0) - - repaired = archive.repair() - - pyfile.setProgress(100) - - if not repaired and not self.getConfig("keepbroken"): - raise CRCError("Archive damaged") - - except PasswordError: - self.logInfo(name, _("Password protected")) - encrypted = True - - except ArchiveError, e: - raise ArchiveError(e) - - self.logDebug("Password: %s" % (password or "No provided")) + self.logDebug("Password: %s" % (password or "None provided")) + passwords = uniqify([password] + self.getPasswords(False)) if self.getConfig("usepasswordfile") else [password] + for pw in passwords: + try: + if self.getConfig("test") or self.repair: + pyfile.setCustomStatus(_("testing")) + if pw: + self.logDebug("Testing with password: %s" % pw) + pyfile.setProgress(0) + archive.test(pw) + pyfile.setProgress(100) + else: + archive.check(pw) + + self.addPassword(pw) + break + + except PasswordError: + if not encrypted: + self.logInfo(name, _("Password protected")) + encrypted = True + + except CRCError, e: + self.logDebug(name, e) + self.logInfo(name, _("CRC Error")) + + if self.repair: + self.logWarning(name, _("Repairing...")) + + pyfile.setCustomStatus(_("repairing")) + pyfile.setProgress(0) + repaired = archive.repair() + pyfile.setProgress(100) + + if not repaired and not self.getConfig("keepbroken"): + raise CRCError("Archive damaged") + + self.addPassword(pw) + break + + raise CRCError("Archive damaged") + + except ArchiveError, e: + raise ArchiveError(e) pyfile.setCustomStatus(_("extracting")) pyfile.setProgress(0) if not encrypted or not self.getConfig("usepasswordfile"): + self.logDebug("Extracting using password: %s" % (password or "None")) archive.extract(password) else: for pw in filter(None, uniqify([password] + self.getPasswords(False))): try: - self.logDebug("Try password: %s" % pw) + self.logDebug("Extracting using password: %s" % pw) - ispw = archive.isPassword(pw) - if ispw or ispw is None: - archive.extract(pw) - self.addPassword(pw) - break + archive.extract(pw) + self.addPassword(pw) + break except PasswordError: self.logDebug("Password was wrong") @@ -419,9 +448,7 @@ class ExtractArchive(Hook): self.logDebug("%s does not exists" % f) self.logInfo(name, _("Extracting finished")) - extracted_files = archive.files or archive.list() - self.manager.dispatchEvent("archive_extracted", pyfile, archive.out, archive.filename, extracted_files) return extracted_files -- cgit v1.2.3 From c2e18511923ebda7a8a87ef9c35a9cb88c799e19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Sun, 22 Feb 2015 17:53:32 +0100 Subject: [ExtractArchive] preventing duplicate files extraction --- module/plugins/hooks/ExtractArchive.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index a1e85ba57..8b84966fd 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -104,7 +104,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.30" + __version__ = "1.31" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), @@ -269,7 +269,8 @@ class ExtractArchive(Hook): matched = False success = True - files_ids = [(save_join(dl, pypack.folder, pylink['name']), pylink['id'], out) for pylink in pypack.getChildren().itervalues()] + files_ids = dict((pylink['name'],((save_join(dl, pypack.folder, pylink['name'])), pylink['id'], out)) for pylink \ + in sorted(pypack.getChildren().itervalues(), key=lambda k: k['name'])).values() # remove duplicates # check as long there are unseen files while files_ids: -- cgit v1.2.3 From 0e7a0295c60e93a7e095207176e465e82c03dc9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Tue, 24 Feb 2015 10:28:47 +0100 Subject: [ExtractArchive] fixes https://github.com/pyload/pyload/issues/1206 --- module/plugins/hooks/ExtractArchive.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 8b84966fd..dc0ee3f41 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -104,7 +104,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.31" + __version__ = "1.32" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), @@ -254,6 +254,7 @@ class ExtractArchive(Hook): pypack = self.core.files.getPackage(pid) if not pypack: + self.queue.remove(pid) continue self.logInfo(_("Check package: %s") % pypack.name) -- cgit v1.2.3 From 93eb54614d512396a5505cb9bdea4e201920b434 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 1 Mar 2015 00:33:21 +0100 Subject: Spare code cosmetics --- module/plugins/hooks/ExtractArchive.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index dc0ee3f41..c9e43eaaf 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -327,10 +327,10 @@ class ExtractArchive(Hook): if recursive and os.path.isfile(file): new_files_ids.append((filename, fid, os.path.dirname(filename))) # append as new target - + pyfile = self.core.files.getFile(fid) self.manager.dispatchEvent("archive_extracted", pyfile, archive.out, archive.filename, new_files) - + files_ids = new_files_ids # also check extracted files if matched: @@ -381,7 +381,7 @@ class ExtractArchive(Hook): pyfile.setProgress(100) else: archive.check(pw) - + self.addPassword(pw) break @@ -389,7 +389,7 @@ class ExtractArchive(Hook): if not encrypted: self.logInfo(name, _("Password protected")) encrypted = True - + except CRCError, e: self.logDebug(name, e) self.logInfo(name, _("CRC Error")) @@ -407,9 +407,9 @@ class ExtractArchive(Hook): self.addPassword(pw) break - + raise CRCError("Archive damaged") - + except ArchiveError, e: raise ArchiveError(e) -- cgit v1.2.3 From 3a9c167ceca27d57dbef927626fe853a3d0e30b2 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 1 Mar 2015 22:41:08 +0100 Subject: [Extractor] Use self.target --- module/plugins/hooks/ExtractArchive.py | 37 +++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 16 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index c9e43eaaf..6a25602e8 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -33,6 +33,7 @@ if sys.version_info < (2, 7) and os.name != "nt": if self.returncode is None: try: pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) + except OSError, e: if e.errno != errno.ECHILD: raise @@ -93,10 +94,13 @@ class ArchiveQueue(object): queue = self.get() try: queue.remove(item) + except ValueError: pass + if queue == []: return self.delete() + return self.set(queue) @@ -106,22 +110,22 @@ class ExtractArchive(Hook): __type__ = "hook" __version__ = "1.32" - __config__ = [("activated" , "bool" , "Activated" , True ), - ("fullpath" , "bool" , "Extract with full paths" , True ), - ("overwrite" , "bool" , "Overwrite files" , False ), - ("keepbroken" , "bool" , "Try to extract broken archives" , False ), - ("repair" , "bool" , "Repair broken archives (rar required)" , False ), - ("test" , "bool" , "Test archive before extracting" , False ), - ("usepasswordfile" , "bool" , "Use password file" , True ), - ("passwordfile" , "file" , "Password file" , "archive_password.txt" ), - ("delete" , "bool" , "Delete archive when successfully extracted", False ), - ("subfolder" , "bool" , "Create subfolder for each package" , False ), - ("destination" , "folder", "Extract files to folder" , "" ), - ("extensions" , "str" , "Extract the following extensions" , "7z,bz2,bzip2,gz,gzip,lha,lzh,lzma,rar,tar,taz,tbz,tbz2,tgz,xar,xz,z,zip"), - ("excludefiles" , "str" , "Don't extract the following files" , "*.nfo,*.DS_Store,index.dat,thumb.db" ), - ("recursive" , "bool" , "Extract archives in archives" , True ), - ("waitall" , "bool" , "Wait for all downloads to be finished" , False ), - ("renice" , "int" , "CPU priority" , 0 )] + __config__ = [("activated" , "bool" , "Activated" , True ), + ("fullpath" , "bool" , "Extract with full paths" , True ), + ("overwrite" , "bool" , "Overwrite files" , False ), + ("keepbroken" , "bool" , "Try to extract broken archives" , False ), + ("repair" , "bool" , "Repair broken archives (rar required)" , False ), + ("test" , "bool" , "Test archive before extracting" , False ), + ("usepasswordfile", "bool" , "Use password file" , True ), + ("passwordfile" , "file" , "Password file" , "archive_password.txt" ), + ("delete" , "bool" , "Delete archive when successfully extracted", False ), + ("subfolder" , "bool" , "Create subfolder for each package" , False ), + ("destination" , "folder", "Extract files to folder" , "" ), + ("extensions" , "str" , "Extract the following extensions" , "7z,bz2,bzip2,gz,gzip,lha,lzh,lzma,rar,tar,taz,tbz,tbz2,tgz,xar,xz,z,zip"), + ("excludefiles" , "str" , "Don't extract the following files" , "*.nfo,*.DS_Store,index.dat,thumb.db" ), + ("recursive" , "bool" , "Extract archives in archives" , True ), + ("waitall" , "bool" , "Wait for all downloads to be finished" , False ), + ("renice" , "int" , "CPU priority" , 0 )] __description__ = """Extract different kind of archives""" __license__ = "GPLv3" @@ -175,6 +179,7 @@ class ExtractArchive(Hook): else: self.logInfo(_("No Extract plugins activated")) + @threaded def extractQueued(self,thread): packages = self.queue.get() -- cgit v1.2.3 From 164ff94bd87b8b0de3c872b0b60153ac5abfad4b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 9 Mar 2015 23:26:18 +0100 Subject: [ExtractArchive] Fix https://github.com/pyload/pyload/issues/1241 --- module/plugins/hooks/ExtractArchive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 6a25602e8..edf25b8c7 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -108,7 +108,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.32" + __version__ = "1.33" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), @@ -474,7 +474,7 @@ class ExtractArchive(Hook): print_exc() finally: - pyfile.finishIfDone() + thread.finishFile(pyfile) self.manager.dispatchEvent("archive_extract_failed", pyfile) -- cgit v1.2.3 From 241d6fcca23656cf9690df455a0b7f9a44b472dd Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 20 Mar 2015 01:37:20 +0100 Subject: [ExtractArchive] Update dispatchEvent --- module/plugins/hooks/ExtractArchive.py | 132 +++++++++++++++++---------------- 1 file changed, 67 insertions(+), 65 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index edf25b8c7..0c163078c 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -12,8 +12,7 @@ from traceback import print_exc # http://bugs.python.org/issue6122 , http://bugs.python.org/issue1236 , http://bugs.python.org/issue1731717 if sys.version_info < (2, 7) and os.name != "nt": import errno - - from subprocess import Popen + import subprocess def _eintr_retry_call(func, *args): while True: @@ -44,13 +43,13 @@ if sys.version_info < (2, 7) and os.name != "nt": self._handle_exitstatus(sts) return self.returncode - Popen.wait = wait + subprocess.Popen.wait = wait if os.name != "nt": from grp import getgrnam from pwd import getpwnam -from module.plugins.Hook import Hook, threaded, Expose +from module.plugins.Hook import Hook, Expose, threaded from module.plugins.internal.Extractor import ArchiveError, CRCError, PasswordError from module.plugins.internal.SimpleHoster import replace_patterns from module.utils import fs_encode, save_join, uniqify @@ -104,28 +103,27 @@ class ArchiveQueue(object): return self.set(queue) - class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.33" - - __config__ = [("activated" , "bool" , "Activated" , True ), - ("fullpath" , "bool" , "Extract with full paths" , True ), - ("overwrite" , "bool" , "Overwrite files" , False ), - ("keepbroken" , "bool" , "Try to extract broken archives" , False ), - ("repair" , "bool" , "Repair broken archives (rar required)" , False ), - ("test" , "bool" , "Test archive before extracting" , False ), - ("usepasswordfile", "bool" , "Use password file" , True ), - ("passwordfile" , "file" , "Password file" , "archive_password.txt" ), - ("delete" , "bool" , "Delete archive when successfully extracted", False ), - ("subfolder" , "bool" , "Create subfolder for each package" , False ), - ("destination" , "folder", "Extract files to folder" , "" ), - ("extensions" , "str" , "Extract the following extensions" , "7z,bz2,bzip2,gz,gzip,lha,lzh,lzma,rar,tar,taz,tbz,tbz2,tgz,xar,xz,z,zip"), - ("excludefiles" , "str" , "Don't extract the following files" , "*.nfo,*.DS_Store,index.dat,thumb.db" ), - ("recursive" , "bool" , "Extract archives in archives" , True ), - ("waitall" , "bool" , "Wait for all downloads to be finished" , False ), - ("renice" , "int" , "CPU priority" , 0 )] + __version__ = "1.34" + + __config__ = [("activated" , "bool" , "Activated" , True ), + ("fullpath" , "bool" , "Extract with full paths" , True ), + ("overwrite" , "bool" , "Overwrite files" , False ), + ("keepbroken" , "bool" , "Try to extract broken archives" , False ), + ("repair" , "bool" , "Repair broken archives (RAR required)" , False ), + ("test" , "bool" , "Test archive before extracting" , False ), + ("usepasswordfile", "bool" , "Use password file" , True ), + ("passwordfile" , "file" , "Password file" , "archive_password.txt" ), + ("delete" , "bool" , "Delete archive after extraction" , False ), + ("subfolder" , "bool" , "Create subfolder for each package" , False ), + ("destination" , "folder", "Extract files to folder" , "" ), + ("extensions" , "str" , "Extract archives ending with extension", "7z,bz2,bzip2,gz,gzip,lha,lzh,lzma,rar,tar,taz,tbz,tbz2,tgz,xar,xz,z,zip"), + ("excludefiles" , "str" , "Don't extract the following files" , "*.nfo,*.DS_Store,index.dat,thumb.db" ), + ("recursive" , "bool" , "Extract archives in archives" , True ), + ("waitall" , "bool" , "Run after all downloads was processed" , False ), + ("renice" , "int" , "CPU priority" , 0 )] __description__ = """Extract different kind of archives""" __license__ = "GPLv3" @@ -139,6 +137,8 @@ class ExtractArchive(Hook): def setup(self): + self.info = {} #@TODO: Remove in 0.4.10 + self.queue = ArchiveQueue(self, "Queue") self.failed = ArchiveQueue(self, "Failed") @@ -158,7 +158,7 @@ class ExtractArchive(Hook): if klass.isUsable(): self.extractors.append(klass) if klass.REPAIR: - self.repair = self.getConfig("repair") + self.repair = self.getConfig('repair') except OSError, e: if e.errno == 2: @@ -181,10 +181,10 @@ class ExtractArchive(Hook): @threaded - def extractQueued(self,thread): + def extractQueued(self, thread): packages = self.queue.get() while packages: - if self.lastPackage: # called from allDownloadsProcessed + if self.lastPackage: #: called from allDownloadsProcessed self.lastPackage = False if self.extract(packages, thread): #@NOTE: check only if all gone fine, no failed reporting for now self.manager.dispatchEvent("all_archives_extracted") @@ -193,7 +193,7 @@ class ExtractArchive(Hook): if self.extract(packages, thread): #@NOTE: check only if all gone fine, no failed reporting for now pass - packages = self.queue.get() # check for packages added during extraction + packages = self.queue.get() #: check for packages added during extraction @Expose @@ -201,7 +201,7 @@ class ExtractArchive(Hook): """ Extract packages with given id""" for id in ids: self.queue.add(id) - if not self.getConfig("waitall") and not self.extracting: + if not self.getConfig('waitall') and not self.extracting: self.extractQueued() @@ -211,7 +211,7 @@ class ExtractArchive(Hook): def packageFinished(self, pypack): self.queue.add(pypack.id) - if not self.getConfig("waitall") and not self.extracting: + if not self.getConfig('waitall') and not self.extracting: self.extractQueued() @@ -221,7 +221,8 @@ class ExtractArchive(Hook): self.extractQueued() - def extract(self, ids, thread=None): + @Expose + def extract(self, ids, thread=None): #@TODO: Use pypack, not pid to improve method usability if not ids: return False @@ -233,17 +234,17 @@ class ExtractArchive(Hook): toList = lambda string: string.replace(' ', '').replace(',', '|').replace(';', '|').split('|') - destination = self.getConfig("destination") - subfolder = self.getConfig("subfolder") - fullpath = self.getConfig("fullpath") - overwrite = self.getConfig("overwrite") - renice = self.getConfig("renice") - recursive = self.getConfig("recursive") - delete = self.getConfig("delete") - keepbroken = self.getConfig("keepbroken") + destination = self.getConfig('destination') + subfolder = self.getConfig('subfolder') + fullpath = self.getConfig('fullpath') + overwrite = self.getConfig('overwrite') + renice = self.getConfig('renice') + recursive = self.getConfig('recursive') + delete = self.getConfig('delete') + keepbroken = self.getConfig('keepbroken') - extensions = [x.lstrip('.').lower() for x in toList(self.getConfig("extensions"))] - excludefiles = toList(self.getConfig("excludefiles")) + extensions = [x.lstrip('.').lower() for x in toList(self.getConfig('extensions'))] + excludefiles = toList(self.getConfig('excludefiles')) if extensions: self.logDebug("Use for extensions: %s" % "|.".join(extensions)) @@ -254,7 +255,7 @@ class ExtractArchive(Hook): # dl folder dl = self.config['general']['download_folder'] - #iterate packages -> extractors -> targets + # iterate packages -> extractors -> targets for pid in ids: pypack = self.core.files.getPackage(pid) @@ -276,7 +277,7 @@ class ExtractArchive(Hook): matched = False success = True files_ids = dict((pylink['name'],((save_join(dl, pypack.folder, pylink['name'])), pylink['id'], out)) for pylink \ - in sorted(pypack.getChildren().itervalues(), key=lambda k: k['name'])).values() # remove duplicates + in sorted(pypack.getChildren().itervalues(), key=lambda k: k['name'])).values() #: remove duplicates # check as long there are unseen files while files_ids: @@ -301,6 +302,7 @@ class ExtractArchive(Hook): self.logInfo(name, _("Extract to: %s") % fout) try: + pyfile = self.core.files.getFile(fid) archive = Extractor(self, fname, fout, @@ -311,16 +313,22 @@ class ExtractArchive(Hook): delete, keepbroken, fid) + + thread.addActive(pyfile) archive.init() - new_files = self._extract(archive, fid, pypack.password, thread) + try: + new_files = self._extract(pyfile, archive, pypack.password) + + finally: + thread.finishFile(pyfile) except Exception, e: self.logError(name, e) success = False continue - files_ids.remove((fname, fid, fout)) # don't let other extractors spam log + files_ids.remove((fname, fid, fout)) #: don't let other extractors spam log self.logDebug("Extracted files: %s" % new_files) self.setPermissions(new_files) @@ -331,12 +339,11 @@ class ExtractArchive(Hook): continue if recursive and os.path.isfile(file): - new_files_ids.append((filename, fid, os.path.dirname(filename))) # append as new target + new_files_ids.append((filename, fid, os.path.dirname(filename))) #: append as new target - pyfile = self.core.files.getFile(fid) - self.manager.dispatchEvent("archive_extracted", pyfile, archive.out, archive.filename, new_files) + self.manager.dispatchEvent("archive_extracted", pyfile, archive) - files_ids = new_files_ids # also check extracted files + files_ids = new_files_ids #: also check extracted files if matched: if success: @@ -364,21 +371,19 @@ class ExtractArchive(Hook): return True if not failed else False - def _extract(self, archive, fid, password, thread): - pyfile = self.core.files.getFile(fid) + def _extract(self, pyfile, archive, password): name = os.path.basename(archive.filename) - thread.addActive(pyfile) pyfile.setStatus("processing") encrypted = False try: self.logDebug("Password: %s" % (password or "None provided")) - passwords = uniqify([password] + self.getPasswords(False)) if self.getConfig("usepasswordfile") else [password] + passwords = uniqify([password] + self.getPasswords(False)) if self.getConfig('usepasswordfile') else [password] for pw in passwords: try: - if self.getConfig("test") or self.repair: - pyfile.setCustomStatus(_("testing")) + if self.getConfig('test') or self.repair: + pyfile.setCustomStatus(_("archive testing")) if pw: self.logDebug("Testing with password: %s" % pw) pyfile.setProgress(0) @@ -402,12 +407,12 @@ class ExtractArchive(Hook): if self.repair: self.logWarning(name, _("Repairing...")) - pyfile.setCustomStatus(_("repairing")) + pyfile.setCustomStatus(_("archive repairing")) pyfile.setProgress(0) repaired = archive.repair() pyfile.setProgress(100) - if not repaired and not self.getConfig("keepbroken"): + if not repaired and not self.getConfig('keepbroken'): raise CRCError("Archive damaged") self.addPassword(pw) @@ -421,7 +426,7 @@ class ExtractArchive(Hook): pyfile.setCustomStatus(_("extracting")) pyfile.setProgress(0) - if not encrypted or not self.getConfig("usepasswordfile"): + if not encrypted or not self.getConfig('usepasswordfile'): self.logDebug("Extracting using password: %s" % (password or "None")) archive.extract(password) else: @@ -439,12 +444,12 @@ class ExtractArchive(Hook): raise PasswordError pyfile.setProgress(100) - pyfile.setCustomStatus(_("finalizing")) + pyfile.setStatus("processing") if self.core.debug: self.logDebug("Would delete: %s" % ", ".join(archive.getDeleteFiles())) - if self.getConfig("delete"): + if self.getConfig('delete'): files = archive.getDeleteFiles() self.logInfo(_("Deleting %s files") % len(files)) for f in files: @@ -473,10 +478,7 @@ class ExtractArchive(Hook): if self.core.debug: print_exc() - finally: - thread.finishFile(pyfile) - - self.manager.dispatchEvent("archive_extract_failed", pyfile) + self.manager.dispatchEvent("archive_extract_failed", pyfile, archive) raise Exception(_("Extract failed")) @@ -494,7 +496,7 @@ class ExtractArchive(Hook): try: passwords = [] - file = fs_encode(self.getConfig("passwordfile")) + file = fs_encode(self.getConfig('passwordfile')) with open(file) as f: for pw in f.read().splitlines(): passwords.append(pw) @@ -512,7 +514,7 @@ class ExtractArchive(Hook): try: self.passwords = uniqify([password] + self.passwords) - file = fs_encode(self.getConfig("passwordfile")) + file = fs_encode(self.getConfig('passwordfile')) with open(file, "wb") as f: for pw in self.passwords: f.write(pw + '\n') -- cgit v1.2.3 From d0ef13775925f15811d2a6744d29190c313b3820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Mon, 23 Mar 2015 00:32:56 +0100 Subject: [ExtractArchive] extract archive even if first part is not in package --- module/plugins/hooks/ExtractArchive.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 0c163078c..d9af2dd49 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -106,7 +106,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.34" + __version__ = "1.35" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), @@ -328,7 +328,9 @@ class ExtractArchive(Hook): success = False continue - files_ids.remove((fname, fid, fout)) #: don't let other extractors spam log + # remove processed file and related multiparts from list + files_ids = [(fname, fid, fout) for fname, fid, fout in files_ids \ + if fname not in archive.getDeleteFiles()] self.logDebug("Extracted files: %s" % new_files) self.setPermissions(new_files) -- cgit v1.2.3 From ffb46ab200df55303836cc49e61b971b02e67ec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Wed, 25 Mar 2015 18:39:54 +0100 Subject: [ExtractArchive] Send2Trash Integration --- module/plugins/hooks/ExtractArchive.py | 63 +++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 24 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index d9af2dd49..c0fe22545 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -106,24 +106,24 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.35" - - __config__ = [("activated" , "bool" , "Activated" , True ), - ("fullpath" , "bool" , "Extract with full paths" , True ), - ("overwrite" , "bool" , "Overwrite files" , False ), - ("keepbroken" , "bool" , "Try to extract broken archives" , False ), - ("repair" , "bool" , "Repair broken archives (RAR required)" , False ), - ("test" , "bool" , "Test archive before extracting" , False ), - ("usepasswordfile", "bool" , "Use password file" , True ), - ("passwordfile" , "file" , "Password file" , "archive_password.txt" ), - ("delete" , "bool" , "Delete archive after extraction" , False ), - ("subfolder" , "bool" , "Create subfolder for each package" , False ), - ("destination" , "folder", "Extract files to folder" , "" ), - ("extensions" , "str" , "Extract archives ending with extension", "7z,bz2,bzip2,gz,gzip,lha,lzh,lzma,rar,tar,taz,tbz,tbz2,tgz,xar,xz,z,zip"), - ("excludefiles" , "str" , "Don't extract the following files" , "*.nfo,*.DS_Store,index.dat,thumb.db" ), - ("recursive" , "bool" , "Extract archives in archives" , True ), - ("waitall" , "bool" , "Run after all downloads was processed" , False ), - ("renice" , "int" , "CPU priority" , 0 )] + __version__ = "1.36" + + __config__ = [("activated" , "bool" , "Activated" , True ), + ("fullpath" , "bool" , "Extract with full paths" , True ), + ("overwrite" , "bool" , "Overwrite files" , False ), + ("keepbroken" , "bool" , "Try to extract broken archives" , False ), + ("repair" , "bool" , "Repair broken archives (RAR required)" , False ), + ("test" , "bool" , "Test archive before extracting" , False ), + ("usepasswordfile", "bool" , "Use password file" , True ), + ("passwordfile" , "file" , "Password file" , "archive_password.txt" ), + ("delete" , "No;Permanent;Trash" , "Delete archive after extraction" , "No" ), + ("subfolder" , "bool" , "Create subfolder for each package" , False ), + ("destination" , "folder" , "Extract files to folder" , "" ), + ("extensions" , "str" , "Extract archives ending with extension", "7z,bz2,bzip2,gz,gzip,lha,lzh,lzma,rar,tar,taz,tbz,tbz2,tgz,xar,xz,z,zip"), + ("excludefiles" , "str" , "Don't extract the following files" , "*.nfo,*.DS_Store,index.dat,thumb.db" ), + ("recursive" , "bool" , "Extract archives in archives" , True ), + ("waitall" , "bool" , "Run after all downloads was processed" , False ), + ("renice" , "int" , "CPU priority" , 0 )] __description__ = """Extract different kind of archives""" __license__ = "GPLv3" @@ -148,6 +148,7 @@ class ExtractArchive(Hook): self.extractors = [] self.passwords = [] self.repair = False + self.trash = False def coreReady(self): @@ -448,15 +449,29 @@ class ExtractArchive(Hook): pyfile.setProgress(100) pyfile.setStatus("processing") + delfiles = archive.getDeleteFiles() if self.core.debug: - self.logDebug("Would delete: %s" % ", ".join(archive.getDeleteFiles())) + self.logDebug("Would delete: %s" % ", ".join(delfiles)) - if self.getConfig('delete'): - files = archive.getDeleteFiles() - self.logInfo(_("Deleting %s files") % len(files)) - for f in files: + if self.getConfig('delete') != 'No': + try: + from send2trash import send2trash + if self.getConfig('delete') == "Trash": + self.trash = True + self.logInfo(_("Sending %s files to trash") % len(delfiles)) + except ImportError: + self.logError(name, _("Send2Trash not installed, no files deleted")) + self.trash = False + + if self.getConfig('delete') == "Permanent": + self.trash = False + self.logInfo(_("Deleting %s files") % len(delfiles)) + + for f in delfiles: file = fs_encode(f) - if os.path.exists(file): + if os.path.exists(file) and self.trash: + send2trash(file) + elif os.path.exists: os.remove(file) else: self.logDebug("%s does not exists" % f) -- cgit v1.2.3 From f4f618f95a2348c8881ae455e1c8ad01ba3eae30 Mon Sep 17 00:00:00 2001 From: Gutz-Pilz Date: Wed, 25 Mar 2015 22:02:11 +0100 Subject: fixed "permanent" issue (forgot to change filversion) --- module/plugins/hooks/ExtractArchive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index c0fe22545..5d71e89ec 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -106,7 +106,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.36" + __version__ = "1.37" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), @@ -471,7 +471,7 @@ class ExtractArchive(Hook): file = fs_encode(f) if os.path.exists(file) and self.trash: send2trash(file) - elif os.path.exists: + elif os.path.exists(file): os.remove(file) else: self.logDebug("%s does not exists" % f) -- cgit v1.2.3 From 329293e0bb0cf4008688d8c73d4b98caa7082141 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 26 Mar 2015 10:40:32 +0100 Subject: [Extractor] Rename method 'test' to 'verify' --- module/plugins/hooks/ExtractArchive.py | 61 +++++++++++++++++----------------- 1 file changed, 30 insertions(+), 31 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 5d71e89ec..c5f842fdd 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -106,24 +106,24 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.37" - - __config__ = [("activated" , "bool" , "Activated" , True ), - ("fullpath" , "bool" , "Extract with full paths" , True ), - ("overwrite" , "bool" , "Overwrite files" , False ), - ("keepbroken" , "bool" , "Try to extract broken archives" , False ), - ("repair" , "bool" , "Repair broken archives (RAR required)" , False ), - ("test" , "bool" , "Test archive before extracting" , False ), - ("usepasswordfile", "bool" , "Use password file" , True ), - ("passwordfile" , "file" , "Password file" , "archive_password.txt" ), - ("delete" , "No;Permanent;Trash" , "Delete archive after extraction" , "No" ), - ("subfolder" , "bool" , "Create subfolder for each package" , False ), - ("destination" , "folder" , "Extract files to folder" , "" ), - ("extensions" , "str" , "Extract archives ending with extension", "7z,bz2,bzip2,gz,gzip,lha,lzh,lzma,rar,tar,taz,tbz,tbz2,tgz,xar,xz,z,zip"), - ("excludefiles" , "str" , "Don't extract the following files" , "*.nfo,*.DS_Store,index.dat,thumb.db" ), - ("recursive" , "bool" , "Extract archives in archives" , True ), - ("waitall" , "bool" , "Run after all downloads was processed" , False ), - ("renice" , "int" , "CPU priority" , 0 )] + __version__ = "1.38" + + __config__ = [("activated" , "bool" , "Activated" , True ), + ("fullpath" , "bool" , "Extract with full paths" , True ), + ("overwrite" , "bool" , "Overwrite files" , False ), + ("keepbroken" , "bool" , "Try to extract broken archives" , False ), + ("repair" , "bool" , "Repair broken archives (RAR required)" , False ), + ("test" , "bool" , "Test archive before extracting" , False ), + ("usepasswordfile", "bool" , "Use password file" , True ), + ("passwordfile" , "file" , "Password file" , "archive_password.txt" ), + ("delete" , "No;Permanent;Trash", "Delete archive after extraction" , "No" ), + ("subfolder" , "bool" , "Create subfolder for each package" , False ), + ("destination" , "folder" , "Extract files to folder" , "" ), + ("extensions" , "str" , "Extract archives ending with extension", "7z,bz2,bzip2,gz,gzip,lha,lzh,lzma,rar,tar,taz,tbz,tbz2,tgz,xar,xz,z,zip"), + ("excludefiles" , "str" , "Don't extract the following files" , "*.nfo,*.DS_Store,index.dat,thumb.db" ), + ("recursive" , "bool" , "Extract archives in archives" , True ), + ("waitall" , "bool" , "Run after all downloads was processed" , False ), + ("renice" , "int" , "CPU priority" , 0 )] __description__ = """Extract different kind of archives""" __license__ = "GPLv3" @@ -235,14 +235,14 @@ class ExtractArchive(Hook): toList = lambda string: string.replace(' ', '').replace(',', '|').replace(';', '|').split('|') - destination = self.getConfig('destination') - subfolder = self.getConfig('subfolder') - fullpath = self.getConfig('fullpath') - overwrite = self.getConfig('overwrite') - renice = self.getConfig('renice') - recursive = self.getConfig('recursive') - delete = self.getConfig('delete') - keepbroken = self.getConfig('keepbroken') + destination = self.getConfig('destination') + subfolder = self.getConfig('subfolder') + fullpath = self.getConfig('fullpath') + overwrite = self.getConfig('overwrite') + renice = self.getConfig('renice') + recursive = self.getConfig('recursive') + delete = self.getConfig('delete') + keepbroken = self.getConfig('keepbroken') extensions = [x.lstrip('.').lower() for x in toList(self.getConfig('extensions'))] excludefiles = toList(self.getConfig('excludefiles')) @@ -253,8 +253,7 @@ class ExtractArchive(Hook): # reload from txt file self.reloadPasswords() - # dl folder - dl = self.config['general']['download_folder'] + download_folder = self.config['general']['download_folder'] # iterate packages -> extractors -> targets for pid in ids: @@ -267,7 +266,7 @@ class ExtractArchive(Hook): self.logInfo(_("Check package: %s") % pypack.name) # determine output folder - out = save_join(dl, pypack.folder, destination, "") #: force trailing slash + out = save_join(download_folder, pypack.folder, destination, "") #: force trailing slash if subfolder: out = save_join(out, pypack.folder) @@ -277,7 +276,7 @@ class ExtractArchive(Hook): matched = False success = True - files_ids = dict((pylink['name'],((save_join(dl, pypack.folder, pylink['name'])), pylink['id'], out)) for pylink \ + files_ids = dict((pylink['name'],((save_join(download_folder, pypack.folder, pylink['name'])), pylink['id'], out)) for pylink \ in sorted(pypack.getChildren().itervalues(), key=lambda k: k['name'])).values() #: remove duplicates # check as long there are unseen files @@ -390,7 +389,7 @@ class ExtractArchive(Hook): if pw: self.logDebug("Testing with password: %s" % pw) pyfile.setProgress(0) - archive.test(pw) + archive.verify(pw) pyfile.setProgress(100) else: archive.check(pw) -- cgit v1.2.3 From 589121e80835c63aea0880a53c6678de5c31c16e Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 28 Mar 2015 01:59:01 +0100 Subject: Spare code cosmetics --- module/plugins/hooks/ExtractArchive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index c5f842fdd..8c40667c2 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -175,7 +175,7 @@ class ExtractArchive(Hook): print_exc() if self.extractors: - self.logInfo(_("Activated") + " " + "|".join("%s %s" % (Extractor.__name__,Extractor.VERSION) for Extractor in self.extractors)) + self.logInfo(_("Activated") + " " + "|".join("%s %s" % (Extractor.__name__, Extractor.VERSION) for Extractor in self.extractors)) self.extractQueued() #: Resume unfinished extractions else: self.logInfo(_("No Extract plugins activated")) -- cgit v1.2.3 From a01a2dcb91bb2020aa4c7548a52ba98dbe43ceef Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 7 Apr 2015 21:12:09 +0200 Subject: [ExtractArchive] Improve send2trash feature --- module/plugins/hooks/ExtractArchive.py | 81 ++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 38 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 8c40667c2..8b6a140cd 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -106,24 +106,25 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.38" - - __config__ = [("activated" , "bool" , "Activated" , True ), - ("fullpath" , "bool" , "Extract with full paths" , True ), - ("overwrite" , "bool" , "Overwrite files" , False ), - ("keepbroken" , "bool" , "Try to extract broken archives" , False ), - ("repair" , "bool" , "Repair broken archives (RAR required)" , False ), - ("test" , "bool" , "Test archive before extracting" , False ), - ("usepasswordfile", "bool" , "Use password file" , True ), - ("passwordfile" , "file" , "Password file" , "archive_password.txt" ), - ("delete" , "No;Permanent;Trash", "Delete archive after extraction" , "No" ), - ("subfolder" , "bool" , "Create subfolder for each package" , False ), - ("destination" , "folder" , "Extract files to folder" , "" ), - ("extensions" , "str" , "Extract archives ending with extension", "7z,bz2,bzip2,gz,gzip,lha,lzh,lzma,rar,tar,taz,tbz,tbz2,tgz,xar,xz,z,zip"), - ("excludefiles" , "str" , "Don't extract the following files" , "*.nfo,*.DS_Store,index.dat,thumb.db" ), - ("recursive" , "bool" , "Extract archives in archives" , True ), - ("waitall" , "bool" , "Run after all downloads was processed" , False ), - ("renice" , "int" , "CPU priority" , 0 )] + __version__ = "1.39" + + __config__ = [("activated" , "bool" , "Activated" , True ), + ("fullpath" , "bool" , "Extract with full paths" , True ), + ("overwrite" , "bool" , "Overwrite files" , False ), + ("keepbroken" , "bool" , "Try to extract broken archives" , False ), + ("repair" , "bool" , "Repair broken archives (RAR required)" , False ), + ("test" , "bool" , "Test archive before extracting" , False ), + ("usepasswordfile", "bool" , "Use password file" , True ), + ("passwordfile" , "file" , "Password file" , "archive_password.txt" ), + ("delete" , "bool" , "Delete archive after extraction" , True ), + ("deltotrash" , "bool" , "Move to trash (recycle bin) instead delete", True ), + ("subfolder" , "bool" , "Create subfolder for each package" , False ), + ("destination" , "folder" , "Extract files to folder" , "" ), + ("extensions" , "str" , "Extract archives ending with extension" , "7z,bz2,bzip2,gz,gzip,lha,lzh,lzma,rar,tar,taz,tbz,tbz2,tgz,xar,xz,z,zip"), + ("excludefiles" , "str" , "Don't extract the following files" , "*.nfo,*.DS_Store,index.dat,thumb.db" ), + ("recursive" , "bool" , "Extract archives in archives" , True ), + ("waitall" , "bool" , "Run after all downloads was processed" , False ), + ("renice" , "int" , "CPU priority" , 0 )] __description__ = """Extract different kind of archives""" __license__ = "GPLv3" @@ -148,7 +149,16 @@ class ExtractArchive(Hook): self.extractors = [] self.passwords = [] self.repair = False - self.trash = False + + try: + import send2trash + + except ImportError: + self.logDebug(name, _("Send2Trash lib not found")) + self.trashable = False + + else: + self.trashable = True def coreReady(self): @@ -321,6 +331,7 @@ class ExtractArchive(Hook): new_files = self._extract(pyfile, archive, pypack.password) finally: + pyfile.setProgress(100) thread.finishFile(pyfile) except Exception, e: @@ -449,31 +460,25 @@ class ExtractArchive(Hook): pyfile.setStatus("processing") delfiles = archive.getDeleteFiles() - if self.core.debug: - self.logDebug("Would delete: %s" % ", ".join(delfiles)) + self.logDebug("Would delete: " + ", ".join(delfiles)) - if self.getConfig('delete') != 'No': - try: - from send2trash import send2trash - if self.getConfig('delete') == "Trash": - self.trash = True - self.logInfo(_("Sending %s files to trash") % len(delfiles)) - except ImportError: - self.logError(name, _("Send2Trash not installed, no files deleted")) - self.trash = False - - if self.getConfig('delete') == "Permanent": - self.trash = False - self.logInfo(_("Deleting %s files") % len(delfiles)) + if self.getConfig('delete'): + self.logInfo(_("Deleting %s files") % len(delfiles)) + deltotrash = self.getConfig('deltotrash') for f in delfiles: file = fs_encode(f) - if os.path.exists(file) and self.trash: - send2trash(file) - elif os.path.exists(file): + if not os.path.exists(file) + continue + + if not deltotrash: os.remove(file) + + elif self.trashable: + send2trash.send2trash(file) + else: - self.logDebug("%s does not exists" % f) + self.logWarning("Unable to move %s to trash" % os.path.basename(f)) self.logInfo(name, _("Extracting finished")) extracted_files = archive.files or archive.list() -- cgit v1.2.3 From 034d87c836f9fba82142a06b5f666c84ca10bd50 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 7 Apr 2015 21:26:23 +0200 Subject: [YadiSk] Fix https://github.com/pyload/pyload/issues/1321 --- module/plugins/hooks/ExtractArchive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 8b6a140cd..2f981d06b 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -478,7 +478,7 @@ class ExtractArchive(Hook): send2trash.send2trash(file) else: - self.logWarning("Unable to move %s to trash" % os.path.basename(f)) + self.logWarning(_("Unable to move %s to trash") % os.path.basename(f)) self.logInfo(name, _("Extracting finished")) extracted_files = archive.files or archive.list() -- cgit v1.2.3 From c99421d6385e0f6b8bd9095c705becbc35f873a0 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 7 Apr 2015 22:08:58 +0200 Subject: Traceback code cosmetics --- module/plugins/hooks/ExtractArchive.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 2f981d06b..595512c12 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -4,9 +4,9 @@ from __future__ import with_statement import os import sys +import traceback from copy import copy -from traceback import print_exc # monkey patch bug in python 2.6 and lower # http://bugs.python.org/issue6122 , http://bugs.python.org/issue1236 , http://bugs.python.org/issue1731717 @@ -177,12 +177,12 @@ class ExtractArchive(Hook): else: self.logWarning(_("Could not activate: %s") % p, e) if self.core.debug: - print_exc() + traceback.print_exc() except Exception, e: self.logWarning(_("Could not activate: %s") % p, e) if self.core.debug: - print_exc() + traceback.print_exc() if self.extractors: self.logInfo(_("Activated") + " " + "|".join("%s %s" % (Extractor.__name__, Extractor.VERSION) for Extractor in self.extractors)) @@ -497,7 +497,7 @@ class ExtractArchive(Hook): except Exception, e: self.logError(name, _("Unknown error"), e) if self.core.debug: - print_exc() + traceback.print_exc() self.manager.dispatchEvent("archive_extract_failed", pyfile, archive) -- cgit v1.2.3 From 41a9fc39c201f663ee308840b8443f760b3678da Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 8 Apr 2015 00:30:57 +0200 Subject: [ExtractArchive] Fix https://github.com/pyload/pyload/issues/1322 --- module/plugins/hooks/ExtractArchive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 595512c12..b418f802f 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -106,7 +106,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.39" + __version__ = "1.40" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), @@ -468,7 +468,7 @@ class ExtractArchive(Hook): deltotrash = self.getConfig('deltotrash') for f in delfiles: file = fs_encode(f) - if not os.path.exists(file) + if not os.path.exists(file): continue if not deltotrash: -- cgit v1.2.3 From c5bd9e25f08248834aa6192b60ab968e95a10dc1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 8 Apr 2015 04:32:51 +0200 Subject: [AntiVirus][ExtractArchive] Fixup --- module/plugins/hooks/ExtractArchive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index b418f802f..9e6f2d379 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -106,7 +106,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.40" + __version__ = "1.41" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), @@ -154,7 +154,7 @@ class ExtractArchive(Hook): import send2trash except ImportError: - self.logDebug(name, _("Send2Trash lib not found")) + self.logDebug("Send2Trash lib not found") self.trashable = False else: -- cgit v1.2.3 From 73f99ea811d01a2e113a882fe5ab86b0aa9c4f4c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 9 Apr 2015 00:22:51 +0200 Subject: [SimpleHoster] Improve file name handling --- module/plugins/hooks/ExtractArchive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 9e6f2d379..d40b52aa7 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -173,7 +173,7 @@ class ExtractArchive(Hook): except OSError, e: if e.errno == 2: - self.logInfo(_("No %s installed") % p) + self.logWarning(_("No %s installed") % p) else: self.logWarning(_("Could not activate: %s") % p, e) if self.core.debug: @@ -185,7 +185,7 @@ class ExtractArchive(Hook): traceback.print_exc() if self.extractors: - self.logInfo(_("Activated") + " " + "|".join("%s %s" % (Extractor.__name__, Extractor.VERSION) for Extractor in self.extractors)) + self.logDebug(*["Found %s %s" % (Extractor.__name__, Extractor.VERSION) for Extractor in self.extractors]) self.extractQueued() #: Resume unfinished extractions else: self.logInfo(_("No Extract plugins activated")) -- cgit v1.2.3 From 711621f3d3a59f7ec5c2684dfac921e2c5902563 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 1 May 2015 02:54:22 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1351 --- module/plugins/hooks/ExtractArchive.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index d40b52aa7..05a1e368a 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -6,8 +6,6 @@ import os import sys import traceback -from copy import copy - # monkey patch bug in python 2.6 and lower # http://bugs.python.org/issue6122 , http://bugs.python.org/issue1236 , http://bugs.python.org/issue1731717 if sys.version_info < (2, 7) and os.name != "nt": @@ -45,6 +43,12 @@ if sys.version_info < (2, 7) and os.name != "nt": subprocess.Popen.wait = wait +try: + import send2trash +except ImportError: + pass + +from copy import copy if os.name != "nt": from grp import getgrnam from pwd import getpwnam @@ -106,7 +110,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.41" + __version__ = "1.42" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), @@ -150,16 +154,6 @@ class ExtractArchive(Hook): self.passwords = [] self.repair = False - try: - import send2trash - - except ImportError: - self.logDebug("Send2Trash lib not found") - self.trashable = False - - else: - self.trashable = True - def coreReady(self): for p in ("UnRar", "SevenZip", "UnZip"): @@ -474,11 +468,12 @@ class ExtractArchive(Hook): if not deltotrash: os.remove(file) - elif self.trashable: - send2trash.send2trash(file) - else: - self.logWarning(_("Unable to move %s to trash") % os.path.basename(f)) + try: + send2trash.send2trash(file) + + except Exception: + self.logWarning(_("Unable to move %s to trash") % os.path.basename(f)) self.logInfo(name, _("Extracting finished")) extracted_files = archive.files or archive.list() -- cgit v1.2.3 From dec360bee84f046bf40612173ec4e74aebd60599 Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Sat, 2 May 2015 22:40:55 +0300 Subject: [ExtractArchive] report missing send2trash --- module/plugins/hooks/ExtractArchive.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 05a1e368a..609d0ad62 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -110,7 +110,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.42" + __version__ = "1.43" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), @@ -472,6 +472,9 @@ class ExtractArchive(Hook): try: send2trash.send2trash(file) + except NameError: + self.logWarning(_("Unable to move %s to trash: Send2Trash lib not found") % os.path.basename(f)) + except Exception: self.logWarning(_("Unable to move %s to trash") % os.path.basename(f)) -- cgit v1.2.3 From c6932a1ea6e44867e2b2caedfad444d4b34aea4f Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Mon, 4 May 2015 01:33:47 +0300 Subject: more verbose --- module/plugins/hooks/ExtractArchive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 609d0ad62..d0a84364a 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -475,8 +475,8 @@ class ExtractArchive(Hook): except NameError: self.logWarning(_("Unable to move %s to trash: Send2Trash lib not found") % os.path.basename(f)) - except Exception: - self.logWarning(_("Unable to move %s to trash") % os.path.basename(f)) + except Exception, e: + self.logWarning(_("Unable to move %s to trash: %s") % (os.path.basename(f), e.message)) self.logInfo(name, _("Extracting finished")) extracted_files = archive.files or archive.list() -- cgit v1.2.3 From e26ba64714e84745db1f6dfc73ccc8c68ad4abcc Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Mon, 4 May 2015 02:22:06 +0300 Subject: even more --- module/plugins/hooks/ExtractArchive.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index d0a84364a..5b90b28fc 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -478,6 +478,9 @@ class ExtractArchive(Hook): except Exception, e: self.logWarning(_("Unable to move %s to trash: %s") % (os.path.basename(f), e.message)) + else: + self.logDebug(_("Successfully moved %s to trash") % os.path.basename(f)) + self.logInfo(name, _("Extracting finished")) extracted_files = archive.files or archive.list() -- cgit v1.2.3 From 8483aacbcd7f3f4c6ad92ef2255b449b88d64a35 Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Tue, 5 May 2015 01:23:31 +0300 Subject: fix bug causing `extractQueued` called twice --- module/plugins/hooks/ExtractArchive.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 05a1e368a..62eb36714 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -187,6 +187,11 @@ class ExtractArchive(Hook): @threaded def extractQueued(self, thread): + if self.extracting: #@NOTE: doing the check here for safty (called by coreReady) + return + + self.extracting = True + packages = self.queue.get() while packages: if self.lastPackage: #: called from allDownloadsProcessed @@ -200,6 +205,8 @@ class ExtractArchive(Hook): packages = self.queue.get() #: check for packages added during extraction + self.extracting = False + @Expose def extractPackage(self, *ids): @@ -222,7 +229,7 @@ class ExtractArchive(Hook): def allDownloadsProcessed(self): self.lastPackage = True - if not self.extracting: + if self.getConfig('waitall') and not self.extracting: self.extractQueued() @@ -231,8 +238,6 @@ class ExtractArchive(Hook): if not ids: return False - self.extracting = True - processed = [] extracted = [] failed = [] @@ -374,7 +379,6 @@ class ExtractArchive(Hook): self.queue.remove(pid) - self.extracting = False return True if not failed else False -- cgit v1.2.3 From e3d3ad079d8af1806a475020aeb7e9fdc5a26600 Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Tue, 5 May 2015 01:33:17 +0300 Subject: Update ExtractArchive.py --- module/plugins/hooks/ExtractArchive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 62eb36714..bd8153a73 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -110,7 +110,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.42" + __version__ = "1.43" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), -- cgit v1.2.3 From 430630fc3158e5dc5412824095f058a0be295edb Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Tue, 5 May 2015 20:50:50 +0300 Subject: Update ExtractArchive.py --- module/plugins/hooks/ExtractArchive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 07a2ba228..5b9e3f30b 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -110,7 +110,7 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.43" + __version__ = "1.44" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), -- cgit v1.2.3 From 9f3ab57ec994deb24cd31a1dfbd338eb71bffc8c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 27 May 2015 23:46:29 +0200 Subject: Spare code cosmetics --- module/plugins/hooks/ExtractArchive.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'module/plugins/hooks/ExtractArchive.py') diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 5b9e3f30b..3e371ec2b 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -136,13 +136,12 @@ class ExtractArchive(Hook): ("Immenz" , "immenz@gmx.net" )] - event_list = ["allDownloadsProcessed","packageDeleted"] - NAME_REPLACEMENTS = [(r'\.part\d+\.rar$', ".part.rar")] def setup(self): self.info = {} #@TODO: Remove in 0.4.10 + self.event_list = ["allDownloadsProcessed","packageDeleted"] self.queue = ArchiveQueue(self, "Queue") self.failed = ArchiveQueue(self, "Failed") -- cgit v1.2.3