From 43460c40d00819535dfeecfdb80f8a608f2190fd Mon Sep 17 00:00:00 2001 From: RaNaN Date: Thu, 13 Oct 2011 15:25:32 +0200 Subject: improvement for hook plugins, new internal plugin type --- module/plugins/Hook.py | 3 +- module/plugins/PluginManager.py | 70 +++++++++--- module/plugins/hooks/UnRar.py | 237 --------------------------------------- module/plugins/internal/UnRar.py | 237 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 292 insertions(+), 255 deletions(-) delete mode 100644 module/plugins/hooks/UnRar.py create mode 100644 module/plugins/internal/UnRar.py (limited to 'module/plugins') diff --git a/module/plugins/Hook.py b/module/plugins/Hook.py index fdcaccfe3..51ebd1aec 100644 --- a/module/plugins/Hook.py +++ b/module/plugins/Hook.py @@ -18,7 +18,6 @@ @interface-version: 0.2 """ -from thread import start_new_thread from traceback import print_exc from Plugin import Base @@ -32,7 +31,7 @@ class Expose(object): def threaded(f): def run(*args,**kwargs): - return start_new_thread(f, args, kwargs) + hookManager.startThread(f, *args, **kwargs) return run class Hook(Base): diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index 0bf15f1a8..4ddb1b1b9 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -53,6 +53,11 @@ class PluginManager(): self.createIndex() + rePattern = re.compile(r'__pattern__.*=.*r("|\')([^"\']+)') + reVersion = re.compile(r'__version__.*=.*("|\')([0-9.]+)') + reConfig = re.compile(r'__config__.*=.*\[([^\]]+)', re.MULTILINE) + reDesc = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)') + def createIndex(self): """create information for all plugins available""" @@ -69,17 +74,18 @@ class PluginManager(): self.reConfig = re.compile(r'__config__.*=.*\[([^\]]+)', re.MULTILINE) self.reDesc = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)') - self.crypterPlugins = self.parse(_("Crypter"), "crypter", pattern=True) - self.containerPlugins = self.parse(_("Container"), "container", pattern=True) - self.hosterPlugins = self.parse(_("Hoster"), "hoster", pattern=True) + self.crypterPlugins = self.parse("crypter", pattern=True) + self.containerPlugins = self.parse("container", pattern=True) + self.hosterPlugins = self.parse("hoster", pattern=True) - self.captchaPlugins = self.parse(_("Captcha"), "captcha") - self.accountPlugins = self.parse(_("Account"), "accounts", create=True) - self.hookPlugins = self.parse(_("Hook"), "hooks") + self.captchaPlugins = self.parse("captcha") + self.accountPlugins = self.parse("accounts") + self.hookPlugins = self.parse("hooks") + self.internalPlugins = self.parse("internal") self.log.debug("created index of plugins") - def parse(self, typ, folder, create=False, pattern=False, home={}): + def parse(self, folder, pattern=False, home={}): """ returns dict with information home contains parsed plugins from module. @@ -108,11 +114,11 @@ class PluginManager(): content = data.read() data.close() - if f.endswith("_25.pyc") and not version_info[0:2] == (2, 5): + if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): continue - elif f.endswith("_26.pyc") and not version_info[0:2] == (2, 6): + elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): continue - elif f.endswith("_27.pyc") and not version_info[0:2] == (2, 7): + elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): continue name = f[:-3] @@ -124,6 +130,7 @@ class PluginManager(): else: version = 0 + # home contains plugins from pyload root if home and name in home: if home[name]["v"] >= version: continue @@ -158,15 +165,17 @@ class PluginManager(): except: self.log.error(_("%s has a invalid pattern.") % name) - config = self.reConfig.findall(content) + # internals have no config + if folder == "internal": + self.core.config.deleteConfig(name) + continue + + config = self.reConfig.findall(content) if config: config = literal_eval(config[0].strip().replace("\n", "").replace("\r", "")) desc = self.reDesc.findall(content) - if desc: - desc = desc[0][1] - else: - desc = "" + desc = desc[0][1] if desc else "" if type(config[0]) == tuple: config = [list(x) for x in config] @@ -188,8 +197,18 @@ class PluginManager(): except : self.log.error("Invalid config in %s: %s" % (name, config)) + elif folder == "hooks": #force config creation + desc = self.reDesc.findall(content) + desc = desc[0][1] if desc else "" + config = (["activated", "bool", "Activated", False],) + + try: + self.core.config.addPluginConfig(name, config, desc) + except : + self.log.error("Invalid config in %s: %s" % (name, config)) + if not home: - temp = self.parse(typ, folder, create, pattern, plugins) + temp = self.parse(folder, pattern, plugins) plugins.update(temp) return plugins @@ -327,6 +346,25 @@ class PluginManager(): return pluginClass + def getInternalModul(self, name): + + if name not in self.internalPlugins: return None + + value = self.internalPlugins[name] + + if "module" in value: + return value["module"] + + try: + module = __import__(value["path"], globals(), locals(), [value["name"]], -1) + except Exception, e: + self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) + if self.core.debug: + print_exc() + return None + + value["module"] = module + return module def reloadPlugins(self): """ reloads and reindexes plugins """ diff --git a/module/plugins/hooks/UnRar.py b/module/plugins/hooks/UnRar.py deleted file mode 100644 index 7ace49a76..000000000 --- a/module/plugins/hooks/UnRar.py +++ /dev/null @@ -1,237 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" -from __future__ import with_statement - -import sys -import os -from os.path import exists, join, isabs, isdir -from os import remove, makedirs, rmdir, listdir, chmod -from traceback import print_exc - -from module.plugins.Hook import Hook -from module.lib.pyunrar import Unrar, WrongPasswordError, CommandError, UnknownError, LowRamError - -from module.utils import save_join - -if os.name != "nt": - from pwd import getpwnam - from os import chown - -import re - -class UnRar(Hook): - __name__ = "UnRar" - __version__ = "0.11" - __description__ = """unrar""" - __config__ = [("activated", "bool", "Activated", False), - ("fullpath", "bool", "extract full path", True), - ("overwrite", "bool", "overwrite files", True), - ("passwordfile", "str", "unrar password file", "unrar_passwords.txt"), - ("deletearchive", "bool", "delete archives when done", False), - ("ramwarning", "bool", "warn about low ram", True), - ("renice", "int", "Cpu Priority", 10), - ("unrar_destination", "str", "Unpack files to", "")] - __threaded__ = ["packageFinished"] - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def setup(self): - self.comments = ["# one password each line"] - self.passwords = [] - if exists(self.getConfig("passwordfile")): - with open(self.getConfig("passwordfile"), "r") as f: - for l in f.readlines(): - l = l.strip("\n\r") - if l and not l.startswith("#"): - self.passwords.append(l) - else: - with open(self.getConfig("passwordfile"), "w") as f: - f.writelines(self.comments) - self.re_splitfile = re.compile("(.*)\.part(\d+)\.rar$") - - self.ram = 0 #ram in kb for unix osses - try: - f = open("/proc/meminfo") - line = True - while line: - line = f.readline() - if line.startswith("MemTotal:"): - self.ram = int(re.search(r"([0-9]+)", line).group(1)) - except: - self.ram = 0 - - self.ram /= 1024 - - def setOwner(self, d, uid, gid, mode): - if not exists(d): - self.core.log.debug(_("Directory %s does not exist!") % d) - return - - for fileEntry in listdir(d): - fullEntryName = join(d, fileEntry) - if isdir(fullEntryName): - self.setOwner(fullEntryName, uid, gid, mode) - try: - chown(fullEntryName, uid, gid) - chmod(fullEntryName, mode) - except: - self.core.log.debug(_("Chown/Chmod for %s failed") % fullEntryName) - self.core.log.debug(_("Exception: %s") % sys.exc_info()[0]) - continue - try: - chown(d, uid, gid) - chmod(d, mode) - except: - self.core.log.debug(_("Chown/Chmod for %s failed") % d) - self.core.log.debug(_("Exception: %s") % sys.exc_info()[0]) - return - - def addPassword(self, pws): - if not type(pws) == list: pws = [pws] - pws.reverse() - for pw in pws: - pw = pw.strip() - if not pw or pw == "None" or pw in self.passwords: continue - self.passwords.insert(0, pw) - - with open(self.getConfig("passwordfile"), "w") as f: - f.writelines([c + "\n" for c in self.comments]) - f.writelines([p + "\n" for p in self.passwords]) - - def removeFiles(self, pack, fname): - if not self.getConfig("deletearchive"): - return - m = self.re_splitfile.search(fname) - - download_folder = self.core.config['general']['download_folder'] - if self.core.config['general']['folder_per_package']: - folder = join(download_folder, pack.folder.decode(sys.getfilesystemencoding())) - else: - folder = download_folder - if m: - nre = re.compile("%s\.part\d+\.rar" % m.group(1)) - for fid, data in pack.getChildren().iteritems(): - if nre.match(data["name"]): - remove(join(folder, data["name"])) - elif not m and fname.endswith(".rar"): - nre = re.compile("^%s\.r..$" % fname.replace(".rar", "")) - for fid, data in pack.getChildren().iteritems(): - if nre.match(data["name"]): - remove(join(folder, data["name"])) - - def packageFinished(self, pack): - if pack.password and pack.password.strip() and pack.password.strip() != "None": - self.addPassword(pack.password.splitlines()) - files = [] - - for fid, data in pack.getChildren().iteritems(): - m = self.re_splitfile.search(data["name"]) - if m and int(m.group(2)) == 1: - files.append((fid, m.group(0))) - elif not m and data["name"].endswith(".rar"): - files.append((fid, data["name"])) - - for fid, fname in files: - self.core.log.info(_("starting Unrar of %s") % fname) - pyfile = self.core.files.getFile(fid) - pyfile.setStatus("processing") - - def s(p): - pyfile.setProgress(p) - - download_folder = self.core.config['general']['download_folder'] - self.core.log.debug(_("download folder %s") % download_folder) - - folder = save_join(download_folder, pack.folder, "") - - - destination = folder - if self.getConfig("unrar_destination") and not self.getConfig("unrar_destination").lower() == "none": - destination = self.getConfig("unrar_destination") - sub = "." - if self.core.config['general']['folder_per_package']: - sub = pack.folder.decode(sys.getfilesystemencoding()) - if isabs(destination): - destination = join(destination, sub, "") - else: - destination = join(folder, destination, sub, "") - - self.core.log.debug(_("Destination folder %s") % destination) - if not exists(destination): - self.core.log.info(_("Creating destination folder %s") % destination) - makedirs(destination) - - u = Unrar(join(folder, fname), tmpdir=join(folder, "tmp"), - ramSize=(self.ram if self.getConfig("ramwarning") else 0), cpu=self.getConfig("renice")) - try: - success = u.crackPassword(passwords=self.passwords, statusFunction=s, overwrite=True, - destination=destination, fullPath=self.getConfig("fullpath")) - except WrongPasswordError: - self.core.log.info(_("Unrar of %s failed (wrong password)") % fname) - continue - except CommandError, e: - if self.core.debug: - print_exc() - if re.search("Cannot find volume", e.stderr): - self.core.log.info(_("Unrar of %s failed (missing volume)") % fname) - continue - try: - if e.getExitCode() == 1 and len(u.listContent(u.getPassword())) == 1: - self.core.log.info(_("Unrar of %s ok") % fname) - self.removeFiles(pack, fname) - except: - if self.core.debug: - print_exc() - self.core.log.info(_("Unrar of %s failed") % fname) - continue - except LowRamError: - self.log.warning(_( - "Your ram amount of %s MB seems not sufficient to unrar this file. You can deactivate this warning and risk instability") % self.ram) - continue - except UnknownError: - if self.core.debug: - print_exc() - self.core.log.info(_("Unrar of %s failed") % fname) - continue - else: - if success: - self.core.log.info(_("Unrar of %s ok") % fname) - self.removeFiles(pack, fname) - - if os.name != "nt" and self.core.config['permission']['change_dl'] and\ - self.core.config['permission']['change_file']: - ownerUser = self.core.config['permission']['user'] - fileMode = int(self.core.config['permission']['file'], 8) - - self.core.log.debug("Setting destination file/directory owner / mode to %s / %s" - % (ownerUser, fileMode)) - - uinfo = getpwnam(ownerUser) - self.core.log.debug("Uid/Gid is %s/%s." % (uinfo.pw_uid, uinfo.pw_gid)) - self.setOwner(destination, uinfo.pw_uid, uinfo.pw_gid, fileMode) - self.core.log.debug("The owner/rights have been successfully changed.") - - self.core.hookManager.unrarFinished(folder, fname) - else: - self.core.log.info(_("Unrar of %s failed (wrong password or bad parts)") % fname) - finally: - pyfile.setProgress(100) - pyfile.setStatus("finished") - pyfile.release() - diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py new file mode 100644 index 000000000..8ff99b5e8 --- /dev/null +++ b/module/plugins/internal/UnRar.py @@ -0,0 +1,237 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" +from __future__ import with_statement + +import sys +import os +from os.path import exists, join, isabs, isdir +from os import remove, makedirs, rmdir, listdir, chmod +from traceback import print_exc + +from module.plugins.Hook import Hook +from module.lib.pyunrar import Unrar, WrongPasswordError, CommandError, UnknownError, LowRamError + +from module.utils import save_join + +if os.name != "nt": + from pwd import getpwnam + from os import chown + +import re + +class UnRar(Hook): + __name__ = "UnRar" + __version__ = "0.1" + __description__ = """Unrar plugin for archive extractor""" + __config__ = [("activated", "bool", "Activated", False), + ("fullpath", "bool", "extract full path", True), + ("overwrite", "bool", "overwrite files", True), + ("passwordfile", "str", "unrar password file", "unrar_passwords.txt"), + ("deletearchive", "bool", "delete archives when done", False), + ("ramwarning", "bool", "warn about low ram", True), + ("renice", "int", "Cpu Priority", 10), + ("unrar_destination", "str", "Unpack files to", "")] + __threaded__ = ["packageFinished"] + __author_name__ = ("mkaay") + __author_mail__ = ("mkaay@mkaay.de") + + def setup(self): + self.comments = ["# one password each line"] + self.passwords = [] + if exists(self.getConfig("passwordfile")): + with open(self.getConfig("passwordfile"), "r") as f: + for l in f.readlines(): + l = l.strip("\n\r") + if l and not l.startswith("#"): + self.passwords.append(l) + else: + with open(self.getConfig("passwordfile"), "w") as f: + f.writelines(self.comments) + self.re_splitfile = re.compile("(.*)\.part(\d+)\.rar$") + + self.ram = 0 #ram in kb for unix osses + try: + f = open("/proc/meminfo") + line = True + while line: + line = f.readline() + if line.startswith("MemTotal:"): + self.ram = int(re.search(r"([0-9]+)", line).group(1)) + except: + self.ram = 0 + + self.ram /= 1024 + + def setOwner(self, d, uid, gid, mode): + if not exists(d): + self.core.log.debug(_("Directory %s does not exist!") % d) + return + + for fileEntry in listdir(d): + fullEntryName = join(d, fileEntry) + if isdir(fullEntryName): + self.setOwner(fullEntryName, uid, gid, mode) + try: + chown(fullEntryName, uid, gid) + chmod(fullEntryName, mode) + except: + self.core.log.debug(_("Chown/Chmod for %s failed") % fullEntryName) + self.core.log.debug(_("Exception: %s") % sys.exc_info()[0]) + continue + try: + chown(d, uid, gid) + chmod(d, mode) + except: + self.core.log.debug(_("Chown/Chmod for %s failed") % d) + self.core.log.debug(_("Exception: %s") % sys.exc_info()[0]) + return + + def addPassword(self, pws): + if not type(pws) == list: pws = [pws] + pws.reverse() + for pw in pws: + pw = pw.strip() + if not pw or pw == "None" or pw in self.passwords: continue + self.passwords.insert(0, pw) + + with open(self.getConfig("passwordfile"), "w") as f: + f.writelines([c + "\n" for c in self.comments]) + f.writelines([p + "\n" for p in self.passwords]) + + def removeFiles(self, pack, fname): + if not self.getConfig("deletearchive"): + return + m = self.re_splitfile.search(fname) + + download_folder = self.core.config['general']['download_folder'] + if self.core.config['general']['folder_per_package']: + folder = join(download_folder, pack.folder.decode(sys.getfilesystemencoding())) + else: + folder = download_folder + if m: + nre = re.compile("%s\.part\d+\.rar" % m.group(1)) + for fid, data in pack.getChildren().iteritems(): + if nre.match(data["name"]): + remove(join(folder, data["name"])) + elif not m and fname.endswith(".rar"): + nre = re.compile("^%s\.r..$" % fname.replace(".rar", "")) + for fid, data in pack.getChildren().iteritems(): + if nre.match(data["name"]): + remove(join(folder, data["name"])) + + def packageFinished(self, pack): + if pack.password and pack.password.strip() and pack.password.strip() != "None": + self.addPassword(pack.password.splitlines()) + files = [] + + for fid, data in pack.getChildren().iteritems(): + m = self.re_splitfile.search(data["name"]) + if m and int(m.group(2)) == 1: + files.append((fid, m.group(0))) + elif not m and data["name"].endswith(".rar"): + files.append((fid, data["name"])) + + for fid, fname in files: + self.core.log.info(_("starting Unrar of %s") % fname) + pyfile = self.core.files.getFile(fid) + pyfile.setStatus("processing") + + def s(p): + pyfile.setProgress(p) + + download_folder = self.core.config['general']['download_folder'] + self.core.log.debug(_("download folder %s") % download_folder) + + folder = save_join(download_folder, pack.folder, "") + + + destination = folder + if self.getConfig("unrar_destination") and not self.getConfig("unrar_destination").lower() == "none": + destination = self.getConfig("unrar_destination") + sub = "." + if self.core.config['general']['folder_per_package']: + sub = pack.folder.decode(sys.getfilesystemencoding()) + if isabs(destination): + destination = join(destination, sub, "") + else: + destination = join(folder, destination, sub, "") + + self.core.log.debug(_("Destination folder %s") % destination) + if not exists(destination): + self.core.log.info(_("Creating destination folder %s") % destination) + makedirs(destination) + + u = Unrar(join(folder, fname), tmpdir=join(folder, "tmp"), + ramSize=(self.ram if self.getConfig("ramwarning") else 0), cpu=self.getConfig("renice")) + try: + success = u.crackPassword(passwords=self.passwords, statusFunction=s, overwrite=True, + destination=destination, fullPath=self.getConfig("fullpath")) + except WrongPasswordError: + self.core.log.info(_("Unrar of %s failed (wrong password)") % fname) + continue + except CommandError, e: + if self.core.debug: + print_exc() + if re.search("Cannot find volume", e.stderr): + self.core.log.info(_("Unrar of %s failed (missing volume)") % fname) + continue + try: + if e.getExitCode() == 1 and len(u.listContent(u.getPassword())) == 1: + self.core.log.info(_("Unrar of %s ok") % fname) + self.removeFiles(pack, fname) + except: + if self.core.debug: + print_exc() + self.core.log.info(_("Unrar of %s failed") % fname) + continue + except LowRamError: + self.log.warning(_( + "Your ram amount of %s MB seems not sufficient to unrar this file. You can deactivate this warning and risk instability") % self.ram) + continue + except UnknownError: + if self.core.debug: + print_exc() + self.core.log.info(_("Unrar of %s failed") % fname) + continue + else: + if success: + self.core.log.info(_("Unrar of %s ok") % fname) + self.removeFiles(pack, fname) + + if os.name != "nt" and self.core.config['permission']['change_dl'] and\ + self.core.config['permission']['change_file']: + ownerUser = self.core.config['permission']['user'] + fileMode = int(self.core.config['permission']['file'], 8) + + self.core.log.debug("Setting destination file/directory owner / mode to %s / %s" + % (ownerUser, fileMode)) + + uinfo = getpwnam(ownerUser) + self.core.log.debug("Uid/Gid is %s/%s." % (uinfo.pw_uid, uinfo.pw_gid)) + self.setOwner(destination, uinfo.pw_uid, uinfo.pw_gid, fileMode) + self.core.log.debug("The owner/rights have been successfully changed.") + + self.core.hookManager.unrarFinished(folder, fname) + else: + self.core.log.info(_("Unrar of %s failed (wrong password or bad parts)") % fname) + finally: + pyfile.setProgress(100) + pyfile.setStatus("finished") + pyfile.release() + -- cgit v1.2.3