From acc46fc3497a66a427b795b4a22c6e71d69185a1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 13 Dec 2014 15:56:57 +0100 Subject: Update --- pyload/manager/Account.py | 191 +++++++++++++++ pyload/manager/AccountManager.py | 191 --------------- pyload/manager/Addon.py | 304 +++++++++++++++++++++++ pyload/manager/AddonManager.py | 304 ----------------------- pyload/manager/Captcha.py | 138 +++++++++++ pyload/manager/CaptchaManager.py | 138 ----------- pyload/manager/Event.py | 104 ++++++++ pyload/manager/Plugin.py | 404 +++++++++++++++++++++++++++++++ pyload/manager/PluginManager.py | 404 ------------------------------- pyload/manager/Remote.py | 76 ++++++ pyload/manager/RemoteManager.py | 76 ------ pyload/manager/Thread.py | 302 +++++++++++++++++++++++ pyload/manager/ThreadManager.py | 302 ----------------------- pyload/manager/event/PullEvents.py | 104 -------- pyload/manager/thread/Addon.py | 69 ++++++ pyload/manager/thread/AddonThread.py | 69 ------ pyload/manager/thread/Decrypter.py | 101 ++++++++ pyload/manager/thread/DecrypterThread.py | 101 -------- pyload/manager/thread/Download.py | 213 ++++++++++++++++ pyload/manager/thread/DownloadThread.py | 213 ---------------- pyload/manager/thread/Info.py | 225 +++++++++++++++++ pyload/manager/thread/InfoThread.py | 225 ----------------- pyload/manager/thread/Plugin.py | 130 ++++++++++ pyload/manager/thread/PluginThread.py | 130 ---------- pyload/manager/thread/Server.py | 111 +++++++++ pyload/manager/thread/ServerThread.py | 111 --------- 26 files changed, 2368 insertions(+), 2368 deletions(-) create mode 100644 pyload/manager/Account.py delete mode 100644 pyload/manager/AccountManager.py create mode 100644 pyload/manager/Addon.py delete mode 100644 pyload/manager/AddonManager.py create mode 100644 pyload/manager/Captcha.py delete mode 100644 pyload/manager/CaptchaManager.py create mode 100644 pyload/manager/Event.py create mode 100644 pyload/manager/Plugin.py delete mode 100644 pyload/manager/PluginManager.py create mode 100644 pyload/manager/Remote.py delete mode 100644 pyload/manager/RemoteManager.py create mode 100644 pyload/manager/Thread.py delete mode 100644 pyload/manager/ThreadManager.py delete mode 100644 pyload/manager/event/PullEvents.py create mode 100644 pyload/manager/thread/Addon.py delete mode 100644 pyload/manager/thread/AddonThread.py create mode 100644 pyload/manager/thread/Decrypter.py delete mode 100644 pyload/manager/thread/DecrypterThread.py create mode 100644 pyload/manager/thread/Download.py delete mode 100644 pyload/manager/thread/DownloadThread.py create mode 100644 pyload/manager/thread/Info.py delete mode 100644 pyload/manager/thread/InfoThread.py create mode 100644 pyload/manager/thread/Plugin.py delete mode 100644 pyload/manager/thread/PluginThread.py create mode 100644 pyload/manager/thread/Server.py delete mode 100644 pyload/manager/thread/ServerThread.py (limited to 'pyload/manager') diff --git a/pyload/manager/Account.py b/pyload/manager/Account.py new file mode 100644 index 000000000..2631e1c7d --- /dev/null +++ b/pyload/manager/Account.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +from os.path import exists +from shutil import copy + +from threading import Lock + +from pyload.manager.Event import AccountUpdateEvent +from pyload.utils import chmod, lock + +ACC_VERSION = 1 + + +class AccountManager(object): + """manages all accounts""" + + #---------------------------------------------------------------------- + def __init__(self, core): + """Constructor""" + + self.core = core + self.lock = Lock() + + self.initPlugins() + self.saveAccounts() # save to add categories to conf + + + def initPlugins(self): + self.accounts = {} # key = ( plugin ) + self.plugins = {} + + self.initAccountPlugins() + self.loadAccounts() + + + def getAccountPlugin(self, plugin): + """get account instance for plugin or None if anonymous""" + try: + if plugin in self.accounts: + if plugin not in self.plugins: + klass = self.core.pluginManager.loadClass("accounts", plugin) + if klass: + self.plugins[plugin] = klass(self, self.accounts[plugin]) + else: #@NOTE: The account class no longer exists (blacklisted plugin). Skipping the account to avoid crash + raise + + return self.plugins[plugin] + else: + raise + except Exception: + return None + + + def getAccountPlugins(self): + """ get all account instances""" + + plugins = [] + for plugin in self.accounts.keys(): + plugins.append(self.getAccountPlugin(plugin)) + + return plugins + + + #---------------------------------------------------------------------- + def loadAccounts(self): + """loads all accounts available""" + + try: + with open("accounts.conf", "a+") as f: + content = f.readlines() + version = content[0].split(":")[1].strip() if content else "" + + if not version or int(version) < ACC_VERSION: + copy("accounts.conf", "accounts.backup") + f.seek(0) + f.write("version: " + str(ACC_VERSION)) + + self.core.log.warning(_("Account settings deleted, due to new config format")) + return + + except IOError, e: + self.core.log.error(str(e)) + return + + plugin = "" + name = "" + + for line in content[1:]: + line = line.strip() + + if not line: continue + if line.startswith("#"): continue + if line.startswith("version"): continue + + if line.endswith(":") and line.count(":") == 1: + plugin = line[:-1] + self.accounts[plugin] = {} + + elif line.startswith("@"): + try: + option = line[1:].split() + self.accounts[plugin][name]['options'][option[0]] = [] if len(option) < 2 else ([option[1]] if len(option) < 3 else option[1:]) + except Exception: + pass + + elif ":" in line: + name, sep, pw = line.partition(":") + self.accounts[plugin][name] = {"password": pw, "options": {}, "valid": True} + + + #---------------------------------------------------------------------- + def saveAccounts(self): + """save all account information""" + + try: + with open("accounts.conf", "wb") as f: + f.write("version: " + str(ACC_VERSION) + "\n") + + for plugin, accounts in self.accounts.iteritems(): + f.write("\n") + f.write(plugin + ":\n") + + for name,data in accounts.iteritems(): + f.write("\n\t%s:%s\n" % (name,data['password']) ) + if data['options']: + for option, values in data['options'].iteritems(): + f.write("\t@%s %s\n" % (option, " ".join(values))) + + chmod(f.name, 0600) + + except Exception, e: + self.core.log.error(str(e)) + + + #---------------------------------------------------------------------- + def initAccountPlugins(self): + """init names""" + for name in self.core.pluginManager.getAccountPlugins(): + self.accounts[name] = {} + + + @lock + def updateAccount(self, plugin , user, password=None, options={}): + """add or update account""" + if plugin in self.accounts: + p = self.getAccountPlugin(plugin) + updated = p.updateAccounts(user, password, options) + #since accounts is a ref in plugin self.accounts doesnt need to be updated here + + self.saveAccounts() + if updated: p.scheduleRefresh(user, force=False) + + + @lock + def removeAccount(self, plugin, user): + """remove account""" + + if plugin in self.accounts: + p = self.getAccountPlugin(plugin) + p.removeAccount(user) + + self.saveAccounts() + + + @lock + def getAccountInfos(self, force=True, refresh=False): + data = {} + + if refresh: + self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) + force = False + + for p in self.accounts.keys(): + if self.accounts[p]: + p = self.getAccountPlugin(p) + if p: + data[p.__name] = p.getAllAccounts(force) + else: #@NOTE: When an account has been skipped, p is None + data[p] = [] + else: + data[p] = [] + e = AccountUpdateEvent() + self.core.pullManager.addEvent(e) + return data + + + def sendChange(self): + e = AccountUpdateEvent() + self.core.pullManager.addEvent(e) diff --git a/pyload/manager/AccountManager.py b/pyload/manager/AccountManager.py deleted file mode 100644 index 22345de8d..000000000 --- a/pyload/manager/AccountManager.py +++ /dev/null @@ -1,191 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import with_statement - -from os.path import exists -from shutil import copy - -from threading import Lock - -from pyload.manager.event.PullEvents import AccountUpdateEvent -from pyload.utils import chmod, lock - -ACC_VERSION = 1 - - -class AccountManager(object): - """manages all accounts""" - - #---------------------------------------------------------------------- - def __init__(self, core): - """Constructor""" - - self.core = core - self.lock = Lock() - - self.initPlugins() - self.saveAccounts() # save to add categories to conf - - - def initPlugins(self): - self.accounts = {} # key = ( plugin ) - self.plugins = {} - - self.initAccountPlugins() - self.loadAccounts() - - - def getAccountPlugin(self, plugin): - """get account instance for plugin or None if anonymous""" - try: - if plugin in self.accounts: - if plugin not in self.plugins: - klass = self.core.pluginManager.loadClass("accounts", plugin) - if klass: - self.plugins[plugin] = klass(self, self.accounts[plugin]) - else: #@NOTE: The account class no longer exists (blacklisted plugin). Skipping the account to avoid crash - raise - - return self.plugins[plugin] - else: - raise - except Exception: - return None - - - def getAccountPlugins(self): - """ get all account instances""" - - plugins = [] - for plugin in self.accounts.keys(): - plugins.append(self.getAccountPlugin(plugin)) - - return plugins - - - #---------------------------------------------------------------------- - def loadAccounts(self): - """loads all accounts available""" - - try: - with open("accounts.conf", "a+") as f: - content = f.readlines() - version = content[0].split(":")[1].strip() if content else "" - - if not version or int(version) < ACC_VERSION: - copy("accounts.conf", "accounts.backup") - f.seek(0) - f.write("version: " + str(ACC_VERSION)) - - self.core.log.warning(_("Account settings deleted, due to new config format")) - return - - except IOError, e: - self.core.log.error(str(e)) - return - - plugin = "" - name = "" - - for line in content[1:]: - line = line.strip() - - if not line: continue - if line.startswith("#"): continue - if line.startswith("version"): continue - - if line.endswith(":") and line.count(":") == 1: - plugin = line[:-1] - self.accounts[plugin] = {} - - elif line.startswith("@"): - try: - option = line[1:].split() - self.accounts[plugin][name]['options'][option[0]] = [] if len(option) < 2 else ([option[1]] if len(option) < 3 else option[1:]) - except Exception: - pass - - elif ":" in line: - name, sep, pw = line.partition(":") - self.accounts[plugin][name] = {"password": pw, "options": {}, "valid": True} - - - #---------------------------------------------------------------------- - def saveAccounts(self): - """save all account information""" - - try: - with open("accounts.conf", "wb") as f: - f.write("version: " + str(ACC_VERSION) + "\n") - - for plugin, accounts in self.accounts.iteritems(): - f.write("\n") - f.write(plugin + ":\n") - - for name,data in accounts.iteritems(): - f.write("\n\t%s:%s\n" % (name,data['password']) ) - if data['options']: - for option, values in data['options'].iteritems(): - f.write("\t@%s %s\n" % (option, " ".join(values))) - - chmod(f.name, 0600) - - except Exception, e: - self.core.log.error(str(e)) - - - #---------------------------------------------------------------------- - def initAccountPlugins(self): - """init names""" - for name in self.core.pluginManager.getAccountPlugins(): - self.accounts[name] = {} - - - @lock - def updateAccount(self, plugin , user, password=None, options={}): - """add or update account""" - if plugin in self.accounts: - p = self.getAccountPlugin(plugin) - updated = p.updateAccounts(user, password, options) - #since accounts is a ref in plugin self.accounts doesnt need to be updated here - - self.saveAccounts() - if updated: p.scheduleRefresh(user, force=False) - - - @lock - def removeAccount(self, plugin, user): - """remove account""" - - if plugin in self.accounts: - p = self.getAccountPlugin(plugin) - p.removeAccount(user) - - self.saveAccounts() - - - @lock - def getAccountInfos(self, force=True, refresh=False): - data = {} - - if refresh: - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - force = False - - for p in self.accounts.keys(): - if self.accounts[p]: - p = self.getAccountPlugin(p) - if p: - data[p.__name] = p.getAllAccounts(force) - else: #@NOTE: When an account has been skipped, p is None - data[p] = [] - else: - data[p] = [] - e = AccountUpdateEvent() - self.core.pullManager.addEvent(e) - return data - - - def sendChange(self): - e = AccountUpdateEvent() - self.core.pullManager.addEvent(e) diff --git a/pyload/manager/Addon.py b/pyload/manager/Addon.py new file mode 100644 index 000000000..164068634 --- /dev/null +++ b/pyload/manager/Addon.py @@ -0,0 +1,304 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN, mkaay +# @interface-version: 0.1 + +import __builtin__ + +import traceback +from threading import RLock, Thread + +from types import MethodType + +from pyload.manager.thread.Addon import AddonThread +from pyload.manager.Plugin import literal_eval +from pyload.utils import lock + + +class AddonManager(object): + """Manages addons, delegates and handles Events. + + Every plugin can define events, \ + but some very usefull events are called by the Core. + Contrary to overwriting addon methods you can use event listener, + which provides additional entry point in the control flow. + Only do very short tasks or use threads. + + **Known Events:** + Most addon methods exists as events. These are the additional known events. + + ======================= ============== ================================== + Name Arguments Description + ======================= ============== ================================== + download-preparing fid A download was just queued and will be prepared now. + download-start fid A plugin will immediately starts the download afterwards. + links-added links, pid Someone just added links, you are able to modify the links. + all_downloads-processed Every link was handled, pyload would idle afterwards. + all_downloads-finished Every download in queue is finished. + config-changed The config was changed via the api. + pluginConfigChanged The plugin config changed, due to api or internal process. + ======================= ============== ================================== + + | Notes: + | all_downloads-processed is *always* called before all_downloads-finished. + | config-changed is *always* called before pluginConfigChanged. + + + """ + + def __init__(self, core): + self.core = core + + __builtin__.addonManager = self #: needed to let addons register themself + + self.plugins = [] + self.pluginMap = {} + self.methods = {} #: dict of names and list of methods usable by rpc + + self.events = {} #: contains events + + # registering callback for config event + self.core.config.pluginCB = MethodType(self.dispatchEvent, "pluginConfigChanged", basestring) #@TODO: Rename event pluginConfigChanged + + self.addEvent("pluginConfigChanged", self.manageAddon) + + self.lock = RLock() + self.createIndex() + + + def try_catch(func): + + def new(*args): + try: + return func(*args) + except Exception, e: + args[0].log.error(_("Error executing addon: %s") % e) + if args[0].core.debug: + traceback.print_exc() + + return new + + + def addRPC(self, plugin, func, doc): + plugin = plugin.rpartition(".")[2] + doc = doc.strip() if doc else "" + + if plugin in self.methods: + self.methods[plugin][func] = doc + else: + self.methods[plugin] = {func: doc} + + + def callRPC(self, plugin, func, args, parse): + if not args: + args = tuple() + if parse: + args = tuple([literal_eval(x) for x in args]) + plugin = self.pluginMap[plugin] + f = getattr(plugin, func) + return f(*args) + + + def createIndex(self): + plugins = [] + active = [] + deactive = [] + + for pluginname in self.core.pluginManager.addonPlugins: + try: + # hookClass = getattr(plugin, plugin.__name) + if self.core.config.getPlugin(pluginname, "activated"): + pluginClass = self.core.pluginManager.loadClass("addon", pluginname) + if not pluginClass: + continue + + plugin = pluginClass(self.core, self) + plugins.append(plugin) + self.pluginMap[pluginClass.__name] = plugin + if plugin.isActivated(): + active.append(pluginClass.__name) + else: + deactive.append(pluginname) + + except Exception: + self.core.log.warning(_("Failed activating %(name)s") % {"name": pluginname}) + if self.core.debug: + traceback.print_exc() + + self.core.log.info(_("Activated addons: %s") % ", ".join(sorted(active))) + self.core.log.info(_("Deactivated addons: %s") % ", ".join(sorted(deactive))) + + self.plugins = plugins + + + def manageAddon(self, plugin, name, value): + if name == "activated" and value: + self.activateAddon(plugin) + + elif name == "activated" and not value: + self.deactivateAddon(plugin) + + + def activateAddon(self, pluginname): + # check if already loaded + for inst in self.plugins: + if inst.__name == pluginname: + return + + pluginClass = self.core.pluginManager.loadClass("addon", pluginname) + + if not pluginClass: + return + + self.core.log.debug("Activate addon: %s" % pluginname) + + addon = pluginClass(self.core, self) + self.plugins.append(addon) + self.pluginMap[pluginClass.__name] = addon + + addon.activate() + + + def deactivateAddon(self, pluginname): + for plugin in self.plugins: + if plugin.__name == pluginname: + addon = plugin + break + else: + return + + self.core.log.debug("Deactivate addon: %s" % pluginname) + + addon.deactivate() + + #remove periodic call + self.core.log.debug("Removed callback: %s" % self.core.scheduler.removeJob(addon.cb)) + + self.plugins.remove(addon) + del self.pluginMap[addon.__name] + + + @try_catch + def coreReady(self): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.activate() + + self.dispatchEvent("addon-start") + + + @try_catch + def coreExiting(self): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.exit() + + self.dispatchEvent("addon-exit") + + + @lock + def downloadPreparing(self, pyfile): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.downloadPreparing(pyfile) + + self.dispatchEvent("download-preparing", pyfile) + + + @lock + def downloadFinished(self, pyfile): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.downloadFinished(pyfile) + + self.dispatchEvent("download-finished", pyfile) + + + @lock + @try_catch + def downloadFailed(self, pyfile): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.downloadFailed(pyfile) + + self.dispatchEvent("download-failed", pyfile) + + + @lock + def packageFinished(self, package): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.packageFinished(package) + + self.dispatchEvent("package-finished", package) + + + @lock + def beforeReconnecting(self, ip): + for plugin in self.plugins: + plugin.beforeReconnecting(ip) + + self.dispatchEvent("beforeReconnecting", ip) + + + @lock + def afterReconnecting(self, ip): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.afterReconnecting(ip) + + self.dispatchEvent("afterReconnecting", ip) + + + def startThread(self, function, *args, **kwargs): + return AddonThread(self.core.threadManager, function, args, kwargs) + + + def activePlugins(self): + """ returns all active plugins """ + return [x for x in self.plugins if x.isActivated()] + + + def getAllInfo(self): + """returns info stored by addon plugins""" + info = {} + for name, plugin in self.pluginMap.iteritems(): + if plugin.info: + # copy and convert so str + info[name] = dict( + [(x, str(y) if not isinstance(y, basestring) else y) for x, y in plugin.info.iteritems()]) + return info + + + def getInfo(self, plugin): + info = {} + if plugin in self.pluginMap and self.pluginMap[plugin].info: + info = dict((x, str(y) if not isinstance(y, basestring) else y) + for x, y in self.pluginMap[plugin].info.iteritems()) + return info + + + def addEvent(self, event, func): + """Adds an event listener for event name""" + if event in self.events: + self.events[event].append(func) + else: + self.events[event] = [func] + + + def removeEvent(self, event, func): + """removes previously added event listener""" + if event in self.events: + self.events[event].remove(func) + + + def dispatchEvent(self, event, *args): + """dispatches event with args""" + if event in self.events: + for f in self.events[event]: + try: + f(*args) + except Exception, e: + self.core.log.warning("Error calling event handler %s: %s, %s, %s" + % (event, f, args, str(e))) + if self.core.debug: + traceback.print_exc() diff --git a/pyload/manager/AddonManager.py b/pyload/manager/AddonManager.py deleted file mode 100644 index 2fd744776..000000000 --- a/pyload/manager/AddonManager.py +++ /dev/null @@ -1,304 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN, mkaay -# @interface-version: 0.1 - -import __builtin__ - -import traceback -from threading import RLock, Thread - -from types import MethodType - -from pyload.manager.thread.AddonThread import AddonThread -from pyload.manager.PluginManager import literal_eval -from pyload.utils import lock - - -class AddonManager(object): - """Manages addons, delegates and handles Events. - - Every plugin can define events, \ - but some very usefull events are called by the Core. - Contrary to overwriting addon methods you can use event listener, - which provides additional entry point in the control flow. - Only do very short tasks or use threads. - - **Known Events:** - Most addon methods exists as events. These are the additional known events. - - ======================= ============== ================================== - Name Arguments Description - ======================= ============== ================================== - download-preparing fid A download was just queued and will be prepared now. - download-start fid A plugin will immediately starts the download afterwards. - links-added links, pid Someone just added links, you are able to modify the links. - all_downloads-processed Every link was handled, pyload would idle afterwards. - all_downloads-finished Every download in queue is finished. - config-changed The config was changed via the api. - pluginConfigChanged The plugin config changed, due to api or internal process. - ======================= ============== ================================== - - | Notes: - | all_downloads-processed is *always* called before all_downloads-finished. - | config-changed is *always* called before pluginConfigChanged. - - - """ - - def __init__(self, core): - self.core = core - - __builtin__.addonManager = self #: needed to let addons register themself - - self.plugins = [] - self.pluginMap = {} - self.methods = {} #: dict of names and list of methods usable by rpc - - self.events = {} #: contains events - - # registering callback for config event - self.core.config.pluginCB = MethodType(self.dispatchEvent, "pluginConfigChanged", basestring) #@TODO: Rename event pluginConfigChanged - - self.addEvent("pluginConfigChanged", self.manageAddon) - - self.lock = RLock() - self.createIndex() - - - def try_catch(func): - - def new(*args): - try: - return func(*args) - except Exception, e: - args[0].log.error(_("Error executing addon: %s") % e) - if args[0].core.debug: - traceback.print_exc() - - return new - - - def addRPC(self, plugin, func, doc): - plugin = plugin.rpartition(".")[2] - doc = doc.strip() if doc else "" - - if plugin in self.methods: - self.methods[plugin][func] = doc - else: - self.methods[plugin] = {func: doc} - - - def callRPC(self, plugin, func, args, parse): - if not args: - args = tuple() - if parse: - args = tuple([literal_eval(x) for x in args]) - plugin = self.pluginMap[plugin] - f = getattr(plugin, func) - return f(*args) - - - def createIndex(self): - plugins = [] - active = [] - deactive = [] - - for pluginname in self.core.pluginManager.addonPlugins: - try: - # hookClass = getattr(plugin, plugin.__name) - if self.core.config.getPlugin(pluginname, "activated"): - pluginClass = self.core.pluginManager.loadClass("addon", pluginname) - if not pluginClass: - continue - - plugin = pluginClass(self.core, self) - plugins.append(plugin) - self.pluginMap[pluginClass.__name] = plugin - if plugin.isActivated(): - active.append(pluginClass.__name) - else: - deactive.append(pluginname) - - except Exception: - self.core.log.warning(_("Failed activating %(name)s") % {"name": pluginname}) - if self.core.debug: - traceback.print_exc() - - self.core.log.info(_("Activated addons: %s") % ", ".join(sorted(active))) - self.core.log.info(_("Deactivated addons: %s") % ", ".join(sorted(deactive))) - - self.plugins = plugins - - - def manageAddon(self, plugin, name, value): - if name == "activated" and value: - self.activateAddon(plugin) - - elif name == "activated" and not value: - self.deactivateAddon(plugin) - - - def activateAddon(self, pluginname): - # check if already loaded - for inst in self.plugins: - if inst.__name == pluginname: - return - - pluginClass = self.core.pluginManager.loadClass("addon", pluginname) - - if not pluginClass: - return - - self.core.log.debug("Activate addon: %s" % pluginname) - - addon = pluginClass(self.core, self) - self.plugins.append(addon) - self.pluginMap[pluginClass.__name] = addon - - addon.activate() - - - def deactivateAddon(self, pluginname): - for plugin in self.plugins: - if plugin.__name == pluginname: - addon = plugin - break - else: - return - - self.core.log.debug("Deactivate addon: %s" % pluginname) - - addon.deactivate() - - #remove periodic call - self.core.log.debug("Removed callback: %s" % self.core.scheduler.removeJob(addon.cb)) - - self.plugins.remove(addon) - del self.pluginMap[addon.__name] - - - @try_catch - def coreReady(self): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.activate() - - self.dispatchEvent("addon-start") - - - @try_catch - def coreExiting(self): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.exit() - - self.dispatchEvent("addon-exit") - - - @lock - def downloadPreparing(self, pyfile): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.downloadPreparing(pyfile) - - self.dispatchEvent("download-preparing", pyfile) - - - @lock - def downloadFinished(self, pyfile): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.downloadFinished(pyfile) - - self.dispatchEvent("download-finished", pyfile) - - - @lock - @try_catch - def downloadFailed(self, pyfile): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.downloadFailed(pyfile) - - self.dispatchEvent("download-failed", pyfile) - - - @lock - def packageFinished(self, package): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.packageFinished(package) - - self.dispatchEvent("package-finished", package) - - - @lock - def beforeReconnecting(self, ip): - for plugin in self.plugins: - plugin.beforeReconnecting(ip) - - self.dispatchEvent("beforeReconnecting", ip) - - - @lock - def afterReconnecting(self, ip): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.afterReconnecting(ip) - - self.dispatchEvent("afterReconnecting", ip) - - - def startThread(self, function, *args, **kwargs): - return AddonThread(self.core.threadManager, function, args, kwargs) - - - def activePlugins(self): - """ returns all active plugins """ - return [x for x in self.plugins if x.isActivated()] - - - def getAllInfo(self): - """returns info stored by addon plugins""" - info = {} - for name, plugin in self.pluginMap.iteritems(): - if plugin.info: - # copy and convert so str - info[name] = dict( - [(x, str(y) if not isinstance(y, basestring) else y) for x, y in plugin.info.iteritems()]) - return info - - - def getInfo(self, plugin): - info = {} - if plugin in self.pluginMap and self.pluginMap[plugin].info: - info = dict((x, str(y) if not isinstance(y, basestring) else y) - for x, y in self.pluginMap[plugin].info.iteritems()) - return info - - - def addEvent(self, event, func): - """Adds an event listener for event name""" - if event in self.events: - self.events[event].append(func) - else: - self.events[event] = [func] - - - def removeEvent(self, event, func): - """removes previously added event listener""" - if event in self.events: - self.events[event].remove(func) - - - def dispatchEvent(self, event, *args): - """dispatches event with args""" - if event in self.events: - for f in self.events[event]: - try: - f(*args) - except Exception, e: - self.core.log.warning("Error calling event handler %s: %s, %s, %s" - % (event, f, args, str(e))) - if self.core.debug: - traceback.print_exc() diff --git a/pyload/manager/Captcha.py b/pyload/manager/Captcha.py new file mode 100644 index 000000000..e54eacf30 --- /dev/null +++ b/pyload/manager/Captcha.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN, mkaay + +from time import time +from traceback import print_exc +from threading import Lock + +from pyload.utils import encode + + +class CaptchaManager(object): + def __init__(self, core): + self.lock = Lock() + self.core = core + self.tasks = [] # task store, for outgoing tasks only + self.ids = 0 # only for internal purpose + + def newTask(self, img, format, file, result_type): + task = CaptchaTask(self.ids, img, format, file, result_type) + self.ids += 1 + return task + + def removeTask(self, task): + self.lock.acquire() + if task in self.tasks: + self.tasks.remove(task) + self.lock.release() + + def getTask(self): + self.lock.acquire() + for task in self.tasks: + if task.status in ("waiting", "shared-user"): + self.lock.release() + return task + self.lock.release() + return None + + def getTaskByID(self, tid): + self.lock.acquire() + for task in self.tasks: + if task.id == str(tid): # task ids are strings + self.lock.release() + return task + self.lock.release() + return None + + def handleCaptcha(self, task, timeout=50): + cli = self.core.isClientConnected() + + if cli: #: client connected -> should solve the captcha + task.setWaiting(timeout) #wait 50 sec for response + + for plugin in self.core.addonManager.activePlugins(): + try: + plugin.captchaTask(task) + except Exception: + if self.core.debug: + print_exc() + + if task.handler or cli: #: the captcha was handled + self.tasks.append(task) + return True + task.error = _("No Client connected for captcha decrypting") + return False + + +class CaptchaTask(object): + def __init__(self, id, img, format, file, result_type='textual'): + self.id = str(id) + self.captchaImg = img + self.captchaFormat = format + self.captchaFile = file + self.captchaResultType = result_type + self.handler = [] #: the hook plugins that will take care of the solution + self.result = None + self.waitUntil = None + self.error = None # error message + self.status = "init" + self.data = {} # handler can store data here + + def getCaptcha(self): + return self.captchaImg, self.captchaFormat, self.captchaResultType + + def setResult(self, text): + if self.isTextual(): + self.result = text + if self.isPositional(): + try: + parts = text.split(',') + self.result = (int(parts[0]), int(parts[1])) + except Exception: + self.result = None + + def getResult(self): + return encode(self.result) + + def getStatus(self): + return self.status + + def setWaiting(self, sec): + """ let the captcha wait secs for the solution """ + self.waitUntil = max(time() + sec, self.waitUntil) + self.status = "waiting" + + def isWaiting(self): + if self.result or self.error or self.timedOut(): + return False + else: + return True + + def isTextual(self): + """ returns if text is written on the captcha """ + return self.captchaResultType == 'textual' + + def isPositional(self): + """ returns if user have to click a specific region on the captcha """ + return self.captchaResultType == 'positional' + + def setWatingForUser(self, exclusive): + if exclusive: + self.status = "user" + else: + self.status = "shared-user" + + def timedOut(self): + return time() > self.waitUntil + + def invalid(self): + """ indicates the captcha was not correct """ + for x in self.handler: + x.captchaInvalid(self) + + def correct(self): + for x in self.handler: + x.captchaCorrect(self) + + def __str__(self): + return "" % self.id diff --git a/pyload/manager/CaptchaManager.py b/pyload/manager/CaptchaManager.py deleted file mode 100644 index e54eacf30..000000000 --- a/pyload/manager/CaptchaManager.py +++ /dev/null @@ -1,138 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN, mkaay - -from time import time -from traceback import print_exc -from threading import Lock - -from pyload.utils import encode - - -class CaptchaManager(object): - def __init__(self, core): - self.lock = Lock() - self.core = core - self.tasks = [] # task store, for outgoing tasks only - self.ids = 0 # only for internal purpose - - def newTask(self, img, format, file, result_type): - task = CaptchaTask(self.ids, img, format, file, result_type) - self.ids += 1 - return task - - def removeTask(self, task): - self.lock.acquire() - if task in self.tasks: - self.tasks.remove(task) - self.lock.release() - - def getTask(self): - self.lock.acquire() - for task in self.tasks: - if task.status in ("waiting", "shared-user"): - self.lock.release() - return task - self.lock.release() - return None - - def getTaskByID(self, tid): - self.lock.acquire() - for task in self.tasks: - if task.id == str(tid): # task ids are strings - self.lock.release() - return task - self.lock.release() - return None - - def handleCaptcha(self, task, timeout=50): - cli = self.core.isClientConnected() - - if cli: #: client connected -> should solve the captcha - task.setWaiting(timeout) #wait 50 sec for response - - for plugin in self.core.addonManager.activePlugins(): - try: - plugin.captchaTask(task) - except Exception: - if self.core.debug: - print_exc() - - if task.handler or cli: #: the captcha was handled - self.tasks.append(task) - return True - task.error = _("No Client connected for captcha decrypting") - return False - - -class CaptchaTask(object): - def __init__(self, id, img, format, file, result_type='textual'): - self.id = str(id) - self.captchaImg = img - self.captchaFormat = format - self.captchaFile = file - self.captchaResultType = result_type - self.handler = [] #: the hook plugins that will take care of the solution - self.result = None - self.waitUntil = None - self.error = None # error message - self.status = "init" - self.data = {} # handler can store data here - - def getCaptcha(self): - return self.captchaImg, self.captchaFormat, self.captchaResultType - - def setResult(self, text): - if self.isTextual(): - self.result = text - if self.isPositional(): - try: - parts = text.split(',') - self.result = (int(parts[0]), int(parts[1])) - except Exception: - self.result = None - - def getResult(self): - return encode(self.result) - - def getStatus(self): - return self.status - - def setWaiting(self, sec): - """ let the captcha wait secs for the solution """ - self.waitUntil = max(time() + sec, self.waitUntil) - self.status = "waiting" - - def isWaiting(self): - if self.result or self.error or self.timedOut(): - return False - else: - return True - - def isTextual(self): - """ returns if text is written on the captcha """ - return self.captchaResultType == 'textual' - - def isPositional(self): - """ returns if user have to click a specific region on the captcha """ - return self.captchaResultType == 'positional' - - def setWatingForUser(self, exclusive): - if exclusive: - self.status = "user" - else: - self.status = "shared-user" - - def timedOut(self): - return time() > self.waitUntil - - def invalid(self): - """ indicates the captcha was not correct """ - for x in self.handler: - x.captchaInvalid(self) - - def correct(self): - for x in self.handler: - x.captchaCorrect(self) - - def __str__(self): - return "" % self.id diff --git a/pyload/manager/Event.py b/pyload/manager/Event.py new file mode 100644 index 000000000..20897290e --- /dev/null +++ b/pyload/manager/Event.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +# @author: mkaay + +from time import time +from pyload.utils import uniqify + +class PullManager(object): + def __init__(self, core): + self.core = core + self.clients = [] + + def newClient(self, uuid): + self.clients.append(Client(uuid)) + + def clean(self): + for n, client in enumerate(self.clients): + if client.lastActive + 30 < time(): + del self.clients[n] + + def getEvents(self, uuid): + events = [] + validUuid = False + for client in self.clients: + if client.uuid == uuid: + client.lastActive = time() + validUuid = True + while client.newEvents(): + events.append(client.popEvent().toList()) + break + if not validUuid: + self.newClient(uuid) + events = [ReloadAllEvent("queue").toList(), ReloadAllEvent("collector").toList()] + return uniqify(events) + + def addEvent(self, event): + for client in self.clients: + client.addEvent(event) + +class Client(object): + def __init__(self, uuid): + self.uuid = uuid + self.lastActive = time() + self.events = [] + + def newEvents(self): + return len(self.events) > 0 + + def popEvent(self): + if not len(self.events): + return None + return self.events.pop(0) + + def addEvent(self, event): + self.events.append(event) + +class UpdateEvent(object): + def __init__(self, itype, iid, destination): + assert itype == "pack" or itype == "file" + assert destination == "queue" or destination == "collector" + self.type = itype + self.id = iid + self.destination = destination + + def toList(self): + return ["update", self.destination, self.type, self.id] + +class RemoveEvent(object): + def __init__(self, itype, iid, destination): + assert itype == "pack" or itype == "file" + assert destination == "queue" or destination == "collector" + self.type = itype + self.id = iid + self.destination = destination + + def toList(self): + return ["remove", self.destination, self.type, self.id] + +class InsertEvent(object): + def __init__(self, itype, iid, after, destination): + assert itype == "pack" or itype == "file" + assert destination == "queue" or destination == "collector" + self.type = itype + self.id = iid + self.after = after + self.destination = destination + + def toList(self): + return ["insert", self.destination, self.type, self.id, self.after] + +class ReloadAllEvent(object): + def __init__(self, destination): + assert destination == "queue" or destination == "collector" + self.destination = destination + + def toList(self): + return ["reload", self.destination] + +class AccountUpdateEvent(object): + def toList(self): + return ["account"] + +class ConfigUpdateEvent(object): + def toList(self): + return ["config"] diff --git a/pyload/manager/Plugin.py b/pyload/manager/Plugin.py new file mode 100644 index 000000000..71a7131c4 --- /dev/null +++ b/pyload/manager/Plugin.py @@ -0,0 +1,404 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import re +import sys + +from itertools import chain +from os import listdir, makedirs +from os.path import isdir, isfile, join, exists, abspath +from sys import version_info +from traceback import print_exc +from urllib import unquote + +from SafeEval import const_eval as literal_eval + + +class PluginManager(object): + ROOT = "pyload.plugin." + USERROOT = "userplugins." + TYPES = ["account", "addon", "container", "crypter", "hook", "hoster", "internal", "ocr"] + + PATTERN = re.compile(r'__pattern\s*=\s*u?r("|\')([^"\']+)') + VERSION = re.compile(r'__version\s*=\s*("|\')([\d.]+)') + CONFIG = re.compile(r'__config\s*=\s*\[([^\]]+)', re.M) + DESC = re.compile(r'__description\s*=\s*("|"""|\')([^"\']+)') + + + def __init__(self, core): + self.core = core + + self.plugins = {} + self.createIndex() + + #register for import addon + sys.meta_path.append(self) + + + def loadTypes(self): + rootdir = join(pypath, "pyload", "plugins") + userdir = "userplugins" + + types = set().union(*[[d for d in listdir(p) if isdir(join(p, d))] + for p in (rootdir, userdir) if exists(p)]) + + if not types: + self.log.critical(_("No plugins found!")) + + self.TYPES = list(set(self.TYPES) | types) + + + def createIndex(self): + """create information for all plugins available""" + + sys.path.append(abspath("")) + + self.loadTypes() + + for type in self.TYPES: + self.plugins[type] = self.parse(type) + setattr(self, "%sPlugins" % type, self.plugins[type]) + + self.plugins['addon'] = self.addonPlugins.update(self.hookPlugins) + + self.core.log.debug("Created index of plugins") + + + def parse(self, folder, rootplugins={}): + """ + returns dict with information + home contains parsed plugins from pyload. + """ + + plugins = {} + + if rootplugins: + try: + pfolder = join("userplugins", folder) + if not exists(pfolder): + makedirs(pfolder) + + for ifile in (join("userplugins", "__init__.py"), + join(pfolder, "__init__.py")): + if not exists(ifile): + f = open(ifile, "wb") + f.close() + + except IOError, e: + self.core.log.critical(str(e)) + return rootplugins + + else: + pfolder = join(pypath, "pyload", "plugins", folder) + + for f in listdir(pfolder): + if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( + "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): + + try: + with open(join(pfolder, f)) as data: + content = data.read() + + except IOError, e: + self.core.log.error(str(e)) + continue + + if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): #@TODO: Remove in 0.4.10 + continue + + elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): #@TODO: Remove in 0.4.10 + continue + + elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): #@TODO: Remove in 0.4.10 + continue + + name = f[:-3] + if name[-1] == ".": + name = name[:-4] + + version = self.VERSION.findall(content) + if version: + version = float(version[0][1]) + else: + version = 0 + + if rootplugins and name in rootplugins: + if rootplugins[name]['version'] >= version: + continue + + plugins[name] = {} + plugins[name]['version'] = version + + module = f.replace(".pyc", "").replace(".py", "") + + # the plugin is loaded from user directory + plugins[name]['user'] = True if rootplugins else False + plugins[name]['name'] = module + + pattern = self.PATTERN.findall(content) + + if pattern: + pattern = pattern[0][1] + + try: + regexp = re.compile(pattern) + except Exception: + self.core.log.error(_("%s has a invalid pattern") % name) + pattern = r'^unmatchable$' + regexp = re.compile(pattern) + + plugins[name]['pattern'] = pattern + plugins[name]['re'] = regexp + + # internals have no config + if folder == "internal": + self.core.config.deleteConfig(name) + continue + + config = self.CONFIG.findall(content) + if config: + try: + config = literal_eval(config[0].strip().replace("\n", "").replace("\r", "")) + desc = self.DESC.findall(content) + desc = desc[0][1] if desc else "" + + if type(config[0]) == tuple: + config = [list(x) for x in config] + else: + config = [list(config)] + + if folder not in ("account", "internal") and not [True for item in config if item[0] == "activated"]: + config.insert(0, ["activated", "bool", "Activated", False if folder in ("addon", "hook") else True]) + + self.core.config.addPluginConfig(name, config, desc) + except Exception: + self.core.log.error("Invalid config in %s: %s" % (name, config)) + + elif folder in ("addon", "hook"): #force config creation + desc = self.DESC.findall(content) + desc = desc[0][1] if desc else "" + config = (["activated", "bool", "Activated", False],) + + try: + self.core.config.addPluginConfig(name, config, desc) + except Exception: + self.core.log.error("Invalid config in %s: %s" % (name, config)) + + if not rootplugins and plugins: #: Double check + plugins.update(self.parse(folder, plugins)) + + return plugins + + + def parseUrls(self, urls): + """parse plugins for given list of urls""" + + last = None + res = [] #: tupels of (url, plugintype, pluginname) + + for url in urls: + if type(url) not in (str, unicode, buffer): + continue + + url = unquote(url) + + if last and last[2]['re'].match(url): + res.append((url, last[0], last[1])) + continue + + for type in self.TYPES: + for name, plugin in self.plugins[type]: + + m = None + try: + if 'pattern' in plugin: + m = plugin['re'].match(url) + + except KeyError: + self.core.log.error(_("Plugin [%(type)s] %(name)s skipped due broken pattern") + % {'name': name, 'type': type}) + + if m: + res.append((url, type, name)) + last = (type, name, plugin) + break + else: + res.append((url, "internal", "BasePlugin")) + + return res + + + def findPlugin(self, type, name): + if type not in self.plugins: + return None + + elif name not in self.plugins[type]: + self.core.log.warning(_("Plugin [%(type)s] %(name)s not found | Using plugin: [internal] BasePlugin") + % {'name': name, 'type': type}) + return self.internalPlugins["BasePlugin"] + + else: + return self.plugins[type][name] + + + def getPlugin(self, type, name, original=False): + """return plugin module from hoster|decrypter|container""" + plugin = self.findPlugin(type, name) + + if plugin is None: + return {} + + if "new_module" in plugin and not original: + return plugin['new_module'] + else: + return self.loadModule(type, name) + + + def getPluginName(self, type, name): + """ used to obtain new name if other plugin was injected""" + plugin = self.findPlugin(type, name) + + if plugin is None: + return "" + + if "new_name" in plugin: + return plugin['new_name'] + + return name + + + def loadModule(self, type, name): + """ Returns loaded module for plugin + + :param type: plugin type, subfolder of pyload.plugins + :param name: + """ + plugins = self.plugins[type] + + if name in plugins: + if "module" in plugins[name]: + return plugins[name]['module'] + + try: + module = __import__(self.ROOT + "%s.%s" % (type, plugins[name]['name']), globals(), locals(), + plugins[name]['name']) + + except Exception, e: + self.core.log.error(_("Error importing plugin: [%(type)s] %(name)s (v%(version).2f) | %(errmsg)s") + % {'name': name, 'type': type, 'version': plugins[name]['version'], "errmsg": str(e)}) + if self.core.debug: + print_exc() + + else: + plugins[name]['module'] = module #: cache import, maybe unneeded + + self.core.log.debug(_("Loaded plugin: [%(type)s] %(name)s (v%(version).2f)") + % {'name': name, 'type': type, 'version': plugins[name]['version']}) + return module + + + def loadClass(self, type, name): + """Returns the class of a plugin with the same name""" + module = self.loadModule(type, name) + if module: + return getattr(module, name) + else: + return None + + + def getAccountPlugins(self): + """return list of account plugin names""" + return self.accountPlugins.keys() + + + def find_module(self, fullname, path=None): + #redirecting imports if necesarry + if fullname.startswith(self.ROOT) or fullname.startswith(self.USERROOT): #seperate pyload plugins + if fullname.startswith(self.USERROOT): user = 1 + else: user = 0 #used as bool and int + + split = fullname.split(".") + if len(split) != 4 - user: return + type, name = split[2 - user:4 - user] + + if type in self.plugins and name in self.plugins[type]: + #userplugin is a newer version + if not user and self.plugins[type][name]['user']: + return self + #imported from userdir, but pyloads is newer + if user and not self.plugins[type][name]['user']: + return self + + + def load_module(self, name, replace=True): + if name not in sys.modules: #could be already in modules + if replace: + if self.ROOT in name: + newname = name.replace(self.ROOT, self.USERROOT) + else: + newname = name.replace(self.USERROOT, self.ROOT) + else: + newname = name + + base, plugin = newname.rsplit(".", 1) + + self.core.log.debug("Redirected import %s -> %s" % (name, newname)) + + module = __import__(newname, globals(), locals(), [plugin]) + #inject under new an old name + sys.modules[name] = module + sys.modules[newname] = module + + return sys.modules[name] + + + def reloadPlugins(self, type_plugins): + """ reload and reindex plugins """ + if not type_plugins: + return None + + self.core.log.debug("Request reload of plugins: %s" % type_plugins) + + reloaded = [] + + as_dict = {} + for t,n in type_plugins: + if t in as_dict: + as_dict[t].append(n) + else: + as_dict[t] = [n] + + for type in as_dict.iterkeys(): + if type in ("addon", "internal"): #: do not reload them because would cause to much side effects + self.core.log.debug("Skipping reload for plugin: [%(type)s] %(name)s" % {'name': plugin, 'type': type}) + continue + + for plugin in as_dict[type]: + if plugin in self.plugins[type] and "module" in self.plugins[type][plugin]: + self.core.log.debug(_("Reloading plugin: [%(type)s] %(name)s") % {'name': plugin, 'type': type}) + + try: + reload(self.plugins[type][plugin]['module']) + + except Exception, e: + self.core.log.error(_("Error when reloading plugin: [%(type)s] %(name)s") % {'name': plugin, 'type': type}, e) + continue + + else: + reloaded.append((type, plugin)) + + #index creation + self.plugins[type] = self.parse(type) + setattr(self, "%sPlugins" % type, self.plugins[type]) + + if "account" in as_dict: #: accounts needs to be reloaded + self.core.accountManager.initPlugins() + self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) + + return reloaded #: return a list of the plugins successfully reloaded + + + def reloadPlugin(self, type_plugin): + """ reload and reindex ONE plugin """ + return True if self.reloadPlugins(type_plugin) else False diff --git a/pyload/manager/PluginManager.py b/pyload/manager/PluginManager.py deleted file mode 100644 index c327c991a..000000000 --- a/pyload/manager/PluginManager.py +++ /dev/null @@ -1,404 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import with_statement - -import re -import sys - -from itertools import chain -from os import listdir, makedirs -from os.path import isdir, isfile, join, exists, abspath -from sys import version_info -from traceback import print_exc -from urllib import unquote - -from SafeEval import const_eval as literal_eval - - -class PluginManager(object): - ROOT = "pyload.plugins." - USERROOT = "userplugins." - TYPES = ["account", "addon", "container", "crypter", "hook", "hoster", "internal", "ocr"] - - PATTERN = re.compile(r'__pattern\s*=\s*u?r("|\')([^"\']+)') - VERSION = re.compile(r'__version\s*=\s*("|\')([\d.]+)') - CONFIG = re.compile(r'__config\s*=\s*\[([^\]]+)', re.M) - DESC = re.compile(r'__description\s*=\s*("|"""|\')([^"\']+)') - - - def __init__(self, core): - self.core = core - - self.plugins = {} - self.createIndex() - - #register for import addon - sys.meta_path.append(self) - - - def loadTypes(self): - rootdir = join(pypath, "pyload", "plugins") - userdir = "userplugins" - - types = set().union(*[[d for d in listdir(p) if isdir(join(p, d))] - for p in (rootdir, userdir) if exists(p)]) - - if not types: - self.log.critical(_("No plugins found!")) - - self.TYPES = list(set(self.TYPES) | types) - - - def createIndex(self): - """create information for all plugins available""" - - sys.path.append(abspath("")) - - self.loadTypes() - - for type in self.TYPES: - self.plugins[type] = self.parse(type) - setattr(self, "%sPlugins" % type, self.plugins[type]) - - self.plugins['addon'] = self.addonPlugins.update(self.hookPlugins) - - self.core.log.debug("Created index of plugins") - - - def parse(self, folder, rootplugins={}): - """ - returns dict with information - home contains parsed plugins from pyload. - """ - - plugins = {} - - if rootplugins: - try: - pfolder = join("userplugins", folder) - if not exists(pfolder): - makedirs(pfolder) - - for ifile in (join("userplugins", "__init__.py"), - join(pfolder, "__init__.py")): - if not exists(ifile): - f = open(ifile, "wb") - f.close() - - except IOError, e: - self.core.log.critical(str(e)) - return rootplugins - - else: - pfolder = join(pypath, "pyload", "plugins", folder) - - for f in listdir(pfolder): - if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( - "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): - - try: - with open(join(pfolder, f)) as data: - content = data.read() - - except IOError, e: - self.core.log.error(str(e)) - continue - - if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): #@TODO: Remove in 0.4.10 - continue - - elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): #@TODO: Remove in 0.4.10 - continue - - elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): #@TODO: Remove in 0.4.10 - continue - - name = f[:-3] - if name[-1] == ".": - name = name[:-4] - - version = self.VERSION.findall(content) - if version: - version = float(version[0][1]) - else: - version = 0 - - if rootplugins and name in rootplugins: - if rootplugins[name]['version'] >= version: - continue - - plugins[name] = {} - plugins[name]['version'] = version - - module = f.replace(".pyc", "").replace(".py", "") - - # the plugin is loaded from user directory - plugins[name]['user'] = True if rootplugins else False - plugins[name]['name'] = module - - pattern = self.PATTERN.findall(content) - - if pattern: - pattern = pattern[0][1] - - try: - regexp = re.compile(pattern) - except Exception: - self.core.log.error(_("%s has a invalid pattern") % name) - pattern = r'^unmatchable$' - regexp = re.compile(pattern) - - plugins[name]['pattern'] = pattern - plugins[name]['re'] = regexp - - # internals have no config - if folder == "internal": - self.core.config.deleteConfig(name) - continue - - config = self.CONFIG.findall(content) - if config: - try: - config = literal_eval(config[0].strip().replace("\n", "").replace("\r", "")) - desc = self.DESC.findall(content) - desc = desc[0][1] if desc else "" - - if type(config[0]) == tuple: - config = [list(x) for x in config] - else: - config = [list(config)] - - if folder not in ("account", "internal") and not [True for item in config if item[0] == "activated"]: - config.insert(0, ["activated", "bool", "Activated", False if folder in ("addon", "hook") else True]) - - self.core.config.addPluginConfig(name, config, desc) - except Exception: - self.core.log.error("Invalid config in %s: %s" % (name, config)) - - elif folder in ("addon", "hook"): #force config creation - desc = self.DESC.findall(content) - desc = desc[0][1] if desc else "" - config = (["activated", "bool", "Activated", False],) - - try: - self.core.config.addPluginConfig(name, config, desc) - except Exception: - self.core.log.error("Invalid config in %s: %s" % (name, config)) - - if not rootplugins and plugins: #: Double check - plugins.update(self.parse(folder, plugins)) - - return plugins - - - def parseUrls(self, urls): - """parse plugins for given list of urls""" - - last = None - res = [] #: tupels of (url, plugintype, pluginname) - - for url in urls: - if type(url) not in (str, unicode, buffer): - continue - - url = unquote(url) - - if last and last[2]['re'].match(url): - res.append((url, last[0], last[1])) - continue - - for type in self.TYPES: - for name, plugin in self.plugins[type]: - - m = None - try: - if 'pattern' in plugin: - m = plugin['re'].match(url) - - except KeyError: - self.core.log.error(_("Plugin [%(type)s] %(name)s skipped due broken pattern") - % {'name': name, 'type': type}) - - if m: - res.append((url, type, name)) - last = (type, name, plugin) - break - else: - res.append((url, "internal", "BasePlugin")) - - return res - - - def findPlugin(self, type, name): - if type not in self.plugins: - return None - - elif name not in self.plugins[type]: - self.core.log.warning(_("Plugin [%(type)s] %(name)s not found | Using plugin: [internal] BasePlugin") - % {'name': name, 'type': type}) - return self.internalPlugins["BasePlugin"] - - else: - return self.plugins[type][name] - - - def getPlugin(self, type, name, original=False): - """return plugin module from hoster|decrypter|container""" - plugin = self.findPlugin(type, name) - - if plugin is None: - return {} - - if "new_module" in plugin and not original: - return plugin['new_module'] - else: - return self.loadModule(type, name) - - - def getPluginName(self, type, name): - """ used to obtain new name if other plugin was injected""" - plugin = self.findPlugin(type, name) - - if plugin is None: - return "" - - if "new_name" in plugin: - return plugin['new_name'] - - return name - - - def loadModule(self, type, name): - """ Returns loaded module for plugin - - :param type: plugin type, subfolder of pyload.plugins - :param name: - """ - plugins = self.plugins[type] - - if name in plugins: - if "module" in plugins[name]: - return plugins[name]['module'] - - try: - module = __import__(self.ROOT + "%s.%s" % (type, plugins[name]['name']), globals(), locals(), - plugins[name]['name']) - - except Exception, e: - self.core.log.error(_("Error importing plugin: [%(type)s] %(name)s (v%(version).2f) | %(errmsg)s") - % {'name': name, 'type': type, 'version': plugins[name]['version'], "errmsg": str(e)}) - if self.core.debug: - print_exc() - - else: - plugins[name]['module'] = module #: cache import, maybe unneeded - - self.core.log.debug(_("Loaded plugin: [%(type)s] %(name)s (v%(version).2f)") - % {'name': name, 'type': type, 'version': plugins[name]['version']}) - return module - - - def loadClass(self, type, name): - """Returns the class of a plugin with the same name""" - module = self.loadModule(type, name) - if module: - return getattr(module, name) - else: - return None - - - def getAccountPlugins(self): - """return list of account plugin names""" - return self.accountPlugins.keys() - - - def find_module(self, fullname, path=None): - #redirecting imports if necesarry - if fullname.startswith(self.ROOT) or fullname.startswith(self.USERROOT): #seperate pyload plugins - if fullname.startswith(self.USERROOT): user = 1 - else: user = 0 #used as bool and int - - split = fullname.split(".") - if len(split) != 4 - user: return - type, name = split[2 - user:4 - user] - - if type in self.plugins and name in self.plugins[type]: - #userplugin is a newer version - if not user and self.plugins[type][name]['user']: - return self - #imported from userdir, but pyloads is newer - if user and not self.plugins[type][name]['user']: - return self - - - def load_module(self, name, replace=True): - if name not in sys.modules: #could be already in modules - if replace: - if self.ROOT in name: - newname = name.replace(self.ROOT, self.USERROOT) - else: - newname = name.replace(self.USERROOT, self.ROOT) - else: - newname = name - - base, plugin = newname.rsplit(".", 1) - - self.core.log.debug("Redirected import %s -> %s" % (name, newname)) - - module = __import__(newname, globals(), locals(), [plugin]) - #inject under new an old name - sys.modules[name] = module - sys.modules[newname] = module - - return sys.modules[name] - - - def reloadPlugins(self, type_plugins): - """ reload and reindex plugins """ - if not type_plugins: - return None - - self.core.log.debug("Request reload of plugins: %s" % type_plugins) - - reloaded = [] - - as_dict = {} - for t,n in type_plugins: - if t in as_dict: - as_dict[t].append(n) - else: - as_dict[t] = [n] - - for type in as_dict.iterkeys(): - if type in ("addon", "internal"): #: do not reload them because would cause to much side effects - self.core.log.debug("Skipping reload for plugin: [%(type)s] %(name)s" % {'name': plugin, 'type': type}) - continue - - for plugin in as_dict[type]: - if plugin in self.plugins[type] and "module" in self.plugins[type][plugin]: - self.core.log.debug(_("Reloading plugin: [%(type)s] %(name)s") % {'name': plugin, 'type': type}) - - try: - reload(self.plugins[type][plugin]['module']) - - except Exception, e: - self.core.log.error(_("Error when reloading plugin: [%(type)s] %(name)s") % {'name': plugin, 'type': type}, e) - continue - - else: - reloaded.append((type, plugin)) - - #index creation - self.plugins[type] = self.parse(type) - setattr(self, "%sPlugins" % type, self.plugins[type]) - - if "account" in as_dict: #: accounts needs to be reloaded - self.core.accountManager.initPlugins() - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - - return reloaded #: return a list of the plugins successfully reloaded - - - def reloadPlugin(self, type_plugin): - """ reload and reindex ONE plugin """ - return True if self.reloadPlugins(type_plugin) else False diff --git a/pyload/manager/Remote.py b/pyload/manager/Remote.py new file mode 100644 index 000000000..910881164 --- /dev/null +++ b/pyload/manager/Remote.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from threading import Thread +from traceback import print_exc + +class BackendBase(Thread): + def __init__(self, manager): + Thread.__init__(self) + self.m = manager + self.core = manager.core + self.enabled = True + self.running = False + + def run(self): + self.running = True + try: + self.serve() + except Exception, e: + self.core.log.error(_("Remote backend error: %s") % e) + if self.core.debug: + print_exc() + finally: + self.running = False + + def setup(self, host, port): + pass + + def checkDeps(self): + return True + + def serve(self): + pass + + def shutdown(self): + pass + + def stop(self): + self.enabled = False# set flag and call shutdowm message, so thread can react + self.shutdown() + + +class RemoteManager(object): + available = [] + + def __init__(self, core): + self.core = core + self.backends = [] + + if self.core.remote: + self.available.append("ThriftBackend") +# else: +# self.available.append("SocketBackend") + + + def startBackends(self): + host = self.core.config["remote"]["listenaddr"] + port = self.core.config["remote"]["port"] + + for b in self.available: + klass = getattr(__import__("pyload.remote.%s" % b, globals(), locals(), [b], -1), b) + backend = klass(self) + if not backend.checkDeps(): + continue + try: + backend.setup(host, port) + self.core.log.info(_("Starting %(name)s: %(addr)s:%(port)s") % {"name": b, "addr": host, "port": port}) + except Exception, e: + self.core.log.error(_("Failed loading backend %(name)s | %(error)s") % {"name": b, "error": str(e)}) + if self.core.debug: + print_exc() + else: + backend.start() + self.backends.append(backend) + + port += 1 diff --git a/pyload/manager/RemoteManager.py b/pyload/manager/RemoteManager.py deleted file mode 100644 index 910881164..000000000 --- a/pyload/manager/RemoteManager.py +++ /dev/null @@ -1,76 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN - -from threading import Thread -from traceback import print_exc - -class BackendBase(Thread): - def __init__(self, manager): - Thread.__init__(self) - self.m = manager - self.core = manager.core - self.enabled = True - self.running = False - - def run(self): - self.running = True - try: - self.serve() - except Exception, e: - self.core.log.error(_("Remote backend error: %s") % e) - if self.core.debug: - print_exc() - finally: - self.running = False - - def setup(self, host, port): - pass - - def checkDeps(self): - return True - - def serve(self): - pass - - def shutdown(self): - pass - - def stop(self): - self.enabled = False# set flag and call shutdowm message, so thread can react - self.shutdown() - - -class RemoteManager(object): - available = [] - - def __init__(self, core): - self.core = core - self.backends = [] - - if self.core.remote: - self.available.append("ThriftBackend") -# else: -# self.available.append("SocketBackend") - - - def startBackends(self): - host = self.core.config["remote"]["listenaddr"] - port = self.core.config["remote"]["port"] - - for b in self.available: - klass = getattr(__import__("pyload.remote.%s" % b, globals(), locals(), [b], -1), b) - backend = klass(self) - if not backend.checkDeps(): - continue - try: - backend.setup(host, port) - self.core.log.info(_("Starting %(name)s: %(addr)s:%(port)s") % {"name": b, "addr": host, "port": port}) - except Exception, e: - self.core.log.error(_("Failed loading backend %(name)s | %(error)s") % {"name": b, "error": str(e)}) - if self.core.debug: - print_exc() - else: - backend.start() - self.backends.append(backend) - - port += 1 diff --git a/pyload/manager/Thread.py b/pyload/manager/Thread.py new file mode 100644 index 000000000..6c9304e87 --- /dev/null +++ b/pyload/manager/Thread.py @@ -0,0 +1,302 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from os.path import exists, join +import re +from subprocess import Popen +from threading import Event, Lock +from time import sleep, time +from traceback import print_exc +from random import choice + +import pycurl + +from pyload.manager.thread.Decrypter import DecrypterThread +from pyload.manager.thread.Download import DownloadThread +from pyload.manager.thread.Info import InfoThread +from pyload.datatype.File import PyFile +from pyload.network.RequestFactory import getURL +from pyload.utils import freeSpace, lock + + +class ThreadManager(object): + """manages the download threads, assign jobs, reconnect etc""" + + + def __init__(self, core): + """Constructor""" + self.core = core + + self.threads = [] #: thread list + self.localThreads = [] #: addon+decrypter threads + + self.pause = True + + self.reconnecting = Event() + self.reconnecting.clear() + self.downloaded = 0 #number of files downloaded since last cleanup + + self.lock = Lock() + + # some operations require to fetch url info from hoster, so we caching them so it wont be done twice + # contains a timestamp and will be purged after timeout + self.infoCache = {} + + # pool of ids for online check + self.resultIDs = 0 + + # threads which are fetching hoster results + self.infoResults = {} + #timeout for cache purge + self.timestamp = 0 + + pycurl.global_init(pycurl.GLOBAL_DEFAULT) + + for i in range(0, self.core.config.get("download", "max_downloads")): + self.createThread() + + + def createThread(self): + """create a download thread""" + + thread = DownloadThread(self) + self.threads.append(thread) + + def createInfoThread(self, data, pid): + """ + start a thread whichs fetches online status and other infos + data = [ .. () .. ] + """ + self.timestamp = time() + 5 * 60 + + InfoThread(self, data, pid) + + @lock + def createResultThread(self, data, add=False): + """ creates a thread to fetch online status, returns result id """ + self.timestamp = time() + 5 * 60 + + rid = self.resultIDs + self.resultIDs += 1 + + InfoThread(self, data, rid=rid, add=add) + + return rid + + + @lock + def getInfoResult(self, rid): + """returns result and clears it""" + self.timestamp = time() + 5 * 60 + + if rid in self.infoResults: + data = self.infoResults[rid] + self.infoResults[rid] = {} + return data + else: + return {} + + @lock + def setInfoResults(self, rid, result): + self.infoResults[rid].update(result) + + def getActiveFiles(self): + active = [x.active for x in self.threads if x.active and isinstance(x.active, PyFile)] + + for t in self.localThreads: + active.extend(t.getActiveFiles()) + + return active + + def processingIds(self): + """get a id list of all pyfiles processed""" + return [x.id for x in self.getActiveFiles()] + + + def work(self): + """run all task which have to be done (this is for repetivive call by core)""" + try: + self.tryReconnect() + except Exception, e: + self.core.log.error(_("Reconnect Failed: %s") % str(e) ) + self.reconnecting.clear() + if self.core.debug: + print_exc() + self.checkThreadCount() + + try: + self.assignJob() + except Exception, e: + self.core.log.warning("Assign job error", e) + if self.core.debug: + print_exc() + + sleep(0.5) + self.assignJob() + #it may be failed non critical so we try it again + + if (self.infoCache or self.infoResults) and self.timestamp < time(): + self.infoCache.clear() + self.infoResults.clear() + self.core.log.debug("Cleared Result cache") + + #-------------------------------------------------------------------------- + def tryReconnect(self): + """checks if reconnect needed""" + + if not (self.core.config["reconnect"]["activated"] and self.core.api.isTimeReconnect()): + return False + + active = [x.active.plugin.wantReconnect and x.active.plugin.waiting for x in self.threads if x.active] + + if not (0 < active.count(True) == len(active)): + return False + + if not exists(self.core.config['reconnect']['method']): + if exists(join(pypath, self.core.config['reconnect']['method'])): + self.core.config['reconnect']['method'] = join(pypath, self.core.config['reconnect']['method']) + else: + self.core.config["reconnect"]["activated"] = False + self.core.log.warning(_("Reconnect script not found!")) + return + + self.reconnecting.set() + + #Do reconnect + self.core.log.info(_("Starting reconnect")) + + while [x.active.plugin.waiting for x in self.threads if x.active].count(True) != 0: + sleep(0.25) + + ip = self.getIP() + + self.core.addonManager.beforeReconnecting(ip) + + self.core.log.debug("Old IP: %s" % ip) + + try: + reconn = Popen(self.core.config['reconnect']['method'], bufsize=-1, shell=True)#, stdout=subprocess.PIPE) + except Exception: + self.core.log.warning(_("Failed executing reconnect script!")) + self.core.config["reconnect"]["activated"] = False + self.reconnecting.clear() + if self.core.debug: + print_exc() + return + + reconn.wait() + sleep(1) + ip = self.getIP() + self.core.addonManager.afterReconnecting(ip) + + self.core.log.info(_("Reconnected, new IP: %s") % ip) + + self.reconnecting.clear() + + def getIP(self): + """retrieve current ip""" + services = [("http://automation.whatismyip.com/n09230945.asp", "(\S+)"), + ("http://checkip.dyndns.org/",".*Current IP Address: (\S+).*")] + + ip = "" + for i in range(10): + try: + sv = choice(services) + ip = getURL(sv[0]) + ip = re.match(sv[1], ip).group(1) + break + except Exception: + ip = "" + sleep(1) + + return ip + + #-------------------------------------------------------------------------- + def checkThreadCount(self): + """checks if there are need for increasing or reducing thread count""" + + if len(self.threads) == self.core.config.get("download", "max_downloads"): + return True + elif len(self.threads) < self.core.config.get("download", "max_downloads"): + self.createThread() + else: + free = [x for x in self.threads if not x.active] + if free: + free[0].put("quit") + + + def cleanPycurl(self): + """ make a global curl cleanup (currently ununused) """ + if self.processingIds(): + return False + pycurl.global_cleanup() + pycurl.global_init(pycurl.GLOBAL_DEFAULT) + self.downloaded = 0 + self.core.log.debug("Cleaned up pycurl") + return True + + #-------------------------------------------------------------------------- + def assignJob(self): + """assing a job to a thread if possible""" + + if self.pause or not self.core.api.isTimeDownload(): return + + #if self.downloaded > 20: + # if not self.cleanPyCurl(): return + + free = [x for x in self.threads if not x.active] + + inuse = set([(x.active.pluginname, self.getLimit(x)) for x in self.threads if x.active and x.active.hasPlugin() and x.active.plugin.account]) + inuse = map(lambda x: (x[0], x[1], len([y for y in self.threads if y.active and y.active.pluginname == x[0]])) ,inuse) + onlimit = [x[0] for x in inuse if x[1] > 0 and x[2] >= x[1]] + + occ = [x.active.pluginname for x in self.threads if x.active and x.active.hasPlugin() and not x.active.plugin.multiDL] + onlimit + + occ.sort() + occ = tuple(set(occ)) + job = self.core.files.getJob(occ) + if job: + try: + job.initPlugin() + except Exception, e: + self.core.log.critical(str(e)) + print_exc() + job.setStatus("failed") + job.error = str(e) + job.release() + return + + if job.plugin.__type == "hoster": + spaceLeft = freeSpace(self.core.config["general"]["download_folder"]) / 1024 / 1024 + if spaceLeft < self.core.config["general"]["min_free_space"]: + self.core.log.warning(_("Not enough space left on device")) + self.pause = True + + if free and not self.pause: + thread = free[0] + #self.downloaded += 1 + + thread.put(job) + else: + #put job back + if occ not in self.core.files.jobCache: + self.core.files.jobCache[occ] = [] + self.core.files.jobCache[occ].append(job.id) + + #check for decrypt jobs + job = self.core.files.getDecryptJob() + if job: + job.initPlugin() + thread = DecrypterThread(self, job) + + + else: + thread = DecrypterThread(self, job) + + def getLimit(self, thread): + limit = thread.active.plugin.account.getAccountData(thread.active.plugin.user)["options"].get("limitDL", ["0"])[0] + return int(limit) + + def cleanup(self): + """do global cleanup, should be called when finished with pycurl""" + pycurl.global_cleanup() diff --git a/pyload/manager/ThreadManager.py b/pyload/manager/ThreadManager.py deleted file mode 100644 index d6f3c0005..000000000 --- a/pyload/manager/ThreadManager.py +++ /dev/null @@ -1,302 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN - -from os.path import exists, join -import re -from subprocess import Popen -from threading import Event, Lock -from time import sleep, time -from traceback import print_exc -from random import choice - -import pycurl - -from pyload.manager.thread.DecrypterThread import DecrypterThread -from pyload.manager.thread.DownloadThread import DownloadThread -from pyload.manager.thread.InfoThread import InfoThread -from pyload.datatype.PyFile import PyFile -from pyload.network.RequestFactory import getURL -from pyload.utils import freeSpace, lock - - -class ThreadManager(object): - """manages the download threads, assign jobs, reconnect etc""" - - - def __init__(self, core): - """Constructor""" - self.core = core - - self.threads = [] #: thread list - self.localThreads = [] #: addon+decrypter threads - - self.pause = True - - self.reconnecting = Event() - self.reconnecting.clear() - self.downloaded = 0 #number of files downloaded since last cleanup - - self.lock = Lock() - - # some operations require to fetch url info from hoster, so we caching them so it wont be done twice - # contains a timestamp and will be purged after timeout - self.infoCache = {} - - # pool of ids for online check - self.resultIDs = 0 - - # threads which are fetching hoster results - self.infoResults = {} - #timeout for cache purge - self.timestamp = 0 - - pycurl.global_init(pycurl.GLOBAL_DEFAULT) - - for i in range(0, self.core.config.get("download", "max_downloads")): - self.createThread() - - - def createThread(self): - """create a download thread""" - - thread = DownloadThread(self) - self.threads.append(thread) - - def createInfoThread(self, data, pid): - """ - start a thread whichs fetches online status and other infos - data = [ .. () .. ] - """ - self.timestamp = time() + 5 * 60 - - InfoThread(self, data, pid) - - @lock - def createResultThread(self, data, add=False): - """ creates a thread to fetch online status, returns result id """ - self.timestamp = time() + 5 * 60 - - rid = self.resultIDs - self.resultIDs += 1 - - InfoThread(self, data, rid=rid, add=add) - - return rid - - - @lock - def getInfoResult(self, rid): - """returns result and clears it""" - self.timestamp = time() + 5 * 60 - - if rid in self.infoResults: - data = self.infoResults[rid] - self.infoResults[rid] = {} - return data - else: - return {} - - @lock - def setInfoResults(self, rid, result): - self.infoResults[rid].update(result) - - def getActiveFiles(self): - active = [x.active for x in self.threads if x.active and isinstance(x.active, PyFile)] - - for t in self.localThreads: - active.extend(t.getActiveFiles()) - - return active - - def processingIds(self): - """get a id list of all pyfiles processed""" - return [x.id for x in self.getActiveFiles()] - - - def work(self): - """run all task which have to be done (this is for repetivive call by core)""" - try: - self.tryReconnect() - except Exception, e: - self.core.log.error(_("Reconnect Failed: %s") % str(e) ) - self.reconnecting.clear() - if self.core.debug: - print_exc() - self.checkThreadCount() - - try: - self.assignJob() - except Exception, e: - self.core.log.warning("Assign job error", e) - if self.core.debug: - print_exc() - - sleep(0.5) - self.assignJob() - #it may be failed non critical so we try it again - - if (self.infoCache or self.infoResults) and self.timestamp < time(): - self.infoCache.clear() - self.infoResults.clear() - self.core.log.debug("Cleared Result cache") - - #-------------------------------------------------------------------------- - def tryReconnect(self): - """checks if reconnect needed""" - - if not (self.core.config["reconnect"]["activated"] and self.core.api.isTimeReconnect()): - return False - - active = [x.active.plugin.wantReconnect and x.active.plugin.waiting for x in self.threads if x.active] - - if not (0 < active.count(True) == len(active)): - return False - - if not exists(self.core.config['reconnect']['method']): - if exists(join(pypath, self.core.config['reconnect']['method'])): - self.core.config['reconnect']['method'] = join(pypath, self.core.config['reconnect']['method']) - else: - self.core.config["reconnect"]["activated"] = False - self.core.log.warning(_("Reconnect script not found!")) - return - - self.reconnecting.set() - - #Do reconnect - self.core.log.info(_("Starting reconnect")) - - while [x.active.plugin.waiting for x in self.threads if x.active].count(True) != 0: - sleep(0.25) - - ip = self.getIP() - - self.core.addonManager.beforeReconnecting(ip) - - self.core.log.debug("Old IP: %s" % ip) - - try: - reconn = Popen(self.core.config['reconnect']['method'], bufsize=-1, shell=True)#, stdout=subprocess.PIPE) - except Exception: - self.core.log.warning(_("Failed executing reconnect script!")) - self.core.config["reconnect"]["activated"] = False - self.reconnecting.clear() - if self.core.debug: - print_exc() - return - - reconn.wait() - sleep(1) - ip = self.getIP() - self.core.addonManager.afterReconnecting(ip) - - self.core.log.info(_("Reconnected, new IP: %s") % ip) - - self.reconnecting.clear() - - def getIP(self): - """retrieve current ip""" - services = [("http://automation.whatismyip.com/n09230945.asp", "(\S+)"), - ("http://checkip.dyndns.org/",".*Current IP Address: (\S+).*")] - - ip = "" - for i in range(10): - try: - sv = choice(services) - ip = getURL(sv[0]) - ip = re.match(sv[1], ip).group(1) - break - except Exception: - ip = "" - sleep(1) - - return ip - - #-------------------------------------------------------------------------- - def checkThreadCount(self): - """checks if there are need for increasing or reducing thread count""" - - if len(self.threads) == self.core.config.get("download", "max_downloads"): - return True - elif len(self.threads) < self.core.config.get("download", "max_downloads"): - self.createThread() - else: - free = [x for x in self.threads if not x.active] - if free: - free[0].put("quit") - - - def cleanPycurl(self): - """ make a global curl cleanup (currently ununused) """ - if self.processingIds(): - return False - pycurl.global_cleanup() - pycurl.global_init(pycurl.GLOBAL_DEFAULT) - self.downloaded = 0 - self.core.log.debug("Cleaned up pycurl") - return True - - #-------------------------------------------------------------------------- - def assignJob(self): - """assing a job to a thread if possible""" - - if self.pause or not self.core.api.isTimeDownload(): return - - #if self.downloaded > 20: - # if not self.cleanPyCurl(): return - - free = [x for x in self.threads if not x.active] - - inuse = set([(x.active.pluginname, self.getLimit(x)) for x in self.threads if x.active and x.active.hasPlugin() and x.active.plugin.account]) - inuse = map(lambda x: (x[0], x[1], len([y for y in self.threads if y.active and y.active.pluginname == x[0]])) ,inuse) - onlimit = [x[0] for x in inuse if x[1] > 0 and x[2] >= x[1]] - - occ = [x.active.pluginname for x in self.threads if x.active and x.active.hasPlugin() and not x.active.plugin.multiDL] + onlimit - - occ.sort() - occ = tuple(set(occ)) - job = self.core.files.getJob(occ) - if job: - try: - job.initPlugin() - except Exception, e: - self.core.log.critical(str(e)) - print_exc() - job.setStatus("failed") - job.error = str(e) - job.release() - return - - if job.plugin.__type == "hoster": - spaceLeft = freeSpace(self.core.config["general"]["download_folder"]) / 1024 / 1024 - if spaceLeft < self.core.config["general"]["min_free_space"]: - self.core.log.warning(_("Not enough space left on device")) - self.pause = True - - if free and not self.pause: - thread = free[0] - #self.downloaded += 1 - - thread.put(job) - else: - #put job back - if occ not in self.core.files.jobCache: - self.core.files.jobCache[occ] = [] - self.core.files.jobCache[occ].append(job.id) - - #check for decrypt jobs - job = self.core.files.getDecryptJob() - if job: - job.initPlugin() - thread = DecrypterThread(self, job) - - - else: - thread = DecrypterThread(self, job) - - def getLimit(self, thread): - limit = thread.active.plugin.account.getAccountData(thread.active.plugin.user)["options"].get("limitDL", ["0"])[0] - return int(limit) - - def cleanup(self): - """do global cleanup, should be called when finished with pycurl""" - pycurl.global_cleanup() diff --git a/pyload/manager/event/PullEvents.py b/pyload/manager/event/PullEvents.py deleted file mode 100644 index 20897290e..000000000 --- a/pyload/manager/event/PullEvents.py +++ /dev/null @@ -1,104 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: mkaay - -from time import time -from pyload.utils import uniqify - -class PullManager(object): - def __init__(self, core): - self.core = core - self.clients = [] - - def newClient(self, uuid): - self.clients.append(Client(uuid)) - - def clean(self): - for n, client in enumerate(self.clients): - if client.lastActive + 30 < time(): - del self.clients[n] - - def getEvents(self, uuid): - events = [] - validUuid = False - for client in self.clients: - if client.uuid == uuid: - client.lastActive = time() - validUuid = True - while client.newEvents(): - events.append(client.popEvent().toList()) - break - if not validUuid: - self.newClient(uuid) - events = [ReloadAllEvent("queue").toList(), ReloadAllEvent("collector").toList()] - return uniqify(events) - - def addEvent(self, event): - for client in self.clients: - client.addEvent(event) - -class Client(object): - def __init__(self, uuid): - self.uuid = uuid - self.lastActive = time() - self.events = [] - - def newEvents(self): - return len(self.events) > 0 - - def popEvent(self): - if not len(self.events): - return None - return self.events.pop(0) - - def addEvent(self, event): - self.events.append(event) - -class UpdateEvent(object): - def __init__(self, itype, iid, destination): - assert itype == "pack" or itype == "file" - assert destination == "queue" or destination == "collector" - self.type = itype - self.id = iid - self.destination = destination - - def toList(self): - return ["update", self.destination, self.type, self.id] - -class RemoveEvent(object): - def __init__(self, itype, iid, destination): - assert itype == "pack" or itype == "file" - assert destination == "queue" or destination == "collector" - self.type = itype - self.id = iid - self.destination = destination - - def toList(self): - return ["remove", self.destination, self.type, self.id] - -class InsertEvent(object): - def __init__(self, itype, iid, after, destination): - assert itype == "pack" or itype == "file" - assert destination == "queue" or destination == "collector" - self.type = itype - self.id = iid - self.after = after - self.destination = destination - - def toList(self): - return ["insert", self.destination, self.type, self.id, self.after] - -class ReloadAllEvent(object): - def __init__(self, destination): - assert destination == "queue" or destination == "collector" - self.destination = destination - - def toList(self): - return ["reload", self.destination] - -class AccountUpdateEvent(object): - def toList(self): - return ["account"] - -class ConfigUpdateEvent(object): - def toList(self): - return ["config"] diff --git a/pyload/manager/thread/Addon.py b/pyload/manager/thread/Addon.py new file mode 100644 index 000000000..7feec227e --- /dev/null +++ b/pyload/manager/thread/Addon.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from Queue import Queue +from threading import Thread +from os import listdir, stat +from os.path import join +from time import sleep, time, strftime, gmtime +from traceback import print_exc, format_exc +from pprint import pformat +from sys import exc_info, exc_clear +from copy import copy +from types import MethodType + +from pycurl import error + +from pyload.manager.thread.Plugin import PluginThread + + +class AddonThread(PluginThread): + """thread for addons""" + + #-------------------------------------------------------------------------- + def __init__(self, m, function, args, kwargs): + """Constructor""" + PluginThread.__init__(self, m) + + self.f = function + self.args = args + self.kwargs = kwargs + + self.active = [] + + m.localThreads.append(self) + + self.start() + + def getActiveFiles(self): + return self.active + + def addActive(self, pyfile): + """ Adds a pyfile to active list and thus will be displayed on overview""" + if pyfile not in self.active: + self.active.append(pyfile) + + def finishFile(self, pyfile): + if pyfile in self.active: + self.active.remove(pyfile) + + pyfile.finishIfDone() + + def run(self): + try: + try: + self.kwargs["thread"] = self + self.f(*self.args, **self.kwargs) + except TypeError, e: + #dirty method to filter out exceptions + if "unexpected keyword argument 'thread'" not in e.args[0]: + raise + + del self.kwargs["thread"] + self.f(*self.args, **self.kwargs) + finally: + local = copy(self.active) + for x in local: + self.finishFile(x) + + self.m.localThreads.remove(self) diff --git a/pyload/manager/thread/AddonThread.py b/pyload/manager/thread/AddonThread.py deleted file mode 100644 index a84856b0a..000000000 --- a/pyload/manager/thread/AddonThread.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN - -from Queue import Queue -from threading import Thread -from os import listdir, stat -from os.path import join -from time import sleep, time, strftime, gmtime -from traceback import print_exc, format_exc -from pprint import pformat -from sys import exc_info, exc_clear -from copy import copy -from types import MethodType - -from pycurl import error - -from pyload.manager.thread.PluginThread import PluginThread - - -class AddonThread(PluginThread): - """thread for addons""" - - #-------------------------------------------------------------------------- - def __init__(self, m, function, args, kwargs): - """Constructor""" - PluginThread.__init__(self, m) - - self.f = function - self.args = args - self.kwargs = kwargs - - self.active = [] - - m.localThreads.append(self) - - self.start() - - def getActiveFiles(self): - return self.active - - def addActive(self, pyfile): - """ Adds a pyfile to active list and thus will be displayed on overview""" - if pyfile not in self.active: - self.active.append(pyfile) - - def finishFile(self, pyfile): - if pyfile in self.active: - self.active.remove(pyfile) - - pyfile.finishIfDone() - - def run(self): - try: - try: - self.kwargs["thread"] = self - self.f(*self.args, **self.kwargs) - except TypeError, e: - #dirty method to filter out exceptions - if "unexpected keyword argument 'thread'" not in e.args[0]: - raise - - del self.kwargs["thread"] - self.f(*self.args, **self.kwargs) - finally: - local = copy(self.active) - for x in local: - self.finishFile(x) - - self.m.localThreads.remove(self) diff --git a/pyload/manager/thread/Decrypter.py b/pyload/manager/thread/Decrypter.py new file mode 100644 index 000000000..51544d1b9 --- /dev/null +++ b/pyload/manager/thread/Decrypter.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from Queue import Queue +from threading import Thread +from os import listdir, stat +from os.path import join +from time import sleep, time, strftime, gmtime +from traceback import print_exc, format_exc +from pprint import pformat +from sys import exc_info, exc_clear +from copy import copy +from types import MethodType + +from pycurl import error + +from pyload.manager.thread.Plugin import PluginThread +from pyload.plugin.Plugin import Abort, Fail, Retry + + +class DecrypterThread(PluginThread): + """thread for decrypting""" + + def __init__(self, manager, pyfile): + """constructor""" + PluginThread.__init__(self, manager) + + self.active = pyfile + manager.localThreads.append(self) + + pyfile.setStatus("decrypting") + + self.start() + + def getActiveFiles(self): + return [self.active] + + def run(self): + """run method""" + + pyfile = self.active + retry = False + + try: + self.m.log.info(_("Decrypting starts: %s") % pyfile.name) + pyfile.error = "" + pyfile.plugin.preprocessing(self) + + except NotImplementedError: + self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) + return + + except Fail, e: + msg = e.args[0] + + if msg == "offline": + pyfile.setStatus("offline") + self.m.log.warning(_("Download is offline: %s") % pyfile.name) + else: + pyfile.setStatus("failed") + self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) + pyfile.error = msg + + if self.m.core.debug: + print_exc() + return + + except Abort: + self.m.log.info(_("Download aborted: %s") % pyfile.name) + pyfile.setStatus("aborted") + + if self.m.core.debug: + print_exc() + return + + except Retry: + self.m.log.info(_("Retrying %s") % pyfile.name) + retry = True + return self.run() + + except Exception, e: + pyfile.setStatus("failed") + self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) + pyfile.error = str(e) + + if self.m.core.debug: + print_exc() + self.writeDebugReport(pyfile) + + return + + finally: + if not retry: + pyfile.release() + self.active = False + self.m.core.files.save() + self.m.localThreads.remove(self) + exc_clear() + + if not retry: + pyfile.delete() diff --git a/pyload/manager/thread/DecrypterThread.py b/pyload/manager/thread/DecrypterThread.py deleted file mode 100644 index 003b5f082..000000000 --- a/pyload/manager/thread/DecrypterThread.py +++ /dev/null @@ -1,101 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN - -from Queue import Queue -from threading import Thread -from os import listdir, stat -from os.path import join -from time import sleep, time, strftime, gmtime -from traceback import print_exc, format_exc -from pprint import pformat -from sys import exc_info, exc_clear -from copy import copy -from types import MethodType - -from pycurl import error - -from pyload.manager.thread.PluginThread import PluginThread -from pyload.plugins.Plugin import Abort, Fail, Retry - - -class DecrypterThread(PluginThread): - """thread for decrypting""" - - def __init__(self, manager, pyfile): - """constructor""" - PluginThread.__init__(self, manager) - - self.active = pyfile - manager.localThreads.append(self) - - pyfile.setStatus("decrypting") - - self.start() - - def getActiveFiles(self): - return [self.active] - - def run(self): - """run method""" - - pyfile = self.active - retry = False - - try: - self.m.log.info(_("Decrypting starts: %s") % pyfile.name) - pyfile.error = "" - pyfile.plugin.preprocessing(self) - - except NotImplementedError: - self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) - return - - except Fail, e: - msg = e.args[0] - - if msg == "offline": - pyfile.setStatus("offline") - self.m.log.warning(_("Download is offline: %s") % pyfile.name) - else: - pyfile.setStatus("failed") - self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) - pyfile.error = msg - - if self.m.core.debug: - print_exc() - return - - except Abort: - self.m.log.info(_("Download aborted: %s") % pyfile.name) - pyfile.setStatus("aborted") - - if self.m.core.debug: - print_exc() - return - - except Retry: - self.m.log.info(_("Retrying %s") % pyfile.name) - retry = True - return self.run() - - except Exception, e: - pyfile.setStatus("failed") - self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) - pyfile.error = str(e) - - if self.m.core.debug: - print_exc() - self.writeDebugReport(pyfile) - - return - - finally: - if not retry: - pyfile.release() - self.active = False - self.m.core.files.save() - self.m.localThreads.remove(self) - exc_clear() - - if not retry: - pyfile.delete() diff --git a/pyload/manager/thread/Download.py b/pyload/manager/thread/Download.py new file mode 100644 index 000000000..c7d21a4ba --- /dev/null +++ b/pyload/manager/thread/Download.py @@ -0,0 +1,213 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from Queue import Queue +from threading import Thread +from os import listdir, stat +from os.path import join +from time import sleep, time, strftime, gmtime +from traceback import print_exc, format_exc +from pprint import pformat +from sys import exc_info, exc_clear +from copy import copy +from types import MethodType + +from pycurl import error + +from pyload.manager.thread.Plugin import PluginThread +from pyload.plugin.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload + + +class DownloadThread(PluginThread): + """thread for downloading files from 'real' hoster plugins""" + + #-------------------------------------------------------------------------- + def __init__(self, manager): + """Constructor""" + PluginThread.__init__(self, manager) + + self.queue = Queue() #: job queue + self.active = False + + self.start() + + #-------------------------------------------------------------------------- + def run(self): + """run method""" + pyfile = None + + while True: + del pyfile + self.active = self.queue.get() + pyfile = self.active + + if self.active == "quit": + self.active = False + self.m.threads.remove(self) + return True + + try: + if not pyfile.hasPlugin(): + continue + #this pyfile was deleted while queueing + + pyfile.plugin.checkForSameFiles(starting=True) + self.m.log.info(_("Download starts: %s" % pyfile.name)) + + # start download + self.m.core.addonManager.downloadPreparing(pyfile) + pyfile.error = "" + pyfile.plugin.preprocessing(self) + + self.m.log.info(_("Download finished: %s") % pyfile.name) + self.m.core.addonManager.downloadFinished(pyfile) + self.m.core.files.checkPackageFinished(pyfile) + + except NotImplementedError: + self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) + pyfile.setStatus("failed") + pyfile.error = "Plugin does not work" + self.clean(pyfile) + continue + + except Abort: + try: + self.m.log.info(_("Download aborted: %s") % pyfile.name) + except Exception: + pass + + pyfile.setStatus("aborted") + + if self.m.core.debug: + print_exc() + + self.clean(pyfile) + continue + + except Reconnect: + self.queue.put(pyfile) + #pyfile.req.clearCookies() + + while self.m.reconnecting.isSet(): + sleep(0.5) + + continue + + except Retry, e: + reason = e.args[0] + self.m.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) + self.queue.put(pyfile) + continue + + except Fail, e: + msg = e.args[0] + + if msg == "offline": + pyfile.setStatus("offline") + self.m.log.warning(_("Download is offline: %s") % pyfile.name) + elif msg == "temp. offline": + pyfile.setStatus("temp. offline") + self.m.log.warning(_("Download is temporary offline: %s") % pyfile.name) + else: + pyfile.setStatus("failed") + self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) + pyfile.error = msg + + if self.m.core.debug: + print_exc() + + self.m.core.addonManager.downloadFailed(pyfile) + self.clean(pyfile) + continue + + except error, e: + if len(e.args) == 2: + code, msg = e.args + else: + code = 0 + msg = e.args + + self.m.log.debug("pycurl exception %s: %s" % (code, msg)) + + if code in (7, 18, 28, 52, 56): + self.m.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry.")) + wait = time() + 60 + + pyfile.waitUntil = wait + pyfile.setStatus("waiting") + while time() < wait: + sleep(1) + if pyfile.abort: + break + + if pyfile.abort: + self.m.log.info(_("Download aborted: %s") % pyfile.name) + pyfile.setStatus("aborted") + + self.clean(pyfile) + else: + self.queue.put(pyfile) + + continue + + else: + pyfile.setStatus("failed") + self.m.log.error("pycurl error %s: %s" % (code, msg)) + if self.m.core.debug: + print_exc() + self.writeDebugReport(pyfile) + + self.m.core.addonManager.downloadFailed(pyfile) + + self.clean(pyfile) + continue + + except SkipDownload, e: + pyfile.setStatus("skipped") + + self.m.log.info( + _("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message}) + + self.clean(pyfile) + + self.m.core.files.checkPackageFinished(pyfile) + + self.active = False + self.m.core.files.save() + + continue + + + except Exception, e: + pyfile.setStatus("failed") + self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) + pyfile.error = str(e) + + if self.m.core.debug: + print_exc() + self.writeDebugReport(pyfile) + + self.m.core.addonManager.downloadFailed(pyfile) + self.clean(pyfile) + continue + + finally: + self.m.core.files.save() + pyfile.checkIfProcessed() + exc_clear() + + #pyfile.plugin.req.clean() + + self.active = False + pyfile.finishIfDone() + self.m.core.files.save() + + + def put(self, job): + """assing job to thread""" + self.queue.put(job) + + + def stop(self): + """stops the thread""" + self.put("quit") diff --git a/pyload/manager/thread/DownloadThread.py b/pyload/manager/thread/DownloadThread.py deleted file mode 100644 index d876000da..000000000 --- a/pyload/manager/thread/DownloadThread.py +++ /dev/null @@ -1,213 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN - -from Queue import Queue -from threading import Thread -from os import listdir, stat -from os.path import join -from time import sleep, time, strftime, gmtime -from traceback import print_exc, format_exc -from pprint import pformat -from sys import exc_info, exc_clear -from copy import copy -from types import MethodType - -from pycurl import error - -from pyload.manager.thread.PluginThread import PluginThread -from pyload.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload - - -class DownloadThread(PluginThread): - """thread for downloading files from 'real' hoster plugins""" - - #-------------------------------------------------------------------------- - def __init__(self, manager): - """Constructor""" - PluginThread.__init__(self, manager) - - self.queue = Queue() #: job queue - self.active = False - - self.start() - - #-------------------------------------------------------------------------- - def run(self): - """run method""" - pyfile = None - - while True: - del pyfile - self.active = self.queue.get() - pyfile = self.active - - if self.active == "quit": - self.active = False - self.m.threads.remove(self) - return True - - try: - if not pyfile.hasPlugin(): - continue - #this pyfile was deleted while queueing - - pyfile.plugin.checkForSameFiles(starting=True) - self.m.log.info(_("Download starts: %s" % pyfile.name)) - - # start download - self.m.core.addonManager.downloadPreparing(pyfile) - pyfile.error = "" - pyfile.plugin.preprocessing(self) - - self.m.log.info(_("Download finished: %s") % pyfile.name) - self.m.core.addonManager.downloadFinished(pyfile) - self.m.core.files.checkPackageFinished(pyfile) - - except NotImplementedError: - self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) - pyfile.setStatus("failed") - pyfile.error = "Plugin does not work" - self.clean(pyfile) - continue - - except Abort: - try: - self.m.log.info(_("Download aborted: %s") % pyfile.name) - except Exception: - pass - - pyfile.setStatus("aborted") - - if self.m.core.debug: - print_exc() - - self.clean(pyfile) - continue - - except Reconnect: - self.queue.put(pyfile) - #pyfile.req.clearCookies() - - while self.m.reconnecting.isSet(): - sleep(0.5) - - continue - - except Retry, e: - reason = e.args[0] - self.m.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) - self.queue.put(pyfile) - continue - - except Fail, e: - msg = e.args[0] - - if msg == "offline": - pyfile.setStatus("offline") - self.m.log.warning(_("Download is offline: %s") % pyfile.name) - elif msg == "temp. offline": - pyfile.setStatus("temp. offline") - self.m.log.warning(_("Download is temporary offline: %s") % pyfile.name) - else: - pyfile.setStatus("failed") - self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) - pyfile.error = msg - - if self.m.core.debug: - print_exc() - - self.m.core.addonManager.downloadFailed(pyfile) - self.clean(pyfile) - continue - - except error, e: - if len(e.args) == 2: - code, msg = e.args - else: - code = 0 - msg = e.args - - self.m.log.debug("pycurl exception %s: %s" % (code, msg)) - - if code in (7, 18, 28, 52, 56): - self.m.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry.")) - wait = time() + 60 - - pyfile.waitUntil = wait - pyfile.setStatus("waiting") - while time() < wait: - sleep(1) - if pyfile.abort: - break - - if pyfile.abort: - self.m.log.info(_("Download aborted: %s") % pyfile.name) - pyfile.setStatus("aborted") - - self.clean(pyfile) - else: - self.queue.put(pyfile) - - continue - - else: - pyfile.setStatus("failed") - self.m.log.error("pycurl error %s: %s" % (code, msg)) - if self.m.core.debug: - print_exc() - self.writeDebugReport(pyfile) - - self.m.core.addonManager.downloadFailed(pyfile) - - self.clean(pyfile) - continue - - except SkipDownload, e: - pyfile.setStatus("skipped") - - self.m.log.info( - _("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message}) - - self.clean(pyfile) - - self.m.core.files.checkPackageFinished(pyfile) - - self.active = False - self.m.core.files.save() - - continue - - - except Exception, e: - pyfile.setStatus("failed") - self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) - pyfile.error = str(e) - - if self.m.core.debug: - print_exc() - self.writeDebugReport(pyfile) - - self.m.core.addonManager.downloadFailed(pyfile) - self.clean(pyfile) - continue - - finally: - self.m.core.files.save() - pyfile.checkIfProcessed() - exc_clear() - - #pyfile.plugin.req.clean() - - self.active = False - pyfile.finishIfDone() - self.m.core.files.save() - - - def put(self, job): - """assing job to thread""" - self.queue.put(job) - - - def stop(self): - """stops the thread""" - self.put("quit") diff --git a/pyload/manager/thread/Info.py b/pyload/manager/thread/Info.py new file mode 100644 index 000000000..4526a07ed --- /dev/null +++ b/pyload/manager/thread/Info.py @@ -0,0 +1,225 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from Queue import Queue +from threading import Thread +from os import listdir, stat +from os.path import join +from time import sleep, time, strftime, gmtime +from traceback import print_exc, format_exc +from pprint import pformat +from sys import exc_info, exc_clear +from copy import copy +from types import MethodType + +from pycurl import error + +from pyload.datatype.File import PyFile +from pyload.manager.thread.Plugin import PluginThread +from pyload.api import OnlineStatus + + +class InfoThread(PluginThread): + + def __init__(self, manager, data, pid=-1, rid=-1, add=False): + """Constructor""" + PluginThread.__init__(self, manager) + + self.data = data + self.pid = pid # package id + # [ .. (name, plugin) .. ] + + self.rid = rid #result id + self.add = add #add packages instead of return result + + self.cache = [] #accumulated data + + self.start() + + def run(self): + """run method""" + + plugins = {} + container = [] + + for url, plugintype, pluginname in data: + try: + plugins[plugintype][pluginname].append(url) + except Exception: + plugins[plugintype][pluginname] = [url] + + # filter out container plugins + for name in self.m.core.pluginManager.containerPlugins: + if name in plugins: + container.extend([(name, url) for url in plugins[name]]) + + del plugins[name] + + #directly write to database + if self.pid > -1: + for plugintype, pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(plugintype, pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) + self.m.core.files.save() + + elif self.add: + for plugintype, pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(plugintype, pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateCache, True) + + else: + #generate default result + result = [(url, 0, 3, url) for url in urls] + + self.updateCache(pluginname, result) + + packs = parseNames([(name, url) for name, x, y, url in self.cache]) + + self.m.log.debug("Fetched and generated %d packages" % len(packs)) + + for k, v in packs: + self.m.core.api.addPackage(k, v) + + #empty cache + del self.cache[:] + + else: #post the results + + + for name, url in container: + #attach container content + try: + data = self.decryptContainer(name, url) + except Exception: + print_exc() + self.m.log.error("Could not decrypt container.") + data = [] + + for url, plugintype, pluginname in data: + try: + plugins[plugintype][pluginname].append(url) + except Exception: + plugins[plugintype][pluginname] = [url] + + self.m.infoResults[self.rid] = {} + + for plugintype, pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(plugintype, pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) + + #force to process cache + if self.cache: + self.updateResult(pluginname, [], True) + + else: + #generate default result + result = [(url, 0, 3, url) for url in urls] + + self.updateResult(pluginname, result, True) + + self.m.infoResults[self.rid]["ALL_INFO_FETCHED"] = {} + + self.m.timestamp = time() + 5 * 60 + + + def updateDB(self, plugin, result): + self.m.core.files.updateFileInfo(result, self.pid) + + def updateResult(self, plugin, result, force=False): + #parse package name and generate result + #accumulate results + + self.cache.extend(result) + + if len(self.cache) >= 20 or force: + #used for package generating + tmp = [(name, (url, OnlineStatus(name, plugin, "unknown", status, int(size)))) + for name, size, status, url in self.cache] + + data = parseNames(tmp) + result = {} + for k, v in data.iteritems(): + for url, status in v: + status.packagename = k + result[url] = status + + self.m.setInfoResults(self.rid, result) + + self.cache = [] + + def updateCache(self, plugin, result): + self.cache.extend(result) + + def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None): + try: + result = [] #result loaded from cache + process = [] #urls to process + for url in urls: + if url in self.m.infoCache: + result.append(self.m.infoCache[url]) + else: + process.append(url) + + if result: + self.m.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname)) + cb(pluginname, result) + + if process: + self.m.log.debug("Run Info Fetching for %s" % pluginname) + for result in plugin.getInfo(process): + #result = [ .. (name, size, status, url) .. ] + if not type(result) == list: + result = [result] + + for res in result: + self.m.infoCache[res[3]] = res #: why don't assign res dict directly? + + cb(pluginname, result) + + self.m.log.debug("Finished Info Fetching for %s" % pluginname) + except Exception, e: + self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % + {"name": pluginname, "err": str(e)}) + if self.m.core.debug: + print_exc() + + # generate default results + if err: + result = [(url, 0, 3, url) for url in urls] + cb(pluginname, result) + + + def decryptContainer(self, plugin, url): + data = [] + # only works on container plugins + + self.m.log.debug("Pre decrypting %s with %s" % (url, plugin)) + + # dummy pyfile + pyfile = PyFile(self.m.core.files, -1, url, url, 0, 0, "", plugin, -1, -1) + + pyfile.initPlugin() + + # little plugin lifecycle + try: + pyfile.plugin.setup() + pyfile.plugin.loadToDisk() + pyfile.plugin.decrypt(pyfile) + pyfile.plugin.deleteTmp() + + for pack in pyfile.plugin.packages: + pyfile.plugin.urls.extend(pack[1]) + + data = self.m.core.pluginManager.parseUrls(pyfile.plugin.urls) + + self.m.log.debug("Got %d links." % len(data)) + + except Exception, e: + self.m.log.debug("Pre decrypting error: %s" % str(e)) + finally: + pyfile.release() + + return data diff --git a/pyload/manager/thread/InfoThread.py b/pyload/manager/thread/InfoThread.py deleted file mode 100644 index 99bf97fc8..000000000 --- a/pyload/manager/thread/InfoThread.py +++ /dev/null @@ -1,225 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN - -from Queue import Queue -from threading import Thread -from os import listdir, stat -from os.path import join -from time import sleep, time, strftime, gmtime -from traceback import print_exc, format_exc -from pprint import pformat -from sys import exc_info, exc_clear -from copy import copy -from types import MethodType - -from pycurl import error - -from pyload.datatype.PyFile import PyFile -from pyload.manager.thread.PluginThread import PluginThread -from pyload.api import OnlineStatus - - -class InfoThread(PluginThread): - - def __init__(self, manager, data, pid=-1, rid=-1, add=False): - """Constructor""" - PluginThread.__init__(self, manager) - - self.data = data - self.pid = pid # package id - # [ .. (name, plugin) .. ] - - self.rid = rid #result id - self.add = add #add packages instead of return result - - self.cache = [] #accumulated data - - self.start() - - def run(self): - """run method""" - - plugins = {} - container = [] - - for url, plugintype, pluginname in data: - try: - plugins[plugintype][pluginname].append(url) - except Exception: - plugins[plugintype][pluginname] = [url] - - # filter out container plugins - for name in self.m.core.pluginManager.containerPlugins: - if name in plugins: - container.extend([(name, url) for url in plugins[name]]) - - del plugins[name] - - #directly write to database - if self.pid > -1: - for plugintype, pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(plugintype, pluginname, True) - if hasattr(plugin, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) - self.m.core.files.save() - - elif self.add: - for plugintype, pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(plugintype, pluginname, True) - if hasattr(plugin, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateCache, True) - - else: - #generate default result - result = [(url, 0, 3, url) for url in urls] - - self.updateCache(pluginname, result) - - packs = parseNames([(name, url) for name, x, y, url in self.cache]) - - self.m.log.debug("Fetched and generated %d packages" % len(packs)) - - for k, v in packs: - self.m.core.api.addPackage(k, v) - - #empty cache - del self.cache[:] - - else: #post the results - - - for name, url in container: - #attach container content - try: - data = self.decryptContainer(name, url) - except Exception: - print_exc() - self.m.log.error("Could not decrypt container.") - data = [] - - for url, plugintype, pluginname in data: - try: - plugins[plugintype][pluginname].append(url) - except Exception: - plugins[plugintype][pluginname] = [url] - - self.m.infoResults[self.rid] = {} - - for plugintype, pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(plugintype, pluginname, True) - if hasattr(plugin, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) - - #force to process cache - if self.cache: - self.updateResult(pluginname, [], True) - - else: - #generate default result - result = [(url, 0, 3, url) for url in urls] - - self.updateResult(pluginname, result, True) - - self.m.infoResults[self.rid]["ALL_INFO_FETCHED"] = {} - - self.m.timestamp = time() + 5 * 60 - - - def updateDB(self, plugin, result): - self.m.core.files.updateFileInfo(result, self.pid) - - def updateResult(self, plugin, result, force=False): - #parse package name and generate result - #accumulate results - - self.cache.extend(result) - - if len(self.cache) >= 20 or force: - #used for package generating - tmp = [(name, (url, OnlineStatus(name, plugin, "unknown", status, int(size)))) - for name, size, status, url in self.cache] - - data = parseNames(tmp) - result = {} - for k, v in data.iteritems(): - for url, status in v: - status.packagename = k - result[url] = status - - self.m.setInfoResults(self.rid, result) - - self.cache = [] - - def updateCache(self, plugin, result): - self.cache.extend(result) - - def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None): - try: - result = [] #result loaded from cache - process = [] #urls to process - for url in urls: - if url in self.m.infoCache: - result.append(self.m.infoCache[url]) - else: - process.append(url) - - if result: - self.m.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname)) - cb(pluginname, result) - - if process: - self.m.log.debug("Run Info Fetching for %s" % pluginname) - for result in plugin.getInfo(process): - #result = [ .. (name, size, status, url) .. ] - if not type(result) == list: - result = [result] - - for res in result: - self.m.infoCache[res[3]] = res #: why don't assign res dict directly? - - cb(pluginname, result) - - self.m.log.debug("Finished Info Fetching for %s" % pluginname) - except Exception, e: - self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % - {"name": pluginname, "err": str(e)}) - if self.m.core.debug: - print_exc() - - # generate default results - if err: - result = [(url, 0, 3, url) for url in urls] - cb(pluginname, result) - - - def decryptContainer(self, plugin, url): - data = [] - # only works on container plugins - - self.m.log.debug("Pre decrypting %s with %s" % (url, plugin)) - - # dummy pyfile - pyfile = PyFile(self.m.core.files, -1, url, url, 0, 0, "", plugin, -1, -1) - - pyfile.initPlugin() - - # little plugin lifecycle - try: - pyfile.plugin.setup() - pyfile.plugin.loadToDisk() - pyfile.plugin.decrypt(pyfile) - pyfile.plugin.deleteTmp() - - for pack in pyfile.plugin.packages: - pyfile.plugin.urls.extend(pack[1]) - - data = self.m.core.pluginManager.parseUrls(pyfile.plugin.urls) - - self.m.log.debug("Got %d links." % len(data)) - - except Exception, e: - self.m.log.debug("Pre decrypting error: %s" % str(e)) - finally: - pyfile.release() - - return data diff --git a/pyload/manager/thread/Plugin.py b/pyload/manager/thread/Plugin.py new file mode 100644 index 000000000..70ee747a8 --- /dev/null +++ b/pyload/manager/thread/Plugin.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from Queue import Queue +from threading import Thread +from os import listdir, stat +from os.path import join +from time import sleep, time, strftime, gmtime +from traceback import print_exc, format_exc +from pprint import pformat +from sys import exc_info, exc_clear +from copy import copy +from types import MethodType + +from pycurl import error + +from pyload.datatype.File import PyFile +from pyload.plugin.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload +from pyload.utils.packagetools import parseNames +from pyload.utils import safe_join +from pyload.api import OnlineStatus + +class PluginThread(Thread): + """abstract base class for thread types""" + + #-------------------------------------------------------------------------- + def __init__(self, manager): + """Constructor""" + Thread.__init__(self) + self.setDaemon(True) + self.m = manager #thread manager + + + def writeDebugReport(self, pyfile): + """ writes a + :return: + """ + + dump_name = "debug_%s_%s.zip" % (pyfile.pluginname, strftime("%d-%m-%Y_%H-%M-%S")) + dump = self.getDebugDump(pyfile) + + try: + import zipfile + + zip = zipfile.ZipFile(dump_name, "w") + + for f in listdir(join("tmp", pyfile.pluginname)): + try: + # avoid encoding errors + zip.write(join("tmp", pyfile.pluginname, f), safe_join(pyfile.pluginname, f)) + except Exception: + pass + + info = zipfile.ZipInfo(safe_join(pyfile.pluginname, "debug_Report.txt"), gmtime()) + info.external_attr = 0644 << 16L # change permissions + + zip.writestr(info, dump) + zip.close() + + if not stat(dump_name).st_size: + raise Exception("Empty Zipfile") + + except Exception, e: + self.m.log.debug("Error creating zip file: %s" % e) + + dump_name = dump_name.replace(".zip", ".txt") + f = open(dump_name, "wb") + f.write(dump) + f.close() + + self.m.core.log.info("Debug Report written to %s" % dump_name) + + def getDebugDump(self, pyfile): + dump = "pyLoad %s Debug Report of %s %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % ( + self.m.core.api.getServerVersion(), pyfile.pluginname, pyfile.plugin.__version, format_exc()) + + tb = exc_info()[2] + stack = [] + while tb: + stack.append(tb.tb_frame) + tb = tb.tb_next + + for frame in stack[1:]: + dump += "\nFrame %s in %s at line %s\n" % (frame.f_code.co_name, + frame.f_code.co_filename, + frame.f_lineno) + + for key, value in frame.f_locals.items(): + dump += "\t%20s = " % key + try: + dump += pformat(value) + "\n" + except Exception, e: + dump += " " + str(e) + "\n" + + del frame + + del stack #delete it just to be sure... + + dump += "\n\nPLUGIN OBJECT DUMP: \n\n" + + for name in dir(pyfile.plugin): + attr = getattr(pyfile.plugin, name) + if not name.endswith("__") and type(attr) != MethodType: + dump += "\t%20s = " % name + try: + dump += pformat(attr) + "\n" + except Exception, e: + dump += " " + str(e) + "\n" + + dump += "\nPYFILE OBJECT DUMP: \n\n" + + for name in dir(pyfile): + attr = getattr(pyfile, name) + if not name.endswith("__") and type(attr) != MethodType: + dump += "\t%20s = " % name + try: + dump += pformat(attr) + "\n" + except Exception, e: + dump += " " + str(e) + "\n" + + if pyfile.pluginname in self.m.core.config.plugin: + dump += "\n\nCONFIG: \n\n" + dump += pformat(self.m.core.config.plugin[pyfile.pluginname]) + "\n" + + return dump + + def clean(self, pyfile): + """ set thread unactive and release pyfile """ + self.active = False + pyfile.release() diff --git a/pyload/manager/thread/PluginThread.py b/pyload/manager/thread/PluginThread.py deleted file mode 100644 index 3cdae122f..000000000 --- a/pyload/manager/thread/PluginThread.py +++ /dev/null @@ -1,130 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN - -from Queue import Queue -from threading import Thread -from os import listdir, stat -from os.path import join -from time import sleep, time, strftime, gmtime -from traceback import print_exc, format_exc -from pprint import pformat -from sys import exc_info, exc_clear -from copy import copy -from types import MethodType - -from pycurl import error - -from pyload.datatype.PyFile import PyFile -from pyload.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload -from pyload.utils.packagetools import parseNames -from pyload.utils import safe_join -from pyload.api import OnlineStatus - -class PluginThread(Thread): - """abstract base class for thread types""" - - #-------------------------------------------------------------------------- - def __init__(self, manager): - """Constructor""" - Thread.__init__(self) - self.setDaemon(True) - self.m = manager #thread manager - - - def writeDebugReport(self, pyfile): - """ writes a - :return: - """ - - dump_name = "debug_%s_%s.zip" % (pyfile.pluginname, strftime("%d-%m-%Y_%H-%M-%S")) - dump = self.getDebugDump(pyfile) - - try: - import zipfile - - zip = zipfile.ZipFile(dump_name, "w") - - for f in listdir(join("tmp", pyfile.pluginname)): - try: - # avoid encoding errors - zip.write(join("tmp", pyfile.pluginname, f), safe_join(pyfile.pluginname, f)) - except Exception: - pass - - info = zipfile.ZipInfo(safe_join(pyfile.pluginname, "debug_Report.txt"), gmtime()) - info.external_attr = 0644 << 16L # change permissions - - zip.writestr(info, dump) - zip.close() - - if not stat(dump_name).st_size: - raise Exception("Empty Zipfile") - - except Exception, e: - self.m.log.debug("Error creating zip file: %s" % e) - - dump_name = dump_name.replace(".zip", ".txt") - f = open(dump_name, "wb") - f.write(dump) - f.close() - - self.m.core.log.info("Debug Report written to %s" % dump_name) - - def getDebugDump(self, pyfile): - dump = "pyLoad %s Debug Report of %s %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % ( - self.m.core.api.getServerVersion(), pyfile.pluginname, pyfile.plugin.__version, format_exc()) - - tb = exc_info()[2] - stack = [] - while tb: - stack.append(tb.tb_frame) - tb = tb.tb_next - - for frame in stack[1:]: - dump += "\nFrame %s in %s at line %s\n" % (frame.f_code.co_name, - frame.f_code.co_filename, - frame.f_lineno) - - for key, value in frame.f_locals.items(): - dump += "\t%20s = " % key - try: - dump += pformat(value) + "\n" - except Exception, e: - dump += " " + str(e) + "\n" - - del frame - - del stack #delete it just to be sure... - - dump += "\n\nPLUGIN OBJECT DUMP: \n\n" - - for name in dir(pyfile.plugin): - attr = getattr(pyfile.plugin, name) - if not name.endswith("__") and type(attr) != MethodType: - dump += "\t%20s = " % name - try: - dump += pformat(attr) + "\n" - except Exception, e: - dump += " " + str(e) + "\n" - - dump += "\nPYFILE OBJECT DUMP: \n\n" - - for name in dir(pyfile): - attr = getattr(pyfile, name) - if not name.endswith("__") and type(attr) != MethodType: - dump += "\t%20s = " % name - try: - dump += pformat(attr) + "\n" - except Exception, e: - dump += " " + str(e) + "\n" - - if pyfile.pluginname in self.m.core.config.plugin: - dump += "\n\nCONFIG: \n\n" - dump += pformat(self.m.core.config.plugin[pyfile.pluginname]) + "\n" - - return dump - - def clean(self, pyfile): - """ set thread unactive and release pyfile """ - self.active = False - pyfile.release() diff --git a/pyload/manager/thread/Server.py b/pyload/manager/thread/Server.py new file mode 100644 index 000000000..f3f174e74 --- /dev/null +++ b/pyload/manager/thread/Server.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +from os.path import exists + +import os +import threading +import logging + +core = None +setup = None +log = logging.getLogger("log") + +class WebServer(threading.Thread): + def __init__(self, pycore): + global core + threading.Thread.__init__(self) + self.core = pycore + core = pycore + self.running = True + self.server = pycore.config['webinterface']['server'] + self.https = pycore.config['webinterface']['https'] + self.cert = pycore.config["ssl"]["cert"] + self.key = pycore.config["ssl"]["key"] + self.host = pycore.config['webinterface']['host'] + self.port = pycore.config['webinterface']['port'] + + self.setDaemon(True) + + def run(self): + import pyload.webui as webinterface + global webinterface + + reset = False + + if self.https and (not exists(self.cert) or not exists(self.key)): + log.warning(_("SSL certificates not found.")) + self.https = False + + if self.server in ("lighttpd", "nginx"): + log.warning(_("Sorry, we dropped support for starting %s directly within pyLoad") % self.server) + log.warning(_("You can use the threaded server which offers good performance and ssl,")) + log.warning(_("of course you can still use your existing %s with pyLoads fastcgi server") % self.server) + log.warning(_("sample configs are located in the pyload/web/servers directory")) + reset = True + elif self.server == "fastcgi": + try: + import flup + except Exception: + log.warning(_("Can't use %(server)s, python-flup is not installed!") % { + "server": self.server}) + reset = True + + if reset or self.server == "lightweight": + if os.name != "nt": + try: + import bjoern + except Exception, e: + log.error(_("Error importing lightweight server: %s") % e) + log.warning(_("You need to download and compile bjoern, https://github.com/jonashaag/bjoern")) + log.warning(_("Copy the boern.so to the lib folder or use setup.py install")) + log.warning(_("Of course you need to be familiar with linux and know how to compile software")) + self.server = "builtin" + else: + self.core.log.info(_("Server set to threaded, due to known performance problems on windows.")) + self.core.config['webinterface']['server'] = "threaded" + self.server = "threaded" + + if self.server == "threaded": + self.start_threaded() + elif self.server == "fastcgi": + self.start_fcgi() + elif self.server == "lightweight": + self.start_lightweight() + else: + self.start_builtin() + + def start_builtin(self): + + if self.https: + log.warning(_("This server offers no SSL, please consider using threaded instead")) + + self.core.log.info(_("Starting builtin webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + webinterface.run_simple(host=self.host, port=self.port) + + def start_threaded(self): + if self.https: + self.core.log.info(_("Starting threaded SSL webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + else: + self.cert = "" + self.key = "" + self.core.log.info(_("Starting threaded webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + + webinterface.run_threaded(host=self.host, port=self.port, cert=self.cert, key=self.key) + + def start_fcgi(self): + + self.core.log.info(_("Starting fastcgi server: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + webinterface.run_fcgi(host=self.host, port=self.port) + + + def start_lightweight(self): + if self.https: + log.warning(_("This server offers no SSL, please consider using threaded instead")) + + self.core.log.info(_("Starting lightweight webserver (bjoern): %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + webinterface.run_lightweight(host=self.host, port=self.port) + + def quit(self): + self.running = False diff --git a/pyload/manager/thread/ServerThread.py b/pyload/manager/thread/ServerThread.py deleted file mode 100644 index a8b95cd56..000000000 --- a/pyload/manager/thread/ServerThread.py +++ /dev/null @@ -1,111 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import with_statement - -from os.path import exists - -import os -import threading -import logging - -core = None -setup = None -log = logging.getLogger("log") - -class WebServer(threading.Thread): - def __init__(self, pycore): - global core - threading.Thread.__init__(self) - self.core = pycore - core = pycore - self.running = True - self.server = pycore.config['webinterface']['server'] - self.https = pycore.config['webinterface']['https'] - self.cert = pycore.config["ssl"]["cert"] - self.key = pycore.config["ssl"]["key"] - self.host = pycore.config['webinterface']['host'] - self.port = pycore.config['webinterface']['port'] - - self.setDaemon(True) - - def run(self): - import pyload.webui as webinterface - global webinterface - - reset = False - - if self.https and (not exists(self.cert) or not exists(self.key)): - log.warning(_("SSL certificates not found.")) - self.https = False - - if self.server in ("lighttpd", "nginx"): - log.warning(_("Sorry, we dropped support for starting %s directly within pyLoad") % self.server) - log.warning(_("You can use the threaded server which offers good performance and ssl,")) - log.warning(_("of course you can still use your existing %s with pyLoads fastcgi server") % self.server) - log.warning(_("sample configs are located in the pyload/web/servers directory")) - reset = True - elif self.server == "fastcgi": - try: - import flup - except Exception: - log.warning(_("Can't use %(server)s, python-flup is not installed!") % { - "server": self.server}) - reset = True - - if reset or self.server == "lightweight": - if os.name != "nt": - try: - import bjoern - except Exception, e: - log.error(_("Error importing lightweight server: %s") % e) - log.warning(_("You need to download and compile bjoern, https://github.com/jonashaag/bjoern")) - log.warning(_("Copy the boern.so to pyload/lib folder or use setup.py install")) - log.warning(_("Of course you need to be familiar with linux and know how to compile software")) - self.server = "builtin" - else: - self.core.log.info(_("Server set to threaded, due to known performance problems on windows.")) - self.core.config['webinterface']['server'] = "threaded" - self.server = "threaded" - - if self.server == "threaded": - self.start_threaded() - elif self.server == "fastcgi": - self.start_fcgi() - elif self.server == "lightweight": - self.start_lightweight() - else: - self.start_builtin() - - def start_builtin(self): - - if self.https: - log.warning(_("This server offers no SSL, please consider using threaded instead")) - - self.core.log.info(_("Starting builtin webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - webinterface.run_simple(host=self.host, port=self.port) - - def start_threaded(self): - if self.https: - self.core.log.info(_("Starting threaded SSL webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - else: - self.cert = "" - self.key = "" - self.core.log.info(_("Starting threaded webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - - webinterface.run_threaded(host=self.host, port=self.port, cert=self.cert, key=self.key) - - def start_fcgi(self): - - self.core.log.info(_("Starting fastcgi server: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - webinterface.run_fcgi(host=self.host, port=self.port) - - - def start_lightweight(self): - if self.https: - log.warning(_("This server offers no SSL, please consider using threaded instead")) - - self.core.log.info(_("Starting lightweight webserver (bjoern): %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - webinterface.run_lightweight(host=self.host, port=self.port) - - def quit(self): - self.running = False -- cgit v1.2.3