diff options
Diffstat (limited to 'pyload/manager')
| -rw-r--r-- | pyload/manager/Account.py | 10 | ||||
| -rw-r--r-- | pyload/manager/Addon.py | 17 | ||||
| -rw-r--r-- | pyload/manager/Captcha.py | 20 | ||||
| -rw-r--r-- | pyload/manager/Event.py | 19 | ||||
| -rw-r--r-- | pyload/manager/Plugin.py | 15 | ||||
| -rw-r--r-- | pyload/manager/Remote.py | 8 | ||||
| -rw-r--r-- | pyload/manager/Thread.py | 21 | ||||
| -rw-r--r-- | pyload/manager/event/Scheduler.py | 14 | ||||
| -rw-r--r-- | pyload/manager/thread/Addon.py | 5 | ||||
| -rw-r--r-- | pyload/manager/thread/Decrypter.py | 2 | ||||
| -rw-r--r-- | pyload/manager/thread/Download.py | 6 | ||||
| -rw-r--r-- | pyload/manager/thread/Info.py | 6 | ||||
| -rw-r--r-- | pyload/manager/thread/Plugin.py | 3 |
13 files changed, 141 insertions, 5 deletions
diff --git a/pyload/manager/Account.py b/pyload/manager/Account.py index 12fc970c2..9b7cb590a 100644 --- a/pyload/manager/Account.py +++ b/pyload/manager/Account.py @@ -16,7 +16,6 @@ ACC_VERSION = 1 class AccountManager(object): """manages all accounts""" - #---------------------------------------------------------------------- def __init__(self, core): """Constructor""" @@ -64,6 +63,7 @@ class AccountManager(object): #---------------------------------------------------------------------- + def loadAccounts(self): """loads all accounts available""" @@ -111,6 +111,7 @@ class AccountManager(object): #---------------------------------------------------------------------- + def saveAccounts(self): """save all account information""" @@ -135,6 +136,7 @@ class AccountManager(object): #---------------------------------------------------------------------- + def initAccountPlugins(self): """init names""" for name in self.core.pluginManager.getAccountPlugins(): @@ -142,6 +144,8 @@ class AccountManager(object): @lock + + def updateAccount(self, plugin , user, password=None, options={}): """add or update account""" if plugin in self.accounts: @@ -154,6 +158,8 @@ class AccountManager(object): @lock + + def removeAccount(self, plugin, user): """remove account""" @@ -165,6 +171,8 @@ class AccountManager(object): @lock + + def getAccountInfos(self, force=True, refresh=False): data = {} diff --git a/pyload/manager/Addon.py b/pyload/manager/Addon.py index 9ce3a4f8f..61585e1cb 100644 --- a/pyload/manager/Addon.py +++ b/pyload/manager/Addon.py @@ -67,6 +67,7 @@ class AddonManager(object): def try_catch(func): + def new(*args): try: return func(*args) @@ -178,6 +179,8 @@ class AddonManager(object): @try_catch + + def coreReady(self): for plugin in self.plugins: if plugin.isActivated(): @@ -187,6 +190,8 @@ class AddonManager(object): @try_catch + + def coreExiting(self): for plugin in self.plugins: if plugin.isActivated(): @@ -196,6 +201,8 @@ class AddonManager(object): @lock + + def downloadPreparing(self, pyfile): for plugin in self.plugins: if plugin.isActivated(): @@ -205,6 +212,8 @@ class AddonManager(object): @lock + + def downloadFinished(self, pyfile): for plugin in self.plugins: if plugin.isActivated(): @@ -215,6 +224,8 @@ class AddonManager(object): @lock @try_catch + + def downloadFailed(self, pyfile): for plugin in self.plugins: if plugin.isActivated(): @@ -224,6 +235,8 @@ class AddonManager(object): @lock + + def packageFinished(self, package): for plugin in self.plugins: if plugin.isActivated(): @@ -233,6 +246,8 @@ class AddonManager(object): @lock + + def beforeReconnecting(self, ip): for plugin in self.plugins: plugin.beforeReconnecting(ip) @@ -241,6 +256,8 @@ class AddonManager(object): @lock + + def afterReconnecting(self, ip): for plugin in self.plugins: if plugin.isActivated(): diff --git a/pyload/manager/Captcha.py b/pyload/manager/Captcha.py index e54eacf30..0814cf78a 100644 --- a/pyload/manager/Captcha.py +++ b/pyload/manager/Captcha.py @@ -9,23 +9,27 @@ from pyload.utils import encode class CaptchaManager(object): + def __init__(self, core): self.lock = Lock() self.core = core self.tasks = [] # task store, for outgoing tasks only self.ids = 0 # only for internal purpose + def newTask(self, img, format, file, result_type): task = CaptchaTask(self.ids, img, format, file, result_type) self.ids += 1 return task + def removeTask(self, task): self.lock.acquire() if task in self.tasks: self.tasks.remove(task) self.lock.release() + def getTask(self): self.lock.acquire() for task in self.tasks: @@ -35,6 +39,7 @@ class CaptchaManager(object): self.lock.release() return None + def getTaskByID(self, tid): self.lock.acquire() for task in self.tasks: @@ -44,6 +49,7 @@ class CaptchaManager(object): self.lock.release() return None + def handleCaptcha(self, task, timeout=50): cli = self.core.isClientConnected() @@ -65,6 +71,7 @@ class CaptchaManager(object): class CaptchaTask(object): + def __init__(self, id, img, format, file, result_type='textual'): self.id = str(id) self.captchaImg = img @@ -78,9 +85,11 @@ class CaptchaTask(object): self.status = "init" self.data = {} # handler can store data here + def getCaptcha(self): return self.captchaImg, self.captchaFormat, self.captchaResultType + def setResult(self, text): if self.isTextual(): self.result = text @@ -91,48 +100,59 @@ class CaptchaTask(object): except Exception: self.result = None + def getResult(self): return encode(self.result) + def getStatus(self): return self.status + def setWaiting(self, sec): """ let the captcha wait secs for the solution """ self.waitUntil = max(time() + sec, self.waitUntil) self.status = "waiting" + def isWaiting(self): if self.result or self.error or self.timedOut(): return False else: return True + def isTextual(self): """ returns if text is written on the captcha """ return self.captchaResultType == 'textual' + def isPositional(self): """ returns if user have to click a specific region on the captcha """ return self.captchaResultType == 'positional' + def setWatingForUser(self, exclusive): if exclusive: self.status = "user" else: self.status = "shared-user" + def timedOut(self): return time() > self.waitUntil + def invalid(self): """ indicates the captcha was not correct """ for x in self.handler: x.captchaInvalid(self) + def correct(self): for x in self.handler: x.captchaCorrect(self) + def __str__(self): return "<CaptchaTask '%s'>" % self.id diff --git a/pyload/manager/Event.py b/pyload/manager/Event.py index 20897290e..90aaaca30 100644 --- a/pyload/manager/Event.py +++ b/pyload/manager/Event.py @@ -5,18 +5,22 @@ from time import time from pyload.utils import uniqify class PullManager(object): + def __init__(self, core): self.core = core self.clients = [] + def newClient(self, uuid): self.clients.append(Client(uuid)) + def clean(self): for n, client in enumerate(self.clients): if client.lastActive + 30 < time(): del self.clients[n] + def getEvents(self, uuid): events = [] validUuid = False @@ -32,28 +36,34 @@ class PullManager(object): events = [ReloadAllEvent("queue").toList(), ReloadAllEvent("collector").toList()] return uniqify(events) + def addEvent(self, event): for client in self.clients: client.addEvent(event) class Client(object): + def __init__(self, uuid): self.uuid = uuid self.lastActive = time() self.events = [] + def newEvents(self): return len(self.events) > 0 + def popEvent(self): if not len(self.events): return None return self.events.pop(0) + def addEvent(self, event): self.events.append(event) class UpdateEvent(object): + def __init__(self, itype, iid, destination): assert itype == "pack" or itype == "file" assert destination == "queue" or destination == "collector" @@ -61,10 +71,12 @@ class UpdateEvent(object): self.id = iid self.destination = destination + def toList(self): return ["update", self.destination, self.type, self.id] class RemoveEvent(object): + def __init__(self, itype, iid, destination): assert itype == "pack" or itype == "file" assert destination == "queue" or destination == "collector" @@ -72,10 +84,12 @@ class RemoveEvent(object): self.id = iid self.destination = destination + def toList(self): return ["remove", self.destination, self.type, self.id] class InsertEvent(object): + def __init__(self, itype, iid, after, destination): assert itype == "pack" or itype == "file" assert destination == "queue" or destination == "collector" @@ -84,21 +98,26 @@ class InsertEvent(object): self.after = after self.destination = destination + def toList(self): return ["insert", self.destination, self.type, self.id, self.after] class ReloadAllEvent(object): + def __init__(self, destination): assert destination == "queue" or destination == "collector" self.destination = destination + def toList(self): return ["reload", self.destination] class AccountUpdateEvent(object): + def toList(self): return ["account"] class ConfigUpdateEvent(object): + def toList(self): return ["config"] diff --git a/pyload/manager/Plugin.py b/pyload/manager/Plugin.py index 918f6de8a..10abbc2ea 100644 --- a/pyload/manager/Plugin.py +++ b/pyload/manager/Plugin.py @@ -25,6 +25,7 @@ class PluginManager(object): CONFIG = re.compile(r'__config\s*=\s*\[([^\]]+)', re.M) DESC = re.compile(r'__description\s*=\s*("|"""|\')([^"\']+)') + def __init__(self, core): self.core = core @@ -34,6 +35,7 @@ class PluginManager(object): # register for import addon sys.meta_path.append(self) + def loadTypes(self): rootdir = join(pypath, "pyload", "plugin") userdir = "userplugins" @@ -46,6 +48,7 @@ class PluginManager(object): self.TYPES = list(set(self.TYPES) | types) + def createIndex(self): """create information for all plugins available""" @@ -64,6 +67,7 @@ class PluginManager(object): self.core.log.debug("Created index of plugins") + def parse(self, folder, rootplugins={}): """ returns dict with information @@ -179,6 +183,7 @@ class PluginManager(object): return plugins + def parseUrls(self, urls): """parse plugins for given list of urls""" @@ -217,6 +222,7 @@ class PluginManager(object): print res return res + def findPlugin(self, type, name): if type not in self.plugins: return None @@ -229,6 +235,7 @@ class PluginManager(object): else: return self.plugins[type][name] + def getPlugin(self, type, name, original=False): """return plugin module from hoster|decrypter|container""" plugin = self.findPlugin(type, name) @@ -241,6 +248,7 @@ class PluginManager(object): else: return self.loadModule(type, name) + def getPluginName(self, type, name): """ used to obtain new name if other plugin was injected""" plugin = self.findPlugin(type, name) @@ -253,6 +261,7 @@ class PluginManager(object): return name + def loadModule(self, type, name): """ Returns loaded module for plugin @@ -282,6 +291,7 @@ class PluginManager(object): % {'name': name, 'type': type, 'version': plugins[name]['version']}) return module + def loadClass(self, type, name): """Returns the class of a plugin with the same name""" module = self.loadModule(type, name) @@ -290,10 +300,12 @@ class PluginManager(object): else: return None + def getAccountPlugins(self): """return list of account plugin names""" return self.accountPlugins.keys() + def find_module(self, fullname, path=None): # redirecting imports if necesarry if fullname.startswith(self.ROOT) or fullname.startswith(self.USERROOT): # seperate pyload plugins @@ -312,6 +324,7 @@ class PluginManager(object): if user and not self.plugins[type][name]['user']: return self + def load_module(self, name, replace=True): if name not in sys.modules: # could be already in modules if replace: @@ -333,6 +346,7 @@ class PluginManager(object): return sys.modules[name] + def reloadPlugins(self, type_plugins): """ reload and reindex plugins """ if not type_plugins: @@ -378,6 +392,7 @@ class PluginManager(object): return reloaded #: return a list of the plugins successfully reloaded + def reloadPlugin(self, type_plugin): """ reload and reindex ONE plugin """ return True if self.reloadPlugins(type_plugin) else False diff --git a/pyload/manager/Remote.py b/pyload/manager/Remote.py index 910881164..4fdb36fc2 100644 --- a/pyload/manager/Remote.py +++ b/pyload/manager/Remote.py @@ -5,6 +5,7 @@ from threading import Thread from traceback import print_exc class BackendBase(Thread): + def __init__(self, manager): Thread.__init__(self) self.m = manager @@ -12,6 +13,7 @@ class BackendBase(Thread): self.enabled = True self.running = False + def run(self): self.running = True try: @@ -23,18 +25,23 @@ class BackendBase(Thread): finally: self.running = False + def setup(self, host, port): pass + def checkDeps(self): return True + def serve(self): pass + def shutdown(self): pass + def stop(self): self.enabled = False# set flag and call shutdowm message, so thread can react self.shutdown() @@ -43,6 +50,7 @@ class BackendBase(Thread): class RemoteManager(object): available = [] + def __init__(self, core): self.core = core self.backends = [] diff --git a/pyload/manager/Thread.py b/pyload/manager/Thread.py index 753a8c251..dd8e6bcce 100644 --- a/pyload/manager/Thread.py +++ b/pyload/manager/Thread.py @@ -54,12 +54,14 @@ class ThreadManager(object): for i in range(0, self.core.config.get("download", "max_downloads")): self.createThread() + def createThread(self): """create a download thread""" thread = DownloadThread(self) self.threads.append(thread) + def createInfoThread(self, data, pid): """ start a thread whichs fetches online status and other infos @@ -70,6 +72,8 @@ class ThreadManager(object): InfoThread(self, data, pid) @lock + + def createResultThread(self, data, add=False): """ creates a thread to fetch online status, returns result id """ self.timestamp = time() + 5 * 60 @@ -82,6 +86,8 @@ class ThreadManager(object): return rid @lock + + def getInfoResult(self, rid): """returns result and clears it""" self.timestamp = time() + 5 * 60 @@ -94,9 +100,12 @@ class ThreadManager(object): return {} @lock + + def setInfoResults(self, rid, result): self.infoResults[rid].update(result) + def getActiveFiles(self): active = [x.active for x in self.threads if x.active and isinstance(x.active, PyFile)] @@ -105,10 +114,12 @@ class ThreadManager(object): return active + def processingIds(self): """get a id list of all pyfiles processed""" return [x.id for x in self.getActiveFiles()] + def work(self): """run all task which have to be done (this is for repetivive call by core)""" try: @@ -136,7 +147,9 @@ class ThreadManager(object): self.infoResults.clear() self.core.log.debug("Cleared Result cache") + #-------------------------------------------------------------------------- + def tryReconnect(self): """checks if reconnect needed""" @@ -189,6 +202,7 @@ class ThreadManager(object): self.reconnecting.clear() + def getIP(self): """retrieve current ip""" services = [("http://automation.whatismyip.com/n09230945.asp", "(\S+)"), @@ -207,7 +221,9 @@ class ThreadManager(object): return ip + #-------------------------------------------------------------------------- + def checkThreadCount(self): """checks if there are need for increasing or reducing thread count""" @@ -220,6 +236,7 @@ class ThreadManager(object): if free: free[0].put("quit") + def cleanPycurl(self): """ make a global curl cleanup (currently ununused) """ if self.processingIds(): @@ -230,7 +247,9 @@ class ThreadManager(object): self.core.log.debug("Cleaned up pycurl") return True + #-------------------------------------------------------------------------- + def assignJob(self): """assing a job to a thread if possible""" @@ -287,10 +306,12 @@ class ThreadManager(object): else: thread = DecrypterThread(self, job) + def getLimit(self, thread): limit = thread.active.plugin.account.getAccountData(thread.active.plugin.user)["options"].get("limitDL", ["0"])[0] return int(limit) + def cleanup(self): """do global cleanup, should be called when finished with pycurl""" pycurl.global_cleanup() diff --git a/pyload/manager/event/Scheduler.py b/pyload/manager/event/Scheduler.py index 2cb537383..d67d9063a 100644 --- a/pyload/manager/event/Scheduler.py +++ b/pyload/manager/event/Scheduler.py @@ -11,13 +11,16 @@ class AlreadyCalled(Exception): class Deferred(object): + def __init__(self): self.call = [] self.result = () + def addCallback(self, f, *cargs, **ckwargs): self.call.append((f, cargs, ckwargs)) + def callback(self, *args, **kwargs): if self.result: raise AlreadyCalled @@ -29,11 +32,13 @@ class Deferred(object): class Scheduler(object): + def __init__(self, core): self.core = core self.queue = PriorityQueue() + def addJob(self, t, call, args=[], kwargs={}, threaded=True): d = Deferred() t += time() @@ -41,6 +46,7 @@ class Scheduler(object): self.queue.put((t, j)) return d + def removeJob(self, d): """ :param d: defered object @@ -58,6 +64,7 @@ class Scheduler(object): return False + def work(self): while True: t, j = self.queue.get() @@ -72,6 +79,7 @@ class Scheduler(object): class Job(object): + def __init__(self, time, call, args=[], kwargs={}, deferred=None, threaded=True): self.time = float(time) self.call = call @@ -80,6 +88,7 @@ class Job(object): self.deferred = deferred self.threaded = threaded + def run(self): ret = self.call(*self.args, **self.kwargs) if self.deferred is None: @@ -87,6 +96,7 @@ class Job(object): else: self.deferred.callback(ret) + def start(self): if self.threaded: t = Thread(target=self.run) @@ -103,17 +113,21 @@ class PriorityQueue(object): self.queue = [] self.lock = Lock() + def __iter__(self): return iter(self.queue) + def __delitem__(self, key): del self.queue[key] + def put(self, element): self.lock.acquire() heappush(self.queue, element) self.lock.release() + def get(self): """ return element or None """ self.lock.acquire() diff --git a/pyload/manager/thread/Addon.py b/pyload/manager/thread/Addon.py index 7feec227e..f3d219989 100644 --- a/pyload/manager/thread/Addon.py +++ b/pyload/manager/thread/Addon.py @@ -20,7 +20,6 @@ from pyload.manager.thread.Plugin import PluginThread class AddonThread(PluginThread): """thread for addons""" - #-------------------------------------------------------------------------- def __init__(self, m, function, args, kwargs): """Constructor""" PluginThread.__init__(self, m) @@ -35,20 +34,24 @@ class AddonThread(PluginThread): self.start() + def getActiveFiles(self): return self.active + def addActive(self, pyfile): """ Adds a pyfile to active list and thus will be displayed on overview""" if pyfile not in self.active: self.active.append(pyfile) + def finishFile(self, pyfile): if pyfile in self.active: self.active.remove(pyfile) pyfile.finishIfDone() + def run(self): try: try: diff --git a/pyload/manager/thread/Decrypter.py b/pyload/manager/thread/Decrypter.py index 12806163c..308e94f10 100644 --- a/pyload/manager/thread/Decrypter.py +++ b/pyload/manager/thread/Decrypter.py @@ -33,9 +33,11 @@ class DecrypterThread(PluginThread): self.start() + def getActiveFiles(self): return [self.active] + def run(self): """run method""" diff --git a/pyload/manager/thread/Download.py b/pyload/manager/thread/Download.py index 37fe844ec..fc76b655e 100644 --- a/pyload/manager/thread/Download.py +++ b/pyload/manager/thread/Download.py @@ -21,8 +21,6 @@ from pyload.plugin.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload class DownloadThread(PluginThread): """thread for downloading files from 'real' hoster plugins""" - #-------------------------------------------------------------------------- - def __init__(self, manager): """Constructor""" PluginThread.__init__(self, manager) @@ -32,7 +30,9 @@ class DownloadThread(PluginThread): self.start() + #-------------------------------------------------------------------------- + def run(self): """run method""" pyfile = None @@ -201,10 +201,12 @@ class DownloadThread(PluginThread): pyfile.finishIfDone() self.m.core.files.save() + def put(self, job): """assing job to thread""" self.queue.put(job) + def stop(self): """stops the thread""" self.put("quit") diff --git a/pyload/manager/thread/Info.py b/pyload/manager/thread/Info.py index edc9489e9..487c3b924 100644 --- a/pyload/manager/thread/Info.py +++ b/pyload/manager/thread/Info.py @@ -36,6 +36,7 @@ class InfoThread(PluginThread): self.start() + def run(self): """run method""" @@ -120,9 +121,11 @@ class InfoThread(PluginThread): self.m.timestamp = time() + 5 * 60 + def updateDB(self, plugin, result): self.m.core.files.updateFileInfo(result, self.pid) + def updateResult(self, plugin, result, force=False): # parse package name and generate result # accumulate results @@ -144,9 +147,11 @@ class InfoThread(PluginThread): self.cache = [] + def updateCache(self, plugin, result): self.cache.extend(result) + def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None): try: result = [] # result loaded from cache @@ -184,6 +189,7 @@ class InfoThread(PluginThread): result = [(url, 0, 3, url) for url in urls] cb(pluginname, result) + def decryptContainer(self, plugin, url): data = [] # only works on container plugins diff --git a/pyload/manager/thread/Plugin.py b/pyload/manager/thread/Plugin.py index 1e7d7b4e4..155e687d4 100644 --- a/pyload/manager/thread/Plugin.py +++ b/pyload/manager/thread/Plugin.py @@ -23,7 +23,6 @@ from pyload.api import OnlineStatus class PluginThread(Thread): """abstract base class for thread types""" - #-------------------------------------------------------------------------- def __init__(self, manager): """Constructor""" Thread.__init__(self) @@ -70,6 +69,7 @@ class PluginThread(Thread): self.m.core.log.info("Debug Report written to %s" % dump_name) + def getDebugDump(self, pyfile): dump = "pyLoad %s Debug Report of %s %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % ( self.m.core.api.getServerVersion(), pyfile.pluginname, pyfile.plugin.__version, format_exc()) @@ -124,6 +124,7 @@ class PluginThread(Thread): return dump + def clean(self, pyfile): """ set thread unactive and release pyfile """ self.active = False |