diff options
Diffstat (limited to 'module/threads')
| -rw-r--r-- | module/threads/BaseThread.py | 30 | ||||
| -rw-r--r-- | module/threads/DecrypterThread.py | 65 | ||||
| -rw-r--r-- | module/threads/DownloadThread.py | 4 | ||||
| -rw-r--r-- | module/threads/InfoThread.py | 51 | ||||
| -rw-r--r-- | module/threads/ThreadManager.py | 11 |
5 files changed, 101 insertions, 60 deletions
diff --git a/module/threads/BaseThread.py b/module/threads/BaseThread.py index b5856c856..1ba3f7a9f 100644 --- a/module/threads/BaseThread.py +++ b/module/threads/BaseThread.py @@ -1,6 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import os +import sys +import locale + from threading import Thread from time import strftime, gmtime from sys import exc_info @@ -14,32 +18,33 @@ class BaseThread(Thread): """abstract base class for thread types""" def __init__(self, manager): - """Constructor""" Thread.__init__(self) self.setDaemon(True) self.m = manager #thread manager self.log = manager.core.log - - def writeDebugReport(self, pyfile): + def writeDebugReport(self, name, pyfile=None, plugin=None): """ writes a debug report to disk """ - dump_name = "debug_%s_%s.zip" % (pyfile.pluginname, strftime("%d-%m-%Y_%H-%M-%S")) - dump = self.getDebugDump(pyfile) + dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S")) + if pyfile: + dump = self.getFileDump(pyfile) + else: + dump = self.getPluginDump(plugin) try: import zipfile zip = zipfile.ZipFile(dump_name, "w") - for f in listdir(join("tmp", pyfile.pluginname)): + for f in listdir(join("tmp", name)): try: # avoid encoding errors - zip.write(join("tmp", pyfile.pluginname, f), save_join(pyfile.pluginname, f)) + zip.write(join("tmp", name, f), save_join(name, f)) except: pass - info = zipfile.ZipInfo(save_join(pyfile.pluginname, "debug_Report.txt"), gmtime()) + info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"), gmtime()) info.external_attr = 0644 << 16L # change permissions zip.writestr(info, dump) @@ -58,7 +63,7 @@ class BaseThread(Thread): self.log.info("Debug Report written to %s" % dump_name) - def getDebugDump(self, pyfile): + def getFileDump(self, pyfile): dump = "pyLoad %s Debug Report of %s %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % ( self.m.core.api.getServerVersion(), pyfile.pluginname, pyfile.plugin.__version__, format_exc()) @@ -111,6 +116,13 @@ class BaseThread(Thread): return dump + #TODO + def getPluginDump(self, plugin): + return "" + + def getSystemDump(self): + return "" + def clean(self, pyfile): """ set thread unactive and release pyfile """ self.active = False diff --git a/module/threads/DecrypterThread.py b/module/threads/DecrypterThread.py index 5ce59a65e..a1b7e4f38 100644 --- a/module/threads/DecrypterThread.py +++ b/module/threads/DecrypterThread.py @@ -1,35 +1,78 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from time import sleep +from traceback import print_exc + +from module.plugins.Base import Retry +from module.plugins.Crypter import Package + from BaseThread import BaseThread class DecrypterThread(BaseThread): """thread for decrypting""" - def __init__(self, manager, data, package): + def __init__(self, manager, data, pid): """constructor""" BaseThread.__init__(self, manager) - self.queue = data - self.package = package - - self.m.log.debug("Starting Decrypt thread") + self.data = data + self.pid = pid self.start() - def add(self, data): - self.queue.extend(data) - def run(self): plugin_map = {} - for plugin, url in self.queue: + for url, plugin in self.data: if plugin in plugin_map: plugin_map[plugin].append(url) else: plugin_map[plugin] = [url] - self.decrypt(plugin_map) def decrypt(self, plugin_map): + pack = self.m.core.files.getPackage(self.pid) + result = [] + for name, urls in plugin_map.iteritems(): - p = self.m.core.pluginManager.loadClass("crypter", name) + klass = self.m.core.pluginManager.loadClass("crypter", name) + plugin = klass(self.m.core, pack, pack.password) + plugin_result = [] + + try: + try: + plugin_result = plugin._decrypt(urls) + except Retry: + sleep(1) + plugin_result = plugin._decrypt(urls) + except Exception, e: + plugin.logError(_("Decrypting failed"), e) + if self.m.core.debug: + print_exc() + self.writeDebugReport(plugin.__name__, plugin=plugin) + + plugin.logDebug("Decrypted", plugin_result) + result.extend(plugin_result) + + pack_names = {} + urls = [] + + for p in result: + if isinstance(p, Package): + if p.name in pack_names: + pack_names[p.name].urls.extend(p.urls) + else: + pack_names[p.name] = p + else: + urls.append(p) + + if urls: + self.log.info(_("Decrypted %(count)d links into package %(name)s") % {"count": len(urls), "name": pack.name}) + self.m.core.api.addFiles(self.pid, urls) + + for p in pack_names: + self.m.core.api.addPackage(p.name, p.urls, p.dest, pack.password) + + if not result: + self.log.info(_("No links decrypted")) + diff --git a/module/threads/DownloadThread.py b/module/threads/DownloadThread.py index 3d444686b..638861338 100644 --- a/module/threads/DownloadThread.py +++ b/module/threads/DownloadThread.py @@ -156,7 +156,7 @@ class DownloadThread(BaseThread): self.m.log.error("pycurl error %s: %s" % (code, msg)) if self.m.core.debug: print_exc() - self.writeDebugReport(pyfile) + self.writeDebugReport(pyfile.pluginname, pyfile) self.m.core.hookManager.downloadFailed(pyfile) @@ -186,7 +186,7 @@ class DownloadThread(BaseThread): if self.m.core.debug: print_exc() - self.writeDebugReport(pyfile) + self.writeDebugReport(pyfile.pluginname, pyfile) self.m.core.hookManager.downloadFailed(pyfile) self.clean(pyfile) diff --git a/module/threads/InfoThread.py b/module/threads/InfoThread.py index 4cba7da38..596153c4b 100644 --- a/module/threads/InfoThread.py +++ b/module/threads/InfoThread.py @@ -7,11 +7,12 @@ from traceback import print_exc from module.Api import OnlineStatus from module.PyFile import PyFile from module.common.packagetools import parseNames +from module.utils import has_method from BaseThread import BaseThread class InfoThread(BaseThread): - def __init__(self, manager, data, pid=-1, rid=-1, add=False): + def __init__(self, manager, data, pid=-1, rid=-1): """Constructor""" BaseThread.__init__(self, manager) @@ -20,7 +21,6 @@ class InfoThread(BaseThread): # [ .. (name, plugin) .. ] self.rid = rid #result id - self.add = add #add packages instead of return result self.cache = [] #accumulated data @@ -39,8 +39,8 @@ class InfoThread(BaseThread): plugins[plugin] = [url] - # filter out container plugins - for name in self.m.core.pluginManager.getPlugins("container"): + # filter out crypter plugins + for name in self.m.core.pluginManager.getPlugins("crypter"): if name in plugins: container.extend([(name, url) for url in plugins[name]]) @@ -50,35 +50,17 @@ class InfoThread(BaseThread): if self.pid > -1: for pluginname, urls in plugins.iteritems(): plugin = self.m.core.pluginManager.getPlugin(pluginname, True) - if hasattr(plugin, "getInfo"): + klass = getattr(plugin, pluginname) + if has_method(klass, "getInfo"): + self.fetchForPlugin(pluginname, klass, urls, self.updateDB) + self.m.core.files.save() + elif has_method(plugin, "getInfo"): + self.log.debug("Deprecated .getInfo() method on module level, use classmethod instead") self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) self.m.core.files.save() - elif self.add: - for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(pluginname, True) - if hasattr(plugin, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateCache, True) - - else: - #generate default result - result = [(url, 0, 3, url) for url in urls] - - self.updateCache(pluginname, result) - - packs = parseNames([(name, url) for name, x, y, url in self.cache]) - - self.m.log.debug("Fetched and generated %d packages" % len(packs)) - - for k, v in packs: - self.m.core.api.addPackage(k, v) - - #empty cache - del self.cache[:] - else: #post the results - - + #TODO: finer crypter control for name, url in container: #attach container content try: @@ -98,13 +80,18 @@ class InfoThread(BaseThread): for pluginname, urls in plugins.iteritems(): plugin = self.m.core.pluginManager.getPlugin(pluginname, True) - if hasattr(plugin, "getInfo"): + klass = getattr(plugin, pluginname) + if has_method(klass, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) + #force to process cache + if self.cache: + self.updateResult(pluginname, [], True) + elif has_method(plugin, "getInfo"): + self.log.debug("Deprecated .getInfo() method on module level, use staticmethod instead") self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) - #force to process cache if self.cache: self.updateResult(pluginname, [], True) - else: #generate default result result = [(url, 0, 3, url) for url in urls] diff --git a/module/threads/ThreadManager.py b/module/threads/ThreadManager.py index c32286eb9..612da2536 100644 --- a/module/threads/ThreadManager.py +++ b/module/threads/ThreadManager.py @@ -71,7 +71,7 @@ class ThreadManager: pycurl.global_init(pycurl.GLOBAL_DEFAULT) - for i in range(0, self.core.config.get("download", "max_downloads")): + for i in range(self.core.config.get("download", "max_downloads")): self.createThread() @@ -84,25 +84,24 @@ class ThreadManager: def createInfoThread(self, data, pid): """ start a thread whichs fetches online status and other infos """ self.timestamp = time() + 5 * 60 - - InfoThread(self, data, pid) + if data: InfoThread(self, data, pid) @lock - def createResultThread(self, data, add=False): + def createResultThread(self, data): """ creates a thread to fetch online status, returns result id """ self.timestamp = time() + 5 * 60 rid = self.resultIDs self.resultIDs += 1 - InfoThread(self, data, rid=rid, add=add) + InfoThread(self, data, rid=rid) return rid @lock def createDecryptThread(self, data, pid): """ Start decrypting of entered data, all links in one package are accumulated to one thread.""" - DecrypterThread(self, data, pid) + if data: DecrypterThread(self, data, pid) @lock |