diff options
Diffstat (limited to 'pyload')
| -rw-r--r-- | pyload/Core.py | 4 | ||||
| -rw-r--r-- | pyload/api/DownloadPreparingApi.py | 18 | ||||
| -rw-r--r-- | pyload/datatypes/OnlineCheck.py | 33 | ||||
| -rw-r--r-- | pyload/datatypes/PyFile.py | 6 | ||||
| -rw-r--r-- | pyload/plugins/Base.py | 30 | ||||
| -rw-r--r-- | pyload/plugins/Crypter.py | 77 | ||||
| -rw-r--r-- | pyload/plugins/Hoster.py | 6 | ||||
| -rw-r--r-- | pyload/plugins/network/CurlRequest.py | 1 | ||||
| -rw-r--r-- | pyload/remote/apitypes.py | 7 | ||||
| -rw-r--r-- | pyload/remote/apitypes_debug.py | 2 | ||||
| -rw-r--r-- | pyload/remote/pyload.thrift | 7 | ||||
| -rw-r--r-- | pyload/remote/wsbackend/AsyncHandler.py | 2 | ||||
| -rw-r--r-- | pyload/threads/BaseThread.py | 41 | ||||
| -rw-r--r-- | pyload/threads/DecrypterThread.py | 61 | ||||
| -rw-r--r-- | pyload/threads/InfoThread.py | 199 | ||||
| -rw-r--r-- | pyload/threads/ThreadManager.py | 46 | ||||
| -rw-r--r-- | pyload/utils/__init__.py | 2 | ||||
| -rw-r--r-- | pyload/web/app/scripts/helpers/linkStatus.js | 2 | ||||
| -rw-r--r-- | pyload/web/app/scripts/models/LinkStatus.js | 5 | ||||
| -rw-r--r-- | pyload/web/app/styles/default/linkgrabber.less | 5 | ||||
| -rw-r--r-- | pyload/web/app/templates/default/linkgrabber/package.html | 4 |
21 files changed, 286 insertions, 272 deletions
diff --git a/pyload/Core.py b/pyload/Core.py index 2d591a2bb..f97cfcf3b 100644 --- a/pyload/Core.py +++ b/pyload/Core.py @@ -599,8 +599,8 @@ class Core(object): if not self.pdb: self.pdb = Pdb() self.pdb.set_trace() - def print_exc(self): - if self.debug: + def print_exc(self, force=False): + if self.debug or force: print_exc() def path(self, *args): diff --git a/pyload/api/DownloadPreparingApi.py b/pyload/api/DownloadPreparingApi.py index 68c83e97d..131f73b1d 100644 --- a/pyload/api/DownloadPreparingApi.py +++ b/pyload/api/DownloadPreparingApi.py @@ -37,16 +37,16 @@ class DownloadPreparingApi(ApiComponent): def checkLinks(self, links): """ initiates online status check, will also decrypt files. - :param urls: + :param links: :return: initial set of data as :class:`OnlineCheck` instance containing the result id """ hoster, crypter = self.core.pluginManager.parseUrls(links) #: TODO: withhold crypter, derypt or add later # initial result does not contain the crypter links - tmp = [(url, LinkStatus(url, url, pluginname, -1, DS.Queued)) for url, pluginname in hoster + crypter] + tmp = [(url, LinkStatus(url, url, -1, DS.Queued, pluginname)) for url, pluginname in hoster + crypter] data = parseNames(tmp) - rid = self.core.threadManager.createResultThread(data) + rid = self.core.threadManager.createResultThread(self.primaryUID, data) return OnlineCheck(rid, data) @@ -54,8 +54,7 @@ class DownloadPreparingApi(ApiComponent): def checkContainer(self, filename, data): """ checks online status of urls and a submitted container file - :param urls: list of urls - :param container: container file name + :param filename: name of the file :param data: file content :return: :class:`OnlineCheck` """ @@ -88,13 +87,8 @@ class DownloadPreparingApi(ApiComponent): :return: `OnlineCheck`, if rid is -1 then there is no more data available """ result = self.core.threadManager.getInfoResult(rid) - - if "ALL_INFO_FETCHED" in result: - del result["ALL_INFO_FETCHED"] - return OnlineCheck(-1, result) - else: - return OnlineCheck(rid, result) - + if result and result.owner == self.primaryUID: + return result.toApiData() @RequirePerm(Permission.Add) def generatePackages(self, links): diff --git a/pyload/datatypes/OnlineCheck.py b/pyload/datatypes/OnlineCheck.py index 4b31e848b..2797828bf 100644 --- a/pyload/datatypes/OnlineCheck.py +++ b/pyload/datatypes/OnlineCheck.py @@ -1 +1,32 @@ -__author__ = 'christian' +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from time import time + +from pyload.Api import OnlineCheck as OC + +class OnlineCheck: + """ Helper class that holds result of an initiated online check """ + + def __init__(self, rid, owner): + self.rid = rid + self.owner = owner + self.result = {} + self.done = False + + self.timestamp = time() + + def update(self, result): + self.timestamp = time() + self.result.update(result) + + def toApiData(self): + self.timestamp = time() + oc = OC(self.rid, self.result) + # getting the results clears the older ones + self.result = {} + # indication for no more data + if self.done: + oc.rid = -1 + + return oc
\ No newline at end of file diff --git a/pyload/datatypes/PyFile.py b/pyload/datatypes/PyFile.py index 2cc28fa24..de7288d22 100644 --- a/pyload/datatypes/PyFile.py +++ b/pyload/datatypes/PyFile.py @@ -49,7 +49,7 @@ class PyFile(object): Represents a file object at runtime """ __slots__ = ("m", "fid", "_name", "_size", "filestatus", "media", "added", "fileorder", - "url", "pluginname", "hash", "status", "error", "packageid", "ownerid", + "url", "pluginname", "hash", "status", "error", "packageid", "owner", "lock", "plugin", "waitUntil", "abort", "statusname", "reconnected", "pluginclass") @@ -83,7 +83,7 @@ class PyFile(object): self.hash = hash self.status = status self.error = error - self.ownerid = owner + self.owner = owner self.packageid = package # database information ends here @@ -183,7 +183,7 @@ class PyFile(object): def toInfoData(self): - return FileInfo(self.fid, self.getName(), self.packageid, self.ownerid, self.getSize(), self.filestatus, + return FileInfo(self.fid, self.getName(), self.packageid, self.owner, self.getSize(), self.filestatus, self.media, self.added, self.fileorder, DownloadInfo( self.url, self.pluginname, self.hash, self.status, self.getStatusName(), self.error ) diff --git a/pyload/plugins/Base.py b/pyload/plugins/Base.py index 3ca8abdd0..abb59a7bc 100644 --- a/pyload/plugins/Base.py +++ b/pyload/plugins/Base.py @@ -1,21 +1,19 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" +############################################################################### +# Copyright(c) 2008-2013 pyLoad Team +# http://www.pyload.org +# +# This file is part of pyLoad. +# pyLoad is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### import sys from time import time, sleep diff --git a/pyload/plugins/Crypter.py b/pyload/plugins/Crypter.py index cce43fb56..2175b5f94 100644 --- a/pyload/plugins/Crypter.py +++ b/pyload/plugins/Crypter.py @@ -1,50 +1,60 @@ # -*- coding: utf-8 -*- -from traceback import print_exc - -from pyload.Api import LinkStatus +from pyload.Api import LinkStatus, DownloadStatus as DS from pyload.utils import to_list, has_method, uniqify from pyload.utils.fs import exists, remove, fs_encode from pyload.utils.packagetools import parseNames -from Base import Base, Retry +from Base import Base, Fail, Retry, Abort class Package: """ Container that indicates that a new package should be created """ - def __init__(self, name=None, urls=None): + def __init__(self, name=None, links=None): self.name = name - self.urls = urls if urls else [] + self.links = [] + + if links: + self.addLinks(links) # nested packages self.packs = [] - def addURL(self, url): - self.urls.append(url) + def addLinks(self, links): + """ Add one or multiple links to the package + + :param links: One or list of urls or :class:`LinkStatus` + """ + links = to_list(links) + for link in links: + if not isinstance(link, LinkStatus): + link = LinkStatus(link, link, -1, DS.Queued) - def addLink(self, url, name, status, size): - # TODO: allow to add urls with known information - pass + self.links.append(link) def addPackage(self, pack): self.packs.append(pack) + def getURLs(self): + return [link.url for link in self.links] + def getAllURLs(self): - urls = self.urls + urls = self.getURLs() for p in self.packs: urls.extend(p.getAllURLs()) return urls # same name and urls is enough to be equal for packages def __eq__(self, other): - return self.name == other.name and self.urls == other.urls + return self.name == other.name and self.links == other.links def __repr__(self): - return u"<CrypterPackage name=%s, links=%s, packs=%s" % (self.name, self.urls, self.packs) + return u"<CrypterPackage name=%s, links=[%s], packs=%s" % (self.name, ",".join(str(l) for l in self.links), + self.packs) def __hash__(self): - return hash(self.name) ^ hash(frozenset(self.urls)) ^ hash(self.name) + return hash(self.name) ^ hash(frozenset(self.links)) ^ hash(self.name) class PyFileMockup: @@ -53,7 +63,7 @@ class PyFileMockup: def __init__(self, url, pack): self.url = url self.name = url - self._package = pack + self._package = None self.packageid = pack.id if pack else -1 def package(self): @@ -91,18 +101,21 @@ class Crypter(Base): QUEUE_DECRYPT = False @classmethod - def decrypt(cls, core, url_or_urls): + def decrypt(cls, core, url_or_urls, password=None): """Static method to decrypt urls or content. Can be used by other plugins. To decrypt file content prefix the string with ``CONTENT_PREFIX `` as seen above. :param core: pyLoad `Core`, needed in decrypt context :param url_or_urls: List of urls or single url/ file content + :param password: optional password used for decrypting + + :raises Exception: No decryption errors are cascaded :return: List of decrypted urls, all package info removed """ urls = to_list(url_or_urls) - p = cls(core) + p = cls(core, password) try: - result = p.processDecrypt(urls) + result = p._decrypt(urls) finally: p.clean() @@ -113,12 +126,12 @@ class Crypter(Base): ret.extend(url_or_pack.getAllURLs()) else: # single url ret.append(url_or_pack) - # eliminate duplicates + # eliminate duplicates return uniqify(ret) # TODO: pass user to crypter # TODO: crypter could not only know url, but also the name and size - def __init__(self, core, package=None, password=None): + def __init__(self, core, password=None): Base.__init__(self, core) self.req = None @@ -131,8 +144,6 @@ class Crypter(Base): if self.req is None: self.req = core.requestFactory.getRequest() - # Package the plugin was initialized for, don't use this, its not guaranteed to be set - self.package = package #: Password supplied by user self.password = password @@ -186,7 +197,7 @@ class Crypter(Base): return [Package(name, purls) for name, purls in parseNames([(url, url) for url in urls]).iteritems()] def _decrypt(self, urls): - """ Internal method to select decrypting method + """Internal method to select decrypting method :param urls: List of urls/content :return: @@ -208,7 +219,7 @@ class Crypter(Base): self.logDebug("Deprecated .decrypt() method in Crypter plugin") result = [] for url in urls: - self.pyfile = PyFileMockup(url, self.package) + self.pyfile = PyFileMockup(url) self.setup() self.decrypt(self.pyfile) result.extend(self.convertPackages()) @@ -223,22 +234,12 @@ class Crypter(Base): result.extend(to_list(self.decryptFile(c))) try: if f.startswith("tmp_"): remove(f) - except: - pass + except IOError: + self.logWarning(_("Could not remove file '%s'") % f) + self.core.print_exc() return result - def processDecrypt(self, urls): - """Catches all exceptions in decrypt methods and return results - - :return: Decrypting results - """ - try: - return to_list(self._decrypt(urls)) - except Exception: - self.core.print_exc() - return [] - def getLocalContent(self, urls): """Load files from disk and separate to file content and url list diff --git a/pyload/plugins/Hoster.py b/pyload/plugins/Hoster.py index bc1e4f9d0..4d96c5730 100644 --- a/pyload/plugins/Hoster.py +++ b/pyload/plugins/Hoster.py @@ -58,16 +58,14 @@ class Hoster(Base): @staticmethod def getInfo(urls): """This method is used to retrieve the online status of files for hoster plugins. - It has to *yield* list of tuples with the result in this format (name, size, status, url), - where status is one of API pyfile statuses. :param urls: List of urls - :return: yield list of tuple with results (name, size, status, url) + :return: yield list of :class:`LinkStatus` as result """ pass def __init__(self, pyfile): - Base.__init__(self, pyfile.m.core) + Base.__init__(self, pyfile.m.core, pyfile.owner) self.wantReconnect = False #: enables simultaneous processing of multiple downloads diff --git a/pyload/plugins/network/CurlRequest.py b/pyload/plugins/network/CurlRequest.py index 182553ed1..8d1f22450 100644 --- a/pyload/plugins/network/CurlRequest.py +++ b/pyload/plugins/network/CurlRequest.py @@ -40,6 +40,7 @@ def myurlencode(data): bad_headers = range(400, 418) + range(500, 506) +pycurl.global_init(pycurl.GLOBAL_DEFAULT) class CurlRequest(Request): """ Request class based on libcurl """ diff --git a/pyload/remote/apitypes.py b/pyload/remote/apitypes.py index d186717d4..53d2de6d2 100644 --- a/pyload/remote/apitypes.py +++ b/pyload/remote/apitypes.py @@ -241,14 +241,15 @@ class InvalidConfigSection(ExceptionObject): self.section = section class LinkStatus(BaseObject): - __slots__ = ['url', 'name', 'plugin', 'size', 'status'] + __slots__ = ['url', 'name', 'size', 'status', 'plugin', 'hash'] - def __init__(self, url=None, name=None, plugin=None, size=None, status=None): + def __init__(self, url=None, name=None, size=None, status=None, plugin=None, hash=None): self.url = url self.name = name - self.plugin = plugin self.size = size self.status = status + self.plugin = plugin + self.hash = hash class OnlineCheck(BaseObject): __slots__ = ['rid', 'data'] diff --git a/pyload/remote/apitypes_debug.py b/pyload/remote/apitypes_debug.py index 1c34b702e..74ea8a6a8 100644 --- a/pyload/remote/apitypes_debug.py +++ b/pyload/remote/apitypes_debug.py @@ -32,7 +32,7 @@ classes = { 'Input' : [int, (None, basestring), (None, basestring)], 'InteractionTask' : [int, int, Input, basestring, basestring, basestring], 'InvalidConfigSection' : [basestring], - 'LinkStatus' : [basestring, basestring, basestring, int, int], + 'LinkStatus' : [basestring, basestring, int, int, (None, basestring), (None, basestring)], 'OnlineCheck' : [int, (dict, basestring, LinkStatus)], 'PackageDoesNotExists' : [int], 'PackageInfo' : [int, basestring, basestring, int, int, basestring, basestring, basestring, int, (list, basestring), int, bool, int, PackageStats, (list, int), (list, int)], diff --git a/pyload/remote/pyload.thrift b/pyload/remote/pyload.thrift index d41b6d10b..905be22b0 100644 --- a/pyload/remote/pyload.thrift +++ b/pyload/remote/pyload.thrift @@ -197,9 +197,10 @@ struct TreeCollection { struct LinkStatus { 1: string url, 2: string name, - 3: PluginName plugin, - 4: ByteCount size, // size <= 0 : unknown - 5: DownloadStatus status, + 3: ByteCount size, // size <= 0 : unknown + 4: DownloadStatus status, + 5: optional PluginName plugin, + 6: optional string hash } struct ServerStatus { diff --git a/pyload/remote/wsbackend/AsyncHandler.py b/pyload/remote/wsbackend/AsyncHandler.py index 136cc060d..c7a26cd6b 100644 --- a/pyload/remote/wsbackend/AsyncHandler.py +++ b/pyload/remote/wsbackend/AsyncHandler.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- ############################################################################### -# Copyright(c) 2008-2012 pyLoad Team +# Copyright(c) 2008-2013 pyLoad Team # http://www.pyload.org # # This file is part of pyLoad. diff --git a/pyload/threads/BaseThread.py b/pyload/threads/BaseThread.py index deaf03461..b7912e924 100644 --- a/pyload/threads/BaseThread.py +++ b/pyload/threads/BaseThread.py @@ -1,9 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import sys from threading import Thread from time import strftime, gmtime -from sys import exc_info from types import MethodType from pprint import pformat from traceback import format_exc @@ -11,17 +11,18 @@ from traceback import format_exc from pyload.utils import primary_uid from pyload.utils.fs import listdir, join, save_join, stat, exists + class BaseThread(Thread): """abstract base class for thread types""" - def __init__(self, manager): + def __init__(self, manager, ower=None): Thread.__init__(self) self.setDaemon(True) self.m = manager #thread manager self.core = manager.core self.log = manager.core.log - #: Owner of the thread, every type should set it + #: Owner of the thread, every type should set it or overwrite user self.owner = None @property @@ -40,13 +41,13 @@ class BaseThread(Thread): dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S")) if pyfile: - dump = self.getFileDump(pyfile) + dump = self.getPluginDump(pyfile.plugin) + "\n" + dump += self.getFileDump(pyfile) else: dump = self.getPluginDump(plugin) try: import zipfile - zip = zipfile.ZipFile(dump_name, "w") if exists(join("tmp", name)): @@ -81,11 +82,11 @@ class BaseThread(Thread): self.log.info("Debug Report written to %s" % dump_name) return dump_name - def getFileDump(self, pyfile): + def getPluginDump(self, plugin): dump = "pyLoad %s Debug Report of %s %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % ( - self.m.core.api.getServerVersion(), pyfile.pluginname, pyfile.plugin.__version__, format_exc()) + self.m.core.api.getServerVersion(), plugin.__name__, plugin.__version__, format_exc()) - tb = exc_info()[2] + tb = sys.exc_info()[2] stack = [] while tb: stack.append(tb.tb_frame) @@ -109,8 +110,8 @@ class BaseThread(Thread): dump += "\n\nPLUGIN OBJECT DUMP: \n\n" - for name in dir(pyfile.plugin): - attr = getattr(pyfile.plugin, name) + for name in dir(plugin): + attr = getattr(plugin, name) if not name.endswith("__") and type(attr) != MethodType: dump += "\t%20s = " % name try: @@ -118,7 +119,10 @@ class BaseThread(Thread): except Exception, e: dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n" - dump += "\nPYFILE OBJECT DUMP: \n\n" + return dump + + def getFileDump(self, pyfile): + dump = "PYFILE OBJECT DUMP: \n\n" for name in dir(pyfile): attr = getattr(pyfile, name) @@ -129,14 +133,13 @@ class BaseThread(Thread): except Exception, e: dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n" - dump += "\n\nCONFIG: \n\n" - dump += pformat(self.m.core.config.values) + "\n" - return dump - #TODO - def getPluginDump(self, plugin): - return "" - def getSystemDump(self): - return "" + dump = "SYSTEM:\n\n" + dump += """Platform: %s +Version: %s +Encoding: %s +FS-Encoding: %s + """ % (sys.platform, sys.version, sys.getdefaultencoding(), sys.getfilesystemencoding()) + return dump diff --git a/pyload/threads/DecrypterThread.py b/pyload/threads/DecrypterThread.py index e4df2ee75..e8b889ac8 100644 --- a/pyload/threads/DecrypterThread.py +++ b/pyload/threads/DecrypterThread.py @@ -2,10 +2,9 @@ # -*- coding: utf-8 -*- from time import sleep -from traceback import print_exc -from pyload.utils import uniqify -from pyload.plugins.Base import Retry +from pyload.utils import uniqify, accumulate +from pyload.plugins.Base import Abort, Retry from pyload.plugins.Crypter import Package from BaseThread import BaseThread @@ -14,30 +13,35 @@ class DecrypterThread(BaseThread): """thread for decrypting""" def __init__(self, manager, data, pid): - """constructor""" + # TODO: owner BaseThread.__init__(self, manager) + # [... (plugin, url) ...] self.data = data self.pid = pid self.start() def run(self): - plugin_map = {} - for url, plugin in self.data: - if plugin in plugin_map: - plugin_map[plugin].append(url) - else: - plugin_map[plugin] = [url] + pack = self.m.core.files.getPackage(self.pid) + links, packages = self.decrypt(accumulate(self.data), pack.password) - self.decrypt(plugin_map) + if links: + self.log.info(_("Decrypted %(count)d links into package %(name)s") % {"count": len(links), "name": pack.name}) + self.m.core.api.addFiles(self.pid, [l.url for l in links]) - def decrypt(self, plugin_map): - pack = self.m.core.files.getPackage(self.pid) + # TODO: add single package into this one and rename it? + # TODO: nested packages + for p in packages: + self.m.core.api.addPackage(p.name, p.getURLs(), pack.password) + + def decrypt(self, plugin_map, password=None): result = [] + # TODO QUEUE_DECRYPT + for name, urls in plugin_map.iteritems(): klass = self.m.core.pluginManager.loadClass("crypter", name) - plugin = klass(self.m.core, pack, pack.password) + plugin = klass(self.m.core, password) plugin_result = [] try: @@ -46,36 +50,37 @@ class DecrypterThread(BaseThread): except Retry: sleep(1) plugin_result = plugin._decrypt(urls) + except Abort: + plugin.logInfo(_("Decrypting aborted")) except Exception, e: plugin.logError(_("Decrypting failed"), e) - if self.m.core.debug: - print_exc() + if self.core.debug: + self.core.print_exc() self.writeDebugReport(plugin.__name__, plugin=plugin) + finally: + plugin.clean() plugin.logDebug("Decrypted", plugin_result) result.extend(plugin_result) - #TODO package names are optional - result = uniqify(result) + # generated packages pack_names = {} + # urls without package urls = [] + # merge urls and packages for p in result: if isinstance(p, Package): if p.name in pack_names: pack_names[p.name].urls.extend(p.urls) else: - pack_names[p.name] = p + if not p.name: + urls.append(p) + else: + pack_names[p.name] = p else: urls.append(p) - if urls: - self.log.info(_("Decrypted %(count)d links into package %(name)s") % {"count": len(urls), "name": pack.name}) - self.m.core.api.addFiles(self.pid, urls) - - for p in pack_names.itervalues(): - self.m.core.api.addPackage(p.name, p.urls, pack.password) - - if not result: - self.log.info(_("No links decrypted")) + urls = uniqify(urls) + return urls, pack_names.values()
\ No newline at end of file diff --git a/pyload/threads/InfoThread.py b/pyload/threads/InfoThread.py index fb4bdf11e..b62596ad3 100644 --- a/pyload/threads/InfoThread.py +++ b/pyload/threads/InfoThread.py @@ -9,25 +9,23 @@ from pyload.utils.packagetools import parseNames from pyload.utils import has_method, accumulate from BaseThread import BaseThread +from DecrypterThread import DecrypterThread -class InfoThread(BaseThread): - def __init__(self, manager, data, pid=-1, rid=-1): - """Constructor""" - BaseThread.__init__(self, manager) - self.data = data - self.pid = pid # package id - # [ .. (name, plugin) .. ] - - self.rid = rid #result id +class InfoThread(DecrypterThread): + def __init__(self, manager, owner, data, pid=-1, oc=None): + BaseThread.__init__(self, manager, owner) - self.cache = [] #accumulated data + # [... (plugin, url) ...] + self.data = data + self.pid = pid + self.oc = oc # online check + # urls that already have a package name + self.names = {} self.start() def run(self): - """run method""" - plugins = accumulate(self.data) crypter = {} @@ -37,93 +35,82 @@ class InfoThread(BaseThread): crypter[name] = plugins[name] del plugins[name] - #directly write to database - if self.pid > -1: - for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPluginModule(pluginname) - klass = self.m.core.pluginManager.getPluginClass(pluginname) - if has_method(klass, "getInfo"): - self.fetchForPlugin(pluginname, klass, urls, self.updateDB) - self.m.core.files.save() - elif has_method(plugin, "getInfo"): - self.log.debug("Deprecated .getInfo() method on module level, use classmethod instead") - self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) - self.m.core.files.save() - - else: #post the results - for name, urls in crypter.iteritems(): - #attach container content - try: - data = self.decrypt(name, urls) - except: - print_exc() - self.m.log.error("Could not decrypt content.") - data = [] - - accumulate(data, plugins) - - self.m.infoResults[self.rid] = {} - - for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPluginModule(pluginname) - klass = self.m.core.pluginManager.getPluginClass(pluginname) - if has_method(klass, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) - #force to process cache - if self.cache: - self.updateResult(pluginname, [], True) - elif has_method(plugin, "getInfo"): - self.log.debug("Deprecated .getInfo() method on module level, use staticmethod instead") - self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) - #force to process cache - if self.cache: - self.updateResult(pluginname, [], True) - else: - #generate default result - result = [(url, 0, 3, url) for url in urls] - - self.updateResult(pluginname, result, True) - - self.m.infoResults[self.rid]["ALL_INFO_FETCHED"] = {} - + if crypter: + # decrypt them + links, packages = self.decrypt(crypter) + # push these as initial result and save package names + self.updateResult(links) + for pack in packages: + for url in pack.getURLs(): + self.names[url] = pack.name + + links.extend(pack.links) + self.updateResult(pack.links) + + # TODO: no plugin information pushed to GUI + # parse links and merge + hoster, crypter = self.m.core.pluginManager.parseUrls([l.url for l in links]) + accumulate(hoster + crypter, plugins) + + # db or info result + cb = self.updateDB if self.pid > 1 else self.updateResult + + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPluginModule(pluginname) + klass = self.m.core.pluginManager.getPluginClass(pluginname) + if has_method(klass, "getInfo"): + self.fetchForPlugin(klass, urls, cb) + # TODO: this branch can be removed in the future + elif has_method(plugin, "getInfo"): + self.log.debug("Deprecated .getInfo() method on module level, use staticmethod instead") + self.fetchForPlugin(plugin, urls, cb) + + self.oc.done = True + self.names.clear() self.m.timestamp = time() + 5 * 60 - - def updateDB(self, plugin, result): - self.m.core.files.updateFileInfo(result, self.pid) - - def updateResult(self, plugin, result, force=False): - #parse package name and generate result - #accumulate results - - self.cache.extend(result) - - if len(self.cache) >= 20 or force: - #used for package generating - tmp = [(name, LinkStatus(url, name, plugin, int(size), status)) - for name, size, status, url in self.cache] - - data = parseNames(tmp) - self.m.setInfoResults(self.rid, data) - - self.cache = [] - - def updateCache(self, plugin, result): - self.cache.extend(result) - - def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None): + def updateDB(self, result): + # writes results to db + # convert link info to tuples + info = [(l.name, l.size, l.status, l.url) for l in result if not l.hash] + info_hash = [(l.name, l.size, l.status, l.hash, l.url) for l in result if l.hash] + if info: + self.m.core.files.updateFileInfo(info, self.pid) + if info_hash: + self.m.core.files.updateFileInfo(info_hash, self.pid) + + def updateResult(self, result): + tmp = {} + parse = [] + # separate these with name and without + for link in result: + if link.url in self.names: + tmp[link] = self.names[link.url] + else: + parse.append(link) + + data = parseNames([(link.name, link) for link in parse]) + # merge in packages that already have a name + data = accumulate(tmp.iteritems(), data) + + self.m.setInfoResults(self.oc, data) + + def fetchForPlugin(self, plugin, urls, cb): + """executes info fetching for given plugin and urls""" + # also works on module names + pluginname = plugin.__name__.split(".")[-1] try: - result = [] #result loaded from cache + cached = [] #results loaded from cache process = [] #urls to process for url in urls: if url in self.m.infoCache: - result.append(self.m.infoCache[url]) + cached.append(self.m.infoCache[url]) else: process.append(url) - if result: - self.m.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname)) - cb(pluginname, result) + if cached: + self.m.log.debug("Fetched %d links from cache for %s" % (len(cached), pluginname)) + cb(cached) if process: self.m.log.debug("Run Info Fetching for %s" % pluginname) @@ -131,26 +118,26 @@ class InfoThread(BaseThread): #result = [ .. (name, size, status, url) .. ] if not type(result) == list: result = [result] + links = [] + # Convert results to link statuses for res in result: - self.m.infoCache[res[3]] = res + if isinstance(res, LinkStatus): + links.append(res) + elif type(res) == tuple and len(res) == 4: + links.append(LinkStatus(res[3], res[0], int(res[1]), res[2], pluginname)) + elif type(res) == tuple and len(res) == 5: + links.append(LinkStatus(res[3], res[0], int(res[1]), res[2], pluginname, res[4])) + else: + self.m.log.debug("Invalid getInfo result: " + result) + + # put them on the cache + for link in links: + self.m.infoCache[link.url] = link - cb(pluginname, result) + cb(links) self.m.log.debug("Finished Info Fetching for %s" % pluginname) except Exception, e: self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % {"name": pluginname, "err": str(e)}) - if self.m.core.debug: - print_exc() - - # generate default results - if err: - result = [(url, 0, 3, url) for url in urls] - cb(pluginname, result) - - def decrypt(self, plugin, urls): - self.m.log.debug("Decrypting %s" % plugin) - klass = self.m.core.pluginManager.loadClass("crypter", plugin) - urls = klass.decrypt(self.core, urls) - data, crypter = self.m.core.pluginManager.parseUrls(urls) - return data + crypter + self.core.print_exc()
\ No newline at end of file diff --git a/pyload/threads/ThreadManager.py b/pyload/threads/ThreadManager.py index 9ad23138a..ff8bfe8d7 100644 --- a/pyload/threads/ThreadManager.py +++ b/pyload/threads/ThreadManager.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- ############################################################################### -# Copyright(c) 2008-2012 pyLoad Team +# Copyright(c) 2008-2013 pyLoad Team # http://www.pyload.org # # This file is part of pyLoad. @@ -24,9 +24,8 @@ from time import sleep, time from traceback import print_exc from random import choice -import pycurl - from pyload.datatypes.PyFile import PyFile +from pyload.datatypes.OnlineCheck import OnlineCheck from pyload.network.RequestFactory import getURL from pyload.utils import lock, uniqify from pyload.utils.fs import free_space @@ -63,13 +62,12 @@ class ThreadManager: # pool of ids for online check self.resultIDs = 0 - # threads which are fetching hoster results + # saved online checks self.infoResults = {} + # timeout for cache purge self.timestamp = 0 - pycurl.global_init(pycurl.GLOBAL_DEFAULT) - for i in range(self.core.config.get("download", "max_downloads")): self.createThread() @@ -83,23 +81,26 @@ class ThreadManager: def createInfoThread(self, data, pid): """ start a thread which fetches online status and other info's """ self.timestamp = time() + 5 * 60 - if data: InfoThread(self, data, pid) + if data: InfoThread(self, None, data, pid) @lock - def createResultThread(self, data): + def createResultThread(self, user, data): """ creates a thread to fetch online status, returns result id """ self.timestamp = time() + 5 * 60 rid = self.resultIDs self.resultIDs += 1 + oc = OnlineCheck(rid, user) + self.infoResults[rid] = oc + # maps url to plugin urls = [] for links in data.itervalues(): for link in links: urls.append((link.url, link.plugin)) - InfoThread(self, urls, rid=rid) + InfoThread(self, user, urls, oc=oc) return rid @@ -108,23 +109,13 @@ class ThreadManager: """ Start decrypting of entered data, all links in one package are accumulated to one thread.""" if data: DecrypterThread(self, data, pid) - @lock def getInfoResult(self, rid): - """returns result and clears it""" - self.timestamp = time() + 5 * 60 - # TODO: restrict user to his own results - if rid in self.infoResults: - data = self.infoResults[rid] - self.infoResults[rid] = {} - return data - else: - return {} + return self.infoResults.get(rid) - @lock - def setInfoResults(self, rid, result): - self.core.evm.dispatchEvent("linkcheck:updated", rid, result) - self.infoResults[rid].update(result) + def setInfoResults(self, oc, result): + self.core.evm.dispatchEvent("linkcheck:updated", oc.rid, result, owner=oc.owner) + oc.update(result) def getActiveDownloads(self, user=None): # TODO: user context @@ -220,8 +211,7 @@ class ThreadManager: self.log.warning(_("Failed executing reconnect script!")) self.core.config["reconnect"]["activated"] = False self.reconnecting.clear() - if self.core.debug: - print_exc() + self.core.print_exc() return reconn.wait() @@ -268,6 +258,8 @@ class ThreadManager: """ make a global curl cleanup (currently unused) """ if self.processingIds(): return False + import pycurl + pycurl.global_cleanup() pycurl.global_init(pycurl.GLOBAL_DEFAULT) self.downloaded = 0 @@ -317,7 +309,3 @@ class ThreadManager: if occ not in self.core.files.jobCache: self.core.files.jobCache[occ] = [] self.core.files.jobCache[occ].append(job.id) - - def cleanup(self): - """do global cleanup, should be called when finished with pycurl""" - pycurl.global_cleanup() diff --git a/pyload/utils/__init__.py b/pyload/utils/__init__.py index bc4e27df4..1655be857 100644 --- a/pyload/utils/__init__.py +++ b/pyload/utils/__init__.py @@ -197,7 +197,7 @@ def has_method(obj, name): def accumulate(it, inv_map=None): """ accumulate (key, value) data to {value : [keylist]} dictionary """ - if not inv_map: + if inv_map is None: inv_map = {} for key, value in it: diff --git a/pyload/web/app/scripts/helpers/linkStatus.js b/pyload/web/app/scripts/helpers/linkStatus.js index 2497785fb..448d63691 100644 --- a/pyload/web/app/scripts/helpers/linkStatus.js +++ b/pyload/web/app/scripts/helpers/linkStatus.js @@ -5,7 +5,7 @@ define('helpers/linkStatus', ['underscore', 'handlebars', 'utils/apitypes', 'uti var s; if (status === Api.DownloadStatus.Online) s = '<span class="text-success">' + i18n.gettext('online') + '</span>'; - else if (status === Api.DownloadState.Offline) + else if (status === Api.DownloadStatus.Offline) s = '<span class="text-error">' + i18n.gettext('offline') + '</span>'; else s = '<span class="text-info">' + i18n.gettext('unknown') + '</span>'; diff --git a/pyload/web/app/scripts/models/LinkStatus.js b/pyload/web/app/scripts/models/LinkStatus.js index 2be1ce368..be6385c9c 100644 --- a/pyload/web/app/scripts/models/LinkStatus.js +++ b/pyload/web/app/scripts/models/LinkStatus.js @@ -8,9 +8,10 @@ define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes'], defaults: { name: '', - plugin: '', size: -1, - status: Api.DownloadStatus.Queued + status: Api.DownloadStatus.Queued, + plugin: '', + hash: null }, destroy: function() { diff --git a/pyload/web/app/styles/default/linkgrabber.less b/pyload/web/app/styles/default/linkgrabber.less index c2e3642ef..010dbadec 100644 --- a/pyload/web/app/styles/default/linkgrabber.less +++ b/pyload/web/app/styles/default/linkgrabber.less @@ -49,6 +49,11 @@ } + .link-name { + .hyphens(); + width: 50%; + } + img { height: 22px; } diff --git a/pyload/web/app/templates/default/linkgrabber/package.html b/pyload/web/app/templates/default/linkgrabber/package.html index ed699f0d5..d5d4c669b 100644 --- a/pyload/web/app/templates/default/linkgrabber/package.html +++ b/pyload/web/app/templates/default/linkgrabber/package.html @@ -7,9 +7,9 @@ <tbody> {{#each links}} <tr> - <td>{{ name }}</td> + <td class="link-name">{{ name }}</td> <td><img src="{{ pluginIcon plugin }}"> {{ plugin }}</td> - <td>{{formatSize size }}</td> + <td>{{ formatSize size }}</td> <td>{{ linkStatus status }}</td> <td><button class="btn btn-danger btn-mini" data-index={{@index}}><i class="icon-trash"></i></button></td> </tr> |