From 967d6dd16c25ceba22dcd105079f72534ddb87e9 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Thu, 26 Sep 2013 16:40:38 +0200 Subject: rewritten decrypter and info fetching thread --- pyload/Core.py | 4 +- pyload/api/DownloadPreparingApi.py | 18 +- pyload/datatypes/OnlineCheck.py | 33 +++- pyload/datatypes/PyFile.py | 6 +- pyload/plugins/Base.py | 30 ++-- pyload/plugins/Crypter.py | 77 ++++---- pyload/plugins/Hoster.py | 6 +- pyload/plugins/network/CurlRequest.py | 1 + pyload/remote/apitypes.py | 7 +- pyload/remote/apitypes_debug.py | 2 +- pyload/remote/pyload.thrift | 7 +- pyload/remote/wsbackend/AsyncHandler.py | 2 +- pyload/threads/BaseThread.py | 41 +++-- pyload/threads/DecrypterThread.py | 61 ++++--- pyload/threads/InfoThread.py | 199 ++++++++++----------- pyload/threads/ThreadManager.py | 46 ++--- pyload/utils/__init__.py | 2 +- pyload/web/app/scripts/helpers/linkStatus.js | 2 +- pyload/web/app/scripts/models/LinkStatus.js | 5 +- pyload/web/app/styles/default/linkgrabber.less | 5 + .../app/templates/default/linkgrabber/package.html | 4 +- 21 files changed, 286 insertions(+), 272 deletions(-) diff --git a/pyload/Core.py b/pyload/Core.py index 2d591a2bb..f97cfcf3b 100644 --- a/pyload/Core.py +++ b/pyload/Core.py @@ -599,8 +599,8 @@ class Core(object): if not self.pdb: self.pdb = Pdb() self.pdb.set_trace() - def print_exc(self): - if self.debug: + def print_exc(self, force=False): + if self.debug or force: print_exc() def path(self, *args): diff --git a/pyload/api/DownloadPreparingApi.py b/pyload/api/DownloadPreparingApi.py index 68c83e97d..131f73b1d 100644 --- a/pyload/api/DownloadPreparingApi.py +++ b/pyload/api/DownloadPreparingApi.py @@ -37,16 +37,16 @@ class DownloadPreparingApi(ApiComponent): def checkLinks(self, links): """ initiates online status check, will also decrypt files. - :param urls: + :param links: :return: initial set of data as :class:`OnlineCheck` instance containing the result id """ hoster, crypter = self.core.pluginManager.parseUrls(links) #: TODO: withhold crypter, derypt or add later # initial result does not contain the crypter links - tmp = [(url, LinkStatus(url, url, pluginname, -1, DS.Queued)) for url, pluginname in hoster + crypter] + tmp = [(url, LinkStatus(url, url, -1, DS.Queued, pluginname)) for url, pluginname in hoster + crypter] data = parseNames(tmp) - rid = self.core.threadManager.createResultThread(data) + rid = self.core.threadManager.createResultThread(self.primaryUID, data) return OnlineCheck(rid, data) @@ -54,8 +54,7 @@ class DownloadPreparingApi(ApiComponent): def checkContainer(self, filename, data): """ checks online status of urls and a submitted container file - :param urls: list of urls - :param container: container file name + :param filename: name of the file :param data: file content :return: :class:`OnlineCheck` """ @@ -88,13 +87,8 @@ class DownloadPreparingApi(ApiComponent): :return: `OnlineCheck`, if rid is -1 then there is no more data available """ result = self.core.threadManager.getInfoResult(rid) - - if "ALL_INFO_FETCHED" in result: - del result["ALL_INFO_FETCHED"] - return OnlineCheck(-1, result) - else: - return OnlineCheck(rid, result) - + if result and result.owner == self.primaryUID: + return result.toApiData() @RequirePerm(Permission.Add) def generatePackages(self, links): diff --git a/pyload/datatypes/OnlineCheck.py b/pyload/datatypes/OnlineCheck.py index 4b31e848b..2797828bf 100644 --- a/pyload/datatypes/OnlineCheck.py +++ b/pyload/datatypes/OnlineCheck.py @@ -1 +1,32 @@ -__author__ = 'christian' +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from time import time + +from pyload.Api import OnlineCheck as OC + +class OnlineCheck: + """ Helper class that holds result of an initiated online check """ + + def __init__(self, rid, owner): + self.rid = rid + self.owner = owner + self.result = {} + self.done = False + + self.timestamp = time() + + def update(self, result): + self.timestamp = time() + self.result.update(result) + + def toApiData(self): + self.timestamp = time() + oc = OC(self.rid, self.result) + # getting the results clears the older ones + self.result = {} + # indication for no more data + if self.done: + oc.rid = -1 + + return oc \ No newline at end of file diff --git a/pyload/datatypes/PyFile.py b/pyload/datatypes/PyFile.py index 2cc28fa24..de7288d22 100644 --- a/pyload/datatypes/PyFile.py +++ b/pyload/datatypes/PyFile.py @@ -49,7 +49,7 @@ class PyFile(object): Represents a file object at runtime """ __slots__ = ("m", "fid", "_name", "_size", "filestatus", "media", "added", "fileorder", - "url", "pluginname", "hash", "status", "error", "packageid", "ownerid", + "url", "pluginname", "hash", "status", "error", "packageid", "owner", "lock", "plugin", "waitUntil", "abort", "statusname", "reconnected", "pluginclass") @@ -83,7 +83,7 @@ class PyFile(object): self.hash = hash self.status = status self.error = error - self.ownerid = owner + self.owner = owner self.packageid = package # database information ends here @@ -183,7 +183,7 @@ class PyFile(object): def toInfoData(self): - return FileInfo(self.fid, self.getName(), self.packageid, self.ownerid, self.getSize(), self.filestatus, + return FileInfo(self.fid, self.getName(), self.packageid, self.owner, self.getSize(), self.filestatus, self.media, self.added, self.fileorder, DownloadInfo( self.url, self.pluginname, self.hash, self.status, self.getStatusName(), self.error ) diff --git a/pyload/plugins/Base.py b/pyload/plugins/Base.py index 3ca8abdd0..abb59a7bc 100644 --- a/pyload/plugins/Base.py +++ b/pyload/plugins/Base.py @@ -1,21 +1,19 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" +############################################################################### +# Copyright(c) 2008-2013 pyLoad Team +# http://www.pyload.org +# +# This file is part of pyLoad. +# pyLoad is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### import sys from time import time, sleep diff --git a/pyload/plugins/Crypter.py b/pyload/plugins/Crypter.py index cce43fb56..2175b5f94 100644 --- a/pyload/plugins/Crypter.py +++ b/pyload/plugins/Crypter.py @@ -1,50 +1,60 @@ # -*- coding: utf-8 -*- -from traceback import print_exc - -from pyload.Api import LinkStatus +from pyload.Api import LinkStatus, DownloadStatus as DS from pyload.utils import to_list, has_method, uniqify from pyload.utils.fs import exists, remove, fs_encode from pyload.utils.packagetools import parseNames -from Base import Base, Retry +from Base import Base, Fail, Retry, Abort class Package: """ Container that indicates that a new package should be created """ - def __init__(self, name=None, urls=None): + def __init__(self, name=None, links=None): self.name = name - self.urls = urls if urls else [] + self.links = [] + + if links: + self.addLinks(links) # nested packages self.packs = [] - def addURL(self, url): - self.urls.append(url) + def addLinks(self, links): + """ Add one or multiple links to the package + + :param links: One or list of urls or :class:`LinkStatus` + """ + links = to_list(links) + for link in links: + if not isinstance(link, LinkStatus): + link = LinkStatus(link, link, -1, DS.Queued) - def addLink(self, url, name, status, size): - # TODO: allow to add urls with known information - pass + self.links.append(link) def addPackage(self, pack): self.packs.append(pack) + def getURLs(self): + return [link.url for link in self.links] + def getAllURLs(self): - urls = self.urls + urls = self.getURLs() for p in self.packs: urls.extend(p.getAllURLs()) return urls # same name and urls is enough to be equal for packages def __eq__(self, other): - return self.name == other.name and self.urls == other.urls + return self.name == other.name and self.links == other.links def __repr__(self): - return u" -1: - for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPluginModule(pluginname) - klass = self.m.core.pluginManager.getPluginClass(pluginname) - if has_method(klass, "getInfo"): - self.fetchForPlugin(pluginname, klass, urls, self.updateDB) - self.m.core.files.save() - elif has_method(plugin, "getInfo"): - self.log.debug("Deprecated .getInfo() method on module level, use classmethod instead") - self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) - self.m.core.files.save() - - else: #post the results - for name, urls in crypter.iteritems(): - #attach container content - try: - data = self.decrypt(name, urls) - except: - print_exc() - self.m.log.error("Could not decrypt content.") - data = [] - - accumulate(data, plugins) - - self.m.infoResults[self.rid] = {} - - for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPluginModule(pluginname) - klass = self.m.core.pluginManager.getPluginClass(pluginname) - if has_method(klass, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) - #force to process cache - if self.cache: - self.updateResult(pluginname, [], True) - elif has_method(plugin, "getInfo"): - self.log.debug("Deprecated .getInfo() method on module level, use staticmethod instead") - self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) - #force to process cache - if self.cache: - self.updateResult(pluginname, [], True) - else: - #generate default result - result = [(url, 0, 3, url) for url in urls] - - self.updateResult(pluginname, result, True) - - self.m.infoResults[self.rid]["ALL_INFO_FETCHED"] = {} - + if crypter: + # decrypt them + links, packages = self.decrypt(crypter) + # push these as initial result and save package names + self.updateResult(links) + for pack in packages: + for url in pack.getURLs(): + self.names[url] = pack.name + + links.extend(pack.links) + self.updateResult(pack.links) + + # TODO: no plugin information pushed to GUI + # parse links and merge + hoster, crypter = self.m.core.pluginManager.parseUrls([l.url for l in links]) + accumulate(hoster + crypter, plugins) + + # db or info result + cb = self.updateDB if self.pid > 1 else self.updateResult + + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPluginModule(pluginname) + klass = self.m.core.pluginManager.getPluginClass(pluginname) + if has_method(klass, "getInfo"): + self.fetchForPlugin(klass, urls, cb) + # TODO: this branch can be removed in the future + elif has_method(plugin, "getInfo"): + self.log.debug("Deprecated .getInfo() method on module level, use staticmethod instead") + self.fetchForPlugin(plugin, urls, cb) + + self.oc.done = True + self.names.clear() self.m.timestamp = time() + 5 * 60 - - def updateDB(self, plugin, result): - self.m.core.files.updateFileInfo(result, self.pid) - - def updateResult(self, plugin, result, force=False): - #parse package name and generate result - #accumulate results - - self.cache.extend(result) - - if len(self.cache) >= 20 or force: - #used for package generating - tmp = [(name, LinkStatus(url, name, plugin, int(size), status)) - for name, size, status, url in self.cache] - - data = parseNames(tmp) - self.m.setInfoResults(self.rid, data) - - self.cache = [] - - def updateCache(self, plugin, result): - self.cache.extend(result) - - def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None): + def updateDB(self, result): + # writes results to db + # convert link info to tuples + info = [(l.name, l.size, l.status, l.url) for l in result if not l.hash] + info_hash = [(l.name, l.size, l.status, l.hash, l.url) for l in result if l.hash] + if info: + self.m.core.files.updateFileInfo(info, self.pid) + if info_hash: + self.m.core.files.updateFileInfo(info_hash, self.pid) + + def updateResult(self, result): + tmp = {} + parse = [] + # separate these with name and without + for link in result: + if link.url in self.names: + tmp[link] = self.names[link.url] + else: + parse.append(link) + + data = parseNames([(link.name, link) for link in parse]) + # merge in packages that already have a name + data = accumulate(tmp.iteritems(), data) + + self.m.setInfoResults(self.oc, data) + + def fetchForPlugin(self, plugin, urls, cb): + """executes info fetching for given plugin and urls""" + # also works on module names + pluginname = plugin.__name__.split(".")[-1] try: - result = [] #result loaded from cache + cached = [] #results loaded from cache process = [] #urls to process for url in urls: if url in self.m.infoCache: - result.append(self.m.infoCache[url]) + cached.append(self.m.infoCache[url]) else: process.append(url) - if result: - self.m.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname)) - cb(pluginname, result) + if cached: + self.m.log.debug("Fetched %d links from cache for %s" % (len(cached), pluginname)) + cb(cached) if process: self.m.log.debug("Run Info Fetching for %s" % pluginname) @@ -131,26 +118,26 @@ class InfoThread(BaseThread): #result = [ .. (name, size, status, url) .. ] if not type(result) == list: result = [result] + links = [] + # Convert results to link statuses for res in result: - self.m.infoCache[res[3]] = res + if isinstance(res, LinkStatus): + links.append(res) + elif type(res) == tuple and len(res) == 4: + links.append(LinkStatus(res[3], res[0], int(res[1]), res[2], pluginname)) + elif type(res) == tuple and len(res) == 5: + links.append(LinkStatus(res[3], res[0], int(res[1]), res[2], pluginname, res[4])) + else: + self.m.log.debug("Invalid getInfo result: " + result) + + # put them on the cache + for link in links: + self.m.infoCache[link.url] = link - cb(pluginname, result) + cb(links) self.m.log.debug("Finished Info Fetching for %s" % pluginname) except Exception, e: self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % {"name": pluginname, "err": str(e)}) - if self.m.core.debug: - print_exc() - - # generate default results - if err: - result = [(url, 0, 3, url) for url in urls] - cb(pluginname, result) - - def decrypt(self, plugin, urls): - self.m.log.debug("Decrypting %s" % plugin) - klass = self.m.core.pluginManager.loadClass("crypter", plugin) - urls = klass.decrypt(self.core, urls) - data, crypter = self.m.core.pluginManager.parseUrls(urls) - return data + crypter + self.core.print_exc() \ No newline at end of file diff --git a/pyload/threads/ThreadManager.py b/pyload/threads/ThreadManager.py index 9ad23138a..ff8bfe8d7 100644 --- a/pyload/threads/ThreadManager.py +++ b/pyload/threads/ThreadManager.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- ############################################################################### -# Copyright(c) 2008-2012 pyLoad Team +# Copyright(c) 2008-2013 pyLoad Team # http://www.pyload.org # # This file is part of pyLoad. @@ -24,9 +24,8 @@ from time import sleep, time from traceback import print_exc from random import choice -import pycurl - from pyload.datatypes.PyFile import PyFile +from pyload.datatypes.OnlineCheck import OnlineCheck from pyload.network.RequestFactory import getURL from pyload.utils import lock, uniqify from pyload.utils.fs import free_space @@ -63,13 +62,12 @@ class ThreadManager: # pool of ids for online check self.resultIDs = 0 - # threads which are fetching hoster results + # saved online checks self.infoResults = {} + # timeout for cache purge self.timestamp = 0 - pycurl.global_init(pycurl.GLOBAL_DEFAULT) - for i in range(self.core.config.get("download", "max_downloads")): self.createThread() @@ -83,23 +81,26 @@ class ThreadManager: def createInfoThread(self, data, pid): """ start a thread which fetches online status and other info's """ self.timestamp = time() + 5 * 60 - if data: InfoThread(self, data, pid) + if data: InfoThread(self, None, data, pid) @lock - def createResultThread(self, data): + def createResultThread(self, user, data): """ creates a thread to fetch online status, returns result id """ self.timestamp = time() + 5 * 60 rid = self.resultIDs self.resultIDs += 1 + oc = OnlineCheck(rid, user) + self.infoResults[rid] = oc + # maps url to plugin urls = [] for links in data.itervalues(): for link in links: urls.append((link.url, link.plugin)) - InfoThread(self, urls, rid=rid) + InfoThread(self, user, urls, oc=oc) return rid @@ -108,23 +109,13 @@ class ThreadManager: """ Start decrypting of entered data, all links in one package are accumulated to one thread.""" if data: DecrypterThread(self, data, pid) - @lock def getInfoResult(self, rid): - """returns result and clears it""" - self.timestamp = time() + 5 * 60 - # TODO: restrict user to his own results - if rid in self.infoResults: - data = self.infoResults[rid] - self.infoResults[rid] = {} - return data - else: - return {} + return self.infoResults.get(rid) - @lock - def setInfoResults(self, rid, result): - self.core.evm.dispatchEvent("linkcheck:updated", rid, result) - self.infoResults[rid].update(result) + def setInfoResults(self, oc, result): + self.core.evm.dispatchEvent("linkcheck:updated", oc.rid, result, owner=oc.owner) + oc.update(result) def getActiveDownloads(self, user=None): # TODO: user context @@ -220,8 +211,7 @@ class ThreadManager: self.log.warning(_("Failed executing reconnect script!")) self.core.config["reconnect"]["activated"] = False self.reconnecting.clear() - if self.core.debug: - print_exc() + self.core.print_exc() return reconn.wait() @@ -268,6 +258,8 @@ class ThreadManager: """ make a global curl cleanup (currently unused) """ if self.processingIds(): return False + import pycurl + pycurl.global_cleanup() pycurl.global_init(pycurl.GLOBAL_DEFAULT) self.downloaded = 0 @@ -317,7 +309,3 @@ class ThreadManager: if occ not in self.core.files.jobCache: self.core.files.jobCache[occ] = [] self.core.files.jobCache[occ].append(job.id) - - def cleanup(self): - """do global cleanup, should be called when finished with pycurl""" - pycurl.global_cleanup() diff --git a/pyload/utils/__init__.py b/pyload/utils/__init__.py index bc4e27df4..1655be857 100644 --- a/pyload/utils/__init__.py +++ b/pyload/utils/__init__.py @@ -197,7 +197,7 @@ def has_method(obj, name): def accumulate(it, inv_map=None): """ accumulate (key, value) data to {value : [keylist]} dictionary """ - if not inv_map: + if inv_map is None: inv_map = {} for key, value in it: diff --git a/pyload/web/app/scripts/helpers/linkStatus.js b/pyload/web/app/scripts/helpers/linkStatus.js index 2497785fb..448d63691 100644 --- a/pyload/web/app/scripts/helpers/linkStatus.js +++ b/pyload/web/app/scripts/helpers/linkStatus.js @@ -5,7 +5,7 @@ define('helpers/linkStatus', ['underscore', 'handlebars', 'utils/apitypes', 'uti var s; if (status === Api.DownloadStatus.Online) s = '' + i18n.gettext('online') + ''; - else if (status === Api.DownloadState.Offline) + else if (status === Api.DownloadStatus.Offline) s = '' + i18n.gettext('offline') + ''; else s = '' + i18n.gettext('unknown') + ''; diff --git a/pyload/web/app/scripts/models/LinkStatus.js b/pyload/web/app/scripts/models/LinkStatus.js index 2be1ce368..be6385c9c 100644 --- a/pyload/web/app/scripts/models/LinkStatus.js +++ b/pyload/web/app/scripts/models/LinkStatus.js @@ -8,9 +8,10 @@ define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes'], defaults: { name: '', - plugin: '', size: -1, - status: Api.DownloadStatus.Queued + status: Api.DownloadStatus.Queued, + plugin: '', + hash: null }, destroy: function() { diff --git a/pyload/web/app/styles/default/linkgrabber.less b/pyload/web/app/styles/default/linkgrabber.less index c2e3642ef..010dbadec 100644 --- a/pyload/web/app/styles/default/linkgrabber.less +++ b/pyload/web/app/styles/default/linkgrabber.less @@ -49,6 +49,11 @@ } + .link-name { + .hyphens(); + width: 50%; + } + img { height: 22px; } diff --git a/pyload/web/app/templates/default/linkgrabber/package.html b/pyload/web/app/templates/default/linkgrabber/package.html index ed699f0d5..d5d4c669b 100644 --- a/pyload/web/app/templates/default/linkgrabber/package.html +++ b/pyload/web/app/templates/default/linkgrabber/package.html @@ -7,9 +7,9 @@ {{#each links}} - {{ name }} + {{ name }} {{ plugin }} - {{formatSize size }} + {{ formatSize size }} {{ linkStatus status }} -- cgit v1.2.3