From 74f2fa861eb59ec1675de8a217b0265ec927815b Mon Sep 17 00:00:00 2001 From: mkaay Date: Tue, 4 May 2010 18:14:58 +0200 Subject: better threadmanager --- module/DownloadThread.py | 179 +++++++++++++++++ module/FileList.py | 502 ++++++++++++++++++++++++++++++++++++++++++++++ module/Plugin.py | 185 ----------------- module/ThreadManager.py | 229 +++++++++++++++++++++ module/download_thread.py | 191 ------------------ module/file_list.py | 502 ---------------------------------------------- module/plugins/Plugin.py | 185 +++++++++++++++++ module/thread_list.py | 310 ---------------------------- pyLoadCore.py | 16 +- 9 files changed, 1104 insertions(+), 1195 deletions(-) create mode 100644 module/DownloadThread.py create mode 100644 module/FileList.py delete mode 100644 module/Plugin.py create mode 100644 module/ThreadManager.py delete mode 100644 module/download_thread.py delete mode 100644 module/file_list.py create mode 100644 module/plugins/Plugin.py delete mode 100644 module/thread_list.py diff --git a/module/DownloadThread.py b/module/DownloadThread.py new file mode 100644 index 000000000..4953ffa53 --- /dev/null +++ b/module/DownloadThread.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay + @author: spoob + @author: sebnapi + @version: v0.3.2 +""" + +from threading import Thread +import traceback +from time import sleep, time + +from module.network.Request import AbortDownload +from module.PullEvents import UpdateEvent + +class Status(object): + """ Saves all status information + """ + def __init__(self, pyfile): + self.pyfile = pyfile + self.type = None + self.status_queue = None + self.filename = None + self.url = None + self.exists = False + self.waituntil = 0 + self.plugin = pyfile.modul.__name__ + self.want_reconnect = False + self.error = "" + + def get_ETA(self): + return self.pyfile.plugin.req.get_ETA() + def get_speed(self): + return self.pyfile.plugin.req.get_speed() + def kB_left(self): + return self.pyfile.plugin.req.kB_left() + def size(self): + return self.pyfile.plugin.req.dl_size / 1024 + def percent(self): + if not self.kB_left() == 0 and not self.size() == 0: + percent = ((self.size()-self.kB_left()) * 100) / self.size() + return percent if percent < 101 else 0 + return 0 + +class Reconnect(Exception): + pass + +class Checksum(Exception): + def __init__(self, code, local_file): + self.code = code + self.file = local_file + + def getCode(self): + return self.code + + def getFile(self): + return self.file + +class CaptchaError(Exception): + pass + +class DownloadThread(Thread): + def __init__(self, parent, job): + Thread.__init__(self) + self.parent = parent + self.setDaemon(True) + self.loadedPyFile = job + + def run(self): + try: + self.download(self.loadedPyFile) + except AbortDownload: + self.loadedPyFile.plugin.req.abort = False + self.loadedPyFile.status.type = "aborted" + except Reconnect: + pass + except Checksum, e: + self.loadedPyFile.status.type = "failed" + self.loadedPyFile.status.error = "Checksum error: %d" % e.getCode() + f = open("%s.info" % e.getFile(), "w") + f.write("Checksum not matched!") + f.close() + except CaptchaError: + self.loadedPyFile.status.type = "failed" + self.loadedPyFile.status.error = "Can't solve captcha" + except Exception, e: + try: + if self.parent.parent.config['general']['debug_mode']: + traceback.print_exc() + code, msg = e + if code == 7: + sleep(60) + self.parent.parent.logger.info(_("Hoster unvailable, wait 60 seconds")) + except Exception, f: + self.parent.parent.logger.debug(_("Error getting error code: %s") % f) + if self.parent.parent.config['general']['debug_mode']: + traceback.print_exc() + self.loadedPyFile.status.type = "failed" + self.loadedPyFile.status.error = str(e) + finally: + self.parent.jobFinished(self.loadedPyFile) + self.parent.parent.pullManager.addEvent(UpdateEvent("file", self.loadedPyFile.id, "queue")) + sleep(0.8) + self.parent.removeThread(self) + + def download(self, pyfile): + status = pyfile.status + status.type = "starting" + self.parent.parent.pullManager.addEvent(UpdateEvent("file", pyfile.id, "queue")) + + pyfile.init_download() + + if not pyfile.plugin.prepare(self): + raise Exception, _("File not found") + + pyfile.plugin.req.set_timeout(self.parent.parent.config['general']['max_download_time']) + + if pyfile.plugin.props["type"] == "container": + status.type = "decrypting" + else: + status.type = "downloading" + self.parent.parent.pullManager.addEvent(UpdateEvent("file", pyfile.id, "queue")) + + + #~ free_file_name = self.get_free_name(status.filename) + #~ location = join(pyfile.folder, status.filename) + pyfile.plugin.proceed(status.url, status.filename) + + if self.parent.parent.xmlconfig.get("general", "checksum", True): + status.type = "checking" + check, code = pyfile.plugin.check_file(status.filename) + """ + return codes: + 0 - checksum ok + 1 - checksum wrong + 5 - can't get checksum + 10 - not implemented + 20 - unknown error + """ + if code == 0: + self.parent.parent.logger.info(_("Checksum ok ('%s')") % status.filename) + elif code == 1: + self.parent.parent.logger.info(_("Checksum not matched! ('%s')") % status.filename) + elif code == 5: + self.parent.parent.logger.debug(_("Can't get checksum for %s") % status.filename) + elif code == 10: + self.parent.parent.logger.debug(_("Checksum not implemented for %s") % status.filename) + if not check: + raise Checksum(code, location) + + status.type = "finished" + + def wait(self, pyfile): + pyfile.status.type = "waiting" + while (time() < pyfile.status.waituntil): + if self.parent.init_reconnect() or self.parent.reconnecting: + pyfile.status.type = "reconnected" + pyfile.status.want_reconnect = False + raise Reconnect + if pyfile.plugin.req.abort: + raise AbortDownload + sleep(1) + pyfile.status.want_reconnect = False + return True diff --git a/module/FileList.py b/module/FileList.py new file mode 100644 index 000000000..5282c0be6 --- /dev/null +++ b/module/FileList.py @@ -0,0 +1,502 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay + @author: RaNaN + @version: v0.3.1 + @list-version: v4 +""" + +LIST_VERSION = 4 + +from operator import attrgetter +from operator import concat +from os.path import join +import re +from threading import RLock +from time import sleep + +import cPickle +from module.DownloadThread import Status +import module.plugins.Plugin +from module.PullEvents import InsertEvent +from module.PullEvents import RemoveEvent +from module.PullEvents import UpdateEvent + +class NoSuchElementException(Exception): + pass + +class FileList(object): + def __init__(self, core): + self.core = core + self.lock = RLock() + self.download_folder = self.core.config["general"]["download_folder"] + self.collector = self.pyLoadCollector(self) + self.packager = self.pyLoadPackager(self) + + self.data = { + "version": LIST_VERSION, + "queue": [], + "packages": [], + "collector": [] + } + self.load() + + def load(self): + self.lock.acquire() + try: + pkl_file = open(join(self.core.configdir, "links.pkl"), "rb") + obj = cPickle.load(pkl_file) + except: + obj = False + if obj != False and obj["version"] == LIST_VERSION: + packages = [] + queue = [] + collector = [] + for n, pd in enumerate(obj["packages"]): + p = PyLoadPackage() + pd.get(p, self) + packages.append(p) + for pd in obj["queue"]: + p = PyLoadPackage() + pd.get(p, self) + queue.append(p) + for fd in obj["collector"]: + f = PyLoadFile("", self) + fd.get(f) + collector.append(f) + obj["packages"] = packages + obj["queue"] = queue + obj["collector"] = collector + self.data = obj + self.lock.release() + + if len(self.data["collector"]) > 0: + self.core.logger.info(_("Found %s links in linkcollector") % len(self.data["collector"])) + if len(self.data["packages"]) > 0: + self.core.logger.info(_("Found %s unqueued packages") % len(self.data["packages"])) + if len(self.data["queue"]) > 0: + self.core.logger.info(_("Added %s packages to queue") % len(self.data["queue"])) + + def save(self): + self.lock.acquire() + + pdata = { + "version": LIST_VERSION, + "queue": [], + "packages": [], + "collector": [] + } + + pdata["packages"] = [PyLoadPackageData().set(x) for x in self.data["packages"]] + pdata["queue"] = [PyLoadPackageData().set(x) for x in self.data["queue"]] + pdata["collector"] = [PyLoadFileData().set(x) for x in self.data["collector"]] + + output = open(join(self.core.configdir, "links.pkl"), "wb") + cPickle.dump(pdata, output, -1) + + self.lock.release() + + def queueEmpty(self): + return (self.data["queue"] == []) + + def getDownloadList(self, occ): + """ + for thread_list only, returns all elements that are suitable for downloadthread + """ + files = [] + files += [[x for x in p.files if x.status.type == None and x.plugin.props['type'] == "container" and not x.active] for p in self.data["queue"] + self.data["packages"]] + files += [[x for x in p.files if (x.status.type == None or x.status.type == "reconnected") and not x.active and not x.modul.__name__ in occ] for p in self.data["queue"]] + + return reduce(concat, files, []) + + def getAllFiles(self): + + return map(attrgetter("files"), self.data["queue"] + self.data["packages"]) + + def countDownloads(self): + """ simply return len of all files in all packages(which have no type) in queue and collector""" + return len(reduce(concat, [[x for x in p.files if x.status.type == None] for p in self.data["queue"] + self.data["packages"]], [])) + + def getFileInfo(self, id): + try: + n, pyfile = self.collector._getFileFromID(id) + except NoSuchElementException: + key, n, pyfile, pypack, pid = self.packager._getFileFromID(id) + info = {} + info["id"] = pyfile.id + info["url"] = pyfile.url + info["folder"] = pyfile.folder + info["filename"] = pyfile.status.filename + info["status_type"] = pyfile.status.type + info["status_url"] = pyfile.status.url + info["status_filename"] = pyfile.status.filename + info["status_error"] = pyfile.status.error + info["size"] = pyfile.status.size() + info["active"] = pyfile.active + info["plugin"] = pyfile.plugin.props['name'] + try: + info["package"] = pypack.data["id"] + except: + pass + return info + + def continueAborted(self): + [[self.packager.resetFileStatus(x.id) for x in p.files if x.status.type == "aborted"] for p in self.data["queue"]] + + class pyLoadCollector(): + def __init__(collector, file_list): + collector.file_list = file_list + + def _getFileFromID(collector, id): + """ + returns PyLoadFile instance and position in collector with given id + """ + for n, pyfile in enumerate(collector.file_list.data["collector"]): + if pyfile.id == id: + return (n, pyfile) + raise NoSuchElementException() + + def _getFreeID(collector): + """ + returns a free id + """ + ids = [] + for pypack in (collector.file_list.data["packages"] + collector.file_list.data["queue"]): + for pyf in pypack.files: + ids.append(pyf.id) + ids += map(attrgetter("id"), collector.file_list.data["collector"]) + id = 1 + while id in ids: + id += 1 + return id + + def getFile(collector, id): + """ + returns PyLoadFile instance from given id + """ + return collector._getFileFromID(id)[1] + + def popFile(collector, id): + """ + returns PyLoadFile instance given id and remove it from the collector + """ + collector.file_list.lock.acquire() + try: + n, pyfile = collector._getFileFromID(id) + del collector.file_list.data["collector"][n] + collector.file_list.core.pullManager.addEvent(RemoveEvent("file", id, "collector")) + except Exception, e: + raise Exception, e + else: + return pyfile + finally: + collector.file_list.lock.release() + + def addLink(collector, url): + """ + appends a new PyLoadFile instance to the end of the collector + """ + pyfile = PyLoadFile(url, collector.file_list) + pyfile.id = collector._getFreeID() + pyfile.folder = collector.file_list.download_folder + collector.file_list.lock.acquire() + collector.file_list.data["collector"].append(pyfile) + collector.file_list.lock.release() + collector.file_list.core.pullManager.addEvent(InsertEvent("file", pyfile.id, -2, "collector")) + return pyfile.id + + def removeFile(collector, id): + """ + removes PyLoadFile instance with the given id from collector + """ + collector.popFile(id) + collector.file_list.core.pullManager.addEvent(RemoveEvent("file", id, "collector")) + + def replaceFile(collector, newpyfile): + """ + replaces PyLoadFile instance with the given PyLoadFile instance at the given id + """ + collector.file_list.lock.acquire() + try: + n, pyfile = collector._getFileFromID(newpyfile.id) + collector.file_list.data["collector"][n] = newpyfile + collector.file_list.core.pullManager.addEvent(UpdateEvent("file", newpyfile.id, "collector")) + finally: + collector.file_list.lock.release() + + class pyLoadPackager(): + def __init__(packager, file_list): + packager.file_list = file_list + + def _getFreeID(packager): + """ + returns a free id + """ + ids = [pypack.data["id"] for pypack in packager.file_list.data["packages"] + packager.file_list.data["queue"]] + + id = 1 + while id in ids: + id += 1 + return id + + def _getPackageFromID(packager, id): + """ + returns PyLoadPackage instance and position with given id + """ + for n, pypack in enumerate(packager.file_list.data["packages"]): + if pypack.data["id"] == id: + return ("packages", n, pypack) + for n, pypack in enumerate(packager.file_list.data["queue"]): + if pypack.data["id"] == id: + return ("queue", n, pypack) + raise NoSuchElementException() + + def _getFileFromID(packager, id): + """ + returns PyLoadFile instance and position with given id + """ + for n, pypack in enumerate(packager.file_list.data["packages"]): + for pyfile in pypack.files: + if pyfile.id == id: + return ("packages", n, pyfile, pypack, pypack.data["id"]) + for n, pypack in enumerate(packager.file_list.data["queue"]): + for pyfile in pypack.files: + if pyfile.id == id: + return ("queue", n, pyfile, pypack, pypack.data["id"]) + raise NoSuchElementException() + + def addNewPackage(packager, package_name=None): + pypack = PyLoadPackage() + pypack.data["id"] = packager._getFreeID() + if package_name is not None: + pypack.data["package_name"] = package_name + packager.file_list.data["packages"].append(pypack) + packager.file_list.core.pullManager.addEvent(InsertEvent("pack", pypack.data["id"], -2, "packages")) + return pypack.data["id"] + + def removePackage(packager, id): + packager.file_list.lock.acquire() + try: + key, n, pypack = packager._getPackageFromID(id) + for pyfile in pypack.files: + pyfile.plugin.req.abort = True + sleep(0.1) + del packager.file_list.data[key][n] + packager.file_list.core.pullManager.addEvent(RemoveEvent("pack", id, key)) + finally: + packager.file_list.lock.release() + + def removeFile(packager, id): + """ + removes PyLoadFile instance with the given id from package + """ + packager.file_list.lock.acquire() + try: + key, n, pyfile, pypack, pid = packager._getFileFromID(id) + pyfile.plugin.req.abort = True + sleep(0.1) + packager.removeFileFromPackage(id, pid) + if not pypack.files: + packager.removePackage(pid) + finally: + packager.file_list.lock.release() + + def pushPackage2Queue(packager, id): + packager.file_list.lock.acquire() + try: + key, n, pypack = packager._getPackageFromID(id) + if key == "packages": + del packager.file_list.data["packages"][n] + packager.file_list.data["queue"].append(pypack) + packager.file_list.core.pullManager.addEvent(RemoveEvent("pack", id, "packages")) + packager.file_list.core.pullManager.addEvent(InsertEvent("pack", id, -2, "queue")) + finally: + packager.file_list.lock.release() + + def pullOutPackage(packager, id): + packager.file_list.lock.acquire() + try: + key, n, pypack = packager._getPackageFromID(id) + if key == "queue": + del packager.file_list.data["queue"][n] + packager.file_list.data["packages"].append(pypack) + packager.file_list.core.pullManager.addEvent(RemoveEvent("pack", id, "queue")) + packager.file_list.core.pullManager.addEvent(InsertEvent("pack", id, -2, "packages")) + finally: + packager.file_list.lock.release() + + def setPackageData(packager, id, package_name=None, folder=None): + packager.file_list.lock.acquire() + try: + key, n, pypack = packager._getPackageFromID(id) + if package_name is not None: + pypack.data["package_name"] = package_name + if folder is not None: + pypack.data["folder"] = folder + packager.file_list.data[key][n] = pypack + packager.file_list.core.pullManager.addEvent(UpdateEvent("pack", id, key)) + finally: + packager.file_list.lock.release() + + def getPackageData(packager, id): + key, n, pypack = packager._getPackageFromID(id) + return pypack.data + + def getPackageFiles(packager, id): + key, n, pypack = packager._getPackageFromID(id) + ids = map(attrgetter("id"), pypack.files) + + return ids + + def addFileToPackage(packager, id, pyfile): + key, n, pypack = packager._getPackageFromID(id) + pyfile.package = pypack + pypack.files.append(pyfile) + packager.file_list.data[key][n] = pypack + packager.file_list.core.pullManager.addEvent(InsertEvent("file", pyfile.id, -2, key)) + + def resetFileStatus(packager, fileid): + packager.file_list.lock.acquire() + try: + key, n, pyfile, pypack, pid = packager._getFileFromID(fileid) + pyfile.init() + pyfile.status.type = None + packager.file_list.core.pullManager.addEvent(UpdateEvent("file", fileid, key)) + finally: + packager.file_list.lock.release() + + def abortFile(packager, fileid): + packager.file_list.lock.acquire() + try: + key, n, pyfile, pypack, pid = packager._getFileFromID(fileid) + pyfile.plugin.req.abort = True + packager.file_list.core.pullManager.addEvent(UpdateEvent("file", fileid, key)) + finally: + packager.file_list.lock.release() + + def removeFileFromPackage(packager, id, pid): + key, n, pypack = packager._getPackageFromID(pid) + for k, pyfile in enumerate(pypack.files): + if id == pyfile.id: + del pypack.files[k] + packager.file_list.core.pullManager.addEvent(RemoveEvent("file", pyfile.id, key)) + if not pypack.files: + packager.removePackage(pid) + return True + raise NoSuchElementException() + +class PyLoadPackage(): + def __init__(self): + self.files = [] + self.data = { + "id": None, + "package_name": "new_package", + "folder": "" + } + +class PyLoadFile(): + def __init__(self, url, file_list): + self.id = None + self.url = url + self.folder = "" + self.file_list = file_list + self.core = file_list.core + self.package = None + self.filename = "n/a" + self.init() + + def init(self): + self.active = False + pluginName = self._get_my_plugin() + if pluginName: + for dir in ["hoster", "decrypter", "container"]: + try: + self.modul = __import__("%s.%s" % (dir, pluginName), globals(), locals(), [pluginName], -1) + except ImportError: + pass + pluginClass = getattr(self.modul, pluginName) + else: + self.modul = module.plugins.Plugin + pluginClass = module.plugins.Plugin.Plugin + self.plugin = pluginClass(self) + self.status = Status(self) + self.status.filename = self.url + + def _get_my_plugin(self): + for plugin, plugin_pattern in self.core.plugins_avaible.items(): + if re.match(plugin_pattern, self.url) != None: + return plugin + + def init_download(self): + if self.core.config['proxy']['activated']: + self.plugin.req.add_proxy(self.core.config['proxy']['protocol'], self.core.config['proxy']['adress']) + +class PyLoadFileData(): + def __init__(self): + self.id = None + self.url = None + self.folder = None + self.pack_id = None + self.filename = None + self.status_type = None + self.status_url = None + + def set(self, pyfile): + self.id = pyfile.id + self.url = pyfile.url + self.folder = pyfile.folder + self.parsePackage(pyfile.package) + self.filename = pyfile.filename + self.status_type = pyfile.status.type + self.status_url = pyfile.status.url + self.status_filename = pyfile.status.filename + self.status_error = pyfile.status.error + + return self + + def get(self, pyfile): + pyfile.id = self.id + pyfile.url = self.url + pyfile.folder = self.folder + pyfile.filename = self.filename + pyfile.status.type = self.status_type + pyfile.status.url = self.status_url + pyfile.status.filename = self.status_filename + pyfile.status.error = self.status_error + + def parsePackage(self, pack): + if pack: + self.pack_id = pack.data["id"] + +class PyLoadPackageData(): + def __init__(self): + self.data = None + self.files = [] + + def set(self, pypack): + self.data = pypack.data + self.files = [PyLoadFileData().set(x) for x in pypack.files] + return self + + def get(self, pypack, fl): + pypack.data = self.data + for fdata in self.files: + pyfile = PyLoadFile(fdata.url, fl) + fdata.get(pyfile) + pyfile.package = pypack + pypack.files.append(pyfile) diff --git a/module/Plugin.py b/module/Plugin.py deleted file mode 100644 index 1428e6235..000000000 --- a/module/Plugin.py +++ /dev/null @@ -1,185 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN, spoob, mkaay -""" - -import logging -import re -from os.path import exists -from os.path import join - -from time import sleep -import sys -from os.path import exists - -from module.network.Request import Request -from os import makedirs - -from module.download_thread import CaptchaError - -class Plugin(): - - def __init__(self, parent): - self.configparser = parent.core.parser_plugins - self.config = {} - props = {} - props['name'] = "BasePlugin" - props['version'] = "0.3" - props['pattern'] = None - props['type'] = "hoster" - props['description'] = """Base Plugin""" - props['author_name'] = ("RaNaN", "spoob", "mkaay") - props['author_mail'] = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de") - self.props = props - self.parent = parent - self.req = Request() - self.html = 0 - self.time_plus_wait = 0 #time() + wait in seconds - self.want_reconnect = False - self.multi_dl = True - self.ocr = None #captcha reader instance - self.logger = logging.getLogger("log") - self.decryptNow = True - self.pyfile = self.parent - - def prepare(self, thread): - self.want_reconnect = False - self.pyfile.status.exists = self.file_exists() - - if not self.pyfile.status.exists: - return False - - self.pyfile.status.filename = self.get_file_name() - self.pyfile.status.waituntil = self.time_plus_wait - self.pyfile.status.url = self.get_file_url() - self.pyfile.status.want_reconnect = self.want_reconnect - thread.wait(self.parent) - - return True - - def set_parent_status(self): - """ sets all available Statusinfos about a File in self.parent.status - """ - pass - - def download_html(self): - """ gets the url from self.parent.url saves html in self.html and parses - """ - self.html = "" - - def file_exists(self): - """ returns True or False - """ - if re.search(r"(?!http://).*\.(dlc|ccf|rsdf|txt)", self.parent.url): - return exists(self.parent.url) - header = self.load(self.parent.url, just_header=True) - try: - if re.search(r"HTTP/1.1 404 Not Found", header): - return False - except: - pass - return True - - def get_file_url(self): - """ returns the absolute downloadable filepath - """ - return self.parent.url - - def get_file_name(self): - try: - return re.findall("([^\/=]+)", self.parent.url)[-1] - except: - return self.parent.url[:20] - - def wait_until(self): - if self.html != None: - self.download_html() - return self.time_plus_wait - - def proceed(self, url, location): - self.download(url, location) - - def set_config(self): - for k, v in self.config.items(): - self.configparser.set(self.props['name'], {"option": k}, v) - - def remove_config(self, option): - self.configparser.remove(self.props['name'], option) - - def get_config(self, value, default=None): - self.configparser.loadData() - return self.configparser.get(self.props['name'], value, default=default) - - def read_config(self): - self.configparser.loadData() - try: - self.verify_config() - self.config = self.configparser.getConfig()[self.props['name']] - except: - pass - - def verify_config(self): - pass - - def init_ocr(self): - modul = __import__("module.plugins.captcha." + self.props['name'], fromlist=['captcha']) - captchaClass = getattr(modul, self.props['name']) - self.ocr = captchaClass() - - def __call__(self): - return self.props['name'] - - def check_file(self, local_file): - """ - return codes: - 0 - checksum ok - 1 - checksum wrong - 5 - can't get checksum - 10 - not implemented - 20 - unknown error - """ - return (True, 10) - - def waitForCaptcha(self, captchaData, imgType): - captchaManager = self.parent.core.captchaManager - task = captchaManager.newTask(self) - task.setCaptcha(captchaData, imgType) - task.setWaiting() - while not task.getStatus() == "done": - if not self.parent.core.isGUIConnected(): - task.removeTask() - raise CaptchaError - sleep(1) - result = task.getResult() - task.removeTask() - return result - - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False): - return self.req.load(url, get, post, ref, cookies, just_header) - - def download(self, url, file_name, get={}, post={}, ref=True, cookies=True): - download_folder = self.parent.core.config['general']['download_folder'] - if self.pyfile.package.data["package_name"] != (self.parent.core.config['general']['link_file']) and self.parent.core.xmlconfig.get("general", "folder_per_package", False): - self.pyfile.folder = self.pyfile.package.data["package_name"] - location = join(download_folder, self.pyfile.folder.decode(sys.getfilesystemencoding())) - if not exists(location): makedirs(location) - file_path = join(location.decode(sys.getfilesystemencoding()), self.pyfile.status.filename.decode(sys.getfilesystemencoding())) - else: - file_path = join(download_folder, self.pyfile.status.filename.decode(sys.getfilesystemencoding())) - file_path = join(download_folder, self.pyfile.status.filename.decode(sys.getfilesystemencoding())) - - self.pyfile.status.filename = self.req.download(url, file_path, get, post, ref, cookies) diff --git a/module/ThreadManager.py b/module/ThreadManager.py new file mode 100644 index 000000000..ab0f99cfa --- /dev/null +++ b/module/ThreadManager.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay + @author: spoob + @author: sebnapi + @author: RaNaN + @version: v0.3.2 +""" + +from __future__ import with_statement +from os.path import exists +import re +import subprocess +from threading import RLock, Thread +from time import sleep +from module.network.Request import getURL +from module.DownloadThread import DownloadThread +from module.SpeedManager import SpeedManager + +class ThreadManager(Thread): + def __init__(self, parent): + Thread.__init__(self) + self.parent = parent + self.list = parent.file_list #file list + self.threads = [] + self.lock = RLock() + self.py_downloading = [] # files downloading + self.occ_plugins = [] #occupied plugins + self.pause = True + self.reconnecting = False + + self.speedManager = SpeedManager(self) + + def run(self): + while True: + if len(self.threads) < int(self.parent.config['general']['max_downloads']) and not self.pause: + job = self.getJob() + if job: + thread = self.createThread(job) + thread.start() + sleep(1) + + def createThread(self, job): + """ creates thread for Py_Load_File and append thread to self.threads + """ + thread = DownloadThread(self, job) + self.threads.append(thread) + return thread + + def removeThread(self, thread): + self.threads.remove(thread) + + def getJob(self): + """return job if suitable, otherwise send thread idle""" + + if not self.parent.server_methods.is_time_download() or self.pause or self.reconnecting or self.list.queueEmpty(): #conditions when threads dont download + return None + + if self.parent.freeSpace() < self.parent.config["general"]["min_free_space"]: + self.parent.logger.debug(_("minimal free space exceeded")) + return None + + self.initReconnect() + + self.lock.acquire() + + pyfile = None + pyfiles = self.list.getDownloadList(self.occ_plugins) + + if pyfiles: + pyfile = pyfiles[0] + self.py_downloading.append(pyfile) + self.parent.hookManager.downloadStarts(pyfile) + if not pyfile.plugin.multi_dl: + self.occ_plugins.append(pyfile.modul.__name__) + pyfile.active = True + if pyfile.plugin.props['type'] == "container": + self.parent.logger.info(_("Get links from: %s") % pyfile.url) + else: + self.parent.logger.info(_("Download starts: %s") % pyfile.url) + + self.lock.release() + return pyfile + + def jobFinished(self, pyfile): + """manage completing download""" + self.lock.acquire() + + if not pyfile.plugin.multi_dl: + self.occ_plugins.remove(pyfile.modul.__name__) + + pyfile.active = False + + if not pyfile.status == "reconnected": + try: + pyfile.plugin.req.pycurl.close() + except: + pass + + self.py_downloading.remove(pyfile) + + if pyfile.status.type == "finished": + if pyfile.plugin.props['type'] == "container": + newLinks = 0 + if pyfile.plugin.links: + if isinstance(pyfile.plugin.links, dict): + packmap = {} + for packname in pyfile.plugin.links.keys(): + packmap[packname] = self.list.packager.addNewPackage(packname) + for packname, links in pyfile.plugin.links.items(): + pid = packmap[packname] + for link in links: + newFile = self.list.collector.addLink(link) + self.list.packager.addFileToPackage(pid, self.list.collector.popFile(newFile)) + newLinks += 1 + else: + for link in pyfile.plugin.links: + newFile = self.list.collector.addLink(link) + self.list.packager.addFileToPackage(pyfile.package.data["id"], self.list.collector.popFile(newFile)) + newLinks += 1 + #self.list.packager.pushPackage2Queue(pyfile.package.data["id"]) + self.list.packager.removeFileFromPackage(pyfile.id, pyfile.package.data["id"]) + + if newLinks: + self.parent.logger.info(_("Parsed links from %s: %i") % (pyfile.status.filename, newLinks)) + else: + self.parent.logger.info(_("No links in %s") % pyfile.status.filename) + #~ self.list.packager.removeFileFromPackage(pyfile.id, pyfile.package.id) + #~ for link in pyfile.plugin.links: + #~ id = self.list.collector.addLink(link) + #~ pyfile.packager.pullOutPackage(pyfile.package.id) + #~ pyfile.packager.addFileToPackage(pyfile.package.id, pyfile.collector.popFile(id)) + else: + self.parent.logger.info(_("Download finished: %s") % pyfile.url) + + elif pyfile.status.type == "reconnected": + pyfile.plugin.req.init_curl() + + elif pyfile.status.type == "failed": + self.parent.logger.warning(_("Download failed: %s | %s") % (pyfile.url, pyfile.status.error)) + with open(self.parent.config['general']['failed_file'], 'a') as f: + f.write(pyfile.url + "\n") + + elif pyfile.status.type == "aborted": + self.parent.logger.info(_("Download aborted: %s") % pyfile.url) + + self.list.save() + + self.parent.hookManager.downloadFinished(pyfile) + + self.lock.release() + return True + + def initReconnect(self): + """initialise a reonnect""" + if not self.parent.config['reconnect']['activated'] or self.reconnecting or not self.parent.server_methods.is_time_reconnect(): + return False + + if not exists(self.parent.config['reconnect']['method']): + self.parent.logger.info(self.parent.config['reconnect']['method'] + " not found") + self.parent.config['reconnect']['activated'] = False + return False + + self.lock.acquire() + + if self.checkReconnect(): + self.reconnecting = True + self.reconnect() + time.sleep(1.1) + + self.reconnecting = False + self.lock.release() + return True + + self.lock.release() + return False + + def checkReconnect(self): + """checks if all files want reconnect""" + + if not self.py_downloading: + return False + + i = 0 + for obj in self.py_downloading: + if obj.status.want_reconnect: + i += 1 + + if len(self.py_downloading) == i: + return True + else: + return False + + def reconnect(self): + self.parent.logger.info(_("Starting reconnect")) + ip = re.match(".*Current IP Address: (.*).*", getURL("http://checkip.dyndns.org/")).group(1) + self.parent.hookManager.beforeReconnecting(ip) + reconn = subprocess.Popen(self.parent.config['reconnect']['method'])#, stdout=subprocess.PIPE) + reconn.wait() + time.sleep(1) + ip = "" + while ip == "": + try: + ip = re.match(".*Current IP Address: (.*).*", getURL("http://checkip.dyndns.org/")).group(1) #versuchen neue ip aus zu lesen + except: + ip = "" + time.sleep(1) + self.parent.hookManager.afterReconnecting(ip) + self.parent.logger.info(_("Reconnected, new IP: %s") % ip) + + def stopAllDownloads(self): + self.pause = True + for pyfile in self.py_downloading: + pyfile.plugin.req.abort = True diff --git a/module/download_thread.py b/module/download_thread.py deleted file mode 100644 index 84f00c99f..000000000 --- a/module/download_thread.py +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay - @author: spoob - @author: sebnapi - @version: v0.3.1 -""" - -import threading -import traceback -from time import sleep, time - -from module.network.Request import AbortDownload -from module.PullEvents import UpdateEvent - - - -class Status(object): - """ Saves all status information - """ - def __init__(self, pyfile): - self.pyfile = pyfile - self.type = None - self.status_queue = None - self.filename = None - self.url = None - self.exists = False - self.waituntil = 0 - self.plugin = pyfile.modul.__name__ - self.want_reconnect = False - self.error = "" - - def get_ETA(self): - return self.pyfile.plugin.req.get_ETA() - def get_speed(self): - return self.pyfile.plugin.req.get_speed() - def kB_left(self): - return self.pyfile.plugin.req.kB_left() - def size(self): - return self.pyfile.plugin.req.dl_size / 1024 - def percent(self): - if not self.kB_left() == 0 and not self.size() == 0: - percent = ((self.size()-self.kB_left()) * 100) / self.size() - return percent if percent < 101 else 0 - return 0 - -class Reconnect(Exception): - pass - -class Checksum(Exception): - def __init__(self, code, local_file): - self.code = code - self.file = local_file - - def getCode(self): - return self.code - - def getFile(self): - return self.file - -class CaptchaError(Exception): - pass - -class Download_Thread(threading.Thread): - def __init__(self, parent): - threading.Thread.__init__(self) - self.shutdown = False - self.parent = parent - self.setDaemon(True) - self.loadedPyFile = None - - self.start() - - def run(self): - while not self.shutdown: - self.loadedPyFile = self.parent.get_job() - if self.loadedPyFile: - try: - self.download(self.loadedPyFile) - except AbortDownload: - self.loadedPyFile.plugin.req.abort = False - self.loadedPyFile.status.type = "aborted" - except Reconnect: - pass - except Checksum, e: - self.loadedPyFile.status.type = "failed" - self.loadedPyFile.status.error = "Checksum error: %d" % e.getCode() - f = open("%s.info" % e.getFile(), "w") - f.write("Checksum not matched!") - f.close() - except CaptchaError: - self.loadedPyFile.status.type = "failed" - self.loadedPyFile.status.error = "Can't solve captcha" - except Exception, e: - try: - if self.parent.parent.config['general']['debug_mode']: - traceback.print_exc() - code, msg = e - if code == 7: - sleep(60) - self.parent.parent.logger.info(_("Hoster unvailable, wait 60 seconds")) - except Exception, f: - self.parent.parent.logger.debug(_("Error getting error code: %s") % f) - if self.parent.parent.config['general']['debug_mode']: - traceback.print_exc() - self.loadedPyFile.status.type = "failed" - self.loadedPyFile.status.error = str(e) - finally: - self.parent.job_finished(self.loadedPyFile) - self.parent.parent.pullManager.addEvent(UpdateEvent("file", self.loadedPyFile.id, "queue")) - else: - sleep(3) - sleep(0.8) - if self.shutdown: - sleep(1) - self.parent.remove_thread(self) - - def download(self, pyfile): - status = pyfile.status - status.type = "starting" - self.parent.parent.pullManager.addEvent(UpdateEvent("file", pyfile.id, "queue")) - - pyfile.init_download() - - if not pyfile.plugin.prepare(self): - raise Exception, _("File not found") - - pyfile.plugin.req.set_timeout(self.parent.parent.config['general']['max_download_time']) - - if pyfile.plugin.props["type"] == "container": - status.type = "decrypting" - else: - status.type = "downloading" - self.parent.parent.pullManager.addEvent(UpdateEvent("file", pyfile.id, "queue")) - - - #~ free_file_name = self.get_free_name(status.filename) - #~ location = join(pyfile.folder, status.filename) - pyfile.plugin.proceed(status.url, status.filename) - - if self.parent.parent.xmlconfig.get("general", "checksum", True): - status.type = "checking" - check, code = pyfile.plugin.check_file(status.filename) - """ - return codes: - 0 - checksum ok - 1 - checksum wrong - 5 - can't get checksum - 10 - not implemented - 20 - unknown error - """ - if code == 0: - self.parent.parent.logger.info(_("Checksum ok ('%s')") % status.filename) - elif code == 1: - self.parent.parent.logger.info(_("Checksum not matched! ('%s')") % status.filename) - elif code == 5: - self.parent.parent.logger.debug(_("Can't get checksum for %s") % status.filename) - elif code == 10: - self.parent.parent.logger.debug(_("Checksum not implemented for %s") % status.filename) - if not check: - raise Checksum(code, location) - - status.type = "finished" - - def wait(self, pyfile): - pyfile.status.type = "waiting" - while (time() < pyfile.status.waituntil): - if self.parent.init_reconnect() or self.parent.reconnecting: - pyfile.status.type = "reconnected" - pyfile.status.want_reconnect = False - raise Reconnect - if pyfile.plugin.req.abort: - raise AbortDownload - sleep(1) - pyfile.status.want_reconnect = False - return True diff --git a/module/file_list.py b/module/file_list.py deleted file mode 100644 index eff96fa9e..000000000 --- a/module/file_list.py +++ /dev/null @@ -1,502 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay - @author: RaNaN - @version: v0.3.1 - @list-version: v4 -""" - -LIST_VERSION = 4 - -from operator import attrgetter -from operator import concat -from os.path import join -import re -from threading import RLock -from time import sleep - -import cPickle -from download_thread import Status -import module.Plugin -from module.PullEvents import InsertEvent -from module.PullEvents import RemoveEvent -from module.PullEvents import UpdateEvent - -class NoSuchElementException(Exception): - pass - -class File_List(object): - def __init__(self, core): - self.core = core - self.lock = RLock() - self.download_folder = self.core.config["general"]["download_folder"] - self.collector = self.pyLoadCollector(self) - self.packager = self.pyLoadPackager(self) - - self.data = { - "version": LIST_VERSION, - "queue": [], - "packages": [], - "collector": [] - } - self.load() - - def load(self): - self.lock.acquire() - try: - pkl_file = open(join(self.core.configdir, "links.pkl"), "rb") - obj = cPickle.load(pkl_file) - except: - obj = False - if obj != False and obj["version"] == LIST_VERSION: - packages = [] - queue = [] - collector = [] - for n, pd in enumerate(obj["packages"]): - p = PyLoadPackage() - pd.get(p, self) - packages.append(p) - for pd in obj["queue"]: - p = PyLoadPackage() - pd.get(p, self) - queue.append(p) - for fd in obj["collector"]: - f = PyLoadFile("", self) - fd.get(f) - collector.append(f) - obj["packages"] = packages - obj["queue"] = queue - obj["collector"] = collector - self.data = obj - self.lock.release() - - if len(self.data["collector"]) > 0: - self.core.logger.info(_("Found %s links in linkcollector") % len(self.data["collector"])) - if len(self.data["packages"]) > 0: - self.core.logger.info(_("Found %s unqueued packages") % len(self.data["packages"])) - if len(self.data["queue"]) > 0: - self.core.logger.info(_("Added %s packages to queue") % len(self.data["queue"])) - - def save(self): - self.lock.acquire() - - pdata = { - "version": LIST_VERSION, - "queue": [], - "packages": [], - "collector": [] - } - - pdata["packages"] = [PyLoadPackageData().set(x) for x in self.data["packages"]] - pdata["queue"] = [PyLoadPackageData().set(x) for x in self.data["queue"]] - pdata["collector"] = [PyLoadFileData().set(x) for x in self.data["collector"]] - - output = open(join(self.core.configdir, "links.pkl"), "wb") - cPickle.dump(pdata, output, -1) - - self.lock.release() - - def queueEmpty(self): - return (self.data["queue"] == []) - - def getDownloadList(self, occ): - """ - for thread_list only, returns all elements that are suitable for downloadthread - """ - files = [] - files += [[x for x in p.files if x.status.type == None and x.plugin.props['type'] == "container" and not x.active] for p in self.data["queue"] + self.data["packages"]] - files += [[x for x in p.files if (x.status.type == None or x.status.type == "reconnected") and not x.active and not x.modul.__name__ in occ] for p in self.data["queue"]] - - return reduce(concat, files, []) - - def getAllFiles(self): - - return map(attrgetter("files"), self.data["queue"] + self.data["packages"]) - - def countDownloads(self): - """ simply return len of all files in all packages(which have no type) in queue and collector""" - return len(reduce(concat, [[x for x in p.files if x.status.type == None] for p in self.data["queue"] + self.data["packages"]], [])) - - def getFileInfo(self, id): - try: - n, pyfile = self.collector._getFileFromID(id) - except NoSuchElementException: - key, n, pyfile, pypack, pid = self.packager._getFileFromID(id) - info = {} - info["id"] = pyfile.id - info["url"] = pyfile.url - info["folder"] = pyfile.folder - info["filename"] = pyfile.status.filename - info["status_type"] = pyfile.status.type - info["status_url"] = pyfile.status.url - info["status_filename"] = pyfile.status.filename - info["status_error"] = pyfile.status.error - info["size"] = pyfile.status.size() - info["active"] = pyfile.active - info["plugin"] = pyfile.plugin.props['name'] - try: - info["package"] = pypack.data["id"] - except: - pass - return info - - def continueAborted(self): - [[self.packager.resetFileStatus(x.id) for x in p.files if x.status.type == "aborted"] for p in self.data["queue"]] - - class pyLoadCollector(): - def __init__(collector, file_list): - collector.file_list = file_list - - def _getFileFromID(collector, id): - """ - returns PyLoadFile instance and position in collector with given id - """ - for n, pyfile in enumerate(collector.file_list.data["collector"]): - if pyfile.id == id: - return (n, pyfile) - raise NoSuchElementException() - - def _getFreeID(collector): - """ - returns a free id - """ - ids = [] - for pypack in (collector.file_list.data["packages"] + collector.file_list.data["queue"]): - for pyf in pypack.files: - ids.append(pyf.id) - ids += map(attrgetter("id"), collector.file_list.data["collector"]) - id = 1 - while id in ids: - id += 1 - return id - - def getFile(collector, id): - """ - returns PyLoadFile instance from given id - """ - return collector._getFileFromID(id)[1] - - def popFile(collector, id): - """ - returns PyLoadFile instance given id and remove it from the collector - """ - collector.file_list.lock.acquire() - try: - n, pyfile = collector._getFileFromID(id) - del collector.file_list.data["collector"][n] - collector.file_list.core.pullManager.addEvent(RemoveEvent("file", id, "collector")) - except Exception, e: - raise Exception, e - else: - return pyfile - finally: - collector.file_list.lock.release() - - def addLink(collector, url): - """ - appends a new PyLoadFile instance to the end of the collector - """ - pyfile = PyLoadFile(url, collector.file_list) - pyfile.id = collector._getFreeID() - pyfile.folder = collector.file_list.download_folder - collector.file_list.lock.acquire() - collector.file_list.data["collector"].append(pyfile) - collector.file_list.lock.release() - collector.file_list.core.pullManager.addEvent(InsertEvent("file", pyfile.id, -2, "collector")) - return pyfile.id - - def removeFile(collector, id): - """ - removes PyLoadFile instance with the given id from collector - """ - collector.popFile(id) - collector.file_list.core.pullManager.addEvent(RemoveEvent("file", id, "collector")) - - def replaceFile(collector, newpyfile): - """ - replaces PyLoadFile instance with the given PyLoadFile instance at the given id - """ - collector.file_list.lock.acquire() - try: - n, pyfile = collector._getFileFromID(newpyfile.id) - collector.file_list.data["collector"][n] = newpyfile - collector.file_list.core.pullManager.addEvent(UpdateEvent("file", newpyfile.id, "collector")) - finally: - collector.file_list.lock.release() - - class pyLoadPackager(): - def __init__(packager, file_list): - packager.file_list = file_list - - def _getFreeID(packager): - """ - returns a free id - """ - ids = [pypack.data["id"] for pypack in packager.file_list.data["packages"] + packager.file_list.data["queue"]] - - id = 1 - while id in ids: - id += 1 - return id - - def _getPackageFromID(packager, id): - """ - returns PyLoadPackage instance and position with given id - """ - for n, pypack in enumerate(packager.file_list.data["packages"]): - if pypack.data["id"] == id: - return ("packages", n, pypack) - for n, pypack in enumerate(packager.file_list.data["queue"]): - if pypack.data["id"] == id: - return ("queue", n, pypack) - raise NoSuchElementException() - - def _getFileFromID(packager, id): - """ - returns PyLoadFile instance and position with given id - """ - for n, pypack in enumerate(packager.file_list.data["packages"]): - for pyfile in pypack.files: - if pyfile.id == id: - return ("packages", n, pyfile, pypack, pypack.data["id"]) - for n, pypack in enumerate(packager.file_list.data["queue"]): - for pyfile in pypack.files: - if pyfile.id == id: - return ("queue", n, pyfile, pypack, pypack.data["id"]) - raise NoSuchElementException() - - def addNewPackage(packager, package_name=None): - pypack = PyLoadPackage() - pypack.data["id"] = packager._getFreeID() - if package_name is not None: - pypack.data["package_name"] = package_name - packager.file_list.data["packages"].append(pypack) - packager.file_list.core.pullManager.addEvent(InsertEvent("pack", pypack.data["id"], -2, "packages")) - return pypack.data["id"] - - def removePackage(packager, id): - packager.file_list.lock.acquire() - try: - key, n, pypack = packager._getPackageFromID(id) - for pyfile in pypack.files: - pyfile.plugin.req.abort = True - sleep(0.1) - del packager.file_list.data[key][n] - packager.file_list.core.pullManager.addEvent(RemoveEvent("pack", id, key)) - finally: - packager.file_list.lock.release() - - def removeFile(packager, id): - """ - removes PyLoadFile instance with the given id from package - """ - packager.file_list.lock.acquire() - try: - key, n, pyfile, pypack, pid = packager._getFileFromID(id) - pyfile.plugin.req.abort = True - sleep(0.1) - packager.removeFileFromPackage(id, pid) - if not pypack.files: - packager.removePackage(pid) - finally: - packager.file_list.lock.release() - - def pushPackage2Queue(packager, id): - packager.file_list.lock.acquire() - try: - key, n, pypack = packager._getPackageFromID(id) - if key == "packages": - del packager.file_list.data["packages"][n] - packager.file_list.data["queue"].append(pypack) - packager.file_list.core.pullManager.addEvent(RemoveEvent("pack", id, "packages")) - packager.file_list.core.pullManager.addEvent(InsertEvent("pack", id, -2, "queue")) - finally: - packager.file_list.lock.release() - - def pullOutPackage(packager, id): - packager.file_list.lock.acquire() - try: - key, n, pypack = packager._getPackageFromID(id) - if key == "queue": - del packager.file_list.data["queue"][n] - packager.file_list.data["packages"].append(pypack) - packager.file_list.core.pullManager.addEvent(RemoveEvent("pack", id, "queue")) - packager.file_list.core.pullManager.addEvent(InsertEvent("pack", id, -2, "packages")) - finally: - packager.file_list.lock.release() - - def setPackageData(packager, id, package_name=None, folder=None): - packager.file_list.lock.acquire() - try: - key, n, pypack = packager._getPackageFromID(id) - if package_name is not None: - pypack.data["package_name"] = package_name - if folder is not None: - pypack.data["folder"] = folder - packager.file_list.data[key][n] = pypack - packager.file_list.core.pullManager.addEvent(UpdateEvent("pack", id, key)) - finally: - packager.file_list.lock.release() - - def getPackageData(packager, id): - key, n, pypack = packager._getPackageFromID(id) - return pypack.data - - def getPackageFiles(packager, id): - key, n, pypack = packager._getPackageFromID(id) - ids = map(attrgetter("id"), pypack.files) - - return ids - - def addFileToPackage(packager, id, pyfile): - key, n, pypack = packager._getPackageFromID(id) - pyfile.package = pypack - pypack.files.append(pyfile) - packager.file_list.data[key][n] = pypack - packager.file_list.core.pullManager.addEvent(InsertEvent("file", pyfile.id, -2, key)) - - def resetFileStatus(packager, fileid): - packager.file_list.lock.acquire() - try: - key, n, pyfile, pypack, pid = packager._getFileFromID(fileid) - pyfile.init() - pyfile.status.type = None - packager.file_list.core.pullManager.addEvent(UpdateEvent("file", fileid, key)) - finally: - packager.file_list.lock.release() - - def abortFile(packager, fileid): - packager.file_list.lock.acquire() - try: - key, n, pyfile, pypack, pid = packager._getFileFromID(fileid) - pyfile.plugin.req.abort = True - packager.file_list.core.pullManager.addEvent(UpdateEvent("file", fileid, key)) - finally: - packager.file_list.lock.release() - - def removeFileFromPackage(packager, id, pid): - key, n, pypack = packager._getPackageFromID(pid) - for k, pyfile in enumerate(pypack.files): - if id == pyfile.id: - del pypack.files[k] - packager.file_list.core.pullManager.addEvent(RemoveEvent("file", pyfile.id, key)) - if not pypack.files: - packager.removePackage(pid) - return True - raise NoSuchElementException() - -class PyLoadPackage(): - def __init__(self): - self.files = [] - self.data = { - "id": None, - "package_name": "new_package", - "folder": "" - } - -class PyLoadFile(): - def __init__(self, url, file_list): - self.id = None - self.url = url - self.folder = "" - self.file_list = file_list - self.core = file_list.core - self.package = None - self.filename = "n/a" - self.init() - - def init(self): - self.active = False - pluginName = self._get_my_plugin() - if pluginName: - for dir in ["hoster", "decrypter", "container"]: - try: - self.modul = __import__("%s.%s" % (dir, pluginName), globals(), locals(), [pluginName], -1) - except ImportError: - pass - pluginClass = getattr(self.modul, pluginName) - else: - self.modul = module.Plugin - pluginClass = module.Plugin.Plugin - self.plugin = pluginClass(self) - self.status = Status(self) - self.status.filename = self.url - - def _get_my_plugin(self): - for plugin, plugin_pattern in self.core.plugins_avaible.items(): - if re.match(plugin_pattern, self.url) != None: - return plugin - - def init_download(self): - if self.core.config['proxy']['activated']: - self.plugin.req.add_proxy(self.core.config['proxy']['protocol'], self.core.config['proxy']['adress']) - -class PyLoadFileData(): - def __init__(self): - self.id = None - self.url = None - self.folder = None - self.pack_id = None - self.filename = None - self.status_type = None - self.status_url = None - - def set(self, pyfile): - self.id = pyfile.id - self.url = pyfile.url - self.folder = pyfile.folder - self.parsePackage(pyfile.package) - self.filename = pyfile.filename - self.status_type = pyfile.status.type - self.status_url = pyfile.status.url - self.status_filename = pyfile.status.filename - self.status_error = pyfile.status.error - - return self - - def get(self, pyfile): - pyfile.id = self.id - pyfile.url = self.url - pyfile.folder = self.folder - pyfile.filename = self.filename - pyfile.status.type = self.status_type - pyfile.status.url = self.status_url - pyfile.status.filename = self.status_filename - pyfile.status.error = self.status_error - - def parsePackage(self, pack): - if pack: - self.pack_id = pack.data["id"] - -class PyLoadPackageData(): - def __init__(self): - self.data = None - self.files = [] - - def set(self, pypack): - self.data = pypack.data - self.files = [PyLoadFileData().set(x) for x in pypack.files] - return self - - def get(self, pypack, fl): - pypack.data = self.data - for fdata in self.files: - pyfile = PyLoadFile(fdata.url, fl) - fdata.get(pyfile) - pyfile.package = pypack - pypack.files.append(pyfile) diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py new file mode 100644 index 000000000..c4ac8ee12 --- /dev/null +++ b/module/plugins/Plugin.py @@ -0,0 +1,185 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN, spoob, mkaay +""" + +import logging +import re +from os.path import exists +from os.path import join + +from time import sleep +import sys +from os.path import exists + +from module.network.Request import Request +from os import makedirs + +from module.DownloadThread import CaptchaError + +class Plugin(): + + def __init__(self, parent): + self.configparser = parent.core.parser_plugins + self.config = {} + props = {} + props['name'] = "BasePlugin" + props['version'] = "0.3" + props['pattern'] = None + props['type'] = "hoster" + props['description'] = """Base Plugin""" + props['author_name'] = ("RaNaN", "spoob", "mkaay") + props['author_mail'] = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de") + self.props = props + self.parent = parent + self.req = Request() + self.html = 0 + self.time_plus_wait = 0 #time() + wait in seconds + self.want_reconnect = False + self.multi_dl = True + self.ocr = None #captcha reader instance + self.logger = logging.getLogger("log") + self.decryptNow = True + self.pyfile = self.parent + + def prepare(self, thread): + self.want_reconnect = False + self.pyfile.status.exists = self.file_exists() + + if not self.pyfile.status.exists: + return False + + self.pyfile.status.filename = self.get_file_name() + self.pyfile.status.waituntil = self.time_plus_wait + self.pyfile.status.url = self.get_file_url() + self.pyfile.status.want_reconnect = self.want_reconnect + thread.wait(self.parent) + + return True + + def set_parent_status(self): + """ sets all available Statusinfos about a File in self.parent.status + """ + pass + + def download_html(self): + """ gets the url from self.parent.url saves html in self.html and parses + """ + self.html = "" + + def file_exists(self): + """ returns True or False + """ + if re.search(r"(?!http://).*\.(dlc|ccf|rsdf|txt)", self.parent.url): + return exists(self.parent.url) + header = self.load(self.parent.url, just_header=True) + try: + if re.search(r"HTTP/1.1 404 Not Found", header): + return False + except: + pass + return True + + def get_file_url(self): + """ returns the absolute downloadable filepath + """ + return self.parent.url + + def get_file_name(self): + try: + return re.findall("([^\/=]+)", self.parent.url)[-1] + except: + return self.parent.url[:20] + + def wait_until(self): + if self.html != None: + self.download_html() + return self.time_plus_wait + + def proceed(self, url, location): + self.download(url, location) + + def set_config(self): + for k, v in self.config.items(): + self.configparser.set(self.props['name'], {"option": k}, v) + + def remove_config(self, option): + self.configparser.remove(self.props['name'], option) + + def get_config(self, value, default=None): + self.configparser.loadData() + return self.configparser.get(self.props['name'], value, default=default) + + def read_config(self): + self.configparser.loadData() + try: + self.verify_config() + self.config = self.configparser.getConfig()[self.props['name']] + except: + pass + + def verify_config(self): + pass + + def init_ocr(self): + modul = __import__("module.plugins.captcha." + self.props['name'], fromlist=['captcha']) + captchaClass = getattr(modul, self.props['name']) + self.ocr = captchaClass() + + def __call__(self): + return self.props['name'] + + def check_file(self, local_file): + """ + return codes: + 0 - checksum ok + 1 - checksum wrong + 5 - can't get checksum + 10 - not implemented + 20 - unknown error + """ + return (True, 10) + + def waitForCaptcha(self, captchaData, imgType): + captchaManager = self.parent.core.captchaManager + task = captchaManager.newTask(self) + task.setCaptcha(captchaData, imgType) + task.setWaiting() + while not task.getStatus() == "done": + if not self.parent.core.isGUIConnected(): + task.removeTask() + raise CaptchaError + sleep(1) + result = task.getResult() + task.removeTask() + return result + + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False): + return self.req.load(url, get, post, ref, cookies, just_header) + + def download(self, url, file_name, get={}, post={}, ref=True, cookies=True): + download_folder = self.parent.core.config['general']['download_folder'] + if self.pyfile.package.data["package_name"] != (self.parent.core.config['general']['link_file']) and self.parent.core.xmlconfig.get("general", "folder_per_package", False): + self.pyfile.folder = self.pyfile.package.data["package_name"] + location = join(download_folder, self.pyfile.folder.decode(sys.getfilesystemencoding())) + if not exists(location): makedirs(location) + file_path = join(location.decode(sys.getfilesystemencoding()), self.pyfile.status.filename.decode(sys.getfilesystemencoding())) + else: + file_path = join(download_folder, self.pyfile.status.filename.decode(sys.getfilesystemencoding())) + file_path = join(download_folder, self.pyfile.status.filename.decode(sys.getfilesystemencoding())) + + self.pyfile.status.filename = self.req.download(url, file_path, get, post, ref, cookies) diff --git a/module/thread_list.py b/module/thread_list.py deleted file mode 100644 index 8f02536de..000000000 --- a/module/thread_list.py +++ /dev/null @@ -1,310 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay - @author: spoob - @author: sebnapi - @author: RaNaN - @version: v0.3.1 -""" - -from __future__ import with_statement -from os.path import exists -import re -import subprocess -from threading import RLock, Thread -import time -import urllib2 -from download_thread import Download_Thread - -class Thread_List(object): - def __init__(self, parent): - self.parent = parent - self.list = parent.file_list #file list - self.threads = [] - self.max_threads = int(self.parent.config['general']['max_downloads']) - self.lock = RLock() - self.py_downloading = [] # files downloading - self.occ_plugins = [] #occupied plugins - self.pause = True - self.reconnecting = False - - self.select_thread() - if self.parent.config['general']['download_speed_limit'] != 0: - self.speedManager = self.SpeedManager(self) - - def create_thread(self): - """ creates thread for Py_Load_File and append thread to self.threads - """ - thread = Download_Thread(self) - self.threads.append(thread) - return True - - def remove_thread(self, thread): - self.threads.remove(thread) - - def select_thread(self): - """ create all threads - """ - while len(self.threads) < self.max_threads: - self.create_thread() - - def get_job(self): - """return job if suitable, otherwise send thread idle""" - - if not self.parent.server_methods.is_time_download() or self.pause or self.reconnecting or self.list.queueEmpty(): #conditions when threads dont download - return None - - if self.parent.freeSpace() < self.parent.config["general"]["min_free_space"]: - self.parent.logger.debug("min free space exceeded") - return None - - self.init_reconnect() - - self.lock.acquire() - - pyfile = None - pyfiles = self.list.getDownloadList(self.occ_plugins) - - if pyfiles: - pyfile = pyfiles[0] - self.py_downloading.append(pyfile) - self.parent.hookManager.downloadStarts(pyfile) - if not pyfile.plugin.multi_dl: - self.occ_plugins.append(pyfile.modul.__name__) - pyfile.active = True - if pyfile.plugin.props['type'] == "container": - self.parent.logger.info(_("Get links from: %s") % pyfile.url) - else: - self.parent.logger.info(_("Download starts: %s") % pyfile.url) - - self.lock.release() - return pyfile - - def job_finished(self, pyfile): - """manage completing download""" - self.lock.acquire() - - if not pyfile.plugin.multi_dl: - self.occ_plugins.remove(pyfile.modul.__name__) - - pyfile.active = False - - if not pyfile.status == "reconnected": - try: - pyfile.plugin.req.pycurl.close() - except: - pass - - self.py_downloading.remove(pyfile) - - if pyfile.status.type == "finished": - if pyfile.plugin.props['type'] == "container": - newLinks = 0 - if pyfile.plugin.links: - if isinstance(pyfile.plugin.links, dict): - packmap = {} - for packname in pyfile.plugin.links.keys(): - packmap[packname] = self.list.packager.addNewPackage(packname) - for packname, links in pyfile.plugin.links.items(): - pid = packmap[packname] - for link in links: - newFile = self.list.collector.addLink(link) - self.list.packager.addFileToPackage(pid, self.list.collector.popFile(newFile)) - newLinks += 1 - else: - for link in pyfile.plugin.links: - newFile = self.list.collector.addLink(link) - self.list.packager.addFileToPackage(pyfile.package.data["id"], self.list.collector.popFile(newFile)) - newLinks += 1 - #self.list.packager.pushPackage2Queue(pyfile.package.data["id"]) - self.list.packager.removeFileFromPackage(pyfile.id, pyfile.package.data["id"]) - - if newLinks: - self.parent.logger.info(_("Parsed links from %s: %i") % (pyfile.status.filename, newLinks)) - else: - self.parent.logger.info(_("No links in %s") % pyfile.status.filename) - #~ self.list.packager.removeFileFromPackage(pyfile.id, pyfile.package.id) - #~ for link in pyfile.plugin.links: - #~ id = self.list.collector.addLink(link) - #~ pyfile.packager.pullOutPackage(pyfile.package.id) - #~ pyfile.packager.addFileToPackage(pyfile.package.id, pyfile.collector.popFile(id)) - else: - self.parent.logger.info(_("Download finished: %s") % pyfile.url) - - elif pyfile.status.type == "reconnected": - pyfile.plugin.req.init_curl() - - elif pyfile.status.type == "failed": - self.parent.logger.warning(_("Download failed: %s | %s") % (pyfile.url, pyfile.status.error)) - with open(self.parent.config['general']['failed_file'], 'a') as f: - f.write(pyfile.url + "\n") - - elif pyfile.status.type == "aborted": - self.parent.logger.info(_("Download aborted: %s") % pyfile.url) - - self.list.save() - - self.parent.hookManager.downloadFinished(pyfile) - - self.lock.release() - return True - - def init_reconnect(self): - """initialise a reonnect""" - if not self.parent.config['reconnect']['activated'] or self.reconnecting or not self.parent.server_methods.is_time_reconnect(): - return False - - if not exists(self.parent.config['reconnect']['method']): - self.parent.logger.info(self.parent.config['reconnect']['method'] + " not found") - self.parent.config['reconnect']['activated'] = False - return False - - self.lock.acquire() - - if self.check_reconnect(): - self.reconnecting = True - self.reconnect() - time.sleep(1.1) - - self.reconnecting = False - self.lock.release() - return True - - self.lock.release() - return False - - def check_reconnect(self): - """checks if all files want reconnect""" - - if not self.py_downloading: - return False - - i = 0 - for obj in self.py_downloading: - if obj.status.want_reconnect: - i += 1 - - if len(self.py_downloading) == i: - return True - else: - return False - - def reconnect(self): - self.parent.logger.info(_("Starting reconnect")) - ip = re.match(".*Current IP Address: (.*).*", urllib2.urlopen("http://checkip.dyndns.org/").read()).group(1) - self.parent.hookManager.beforeReconnecting(ip) - reconn = subprocess.Popen(self.parent.config['reconnect']['method'])#, stdout=subprocess.PIPE) - reconn.wait() - time.sleep(1) - ip = "" - while ip == "": - try: - ip = re.match(".*Current IP Address: (.*).*", urllib2.urlopen("http://checkip.dyndns.org/").read()).group(1) #versuchen neue ip aus zu lesen - except: - ip = "" - time.sleep(1) - self.parent.hookManager.afterReconnecting(ip) - self.parent.logger.info(_("Reconnected, new IP: %s") % ip) - - def stopAllDownloads(self): - self.pause = True - for pyfile in self.py_downloading: - pyfile.plugin.req.abort = True - - class SpeedManager(Thread): - def __init__(self, parent): - Thread.__init__(self) - self.parent = parent - self.running = True - self.lastSlowCheck = 0.0 - - stat = {} - stat["slow_downloads"] = None - stat["each_speed"] = None - stat["each_speed_optimized"] = None - self.stat = stat - - self.slowCheckInterval = 60 - self.slowCheckTestTime = 25 - - self.logger = self.parent.parent.logger - self.start() - - def run(self): - while self.running: - time.sleep(1) - self.manageSpeed() - - def getMaxSpeed(self): - return self.parent.parent.getMaxSpeed() - - def manageSpeed(self): - maxSpeed = self.getMaxSpeed() - if maxSpeed <= 0: - for thread in self.parent.py_downloading: - thread.plugin.req.speedLimitActive = False - return - threads = self.parent.py_downloading - threadCount = len(threads) - if threadCount <= 0: - return - eachSpeed = maxSpeed/threadCount - - currentOverallSpeed = 0 - restSpeed = maxSpeed - currentOverallSpeed - speeds = [] - for thread in threads: - currentOverallSpeed += thread.plugin.req.dl_speed - speeds.append((thread.plugin.req.dl_speed, thread.plugin.req.averageSpeed, thread)) - thread.plugin.req.speedLimitActive = True - - if currentOverallSpeed+50 < maxSpeed: - for thread in self.parent.py_downloading: - thread.plugin.req.speedLimitActive = False - return - - slowCount = 0 - slowSpeed = 0 - if self.lastSlowCheck + self.slowCheckInterval + self.slowCheckTestTime < time.time(): - self.lastSlowCheck = time.time() - if self.lastSlowCheck + self.slowCheckInterval < time.time() < self.lastSlowCheck + self.slowCheckInterval + self.slowCheckTestTime: - for speed in speeds: - speed[2].plugin.req.isSlow = False - else: - for speed in speeds: - if speed[0] <= eachSpeed-7: - if speed[1] < eachSpeed-15: - if speed[2].plugin.req.dl_time > 0 and speed[2].plugin.req.dl_time+30 < time.time(): - speed[2].plugin.req.isSlow = True - if not speed[1]-5 < speed[2].plugin.req.maxSpeed/1024 < speed[1]+5: - speed[2].plugin.req.maxSpeed = (speed[1]+10)*1024 - if speed[2].plugin.req.isSlow: - slowCount += 1 - slowSpeed += speed[2].plugin.req.maxSpeed/1024 - stat = {} - stat["slow_downloads"] = slowCount - stat["each_speed"] = eachSpeed - eachSpeed = (maxSpeed - slowSpeed) / (threadCount - slowCount) - stat["each_speed_optimized"] = eachSpeed - self.stat = stat - - for speed in speeds: - if speed[2].plugin.req.isSlow: - continue - speed[2].plugin.req.maxSpeed = eachSpeed*1024 - print "max", speed[2].plugin.req.maxSpeed, "current", speed[2].plugin.req.dl_speed diff --git a/pyLoadCore.py b/pyLoadCore.py index 8d8793b33..8fcfa6c82 100755 --- a/pyLoadCore.py +++ b/pyLoadCore.py @@ -61,16 +61,17 @@ import time from time import sleep from xmlrpclib import Binary -from module.CaptchaManager import CaptchaManager -from module.HookManager import HookManager -from module.PullEvents import PullManager from module.XMLConfigParser import XMLConfigParser -from module.file_list import File_List from module.network.Request import getURL import module.remote.SecureXMLRPCServer as Server -from module.thread_list import Thread_List from module.web.ServerThread import WebServer +from module.ThreadManager import ThreadManager +from module.CaptchaManager import CaptchaManager +from module.HookManager import HookManager +from module.PullEvents import PullManager +from module.FileList import FileList + class Core(object): """ pyLoad Core """ @@ -250,9 +251,9 @@ class Core(object): self.lastGuiConnected = 0 self.server_methods = ServerMethods(self) - self.file_list = File_List(self) + self.file_list = FileList(self) self.pullManager = PullManager(self) - self.thread_list = Thread_List(self) + self.thread_list = ThreadManager(self) self.captchaManager = CaptchaManager(self) self.last_update_check = 0 @@ -295,6 +296,7 @@ class Core(object): self.logger.info(_("Free space: %sMB") % freeSpace) self.thread_list.pause = False + self.thread_list.start() self.hookManager.coreReady() -- cgit v1.2.3