diff options
Diffstat (limited to 'module')
| -rw-r--r-- | module/file_list.py | 379 | ||||
| -rw-r--r-- | module/remote/ClientHandler.py | 24 | ||||
| -rw-r--r-- | module/remote/ClientSocket.py | 63 | ||||
| -rw-r--r-- | module/remote/RequestHandler.py | 64 | ||||
| -rw-r--r-- | module/remote/RequestObject.py | 18 | ||||
| -rw-r--r-- | module/thread_list.py | 25 |
6 files changed, 257 insertions, 316 deletions
diff --git a/module/file_list.py b/module/file_list.py index 1701b801b..6e1984704 100644 --- a/module/file_list.py +++ b/module/file_list.py @@ -1,159 +1,274 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# -#Copyright (C) 2009 RaNaN -# -#This program is free software; you can redistribute it and/or modify -#it under the terms of the GNU General Public License as published by -#the Free Software Foundation; either version 3 of the License, -#or (at your option) any later version. -# -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#See the GNU General Public License for more details. -# -#You should have received a copy of the GNU General Public License -# along with this program; if not, see <http://www.gnu.org/licenses/>. -# -### -LIST_VERSION = 3 +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: mkaay + @author: RaNaN + @version: v0.3 + @list-version: v4 +""" + +LIST_VERSION = 4 from threading import RLock from download_thread import Status import cPickle import re -from module.remote.RequestObject import RequestObject import module.Plugin +class NoSuchElementException(Exception): + pass + class File_List(object): def __init__(self, core): self.core = core - self.files = [] - self.data = {'version': LIST_VERSION, 'order': []} self.lock = RLock() - self.load() - - def new_pyfile(self, url, folder): - url = url.replace("\n", "") - pyfile = PyLoadFile(self.core, url) - pyfile.download_folder = self.core.config['general']['download_folder'] - pyfile.id = self.get_id() - pyfile.folder = folder - - return pyfile - - def append(self, url, folder=""): - if not url: - return False - #@TODO: filter non existence and invalid links - #re.compile("https?://[-a-z0-9\.]{4,}(?::\d+)?/[^#?]+(?:#\S+)?",re.IGNORECASE) - new_file = self.new_pyfile(url, folder) - self.files.append(new_file) - self.data[new_file.id] = Data(url, folder) - self.data['order'].append(int(new_file.id)) - - def extend(self, urls): - for url in urls: - self.append(url) - - def remove(self, pyfile): - if not self.core.config['general']['debug_mode']: - if pyfile in self.files: - self.files.remove(pyfile) - - self.data['order'].remove(pyfile.id) - del self.data[pyfile.id] - - def remove_id(self, pyid): - #also abort download - pyid = int(pyid) - found = False - for pyfile in self.files: - if pyfile.id == pyid: - self.files.remove(pyfile) - found = True - break - - if not found: - for pyfile in self.core.thread_list.py_downloading: - if pyfile.id == pyid: - pyfile.plugin.req.abort = True - break - return False - - self.data['order'].remove(pyid) - del self.data[pyid] - - def get_id(self): - """return a free id""" - id = 1 - while id in self.data.keys(): - id += 1 - - return id - - def move(self, id, offset=-1): - - for pyfile in self.files: - if pyfile.id == id: - index = self.files.index(pyfile) - pyfile = self.files.pop(index) - self.files.insert(index + offset, pyfile) - break - - index = self.data['order'].index(id) - pyfile = self.data['order'].pop(index) - self.data['order'].insert(index + offset, pyfile) - - def save(self): - self.lock.acquire() - - output = open('links.pkl', 'wb') - cPickle.dump(self.data, output, -1) - - #self.inform_client() + self.download_folder = self.core.config['general']['download_folder'] + self.collector = self.pyLoadCollector(self) + self.packager = self.pyLoadPackager(self) + + self.data = { + "version": LIST_VERSION, + "queue": [], + "packages": [], + "collector": [] + } - self.lock.release() - def load(self): + self.lock.acquire() try: pkl_file = open('links.pkl', 'rb') obj = cPickle.load(pkl_file) except: - obj = {'version': LIST_VERSION, 'order': []} - - if obj['version'] < LIST_VERSION: - obj = {'version': LIST_VERSION, 'order': []} - - for i in obj['order']: - self.append(obj[i].url, obj[i].folder) - - self.core.logger.info("Links loaded: " + str(int(len(obj) - 2))) - - def inform_client(self): - obj = RequestObject() - obj.command = "file_list" - obj.data = self.data + obj = False + if obj['version'] == LIST_VERSION and obj: + self.data = obj + self.lock.release() + + if len(self.data["collector"]) > 0: + self.core.logger.info("Found %s links in linkcollector" % len(self.data["collector"])) + if len(self.data["packages"]) > 0: + self.core.logger.info("Found %s unqueued packages" % len(self.data["packages"])) + if len(self.data["queue"]) > 0: + self.core.logger.info("Added %s packages to queue" % len(self.data["queue"])) + + def save(self): + self.lock.acquire() - self.core.server.push_all(obj) + output = open('links.pkl', 'wb') + cPickle.dump(self.data, output, -1) + + self.lock.release() + + def queueEmpty(self): + return (self.data["queue"] == []) + + def getDownloadList(self): + """ + for thread_list only + """ + files = [] + for pypack in self.data["queue"]: + for pyfile in pypack.files: + if pyfile.status.type == "reconnected" or pyfile.status.type == None: + files.append(pyfile) + return files + + class pyLoadCollector(): + def __init__(collector, file_list): + collector.file_list = file_list + + def _getFileFromID(collector, id): + """ + returns PyLoadFile instance and position in collector with given id + """ + for n, pyfile in enumerate(collector.file_list.data["collector"]): + if pyfile.id == id: + return (n, pyfile) + raise NoSuchElementException() + + def _getFreeID(collector): + """ + returns a free id + """ + ids = [] + for pyfile in collector.file_list.data["collector"]: + ids.append(pyfile.id) + id = 1 + while id in ids: + id += 1 + return id + + def getFile(collector, id): + """ + returns PyLoadFile instance from given id + """ + return collector._getFileFromID(id)[1] + + def popFile(collector, id): + """ + returns PyLoadFile instance given id and remove it from the collector + """ + collector.file_list.lock.acquire() + try: + n, pyfile = collector._getFileFromID(id) + del collector.file_list.data["collector"][n] + collector.file_list.lock.release() + except: + collector.file_list.lock.release() + else: + return pyfile + + def addLink(collector, url): + """ + appends a new PyLoadFile instance to the end of the collector + """ + pyfile = PyLoadFile(url) + pyfile.id = collector._getFreeID() + pyfile.download_folder = collector.file_list.download_folder + collector.file_list.lock.acquire() + collector.file_list.data["collector"].append(pyfile) + collector.file_list.lock.release() + return pyfile.id + + def removeFile(collector, id): + """ + removes PyLoadFile instance with the given id from collector + """ + collector.popFile(id) + + def replaceFile(collector, newpyfile): + """ + replaces PyLoadFile instance with the given PyLoadFile instance at the given id + """ + collector.file_list.lock.acquire() + try: + n, pyfile = collector._getFileFromID(newpyfile.id) + collector.file_list.data["collector"][n] = newpyfile + finally: + collector.file_list.lock.release() + + class pyLoadPackager(): + def __init__(packager, file_list): + packager.file_list = file_list + + def _getFreeID(packager): + """ + returns a free id + """ + ids = [] + for pypack in (packager.file_list.data["packages"] + packager.file_list.data["queue"]): + ids.append(pypack.id) + id = 1 + while id in ids: + id += 1 + return id + + def _getPackageFromID(packager, id): + """ + returns PyLoadPackage instance and position with given id + """ + for n, pypack in enumerate(packager.file_list.data["packages"]): + if pypack.id == id: + return ("packages", n, pypack) + for n, pypack in enumerate(packager.file_list.data["queue"]): + if pypack.id == id: + return ("queue", n, pypack) + raise NoSuchElementException() + + def addNewPackage(packager, package_name=None): + pypack = PyLoadPackage() + pypack.id = packager._getFreeID() + if package_name is not None: + pypack.data["package_name"] = package_name + packager.file_list.data["packages"].append(pypack) + return pypack.id + + def removePackage(packager, id): + packager.file_list.lock.acquire() + try: + key, n, pypack = packager._getPackageFromID(id) + del packager.file_list.data[key][n] + finally: + packager.file_list.lock.release() + + def pushPackage2Queue(packager, id): + packager.file_list.lock.acquire() + try: + key, n, pypack = packager._getPackageFromID(id) + if key == "packages": + del packager.file_list.data["packages"][n] + packager.file_list.data["queue"].append(pypack) + finally: + packager.file_list.lock.release() + + def pullOutPackage(packager, id): + packager.file_list.lock.acquire() + try: + key, n, pypack = packager._getPackageFromID(id) + if key == "queue": + del packager.file_list.data["queue"][n] + packager.file_list.data["packages"].append(pypack) + finally: + packager.file_list.lock.release() + + def setPackageData(packager, id, package_name=None, folder=None): + packager.file_list.lock.acquire() + try: + key, n, pypack = packager._getPackageFromID(id) + if package_name is not None: + pypack.data["package_name"] = package_name + if folder is not None: + pypack.data["folder"] = folder + packager.file_list.data[key][n] = pypack + finally: + packager.file_list.lock.release() + + def addFileToPackage(packager, id, pyfile): + key, n, pypack = packager._getPackageFromID(id) + pyfile.package = pypack + pypack.files.append(pyfile) + packager.file_list.data[key][n] = pypack + + def removeFileFromPackage(packager, id, pid): + key, n, pypack = packager._getPackageFromID(pid) + for k, pyfile in enumerate(pypack.files): + if id == pyfile.id: + del pypack.files[k] + return True + raise NoSuchElementException() -class Data(): - def __init__(self, url, folder=""): - self.url = url - self.folder = folder +class PyLoadPackage(): + def __init__(self): + self.files = [] + self.data = { + "id": None, + "package_name": "", + "folder": "" + } -class PyLoadFile: - """ represents the url or file - """ - def __init__(self, parent, url): - self.parent = parent +class PyLoadFile(): + def __init__(self, url): self.id = None self.url = url self.folder = None + self.package = None self.filename = "filename" self.download_folder = "" + self.active = False pluginName = self._get_my_plugin() if pluginName: self.modul = __import__(pluginName) @@ -163,18 +278,12 @@ class PyLoadFile: pluginClass = module.Plugin.Plugin self.plugin = pluginClass(self) self.status = Status(self) + def _get_my_plugin(self): - - """ searches the right plugin for an url - """ for plugin, plugin_pattern in self.parent.plugins_avaible.items(): if re.match(plugin_pattern, self.url) != None: return plugin def init_download(self): - if self.parent.config['proxy']['activated']: self.plugin.req.add_proxy(self.parent.config['proxy']['protocol'], self.parent.config['proxy']['adress']) - - #@TODO: check dependicies, ocr etc - diff --git a/module/remote/ClientHandler.py b/module/remote/ClientHandler.py deleted file mode 100644 index 9653db9ae..000000000 --- a/module/remote/ClientHandler.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -authored by: RaNaN - -this module handels the incoming requests - -""" -import hashlib - -from Crypto.Cipher import Blowfish -from RequestHandler import RequestHandler - -class ClientHandler(RequestHandler): - def __init__(self, client, pw): - self.client = client - key = hashlib.sha256(pw) - self.bf = Blowfish.new(key.hexdigest(), Blowfish.MODE_ECB) - - def proceed(self, data): - obj = self.decrypt(data) - - self.client.data_arrived(obj)
\ No newline at end of file diff --git a/module/remote/ClientSocket.py b/module/remote/ClientSocket.py deleted file mode 100644 index 0e6a5ed53..000000000 --- a/module/remote/ClientSocket.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -authored by: RaNaN - -socket for connecting to the core's server - -""" -import asynchat -import asyncore -import socket -import threading - -from ClientHandler import ClientHandler -from RequestObject import RequestObject - -class SocketThread(threading.Thread): - def __init__(self, adress, port, pw, client): - threading.Thread.__init__(self) - self.setDaemon(True) - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((adress, port)) - self.socket = ClientSocket(sock, pw, client) - self.start() - - def run(self): - asyncore.loop() - print "loop closed" - - def push_exec(self, function, args=[]): - obj = RequestObject() - obj.command = "exec" - obj.function = function - obj.args = args - self.push(obj) - - def push(self, obj): - self.socket.push_obj(obj) - - -class ClientSocket(asynchat.async_chat): - def __init__(self, sock, pw, client): - asynchat.async_chat.__init__(self, sock) - self.data = "" - self.handler = ClientHandler(client, pw) - self.set_terminator("\n") - #self.create_socket(socket.AF_INET, socket.SOCK_STREAM) - - def handle_close(self): - print "Disconnected from", self.getpeername() - self.close() - - def collect_incoming_data(self, data): - self.data += data - - def found_terminator(self): - self.handler.proceed(self.data) - self.data = "" - - def push_obj(self, obj): - string = self.handler.encrypt(obj) - self.push(string)
\ No newline at end of file diff --git a/module/remote/RequestHandler.py b/module/remote/RequestHandler.py deleted file mode 100644 index a879d2297..000000000 --- a/module/remote/RequestHandler.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -authored by: RaNaN - -this module handels the incoming requests - -""" - -import base64 -import hashlib -import random -import string - -import cPickle -from Crypto.Cipher import Blowfish -from RequestObject import RequestObject - - -class RequestHandler: - def __init__(self, core): - self.core = core - key = hashlib.sha256(core.config['remote']['password']) - self.bf = Blowfish.new(key.hexdigest(), Blowfish.MODE_ECB) - - def proceed(self, data): - obj = self.decrypt(data) - - if obj.command == "exec": - func = getattr(self.core, obj.function) - obj.data = func( * obj.args) - else: - obj.data = "error happend" - - return self.encrypt(obj) - - - def decrypt(self, dec_str): - try: - dec_str = base64.standard_b64decode(dec_str) - dec_str = self.bf.decrypt(dec_str) - - dec_str = dec_str[:-(int(dec_str[-1], 16) + 1)] - obj = cPickle.loads(dec_str) - except: - obj = RequestObject() - - return obj - - def encrypt(self, obj): - enc_str = cPickle.dumps(obj, 1) - padding = len(enc_str) % 16 - padding = 16 - padding - - p_str = "" - for i in range(padding - 1): - p_str += random.choice(string.letters + string.digits + string.punctuation) - p_str += hex(len(p_str)).replace("0x", "") - enc_str += p_str - - enc_str = self.bf.encrypt(enc_str) - enc_str = base64.standard_b64encode(enc_str) - return enc_str + "\n" diff --git a/module/remote/RequestObject.py b/module/remote/RequestObject.py deleted file mode 100644 index 54ea837a6..000000000 --- a/module/remote/RequestObject.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -authored by: RaNaN - -represents the object for interaction - -""" -class RequestObject(object): - def __init__(self): - self.version = 0 - self.sender = "ip" - self.status = {} - self.command = None - self.function = "" - self.args = [] - self.data = None diff --git a/module/thread_list.py b/module/thread_list.py index da13afba6..aa50a654f 100644 --- a/module/thread_list.py +++ b/module/thread_list.py @@ -60,7 +60,7 @@ class Thread_List(object): def get_job(self): """return job if suitable, otherwise send thread idle""" - if not self.parent.is_time_download() or self.pause or self.reconnecting or not self.list.files: #conditions when threads dont download + if not self.parent.is_time_download() or self.pause or self.reconnecting or self.list.queueEmpty(): #conditions when threads dont download return None self.init_reconnect() @@ -68,15 +68,16 @@ class Thread_List(object): self.lock.acquire() pyfile = None - for i in range(len(self.list.files)): - if not self.list.files[i].modul.__name__ in self.occ_plugins: - pyfile = self.list.files.pop(i) + for f in self.list.getDownloadList(): + if not f.modul.__name__ in self.occ_plugins: + pyfile = f break if pyfile: self.py_downloading.append(pyfile) if not pyfile.plugin.multi_dl: self.occ_plugins.append(pyfile.modul.__name__) + pyfile.active = True self.parent.logger.info('Download starts: ' + pyfile.url) self.lock.release() @@ -89,6 +90,8 @@ class Thread_List(object): if not pyfile.plugin.multi_dl: self.occ_plugins.remove(pyfile.modul.__name__) + + pyfile.active = False if pyfile.plugin.req.curl and not pyfile.status == "reconnected": try: @@ -101,25 +104,23 @@ class Thread_List(object): if pyfile.status.type == "finished": self.parent.logger.info('Download finished: ' + pyfile.url + ' @' + str(pyfile.status.get_speed()) + 'kb/s') - self.list.remove(pyfile) - if pyfile.plugin.props['type'] == "container": - self.list.extend(pyfile.plugin.links) - + self.list.packager.removeFileFromPackage(pyfile.id, pyfile.package.id) + for link in pyfile.plugin.links: + id = self.list.collector.addLink(link) + pyfile.packager.pullOutPackage(pyfile.package.id) + pyfile.packager.addFileToPackage(pyfile.package.id, pyfile.collector.popFile(id)) - elif pyfile.status.type == "reconnected":#put it back in queque + elif pyfile.status.type == "reconnected": pyfile.plugin.req.init_curl() - self.list.files.insert(0, pyfile) elif pyfile.status.type == "failed": self.parent.logger.warning("Download failed: " + pyfile.url+ " | " + pyfile.status.error) with open(self.parent.config['general']['failed_file'], 'a') as f: f.write(pyfile.url + "\n") - self.list.remove(pyfile) elif pyfile.status.type == "aborted": self.parent.logger.info("Download aborted: " + pyfile.url) - self.list.remove(pyfile) self.list.save() |