diff options
Diffstat (limited to 'module')
| -rw-r--r-- | module/Api.py | 15 | ||||
| -rw-r--r-- | module/PluginThread.py | 29 | ||||
| -rw-r--r-- | module/ThreadManager.py | 4 | ||||
| -rw-r--r-- | module/common/APIExerciser.py | 23 | ||||
| -rw-r--r-- | module/plugins/hooks/UnRar.py | 131 | ||||
| -rw-r--r-- | module/remote/thriftbackend/ThriftClient.py | 1 |
6 files changed, 135 insertions, 68 deletions
diff --git a/module/Api.py b/module/Api.py index be72ee359..221809dc2 100644 --- a/module/Api.py +++ b/module/Api.py @@ -240,7 +240,7 @@ class Api(Iface): return data - def addPackage(self, name, links, dest): + def addPackage(self, name, links, dest=Destination.Queue): """Adds a package, with links to desired destination. :param name: name of the new package @@ -343,6 +343,19 @@ class Api(Iface): return [self.addPackage(name, urls, dest) for name, urls in self.generatePackages(links).iteritems()] + def checkAndAddPackages(self, links, dest=Destination.Queue): + """Checks online status, retrieves names, and will add packages.\ + Because of this packages are not added immediatly, only for internal use. + + :param links: list of urls + :param dest: `Destination` + :return: None + """ + data = self.core.pluginManager.parseUrls(urls) + self.core.threadManager.createResultThread(data, True) + + + def getPackageData(self, pid): """Returns complete information about package, and included files. diff --git a/module/PluginThread.py b/module/PluginThread.py index a036b4878..0a8151a25 100644 --- a/module/PluginThread.py +++ b/module/PluginThread.py @@ -407,7 +407,7 @@ class HookThread(PluginThread): class InfoThread(PluginThread): - def __init__(self, manager, data, pid=-1, rid=-1): + def __init__(self, manager, data, pid=-1, rid=-1, add=False): """Constructor""" PluginThread.__init__(self, manager) @@ -416,6 +416,7 @@ class InfoThread(PluginThread): # [ .. (name, plugin) .. ] self.rid = rid #result id + self.add = add #add packages instead of return result self.cache = [] #accumulated data @@ -440,6 +441,29 @@ class InfoThread(PluginThread): self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) self.m.core.files.save() + elif self.add: + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateCache, True) + + else: + #generate default result + result = [(url, 0, 3, url) for url in urls] + + self.updateCache(pluginname, result) + + + packs = parseNames([(name, url) for name, x,y, url in self.cache]) + + self.m.core.log.debug("Fetched and generated %d packages" % len(packs)) + + for k, v in packs: + self.m.core.api.addPackage(k, v) + + #empty cache + del self.cache[:] + else: #post the results self.m.infoResults[self.rid] = {} @@ -489,6 +513,9 @@ class InfoThread(PluginThread): self.cache = [] + def updateCache(self, plugin, result): + self.cache.extend(result) + def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None): try: result = [] #result loaded from cache diff --git a/module/ThreadManager.py b/module/ThreadManager.py index 07ab69548..ce5b7bd1d 100644 --- a/module/ThreadManager.py +++ b/module/ThreadManager.py @@ -88,14 +88,14 @@ class ThreadManager: PluginThread.InfoThread(self, data, pid) @lock - def createResultThread(self, data): + def createResultThread(self, data, add=False): """ creates a thread to fetch online status, returns result id """ self.timestamp = time() + 5 * 60 rid = self.resultIDs self.resultIDs += 1 - PluginThread.InfoThread(self, data, rid=rid) + PluginThread.InfoThread(self, data, rid=rid, add=add) return rid diff --git a/module/common/APIExerciser.py b/module/common/APIExerciser.py index b9e67d824..d86e7266b 100644 --- a/module/common/APIExerciser.py +++ b/module/common/APIExerciser.py @@ -4,10 +4,12 @@ import string from threading import Thread from random import choice, random, sample, randint from time import time, sleep +from math import floor +import gc from traceback import print_exc, format_exc -from module.remote.thriftbackend.ThriftClient import ThriftClient +from module.remote.thriftbackend.ThriftClient import ThriftClient, Destination def createURLs(): """ create some urls, some may fail """ @@ -24,6 +26,9 @@ def createURLs(): AVOID = (0,3,8) class APIExerciser(Thread): + + idPool = 0 + def __init__(self, core, thrift=False): Thread.__init__(self) self.setDaemon(True) @@ -37,6 +42,12 @@ class APIExerciser(Thread): else: self.api = core.api + + self.id = self.idPool + + self.core.log.info("API Excerciser started %d" % self.id) + APIExerciser.idPool += 1 + self.start() def run(self): @@ -49,15 +60,17 @@ class APIExerciser(Thread): try: self.testAPI() except Exception: + self.core.log.error("Excerciser %d throw an execption" % self.id) print_exc() out.write(format_exc() + 2 * "\n") out.flush() if not self.count % 100: - print "Tested %s api calls" % self.count + self.core.log.info("Exerciser %d tested %d api calls" % (self.id, self.count)) if not self.count % 1000: out.write("Tested %s api calls\n" % self.count) out.flush() + self.core.log.info("Collected garbage: %d" % gc.collect()) #sleep(random() / 500) @@ -82,7 +95,7 @@ class APIExerciser(Thread): name = "".join(sample(string.ascii_letters, 10)) urls = createURLs() - self.api.addPackage(name, urls, 0) + self.api.addPackage(name, urls, choice([Destination.Queue, Destination.Collector])) def deleteFiles(self): @@ -98,12 +111,12 @@ class APIExerciser(Thread): def deletePackages(self): - info = self.api.getQueue() + info = choice([self.api.getQueue(), self.api.getCollector()]) if not info: return pids = [p.pid for p in info] if len(pids): - pids = sample(pids, randint(1, max(len(pids) / 2, 1))) + pids = sample(pids, randint(1, max(floor(len(pids) / 2.5), 1))) self.api.deletePackages(pids) def getFileData(self): diff --git a/module/plugins/hooks/UnRar.py b/module/plugins/hooks/UnRar.py index 6dfe2c195..c636664ee 100644 --- a/module/plugins/hooks/UnRar.py +++ b/module/plugins/hooks/UnRar.py @@ -24,27 +24,31 @@ from module.plugins.Hook import Hook from module.lib.pyunrar import Unrar, WrongPasswordError, CommandError, UnknownError, LowRamError from traceback import print_exc +import os from os.path import exists, join, isabs, isdir from os import remove, makedirs, rmdir, listdir, chown, chmod -from pwd import getpwnam + +if os.name != "nt": + from pwd import getpwnam + import re class UnRar(Hook): __name__ = "UnRar" __version__ = "0.11" __description__ = """unrar""" - __config__ = [ ("activated", "bool", "Activated", False), - ("fullpath", "bool", "extract full path", True), - ("overwrite", "bool", "overwrite files", True), - ("passwordfile", "str", "unrar password file", "unrar_passwords.txt"), - ("deletearchive", "bool", "delete archives when done", False), - ("ramwarning", "bool", "warn about low ram", True), - ("renice", "int", "Cpu Priority", 10), - ("unrar_destination", "str", "Unpack files to", "")] + __config__ = [("activated", "bool", "Activated", False), + ("fullpath", "bool", "extract full path", True), + ("overwrite", "bool", "overwrite files", True), + ("passwordfile", "str", "unrar password file", "unrar_passwords.txt"), + ("deletearchive", "bool", "delete archives when done", False), + ("ramwarning", "bool", "warn about low ram", True), + ("renice", "int", "Cpu Priority", 10), + ("unrar_destination", "str", "Unpack files to", "")] __threaded__ = ["packageFinished"] __author_name__ = ("mkaay") __author_mail__ = ("mkaay@mkaay.de") - + def setup(self): self.comments = ["# one password each line"] self.passwords = [] @@ -71,31 +75,31 @@ class UnRar(Hook): self.ram = 0 self.ram /= 1024 - - def setOwner(self,d,uid,gid,mode): + + def setOwner(self, d, uid, gid, mode): if not exists(d): - self.core.log.debug(_("Directory %s does not exist!") % d) - return - fileList=listdir(d) - for fileEntry in fileList: - fullEntryName=join(d,fileEntry) - if isdir(fullEntryName): - self.setOwner(fullEntryName,uid,gid,mode) - try: - chown(fullEntryName,uid,gid) - chmod(fullEntryName,mode) - except: - self.core.log.debug(_("Chown/Chmod for %s failed") % fullEntryName) - self.core.log.debug(_("Exception: %s") % sys.exc_info()[0]) - continue + self.core.log.debug(_("Directory %s does not exist!") % d) + return + + for fileEntry in listdir(d): + fullEntryName = join(d, fileEntry) + if isdir(fullEntryName): + self.setOwner(fullEntryName, uid, gid, mode) + try: + chown(fullEntryName, uid, gid) + chmod(fullEntryName, mode) + except: + self.core.log.debug(_("Chown/Chmod for %s failed") % fullEntryName) + self.core.log.debug(_("Exception: %s") % sys.exc_info()[0]) + continue try: - chown(d,uid,gid) - chmod(d,mode) + chown(d, uid, gid) + chmod(d, mode) except: - self.core.log.debug(_("Chown/Chmod for %s failed") % d) - self.core.log.debug(_("Exception: %s") % sys.exc_info()[0]) - return - + self.core.log.debug(_("Chown/Chmod for %s failed") % d) + self.core.log.debug(_("Exception: %s") % sys.exc_info()[0]) + return + def addPassword(self, pws): if not type(pws) == list: pws = [pws] pws.reverse() @@ -105,14 +109,14 @@ class UnRar(Hook): self.passwords.insert(0, pw) with open(self.getConfig("passwordfile"), "w") as f: - f.writelines([c+"\n" for c in self.comments]) - f.writelines([p+"\n" for p in self.passwords]) - + f.writelines([c + "\n" for c in self.comments]) + f.writelines([p + "\n" for p in self.passwords]) + def removeFiles(self, pack, fname): if not self.getConfig("deletearchive"): return m = self.re_splitfile.search(fname) - + download_folder = self.core.config['general']['download_folder'] if self.core.config['general']['folder_per_package']: folder = join(download_folder, pack.folder.decode(sys.getfilesystemencoding())) @@ -124,11 +128,11 @@ class UnRar(Hook): if nre.match(data["name"]): remove(join(folder, data["name"])) elif not m and fname.endswith(".rar"): - nre = re.compile("^%s\.r..$" % fname.replace(".rar","")) + nre = re.compile("^%s\.r..$" % fname.replace(".rar", "")) for fid, data in pack.getChildren().iteritems(): if nre.match(data["name"]): remove(join(folder, data["name"])) - + def packageFinished(self, pack): if pack.password and pack.password.strip() and pack.password.strip() != "None": self.addPassword(pack.password.splitlines()) @@ -137,15 +141,16 @@ class UnRar(Hook): for fid, data in pack.getChildren().iteritems(): m = self.re_splitfile.search(data["name"]) if m and int(m.group(2)) == 1: - files.append((fid,m.group(0))) + files.append((fid, m.group(0))) elif not m and data["name"].endswith(".rar"): - files.append((fid,data["name"])) - + files.append((fid, data["name"])) + for fid, fname in files: self.core.log.info(_("starting Unrar of %s") % fname) pyfile = self.core.files.getFile(fid) pyfile.setStatus("processing") pyfile.progress.setRange(0, 100) + def s(p): pyfile.progress.setValue(p) @@ -170,12 +175,14 @@ class UnRar(Hook): self.core.log.debug(_("Destination folder %s") % destination) if not exists(destination): - self.core.log.info(_("Creating destination folder %s") % destination) - makedirs(destination) + self.core.log.info(_("Creating destination folder %s") % destination) + makedirs(destination) - u = Unrar(join(folder, fname), tmpdir=join(folder, "tmp"), ramSize=(self.ram if self.getConfig("ramwarning") else 0), cpu=self.getConfig("renice")) + u = Unrar(join(folder, fname), tmpdir=join(folder, "tmp"), + ramSize=(self.ram if self.getConfig("ramwarning") else 0), cpu=self.getConfig("renice")) try: - success = u.crackPassword(passwords=self.passwords, statusFunction=s, overwrite=True, destination=destination, fullPath=self.getConfig("fullpath")) + success = u.crackPassword(passwords=self.passwords, statusFunction=s, overwrite=True, + destination=destination, fullPath=self.getConfig("fullpath")) except WrongPasswordError: self.core.log.info(_("Unrar of %s failed (wrong password)") % fname) continue @@ -195,7 +202,8 @@ class UnRar(Hook): self.core.log.info(_("Unrar of %s failed") % fname) continue except LowRamError: - self.log.warning(_("Your ram amount of %s MB seems not sufficient to unrar this file. You can deactivate this warning and risk instability") % self.ram) + self.log.warning(_( + "Your ram amount of %s MB seems not sufficient to unrar this file. You can deactivate this warning and risk instability") % self.ram) continue except UnknownError: if self.core.debug: @@ -207,19 +215,24 @@ class UnRar(Hook): self.core.log.info(_("Unrar of %s ok") % fname) self.removeFiles(pack, fname) if self.core.config['general']['folder_per_package']: - if self.getConfig("deletearchive"): - self.core.log.debug(_("Deleting package directory %s...") % folder) - rmdir(folder) - self.core.log.debug(_("Package directory %s has been deleted.") % folder) - ownerUser=self.core.config['permission']['user'] - uinfo=getpwnam(ownerUser) - fileMode=int(self.core.config['permission']['file'],8) - self.core.log.debug(_("Setting destination file/directory owner to %s.") % ownerUser) - self.core.log.debug(_("Setting destination file/directory mode to %s.") % fileMode) - self.core.log.debug(_("Uid is %s.") % uinfo.pw_uid) - self.core.log.debug(_("Gid is %s.") % uinfo.pw_gid) - self.setOwner(destination,uinfo.pw_uid,uinfo.pw_gid,fileMode) - self.core.log.debug(_("The owner/rights have been successfully changed.")) + if self.getConfig("deletearchive"): + self.core.log.debug(_("Deleting package directory %s...") % folder) + rmdir(folder) + self.core.log.debug(_("Package directory %s has been deleted.") % folder) + + if os.name != "nt" and self.core.config['permission']['change_dl'] and\ + self.core.config['permission']['change_file']: + ownerUser = self.core.config['permission']['user'] + fileMode = int(self.core.config['permission']['file'], 8) + + self.core.log.debug("Setting destination file/directory owner / mode to %s / %s" + % (ownerUser, fileMode)) + + uinfo = getpwnam(ownerUser) + self.core.log.debug("Uid/Gid is %s/%s." % (uinfo.pw_uid, uinfo.pw_gid)) + self.setOwner(destination, uinfo.pw_uid, uinfo.pw_gid, fileMode) + self.core.log.debug("The owner/rights have been successfully changed.") + self.core.hookManager.unrarFinished(folder, fname) else: self.core.log.info(_("Unrar of %s failed (wrong password or bad parts)") % fname) diff --git a/module/remote/thriftbackend/ThriftClient.py b/module/remote/thriftbackend/ThriftClient.py index b3e6ac6ed..c8e8f2bd7 100644 --- a/module/remote/thriftbackend/ThriftClient.py +++ b/module/remote/thriftbackend/ThriftClient.py @@ -16,6 +16,7 @@ from Socket import Socket from Protocol import Protocol from thriftgen.pyload import Pyload +from thriftgen.pyload.ttypes import * from thriftgen.pyload.Pyload import PackageDoesNotExists from thriftgen.pyload.Pyload import FileDoesNotExists |