diff options
Diffstat (limited to 'module')
| -rw-r--r-- | module/HookManager.py | 4 | ||||
| -rw-r--r-- | module/PluginThread.py | 14 | ||||
| -rw-r--r-- | module/database/FileDatabase.py | 7 | ||||
| -rw-r--r-- | module/plugins/Plugin.py | 128 | ||||
| -rw-r--r-- | module/web/json_app.py | 2 | ||||
| -rw-r--r-- | module/web/media/default/img/arrow_right.png | bin | 0 -> 349 bytes |
6 files changed, 96 insertions, 59 deletions
diff --git a/module/HookManager.py b/module/HookManager.py index 9688d8c60..29756f8c5 100644 --- a/module/HookManager.py +++ b/module/HookManager.py @@ -109,8 +109,8 @@ class HookManager: if self.core.debug: traceback.print_exc() - self.log.info(_("Activated plugins: %s") % ", ".join(active)) - self.log.info(_("Deactivate plugins: %s") % ", ".join(deactive)) + self.log.info(_("Activated plugins: %s") % ", ".join(sorted(active))) + self.log.info(_("Deactivate plugins: %s") % ", ".join(sorted(deactive))) #self.log.info(_("Not loaded plugins: %s") % ", ".join(unloaded)) self.plugins = plugins diff --git a/module/PluginThread.py b/module/PluginThread.py index 7f967f989..33701ee69 100644 --- a/module/PluginThread.py +++ b/module/PluginThread.py @@ -143,7 +143,7 @@ class DownloadThread(PluginThread): try: - pyfile.plugin.checkForSameFiles() + pyfile.plugin.checkForSameFiles(starting=True) self.m.log.info(_("Download starts: %s" % pyfile.name)) # start download @@ -246,12 +246,18 @@ class DownloadThread(PluginThread): pyfile.setStatus("skipped") - self.m.log.info(_("Download skipped: %(name)s") % pyfile.name) - if self.m.core.debug: - self.m.log.debug("Skipped due to %s" % e.message) + self.m.log.info(_("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message}) self.clean(pyfile) + self.m.core.files.checkPackageFinished(pyfile) + + self.active = False + pyfile.finishIfDone() + self.m.core.files.save() + + continue + except Exception, e: pyfile.setStatus("failed") diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 99812d2c9..3ba43f881 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -814,6 +814,13 @@ class FileMethods(): def restartFailed(self): self.c.execute("UPDATE links SET status=3,error='' WHERE status IN (8, 9)") + + @style.queue + def findDuplicates(self, id, pid, filename): + """ checks if filename exists with different id and same package """ + self.c.execute("SELECT plugin FROM links where id!=? AND status=0 AND package=? AND name=?", (id, pid, filename)) + return self.c.fetchone() + DatabaseBackend.registerSub(FileMethods) if __name__ == "__main__": diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index c2e1c3044..cffbb8acb 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -40,34 +40,39 @@ from itertools import islice from module.utils import save_join, decode, removeChars def chunks(iterable, size): - it = iter(iterable) - item = list(islice(it, size)) - while item: - yield item + it = iter(iterable) item = list(islice(it, size)) + while item: + yield item + item = list(islice(it, size)) class Abort(Exception): """ raised when aborted """ + class Fail(Exception): """ raised when failed """ - + + class Reconnect(Exception): """ raised when reconnected """ + class Retry(Exception): """ raised when start again from beginning """ + class SkipDownload(Exception): """ raised when download should be skipped """ + class Plugin(object): __name__ = "Plugin" __version__ = "0.4" __pattern__ = None __type__ = "hoster" - __config__ = [ ("name", "type", "desc" , "default") ] + __config__ = [("name", "type", "desc", "default")] __description__ = """Base Plugin""" __author_name__ = ("RaNaN", "spoob", "mkaay") __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de") @@ -101,7 +106,7 @@ class Plugin(object): self.premium = self.account.isPremium(self.user) #premium status else: self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) - + self.log = pyfile.m.core.log self.pyfile = pyfile @@ -115,12 +120,12 @@ class Plugin(object): self.html = None #some plugins store html code here self.init() - + def getChunkCount(self): if self.chunkLimit <= 0: return self.config["download"]["chunks"] return min(self.config["download"]["chunks"], self.chunkLimit) - + def __call__(self): return self.__name__ @@ -151,13 +156,13 @@ class Plugin(object): def process(self, pyfile): """the 'main' method of every plugin""" raise NotImplementedError - + def resetAccount(self): """ dont use account and retry download """ self.account = None self.req = self.core.requestFactory.getRequest(self.__name__) self.retry() - + def checksum(self, local_file=None): """ return codes: @@ -181,14 +186,14 @@ class Plugin(object): return self.config.getPlugin(self.__name__, option) def setConfig(self, option, value): - """ sets a config value """ - self.setConf(option, value) - + """ sets a config value """ + self.setConf(option, value) + def getConfig(self, option): - """ gets a config value """ - return self.getConf(option) - - + """ gets a config value """ + return self.getConf(option) + + def setWait(self, seconds, reconnect=False): """ set the wait time to specified seconds """ if reconnect: @@ -199,16 +204,16 @@ class Plugin(object): """ waits the time previously set """ self.waiting = True self.pyfile.setStatus("waiting") - + while self.pyfile.waitUntil > time(): self.thread.m.reconnecting.wait(2) - + if self.pyfile.abort: raise Abort if self.thread.m.reconnecting.isSet(): self.waiting = False self.wantReconnect = False raise Reconnect - + self.waiting = False self.pyfile.setStatus("starting") @@ -227,28 +232,28 @@ class Plugin(object): def retry(self): """ begin again from the beginning """ raise Retry - + def invalidCaptcha(self): if self.cTask: self.cTask.invalid() - + def correctCaptcha(self): if self.cTask: self.cTask.correct() def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype="jpg"): """ loads the catpcha and decrypt it or ask the user for input """ - + content = self.load(url, get=get, post=post, cookies=cookies) - id = ("%.2f" % time())[-6:].replace(".","") - temp = open(join("tmp","tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") - + id = ("%.2f" % time())[-6:].replace(".", "") + temp = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") + temp.write(content) temp.close() has_plugin = self.core.pluginManager.captchaPlugins.has_key(self.__name__) - + if self.core.captcha: Ocr = self.core.pluginManager.getCaptchaPlugin(self.__name__) else: @@ -257,16 +262,15 @@ class Plugin(object): if Ocr and not forceUser: sleep(randint(3000, 5000) / 1000.0) if self.pyfile.abort: raise Abort - + ocr = Ocr() result = ocr.get_captcha(temp.name) else: - captchaManager = self.core.captchaManager task = captchaManager.newTask(content, imgtype, temp.name) self.cTask = task captchaManager.handleCaptcha(task) - + while task.isWaiting(): if self.pyfile.abort: captchaManager.removeTask(task) @@ -282,16 +286,15 @@ class Plugin(object): elif not task.result: self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) - result = task.result self.log.debug("Received captcha result: %s" % result) if not self.core.debug: - try: - remove(temp.name) - except: - pass - + try: + remove(temp.name) + except: + pass + return result @@ -307,25 +310,27 @@ class Plugin(object): if self.core.debug: from inspect import currentframe + frame = currentframe() if not exists(join("tmp", self.__name__)): makedirs(join("tmp", self.__name__)) - f = open(join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)), "wb") + f = open( + join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) + , "wb") del frame # delete the frame or it wont be cleaned - + try: tmp = res.encode("utf8") except: tmp = res - + f.write(tmp) f.close() - if just_header: #parse header - header = {"code" : self.req.code} + header = {"code": self.req.code} for line in res.splitlines(): line = line.strip() if not line or ":" not in line: continue @@ -338,7 +343,7 @@ class Plugin(object): if type(header[key]) == list: header[key].append(value) else: - header[key] = [header[key],value] + header[key] = [header[key], value] else: header[key] = value res = header @@ -353,11 +358,11 @@ class Plugin(object): self.pyfile.setStatus("downloading") download_folder = self.config['general']['download_folder'] - + location = save_join(download_folder, self.pyfile.package().folder) if not exists(location): - makedirs(location, int(self.core.config["permission"]["folder"],8)) + makedirs(location, int(self.core.config["permission"]["folder"], 8)) if self.core.config["permission"]["change_dl"] and os.name != "nt": try: @@ -365,7 +370,7 @@ class Plugin(object): gid = getgrnam(self.config["permission"]["group"])[2] chown(location, uid, gid) - except Exception,e: + except Exception, e: self.log.warning(_("Setting User and Group failed: %s") % str(e)) name = self.pyfile.name @@ -377,7 +382,9 @@ class Plugin(object): filename = save_join(location, name) try: - newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, chunks=self.getChunkCount(), resume=self.resumeDownload, progressNotify=self.pyfile.progress.setValue, disposition=disposition) + newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, + chunks=self.getChunkCount(), resume=self.resumeDownload, + progressNotify=self.pyfile.progress.setValue, disposition=disposition) finally: self.pyfile.size = self.req.size @@ -387,7 +394,7 @@ class Plugin(object): filename = save_join(location, newname) if self.core.config["permission"]["change_file"]: - chmod(filename, int(self.core.config["permission"]["file"],8)) + chmod(filename, int(self.core.config["permission"]["file"], 8)) if self.core.config["permission"]["change_dl"] and os.name != "nt": try: @@ -395,20 +402,20 @@ class Plugin(object): gid = getgrnam(self.config["permission"]["group"])[2] chown(filename, uid, gid) - except Exception,e: + except Exception, e: self.log.warning(_("Setting User and Group failed: %s") % str(e)) self.lastDownload = filename return self.lastDownload - def checkDownload(self, rules, api_size=0 ,max_size=50000, delete=True, read_size=0): + def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): """ checks the content of the last downloaded file rules - dict with names and rules to match(re or strings) size - excpected size @return name of first rule matched or None""" if not exists(self.lastDownload): return None - + size = stat(self.lastDownload) size = size.st_size @@ -441,9 +448,24 @@ class Plugin(object): return password - def checkForSameFiles(self): + def checkForSameFiles(self, starting=False): """ checks if same file was/is downloaded within same package and raise exception """ - pass + + pack = self.pyfile.package() + + cache = self.core.files.cache.values() + + for pyfile in cache: + if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: + if pyfile.status in (0, 12): #finished or downloading + raise SkipDownload(pyfile.pluginname) + elif pyfile.status in (5, 7) and starting: #a download is waiting and was appenrently started before + raise SkipDownload(pyfile.pluginname) + + #TODO check same packagenames + pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.packageid, self.pyfile.name) + if pyfile: + raise SkipDownload(pyfile[0]) def clean(self): """ clean everything and remove references """ diff --git a/module/web/json_app.py b/module/web/json_app.py index 9e9536a40..738f5ac63 100644 --- a/module/web/json_app.py +++ b/module/web/json_app.py @@ -142,6 +142,8 @@ def package(id): pyfile["icon"] = "status_waiting.png" elif pyfile["status"] == 8: pyfile["icon"] = "status_failed.png" + elif pyfile["status"] == 4: + pyfile["icon"] = "arrow_right.png" elif pyfile["status"] in (11, 13): pyfile["icon"] = "status_proc.png" else: diff --git a/module/web/media/default/img/arrow_right.png b/module/web/media/default/img/arrow_right.png Binary files differnew file mode 100644 index 000000000..b1a181923 --- /dev/null +++ b/module/web/media/default/img/arrow_right.png |
