From 567c1d512e2e7452969d42d272d600d694e40058 Mon Sep 17 00:00:00 2001 From: zoidberg10 Date: Fri, 20 Jan 2012 01:58:41 +0100 Subject: fix threadmanager limitdl, add zevera.com --- module/plugins/accounts/ZeveraCom.py | 105 +++++++++++++++++++++++++++++++++++ module/plugins/hoster/DlFreeFr.py | 82 ++++++++------------------- module/plugins/hoster/ZeveraCom.py | 83 +++++++++++++++++++++++++++ module/threads/ThreadManager.py | 6 +- 4 files changed, 212 insertions(+), 64 deletions(-) create mode 100644 module/plugins/accounts/ZeveraCom.py create mode 100644 module/plugins/hoster/ZeveraCom.py diff --git a/module/plugins/accounts/ZeveraCom.py b/module/plugins/accounts/ZeveraCom.py new file mode 100644 index 000000000..26eac64b6 --- /dev/null +++ b/module/plugins/accounts/ZeveraCom.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +from module.plugins.MultiHoster import MultiHoster + +import re +from time import mktime, strptime + +class ZeveraCom(MultiHoster): + __name__ = "ZeveraCom" + __version__ = "0.11" + __type__ = "account" + __description__ = """Zevera.com account plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + api_url = "http://zevera.com/API.ashx" + + def loadAccountInfo(self, req): + dataRet = self.loadAPIRequest(req) + account_info = { + "trafficleft": dataRet['AccountInfo']['AvailableTODAYTrafficForUseInMBytes'] * 1024, + "validuntil": -1 #dataRet['AccountInfo']['EndSubscriptionDate'] + } + + return account_info + + def login(self, req): + if self.loadAPIRequest(req, parse = False) == 'Login Error': + self.wrongPassword() + + def loadHosterList(self, req): + page = req.load("http://www.zevera.com/jDownloader.ashx?cmd=gethosters") + return [x.strip() for x in page.replace("\"", "").split(",")] + + def loadAPIRequest(self, req, parse = True, **kwargs): + get_dict = { + 'cmd': 'download_request', + 'login': self.loginname, + 'pass': self.password + } + get_dict.update(kwargs) + + response = req.load(self.api_url, get = get_dict, decode = True) + self.logDebug(response) + return self.parseAPIRequest(response) if parse else response + + def parseAPIRequest(self, api_response): + + try: + arFields = iter(api_response.split('TAG BEGIN DATA#')[1].split('#END DATA')[0].split('#')) + + retData = { + 'VersionMajor': arFields.next(), + 'VersionMinor': arFields.next(), + 'ErrorCode': int(arFields.next()), + 'ErrorMessage': arFields.next(), + 'Update_Wait': arFields.next() + } + serverInfo = { + 'DateTimeOnServer': mktime(strptime(arFields.next(),"%Y/%m/%d %H:%M:%S")), + 'DAY_Traffic_LimitInMBytes': int(arFields.next()) + } + accountInfo = { + 'EndSubscriptionDate': mktime(strptime(arFields.next(),"%Y/%m/%d %H:%M:%S")), + 'TrafficUsedInMBytesDayToday': int(arFields.next()), + 'AvailableEXTRATrafficForUseInMBytes': int(arFields.next()), + 'AvailableTODAYTrafficForUseInMBytes': int(arFields.next()) + } + fileInfo = { + 'FileID': arFields.next(), + 'Title': arFields.next(), + 'RealFileName': arFields.next(), + 'FileNameOnServer': arFields.next(), + 'StorageServerURL': arFields.next(), + 'Token': arFields.next(), + 'FileSizeInBytes': int(arFields.next()), + 'StatusID': int(arFields.next()) + } + progress = { + 'BytesReceived': int(arFields.next()), + 'TotalBytesToReceive': int(arFields.next()), + 'Percentage': arFields.next(), + 'StatusText': arFields.next(), + 'ProgressText': arFields.next() + } + fileInfo.update({ + 'Progress': progress, + 'FilePassword': arFields.next(), + 'Keywords': arFields.next(), + 'ImageURL4Download': arFields.next(), + 'CategoryID': arFields.next(), + 'CategoryText': arFields.next(), + 'Notes': arFields.next() + }) + retData.update({ + 'ServerInfo': serverInfo, + 'AccountInfo': accountInfo, + 'FileInfo': fileInfo + }) + + except Exception, e: + self.logError(e) + return None + + self.logDebug(retData) + return retData \ No newline at end of file diff --git a/module/plugins/hoster/DlFreeFr.py b/module/plugins/hoster/DlFreeFr.py index 8ab9dd8eb..7cb58e6f4 100644 --- a/module/plugins/hoster/DlFreeFr.py +++ b/module/plugins/hoster/DlFreeFr.py @@ -2,72 +2,36 @@ # -*- coding: utf-8 -*- import re +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.plugins.Hoster import Hoster - -class DlFreeFr(Hoster): +class DlFreeFr(SimpleHoster): __name__ = "DlFreeFr" __type__ = "hoster" __pattern__ = r"http://dl\.free\.fr/([a-zA-Z0-9]+|getfile\.pl\?file=/[a-zA-Z0-9]+)$" - __version__ = "0.1" + __version__ = "0.2" __description__ = """dl.free.fr download hoster""" - __author_name__ = ("the-razer") - __author_mail__ = ("daniel_ AT gmx DOT net") - + __author_name__ = ("the-razer", "zoidberg") + __author_mail__ = ("daniel_ AT gmx DOT net", "zoidberg@mujmail.cz") + + FILE_NAME_PATTERN = r"Fichier:\s*]*>(?P[^>]*)" + FILE_SIZE_PATTERN = r"Taille:\s*]*>(?P[\d.]+[KMG])" + FILE_OFFLINE_PATTERN = r"Erreur 404 - Document non trouv|Fichier inexistant|Le fichier demandé n'a pas été trouvé" + FILE_URL_PATTERN = r'href="(?Phttp://.*?)">Télécharger ce fichier' + def setup(self): - self.html = None - self.multiDL = False - - def process(self, pyfile): - - self.download_html() - - if not self.file_exists(): - self.log.debug(self.__name__+": File not yet available.") - self.offline() - - pyfile.name = self.get_file_name() - - url = self.get_file_url() - if url: - self.download(url) - else: - self.offline() + self.limitDL = 5 + self.resumeDownload = True + self.chunkLimit = 1 - def download_html(self): - self.html = self.load(self.pyfile.url, cookies=False) - - def file_exists(self): - warnings = (r"Erreur 404 - Document non trouv", - r"Fichier inexistant.", - r"Le fichier demandé n'a pas été trouvé") - expr = '(' + '|'.join(warnings) + ')' - if re.search(expr, self.html) is not None: - return False - return True + def handleFree(self): + if "Trop de slots utilisés" in self.html: + self.retry(300) - def get_file_url(self): - self.log.debug(self.__name__+": Getting file URL") - file_url_pattern = r'href="(?Phttp://.*?)">Télécharger ce fichier' + m = re.search(self.FILE_URL_PATTERN, self.html) + if not m: self.parseError('URL') - m = re.search(file_url_pattern, self.html) - if m is not None: - url = m.group('url') - self.log.debug(self.__name__+": File URL [%s]" % url) - return url - else: - self.log.debug(self.__name__+": Error getting URL") - return False + url = m.group('url') + self.logDebug("File URL [%s]" % url) + self.download(url) - def get_file_name(self): - self.log.debug(self.__name__+": Getting file name") - - file_name_pattern = r"Fichier:\s*(?P.*?)" - m = re.search(file_name_pattern, self.html) - - if m is not None: - name = m.group('name').strip() - self.log.debug(self.__name__+": File name [%s]" % name) - return name - else: - self.log.debug(self.__name__+": Could not find filename") +getInfo = create_getInfo(DlFreeFr) \ No newline at end of file diff --git a/module/plugins/hoster/ZeveraCom.py b/module/plugins/hoster/ZeveraCom.py new file mode 100644 index 000000000..d1fa80802 --- /dev/null +++ b/module/plugins/hoster/ZeveraCom.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from module.plugins.Hoster import Hoster +from module.utils import html_unescape +from urllib import quote, unquote +from time import sleep + +class ZeveraCom(Hoster): + __name__ = "ZeveraCom" + __version__ = "0.11" + __type__ = "hoster" + __pattern__ = r"http://zevera.com/.*" + __description__ = """zevera.com hoster plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + api_url = "http://zevera.com/API.ashx" + + def process(self, pyfile): + if not self.account: + self.logError(_("Please enter your zevera.com account or deactivate this plugin")) + self.fail("No zevera.com account provided") + + self.logDebug("zevera.com: Old URL: %s" % pyfile.url) + + last_size = retries = 0 + olink = self.pyfile.url #quote(self.pyfile.url.encode('utf_8')) + + for i in range(100): + self.retData = self.account.loadAPIRequest(self.req, cmd = 'download_request', olink = olink) + self.checkAPIErrors(self.retData) + + if self.retData['FileInfo']['StatusID'] == 100: + break + elif self.retData['FileInfo']['StatusID'] == 99: + self.fail('Failed to initialize download (99)') + else: + if self.retData['FileInfo']['Progress']['BytesReceived'] <= last_size: + if retries >= 6: + self.fail('Failed to initialize download (%d)' % self.retData['FileInfo']['StatusID'] ) + retries += 1 + else: + retries = 0 + + last_size = self.retData['FileInfo']['Progress']['BytesReceived'] + + pyfile.progress = self.retData['FileInfo']['Progress']['Percentage'] + + self.setWait(self.retData['Update_Wait']) + self.wait() + + pyfile.progress = 0 + pyfile.name = self.crazyDecode(self.retData['FileInfo']['RealFileName']) + pyfile.size = self.retData['FileInfo']['FileSizeInBytes'] + + self.retData = self.account.loadAPIRequest(self.req, cmd = 'download_start', FileID = self.retData['FileInfo']['FileID']) + self.checkAPIErrors(self.retData) + + self.download(self.api_url, get = { + 'cmd': "open_stream", + 'login': self.account.loginname, + 'pass': self.account.password, + 'FileID': self.retData['FileInfo']['FileID'], + 'startBytes': 0 + } + ) + + def checkAPIErrors(self, retData): + if not retData: + self.fail('Unknown API response') + + if retData['ErrorCode']: + self.logError(retData['ErrorCode'], retData['ErrorMessage']) + self.fail('ERROR: ' + retData['ErrorMessage']) + + if self.pyfile.size / 1024000 > retData['AccountInfo']['AvailableTODAYTrafficForUseInMBytes']: + self.logWarning("Not enough data left to download the file") + + def crazyDecode(self, ustring): + # accepts decoded ie. unicode string - API response is double-quoted, double-utf8-encoded + # no idea what the proper order of calling these functions would be :-/ + return html_unescape(unquote(unquote(ustring.replace('@DELIMITER@','#'))).encode('raw_unicode_escape').decode('utf-8')) \ No newline at end of file diff --git a/module/threads/ThreadManager.py b/module/threads/ThreadManager.py index 6a5096b8c..a42d6df16 100644 --- a/module/threads/ThreadManager.py +++ b/module/threads/ThreadManager.py @@ -266,12 +266,8 @@ class ThreadManager: inuse = uniqify([(x.active.pluginname, x.active.plugin.getDownloadLimit()) for x in self.threads if x.active and x.active.hasPlugin()]) inuse = map(lambda x : (x[0], x[1], len([y for y in self.threads if y.active and y.active.pluginname == x[0]])) ,inuse) - onlimit = [x[0] for x in inuse if 0 < x[1] <= x[2]] - - occ = [x.active.pluginname for x in self.threads if x.active and x.active.hasPlugin() and not x.active.plugin.multiDL] + onlimit + occ = tuple(sorted([x[0] for x in inuse if 0 < x[1] <= x[2]])) - occ.sort() - occ = tuple(uniqify(occ)) job = self.core.files.getJob(occ) if job: try: -- cgit v1.2.3