From 8e7d14bae4d3c836f029a1235eb227380acc3f75 Mon Sep 17 00:00:00 2001
From: Walter Purcaro <vuolter@gmail.com>
Date: Mon, 16 Feb 2015 21:59:10 +0100
Subject: Fix plugins to work on 0.4.10

---
 module/plugins/internal/SimpleHoster.py | 701 --------------------------------
 1 file changed, 701 deletions(-)
 delete mode 100644 module/plugins/internal/SimpleHoster.py

(limited to 'module/plugins/internal/SimpleHoster.py')

diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py
deleted file mode 100644
index e4ff1a2d8..000000000
--- a/module/plugins/internal/SimpleHoster.py
+++ /dev/null
@@ -1,701 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import mimetypes
-import os
-import re
-
-from datetime import datetime, timedelta
-from inspect import isclass
-from time import time
-from urllib import unquote
-from urlparse import urljoin, urlparse
-
-from module.PyFile import statusMap as _statusMap
-from module.network.CookieJar import CookieJar
-from module.network.HTTPRequest import BadHeader
-from module.network.RequestFactory import getURL
-from module.plugins.Hoster import Hoster
-from module.plugins.Plugin import Fail
-from module.utils import fixup, fs_encode, parseFileSize
-
-
-#@TODO: Adapt and move to PyFile in 0.4.10
-statusMap = dict((v, k) for k, v in _statusMap.iteritems())
-
-
-#@TODO: Remove in 0.4.10 and redirect to self.error instead
-def _error(self, reason, type):
-        if not reason and not type:
-            type = "unknown"
-
-        msg  = _("%s error") % type.strip().capitalize() if type else _("Error")
-        msg += ": %s" % reason.strip() if reason else ""
-        msg += _(" | Plugin may be out of date")
-
-        raise Fail(msg)
-
-
-#@TODO: Remove in 0.4.10
-def _wait(self, seconds, reconnect):
-    if seconds:
-        self.setWait(int(seconds) + 1)
-
-    if reconnect is not None:
-        self.wantReconnect = reconnect
-
-    super(SimpleHoster, self).wait()
-
-
-def replace_patterns(string, ruleslist):
-    for r in ruleslist:
-        rf, rt = r
-        string = re.sub(rf, rt, string)
-    return string
-
-
-def set_cookies(cj, cookies):
-    for cookie in cookies:
-        if isinstance(cookie, tuple) and len(cookie) == 3:
-            domain, name, value = cookie
-            cj.setCookie(domain, name, value)
-
-
-def parseHtmlTagAttrValue(attr_name, tag):
-    m = re.search(r"%s\s*=\s*([\"']?)((?<=\")[^\"]+|(?<=')[^']+|[^>\s\"'][^>\s]*)\1" % attr_name, tag, re.I)
-    return m.group(2) if m else None
-
-
-def parseHtmlForm(attr_str, html, input_names={}):
-    for form in re.finditer(r"(?P<TAG><form[^>]*%s[^>]*>)(?P<CONTENT>.*?)</?(form|body|html)[^>]*>" % attr_str,
-                            html, re.S | re.I):
-        inputs = {}
-        action = parseHtmlTagAttrValue("action", form.group('TAG'))
-
-        for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=</\2)|)', form.group('CONTENT'), re.S | re.I):
-            name = parseHtmlTagAttrValue("name", inputtag.group(1))
-            if name:
-                value = parseHtmlTagAttrValue("value", inputtag.group(1))
-                if not value:
-                    inputs[name] = inputtag.group(3) or ''
-                else:
-                    inputs[name] = value
-
-        if input_names:
-            # check input attributes
-            for key, val in input_names.iteritems():
-                if key in inputs:
-                    if isinstance(val, basestring) and inputs[key] == val:
-                        continue
-                    elif isinstance(val, tuple) and inputs[key] in val:
-                        continue
-                    elif hasattr(val, "search") and re.match(val, inputs[key]):
-                        continue
-                    break  #: attibute value does not match
-                else:
-                    break  #: attibute name does not match
-            else:
-                return action, inputs  #: passed attribute check
-        else:
-            # no attribute check
-            return action, inputs
-
-    return {}, None  #: no matching form found
-
-
-#: Deprecated
-def parseFileInfo(plugin, url="", html=""):
-    if hasattr(plugin, "getInfo"):
-        info = plugin.getInfo(url, html)
-        res  = info['name'], info['size'], info['status'], info['url']
-    else:
-        url   = unquote(url)
-        url_p = urlparse(url)
-        res   = ((url_p.path.split('/')[-1]
-                  or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0]
-                  or url_p.netloc.split('.', 1)[0]),
-                 0,
-                 3 if url else 8,
-                 url)
-
-    return res
-
-
-#@TODO: Remove in 0.4.10
-#@NOTE: Every plugin must have own parseInfos classmethod to work with 0.4.10
-def create_getInfo(plugin):
-
-    def generator(list):
-        for x in list:
-            yield x
-
-    if hasattr(plugin, "parseInfos"):
-        fn = lambda urls: generator((info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfos(urls))
-    else:
-        fn = lambda urls: generator(parseFileInfo(url) for url in urls)
-
-    return fn
-
-
-def timestamp():
-    return int(time() * 1000)
-
-
-#@TODO: Move to hoster class in 0.4.10
-def fileUrl(self, url, follow_location=None):
-    link     = ""
-    redirect = 1
-
-    if type(follow_location) is int:
-        redirect = max(follow_location, 1)
-    else:
-        redirect = 5
-
-    for i in xrange(redirect):
-        try:
-            self.logDebug("Redirect #%d to: %s" % (i, url))
-            header = self.load(url, ref=True, cookies=True, just_header=True, decode=True)
-
-        except Exception:  #: Bad bad bad...
-            req = pyreq.getHTTPRequest()
-            res = req.load(url, cookies=True, just_header=True, decode=True)
-
-            req.close()
-
-            header = {"code": req.code}
-            for line in res.splitlines():
-                line = line.strip()
-                if not line or ":" not in line:
-                    continue
-
-                key, none, value = line.partition(":")
-                key              = key.lower().strip()
-                value            = value.strip()
-
-                if key in header:
-                    if type(header[key]) == list:
-                        header[key].append(value)
-                    else:
-                        header[key] = [header[key], value]
-                else:
-                    header[key] = value
-
-        if 'content-disposition' in header:
-            link = url
-
-        elif 'location' in header and header['location'].strip():
-            location = header['location']
-
-            if not urlparse(location).scheme:
-                url_p    = urlparse(url)
-                baseurl  = "%s://%s" % (url_p.scheme, url_p.netloc)
-                location = urljoin(baseurl, location)
-
-            if 'code' in header and header['code'] == 302:
-                link = location
-
-            if follow_location:
-                url = location
-                continue
-
-        else:
-            extension = os.path.splitext(urlparse(url).path.split('/')[-1])[-1]
-
-            if 'content-type' in header and header['content-type'].strip():
-                mimetype = header['content-type'].split(';')[0].strip()
-
-            elif extension:
-                mimetype = mimetypes.guess_type(extension, False)[0] or "application/octet-stream"
-
-            else:
-                mimetype = ""
-
-            if mimetype and (link or 'html' not in mimetype):
-                link = url
-            else:
-                link = ""
-
-        break
-
-    else:
-        try:
-            self.logError(_("Too many redirects"))
-        except Exception:
-            pass
-
-    return link
-
-
-def secondsToMidnight(gmt=0):
-    now = datetime.utcnow() + timedelta(hours=gmt)
-
-    if now.hour is 0 and now.minute < 10:
-        midnight = now
-    else:
-        midnight = now + timedelta(days=1)
-
-    td = midnight.replace(hour=0, minute=10, second=0, microsecond=0) - now
-
-    if hasattr(td, 'total_seconds'):
-        res = td.total_seconds()
-    else:  #@NOTE: work-around for python 2.5 and 2.6 missing timedelta.total_seconds
-        res = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6
-
-    return int(res)
-
-
-class SimpleHoster(Hoster):
-    __name__    = "SimpleHoster"
-    __type__    = "hoster"
-    __version__ = "1.15"
-
-    __pattern__ = r'^unmatchable$'
-
-    __description__ = """Simple hoster plugin"""
-    __license__     = "GPLv3"
-    __authors__     = [("zoidberg", "zoidberg@mujmail.cz"),
-                       ("stickell", "l.stickell@yahoo.it"),
-                       ("Walter Purcaro", "vuolter@gmail.com")]
-
-
-    """
-    Info patterns should be defined by each hoster:
-
-      INFO_PATTERN: (optional) Name and Size of the file
-        example: INFO_PATTERN = r'(?P<N>file_name) (?P<S>file_size) (?P<U>size_unit)'
-      or
-        NAME_PATTERN: (optional) Name that will be set for the file
-          example: NAME_PATTERN = r'(?P<N>file_name)'
-        SIZE_PATTERN: (optional) Size that will be checked for the file
-          example: SIZE_PATTERN = r'(?P<S>file_size) (?P<U>size_unit)'
-
-      HASHSUM_PATTERN: (optional) Hash code and type of the file
-        example: HASHSUM_PATTERN = r'(?P<H>hash_code) (?P<T>MD5)'
-
-      OFFLINE_PATTERN: (optional) Check if the page is unreachable
-        example: OFFLINE_PATTERN = r'File (deleted|not found)'
-
-      TEMP_OFFLINE_PATTERN: (optional) Check if the page is temporarily unreachable
-        example: TEMP_OFFLINE_PATTERN = r'Server (maintenance|maintainance)'
-
-
-    Error handling patterns are all optional:
-
-      WAIT_PATTERN: (optional) Detect waiting time
-        example: WAIT_PATTERN = r''
-
-      PREMIUM_ONLY_PATTERN: (optional) Check if the file can be downloaded only with a premium account
-        example: PREMIUM_ONLY_PATTERN = r'Premium account required'
-
-      ERROR_PATTERN: (optional) Detect any error preventing download
-        example: ERROR_PATTERN = r''
-
-
-    Instead overriding handleFree and handlePremium methods you can define the following patterns for direct download:
-
-      LINK_FREE_PATTERN: (optional) group(1) should be the direct link for free download
-        example: LINK_FREE_PATTERN = r'<div class="link"><a href="(.+?)"'
-
-      LINK_PREMIUM_PATTERN: (optional) group(1) should be the direct link for premium download
-        example: LINK_PREMIUM_PATTERN = r'<div class="link"><a href="(.+?)"'
-    """
-
-    NAME_REPLACEMENTS = [("&#?\w+;", fixup)]
-    SIZE_REPLACEMENTS = []
-    URL_REPLACEMENTS  = []
-
-    TEXT_ENCODING = False  #: Set to True or encoding name if encoding value in http header is not correct
-    COOKIES       = True   #: or False or list of tuples [(domain, name, value)]
-    CHECK_TRAFFIC = False  #: Set to True to force checking traffic left for premium account
-    DIRECT_LINK   = None   #: Set to True to looking for direct link (as defined in handleDirect method), set to None to do it if self.account is True else False
-    MULTI_HOSTER  = False  #: Set to True to leech other hoster link (as defined in handleMulti method)
-    LOGIN_ACCOUNT = False  #: Set to True to require account login
-    DISPOSITION   = True   #: Work-around to `filename*=UTF-8` bug; remove in 0.4.10
-
-    directLink = fileUrl  #@TODO: Remove in 0.4.10
-
-
-    @classmethod
-    def parseInfos(cls, urls):  #@TODO: Built-in in 0.4.10 core, then remove from plugins
-        for url in urls:
-            url = replace_patterns(url, cls.URL_REPLACEMENTS)
-            yield cls.getInfo(url)
-
-
-    @classmethod
-    def apiInfo(cls, url="", get={}, post={}):
-        url   = unquote(url)
-        url_p = urlparse(url)
-        return {'name'  : (url_p.path.split('/')[-1]
-                           or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0]
-                           or url_p.netloc.split('.', 1)[0]),
-                'size'  : 0,
-                'status': 3 if url else 8,
-                'url'   : url}
-
-
-    @classmethod
-    def getInfo(cls, url="", html=""):
-        info   = cls.apiInfo(url)
-        online = False if info['status'] != 2 else True
-
-        try:
-            info['pattern'] = re.match(cls.__pattern__, url).groupdict()  #: pattern groups will be saved here
-
-        except Exception:
-            info['pattern'] = {}
-
-        if not html and not online:
-            if not url:
-                info['error']  = "missing url"
-                info['status'] = 1
-
-            elif info['status'] is 3 and not fileUrl(None, url):
-                try:
-                    html = getURL(url, cookies=cls.COOKIES, decode=not cls.TEXT_ENCODING)
-
-                    if isinstance(cls.TEXT_ENCODING, basestring):
-                        html = unicode(html, cls.TEXT_ENCODING)
-
-                except BadHeader, e:
-                    info['error'] = "%d: %s" % (e.code, e.content)
-
-                    if e.code is 404:
-                        info['status'] = 1
-
-                    elif e.code is 503:
-                        info['status'] = 6
-
-        if html:
-            if hasattr(cls, "OFFLINE_PATTERN") and re.search(cls.OFFLINE_PATTERN, html):
-                info['status'] = 1
-
-            elif hasattr(cls, "TEMP_OFFLINE_PATTERN") and re.search(cls.TEMP_OFFLINE_PATTERN, html):
-                info['status'] = 6
-
-            else:
-                for pattern in ("INFO_PATTERN", "NAME_PATTERN", "SIZE_PATTERN", "HASHSUM_PATTERN"):
-                    try:
-                        attr  = getattr(cls, pattern)
-                        pdict = re.search(attr, html).groupdict()
-
-                        if all(True for k in pdict if k not in info['pattern']):
-                            info['pattern'].update(pdict)
-
-                    except AttributeError:
-                        continue
-
-                    else:
-                        online = True
-
-        if online:
-            info['status'] = 2
-
-            if 'N' in info['pattern']:
-                info['name'] = replace_patterns(unquote(info['pattern']['N'].strip()),
-                                                cls.NAME_REPLACEMENTS)
-
-            if 'S' in info['pattern']:
-                size = replace_patterns(info['pattern']['S'] + info['pattern']['U'] if 'U' in info['pattern'] else info['pattern']['S'],
-                                        cls.SIZE_REPLACEMENTS)
-                info['size'] = parseFileSize(size)
-
-            elif isinstance(info['size'], basestring):
-                unit = info['units'] if 'units' in info else None
-                info['size'] = parseFileSize(info['size'], unit)
-
-            if 'H' in info['pattern']:
-                hashtype = info['pattern']['T'] if 'T' in info['pattern'] else "hash"
-                info[hashtype] = info['pattern']['H']
-
-        if not info['pattern']:
-            info.pop('pattern', None)
-
-        return info
-
-
-    def setup(self):
-        self.resumeDownload = self.multiDL = self.premium
-
-
-    def prepare(self):
-        self.pyfile.error = ""  #@TODO: Remove in 0.4.10
-
-        self.info      = {}
-        self.html      = ""
-        self.link      = ""     #@TODO: Move to hoster class in 0.4.10
-        self.directDL  = False  #@TODO: Move to hoster class in 0.4.10
-        self.multihost = False  #@TODO: Move to hoster class in 0.4.10
-
-        if self.LOGIN_ACCOUNT and not self.account:
-            self.fail(_("Required account not found"))
-
-        self.req.setOption("timeout", 120)
-
-        if isinstance(self.COOKIES, list):
-            set_cookies(self.req.cj, self.COOKIES)
-
-        if (self.MULTI_HOSTER
-            and (self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern']
-                 or re.match(self.__pattern__, self.pyfile.url) is None)):
-            self.multihost = True
-            return
-
-        if self.DIRECT_LINK is None:
-            self.directDL = bool(self.account)
-        else:
-            self.directDL = self.DIRECT_LINK
-
-        self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS)
-
-
-    def preload(self):
-        self.html = self.load(self.pyfile.url, cookies=bool(self.COOKIES), decode=not self.TEXT_ENCODING)
-
-        if isinstance(self.TEXT_ENCODING, basestring):
-            self.html = unicode(self.html, self.TEXT_ENCODING)
-
-
-    def process(self, pyfile):
-        self.prepare()
-        self.checkInfo()
-
-        if self.directDL:
-            self.logDebug("Looking for direct download link...")
-            self.handleDirect(pyfile)
-
-        if self.multihost and not self.link and not self.lastDownload:
-            self.logDebug("Looking for leeched download link...")
-            self.handleMulti(pyfile)
-
-            if not self.link and not self.lastDownload:
-                self.MULTI_HOSTER = False
-                self.retry(1, reason="Multi hoster fails")
-
-        if not self.link and not self.lastDownload:
-            self.preload()
-            self.checkInfo()
-
-            if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()):
-                self.logDebug("Handled as premium download")
-                self.handlePremium(pyfile)
-
-            elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()):
-                self.logDebug("Handled as free download")
-                self.handleFree(pyfile)
-
-        self.downloadLink(self.link, self.DISPOSITION)  #: Remove `self.DISPOSITION` in 0.4.10
-        self.checkFile()
-
-
-    def downloadLink(self, link, disposition=True):
-        if link and isinstance(link, basestring):
-            self.correctCaptcha()
-
-            if not urlparse(link).scheme:
-                url_p   = urlparse(self.pyfile.url)
-                baseurl = "%s://%s" % (url_p.scheme, url_p.netloc)
-                link    = urljoin(baseurl, link)
-
-            self.download(link, ref=False, disposition=disposition)
-
-
-    def checkFile(self):
-        if self.cTask and not self.lastDownload:
-            self.invalidCaptcha()
-            self.retry(10, reason=_("Wrong captcha"))
-
-        elif not self.lastDownload or not os.path.exists(fs_encode(self.lastDownload)):
-            self.lastDownload = ""
-            self.error(self.pyfile.error or _("No file downloaded"))
-
-        else:
-            rules = {'empty file': re.compile(r'\A\Z'),
-                     'html file' : re.compile(r'\A\s*<!DOCTYPE html'),
-                     'html error': re.compile(r'\A\s*(<.+>)?\d{3}(\Z|\s+)')}
-
-            if hasattr(self, 'ERROR_PATTERN'):
-                rules['error'] = re.compile(self.ERROR_PATTERN)
-
-            check = self.checkDownload(rules)
-            if check:  #@TODO: Move to hoster in 0.4.10
-                errmsg = check.strip().capitalize()
-                if self.lastCheck:
-                    errmsg += " | " + self.lastCheck.group(0).strip()
-
-                self.lastDownload = ""
-                self.retry(10, 60, errmsg)
-
-
-    def checkErrors(self):
-        if not self.html:
-            self.logWarning(_("No html code to check"))
-            return
-
-        if hasattr(self, 'PREMIUM_ONLY_PATTERN') and not self.premium and re.search(self.PREMIUM_ONLY_PATTERN, self.html):
-            self.fail(_("Link require a premium account to be handled"))
-
-        elif hasattr(self, 'ERROR_PATTERN'):
-            m = re.search(self.ERROR_PATTERN, self.html)
-            if m:
-                errmsg = self.info['error'] = m.group(1)
-                self.error(errmsg)
-
-        elif hasattr(self, 'WAIT_PATTERN'):
-            m = re.search(self.WAIT_PATTERN, self.html)
-            if m:
-                wait_time = sum(int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in
-                                re.findall(r'(\d+)\s*(hr|hour|min|sec)', m.group(0), re.I))
-                self.wait(wait_time, wait_time > 300)
-                return
-
-        self.info.pop('error', None)
-
-
-    def checkStatus(self, getinfo=True):
-        if not self.info or getinfo:
-            self.logDebug("Update file info...")
-            self.logDebug("Previous file info: %s" % self.info)
-            self.info.update(self.getInfo(self.pyfile.url, self.html))
-            self.logDebug("Current file info: %s"  % self.info)
-
-        try:
-            status = self.info['status']
-
-            if status is 1:
-                self.offline()
-
-            elif status is 6:
-                self.tempOffline()
-
-            elif status is 8:
-                self.fail()
-
-        finally:
-            self.logDebug("File status: %s" % statusMap[status])
-
-
-    def checkNameSize(self, getinfo=True):
-        if not self.info or getinfo:
-            self.logDebug("Update file info...")
-            self.logDebug("Previous file info: %s" % self.info)
-            self.info.update(self.getInfo(self.pyfile.url, self.html))
-            self.logDebug("Current file info: %s"  % self.info)
-
-        try:
-            url  = self.info['url'].strip()
-            name = self.info['name'].strip()
-            if name and name != url:
-                self.pyfile.name = name
-
-        except Exception:
-            pass
-
-        try:
-            size = self.info['size']
-            if size > 0:
-                self.pyfile.size = size
-
-        except Exception:
-            pass
-
-        self.logDebug("File name: %s" % self.pyfile.name,
-                      "File size: %s byte" % self.pyfile.size if self.pyfile.size > 0 else "File size: Unknown")
-
-
-    def checkInfo(self):
-        self.checkNameSize()
-
-        if self.html:
-            self.checkErrors()
-            self.checkNameSize()
-
-        self.checkStatus(getinfo=False)
-
-
-    #: Deprecated
-    def getFileInfo(self):
-        self.info = {}
-        self.checkInfo()
-        return self.info
-
-
-    def handleDirect(self, pyfile):
-        link = self.directLink(pyfile.url, self.resumeDownload)
-
-        if link:
-            self.logInfo(_("Direct download link detected"))
-
-            self.link = link
-        else:
-            self.logDebug("Direct download link not found")
-
-
-    def handleMulti(self, pyfile):  #: Multi-hoster handler
-        pass
-
-
-    def handleFree(self, pyfile):
-        if not hasattr(self, 'LINK_FREE_PATTERN'):
-            self.logError(_("Free download not implemented"))
-
-        m = re.search(self.LINK_FREE_PATTERN, self.html)
-        if m is None:
-            self.error(_("Free download link not found"))
-        else:
-            self.link = m.group(1)
-
-
-    def handlePremium(self, pyfile):
-        if not hasattr(self, 'LINK_PREMIUM_PATTERN'):
-            self.logError(_("Premium download not implemented"))
-            self.logDebug("Handled as free download")
-            self.handleFree(pyfile)
-
-        m = re.search(self.LINK_PREMIUM_PATTERN, self.html)
-        if m is None:
-            self.error(_("Premium download link not found"))
-        else:
-            self.link = m.group(1)
-
-
-    def longWait(self, wait_time=None, max_tries=3):
-        if wait_time and isinstance(wait_time, (int, long, float)):
-            time_str  = "%dh %dm" % divmod(wait_time / 60, 60)
-        else:
-            wait_time = 900
-            time_str  = _("(unknown time)")
-            max_tries = 100
-
-        self.logInfo(_("Download limit reached, reconnect or wait %s") % time_str)
-
-        self.wait(wait_time, True)
-        self.retry(max_tries=max_tries, reason=_("Download limit reached"))
-
-
-    def parseHtmlForm(self, attr_str="", input_names={}):
-        return parseHtmlForm(attr_str, self.html, input_names)
-
-
-    def checkTrafficLeft(self):
-        if not self.account:
-            return True
-
-        traffic = self.account.getAccountInfo(self.user, True)['trafficleft']
-
-        if traffic is None:
-            return False
-        elif traffic == -1:
-            return True
-        else:
-            size = self.pyfile.size / 1024
-            self.logInfo(_("Filesize: %i KiB, Traffic left for user %s: %i KiB") % (size, self.user, traffic))
-            return size <= traffic
-
-
-    #@TODO: Remove in 0.4.10
-    def wait(self, seconds=0, reconnect=None):
-        return _wait(self, seconds, reconnect)
-
-
-    def error(self, reason="", type="parse"):
-        return _error(self, reason, type)
-- 
cgit v1.2.3