From 1bd8de6bebb3f76bc5c09566aec682374150f2f8 Mon Sep 17 00:00:00 2001
From: Walter Purcaro <vuolter@gmail.com>
Date: Sun, 28 Sep 2014 21:32:34 +0200
Subject: Changed XFileSharingPro and UpdateManager to internal plugins

---
 pyload/plugins/internal/UpdateManager.py   | 281 +++++++++++++++++++++++
 pyload/plugins/internal/XFileSharingPro.py | 352 +++++++++++++++++++++++++++++
 2 files changed, 633 insertions(+)
 create mode 100644 pyload/plugins/internal/UpdateManager.py
 create mode 100644 pyload/plugins/internal/XFileSharingPro.py

(limited to 'pyload/plugins/internal')

diff --git a/pyload/plugins/internal/UpdateManager.py b/pyload/plugins/internal/UpdateManager.py
new file mode 100644
index 000000000..eb9411236
--- /dev/null
+++ b/pyload/plugins/internal/UpdateManager.py
@@ -0,0 +1,281 @@
+# -*- coding: utf-8 -*-
+
+import re
+import sys
+
+from operator import itemgetter
+from os import path, remove, stat
+
+from pyload.network.RequestFactory import getURL
+from pyload.plugins.Hook import Expose, Hook, threaded
+from pyload.utils import safe_join
+
+
+class UpdateManager(Hook):
+    __name__ = "UpdateManager"
+    __type__ = "hook"
+    __version__ = "0.35"
+
+    __config__ = [("activated", "bool", "Activated", True),
+                  ("mode", "pyLoad + plugins;plugins only", "Check updates for", "pyLoad + plugins"),
+                  ("interval", "int", "Check interval in hours", 8),
+                  ("reloadplugins", "bool", "Monitor plugins for code changes (debug mode only)", True),
+                  ("nodebugupdate", "bool", "Don't check for updates in debug mode", True)]
+
+    __description__ = """ Check for updates """
+    __author_name__ = "Walter Purcaro"
+    __author_mail__ = "vuolter@gmail.com"
+
+
+    event_list = ["pluginConfigChanged"]
+
+    SERVER_URL = "http://updatemanager.pyload.org"
+    MIN_INTERVAL = 3 * 60 * 60  #: 3h minimum check interval (value is in seconds)
+
+
+    def pluginConfigChanged(self, plugin, name, value):
+        if name == "interval":
+            interval = value * 60 * 60
+            if self.MIN_INTERVAL <= interval != self.interval:
+                self.core.scheduler.removeJob(self.cb)
+                self.interval = interval
+                self.initPeriodical()
+            else:
+                self.logDebug("Invalid interval value, kept current")
+        elif name == "reloadplugins":
+            if self.cb2:
+                self.core.scheduler.removeJob(self.cb2)
+            if value is True and self.core.debug:
+                self.periodical2()
+
+    def coreReady(self):
+        self.pluginConfigChanged(self.__name__, "interval", self.getConfig("interval"))
+        x = lambda: self.pluginConfigChanged(self.__name__, "reloadplugins", self.getConfig("reloadplugins"))
+        self.core.scheduler.addJob(10, x, threaded=False)
+
+    def unload(self):
+        self.pluginConfigChanged(self.__name__, "reloadplugins", False)
+
+    def setup(self):
+        self.cb2 = None
+        self.interval = self.MIN_INTERVAL
+        self.updating = False
+        self.info = {'pyload': False, 'version': None, 'plugins': False}
+        self.mtimes = {}  #: store modification time for each plugin
+
+    def periodical2(self):
+        if not self.updating:
+            self.autoreloadPlugins()
+        self.cb2 = self.core.scheduler.addJob(4, self.periodical2, threaded=False)
+
+    @Expose
+    def autoreloadPlugins(self):
+        """ reload and reindex all modified plugins """
+        modules = filter(
+            lambda m: m and (m.__name__.startswith("pyload.plugins.") or
+                             m.__name__.startswith("userplugins.")) and
+                             m.__name__.count(".") >= 2, sys.modules.itervalues()
+        )
+
+        reloads = []
+
+        for m in modules:
+            root, type, name = m.__name__.rsplit(".", 2)
+            id = (type, name)
+            if type in self.core.pluginManager.plugins:
+                f = m.__file__.replace(".pyc", ".py")
+                if not path.isfile(f):
+                    continue
+
+                mtime = stat(f).st_mtime
+
+                if id not in self.mtimes:
+                    self.mtimes[id] = mtime
+                elif self.mtimes[id] < mtime:
+                    reloads.append(id)
+                    self.mtimes[id] = mtime
+
+        return True if self.core.pluginManager.reloadPlugins(reloads) else False
+
+    def periodical(self):
+        if not self.info['pyload'] and not (self.getConfig("nodebugupdate") and self.core.debug):
+            self.updateThread()
+
+    def server_request(self):
+        try:
+            return getURL(self.SERVER_URL, get={'v': self.core.api.getServerVersion()}).splitlines()
+        except:
+            self.logWarning(_("Unable to contact server to get updates"))
+
+    @threaded
+    def updateThread(self):
+        self.updating = True
+        status = self.update(onlyplugin=self.getConfig("mode") == "plugins only")
+        if status == 2:
+            self.core.api.restart()
+        else:
+            self.updating = False
+
+    @Expose
+    def updatePlugins(self):
+        """ simple wrapper for calling plugin update quickly """
+        return self.update(onlyplugin=True)
+
+    @Expose
+    def update(self, onlyplugin=False):
+        """ check for updates """
+        data = self.server_request()
+        if not data:
+            exitcode = 0
+        elif data[0] == "None":
+            self.logInfo(_("No new pyLoad version available"))
+            updates = data[1:]
+            exitcode = self._updatePlugins(updates)
+        elif onlyplugin:
+            exitcode = 0
+        else:
+            newversion = data[0]
+            self.logInfo(_("***  New pyLoad Version %s available  ***") % newversion)
+            self.logInfo(_("***  Get it here: https://github.com/pyload/pyload/releases  ***"))
+            exitcode = 3
+            self.info['pyload'] = True
+            self.info['version'] = newversion
+        return exitcode  #: 0 = No plugins updated; 1 = Plugins updated; 2 = Plugins updated, but restart required; 3 = No plugins updated, new pyLoad version available
+
+    def _updatePlugins(self, updates):
+        """ check for plugin updates """
+
+        if self.info['plugins']:
+            return False  #: plugins were already updated
+
+        updated = []
+
+        vre = re.compile(r'__version__.*=.*("|\')([0-9.]+)')
+        url = updates[0]
+        schema = updates[1].split('|')
+        if "BLACKLIST" in updates:
+            blacklist = updates[updates.index('BLACKLIST') + 1:]
+            updates = updates[2:updates.index('BLACKLIST')]
+        else:
+            blacklist = None
+            updates = updates[2:]
+
+        upgradable = sorted(map(lambda x: dict(zip(schema, x.split('|'))), updates), key=itemgetter("type", "name"))
+        for plugin in upgradable:
+            filename = plugin['name']
+            prefix = plugin['type']
+            version = plugin['version']
+
+            if filename.endswith(".pyc"):
+                name = filename[:filename.find("_")]
+            else:
+                name = filename.replace(".py", "")
+
+            #@TODO: obsolete after 0.4.10
+            if prefix.endswith("s"):
+                type = prefix[:-1]
+            else:
+                type = prefix
+
+            plugins = getattr(self.core.pluginManager, "%sPlugins" % type)
+
+            oldver = float(plugins[name]['v']) if name in plugins else None
+            newver = float(version)
+
+            if not oldver:
+                msg = "New [%(type)s] %(name)s (v%(newver)s)"
+            elif newver > oldver:
+                msg = "New version of [%(type)s] %(name)s (v%(oldver)s -> v%(newver)s)"
+            else:
+                continue
+
+            self.logInfo(_(msg) % {
+                'type': type,
+                'name': name,
+                'oldver': oldver,
+                'newver': newver,
+            })
+
+            try:
+                content = getURL(url % plugin)
+                m = vre.search(content)
+                if m and m.group(2) == version:
+                    f = open(safe_join("userplugins", prefix, filename), "wb")
+                    f.write(content)
+                    f.close()
+                    updated.append((prefix, name))
+                else:
+                    raise Exception, _("Version mismatch")
+            except Exception, e:
+                self.logError(_("Error updating plugin %s") % filename, e)
+
+        if blacklist:
+            blacklisted = sorted(map(lambda x: (x.split('|')[0], x.split('|')[1].rsplit('.', 1)[0]), blacklist))
+
+            # Always protect UpdateManager from self-removing
+            try:
+                blacklisted.remove(("internal", "UpdateManager"))
+            except:
+                pass
+
+            removed = self.removePlugins(blacklisted)
+            for t, n in removed:
+                self.logInfo(_("Removed blacklisted plugin [%(type)s] %(name)s") % {
+                    'type': t,
+                    'name': n,
+                })
+
+        if updated:
+            reloaded = self.core.pluginManager.reloadPlugins(updated)
+            if reloaded:
+                self.logInfo(_("Plugins updated and reloaded"))
+                exitcode = 1
+            else:
+                self.logInfo(_("*** Plugins have been updated, but need a pyLoad restart to be reloaded ***"))
+                self.info['plugins'] = True
+                exitcode = 2
+        else:
+            self.logInfo(_("No plugin updates available"))
+            exitcode = 0
+
+        return exitcode  #: 0 = No plugins updated; 1 = Plugins updated; 2 = Plugins updated, but restart required
+
+    @Expose
+    def removePlugins(self, type_plugins):
+        """ delete plugins from disk """
+
+        if not type_plugins:
+            return
+
+        self.logDebug("Requested deletion of plugins", type_plugins)
+
+        removed = []
+
+        for type, name in type_plugins:
+            err = False
+            file = name + ".py"
+
+            for root in ("userplugins", path.join(pypath, "pyload", "plugins")):
+
+                filename = safe_join(root, type, file)
+                try:
+                    remove(filename)
+                except Exception, e:
+                    self.logDebug("Error deleting", path.basename(filename), e)
+                    err = True
+
+                filename += "c"
+                if path.isfile(filename):
+                    try:
+                        if type == "hook":
+                            self.manager.deactivateHook(name)
+                        remove(filename)
+                    except Exception, e:
+                        self.logDebug("Error deleting", path.basename(filename), e)
+                        err = True
+
+            if not err:
+                id = (type, name)
+                removed.append(id)
+
+        return removed  #: return a list of the plugins successfully removed
diff --git a/pyload/plugins/internal/XFileSharingPro.py b/pyload/plugins/internal/XFileSharingPro.py
new file mode 100644
index 000000000..212ef23ef
--- /dev/null
+++ b/pyload/plugins/internal/XFileSharingPro.py
@@ -0,0 +1,352 @@
+# -*- coding: utf-8 -*-
+
+import re
+
+from pycurl import FOLLOWLOCATION, LOW_SPEED_TIME
+from random import random
+from urllib import unquote
+from urlparse import urlparse
+
+from pyload.network.RequestFactory import getURL
+from pyload.plugins.internal.CaptchaService import ReCaptcha, SolveMedia
+from pyload.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, PluginParseError, replace_patterns
+from pyload.utils import html_unescape
+
+
+class XFileSharingPro(SimpleHoster):
+    """
+    Common base for XFileSharingPro hosters like EasybytezCom, CramitIn, FiledinoCom...
+    Some hosters may work straight away when added to __pattern__
+    However, most of them will NOT work because they are either down or running a customized version
+    """
+    __name__ = "XFileSharingPro"
+    __type__ = "hoster"
+    __version__ = "0.36"
+
+    __pattern__ = r'^unmatchable$'
+
+    __description__ = """XFileSharingPro base hoster plugin"""
+    __author_name__ = ("zoidberg", "stickell", "Walter Purcaro")
+    __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it", "vuolter@gmail.com")
+
+
+    HOSTER_NAME = None
+
+    FILE_URL_REPLACEMENTS = [(r'/embed-(\w{12}).*', r'/\1')]  #: support embedded files
+
+    COOKIES = [(HOSTER_NAME, "lang", "english")]
+
+    FILE_INFO_PATTERN = r'<tr><td align=right><b>Filename:</b></td><td nowrap>(?P<N>[^<]+)</td></tr>\s*.*?<small>\((?P<S>[^<]+)\)</small>'
+    FILE_NAME_PATTERN = r'<input type="hidden" name="fname" value="(?P<N>[^"]+)"'
+    FILE_SIZE_PATTERN = r'You have requested .*\((?P<S>[\d\.\,]+) ?(?P<U>\w+)?\)</font>'
+
+    OFFLINE_PATTERN = r'>\w+ (Not Found|file (was|has been) removed)'
+
+    WAIT_PATTERN = r'<span id="countdown_str">.*?>(\d+)</span>'
+
+    OVR_LINK_PATTERN = r'<h2>Download Link</h2>\s*<textarea[^>]*>([^<]+)'
+    LINK_PATTERN = None  #: final download url pattern
+
+    CAPTCHA_URL_PATTERN = r'(http://[^"\']+?/captchas?/[^"\']+)'
+    RECAPTCHA_URL_PATTERN = r'http://[^"\']+?recaptcha[^"\']+?\?k=([^"\']+)"'
+    CAPTCHA_DIV_PATTERN = r'>Enter code.*?<div.*?>(.*?)</div>'
+    SOLVEMEDIA_PATTERN = r'http:\/\/api\.solvemedia\.com\/papi\/challenge\.script\?k=(.*?)"'
+
+    ERROR_PATTERN = r'class=["\']err["\'][^>]*>(.*?)</'
+
+
+    def setup(self):
+        self.chunkLimit = 1
+
+        if self.__name__ == "XFileSharingPro":
+            self.multiDL = True
+            self.__pattern__ = self.core.pluginManager.hosterPlugins[self.__name__]['pattern']
+            self.HOSTER_NAME = re.match(self.__pattern__, self.pyfile.url).group(1).lower()
+            self.COOKIES = [(self.HOSTER_NAME, "lang", "english")]
+        else:
+            self.resumeDownload = self.multiDL = self.premium
+
+
+    def prepare(self):
+        """ Initialize important variables """
+        if not self.HOSTER_NAME:
+            self.fail("Missing HOSTER_NAME")
+
+        if not self.LINK_PATTERN:
+            pattr = r'(http://([^/]*?%s|\d+\.\d+\.\d+\.\d+)(:\d+)?(/d/|(?:/files)?/\d+/\w+/)[^"\'<]+)'
+            self.LINK_PATTERN = pattr % self.HOSTER_NAME
+
+        if isinstance(self.COOKIES, list):
+            set_cookies(self.req.cj, self.COOKIES)
+
+        self.captcha = None
+        self.errmsg = None
+        self.passwords = self.getPassword().splitlines()
+
+
+    def process(self, pyfile):
+        self.prepare()
+
+        pyfile.url = replace_patterns(pyfile.url, self.FILE_URL_REPLACEMENTS)
+
+        if not re.match(self.__pattern__, pyfile.url):
+            if self.premium:
+                self.handleOverriden()
+            else:
+                self.fail("Only premium users can download from other hosters with %s" % self.HOSTER_NAME)
+        else:
+            try:
+                # Due to a 0.4.9 core bug self.load would use cookies even if
+                # cookies=False. Workaround using getURL to avoid cookies.
+                # Can be reverted in 0.4.10 as the cookies bug has been fixed.
+                self.html = getURL(pyfile.url, decode=True, cookies=self.COOKIES)
+                self.file_info = self.getFileInfo()
+            except PluginParseError:
+                self.file_info = None
+
+            self.location = self.getDirectDownloadLink()
+
+            if not self.file_info:
+                pyfile.name = html_unescape(unquote(urlparse(
+                    self.location if self.location else pyfile.url).path.split("/")[-1]))
+
+            if self.location:
+                self.startDownload(self.location)
+            elif self.premium:
+                self.handlePremium()
+            else:
+                self.handleFree()
+
+
+    def getDirectDownloadLink(self):
+        """ Get download link for premium users with direct download enabled """
+        self.req.http.lastURL = self.pyfile.url
+
+        self.req.http.c.setopt(FOLLOWLOCATION, 0)
+        self.html = self.load(self.pyfile.url, decode=True)
+        self.header = self.req.http.header
+        self.req.http.c.setopt(FOLLOWLOCATION, 1)
+
+        location = None
+        m = re.search(r"Location\s*:\s*(.*)", self.header, re.I)
+        if m and re.match(self.LINK_PATTERN, m.group(1)):
+            location = m.group(1).strip()
+
+        return location
+
+
+    def handleFree(self):
+        url = self.getDownloadLink()
+        self.logDebug("Download URL: %s" % url)
+        self.startDownload(url)
+
+
+    def getDownloadLink(self):
+        for i in xrange(5):
+            self.logDebug("Getting download link: #%d" % i)
+            data = self.getPostParameters()
+
+            self.req.http.c.setopt(FOLLOWLOCATION, 0)
+            self.html = self.load(self.pyfile.url, post=data, ref=True, decode=True)
+            self.header = self.req.http.header
+            self.req.http.c.setopt(FOLLOWLOCATION, 1)
+
+            m = re.search(r"Location\s*:\s*(.*)", self.header, re.I)
+            if m:
+                break
+
+            m = re.search(self.LINK_PATTERN, self.html, re.S)
+            if m:
+                break
+
+        else:
+            if self.errmsg and 'captcha' in self.errmsg:
+                self.fail("No valid captcha code entered")
+            else:
+                self.fail("Download link not found")
+
+        return m.group(1)
+
+
+    def handlePremium(self):
+        self.html = self.load(self.pyfile.url, post=self.getPostParameters())
+        m = re.search(self.LINK_PATTERN, self.html)
+        if m is None:
+            self.parseError('DIRECT LINK')
+        self.startDownload(m.group(1))
+
+
+    def handleOverriden(self):
+        #only tested with easybytez.com
+        self.html = self.load("http://www.%s/" % self.HOSTER_NAME)
+        action, inputs = self.parseHtmlForm('')
+        upload_id = "%012d" % int(random() * 10 ** 12)
+        action += upload_id + "&js_on=1&utype=prem&upload_type=url"
+        inputs['tos'] = '1'
+        inputs['url_mass'] = self.pyfile.url
+        inputs['up1oad_type'] = 'url'
+
+        self.logDebug(self.HOSTER_NAME, action, inputs)
+        #wait for file to upload to easybytez.com
+        self.req.http.c.setopt(LOW_SPEED_TIME, 600)
+        self.html = self.load(action, post=inputs)
+
+        action, inputs = self.parseHtmlForm('F1')
+        if not inputs:
+            self.parseError('TEXTAREA')
+        self.logDebug(self.HOSTER_NAME, inputs)
+        if inputs['st'] == 'OK':
+            self.html = self.load(action, post=inputs)
+        elif inputs['st'] == 'Can not leech file':
+            self.retry(max_tries=20, wait_time=3 * 60, reason=inputs['st'])
+        else:
+            self.fail(inputs['st'])
+
+        #get easybytez.com link for uploaded file
+        m = re.search(self.OVR_LINK_PATTERN, self.html)
+        if m is None:
+            self.parseError('DIRECT LINK (OVR)')
+        self.pyfile.url = m.group(1)
+        header = self.load(self.pyfile.url, just_header=True)
+        if 'location' in header:  # Direct link
+            self.startDownload(self.pyfile.url)
+        else:
+            self.retry()
+
+
+    def startDownload(self, link):
+        link = link.strip()
+        if self.captcha:
+            self.correctCaptcha()
+        self.logDebug("DIRECT LINK: %s" % link)
+        self.download(link, disposition=True)
+
+
+    def checkErrors(self):
+        m = re.search(self.ERROR_PATTERN, self.html)
+        if m:
+            self.errmsg = m.group(1)
+            self.logWarning(re.sub(r"<.*?>", " ", self.errmsg))
+
+            if 'wait' in self.errmsg:
+                wait_time = sum([int(v) * {"hour": 3600, "minute": 60, "second": 1}[u] for v, u in
+                                 re.findall(r'(\d+)\s*(hour|minute|second)', self.errmsg)])
+                self.wait(wait_time, True)
+            elif 'captcha' in self.errmsg:
+                self.invalidCaptcha()
+            elif 'premium' in self.errmsg and 'require' in self.errmsg:
+                self.fail("File can be downloaded by premium users only")
+            elif 'limit' in self.errmsg:
+                self.wait(1 * 60 * 60, True)
+                self.retry(25)
+            elif 'countdown' in self.errmsg or 'Expired' in self.errmsg:
+                self.retry()
+            elif 'maintenance' in self.errmsg:
+                self.tempOffline()
+            elif 'download files up to' in self.errmsg:
+                self.fail("File too large for free download")
+            else:
+                self.fail(self.errmsg)
+
+        else:
+            self.errmsg = None
+
+        return self.errmsg
+
+
+    def getPostParameters(self):
+        for _ in xrange(3):
+            if not self.errmsg:
+                self.checkErrors()
+
+            if hasattr(self, "FORM_PATTERN"):
+                action, inputs = self.parseHtmlForm(self.FORM_PATTERN)
+            else:
+                action, inputs = self.parseHtmlForm(input_names={"op": re.compile("^download")})
+
+            if not inputs:
+                action, inputs = self.parseHtmlForm('F1')
+                if not inputs:
+                    if self.errmsg:
+                        self.retry()
+                    else:
+                        self.parseError("Form not found")
+
+            self.logDebug(self.HOSTER_NAME, inputs)
+
+            if 'op' in inputs and inputs['op'] in ("download2", "download3"):
+                if "password" in inputs:
+                    if self.passwords:
+                        inputs['password'] = self.passwords.pop(0)
+                    else:
+                        self.fail("No or invalid passport")
+
+                if not self.premium:
+                    m = re.search(self.WAIT_PATTERN, self.html)
+                    if m:
+                        wait_time = int(m.group(1)) + 1
+                        self.setWait(wait_time, False)
+                    else:
+                        wait_time = 0
+
+                    self.captcha = self.handleCaptcha(inputs)
+
+                    if wait_time:
+                        self.wait()
+
+                self.errmsg = None
+                return inputs
+
+            else:
+                inputs['referer'] = self.pyfile.url
+
+                if self.premium:
+                    inputs['method_premium'] = "Premium Download"
+                    if 'method_free' in inputs:
+                        del inputs['method_free']
+                else:
+                    inputs['method_free'] = "Free Download"
+                    if 'method_premium' in inputs:
+                        del inputs['method_premium']
+
+                self.html = self.load(self.pyfile.url, post=inputs, ref=True)
+                self.errmsg = None
+
+        else:
+            self.parseError('FORM: %s' % (inputs['op'] if 'op' in inputs else 'UNKNOWN'))
+
+
+    def handleCaptcha(self, inputs):
+        m = re.search(self.RECAPTCHA_URL_PATTERN, self.html)
+        if m:
+            recaptcha_key = unquote(m.group(1))
+            self.logDebug("RECAPTCHA KEY: %s" % recaptcha_key)
+            recaptcha = ReCaptcha(self)
+            inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(recaptcha_key)
+            return 1
+        else:
+            m = re.search(self.CAPTCHA_URL_PATTERN, self.html)
+            if m:
+                captcha_url = m.group(1)
+                inputs['code'] = self.decryptCaptcha(captcha_url)
+                return 2
+            else:
+                m = re.search(self.CAPTCHA_DIV_PATTERN, self.html, re.DOTALL)
+                if m:
+                    captcha_div = m.group(1)
+                    self.logDebug(captcha_div)
+                    numerals = re.findall(r'<span.*?padding-left\s*:\s*(\d+).*?>(\d)</span>', html_unescape(captcha_div))
+                    inputs['code'] = "".join([a[1] for a in sorted(numerals, key=lambda num: int(num[0]))])
+                    self.logDebug("CAPTCHA", inputs['code'], numerals)
+                    return 3
+                else:
+                    m = re.search(self.SOLVEMEDIA_PATTERN, self.html)
+                    if m:
+                        captcha_key = m.group(1)
+                        captcha = SolveMedia(self)
+                        inputs['adcopy_challenge'], inputs['adcopy_response'] = captcha.challenge(captcha_key)
+                        return 4
+        return 0
+
+
+getInfo = create_getInfo(XFileSharingPro)
-- 
cgit v1.2.3