From 16af85004c84d0d6c626b4f8424ce9647669a0c1 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 9 Jun 2013 18:10:22 +0200 Subject: moved everything from module to pyload --- .gitignore | 5 +- docs/api/components.rst | 18 +- docs/api/datatypes.rst | 4 +- docs/api/json_api.rst | 6 +- docs/api/overview.rst | 6 +- docs/conf.py | 6 +- docs/docs.conf | 4 +- docs/index.rst | 2 +- docs/module_overview.rst | 30 +- docs/plugins/addon_plugin.rst | 40 +- docs/plugins/base_plugin.rst | 16 +- docs/plugins/crypter_plugin.rst | 12 +- docs/plugins/hoster_plugin.rst | 8 +- module/AccountManager.py | 141 - module/AddonManager.py | 248 -- module/Api.py | 206 -- module/FileManager.py | 582 ---- module/InitHomeDir.py | 87 - module/PluginManager.py | 407 --- module/Scheduler.py | 141 - module/Setup.py | 418 --- module/__init__.py | 0 module/api/AccountApi.py | 54 - module/api/AddonApi.py | 27 - module/api/ApiComponent.py | 23 - module/api/CollectorApi.py | 37 - module/api/ConfigApi.py | 134 - module/api/CoreApi.py | 131 - module/api/DownloadApi.py | 182 -- module/api/DownloadPreparingApi.py | 121 - module/api/FileApi.py | 169 - module/api/UserInteractionApi.py | 61 - module/api/__init__.py | 8 - module/cli/AddPackage.py | 66 - module/cli/Handler.py | 48 - module/cli/ManageFiles.py | 204 -- module/cli/__init__.py | 2 - module/cli/printer.py | 26 - module/config/ConfigManager.py | 139 - module/config/ConfigParser.py | 207 -- module/config/__init__.py | 1 - module/config/default.py | 107 - module/database/AccountDatabase.py | 25 - module/database/ConfigDatabase.py | 56 - module/database/DatabaseBackend.py | 492 --- module/database/FileDatabase.py | 448 --- module/database/StatisticDatabase.py | 13 - module/database/StorageDatabase.py | 48 - module/database/UserDatabase.py | 125 - module/database/__init__.py | 8 - module/datatypes/PyFile.py | 268 -- module/datatypes/PyPackage.py | 115 - module/datatypes/User.py | 63 - module/datatypes/__init__.py | 0 module/interaction/EventManager.py | 84 - module/interaction/InteractionManager.py | 166 - module/interaction/InteractionTask.py | 100 - module/interaction/__init__.py | 2 - module/lib/Getch.py | 76 - module/lib/ReadWriteLock.py | 232 -- module/lib/SafeEval.py | 47 - module/lib/__init__.py | 0 module/lib/beaker/__init__.py | 1 - module/lib/beaker/cache.py | 459 --- module/lib/beaker/container.py | 633 ---- module/lib/beaker/converters.py | 26 - module/lib/beaker/crypto/__init__.py | 40 - module/lib/beaker/crypto/jcecrypto.py | 30 - module/lib/beaker/crypto/pbkdf2.py | 342 -- module/lib/beaker/crypto/pycrypto.py | 31 - module/lib/beaker/crypto/util.py | 30 - module/lib/beaker/exceptions.py | 24 - module/lib/beaker/ext/__init__.py | 0 module/lib/beaker/ext/database.py | 165 - module/lib/beaker/ext/google.py | 120 - module/lib/beaker/ext/memcached.py | 82 - module/lib/beaker/ext/sqla.py | 133 - module/lib/beaker/middleware.py | 165 - module/lib/beaker/session.py | 618 ---- module/lib/beaker/synchronization.py | 381 --- module/lib/beaker/util.py | 302 -- module/lib/bottle.py | 3251 -------------------- module/lib/forwarder.py | 73 - module/lib/hg_tool.py | 133 - module/lib/mod_pywebsocket/COPYING | 28 - module/lib/mod_pywebsocket/__init__.py | 197 -- module/lib/mod_pywebsocket/_stream_base.py | 165 - module/lib/mod_pywebsocket/_stream_hixie75.py | 229 -- module/lib/mod_pywebsocket/_stream_hybi.py | 915 ------ module/lib/mod_pywebsocket/common.py | 307 -- module/lib/mod_pywebsocket/dispatch.py | 387 --- module/lib/mod_pywebsocket/extensions.py | 727 ----- module/lib/mod_pywebsocket/handshake/__init__.py | 110 - module/lib/mod_pywebsocket/handshake/_base.py | 226 -- module/lib/mod_pywebsocket/handshake/hybi.py | 404 --- module/lib/mod_pywebsocket/handshake/hybi00.py | 242 -- module/lib/mod_pywebsocket/headerparserhandler.py | 244 -- module/lib/mod_pywebsocket/http_header_util.py | 263 -- module/lib/mod_pywebsocket/memorizingfile.py | 99 - module/lib/mod_pywebsocket/msgutil.py | 219 -- module/lib/mod_pywebsocket/mux.py | 1636 ---------- module/lib/mod_pywebsocket/standalone.py | 998 ------ module/lib/mod_pywebsocket/stream.py | 57 - module/lib/mod_pywebsocket/util.py | 515 ---- module/lib/new_collections.py | 375 --- module/lib/rename_process.py | 14 - module/lib/simplejson/__init__.py | 466 --- module/lib/simplejson/decoder.py | 421 --- module/lib/simplejson/encoder.py | 534 ---- module/lib/simplejson/ordered_dict.py | 119 - module/lib/simplejson/scanner.py | 77 - module/lib/simplejson/tool.py | 39 - module/lib/wsgiserver/LICENSE.txt | 25 - module/lib/wsgiserver/__init__.py | 1794 ----------- module/network/Browser.py | 153 - module/network/Bucket.py | 63 - module/network/CookieJar.py | 53 - module/network/HTTPChunk.py | 298 -- module/network/HTTPDownload.py | 338 -- module/network/HTTPRequest.py | 324 -- module/network/RequestFactory.py | 114 - module/network/XDCCRequest.py | 162 - module/network/__init__.py | 1 - module/plugins/Account.py | 295 -- module/plugins/Addon.py | 205 -- module/plugins/Base.py | 343 --- module/plugins/Crypter.py | 271 -- module/plugins/Hoster.py | 400 --- module/plugins/MultiHoster.py | 73 - module/plugins/ReCaptcha.py | 22 - module/plugins/__init__.py | 0 module/plugins/accounts/AlldebridCom.py | 49 - module/plugins/accounts/BayfilesCom.py | 51 - module/plugins/accounts/BitshareCom.py | 44 - module/plugins/accounts/BoltsharingCom.py | 12 - module/plugins/accounts/CramitIn.py | 12 - module/plugins/accounts/CyberlockerCh.py | 31 - module/plugins/accounts/CzshareCom.py | 57 - module/plugins/accounts/DdlstorageCom.py | 12 - module/plugins/accounts/DebridItaliaCom.py | 50 - module/plugins/accounts/DepositfilesCom.py | 47 - module/plugins/accounts/EasybytezCom.py | 73 - module/plugins/accounts/EgoFilesCom.py | 40 - module/plugins/accounts/EuroshareEu.py | 55 - module/plugins/accounts/FastshareCz.py | 54 - module/plugins/accounts/FilebeerInfo.py | 57 - module/plugins/accounts/FilecloudIo.py | 49 - module/plugins/accounts/FilefactoryCom.py | 54 - module/plugins/accounts/FilejungleCom.py | 60 - module/plugins/accounts/FilerioCom.py | 12 - module/plugins/accounts/FilesMailRu.py | 41 - module/plugins/accounts/FileserveCom.py | 58 - module/plugins/accounts/FourSharedCom.py | 48 - module/plugins/accounts/FreakshareCom.py | 51 - module/plugins/accounts/FshareVn.py | 62 - module/plugins/accounts/Ftp.py | 13 - module/plugins/accounts/HellshareCz.py | 87 - module/plugins/accounts/HellspyCz.py | 70 - module/plugins/accounts/HotfileCom.py | 86 - module/plugins/accounts/Http.py | 13 - module/plugins/accounts/MegasharesCom.py | 42 - module/plugins/accounts/MultiDebridCom.py | 47 - module/plugins/accounts/MultishareCz.py | 58 - module/plugins/accounts/NetloadIn.py | 49 - module/plugins/accounts/Premium4Me.py | 27 - module/plugins/accounts/PremiumizeMe.py | 44 - module/plugins/accounts/QuickshareCz.py | 53 - module/plugins/accounts/RapidgatorNet.py | 74 - module/plugins/accounts/RapidshareCom.py | 68 - module/plugins/accounts/RarefileNet.py | 12 - module/plugins/accounts/RealdebridCom.py | 35 - module/plugins/accounts/RehostTo.py | 37 - module/plugins/accounts/ReloadCc.py | 73 - module/plugins/accounts/RyushareCom.py | 18 - module/plugins/accounts/Share76Com.py | 11 - module/plugins/accounts/ShareFilesCo.py | 12 - module/plugins/accounts/ShareRapidCom.py | 47 - module/plugins/accounts/ShareonlineBiz.py | 56 - module/plugins/accounts/SpeedLoadOrg.py | 12 - module/plugins/accounts/StahnuTo.py | 49 - module/plugins/accounts/TurbobitNet.py | 56 - module/plugins/accounts/UlozTo.py | 36 - module/plugins/accounts/UploadedTo.py | 65 - module/plugins/accounts/UploadheroCom.py | 35 - module/plugins/accounts/UploadingCom.py | 52 - module/plugins/accounts/UploadstationCom.py | 13 - module/plugins/accounts/UptoboxCom.py | 12 - module/plugins/accounts/WarserverCz.py | 70 - module/plugins/accounts/WuploadCom.py | 47 - module/plugins/accounts/X7To.py | 66 - module/plugins/accounts/YibaishiwuCom.py | 51 - module/plugins/accounts/ZeveraCom.py | 49 - module/plugins/accounts/__init__.py | 0 module/plugins/addons/AlldebridCom.py | 28 - module/plugins/addons/BypassCaptcha.py | 143 - module/plugins/addons/CaptchaBrotherhood.py | 169 - module/plugins/addons/CaptchaTrader.py | 159 - module/plugins/addons/Checksum.py | 169 - module/plugins/addons/ClickAndLoad.py | 89 - module/plugins/addons/DeathByCaptcha.py | 210 -- module/plugins/addons/DebridItaliaCom.py | 41 - module/plugins/addons/DeleteFinished.py | 86 - module/plugins/addons/DownloadScheduler.py | 86 - module/plugins/addons/EasybytezCom.py | 31 - module/plugins/addons/Ev0InFetcher.py | 87 - module/plugins/addons/ExpertDecoders.py | 112 - module/plugins/addons/ExternalScripts.py | 118 - module/plugins/addons/ExtractArchive.py | 312 -- module/plugins/addons/HotFolder.py | 85 - module/plugins/addons/IRCInterface.py | 431 --- module/plugins/addons/ImageTyperz.py | 160 - module/plugins/addons/LinkdecrypterCom.py | 59 - module/plugins/addons/MergeFiles.py | 94 - module/plugins/addons/MultiDebridCom.py | 42 - module/plugins/addons/MultiHome.py | 82 - module/plugins/addons/MultiHoster.py | 102 - module/plugins/addons/MultishareCz.py | 23 - module/plugins/addons/Premium4Me.py | 33 - module/plugins/addons/PremiumizeMe.py | 50 - module/plugins/addons/RealdebridCom.py | 25 - module/plugins/addons/RehostTo.py | 41 - module/plugins/addons/RestartFailed.py | 124 - module/plugins/addons/UnSkipOnFail.py | 97 - module/plugins/addons/UpdateManager.py | 201 -- module/plugins/addons/XFileSharingPro.py | 70 - module/plugins/addons/XMPPInterface.py | 276 -- module/plugins/addons/ZeveraCom.py | 19 - module/plugins/addons/__init__.py | 0 module/plugins/crypter/C1neonCom.py | 133 - module/plugins/crypter/CCF.py | 42 - module/plugins/crypter/CrockoComFolder.py | 14 - module/plugins/crypter/CryptItCom.py | 38 - module/plugins/crypter/CzshareComFolder.py | 30 - module/plugins/crypter/DDLMusicOrg.py | 42 - module/plugins/crypter/DataHuFolder.py | 55 - module/plugins/crypter/DdlstorageComFolder.py | 32 - module/plugins/crypter/DepositfilesComFolder.py | 14 - module/plugins/crypter/Dereferer.py | 34 - module/plugins/crypter/DontKnowMe.py | 21 - module/plugins/crypter/DownloadVimeoCom.py | 30 - module/plugins/crypter/DuckCryptInfo.py | 58 - module/plugins/crypter/EasybytezComFolder.py | 60 - module/plugins/crypter/EmbeduploadCom.py | 54 - module/plugins/crypter/FilebeerInfoFolder.py | 35 - module/plugins/crypter/FilefactoryComFolder.py | 44 - module/plugins/crypter/FileserveComFolder.py | 32 - module/plugins/crypter/FourChanOrg.py | 25 - module/plugins/crypter/FshareVnFolder.py | 14 - module/plugins/crypter/GooGl.py | 41 - module/plugins/crypter/HoerbuchIn.py | 55 - module/plugins/crypter/HotfileFolderCom.py | 29 - module/plugins/crypter/ILoadTo.py | 62 - module/plugins/crypter/LetitbitNetFolder.py | 33 - module/plugins/crypter/LinkList.py | 55 - module/plugins/crypter/LinkSaveIn.py | 227 -- module/plugins/crypter/LinkdecrypterCom.py | 100 - module/plugins/crypter/LixIn.py | 60 - module/plugins/crypter/LofCc.py | 49 - module/plugins/crypter/MBLinkInfo.py | 27 - module/plugins/crypter/MediafireComFolder.py | 55 - module/plugins/crypter/Movie2kTo.py | 151 - module/plugins/crypter/MultiloadCz.py | 41 - module/plugins/crypter/MultiuploadCom.py | 58 - module/plugins/crypter/NCryptIn.py | 251 -- module/plugins/crypter/NetfolderIn.py | 71 - module/plugins/crypter/OneKhDe.py | 36 - module/plugins/crypter/OronComFolder.py | 46 - module/plugins/crypter/QuickshareCzFolder.py | 30 - module/plugins/crypter/RSDF.py | 49 - module/plugins/crypter/RSLayerCom.py | 49 - module/plugins/crypter/RelinkUs.py | 264 -- module/plugins/crypter/SecuredIn.py | 334 -- module/plugins/crypter/SerienjunkiesOrg.py | 321 -- module/plugins/crypter/ShareLinksBiz.py | 269 -- module/plugins/crypter/ShareRapidComFolder.py | 14 - module/plugins/crypter/SpeedLoadOrgFolder.py | 30 - module/plugins/crypter/StealthTo.py | 45 - module/plugins/crypter/TrailerzoneInfo.py | 45 - module/plugins/crypter/UlozToFolder.py | 40 - module/plugins/crypter/UploadedToFolder.py | 50 - module/plugins/crypter/WiiReloadedOrg.py | 52 - module/plugins/crypter/XfilesharingProFolder.py | 34 - module/plugins/crypter/YoutubeBatch.py | 42 - module/plugins/crypter/__init__.py | 0 module/plugins/hooks/Captcha9kw.py | 162 - module/plugins/hooks/ReloadCc.py | 65 - module/plugins/hoster/ARD.py | 80 - module/plugins/hoster/AlldebridCom.py | 84 - module/plugins/hoster/BasePlugin.py | 104 - module/plugins/hoster/BayfilesCom.py | 93 - module/plugins/hoster/BezvadataCz.py | 94 - module/plugins/hoster/BillionuploadsCom.py | 17 - module/plugins/hoster/BitshareCom.py | 179 -- module/plugins/hoster/BoltsharingCom.py | 15 - module/plugins/hoster/CatShareNet.py | 38 - module/plugins/hoster/ChipDe.py | 24 - module/plugins/hoster/CloudzerNet.py | 65 - module/plugins/hoster/CramitIn.py | 20 - module/plugins/hoster/CrockoCom.py | 71 - module/plugins/hoster/CyberlockerCh.py | 15 - module/plugins/hoster/CzshareCom.py | 160 - module/plugins/hoster/DailymotionCom.py | 47 - module/plugins/hoster/DataHu.py | 53 - module/plugins/hoster/DataportCz.py | 68 - module/plugins/hoster/DateiTo.py | 94 - module/plugins/hoster/DdlstorageCom.py | 20 - module/plugins/hoster/DebridItaliaCom.py | 61 - module/plugins/hoster/DepositfilesCom.py | 112 - module/plugins/hoster/DlFreeFr.py | 183 -- module/plugins/hoster/EasybytezCom.py | 47 - module/plugins/hoster/EdiskCz.py | 62 - module/plugins/hoster/EgoFilesCom.py | 103 - module/plugins/hoster/EuroshareEu.py | 74 - module/plugins/hoster/ExtabitCom.py | 87 - module/plugins/hoster/FastshareCz.py | 96 - module/plugins/hoster/FileApeCom.py | 62 - module/plugins/hoster/FilebeerInfo.py | 15 - module/plugins/hoster/FilecloudIo.py | 112 - module/plugins/hoster/FilefactoryCom.py | 133 - module/plugins/hoster/FilejungleCom.py | 38 - module/plugins/hoster/FilepostCom.py | 135 - module/plugins/hoster/FilerNet.py | 103 - module/plugins/hoster/FilerioCom.py | 20 - module/plugins/hoster/FilesMailRu.py | 99 - module/plugins/hoster/FileserveCom.py | 211 -- module/plugins/hoster/FileshareInUa.py | 78 - module/plugins/hoster/FlyFilesNet.py | 41 - module/plugins/hoster/FourSharedCom.py | 53 - module/plugins/hoster/FreakshareCom.py | 167 - module/plugins/hoster/FreevideoCz.py | 64 - module/plugins/hoster/FshareVn.py | 111 - module/plugins/hoster/Ftp.py | 91 - module/plugins/hoster/GamefrontCom.py | 80 - module/plugins/hoster/GigapetaCom.py | 73 - module/plugins/hoster/HellshareCz.py | 56 - module/plugins/hoster/HellspyCz.py | 70 - module/plugins/hoster/HotfileCom.py | 137 - module/plugins/hoster/IFileWs.py | 20 - module/plugins/hoster/IcyFilesCom.py | 112 - module/plugins/hoster/IfileIt.py | 74 - module/plugins/hoster/IfolderRu.py | 90 - module/plugins/hoster/JumbofilesCom.py | 31 - module/plugins/hoster/LetitbitNet.py | 125 - module/plugins/hoster/LoadTo.py | 83 - module/plugins/hoster/LuckyShareNet.py | 72 - module/plugins/hoster/MediafireCom.py | 135 - module/plugins/hoster/MegaNz.py | 125 - module/plugins/hoster/MegasharesCom.py | 108 - module/plugins/hoster/MovReelCom.py | 106 - module/plugins/hoster/MultiDebridCom.py | 57 - module/plugins/hoster/MultishareCz.py | 76 - module/plugins/hoster/MyvideoDe.py | 43 - module/plugins/hoster/NarodRu.py | 66 - module/plugins/hoster/NetloadIn.py | 252 -- module/plugins/hoster/NovafileCom.py | 24 - module/plugins/hoster/NowDownloadEu.py | 66 - module/plugins/hoster/OneFichierCom.py | 58 - module/plugins/hoster/PornhostCom.py | 76 - module/plugins/hoster/PornhubCom.py | 83 - module/plugins/hoster/Premium4Me.py | 70 - module/plugins/hoster/PremiumizeMe.py | 50 - module/plugins/hoster/PutlockerCom.py | 78 - module/plugins/hoster/QuickshareCz.py | 99 - module/plugins/hoster/RapidgatorNet.py | 192 -- module/plugins/hoster/RapidshareCom.py | 225 -- module/plugins/hoster/RarefileNet.py | 34 - module/plugins/hoster/RealdebridCom.py | 88 - module/plugins/hoster/RedtubeCom.py | 56 - module/plugins/hoster/RehostTo.py | 37 - module/plugins/hoster/ReloadCc.py | 103 - module/plugins/hoster/RyushareCom.py | 55 - module/plugins/hoster/SecureUploadEu.py | 18 - module/plugins/hoster/SendmywayCom.py | 18 - module/plugins/hoster/SendspaceCom.py | 67 - module/plugins/hoster/Share4webCom.py | 16 - module/plugins/hoster/Share76Com.py | 19 - module/plugins/hoster/ShareFilesCo.py | 24 - module/plugins/hoster/ShareRapidCom.py | 104 - module/plugins/hoster/SharebeesCom.py | 19 - module/plugins/hoster/ShareonlineBiz.py | 203 -- module/plugins/hoster/ShareplaceCom.py | 84 - module/plugins/hoster/ShragleCom.py | 106 - module/plugins/hoster/SpeedLoadOrg.py | 21 - module/plugins/hoster/SpeedfileCz.py | 65 - module/plugins/hoster/StreamCz.py | 76 - module/plugins/hoster/TurbobitNet.py | 170 - module/plugins/hoster/TurbouploadCom.py | 45 - module/plugins/hoster/TusfilesNet.py | 18 - module/plugins/hoster/TwoSharedCom.py | 33 - module/plugins/hoster/UlozTo.py | 156 - module/plugins/hoster/UloziskoSk.py | 75 - module/plugins/hoster/UnibytesCom.py | 80 - module/plugins/hoster/UploadStationCom.py | 21 - module/plugins/hoster/UploadedTo.py | 238 -- module/plugins/hoster/UploadheroCom.py | 84 - module/plugins/hoster/UploadingCom.py | 110 - module/plugins/hoster/UptoboxCom.py | 21 - module/plugins/hoster/VeehdCom.py | 80 - module/plugins/hoster/WarserverCz.py | 70 - module/plugins/hoster/WebshareCz.py | 48 - module/plugins/hoster/WrzucTo.py | 58 - module/plugins/hoster/WuploadCom.py | 241 -- module/plugins/hoster/X7To.py | 93 - module/plugins/hoster/XFileSharingPro.py | 314 -- module/plugins/hoster/XHamsterCom.py | 120 - module/plugins/hoster/XVideosCom.py | 19 - module/plugins/hoster/Xdcc.py | 229 -- module/plugins/hoster/XvidstageCom.py | 114 - module/plugins/hoster/YibaishiwuCom.py | 54 - module/plugins/hoster/YoupornCom.py | 56 - module/plugins/hoster/YourfilesTo.py | 83 - module/plugins/hoster/YoutubeCom.py | 164 - module/plugins/hoster/ZDF.py | 46 - module/plugins/hoster/ZeveraCom.py | 108 - module/plugins/hoster/ZippyshareCom.py | 187 -- module/plugins/hoster/__init__.py | 0 module/plugins/internal/AbstractExtractor.py | 93 - module/plugins/internal/CaptchaService.py | 77 - module/plugins/internal/DeadHoster.py | 18 - module/plugins/internal/NetloadInOCR.py | 27 - module/plugins/internal/OCR.py | 314 -- module/plugins/internal/ShareonlineBizOCR.py | 53 - module/plugins/internal/SimpleCrypter.py | 68 - module/plugins/internal/SimpleHoster.py | 251 -- module/plugins/internal/UnRar.py | 212 -- module/plugins/internal/UnZip.py | 49 - module/plugins/internal/XFSPAccount.py | 79 - module/plugins/internal/__init__.py | 0 module/remote/ClickAndLoadBackend.py | 170 - module/remote/JSONClient.py | 56 - module/remote/RemoteManager.py | 89 - module/remote/WSClient.py | 59 - module/remote/WebSocketBackend.py | 49 - module/remote/__init__.py | 0 module/remote/apitypes.py | 536 ---- module/remote/apitypes_debug.py | 135 - module/remote/create_apitypes.py | 180 -- module/remote/create_jstypes.py | 36 - module/remote/json_converter.py | 64 - module/remote/pyload.thrift | 538 ---- module/remote/ttypes.py | 534 ---- module/remote/wsbackend/AbstractHandler.py | 133 - module/remote/wsbackend/ApiHandler.py | 81 - module/remote/wsbackend/AsyncHandler.py | 167 - module/remote/wsbackend/Dispatcher.py | 31 - module/remote/wsbackend/Server.py | 733 ----- module/remote/wsbackend/__init__.py | 2 - module/setup/System_Checks.py | 126 - module/threads/AddonThread.py | 65 - module/threads/BaseThread.py | 142 - module/threads/DecrypterThread.py | 81 - module/threads/DownloadThread.py | 231 -- module/threads/InfoThread.py | 168 - module/threads/ThreadManager.py | 313 -- module/threads/__init__.py | 0 module/utils/ImportDebugger.py | 19 - module/utils/JsEngine.py | 195 -- module/utils/__init__.py | 253 -- module/utils/fs.py | 78 - module/utils/json_layer.py | 15 - module/utils/packagetools.py | 155 - module/utils/pylgettext.py | 61 - module/web/.bowerrc | 3 - module/web/.jshintrc | 29 - module/web/Gruntfile.js | 361 --- module/web/ServerThread.py | 143 - module/web/__init__.py | 0 module/web/api_app.py | 112 - module/web/app/favicon.ico | Bin 6006 -> 0 bytes module/web/app/fonts/Sansation_Bold-webfont.eot | Bin 35336 -> 0 bytes module/web/app/fonts/Sansation_Bold-webfont.ttf | Bin 35160 -> 0 bytes module/web/app/fonts/Sansation_Bold-webfont.woff | Bin 18496 -> 0 bytes module/web/app/fonts/Sansation_Light-webfont.eot | Bin 36700 -> 0 bytes module/web/app/fonts/Sansation_Light-webfont.ttf | Bin 36520 -> 0 bytes module/web/app/fonts/Sansation_Light-webfont.woff | Bin 18408 -> 0 bytes module/web/app/fonts/Sansation_Regular-webfont.eot | Bin 36368 -> 0 bytes module/web/app/fonts/Sansation_Regular-webfont.ttf | Bin 36180 -> 0 bytes .../web/app/fonts/Sansation_Regular-webfont.woff | Bin 18316 -> 0 bytes module/web/app/images/default/bgpattern.png | Bin 2487 -> 0 bytes module/web/app/images/default/checks_sheet.png | Bin 1145 -> 0 bytes module/web/app/images/default/fancy_deboss.png | Bin 265 -> 0 bytes module/web/app/images/default/logo.png | Bin 5329 -> 0 bytes module/web/app/images/default/logo_grey.png | Bin 1141 -> 0 bytes module/web/app/images/icon.png | Bin 1912 -> 0 bytes module/web/app/index.html | 107 - module/web/app/scripts/app.js | 105 - module/web/app/scripts/collections/AccountList.js | 24 - module/web/app/scripts/collections/FileList.js | 18 - .../web/app/scripts/collections/InteractionList.js | 49 - module/web/app/scripts/collections/PackageList.js | 16 - module/web/app/scripts/collections/ProgressList.js | 18 - module/web/app/scripts/config.js | 73 - module/web/app/scripts/controller.js | 67 - module/web/app/scripts/default.js | 30 - module/web/app/scripts/helpers/fileHelper.js | 55 - module/web/app/scripts/helpers/formatSize.js | 15 - module/web/app/scripts/helpers/formatTime.js | 17 - module/web/app/scripts/helpers/pluginIcon.js | 14 - module/web/app/scripts/models/Account.js | 51 - module/web/app/scripts/models/ConfigHolder.js | 68 - module/web/app/scripts/models/ConfigItem.js | 40 - module/web/app/scripts/models/File.js | 92 - module/web/app/scripts/models/InteractionTask.js | 41 - module/web/app/scripts/models/Package.js | 119 - module/web/app/scripts/models/Progress.js | 50 - module/web/app/scripts/models/ServerStatus.js | 47 - module/web/app/scripts/models/TreeCollection.js | 50 - module/web/app/scripts/models/UserSession.js | 20 - module/web/app/scripts/router.js | 29 - module/web/app/scripts/routers/defaultRouter.js | 30 - module/web/app/scripts/routers/mobileRouter.js | 56 - module/web/app/scripts/utils/animations.js | 129 - module/web/app/scripts/utils/apitypes.js | 16 - module/web/app/scripts/utils/dialogs.js | 16 - module/web/app/scripts/utils/initHB.js | 11 - module/web/app/scripts/utils/lazyRequire.js | 97 - module/web/app/scripts/vendor/Handlebars-1.0rc1.js | 1927 ------------ module/web/app/scripts/vendor/bootstrap-2.3.2.js | 2291 -------------- module/web/app/scripts/vendor/jquery.omniwindow.js | 141 - module/web/app/scripts/vendor/remaining.js | 149 - module/web/app/scripts/views/abstract/itemView.js | 47 - module/web/app/scripts/views/abstract/modalView.js | 125 - .../app/scripts/views/accounts/accountListView.js | 52 - .../web/app/scripts/views/accounts/accountModal.js | 72 - .../web/app/scripts/views/accounts/accountView.js | 18 - .../app/scripts/views/dashboard/dashboardView.js | 168 - module/web/app/scripts/views/dashboard/fileView.js | 102 - .../web/app/scripts/views/dashboard/filterView.js | 133 - .../web/app/scripts/views/dashboard/packageView.js | 75 - .../app/scripts/views/dashboard/selectionView.js | 155 - module/web/app/scripts/views/headerView.js | 240 -- module/web/app/scripts/views/input/inputLoader.js | 8 - module/web/app/scripts/views/input/inputView.js | 86 - module/web/app/scripts/views/input/textInput.js | 36 - module/web/app/scripts/views/linkGrabberModal.js | 49 - module/web/app/scripts/views/loginView.js | 37 - module/web/app/scripts/views/notificationView.js | 83 - module/web/app/scripts/views/progressView.js | 33 - module/web/app/scripts/views/queryModal.js | 69 - .../scripts/views/settings/configSectionView.js | 99 - .../scripts/views/settings/pluginChooserModal.js | 69 - .../web/app/scripts/views/settings/settingsView.js | 184 -- module/web/app/styles/default/accounts.less | 6 - module/web/app/styles/default/admin.less | 17 - module/web/app/styles/default/base.less | 163 - module/web/app/styles/default/common.less | 90 - module/web/app/styles/default/dashboard.less | 331 -- module/web/app/styles/default/main.less | 11 - module/web/app/styles/default/settings.less | 121 - module/web/app/styles/default/style.less | 366 --- module/web/app/styles/font.css | 37 - .../app/templates/default/accounts/account.html | 10 - .../app/templates/default/accounts/actionbar.html | 5 - .../web/app/templates/default/accounts/layout.html | 19 - module/web/app/templates/default/admin.html | 223 -- .../app/templates/default/dashboard/actionbar.html | 54 - .../web/app/templates/default/dashboard/file.html | 34 - .../app/templates/default/dashboard/layout.html | 35 - .../app/templates/default/dashboard/package.html | 50 - .../app/templates/default/dashboard/select.html | 11 - .../app/templates/default/dialogs/addAccount.html | 42 - .../templates/default/dialogs/addPluginConfig.html | 26 - .../templates/default/dialogs/confirmDelete.html | 11 - .../templates/default/dialogs/interactionTask.html | 37 - .../app/templates/default/dialogs/linkgrabber.html | 49 - .../web/app/templates/default/dialogs/modal.html | 10 - .../web/app/templates/default/header/layout.html | 62 - .../web/app/templates/default/header/progress.html | 14 - .../app/templates/default/header/progressbar.html | 27 - .../web/app/templates/default/header/status.html | 3 - module/web/app/templates/default/login.html | 28 - module/web/app/templates/default/notification.html | 11 - .../app/templates/default/settings/actionbar.html | 5 - .../web/app/templates/default/settings/config.html | 17 - .../app/templates/default/settings/configItem.html | 7 - .../web/app/templates/default/settings/layout.html | 11 - .../web/app/templates/default/settings/menu.html | 40 - module/web/app/templates/default/setup.html | 16 - module/web/app/unavailable.html | 18 - module/web/bower.json | 23 - module/web/cnl_app.py | 166 - module/web/middlewares.py | 134 - module/web/package.json | 32 - module/web/pyload_app.py | 78 - module/web/servers.py | 162 - module/web/setup_app.py | 21 - module/web/utils.py | 78 - module/web/webinterface.py | 98 - pavement.py | 67 +- pyload-cli.py | 596 +--- pyload.py | 623 +--- pyload/AccountManager.py | 141 + pyload/AddonManager.py | 248 ++ pyload/Api.py | 206 ++ pyload/FileManager.py | 582 ++++ pyload/InitHomeDir.py | 87 + pyload/PluginManager.py | 426 +++ pyload/Scheduler.py | 141 + pyload/Setup.py | 418 +++ pyload/__init__.py | 0 pyload/api/AccountApi.py | 54 + pyload/api/AddonApi.py | 27 + pyload/api/ApiComponent.py | 23 + pyload/api/CollectorApi.py | 37 + pyload/api/ConfigApi.py | 134 + pyload/api/CoreApi.py | 131 + pyload/api/DownloadApi.py | 182 ++ pyload/api/DownloadPreparingApi.py | 121 + pyload/api/FileApi.py | 169 + pyload/api/UserInteractionApi.py | 61 + pyload/api/__init__.py | 8 + pyload/cli/AddPackage.py | 66 + pyload/cli/Handler.py | 48 + pyload/cli/ManageFiles.py | 204 ++ pyload/cli/__init__.py | 2 + pyload/cli/printer.py | 26 + pyload/config/ConfigManager.py | 139 + pyload/config/ConfigParser.py | 207 ++ pyload/config/__init__.py | 1 + pyload/config/default.py | 107 + pyload/database/AccountDatabase.py | 25 + pyload/database/ConfigDatabase.py | 56 + pyload/database/DatabaseBackend.py | 492 +++ pyload/database/FileDatabase.py | 448 +++ pyload/database/StatisticDatabase.py | 13 + pyload/database/StorageDatabase.py | 48 + pyload/database/UserDatabase.py | 125 + pyload/database/__init__.py | 8 + pyload/datatypes/PyFile.py | 268 ++ pyload/datatypes/PyPackage.py | 115 + pyload/datatypes/User.py | 63 + pyload/datatypes/__init__.py | 0 pyload/interaction/EventManager.py | 84 + pyload/interaction/InteractionManager.py | 166 + pyload/interaction/InteractionTask.py | 100 + pyload/interaction/__init__.py | 2 + pyload/lib/Getch.py | 76 + pyload/lib/ReadWriteLock.py | 232 ++ pyload/lib/SafeEval.py | 47 + pyload/lib/__init__.py | 0 pyload/lib/beaker/__init__.py | 1 + pyload/lib/beaker/cache.py | 459 +++ pyload/lib/beaker/container.py | 633 ++++ pyload/lib/beaker/converters.py | 26 + pyload/lib/beaker/crypto/__init__.py | 40 + pyload/lib/beaker/crypto/jcecrypto.py | 30 + pyload/lib/beaker/crypto/pbkdf2.py | 342 ++ pyload/lib/beaker/crypto/pycrypto.py | 31 + pyload/lib/beaker/crypto/util.py | 30 + pyload/lib/beaker/exceptions.py | 24 + pyload/lib/beaker/ext/__init__.py | 0 pyload/lib/beaker/ext/database.py | 165 + pyload/lib/beaker/ext/google.py | 120 + pyload/lib/beaker/ext/memcached.py | 82 + pyload/lib/beaker/ext/sqla.py | 133 + pyload/lib/beaker/middleware.py | 165 + pyload/lib/beaker/session.py | 618 ++++ pyload/lib/beaker/synchronization.py | 381 +++ pyload/lib/beaker/util.py | 302 ++ pyload/lib/bottle.py | 3251 ++++++++++++++++++++ pyload/lib/forwarder.py | 73 + pyload/lib/hg_tool.py | 133 + pyload/lib/mod_pywebsocket/COPYING | 28 + pyload/lib/mod_pywebsocket/__init__.py | 197 ++ pyload/lib/mod_pywebsocket/_stream_base.py | 165 + pyload/lib/mod_pywebsocket/_stream_hixie75.py | 229 ++ pyload/lib/mod_pywebsocket/_stream_hybi.py | 915 ++++++ pyload/lib/mod_pywebsocket/common.py | 307 ++ pyload/lib/mod_pywebsocket/dispatch.py | 387 +++ pyload/lib/mod_pywebsocket/extensions.py | 727 +++++ pyload/lib/mod_pywebsocket/handshake/__init__.py | 110 + pyload/lib/mod_pywebsocket/handshake/_base.py | 226 ++ pyload/lib/mod_pywebsocket/handshake/hybi.py | 404 +++ pyload/lib/mod_pywebsocket/handshake/hybi00.py | 242 ++ pyload/lib/mod_pywebsocket/headerparserhandler.py | 244 ++ pyload/lib/mod_pywebsocket/http_header_util.py | 263 ++ pyload/lib/mod_pywebsocket/memorizingfile.py | 99 + pyload/lib/mod_pywebsocket/msgutil.py | 219 ++ pyload/lib/mod_pywebsocket/mux.py | 1636 ++++++++++ pyload/lib/mod_pywebsocket/standalone.py | 998 ++++++ pyload/lib/mod_pywebsocket/stream.py | 57 + pyload/lib/mod_pywebsocket/util.py | 515 ++++ pyload/lib/new_collections.py | 375 +++ pyload/lib/rename_process.py | 14 + pyload/lib/simplejson/__init__.py | 466 +++ pyload/lib/simplejson/decoder.py | 421 +++ pyload/lib/simplejson/encoder.py | 534 ++++ pyload/lib/simplejson/ordered_dict.py | 119 + pyload/lib/simplejson/scanner.py | 77 + pyload/lib/simplejson/tool.py | 39 + pyload/lib/wsgiserver/LICENSE.txt | 25 + pyload/lib/wsgiserver/__init__.py | 1794 +++++++++++ pyload/network/Browser.py | 153 + pyload/network/Bucket.py | 63 + pyload/network/CookieJar.py | 53 + pyload/network/HTTPChunk.py | 298 ++ pyload/network/HTTPDownload.py | 338 ++ pyload/network/HTTPRequest.py | 324 ++ pyload/network/RequestFactory.py | 114 + pyload/network/XDCCRequest.py | 162 + pyload/network/__init__.py | 1 + pyload/plugins/Account.py | 295 ++ pyload/plugins/Addon.py | 205 ++ pyload/plugins/Base.py | 343 +++ pyload/plugins/Crypter.py | 271 ++ pyload/plugins/Hoster.py | 400 +++ pyload/plugins/MultiHoster.py | 73 + pyload/plugins/ReCaptcha.py | 22 + pyload/plugins/__init__.py | 0 pyload/plugins/accounts/AlldebridCom.py | 49 + pyload/plugins/accounts/BayfilesCom.py | 51 + pyload/plugins/accounts/BitshareCom.py | 44 + pyload/plugins/accounts/BoltsharingCom.py | 12 + pyload/plugins/accounts/CramitIn.py | 12 + pyload/plugins/accounts/CyberlockerCh.py | 31 + pyload/plugins/accounts/CzshareCom.py | 57 + pyload/plugins/accounts/DdlstorageCom.py | 12 + pyload/plugins/accounts/DebridItaliaCom.py | 50 + pyload/plugins/accounts/DepositfilesCom.py | 47 + pyload/plugins/accounts/EasybytezCom.py | 73 + pyload/plugins/accounts/EgoFilesCom.py | 40 + pyload/plugins/accounts/EuroshareEu.py | 55 + pyload/plugins/accounts/FastshareCz.py | 54 + pyload/plugins/accounts/FilebeerInfo.py | 57 + pyload/plugins/accounts/FilecloudIo.py | 49 + pyload/plugins/accounts/FilefactoryCom.py | 54 + pyload/plugins/accounts/FilejungleCom.py | 60 + pyload/plugins/accounts/FilerioCom.py | 12 + pyload/plugins/accounts/FilesMailRu.py | 41 + pyload/plugins/accounts/FileserveCom.py | 58 + pyload/plugins/accounts/FourSharedCom.py | 48 + pyload/plugins/accounts/FreakshareCom.py | 51 + pyload/plugins/accounts/FshareVn.py | 62 + pyload/plugins/accounts/Ftp.py | 13 + pyload/plugins/accounts/HellshareCz.py | 87 + pyload/plugins/accounts/HellspyCz.py | 70 + pyload/plugins/accounts/HotfileCom.py | 86 + pyload/plugins/accounts/Http.py | 13 + pyload/plugins/accounts/MegasharesCom.py | 42 + pyload/plugins/accounts/MultiDebridCom.py | 47 + pyload/plugins/accounts/MultishareCz.py | 58 + pyload/plugins/accounts/NetloadIn.py | 49 + pyload/plugins/accounts/Premium4Me.py | 27 + pyload/plugins/accounts/PremiumizeMe.py | 44 + pyload/plugins/accounts/QuickshareCz.py | 53 + pyload/plugins/accounts/RapidgatorNet.py | 74 + pyload/plugins/accounts/RapidshareCom.py | 68 + pyload/plugins/accounts/RarefileNet.py | 12 + pyload/plugins/accounts/RealdebridCom.py | 35 + pyload/plugins/accounts/RehostTo.py | 37 + pyload/plugins/accounts/ReloadCc.py | 73 + pyload/plugins/accounts/RyushareCom.py | 18 + pyload/plugins/accounts/Share76Com.py | 11 + pyload/plugins/accounts/ShareFilesCo.py | 12 + pyload/plugins/accounts/ShareRapidCom.py | 47 + pyload/plugins/accounts/ShareonlineBiz.py | 56 + pyload/plugins/accounts/SpeedLoadOrg.py | 12 + pyload/plugins/accounts/StahnuTo.py | 49 + pyload/plugins/accounts/TurbobitNet.py | 56 + pyload/plugins/accounts/UlozTo.py | 36 + pyload/plugins/accounts/UploadedTo.py | 65 + pyload/plugins/accounts/UploadheroCom.py | 35 + pyload/plugins/accounts/UploadingCom.py | 52 + pyload/plugins/accounts/UploadstationCom.py | 13 + pyload/plugins/accounts/UptoboxCom.py | 12 + pyload/plugins/accounts/WarserverCz.py | 70 + pyload/plugins/accounts/WuploadCom.py | 47 + pyload/plugins/accounts/X7To.py | 66 + pyload/plugins/accounts/YibaishiwuCom.py | 51 + pyload/plugins/accounts/ZeveraCom.py | 49 + pyload/plugins/accounts/__init__.py | 0 pyload/plugins/addons/AlldebridCom.py | 28 + pyload/plugins/addons/BypassCaptcha.py | 143 + pyload/plugins/addons/CaptchaBrotherhood.py | 169 + pyload/plugins/addons/CaptchaTrader.py | 159 + pyload/plugins/addons/Checksum.py | 169 + pyload/plugins/addons/ClickAndLoad.py | 89 + pyload/plugins/addons/DeathByCaptcha.py | 210 ++ pyload/plugins/addons/DebridItaliaCom.py | 41 + pyload/plugins/addons/DeleteFinished.py | 86 + pyload/plugins/addons/DownloadScheduler.py | 86 + pyload/plugins/addons/EasybytezCom.py | 31 + pyload/plugins/addons/Ev0InFetcher.py | 87 + pyload/plugins/addons/ExpertDecoders.py | 112 + pyload/plugins/addons/ExternalScripts.py | 118 + pyload/plugins/addons/ExtractArchive.py | 312 ++ pyload/plugins/addons/HotFolder.py | 85 + pyload/plugins/addons/IRCInterface.py | 431 +++ pyload/plugins/addons/ImageTyperz.py | 160 + pyload/plugins/addons/LinkdecrypterCom.py | 59 + pyload/plugins/addons/MergeFiles.py | 94 + pyload/plugins/addons/MultiDebridCom.py | 42 + pyload/plugins/addons/MultiHome.py | 82 + pyload/plugins/addons/MultiHoster.py | 102 + pyload/plugins/addons/MultishareCz.py | 23 + pyload/plugins/addons/Premium4Me.py | 33 + pyload/plugins/addons/PremiumizeMe.py | 50 + pyload/plugins/addons/RealdebridCom.py | 25 + pyload/plugins/addons/RehostTo.py | 41 + pyload/plugins/addons/RestartFailed.py | 124 + pyload/plugins/addons/UnSkipOnFail.py | 97 + pyload/plugins/addons/UpdateManager.py | 201 ++ pyload/plugins/addons/XFileSharingPro.py | 70 + pyload/plugins/addons/XMPPInterface.py | 276 ++ pyload/plugins/addons/ZeveraCom.py | 19 + pyload/plugins/addons/__init__.py | 0 pyload/plugins/crypter/C1neonCom.py | 133 + pyload/plugins/crypter/CCF.py | 42 + pyload/plugins/crypter/CrockoComFolder.py | 14 + pyload/plugins/crypter/CryptItCom.py | 38 + pyload/plugins/crypter/CzshareComFolder.py | 30 + pyload/plugins/crypter/DDLMusicOrg.py | 42 + pyload/plugins/crypter/DataHuFolder.py | 55 + pyload/plugins/crypter/DdlstorageComFolder.py | 32 + pyload/plugins/crypter/DepositfilesComFolder.py | 14 + pyload/plugins/crypter/Dereferer.py | 34 + pyload/plugins/crypter/DontKnowMe.py | 21 + pyload/plugins/crypter/DownloadVimeoCom.py | 30 + pyload/plugins/crypter/DuckCryptInfo.py | 58 + pyload/plugins/crypter/EasybytezComFolder.py | 60 + pyload/plugins/crypter/EmbeduploadCom.py | 54 + pyload/plugins/crypter/FilebeerInfoFolder.py | 35 + pyload/plugins/crypter/FilefactoryComFolder.py | 44 + pyload/plugins/crypter/FileserveComFolder.py | 32 + pyload/plugins/crypter/FourChanOrg.py | 25 + pyload/plugins/crypter/FshareVnFolder.py | 14 + pyload/plugins/crypter/GooGl.py | 41 + pyload/plugins/crypter/HoerbuchIn.py | 55 + pyload/plugins/crypter/HotfileFolderCom.py | 29 + pyload/plugins/crypter/ILoadTo.py | 62 + pyload/plugins/crypter/LetitbitNetFolder.py | 33 + pyload/plugins/crypter/LinkList.py | 55 + pyload/plugins/crypter/LinkSaveIn.py | 227 ++ pyload/plugins/crypter/LinkdecrypterCom.py | 100 + pyload/plugins/crypter/LixIn.py | 60 + pyload/plugins/crypter/LofCc.py | 49 + pyload/plugins/crypter/MBLinkInfo.py | 27 + pyload/plugins/crypter/MediafireComFolder.py | 55 + pyload/plugins/crypter/Movie2kTo.py | 151 + pyload/plugins/crypter/MultiloadCz.py | 41 + pyload/plugins/crypter/MultiuploadCom.py | 58 + pyload/plugins/crypter/NCryptIn.py | 251 ++ pyload/plugins/crypter/NetfolderIn.py | 71 + pyload/plugins/crypter/OneKhDe.py | 36 + pyload/plugins/crypter/OronComFolder.py | 46 + pyload/plugins/crypter/QuickshareCzFolder.py | 30 + pyload/plugins/crypter/RSDF.py | 49 + pyload/plugins/crypter/RSLayerCom.py | 49 + pyload/plugins/crypter/RelinkUs.py | 264 ++ pyload/plugins/crypter/SecuredIn.py | 334 ++ pyload/plugins/crypter/SerienjunkiesOrg.py | 321 ++ pyload/plugins/crypter/ShareLinksBiz.py | 269 ++ pyload/plugins/crypter/ShareRapidComFolder.py | 14 + pyload/plugins/crypter/SpeedLoadOrgFolder.py | 30 + pyload/plugins/crypter/StealthTo.py | 45 + pyload/plugins/crypter/TrailerzoneInfo.py | 45 + pyload/plugins/crypter/UlozToFolder.py | 40 + pyload/plugins/crypter/UploadedToFolder.py | 50 + pyload/plugins/crypter/WiiReloadedOrg.py | 52 + pyload/plugins/crypter/XfilesharingProFolder.py | 34 + pyload/plugins/crypter/YoutubeBatch.py | 42 + pyload/plugins/crypter/__init__.py | 0 pyload/plugins/hooks/Captcha9kw.py | 162 + pyload/plugins/hooks/ReloadCc.py | 65 + pyload/plugins/hoster/ARD.py | 80 + pyload/plugins/hoster/AlldebridCom.py | 84 + pyload/plugins/hoster/BasePlugin.py | 104 + pyload/plugins/hoster/BayfilesCom.py | 93 + pyload/plugins/hoster/BezvadataCz.py | 94 + pyload/plugins/hoster/BillionuploadsCom.py | 17 + pyload/plugins/hoster/BitshareCom.py | 179 ++ pyload/plugins/hoster/BoltsharingCom.py | 15 + pyload/plugins/hoster/CatShareNet.py | 38 + pyload/plugins/hoster/ChipDe.py | 24 + pyload/plugins/hoster/CloudzerNet.py | 65 + pyload/plugins/hoster/CramitIn.py | 20 + pyload/plugins/hoster/CrockoCom.py | 71 + pyload/plugins/hoster/CyberlockerCh.py | 15 + pyload/plugins/hoster/CzshareCom.py | 160 + pyload/plugins/hoster/DailymotionCom.py | 47 + pyload/plugins/hoster/DataHu.py | 53 + pyload/plugins/hoster/DataportCz.py | 68 + pyload/plugins/hoster/DateiTo.py | 94 + pyload/plugins/hoster/DdlstorageCom.py | 20 + pyload/plugins/hoster/DebridItaliaCom.py | 61 + pyload/plugins/hoster/DepositfilesCom.py | 112 + pyload/plugins/hoster/DlFreeFr.py | 183 ++ pyload/plugins/hoster/EasybytezCom.py | 47 + pyload/plugins/hoster/EdiskCz.py | 62 + pyload/plugins/hoster/EgoFilesCom.py | 103 + pyload/plugins/hoster/EuroshareEu.py | 74 + pyload/plugins/hoster/ExtabitCom.py | 87 + pyload/plugins/hoster/FastshareCz.py | 96 + pyload/plugins/hoster/FileApeCom.py | 62 + pyload/plugins/hoster/FilebeerInfo.py | 15 + pyload/plugins/hoster/FilecloudIo.py | 112 + pyload/plugins/hoster/FilefactoryCom.py | 133 + pyload/plugins/hoster/FilejungleCom.py | 38 + pyload/plugins/hoster/FilepostCom.py | 135 + pyload/plugins/hoster/FilerNet.py | 103 + pyload/plugins/hoster/FilerioCom.py | 20 + pyload/plugins/hoster/FilesMailRu.py | 99 + pyload/plugins/hoster/FileserveCom.py | 211 ++ pyload/plugins/hoster/FileshareInUa.py | 78 + pyload/plugins/hoster/FlyFilesNet.py | 41 + pyload/plugins/hoster/FourSharedCom.py | 53 + pyload/plugins/hoster/FreakshareCom.py | 167 + pyload/plugins/hoster/FreevideoCz.py | 64 + pyload/plugins/hoster/FshareVn.py | 111 + pyload/plugins/hoster/Ftp.py | 91 + pyload/plugins/hoster/GamefrontCom.py | 80 + pyload/plugins/hoster/GigapetaCom.py | 73 + pyload/plugins/hoster/HellshareCz.py | 56 + pyload/plugins/hoster/HellspyCz.py | 70 + pyload/plugins/hoster/HotfileCom.py | 137 + pyload/plugins/hoster/IFileWs.py | 20 + pyload/plugins/hoster/IcyFilesCom.py | 112 + pyload/plugins/hoster/IfileIt.py | 74 + pyload/plugins/hoster/IfolderRu.py | 90 + pyload/plugins/hoster/JumbofilesCom.py | 31 + pyload/plugins/hoster/LetitbitNet.py | 125 + pyload/plugins/hoster/LoadTo.py | 83 + pyload/plugins/hoster/LuckyShareNet.py | 72 + pyload/plugins/hoster/MediafireCom.py | 135 + pyload/plugins/hoster/MegaNz.py | 125 + pyload/plugins/hoster/MegasharesCom.py | 108 + pyload/plugins/hoster/MovReelCom.py | 106 + pyload/plugins/hoster/MultiDebridCom.py | 57 + pyload/plugins/hoster/MultishareCz.py | 76 + pyload/plugins/hoster/MyvideoDe.py | 43 + pyload/plugins/hoster/NarodRu.py | 66 + pyload/plugins/hoster/NetloadIn.py | 252 ++ pyload/plugins/hoster/NovafileCom.py | 24 + pyload/plugins/hoster/NowDownloadEu.py | 66 + pyload/plugins/hoster/OneFichierCom.py | 58 + pyload/plugins/hoster/PornhostCom.py | 76 + pyload/plugins/hoster/PornhubCom.py | 83 + pyload/plugins/hoster/Premium4Me.py | 70 + pyload/plugins/hoster/PremiumizeMe.py | 50 + pyload/plugins/hoster/PutlockerCom.py | 78 + pyload/plugins/hoster/QuickshareCz.py | 99 + pyload/plugins/hoster/RapidgatorNet.py | 192 ++ pyload/plugins/hoster/RapidshareCom.py | 225 ++ pyload/plugins/hoster/RarefileNet.py | 34 + pyload/plugins/hoster/RealdebridCom.py | 88 + pyload/plugins/hoster/RedtubeCom.py | 56 + pyload/plugins/hoster/RehostTo.py | 37 + pyload/plugins/hoster/ReloadCc.py | 103 + pyload/plugins/hoster/RyushareCom.py | 55 + pyload/plugins/hoster/SecureUploadEu.py | 18 + pyload/plugins/hoster/SendmywayCom.py | 18 + pyload/plugins/hoster/SendspaceCom.py | 67 + pyload/plugins/hoster/Share4webCom.py | 16 + pyload/plugins/hoster/Share76Com.py | 19 + pyload/plugins/hoster/ShareFilesCo.py | 24 + pyload/plugins/hoster/ShareRapidCom.py | 104 + pyload/plugins/hoster/SharebeesCom.py | 19 + pyload/plugins/hoster/ShareonlineBiz.py | 203 ++ pyload/plugins/hoster/ShareplaceCom.py | 84 + pyload/plugins/hoster/ShragleCom.py | 106 + pyload/plugins/hoster/SpeedLoadOrg.py | 21 + pyload/plugins/hoster/SpeedfileCz.py | 65 + pyload/plugins/hoster/StreamCz.py | 76 + pyload/plugins/hoster/TurbobitNet.py | 170 + pyload/plugins/hoster/TurbouploadCom.py | 45 + pyload/plugins/hoster/TusfilesNet.py | 18 + pyload/plugins/hoster/TwoSharedCom.py | 33 + pyload/plugins/hoster/UlozTo.py | 156 + pyload/plugins/hoster/UloziskoSk.py | 75 + pyload/plugins/hoster/UnibytesCom.py | 80 + pyload/plugins/hoster/UploadStationCom.py | 21 + pyload/plugins/hoster/UploadedTo.py | 238 ++ pyload/plugins/hoster/UploadheroCom.py | 84 + pyload/plugins/hoster/UploadingCom.py | 110 + pyload/plugins/hoster/UptoboxCom.py | 21 + pyload/plugins/hoster/VeehdCom.py | 80 + pyload/plugins/hoster/WarserverCz.py | 70 + pyload/plugins/hoster/WebshareCz.py | 48 + pyload/plugins/hoster/WrzucTo.py | 58 + pyload/plugins/hoster/WuploadCom.py | 241 ++ pyload/plugins/hoster/X7To.py | 93 + pyload/plugins/hoster/XFileSharingPro.py | 314 ++ pyload/plugins/hoster/XHamsterCom.py | 120 + pyload/plugins/hoster/XVideosCom.py | 19 + pyload/plugins/hoster/Xdcc.py | 229 ++ pyload/plugins/hoster/XvidstageCom.py | 114 + pyload/plugins/hoster/YibaishiwuCom.py | 54 + pyload/plugins/hoster/YoupornCom.py | 56 + pyload/plugins/hoster/YourfilesTo.py | 83 + pyload/plugins/hoster/YoutubeCom.py | 164 + pyload/plugins/hoster/ZDF.py | 46 + pyload/plugins/hoster/ZeveraCom.py | 108 + pyload/plugins/hoster/ZippyshareCom.py | 187 ++ pyload/plugins/hoster/__init__.py | 0 pyload/plugins/internal/AbstractExtractor.py | 93 + pyload/plugins/internal/CaptchaService.py | 77 + pyload/plugins/internal/DeadHoster.py | 18 + pyload/plugins/internal/NetloadInOCR.py | 27 + pyload/plugins/internal/OCR.py | 314 ++ pyload/plugins/internal/ShareonlineBizOCR.py | 53 + pyload/plugins/internal/SimpleCrypter.py | 68 + pyload/plugins/internal/SimpleHoster.py | 251 ++ pyload/plugins/internal/UnRar.py | 212 ++ pyload/plugins/internal/UnZip.py | 49 + pyload/plugins/internal/XFSPAccount.py | 79 + pyload/plugins/internal/__init__.py | 0 pyload/remote/ClickAndLoadBackend.py | 170 + pyload/remote/JSONClient.py | 56 + pyload/remote/RemoteManager.py | 89 + pyload/remote/WSClient.py | 59 + pyload/remote/WebSocketBackend.py | 49 + pyload/remote/__init__.py | 0 pyload/remote/apitypes.py | 536 ++++ pyload/remote/apitypes_debug.py | 135 + pyload/remote/create_apitypes.py | 180 ++ pyload/remote/create_jstypes.py | 36 + pyload/remote/json_converter.py | 64 + pyload/remote/pyload.thrift | 538 ++++ pyload/remote/ttypes.py | 534 ++++ pyload/remote/wsbackend/AbstractHandler.py | 133 + pyload/remote/wsbackend/ApiHandler.py | 81 + pyload/remote/wsbackend/AsyncHandler.py | 167 + pyload/remote/wsbackend/Dispatcher.py | 31 + pyload/remote/wsbackend/Server.py | 733 +++++ pyload/remote/wsbackend/__init__.py | 0 pyload/setup/System_Checks.py | 126 + pyload/threads/AddonThread.py | 65 + pyload/threads/BaseThread.py | 142 + pyload/threads/DecrypterThread.py | 81 + pyload/threads/DownloadThread.py | 231 ++ pyload/threads/InfoThread.py | 168 + pyload/threads/ThreadManager.py | 313 ++ pyload/threads/__init__.py | 0 pyload/utils/ImportDebugger.py | 19 + pyload/utils/JsEngine.py | 195 ++ pyload/utils/__init__.py | 253 ++ pyload/utils/fs.py | 78 + pyload/utils/json_layer.py | 15 + pyload/utils/packagetools.py | 155 + pyload/utils/pylgettext.py | 61 + pyload/web/.bowerrc | 3 + pyload/web/.jshintrc | 29 + pyload/web/Gruntfile.js | 361 +++ pyload/web/ServerThread.py | 143 + pyload/web/__init__.py | 0 pyload/web/api_app.py | 112 + pyload/web/app/favicon.ico | Bin 0 -> 6006 bytes pyload/web/app/fonts/Sansation_Bold-webfont.eot | Bin 0 -> 35336 bytes pyload/web/app/fonts/Sansation_Bold-webfont.ttf | Bin 0 -> 35160 bytes pyload/web/app/fonts/Sansation_Bold-webfont.woff | Bin 0 -> 18496 bytes pyload/web/app/fonts/Sansation_Light-webfont.eot | Bin 0 -> 36700 bytes pyload/web/app/fonts/Sansation_Light-webfont.ttf | Bin 0 -> 36520 bytes pyload/web/app/fonts/Sansation_Light-webfont.woff | Bin 0 -> 18408 bytes pyload/web/app/fonts/Sansation_Regular-webfont.eot | Bin 0 -> 36368 bytes pyload/web/app/fonts/Sansation_Regular-webfont.ttf | Bin 0 -> 36180 bytes .../web/app/fonts/Sansation_Regular-webfont.woff | Bin 0 -> 18316 bytes pyload/web/app/images/default/bgpattern.png | Bin 0 -> 2487 bytes pyload/web/app/images/default/checks_sheet.png | Bin 0 -> 1145 bytes pyload/web/app/images/default/fancy_deboss.png | Bin 0 -> 265 bytes pyload/web/app/images/default/logo.png | Bin 0 -> 5329 bytes pyload/web/app/images/default/logo_grey.png | Bin 0 -> 1141 bytes pyload/web/app/images/icon.png | Bin 0 -> 1912 bytes pyload/web/app/index.html | 107 + pyload/web/app/scripts/app.js | 105 + pyload/web/app/scripts/collections/AccountList.js | 24 + pyload/web/app/scripts/collections/FileList.js | 18 + .../web/app/scripts/collections/InteractionList.js | 49 + pyload/web/app/scripts/collections/PackageList.js | 16 + pyload/web/app/scripts/collections/ProgressList.js | 18 + pyload/web/app/scripts/config.js | 73 + pyload/web/app/scripts/controller.js | 67 + pyload/web/app/scripts/default.js | 30 + pyload/web/app/scripts/helpers/fileHelper.js | 55 + pyload/web/app/scripts/helpers/formatSize.js | 15 + pyload/web/app/scripts/helpers/formatTime.js | 17 + pyload/web/app/scripts/helpers/pluginIcon.js | 14 + pyload/web/app/scripts/models/Account.js | 51 + pyload/web/app/scripts/models/ConfigHolder.js | 68 + pyload/web/app/scripts/models/ConfigItem.js | 40 + pyload/web/app/scripts/models/File.js | 92 + pyload/web/app/scripts/models/InteractionTask.js | 41 + pyload/web/app/scripts/models/Package.js | 119 + pyload/web/app/scripts/models/Progress.js | 50 + pyload/web/app/scripts/models/ServerStatus.js | 47 + pyload/web/app/scripts/models/TreeCollection.js | 50 + pyload/web/app/scripts/models/UserSession.js | 20 + pyload/web/app/scripts/router.js | 29 + pyload/web/app/scripts/routers/defaultRouter.js | 30 + pyload/web/app/scripts/routers/mobileRouter.js | 56 + pyload/web/app/scripts/utils/animations.js | 129 + pyload/web/app/scripts/utils/apitypes.js | 16 + pyload/web/app/scripts/utils/dialogs.js | 16 + pyload/web/app/scripts/utils/initHB.js | 11 + pyload/web/app/scripts/utils/lazyRequire.js | 97 + pyload/web/app/scripts/vendor/Handlebars-1.0rc1.js | 1927 ++++++++++++ pyload/web/app/scripts/vendor/bootstrap-2.3.2.js | 2291 ++++++++++++++ pyload/web/app/scripts/vendor/jquery.omniwindow.js | 141 + pyload/web/app/scripts/vendor/remaining.js | 149 + pyload/web/app/scripts/views/abstract/itemView.js | 47 + pyload/web/app/scripts/views/abstract/modalView.js | 125 + .../app/scripts/views/accounts/accountListView.js | 52 + .../web/app/scripts/views/accounts/accountModal.js | 72 + .../web/app/scripts/views/accounts/accountView.js | 18 + .../app/scripts/views/dashboard/dashboardView.js | 168 + pyload/web/app/scripts/views/dashboard/fileView.js | 102 + .../web/app/scripts/views/dashboard/filterView.js | 133 + .../web/app/scripts/views/dashboard/packageView.js | 75 + .../app/scripts/views/dashboard/selectionView.js | 155 + pyload/web/app/scripts/views/headerView.js | 240 ++ pyload/web/app/scripts/views/input/inputLoader.js | 8 + pyload/web/app/scripts/views/input/inputView.js | 86 + pyload/web/app/scripts/views/input/textInput.js | 36 + pyload/web/app/scripts/views/linkGrabberModal.js | 49 + pyload/web/app/scripts/views/loginView.js | 37 + pyload/web/app/scripts/views/notificationView.js | 83 + pyload/web/app/scripts/views/progressView.js | 33 + pyload/web/app/scripts/views/queryModal.js | 69 + .../scripts/views/settings/configSectionView.js | 99 + .../scripts/views/settings/pluginChooserModal.js | 69 + .../web/app/scripts/views/settings/settingsView.js | 184 ++ pyload/web/app/styles/default/accounts.less | 6 + pyload/web/app/styles/default/admin.less | 17 + pyload/web/app/styles/default/base.less | 163 + pyload/web/app/styles/default/common.less | 90 + pyload/web/app/styles/default/dashboard.less | 331 ++ pyload/web/app/styles/default/main.less | 11 + pyload/web/app/styles/default/settings.less | 121 + pyload/web/app/styles/default/style.less | 366 +++ pyload/web/app/styles/font.css | 37 + .../app/templates/default/accounts/account.html | 10 + .../app/templates/default/accounts/actionbar.html | 5 + .../web/app/templates/default/accounts/layout.html | 19 + pyload/web/app/templates/default/admin.html | 223 ++ .../app/templates/default/dashboard/actionbar.html | 54 + .../web/app/templates/default/dashboard/file.html | 34 + .../app/templates/default/dashboard/layout.html | 35 + .../app/templates/default/dashboard/package.html | 50 + .../app/templates/default/dashboard/select.html | 11 + .../app/templates/default/dialogs/addAccount.html | 42 + .../templates/default/dialogs/addPluginConfig.html | 26 + .../templates/default/dialogs/confirmDelete.html | 11 + .../templates/default/dialogs/interactionTask.html | 37 + .../app/templates/default/dialogs/linkgrabber.html | 49 + .../web/app/templates/default/dialogs/modal.html | 10 + .../web/app/templates/default/header/layout.html | 62 + .../web/app/templates/default/header/progress.html | 14 + .../app/templates/default/header/progressbar.html | 27 + .../web/app/templates/default/header/status.html | 3 + pyload/web/app/templates/default/login.html | 28 + pyload/web/app/templates/default/notification.html | 11 + .../app/templates/default/settings/actionbar.html | 5 + .../web/app/templates/default/settings/config.html | 17 + .../app/templates/default/settings/configItem.html | 7 + .../web/app/templates/default/settings/layout.html | 11 + .../web/app/templates/default/settings/menu.html | 40 + pyload/web/app/templates/default/setup.html | 16 + pyload/web/app/unavailable.html | 18 + pyload/web/bower.json | 23 + pyload/web/cnl_app.py | 166 + pyload/web/middlewares.py | 134 + pyload/web/package.json | 32 + pyload/web/pyload_app.py | 78 + pyload/web/servers.py | 162 + pyload/web/setup_app.py | 21 + pyload/web/utils.py | 78 + pyload/web/webinterface.py | 98 + 1166 files changed, 68467 insertions(+), 69649 deletions(-) delete mode 100644 module/AccountManager.py delete mode 100644 module/AddonManager.py delete mode 100644 module/Api.py delete mode 100644 module/FileManager.py delete mode 100644 module/InitHomeDir.py delete mode 100644 module/PluginManager.py delete mode 100644 module/Scheduler.py delete mode 100644 module/Setup.py delete mode 100644 module/__init__.py delete mode 100644 module/api/AccountApi.py delete mode 100644 module/api/AddonApi.py delete mode 100644 module/api/ApiComponent.py delete mode 100644 module/api/CollectorApi.py delete mode 100644 module/api/ConfigApi.py delete mode 100644 module/api/CoreApi.py delete mode 100644 module/api/DownloadApi.py delete mode 100644 module/api/DownloadPreparingApi.py delete mode 100644 module/api/FileApi.py delete mode 100644 module/api/UserInteractionApi.py delete mode 100644 module/api/__init__.py delete mode 100644 module/cli/AddPackage.py delete mode 100644 module/cli/Handler.py delete mode 100644 module/cli/ManageFiles.py delete mode 100644 module/cli/__init__.py delete mode 100644 module/cli/printer.py delete mode 100644 module/config/ConfigManager.py delete mode 100644 module/config/ConfigParser.py delete mode 100644 module/config/__init__.py delete mode 100644 module/config/default.py delete mode 100644 module/database/AccountDatabase.py delete mode 100644 module/database/ConfigDatabase.py delete mode 100644 module/database/DatabaseBackend.py delete mode 100644 module/database/FileDatabase.py delete mode 100644 module/database/StatisticDatabase.py delete mode 100644 module/database/StorageDatabase.py delete mode 100644 module/database/UserDatabase.py delete mode 100644 module/database/__init__.py delete mode 100644 module/datatypes/PyFile.py delete mode 100644 module/datatypes/PyPackage.py delete mode 100644 module/datatypes/User.py delete mode 100644 module/datatypes/__init__.py delete mode 100644 module/interaction/EventManager.py delete mode 100644 module/interaction/InteractionManager.py delete mode 100644 module/interaction/InteractionTask.py delete mode 100644 module/interaction/__init__.py delete mode 100644 module/lib/Getch.py delete mode 100644 module/lib/ReadWriteLock.py delete mode 100644 module/lib/SafeEval.py delete mode 100644 module/lib/__init__.py delete mode 100644 module/lib/beaker/__init__.py delete mode 100644 module/lib/beaker/cache.py delete mode 100644 module/lib/beaker/container.py delete mode 100644 module/lib/beaker/converters.py delete mode 100644 module/lib/beaker/crypto/__init__.py delete mode 100644 module/lib/beaker/crypto/jcecrypto.py delete mode 100644 module/lib/beaker/crypto/pbkdf2.py delete mode 100644 module/lib/beaker/crypto/pycrypto.py delete mode 100644 module/lib/beaker/crypto/util.py delete mode 100644 module/lib/beaker/exceptions.py delete mode 100644 module/lib/beaker/ext/__init__.py delete mode 100644 module/lib/beaker/ext/database.py delete mode 100644 module/lib/beaker/ext/google.py delete mode 100644 module/lib/beaker/ext/memcached.py delete mode 100644 module/lib/beaker/ext/sqla.py delete mode 100644 module/lib/beaker/middleware.py delete mode 100644 module/lib/beaker/session.py delete mode 100644 module/lib/beaker/synchronization.py delete mode 100644 module/lib/beaker/util.py delete mode 100644 module/lib/bottle.py delete mode 100644 module/lib/forwarder.py delete mode 100644 module/lib/hg_tool.py delete mode 100644 module/lib/mod_pywebsocket/COPYING delete mode 100644 module/lib/mod_pywebsocket/__init__.py delete mode 100644 module/lib/mod_pywebsocket/_stream_base.py delete mode 100644 module/lib/mod_pywebsocket/_stream_hixie75.py delete mode 100644 module/lib/mod_pywebsocket/_stream_hybi.py delete mode 100644 module/lib/mod_pywebsocket/common.py delete mode 100644 module/lib/mod_pywebsocket/dispatch.py delete mode 100644 module/lib/mod_pywebsocket/extensions.py delete mode 100644 module/lib/mod_pywebsocket/handshake/__init__.py delete mode 100644 module/lib/mod_pywebsocket/handshake/_base.py delete mode 100644 module/lib/mod_pywebsocket/handshake/hybi.py delete mode 100644 module/lib/mod_pywebsocket/handshake/hybi00.py delete mode 100644 module/lib/mod_pywebsocket/headerparserhandler.py delete mode 100644 module/lib/mod_pywebsocket/http_header_util.py delete mode 100644 module/lib/mod_pywebsocket/memorizingfile.py delete mode 100644 module/lib/mod_pywebsocket/msgutil.py delete mode 100644 module/lib/mod_pywebsocket/mux.py delete mode 100755 module/lib/mod_pywebsocket/standalone.py delete mode 100644 module/lib/mod_pywebsocket/stream.py delete mode 100644 module/lib/mod_pywebsocket/util.py delete mode 100644 module/lib/new_collections.py delete mode 100644 module/lib/rename_process.py delete mode 100644 module/lib/simplejson/__init__.py delete mode 100644 module/lib/simplejson/decoder.py delete mode 100644 module/lib/simplejson/encoder.py delete mode 100644 module/lib/simplejson/ordered_dict.py delete mode 100644 module/lib/simplejson/scanner.py delete mode 100644 module/lib/simplejson/tool.py delete mode 100644 module/lib/wsgiserver/LICENSE.txt delete mode 100644 module/lib/wsgiserver/__init__.py delete mode 100644 module/network/Browser.py delete mode 100644 module/network/Bucket.py delete mode 100644 module/network/CookieJar.py delete mode 100644 module/network/HTTPChunk.py delete mode 100644 module/network/HTTPDownload.py delete mode 100644 module/network/HTTPRequest.py delete mode 100644 module/network/RequestFactory.py delete mode 100644 module/network/XDCCRequest.py delete mode 100644 module/network/__init__.py delete mode 100644 module/plugins/Account.py delete mode 100644 module/plugins/Addon.py delete mode 100644 module/plugins/Base.py delete mode 100644 module/plugins/Crypter.py delete mode 100644 module/plugins/Hoster.py delete mode 100644 module/plugins/MultiHoster.py delete mode 100644 module/plugins/ReCaptcha.py delete mode 100644 module/plugins/__init__.py delete mode 100644 module/plugins/accounts/AlldebridCom.py delete mode 100644 module/plugins/accounts/BayfilesCom.py delete mode 100644 module/plugins/accounts/BitshareCom.py delete mode 100644 module/plugins/accounts/BoltsharingCom.py delete mode 100644 module/plugins/accounts/CramitIn.py delete mode 100644 module/plugins/accounts/CyberlockerCh.py delete mode 100644 module/plugins/accounts/CzshareCom.py delete mode 100644 module/plugins/accounts/DdlstorageCom.py delete mode 100644 module/plugins/accounts/DebridItaliaCom.py delete mode 100644 module/plugins/accounts/DepositfilesCom.py delete mode 100644 module/plugins/accounts/EasybytezCom.py delete mode 100644 module/plugins/accounts/EgoFilesCom.py delete mode 100644 module/plugins/accounts/EuroshareEu.py delete mode 100644 module/plugins/accounts/FastshareCz.py delete mode 100644 module/plugins/accounts/FilebeerInfo.py delete mode 100644 module/plugins/accounts/FilecloudIo.py delete mode 100644 module/plugins/accounts/FilefactoryCom.py delete mode 100644 module/plugins/accounts/FilejungleCom.py delete mode 100644 module/plugins/accounts/FilerioCom.py delete mode 100644 module/plugins/accounts/FilesMailRu.py delete mode 100644 module/plugins/accounts/FileserveCom.py delete mode 100644 module/plugins/accounts/FourSharedCom.py delete mode 100644 module/plugins/accounts/FreakshareCom.py delete mode 100644 module/plugins/accounts/FshareVn.py delete mode 100644 module/plugins/accounts/Ftp.py delete mode 100644 module/plugins/accounts/HellshareCz.py delete mode 100644 module/plugins/accounts/HellspyCz.py delete mode 100644 module/plugins/accounts/HotfileCom.py delete mode 100644 module/plugins/accounts/Http.py delete mode 100644 module/plugins/accounts/MegasharesCom.py delete mode 100644 module/plugins/accounts/MultiDebridCom.py delete mode 100644 module/plugins/accounts/MultishareCz.py delete mode 100755 module/plugins/accounts/NetloadIn.py delete mode 100644 module/plugins/accounts/Premium4Me.py delete mode 100644 module/plugins/accounts/PremiumizeMe.py delete mode 100644 module/plugins/accounts/QuickshareCz.py delete mode 100644 module/plugins/accounts/RapidgatorNet.py delete mode 100644 module/plugins/accounts/RapidshareCom.py delete mode 100644 module/plugins/accounts/RarefileNet.py delete mode 100644 module/plugins/accounts/RealdebridCom.py delete mode 100644 module/plugins/accounts/RehostTo.py delete mode 100644 module/plugins/accounts/ReloadCc.py delete mode 100644 module/plugins/accounts/RyushareCom.py delete mode 100644 module/plugins/accounts/Share76Com.py delete mode 100644 module/plugins/accounts/ShareFilesCo.py delete mode 100644 module/plugins/accounts/ShareRapidCom.py delete mode 100644 module/plugins/accounts/ShareonlineBiz.py delete mode 100644 module/plugins/accounts/SpeedLoadOrg.py delete mode 100644 module/plugins/accounts/StahnuTo.py delete mode 100644 module/plugins/accounts/TurbobitNet.py delete mode 100644 module/plugins/accounts/UlozTo.py delete mode 100644 module/plugins/accounts/UploadedTo.py delete mode 100644 module/plugins/accounts/UploadheroCom.py delete mode 100644 module/plugins/accounts/UploadingCom.py delete mode 100644 module/plugins/accounts/UploadstationCom.py delete mode 100644 module/plugins/accounts/UptoboxCom.py delete mode 100644 module/plugins/accounts/WarserverCz.py delete mode 100644 module/plugins/accounts/WuploadCom.py delete mode 100644 module/plugins/accounts/X7To.py delete mode 100644 module/plugins/accounts/YibaishiwuCom.py delete mode 100644 module/plugins/accounts/ZeveraCom.py delete mode 100644 module/plugins/accounts/__init__.py delete mode 100644 module/plugins/addons/AlldebridCom.py delete mode 100644 module/plugins/addons/BypassCaptcha.py delete mode 100644 module/plugins/addons/CaptchaBrotherhood.py delete mode 100644 module/plugins/addons/CaptchaTrader.py delete mode 100644 module/plugins/addons/Checksum.py delete mode 100644 module/plugins/addons/ClickAndLoad.py delete mode 100644 module/plugins/addons/DeathByCaptcha.py delete mode 100644 module/plugins/addons/DebridItaliaCom.py delete mode 100644 module/plugins/addons/DeleteFinished.py delete mode 100644 module/plugins/addons/DownloadScheduler.py delete mode 100644 module/plugins/addons/EasybytezCom.py delete mode 100644 module/plugins/addons/Ev0InFetcher.py delete mode 100644 module/plugins/addons/ExpertDecoders.py delete mode 100644 module/plugins/addons/ExternalScripts.py delete mode 100644 module/plugins/addons/ExtractArchive.py delete mode 100644 module/plugins/addons/HotFolder.py delete mode 100644 module/plugins/addons/IRCInterface.py delete mode 100644 module/plugins/addons/ImageTyperz.py delete mode 100644 module/plugins/addons/LinkdecrypterCom.py delete mode 100644 module/plugins/addons/MergeFiles.py delete mode 100644 module/plugins/addons/MultiDebridCom.py delete mode 100644 module/plugins/addons/MultiHome.py delete mode 100644 module/plugins/addons/MultiHoster.py delete mode 100644 module/plugins/addons/MultishareCz.py delete mode 100644 module/plugins/addons/Premium4Me.py delete mode 100644 module/plugins/addons/PremiumizeMe.py delete mode 100644 module/plugins/addons/RealdebridCom.py delete mode 100644 module/plugins/addons/RehostTo.py delete mode 100644 module/plugins/addons/RestartFailed.py delete mode 100644 module/plugins/addons/UnSkipOnFail.py delete mode 100644 module/plugins/addons/UpdateManager.py delete mode 100644 module/plugins/addons/XFileSharingPro.py delete mode 100644 module/plugins/addons/XMPPInterface.py delete mode 100644 module/plugins/addons/ZeveraCom.py delete mode 100644 module/plugins/addons/__init__.py delete mode 100644 module/plugins/crypter/C1neonCom.py delete mode 100644 module/plugins/crypter/CCF.py delete mode 100644 module/plugins/crypter/CrockoComFolder.py delete mode 100644 module/plugins/crypter/CryptItCom.py delete mode 100644 module/plugins/crypter/CzshareComFolder.py delete mode 100644 module/plugins/crypter/DDLMusicOrg.py delete mode 100644 module/plugins/crypter/DataHuFolder.py delete mode 100644 module/plugins/crypter/DdlstorageComFolder.py delete mode 100644 module/plugins/crypter/DepositfilesComFolder.py delete mode 100644 module/plugins/crypter/Dereferer.py delete mode 100644 module/plugins/crypter/DontKnowMe.py delete mode 100644 module/plugins/crypter/DownloadVimeoCom.py delete mode 100644 module/plugins/crypter/DuckCryptInfo.py delete mode 100644 module/plugins/crypter/EasybytezComFolder.py delete mode 100644 module/plugins/crypter/EmbeduploadCom.py delete mode 100644 module/plugins/crypter/FilebeerInfoFolder.py delete mode 100644 module/plugins/crypter/FilefactoryComFolder.py delete mode 100644 module/plugins/crypter/FileserveComFolder.py delete mode 100644 module/plugins/crypter/FourChanOrg.py delete mode 100644 module/plugins/crypter/FshareVnFolder.py delete mode 100644 module/plugins/crypter/GooGl.py delete mode 100644 module/plugins/crypter/HoerbuchIn.py delete mode 100644 module/plugins/crypter/HotfileFolderCom.py delete mode 100644 module/plugins/crypter/ILoadTo.py delete mode 100644 module/plugins/crypter/LetitbitNetFolder.py delete mode 100644 module/plugins/crypter/LinkList.py delete mode 100644 module/plugins/crypter/LinkSaveIn.py delete mode 100644 module/plugins/crypter/LinkdecrypterCom.py delete mode 100644 module/plugins/crypter/LixIn.py delete mode 100644 module/plugins/crypter/LofCc.py delete mode 100644 module/plugins/crypter/MBLinkInfo.py delete mode 100644 module/plugins/crypter/MediafireComFolder.py delete mode 100644 module/plugins/crypter/Movie2kTo.py delete mode 100644 module/plugins/crypter/MultiloadCz.py delete mode 100644 module/plugins/crypter/MultiuploadCom.py delete mode 100644 module/plugins/crypter/NCryptIn.py delete mode 100644 module/plugins/crypter/NetfolderIn.py delete mode 100644 module/plugins/crypter/OneKhDe.py delete mode 100755 module/plugins/crypter/OronComFolder.py delete mode 100644 module/plugins/crypter/QuickshareCzFolder.py delete mode 100644 module/plugins/crypter/RSDF.py delete mode 100644 module/plugins/crypter/RSLayerCom.py delete mode 100644 module/plugins/crypter/RelinkUs.py delete mode 100644 module/plugins/crypter/SecuredIn.py delete mode 100644 module/plugins/crypter/SerienjunkiesOrg.py delete mode 100644 module/plugins/crypter/ShareLinksBiz.py delete mode 100644 module/plugins/crypter/ShareRapidComFolder.py delete mode 100644 module/plugins/crypter/SpeedLoadOrgFolder.py delete mode 100644 module/plugins/crypter/StealthTo.py delete mode 100644 module/plugins/crypter/TrailerzoneInfo.py delete mode 100644 module/plugins/crypter/UlozToFolder.py delete mode 100644 module/plugins/crypter/UploadedToFolder.py delete mode 100644 module/plugins/crypter/WiiReloadedOrg.py delete mode 100644 module/plugins/crypter/XfilesharingProFolder.py delete mode 100644 module/plugins/crypter/YoutubeBatch.py delete mode 100644 module/plugins/crypter/__init__.py delete mode 100755 module/plugins/hooks/Captcha9kw.py delete mode 100644 module/plugins/hooks/ReloadCc.py delete mode 100644 module/plugins/hoster/ARD.py delete mode 100644 module/plugins/hoster/AlldebridCom.py delete mode 100644 module/plugins/hoster/BasePlugin.py delete mode 100644 module/plugins/hoster/BayfilesCom.py delete mode 100644 module/plugins/hoster/BezvadataCz.py delete mode 100644 module/plugins/hoster/BillionuploadsCom.py delete mode 100644 module/plugins/hoster/BitshareCom.py delete mode 100644 module/plugins/hoster/BoltsharingCom.py delete mode 100644 module/plugins/hoster/CatShareNet.py delete mode 100644 module/plugins/hoster/ChipDe.py delete mode 100644 module/plugins/hoster/CloudzerNet.py delete mode 100644 module/plugins/hoster/CramitIn.py delete mode 100644 module/plugins/hoster/CrockoCom.py delete mode 100644 module/plugins/hoster/CyberlockerCh.py delete mode 100644 module/plugins/hoster/CzshareCom.py delete mode 100644 module/plugins/hoster/DailymotionCom.py delete mode 100644 module/plugins/hoster/DataHu.py delete mode 100644 module/plugins/hoster/DataportCz.py delete mode 100644 module/plugins/hoster/DateiTo.py delete mode 100644 module/plugins/hoster/DdlstorageCom.py delete mode 100644 module/plugins/hoster/DebridItaliaCom.py delete mode 100644 module/plugins/hoster/DepositfilesCom.py delete mode 100644 module/plugins/hoster/DlFreeFr.py delete mode 100644 module/plugins/hoster/EasybytezCom.py delete mode 100644 module/plugins/hoster/EdiskCz.py delete mode 100644 module/plugins/hoster/EgoFilesCom.py delete mode 100644 module/plugins/hoster/EuroshareEu.py delete mode 100644 module/plugins/hoster/ExtabitCom.py delete mode 100644 module/plugins/hoster/FastshareCz.py delete mode 100644 module/plugins/hoster/FileApeCom.py delete mode 100644 module/plugins/hoster/FilebeerInfo.py delete mode 100644 module/plugins/hoster/FilecloudIo.py delete mode 100644 module/plugins/hoster/FilefactoryCom.py delete mode 100644 module/plugins/hoster/FilejungleCom.py delete mode 100644 module/plugins/hoster/FilepostCom.py delete mode 100644 module/plugins/hoster/FilerNet.py delete mode 100644 module/plugins/hoster/FilerioCom.py delete mode 100644 module/plugins/hoster/FilesMailRu.py delete mode 100644 module/plugins/hoster/FileserveCom.py delete mode 100644 module/plugins/hoster/FileshareInUa.py delete mode 100644 module/plugins/hoster/FlyFilesNet.py delete mode 100644 module/plugins/hoster/FourSharedCom.py delete mode 100644 module/plugins/hoster/FreakshareCom.py delete mode 100644 module/plugins/hoster/FreevideoCz.py delete mode 100644 module/plugins/hoster/FshareVn.py delete mode 100644 module/plugins/hoster/Ftp.py delete mode 100644 module/plugins/hoster/GamefrontCom.py delete mode 100644 module/plugins/hoster/GigapetaCom.py delete mode 100644 module/plugins/hoster/HellshareCz.py delete mode 100644 module/plugins/hoster/HellspyCz.py delete mode 100644 module/plugins/hoster/HotfileCom.py delete mode 100644 module/plugins/hoster/IFileWs.py delete mode 100644 module/plugins/hoster/IcyFilesCom.py delete mode 100644 module/plugins/hoster/IfileIt.py delete mode 100644 module/plugins/hoster/IfolderRu.py delete mode 100644 module/plugins/hoster/JumbofilesCom.py delete mode 100644 module/plugins/hoster/LetitbitNet.py delete mode 100644 module/plugins/hoster/LoadTo.py delete mode 100644 module/plugins/hoster/LuckyShareNet.py delete mode 100644 module/plugins/hoster/MediafireCom.py delete mode 100644 module/plugins/hoster/MegaNz.py delete mode 100644 module/plugins/hoster/MegasharesCom.py delete mode 100644 module/plugins/hoster/MovReelCom.py delete mode 100644 module/plugins/hoster/MultiDebridCom.py delete mode 100644 module/plugins/hoster/MultishareCz.py delete mode 100644 module/plugins/hoster/MyvideoDe.py delete mode 100644 module/plugins/hoster/NarodRu.py delete mode 100644 module/plugins/hoster/NetloadIn.py delete mode 100644 module/plugins/hoster/NovafileCom.py delete mode 100644 module/plugins/hoster/NowDownloadEu.py delete mode 100644 module/plugins/hoster/OneFichierCom.py delete mode 100644 module/plugins/hoster/PornhostCom.py delete mode 100644 module/plugins/hoster/PornhubCom.py delete mode 100644 module/plugins/hoster/Premium4Me.py delete mode 100644 module/plugins/hoster/PremiumizeMe.py delete mode 100644 module/plugins/hoster/PutlockerCom.py delete mode 100644 module/plugins/hoster/QuickshareCz.py delete mode 100644 module/plugins/hoster/RapidgatorNet.py delete mode 100644 module/plugins/hoster/RapidshareCom.py delete mode 100644 module/plugins/hoster/RarefileNet.py delete mode 100644 module/plugins/hoster/RealdebridCom.py delete mode 100644 module/plugins/hoster/RedtubeCom.py delete mode 100644 module/plugins/hoster/RehostTo.py delete mode 100644 module/plugins/hoster/ReloadCc.py delete mode 100644 module/plugins/hoster/RyushareCom.py delete mode 100644 module/plugins/hoster/SecureUploadEu.py delete mode 100644 module/plugins/hoster/SendmywayCom.py delete mode 100644 module/plugins/hoster/SendspaceCom.py delete mode 100644 module/plugins/hoster/Share4webCom.py delete mode 100644 module/plugins/hoster/Share76Com.py delete mode 100644 module/plugins/hoster/ShareFilesCo.py delete mode 100644 module/plugins/hoster/ShareRapidCom.py delete mode 100644 module/plugins/hoster/SharebeesCom.py delete mode 100644 module/plugins/hoster/ShareonlineBiz.py delete mode 100644 module/plugins/hoster/ShareplaceCom.py delete mode 100644 module/plugins/hoster/ShragleCom.py delete mode 100644 module/plugins/hoster/SpeedLoadOrg.py delete mode 100644 module/plugins/hoster/SpeedfileCz.py delete mode 100644 module/plugins/hoster/StreamCz.py delete mode 100644 module/plugins/hoster/TurbobitNet.py delete mode 100644 module/plugins/hoster/TurbouploadCom.py delete mode 100644 module/plugins/hoster/TusfilesNet.py delete mode 100644 module/plugins/hoster/TwoSharedCom.py delete mode 100644 module/plugins/hoster/UlozTo.py delete mode 100644 module/plugins/hoster/UloziskoSk.py delete mode 100644 module/plugins/hoster/UnibytesCom.py delete mode 100644 module/plugins/hoster/UploadStationCom.py delete mode 100644 module/plugins/hoster/UploadedTo.py delete mode 100644 module/plugins/hoster/UploadheroCom.py delete mode 100644 module/plugins/hoster/UploadingCom.py delete mode 100644 module/plugins/hoster/UptoboxCom.py delete mode 100644 module/plugins/hoster/VeehdCom.py delete mode 100644 module/plugins/hoster/WarserverCz.py delete mode 100644 module/plugins/hoster/WebshareCz.py delete mode 100644 module/plugins/hoster/WrzucTo.py delete mode 100644 module/plugins/hoster/WuploadCom.py delete mode 100644 module/plugins/hoster/X7To.py delete mode 100644 module/plugins/hoster/XFileSharingPro.py delete mode 100644 module/plugins/hoster/XHamsterCom.py delete mode 100644 module/plugins/hoster/XVideosCom.py delete mode 100644 module/plugins/hoster/Xdcc.py delete mode 100644 module/plugins/hoster/XvidstageCom.py delete mode 100644 module/plugins/hoster/YibaishiwuCom.py delete mode 100644 module/plugins/hoster/YoupornCom.py delete mode 100644 module/plugins/hoster/YourfilesTo.py delete mode 100644 module/plugins/hoster/YoutubeCom.py delete mode 100644 module/plugins/hoster/ZDF.py delete mode 100644 module/plugins/hoster/ZeveraCom.py delete mode 100644 module/plugins/hoster/ZippyshareCom.py delete mode 100644 module/plugins/hoster/__init__.py delete mode 100644 module/plugins/internal/AbstractExtractor.py delete mode 100644 module/plugins/internal/CaptchaService.py delete mode 100644 module/plugins/internal/DeadHoster.py delete mode 100644 module/plugins/internal/NetloadInOCR.py delete mode 100644 module/plugins/internal/OCR.py delete mode 100644 module/plugins/internal/ShareonlineBizOCR.py delete mode 100644 module/plugins/internal/SimpleCrypter.py delete mode 100644 module/plugins/internal/SimpleHoster.py delete mode 100644 module/plugins/internal/UnRar.py delete mode 100644 module/plugins/internal/UnZip.py delete mode 100644 module/plugins/internal/XFSPAccount.py delete mode 100644 module/plugins/internal/__init__.py delete mode 100644 module/remote/ClickAndLoadBackend.py delete mode 100644 module/remote/JSONClient.py delete mode 100644 module/remote/RemoteManager.py delete mode 100644 module/remote/WSClient.py delete mode 100644 module/remote/WebSocketBackend.py delete mode 100644 module/remote/__init__.py delete mode 100644 module/remote/apitypes.py delete mode 100644 module/remote/apitypes_debug.py delete mode 100644 module/remote/create_apitypes.py delete mode 100644 module/remote/create_jstypes.py delete mode 100644 module/remote/json_converter.py delete mode 100644 module/remote/pyload.thrift delete mode 100644 module/remote/ttypes.py delete mode 100644 module/remote/wsbackend/AbstractHandler.py delete mode 100644 module/remote/wsbackend/ApiHandler.py delete mode 100644 module/remote/wsbackend/AsyncHandler.py delete mode 100644 module/remote/wsbackend/Dispatcher.py delete mode 100644 module/remote/wsbackend/Server.py delete mode 100644 module/remote/wsbackend/__init__.py delete mode 100644 module/setup/System_Checks.py delete mode 100644 module/threads/AddonThread.py delete mode 100644 module/threads/BaseThread.py delete mode 100644 module/threads/DecrypterThread.py delete mode 100644 module/threads/DownloadThread.py delete mode 100644 module/threads/InfoThread.py delete mode 100644 module/threads/ThreadManager.py delete mode 100644 module/threads/__init__.py delete mode 100644 module/utils/ImportDebugger.py delete mode 100644 module/utils/JsEngine.py delete mode 100644 module/utils/__init__.py delete mode 100644 module/utils/fs.py delete mode 100644 module/utils/json_layer.py delete mode 100644 module/utils/packagetools.py delete mode 100644 module/utils/pylgettext.py delete mode 100644 module/web/.bowerrc delete mode 100644 module/web/.jshintrc delete mode 100644 module/web/Gruntfile.js delete mode 100644 module/web/ServerThread.py delete mode 100644 module/web/__init__.py delete mode 100644 module/web/api_app.py delete mode 100644 module/web/app/favicon.ico delete mode 100644 module/web/app/fonts/Sansation_Bold-webfont.eot delete mode 100644 module/web/app/fonts/Sansation_Bold-webfont.ttf delete mode 100644 module/web/app/fonts/Sansation_Bold-webfont.woff delete mode 100644 module/web/app/fonts/Sansation_Light-webfont.eot delete mode 100644 module/web/app/fonts/Sansation_Light-webfont.ttf delete mode 100644 module/web/app/fonts/Sansation_Light-webfont.woff delete mode 100644 module/web/app/fonts/Sansation_Regular-webfont.eot delete mode 100644 module/web/app/fonts/Sansation_Regular-webfont.ttf delete mode 100644 module/web/app/fonts/Sansation_Regular-webfont.woff delete mode 100644 module/web/app/images/default/bgpattern.png delete mode 100644 module/web/app/images/default/checks_sheet.png delete mode 100644 module/web/app/images/default/fancy_deboss.png delete mode 100644 module/web/app/images/default/logo.png delete mode 100644 module/web/app/images/default/logo_grey.png delete mode 100644 module/web/app/images/icon.png delete mode 100644 module/web/app/index.html delete mode 100644 module/web/app/scripts/app.js delete mode 100644 module/web/app/scripts/collections/AccountList.js delete mode 100644 module/web/app/scripts/collections/FileList.js delete mode 100644 module/web/app/scripts/collections/InteractionList.js delete mode 100644 module/web/app/scripts/collections/PackageList.js delete mode 100644 module/web/app/scripts/collections/ProgressList.js delete mode 100644 module/web/app/scripts/config.js delete mode 100644 module/web/app/scripts/controller.js delete mode 100644 module/web/app/scripts/default.js delete mode 100644 module/web/app/scripts/helpers/fileHelper.js delete mode 100644 module/web/app/scripts/helpers/formatSize.js delete mode 100644 module/web/app/scripts/helpers/formatTime.js delete mode 100644 module/web/app/scripts/helpers/pluginIcon.js delete mode 100644 module/web/app/scripts/models/Account.js delete mode 100644 module/web/app/scripts/models/ConfigHolder.js delete mode 100644 module/web/app/scripts/models/ConfigItem.js delete mode 100644 module/web/app/scripts/models/File.js delete mode 100644 module/web/app/scripts/models/InteractionTask.js delete mode 100644 module/web/app/scripts/models/Package.js delete mode 100644 module/web/app/scripts/models/Progress.js delete mode 100644 module/web/app/scripts/models/ServerStatus.js delete mode 100644 module/web/app/scripts/models/TreeCollection.js delete mode 100644 module/web/app/scripts/models/UserSession.js delete mode 100644 module/web/app/scripts/router.js delete mode 100644 module/web/app/scripts/routers/defaultRouter.js delete mode 100644 module/web/app/scripts/routers/mobileRouter.js delete mode 100644 module/web/app/scripts/utils/animations.js delete mode 100644 module/web/app/scripts/utils/apitypes.js delete mode 100644 module/web/app/scripts/utils/dialogs.js delete mode 100644 module/web/app/scripts/utils/initHB.js delete mode 100644 module/web/app/scripts/utils/lazyRequire.js delete mode 100644 module/web/app/scripts/vendor/Handlebars-1.0rc1.js delete mode 100755 module/web/app/scripts/vendor/bootstrap-2.3.2.js delete mode 100644 module/web/app/scripts/vendor/jquery.omniwindow.js delete mode 100644 module/web/app/scripts/vendor/remaining.js delete mode 100644 module/web/app/scripts/views/abstract/itemView.js delete mode 100644 module/web/app/scripts/views/abstract/modalView.js delete mode 100644 module/web/app/scripts/views/accounts/accountListView.js delete mode 100644 module/web/app/scripts/views/accounts/accountModal.js delete mode 100644 module/web/app/scripts/views/accounts/accountView.js delete mode 100644 module/web/app/scripts/views/dashboard/dashboardView.js delete mode 100644 module/web/app/scripts/views/dashboard/fileView.js delete mode 100644 module/web/app/scripts/views/dashboard/filterView.js delete mode 100644 module/web/app/scripts/views/dashboard/packageView.js delete mode 100644 module/web/app/scripts/views/dashboard/selectionView.js delete mode 100644 module/web/app/scripts/views/headerView.js delete mode 100644 module/web/app/scripts/views/input/inputLoader.js delete mode 100644 module/web/app/scripts/views/input/inputView.js delete mode 100644 module/web/app/scripts/views/input/textInput.js delete mode 100644 module/web/app/scripts/views/linkGrabberModal.js delete mode 100644 module/web/app/scripts/views/loginView.js delete mode 100644 module/web/app/scripts/views/notificationView.js delete mode 100644 module/web/app/scripts/views/progressView.js delete mode 100644 module/web/app/scripts/views/queryModal.js delete mode 100644 module/web/app/scripts/views/settings/configSectionView.js delete mode 100644 module/web/app/scripts/views/settings/pluginChooserModal.js delete mode 100644 module/web/app/scripts/views/settings/settingsView.js delete mode 100644 module/web/app/styles/default/accounts.less delete mode 100644 module/web/app/styles/default/admin.less delete mode 100644 module/web/app/styles/default/base.less delete mode 100644 module/web/app/styles/default/common.less delete mode 100644 module/web/app/styles/default/dashboard.less delete mode 100644 module/web/app/styles/default/main.less delete mode 100644 module/web/app/styles/default/settings.less delete mode 100644 module/web/app/styles/default/style.less delete mode 100644 module/web/app/styles/font.css delete mode 100644 module/web/app/templates/default/accounts/account.html delete mode 100644 module/web/app/templates/default/accounts/actionbar.html delete mode 100644 module/web/app/templates/default/accounts/layout.html delete mode 100644 module/web/app/templates/default/admin.html delete mode 100644 module/web/app/templates/default/dashboard/actionbar.html delete mode 100644 module/web/app/templates/default/dashboard/file.html delete mode 100644 module/web/app/templates/default/dashboard/layout.html delete mode 100644 module/web/app/templates/default/dashboard/package.html delete mode 100644 module/web/app/templates/default/dashboard/select.html delete mode 100755 module/web/app/templates/default/dialogs/addAccount.html delete mode 100755 module/web/app/templates/default/dialogs/addPluginConfig.html delete mode 100644 module/web/app/templates/default/dialogs/confirmDelete.html delete mode 100755 module/web/app/templates/default/dialogs/interactionTask.html delete mode 100755 module/web/app/templates/default/dialogs/linkgrabber.html delete mode 100755 module/web/app/templates/default/dialogs/modal.html delete mode 100644 module/web/app/templates/default/header/layout.html delete mode 100644 module/web/app/templates/default/header/progress.html delete mode 100644 module/web/app/templates/default/header/progressbar.html delete mode 100644 module/web/app/templates/default/header/status.html delete mode 100644 module/web/app/templates/default/login.html delete mode 100644 module/web/app/templates/default/notification.html delete mode 100644 module/web/app/templates/default/settings/actionbar.html delete mode 100644 module/web/app/templates/default/settings/config.html delete mode 100644 module/web/app/templates/default/settings/configItem.html delete mode 100644 module/web/app/templates/default/settings/layout.html delete mode 100644 module/web/app/templates/default/settings/menu.html delete mode 100644 module/web/app/templates/default/setup.html delete mode 100644 module/web/app/unavailable.html delete mode 100644 module/web/bower.json delete mode 100644 module/web/cnl_app.py delete mode 100644 module/web/middlewares.py delete mode 100644 module/web/package.json delete mode 100644 module/web/pyload_app.py delete mode 100644 module/web/servers.py delete mode 100644 module/web/setup_app.py delete mode 100644 module/web/utils.py delete mode 100644 module/web/webinterface.py create mode 100644 pyload/AccountManager.py create mode 100644 pyload/AddonManager.py create mode 100644 pyload/Api.py create mode 100644 pyload/FileManager.py create mode 100644 pyload/InitHomeDir.py create mode 100644 pyload/PluginManager.py create mode 100644 pyload/Scheduler.py create mode 100644 pyload/Setup.py create mode 100644 pyload/__init__.py create mode 100644 pyload/api/AccountApi.py create mode 100644 pyload/api/AddonApi.py create mode 100644 pyload/api/ApiComponent.py create mode 100644 pyload/api/CollectorApi.py create mode 100644 pyload/api/ConfigApi.py create mode 100644 pyload/api/CoreApi.py create mode 100644 pyload/api/DownloadApi.py create mode 100644 pyload/api/DownloadPreparingApi.py create mode 100644 pyload/api/FileApi.py create mode 100644 pyload/api/UserInteractionApi.py create mode 100644 pyload/api/__init__.py create mode 100644 pyload/cli/AddPackage.py create mode 100644 pyload/cli/Handler.py create mode 100644 pyload/cli/ManageFiles.py create mode 100644 pyload/cli/__init__.py create mode 100644 pyload/cli/printer.py create mode 100644 pyload/config/ConfigManager.py create mode 100644 pyload/config/ConfigParser.py create mode 100644 pyload/config/__init__.py create mode 100644 pyload/config/default.py create mode 100644 pyload/database/AccountDatabase.py create mode 100644 pyload/database/ConfigDatabase.py create mode 100644 pyload/database/DatabaseBackend.py create mode 100644 pyload/database/FileDatabase.py create mode 100644 pyload/database/StatisticDatabase.py create mode 100644 pyload/database/StorageDatabase.py create mode 100644 pyload/database/UserDatabase.py create mode 100644 pyload/database/__init__.py create mode 100644 pyload/datatypes/PyFile.py create mode 100644 pyload/datatypes/PyPackage.py create mode 100644 pyload/datatypes/User.py create mode 100644 pyload/datatypes/__init__.py create mode 100644 pyload/interaction/EventManager.py create mode 100644 pyload/interaction/InteractionManager.py create mode 100644 pyload/interaction/InteractionTask.py create mode 100644 pyload/interaction/__init__.py create mode 100644 pyload/lib/Getch.py create mode 100644 pyload/lib/ReadWriteLock.py create mode 100644 pyload/lib/SafeEval.py create mode 100644 pyload/lib/__init__.py create mode 100644 pyload/lib/beaker/__init__.py create mode 100644 pyload/lib/beaker/cache.py create mode 100644 pyload/lib/beaker/container.py create mode 100644 pyload/lib/beaker/converters.py create mode 100644 pyload/lib/beaker/crypto/__init__.py create mode 100644 pyload/lib/beaker/crypto/jcecrypto.py create mode 100644 pyload/lib/beaker/crypto/pbkdf2.py create mode 100644 pyload/lib/beaker/crypto/pycrypto.py create mode 100644 pyload/lib/beaker/crypto/util.py create mode 100644 pyload/lib/beaker/exceptions.py create mode 100644 pyload/lib/beaker/ext/__init__.py create mode 100644 pyload/lib/beaker/ext/database.py create mode 100644 pyload/lib/beaker/ext/google.py create mode 100644 pyload/lib/beaker/ext/memcached.py create mode 100644 pyload/lib/beaker/ext/sqla.py create mode 100644 pyload/lib/beaker/middleware.py create mode 100644 pyload/lib/beaker/session.py create mode 100644 pyload/lib/beaker/synchronization.py create mode 100644 pyload/lib/beaker/util.py create mode 100644 pyload/lib/bottle.py create mode 100644 pyload/lib/forwarder.py create mode 100644 pyload/lib/hg_tool.py create mode 100644 pyload/lib/mod_pywebsocket/COPYING create mode 100644 pyload/lib/mod_pywebsocket/__init__.py create mode 100644 pyload/lib/mod_pywebsocket/_stream_base.py create mode 100644 pyload/lib/mod_pywebsocket/_stream_hixie75.py create mode 100644 pyload/lib/mod_pywebsocket/_stream_hybi.py create mode 100644 pyload/lib/mod_pywebsocket/common.py create mode 100644 pyload/lib/mod_pywebsocket/dispatch.py create mode 100644 pyload/lib/mod_pywebsocket/extensions.py create mode 100644 pyload/lib/mod_pywebsocket/handshake/__init__.py create mode 100644 pyload/lib/mod_pywebsocket/handshake/_base.py create mode 100644 pyload/lib/mod_pywebsocket/handshake/hybi.py create mode 100644 pyload/lib/mod_pywebsocket/handshake/hybi00.py create mode 100644 pyload/lib/mod_pywebsocket/headerparserhandler.py create mode 100644 pyload/lib/mod_pywebsocket/http_header_util.py create mode 100644 pyload/lib/mod_pywebsocket/memorizingfile.py create mode 100644 pyload/lib/mod_pywebsocket/msgutil.py create mode 100644 pyload/lib/mod_pywebsocket/mux.py create mode 100755 pyload/lib/mod_pywebsocket/standalone.py create mode 100644 pyload/lib/mod_pywebsocket/stream.py create mode 100644 pyload/lib/mod_pywebsocket/util.py create mode 100644 pyload/lib/new_collections.py create mode 100644 pyload/lib/rename_process.py create mode 100644 pyload/lib/simplejson/__init__.py create mode 100644 pyload/lib/simplejson/decoder.py create mode 100644 pyload/lib/simplejson/encoder.py create mode 100644 pyload/lib/simplejson/ordered_dict.py create mode 100644 pyload/lib/simplejson/scanner.py create mode 100644 pyload/lib/simplejson/tool.py create mode 100644 pyload/lib/wsgiserver/LICENSE.txt create mode 100644 pyload/lib/wsgiserver/__init__.py create mode 100644 pyload/network/Browser.py create mode 100644 pyload/network/Bucket.py create mode 100644 pyload/network/CookieJar.py create mode 100644 pyload/network/HTTPChunk.py create mode 100644 pyload/network/HTTPDownload.py create mode 100644 pyload/network/HTTPRequest.py create mode 100644 pyload/network/RequestFactory.py create mode 100644 pyload/network/XDCCRequest.py create mode 100644 pyload/network/__init__.py create mode 100644 pyload/plugins/Account.py create mode 100644 pyload/plugins/Addon.py create mode 100644 pyload/plugins/Base.py create mode 100644 pyload/plugins/Crypter.py create mode 100644 pyload/plugins/Hoster.py create mode 100644 pyload/plugins/MultiHoster.py create mode 100644 pyload/plugins/ReCaptcha.py create mode 100644 pyload/plugins/__init__.py create mode 100644 pyload/plugins/accounts/AlldebridCom.py create mode 100644 pyload/plugins/accounts/BayfilesCom.py create mode 100644 pyload/plugins/accounts/BitshareCom.py create mode 100644 pyload/plugins/accounts/BoltsharingCom.py create mode 100644 pyload/plugins/accounts/CramitIn.py create mode 100644 pyload/plugins/accounts/CyberlockerCh.py create mode 100644 pyload/plugins/accounts/CzshareCom.py create mode 100644 pyload/plugins/accounts/DdlstorageCom.py create mode 100644 pyload/plugins/accounts/DebridItaliaCom.py create mode 100644 pyload/plugins/accounts/DepositfilesCom.py create mode 100644 pyload/plugins/accounts/EasybytezCom.py create mode 100644 pyload/plugins/accounts/EgoFilesCom.py create mode 100644 pyload/plugins/accounts/EuroshareEu.py create mode 100644 pyload/plugins/accounts/FastshareCz.py create mode 100644 pyload/plugins/accounts/FilebeerInfo.py create mode 100644 pyload/plugins/accounts/FilecloudIo.py create mode 100644 pyload/plugins/accounts/FilefactoryCom.py create mode 100644 pyload/plugins/accounts/FilejungleCom.py create mode 100644 pyload/plugins/accounts/FilerioCom.py create mode 100644 pyload/plugins/accounts/FilesMailRu.py create mode 100644 pyload/plugins/accounts/FileserveCom.py create mode 100644 pyload/plugins/accounts/FourSharedCom.py create mode 100644 pyload/plugins/accounts/FreakshareCom.py create mode 100644 pyload/plugins/accounts/FshareVn.py create mode 100644 pyload/plugins/accounts/Ftp.py create mode 100644 pyload/plugins/accounts/HellshareCz.py create mode 100644 pyload/plugins/accounts/HellspyCz.py create mode 100644 pyload/plugins/accounts/HotfileCom.py create mode 100644 pyload/plugins/accounts/Http.py create mode 100644 pyload/plugins/accounts/MegasharesCom.py create mode 100644 pyload/plugins/accounts/MultiDebridCom.py create mode 100644 pyload/plugins/accounts/MultishareCz.py create mode 100755 pyload/plugins/accounts/NetloadIn.py create mode 100644 pyload/plugins/accounts/Premium4Me.py create mode 100644 pyload/plugins/accounts/PremiumizeMe.py create mode 100644 pyload/plugins/accounts/QuickshareCz.py create mode 100644 pyload/plugins/accounts/RapidgatorNet.py create mode 100644 pyload/plugins/accounts/RapidshareCom.py create mode 100644 pyload/plugins/accounts/RarefileNet.py create mode 100644 pyload/plugins/accounts/RealdebridCom.py create mode 100644 pyload/plugins/accounts/RehostTo.py create mode 100644 pyload/plugins/accounts/ReloadCc.py create mode 100644 pyload/plugins/accounts/RyushareCom.py create mode 100644 pyload/plugins/accounts/Share76Com.py create mode 100644 pyload/plugins/accounts/ShareFilesCo.py create mode 100644 pyload/plugins/accounts/ShareRapidCom.py create mode 100644 pyload/plugins/accounts/ShareonlineBiz.py create mode 100644 pyload/plugins/accounts/SpeedLoadOrg.py create mode 100644 pyload/plugins/accounts/StahnuTo.py create mode 100644 pyload/plugins/accounts/TurbobitNet.py create mode 100644 pyload/plugins/accounts/UlozTo.py create mode 100644 pyload/plugins/accounts/UploadedTo.py create mode 100644 pyload/plugins/accounts/UploadheroCom.py create mode 100644 pyload/plugins/accounts/UploadingCom.py create mode 100644 pyload/plugins/accounts/UploadstationCom.py create mode 100644 pyload/plugins/accounts/UptoboxCom.py create mode 100644 pyload/plugins/accounts/WarserverCz.py create mode 100644 pyload/plugins/accounts/WuploadCom.py create mode 100644 pyload/plugins/accounts/X7To.py create mode 100644 pyload/plugins/accounts/YibaishiwuCom.py create mode 100644 pyload/plugins/accounts/ZeveraCom.py create mode 100644 pyload/plugins/accounts/__init__.py create mode 100644 pyload/plugins/addons/AlldebridCom.py create mode 100644 pyload/plugins/addons/BypassCaptcha.py create mode 100644 pyload/plugins/addons/CaptchaBrotherhood.py create mode 100644 pyload/plugins/addons/CaptchaTrader.py create mode 100644 pyload/plugins/addons/Checksum.py create mode 100644 pyload/plugins/addons/ClickAndLoad.py create mode 100644 pyload/plugins/addons/DeathByCaptcha.py create mode 100644 pyload/plugins/addons/DebridItaliaCom.py create mode 100644 pyload/plugins/addons/DeleteFinished.py create mode 100644 pyload/plugins/addons/DownloadScheduler.py create mode 100644 pyload/plugins/addons/EasybytezCom.py create mode 100644 pyload/plugins/addons/Ev0InFetcher.py create mode 100644 pyload/plugins/addons/ExpertDecoders.py create mode 100644 pyload/plugins/addons/ExternalScripts.py create mode 100644 pyload/plugins/addons/ExtractArchive.py create mode 100644 pyload/plugins/addons/HotFolder.py create mode 100644 pyload/plugins/addons/IRCInterface.py create mode 100644 pyload/plugins/addons/ImageTyperz.py create mode 100644 pyload/plugins/addons/LinkdecrypterCom.py create mode 100644 pyload/plugins/addons/MergeFiles.py create mode 100644 pyload/plugins/addons/MultiDebridCom.py create mode 100644 pyload/plugins/addons/MultiHome.py create mode 100644 pyload/plugins/addons/MultiHoster.py create mode 100644 pyload/plugins/addons/MultishareCz.py create mode 100644 pyload/plugins/addons/Premium4Me.py create mode 100644 pyload/plugins/addons/PremiumizeMe.py create mode 100644 pyload/plugins/addons/RealdebridCom.py create mode 100644 pyload/plugins/addons/RehostTo.py create mode 100644 pyload/plugins/addons/RestartFailed.py create mode 100644 pyload/plugins/addons/UnSkipOnFail.py create mode 100644 pyload/plugins/addons/UpdateManager.py create mode 100644 pyload/plugins/addons/XFileSharingPro.py create mode 100644 pyload/plugins/addons/XMPPInterface.py create mode 100644 pyload/plugins/addons/ZeveraCom.py create mode 100644 pyload/plugins/addons/__init__.py create mode 100644 pyload/plugins/crypter/C1neonCom.py create mode 100644 pyload/plugins/crypter/CCF.py create mode 100644 pyload/plugins/crypter/CrockoComFolder.py create mode 100644 pyload/plugins/crypter/CryptItCom.py create mode 100644 pyload/plugins/crypter/CzshareComFolder.py create mode 100644 pyload/plugins/crypter/DDLMusicOrg.py create mode 100644 pyload/plugins/crypter/DataHuFolder.py create mode 100644 pyload/plugins/crypter/DdlstorageComFolder.py create mode 100644 pyload/plugins/crypter/DepositfilesComFolder.py create mode 100644 pyload/plugins/crypter/Dereferer.py create mode 100644 pyload/plugins/crypter/DontKnowMe.py create mode 100644 pyload/plugins/crypter/DownloadVimeoCom.py create mode 100644 pyload/plugins/crypter/DuckCryptInfo.py create mode 100644 pyload/plugins/crypter/EasybytezComFolder.py create mode 100644 pyload/plugins/crypter/EmbeduploadCom.py create mode 100644 pyload/plugins/crypter/FilebeerInfoFolder.py create mode 100644 pyload/plugins/crypter/FilefactoryComFolder.py create mode 100644 pyload/plugins/crypter/FileserveComFolder.py create mode 100644 pyload/plugins/crypter/FourChanOrg.py create mode 100644 pyload/plugins/crypter/FshareVnFolder.py create mode 100644 pyload/plugins/crypter/GooGl.py create mode 100644 pyload/plugins/crypter/HoerbuchIn.py create mode 100644 pyload/plugins/crypter/HotfileFolderCom.py create mode 100644 pyload/plugins/crypter/ILoadTo.py create mode 100644 pyload/plugins/crypter/LetitbitNetFolder.py create mode 100644 pyload/plugins/crypter/LinkList.py create mode 100644 pyload/plugins/crypter/LinkSaveIn.py create mode 100644 pyload/plugins/crypter/LinkdecrypterCom.py create mode 100644 pyload/plugins/crypter/LixIn.py create mode 100644 pyload/plugins/crypter/LofCc.py create mode 100644 pyload/plugins/crypter/MBLinkInfo.py create mode 100644 pyload/plugins/crypter/MediafireComFolder.py create mode 100644 pyload/plugins/crypter/Movie2kTo.py create mode 100644 pyload/plugins/crypter/MultiloadCz.py create mode 100644 pyload/plugins/crypter/MultiuploadCom.py create mode 100644 pyload/plugins/crypter/NCryptIn.py create mode 100644 pyload/plugins/crypter/NetfolderIn.py create mode 100644 pyload/plugins/crypter/OneKhDe.py create mode 100755 pyload/plugins/crypter/OronComFolder.py create mode 100644 pyload/plugins/crypter/QuickshareCzFolder.py create mode 100644 pyload/plugins/crypter/RSDF.py create mode 100644 pyload/plugins/crypter/RSLayerCom.py create mode 100644 pyload/plugins/crypter/RelinkUs.py create mode 100644 pyload/plugins/crypter/SecuredIn.py create mode 100644 pyload/plugins/crypter/SerienjunkiesOrg.py create mode 100644 pyload/plugins/crypter/ShareLinksBiz.py create mode 100644 pyload/plugins/crypter/ShareRapidComFolder.py create mode 100644 pyload/plugins/crypter/SpeedLoadOrgFolder.py create mode 100644 pyload/plugins/crypter/StealthTo.py create mode 100644 pyload/plugins/crypter/TrailerzoneInfo.py create mode 100644 pyload/plugins/crypter/UlozToFolder.py create mode 100644 pyload/plugins/crypter/UploadedToFolder.py create mode 100644 pyload/plugins/crypter/WiiReloadedOrg.py create mode 100644 pyload/plugins/crypter/XfilesharingProFolder.py create mode 100644 pyload/plugins/crypter/YoutubeBatch.py create mode 100644 pyload/plugins/crypter/__init__.py create mode 100755 pyload/plugins/hooks/Captcha9kw.py create mode 100644 pyload/plugins/hooks/ReloadCc.py create mode 100644 pyload/plugins/hoster/ARD.py create mode 100644 pyload/plugins/hoster/AlldebridCom.py create mode 100644 pyload/plugins/hoster/BasePlugin.py create mode 100644 pyload/plugins/hoster/BayfilesCom.py create mode 100644 pyload/plugins/hoster/BezvadataCz.py create mode 100644 pyload/plugins/hoster/BillionuploadsCom.py create mode 100644 pyload/plugins/hoster/BitshareCom.py create mode 100644 pyload/plugins/hoster/BoltsharingCom.py create mode 100644 pyload/plugins/hoster/CatShareNet.py create mode 100644 pyload/plugins/hoster/ChipDe.py create mode 100644 pyload/plugins/hoster/CloudzerNet.py create mode 100644 pyload/plugins/hoster/CramitIn.py create mode 100644 pyload/plugins/hoster/CrockoCom.py create mode 100644 pyload/plugins/hoster/CyberlockerCh.py create mode 100644 pyload/plugins/hoster/CzshareCom.py create mode 100644 pyload/plugins/hoster/DailymotionCom.py create mode 100644 pyload/plugins/hoster/DataHu.py create mode 100644 pyload/plugins/hoster/DataportCz.py create mode 100644 pyload/plugins/hoster/DateiTo.py create mode 100644 pyload/plugins/hoster/DdlstorageCom.py create mode 100644 pyload/plugins/hoster/DebridItaliaCom.py create mode 100644 pyload/plugins/hoster/DepositfilesCom.py create mode 100644 pyload/plugins/hoster/DlFreeFr.py create mode 100644 pyload/plugins/hoster/EasybytezCom.py create mode 100644 pyload/plugins/hoster/EdiskCz.py create mode 100644 pyload/plugins/hoster/EgoFilesCom.py create mode 100644 pyload/plugins/hoster/EuroshareEu.py create mode 100644 pyload/plugins/hoster/ExtabitCom.py create mode 100644 pyload/plugins/hoster/FastshareCz.py create mode 100644 pyload/plugins/hoster/FileApeCom.py create mode 100644 pyload/plugins/hoster/FilebeerInfo.py create mode 100644 pyload/plugins/hoster/FilecloudIo.py create mode 100644 pyload/plugins/hoster/FilefactoryCom.py create mode 100644 pyload/plugins/hoster/FilejungleCom.py create mode 100644 pyload/plugins/hoster/FilepostCom.py create mode 100644 pyload/plugins/hoster/FilerNet.py create mode 100644 pyload/plugins/hoster/FilerioCom.py create mode 100644 pyload/plugins/hoster/FilesMailRu.py create mode 100644 pyload/plugins/hoster/FileserveCom.py create mode 100644 pyload/plugins/hoster/FileshareInUa.py create mode 100644 pyload/plugins/hoster/FlyFilesNet.py create mode 100644 pyload/plugins/hoster/FourSharedCom.py create mode 100644 pyload/plugins/hoster/FreakshareCom.py create mode 100644 pyload/plugins/hoster/FreevideoCz.py create mode 100644 pyload/plugins/hoster/FshareVn.py create mode 100644 pyload/plugins/hoster/Ftp.py create mode 100644 pyload/plugins/hoster/GamefrontCom.py create mode 100644 pyload/plugins/hoster/GigapetaCom.py create mode 100644 pyload/plugins/hoster/HellshareCz.py create mode 100644 pyload/plugins/hoster/HellspyCz.py create mode 100644 pyload/plugins/hoster/HotfileCom.py create mode 100644 pyload/plugins/hoster/IFileWs.py create mode 100644 pyload/plugins/hoster/IcyFilesCom.py create mode 100644 pyload/plugins/hoster/IfileIt.py create mode 100644 pyload/plugins/hoster/IfolderRu.py create mode 100644 pyload/plugins/hoster/JumbofilesCom.py create mode 100644 pyload/plugins/hoster/LetitbitNet.py create mode 100644 pyload/plugins/hoster/LoadTo.py create mode 100644 pyload/plugins/hoster/LuckyShareNet.py create mode 100644 pyload/plugins/hoster/MediafireCom.py create mode 100644 pyload/plugins/hoster/MegaNz.py create mode 100644 pyload/plugins/hoster/MegasharesCom.py create mode 100644 pyload/plugins/hoster/MovReelCom.py create mode 100644 pyload/plugins/hoster/MultiDebridCom.py create mode 100644 pyload/plugins/hoster/MultishareCz.py create mode 100644 pyload/plugins/hoster/MyvideoDe.py create mode 100644 pyload/plugins/hoster/NarodRu.py create mode 100644 pyload/plugins/hoster/NetloadIn.py create mode 100644 pyload/plugins/hoster/NovafileCom.py create mode 100644 pyload/plugins/hoster/NowDownloadEu.py create mode 100644 pyload/plugins/hoster/OneFichierCom.py create mode 100644 pyload/plugins/hoster/PornhostCom.py create mode 100644 pyload/plugins/hoster/PornhubCom.py create mode 100644 pyload/plugins/hoster/Premium4Me.py create mode 100644 pyload/plugins/hoster/PremiumizeMe.py create mode 100644 pyload/plugins/hoster/PutlockerCom.py create mode 100644 pyload/plugins/hoster/QuickshareCz.py create mode 100644 pyload/plugins/hoster/RapidgatorNet.py create mode 100644 pyload/plugins/hoster/RapidshareCom.py create mode 100644 pyload/plugins/hoster/RarefileNet.py create mode 100644 pyload/plugins/hoster/RealdebridCom.py create mode 100644 pyload/plugins/hoster/RedtubeCom.py create mode 100644 pyload/plugins/hoster/RehostTo.py create mode 100644 pyload/plugins/hoster/ReloadCc.py create mode 100644 pyload/plugins/hoster/RyushareCom.py create mode 100644 pyload/plugins/hoster/SecureUploadEu.py create mode 100644 pyload/plugins/hoster/SendmywayCom.py create mode 100644 pyload/plugins/hoster/SendspaceCom.py create mode 100644 pyload/plugins/hoster/Share4webCom.py create mode 100644 pyload/plugins/hoster/Share76Com.py create mode 100644 pyload/plugins/hoster/ShareFilesCo.py create mode 100644 pyload/plugins/hoster/ShareRapidCom.py create mode 100644 pyload/plugins/hoster/SharebeesCom.py create mode 100644 pyload/plugins/hoster/ShareonlineBiz.py create mode 100644 pyload/plugins/hoster/ShareplaceCom.py create mode 100644 pyload/plugins/hoster/ShragleCom.py create mode 100644 pyload/plugins/hoster/SpeedLoadOrg.py create mode 100644 pyload/plugins/hoster/SpeedfileCz.py create mode 100644 pyload/plugins/hoster/StreamCz.py create mode 100644 pyload/plugins/hoster/TurbobitNet.py create mode 100644 pyload/plugins/hoster/TurbouploadCom.py create mode 100644 pyload/plugins/hoster/TusfilesNet.py create mode 100644 pyload/plugins/hoster/TwoSharedCom.py create mode 100644 pyload/plugins/hoster/UlozTo.py create mode 100644 pyload/plugins/hoster/UloziskoSk.py create mode 100644 pyload/plugins/hoster/UnibytesCom.py create mode 100644 pyload/plugins/hoster/UploadStationCom.py create mode 100644 pyload/plugins/hoster/UploadedTo.py create mode 100644 pyload/plugins/hoster/UploadheroCom.py create mode 100644 pyload/plugins/hoster/UploadingCom.py create mode 100644 pyload/plugins/hoster/UptoboxCom.py create mode 100644 pyload/plugins/hoster/VeehdCom.py create mode 100644 pyload/plugins/hoster/WarserverCz.py create mode 100644 pyload/plugins/hoster/WebshareCz.py create mode 100644 pyload/plugins/hoster/WrzucTo.py create mode 100644 pyload/plugins/hoster/WuploadCom.py create mode 100644 pyload/plugins/hoster/X7To.py create mode 100644 pyload/plugins/hoster/XFileSharingPro.py create mode 100644 pyload/plugins/hoster/XHamsterCom.py create mode 100644 pyload/plugins/hoster/XVideosCom.py create mode 100644 pyload/plugins/hoster/Xdcc.py create mode 100644 pyload/plugins/hoster/XvidstageCom.py create mode 100644 pyload/plugins/hoster/YibaishiwuCom.py create mode 100644 pyload/plugins/hoster/YoupornCom.py create mode 100644 pyload/plugins/hoster/YourfilesTo.py create mode 100644 pyload/plugins/hoster/YoutubeCom.py create mode 100644 pyload/plugins/hoster/ZDF.py create mode 100644 pyload/plugins/hoster/ZeveraCom.py create mode 100644 pyload/plugins/hoster/ZippyshareCom.py create mode 100644 pyload/plugins/hoster/__init__.py create mode 100644 pyload/plugins/internal/AbstractExtractor.py create mode 100644 pyload/plugins/internal/CaptchaService.py create mode 100644 pyload/plugins/internal/DeadHoster.py create mode 100644 pyload/plugins/internal/NetloadInOCR.py create mode 100644 pyload/plugins/internal/OCR.py create mode 100644 pyload/plugins/internal/ShareonlineBizOCR.py create mode 100644 pyload/plugins/internal/SimpleCrypter.py create mode 100644 pyload/plugins/internal/SimpleHoster.py create mode 100644 pyload/plugins/internal/UnRar.py create mode 100644 pyload/plugins/internal/UnZip.py create mode 100644 pyload/plugins/internal/XFSPAccount.py create mode 100644 pyload/plugins/internal/__init__.py create mode 100644 pyload/remote/ClickAndLoadBackend.py create mode 100644 pyload/remote/JSONClient.py create mode 100644 pyload/remote/RemoteManager.py create mode 100644 pyload/remote/WSClient.py create mode 100644 pyload/remote/WebSocketBackend.py create mode 100644 pyload/remote/__init__.py create mode 100644 pyload/remote/apitypes.py create mode 100644 pyload/remote/apitypes_debug.py create mode 100644 pyload/remote/create_apitypes.py create mode 100644 pyload/remote/create_jstypes.py create mode 100644 pyload/remote/json_converter.py create mode 100644 pyload/remote/pyload.thrift create mode 100644 pyload/remote/ttypes.py create mode 100644 pyload/remote/wsbackend/AbstractHandler.py create mode 100644 pyload/remote/wsbackend/ApiHandler.py create mode 100644 pyload/remote/wsbackend/AsyncHandler.py create mode 100644 pyload/remote/wsbackend/Dispatcher.py create mode 100644 pyload/remote/wsbackend/Server.py create mode 100644 pyload/remote/wsbackend/__init__.py create mode 100644 pyload/setup/System_Checks.py create mode 100644 pyload/threads/AddonThread.py create mode 100644 pyload/threads/BaseThread.py create mode 100644 pyload/threads/DecrypterThread.py create mode 100644 pyload/threads/DownloadThread.py create mode 100644 pyload/threads/InfoThread.py create mode 100644 pyload/threads/ThreadManager.py create mode 100644 pyload/threads/__init__.py create mode 100644 pyload/utils/ImportDebugger.py create mode 100644 pyload/utils/JsEngine.py create mode 100644 pyload/utils/__init__.py create mode 100644 pyload/utils/fs.py create mode 100644 pyload/utils/json_layer.py create mode 100644 pyload/utils/packagetools.py create mode 100644 pyload/utils/pylgettext.py create mode 100644 pyload/web/.bowerrc create mode 100644 pyload/web/.jshintrc create mode 100644 pyload/web/Gruntfile.js create mode 100644 pyload/web/ServerThread.py create mode 100644 pyload/web/__init__.py create mode 100644 pyload/web/api_app.py create mode 100644 pyload/web/app/favicon.ico create mode 100644 pyload/web/app/fonts/Sansation_Bold-webfont.eot create mode 100644 pyload/web/app/fonts/Sansation_Bold-webfont.ttf create mode 100644 pyload/web/app/fonts/Sansation_Bold-webfont.woff create mode 100644 pyload/web/app/fonts/Sansation_Light-webfont.eot create mode 100644 pyload/web/app/fonts/Sansation_Light-webfont.ttf create mode 100644 pyload/web/app/fonts/Sansation_Light-webfont.woff create mode 100644 pyload/web/app/fonts/Sansation_Regular-webfont.eot create mode 100644 pyload/web/app/fonts/Sansation_Regular-webfont.ttf create mode 100644 pyload/web/app/fonts/Sansation_Regular-webfont.woff create mode 100644 pyload/web/app/images/default/bgpattern.png create mode 100644 pyload/web/app/images/default/checks_sheet.png create mode 100644 pyload/web/app/images/default/fancy_deboss.png create mode 100644 pyload/web/app/images/default/logo.png create mode 100644 pyload/web/app/images/default/logo_grey.png create mode 100644 pyload/web/app/images/icon.png create mode 100644 pyload/web/app/index.html create mode 100644 pyload/web/app/scripts/app.js create mode 100644 pyload/web/app/scripts/collections/AccountList.js create mode 100644 pyload/web/app/scripts/collections/FileList.js create mode 100644 pyload/web/app/scripts/collections/InteractionList.js create mode 100644 pyload/web/app/scripts/collections/PackageList.js create mode 100644 pyload/web/app/scripts/collections/ProgressList.js create mode 100644 pyload/web/app/scripts/config.js create mode 100644 pyload/web/app/scripts/controller.js create mode 100644 pyload/web/app/scripts/default.js create mode 100644 pyload/web/app/scripts/helpers/fileHelper.js create mode 100644 pyload/web/app/scripts/helpers/formatSize.js create mode 100644 pyload/web/app/scripts/helpers/formatTime.js create mode 100644 pyload/web/app/scripts/helpers/pluginIcon.js create mode 100644 pyload/web/app/scripts/models/Account.js create mode 100644 pyload/web/app/scripts/models/ConfigHolder.js create mode 100644 pyload/web/app/scripts/models/ConfigItem.js create mode 100644 pyload/web/app/scripts/models/File.js create mode 100644 pyload/web/app/scripts/models/InteractionTask.js create mode 100644 pyload/web/app/scripts/models/Package.js create mode 100644 pyload/web/app/scripts/models/Progress.js create mode 100644 pyload/web/app/scripts/models/ServerStatus.js create mode 100644 pyload/web/app/scripts/models/TreeCollection.js create mode 100644 pyload/web/app/scripts/models/UserSession.js create mode 100644 pyload/web/app/scripts/router.js create mode 100644 pyload/web/app/scripts/routers/defaultRouter.js create mode 100644 pyload/web/app/scripts/routers/mobileRouter.js create mode 100644 pyload/web/app/scripts/utils/animations.js create mode 100644 pyload/web/app/scripts/utils/apitypes.js create mode 100644 pyload/web/app/scripts/utils/dialogs.js create mode 100644 pyload/web/app/scripts/utils/initHB.js create mode 100644 pyload/web/app/scripts/utils/lazyRequire.js create mode 100644 pyload/web/app/scripts/vendor/Handlebars-1.0rc1.js create mode 100755 pyload/web/app/scripts/vendor/bootstrap-2.3.2.js create mode 100644 pyload/web/app/scripts/vendor/jquery.omniwindow.js create mode 100644 pyload/web/app/scripts/vendor/remaining.js create mode 100644 pyload/web/app/scripts/views/abstract/itemView.js create mode 100644 pyload/web/app/scripts/views/abstract/modalView.js create mode 100644 pyload/web/app/scripts/views/accounts/accountListView.js create mode 100644 pyload/web/app/scripts/views/accounts/accountModal.js create mode 100644 pyload/web/app/scripts/views/accounts/accountView.js create mode 100644 pyload/web/app/scripts/views/dashboard/dashboardView.js create mode 100644 pyload/web/app/scripts/views/dashboard/fileView.js create mode 100644 pyload/web/app/scripts/views/dashboard/filterView.js create mode 100644 pyload/web/app/scripts/views/dashboard/packageView.js create mode 100644 pyload/web/app/scripts/views/dashboard/selectionView.js create mode 100644 pyload/web/app/scripts/views/headerView.js create mode 100644 pyload/web/app/scripts/views/input/inputLoader.js create mode 100644 pyload/web/app/scripts/views/input/inputView.js create mode 100644 pyload/web/app/scripts/views/input/textInput.js create mode 100644 pyload/web/app/scripts/views/linkGrabberModal.js create mode 100644 pyload/web/app/scripts/views/loginView.js create mode 100644 pyload/web/app/scripts/views/notificationView.js create mode 100644 pyload/web/app/scripts/views/progressView.js create mode 100644 pyload/web/app/scripts/views/queryModal.js create mode 100644 pyload/web/app/scripts/views/settings/configSectionView.js create mode 100644 pyload/web/app/scripts/views/settings/pluginChooserModal.js create mode 100644 pyload/web/app/scripts/views/settings/settingsView.js create mode 100644 pyload/web/app/styles/default/accounts.less create mode 100644 pyload/web/app/styles/default/admin.less create mode 100644 pyload/web/app/styles/default/base.less create mode 100644 pyload/web/app/styles/default/common.less create mode 100644 pyload/web/app/styles/default/dashboard.less create mode 100644 pyload/web/app/styles/default/main.less create mode 100644 pyload/web/app/styles/default/settings.less create mode 100644 pyload/web/app/styles/default/style.less create mode 100644 pyload/web/app/styles/font.css create mode 100644 pyload/web/app/templates/default/accounts/account.html create mode 100644 pyload/web/app/templates/default/accounts/actionbar.html create mode 100644 pyload/web/app/templates/default/accounts/layout.html create mode 100644 pyload/web/app/templates/default/admin.html create mode 100644 pyload/web/app/templates/default/dashboard/actionbar.html create mode 100644 pyload/web/app/templates/default/dashboard/file.html create mode 100644 pyload/web/app/templates/default/dashboard/layout.html create mode 100644 pyload/web/app/templates/default/dashboard/package.html create mode 100644 pyload/web/app/templates/default/dashboard/select.html create mode 100755 pyload/web/app/templates/default/dialogs/addAccount.html create mode 100755 pyload/web/app/templates/default/dialogs/addPluginConfig.html create mode 100644 pyload/web/app/templates/default/dialogs/confirmDelete.html create mode 100755 pyload/web/app/templates/default/dialogs/interactionTask.html create mode 100755 pyload/web/app/templates/default/dialogs/linkgrabber.html create mode 100755 pyload/web/app/templates/default/dialogs/modal.html create mode 100644 pyload/web/app/templates/default/header/layout.html create mode 100644 pyload/web/app/templates/default/header/progress.html create mode 100644 pyload/web/app/templates/default/header/progressbar.html create mode 100644 pyload/web/app/templates/default/header/status.html create mode 100644 pyload/web/app/templates/default/login.html create mode 100644 pyload/web/app/templates/default/notification.html create mode 100644 pyload/web/app/templates/default/settings/actionbar.html create mode 100644 pyload/web/app/templates/default/settings/config.html create mode 100644 pyload/web/app/templates/default/settings/configItem.html create mode 100644 pyload/web/app/templates/default/settings/layout.html create mode 100644 pyload/web/app/templates/default/settings/menu.html create mode 100644 pyload/web/app/templates/default/setup.html create mode 100644 pyload/web/app/unavailable.html create mode 100644 pyload/web/bower.json create mode 100644 pyload/web/cnl_app.py create mode 100644 pyload/web/middlewares.py create mode 100644 pyload/web/package.json create mode 100644 pyload/web/pyload_app.py create mode 100644 pyload/web/servers.py create mode 100644 pyload/web/setup_app.py create mode 100644 pyload/web/utils.py create mode 100644 pyload/web/webinterface.py diff --git a/.gitignore b/.gitignore index a5b06fed6..2384b1ff3 100644 --- a/.gitignore +++ b/.gitignore @@ -47,14 +47,15 @@ Downloads/* container/* Logs/* docs/module/ +docs/pyload/ docs/_build -module/plugins/container/DLC_*.py +pyload/plugins/container/DLC_*.py failed_links.txt links.txt ssl.crt ssl.key cert.pem -module/web/pyload.db +pyload/web/pyload.db *.prefs *.po *.orig diff --git a/docs/api/components.rst b/docs/api/components.rst index 116cfa90b..08560b535 100644 --- a/docs/api/components.rst +++ b/docs/api/components.rst @@ -9,12 +9,12 @@ of the available components listed by topic can be found below. .. autosummary:: :toctree: module - module.api.CoreApi.CoreApi - module.api.ConfigApi.ConfigApi - module.api.DownloadPreparingApi.DownloadPreparingApi - module.api.DownloadApi.DownloadApi - module.api.FileApi.FileApi - module.api.CollectorApi.CollectorApi - module.api.AccountApi.AccountApi - module.api.UserInteractionApi.UserInteractionApi - module.api.AddonApi.AddonApi \ No newline at end of file + pyload.api.CoreApi.CoreApi + pyload.api.ConfigApi.ConfigApi + pyload.api.DownloadPreparingApi.DownloadPreparingApi + pyload.api.DownloadApi.DownloadApi + pyload.api.FileApi.FileApi + pyload.api.CollectorApi.CollectorApi + pyload.api.AccountApi.AccountApi + pyload.api.UserInteractionApi.UserInteractionApi + pyload.api.AddonApi.AddonApi \ No newline at end of file diff --git a/docs/api/datatypes.rst b/docs/api/datatypes.rst index 5516189f3..cc07a5f28 100644 --- a/docs/api/datatypes.rst +++ b/docs/api/datatypes.rst @@ -4,12 +4,12 @@ Datatype Definition ******************* -Below you find a copy of :file:`module/remote/thriftbackend/pyload.thrift`, which is used to generate the data structs +Below you find a copy of :file:`pyload/remote/thriftbackend/pyload.thrift`, which is used to generate the data structs for various languages. It is also a good overview of avaible methods and return data. .. code-block:: c - .. [[[cog cog.out(open('module/remote/pyload.thrift', 'rb').read()) ]]] + .. [[[cog cog.out(open('pyload/remote/pyload.thrift', 'rb').read()) ]]] namespace java org.pyload.thrift typedef i32 FileID diff --git a/docs/api/json_api.rst b/docs/api/json_api.rst index 1f54e40aa..504de20bf 100644 --- a/docs/api/json_api.rst +++ b/docs/api/json_api.rst @@ -6,7 +6,7 @@ JSON API JSON [1]_ is a lightweight object notation and wrappers exists for nearly every programming language. Every modern browser is able to load JSON objects with JavaScript. Unlike other RPC methods you don't need to generate or precompile -any stub methods. The JSON :class:`Api ` is ready to be used in most languages and most JSON libraries are lightweight +any stub methods. The JSON :class:`Api ` is ready to be used in most languages and most JSON libraries are lightweight enough to build very small and performant scripts. Because of the builtin support, JSON is the first choice for all browser applications. @@ -28,7 +28,7 @@ so pyLoad can authenticate you. Calling Methods --------------- -In general you can use any method listed at the :class:`Api ` documentation, which is also available to +In general you can use any method listed at the :class:`Api ` documentation, which is also available to the thrift backend. Access works simply via ``http://pyload-core/api/methodName``, where ``pyload-core`` is the ip address @@ -43,7 +43,7 @@ Passing parameters To pass arguments you have two choices: Either use positional arguments, e.g.: ``http://pyload-core/api/getFileData/1``, where 1 is the FileID, or use keyword arguments supplied via GET or POST ``http://pyload-core/api/getFileData?fid=1``. You can find the argument names -in the :class:`Api ` documentation. +in the :class:`Api ` documentation. It is important that *all* arguments are in JSON format. So ``http://pyload-core/api/getFileData/1`` is valid because 1 represents an integer in json format. On the other hand if the method is expecting strings, this would be correct: diff --git a/docs/api/overview.rst b/docs/api/overview.rst index 99e7075ca..3b65a45b0 100644 --- a/docs/api/overview.rst +++ b/docs/api/overview.rst @@ -12,10 +12,10 @@ From Wikipedia, the free encyclopedia [1]_: .. rubric:: Motivation -The idea of the centralized pyLoad :class:`Api ` is to give uniform access to all integral parts +The idea of the centralized pyLoad :class:`Api ` is to give uniform access to all integral parts and plugins in pyLoad as well as other clients written in arbitrary programming languages. -Most of the :class:`Api ` functionality is exposed via HTTP or WebSocktes [2]_ as -simple JSON objects [3]_. In conclusion the :class:`Api ` is accessible via many programming languages, +Most of the :class:`Api ` functionality is exposed via HTTP or WebSocktes [2]_ as +simple JSON objects [3]_. In conclusion the :class:`Api ` is accessible via many programming languages, over network from remote machines and over browser with javascript. diff --git a/docs/conf.py b/docs/conf.py index 258dab745..8bd7c5f56 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,7 +16,7 @@ from os.path import dirname, join, abspath dir_name = join(dirname(abspath(""))) sys.path.append(dir_name) -sys.path.append(join(dir_name, "module", "lib")) +sys.path.append(join(dir_name, "pyload", "lib")) # If extensions (or modules to document with autodoc) are in another directory, @@ -129,12 +129,12 @@ html_theme = 'default' # The name of an image file (relative to this directory) to place at the top # of the sidebar. -html_logo = join(dir_name, "module", "web", "static", "img", "default", "logo.png") +html_logo = join(dir_name, "pyload", "web", "static", "img", "default", "logo.png") # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -html_favicon = join(dir_name, "module", "web", "static", "img", "favicon.ico") +html_favicon = join(dir_name, "pyload", "web", "static", "img", "favicon.ico") # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/docs/docs.conf b/docs/docs.conf index e197cfa43..d41ad8bc6 100644 --- a/docs/docs.conf +++ b/docs/docs.conf @@ -3,12 +3,12 @@ [epydoc] -modules: pyLoadCore.py, pyLoadCli.py, pyloadGui.py, module +modules: pyLoadCore.py, pyLoadCli.py, pyloadGui.py, pyload output: html target: docs docformat: restructuredtext -exclude: module\.lib|module\.remote\.thriftbackend\.thriftgen|PyQt4|\.pyc|\.pyo|module\.plugins\.(accounts|captcha|container|crypter|hooks|hoster) +exclude: pyload\.lib|pyload\.remote\.thriftbackend\.thriftgen|PyQt4|\.pyc|\.pyo|pyload\.plugins\.(accounts|captcha|container|crypter|hooks|hoster) name: pyLoad Documentation url: http://docs.pyload.org diff --git a/docs/index.rst b/docs/index.rst index 37fba3501..0b5885f0b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -37,7 +37,7 @@ Contents module_overview.rst -.. currentmodule:: module +.. currentmodule:: pyload .. rubric:: Footnotes diff --git a/docs/module_overview.rst b/docs/module_overview.rst index 56934e298..2876e465e 100644 --- a/docs/module_overview.rst +++ b/docs/module_overview.rst @@ -5,19 +5,19 @@ Module Overview A little selection of most important modules in pyLoad. .. autosummary:: - :toctree: module + :toctree: pyload - module.Api.Api - module.plugins.Base.Base - module.plugins.Hoster.Hoster - module.plugins.internal.SimpleHoster.SimpleHoster - module.plugins.Crypter.Crypter - module.plugins.internal.SimpleCrypter.SimpleCrypter - module.plugins.Addon.Addon - module.plugins.Account.Account - module.plugins.MultiHoster.MultiHoster - module.AddonManager.AddonManager - module.interaction.EventManager.EventManager - module.interaction.InteractionManager.InteractionManager - module.interaction.InteractionTask.InteractionTask - module.remote.ttypes + pyload.Api.Api + pyload.plugins.Base.Base + pyload.plugins.Hoster.Hoster + pyload.plugins.internal.SimpleHoster.SimpleHoster + pyload.plugins.Crypter.Crypter + pyload.plugins.internal.SimpleCrypter.SimpleCrypter + pyload.plugins.Addon.Addon + pyload.plugins.Account.Account + pyload.plugins.MultiHoster.MultiHoster + pyload.AddonManager.AddonManager + pyload.interaction.EventManager.EventManager + pyload.interaction.InteractionManager.InteractionManager + pyload.interaction.InteractionTask.InteractionTask + pyload.remote.ttypes diff --git a/docs/plugins/addon_plugin.rst b/docs/plugins/addon_plugin.rst index e18bd6b0f..bef5b115e 100644 --- a/docs/plugins/addon_plugin.rst +++ b/docs/plugins/addon_plugin.rst @@ -3,20 +3,20 @@ Addon - Add new functionality ============================= -A Hook is a python file which is located at :file:`module/plugins/hooks`. -The :class:`HookManager ` will load it automatically on startup. Only one instance exists -over the complete lifetime of pyload. Your hook can interact on various events called by the :class:`HookManager `, -do something completely autonomic and has full access to the :class:`Api ` and every detail of pyLoad. +A Hook is a python file which is located at :file:`pyload/plugins/hooks`. +The :class:`HookManager ` will load it automatically on startup. Only one instance exists +over the complete lifetime of pyload. Your hook can interact on various events called by the :class:`HookManager `, +do something completely autonomic and has full access to the :class:`Api ` and every detail of pyLoad. The UpdateManager, CaptchaTrader, UnRar and many more are implemented as hooks. Hook header ----------- -Your hook needs to subclass :class:`Hook ` and will inherit all of its methods, so make sure to check it's documentation! +Your hook needs to subclass :class:`Hook ` and will inherit all of its methods, so make sure to check it's documentation! All hooks should start with something like this: :: - from module.plugins.Hook import Hook + from pyload.plugins.Hook import Hook class YourHook(Hook): __name__ = "YourHook" @@ -43,16 +43,16 @@ Interacting on Events The next step is to think about where your Hook action takes place. -The easiest way is to overwrite specific methods defined by the :class:`Hook ` base class. +The easiest way is to overwrite specific methods defined by the :class:`Hook ` base class. The name is indicating when the function gets called. -See :class:`Hook ` page for a complete listing. +See :class:`Hook ` page for a complete listing. -You should be aware of the arguments the hooks are called with, whether its a :class:`PyFile ` -or :class:`PyPackage ` you should read the relevant documentation to know how to access it's great power and manipulate them. +You should be aware of the arguments the hooks are called with, whether its a :class:`PyFile ` +or :class:`PyPackage ` you should read the relevant documentation to know how to access it's great power and manipulate them. What a basic excerpt would look like: :: - from module.plugins.Hook import Hook + from pyload.plugins.Hook import Hook class YourHook(Hook): """ @@ -70,12 +70,12 @@ in a thread, in order to not block the main thread. This should be used for all Another and more flexible and powerful way is to use the event listener. All hook methods exists as event and very useful additional events are dispatched by the core. For a little overview look -at :class:`HookManager `. Keep in mind that you can define your own events and other people may listen on them. +at :class:`HookManager `. Keep in mind that you can define your own events and other people may listen on them. For your convenience it's possible to register listeners automatically via the ``event_map`` attribute. It requires a `dict` that maps event names to function names or a `list` of function names. It's important that all names are strings :: - from module.plugins.Hook import Hook + from pyload.plugins.Hook import Hook class YourHook(Hook): """ @@ -96,7 +96,7 @@ It requires a `dict` that maps event names to function names or a `list` of func An advantage of the event listener is that you are able to register and remove the listeners at runtime. Use `self.manager.addEvent("name", function)`, `self.manager.removeEvent("name", function)` and see doc for -:class:`HookManager `. Contrary to ``event_map``, ``function`` has to be a reference +:class:`HookManager `. Contrary to ``event_map``, ``function`` has to be a reference and **not** a `string`. We introduced events because it scales better if there is a huge amount of events and hooks. So all future interactions will be exclusively @@ -107,12 +107,12 @@ Providing RPC services ---------------------- -You may have noticed that pyLoad has an :class:`Api `, which can be used internal or called by clients via RPC. +You may have noticed that pyLoad has an :class:`Api `, which can be used internal or called by clients via RPC. So probably clients want to be able to interact with your hook to request it's state or invoke some action. Sounds complicated but is very easy to do. Just use the ``Expose`` decorator: :: - from module.plugins.Hook import Hook, Expose + from pyload.plugins.Hook import Hook, Expose class YourHook(Hook): """ @@ -123,7 +123,7 @@ Sounds complicated but is very easy to do. Just use the ``Expose`` decorator: :: def invoke(self, arg): print "Invoked with", arg -Thats all, it's available via the :class:`Api ` now. If you want to use it read :ref:`access_api`. +Thats all, it's available via the :class:`Api ` now. If you want to use it read :ref:`access_api`. Here is a basic example: :: #Assuming client is a ThriftClient or Api object @@ -133,11 +133,11 @@ Here is a basic example: :: Providing status information ---------------------------- -Your hook can store information in a ``dict`` that can easily be retrievied via the :class:`Api `. +Your hook can store information in a ``dict`` that can easily be retrievied via the :class:`Api `. Just store everything in ``self.info``. :: - from module.plugins.Hook import Hook + from pyload.plugins.Hook import Hook class YourHook(Hook): """ @@ -160,4 +160,4 @@ Example ------- Sorry but you won't find an example here ;-) - Look at :file:`module/plugins/hooks` and you will find plenty examples there. + Look at :file:`pyload/plugins/hooks` and you will find plenty examples there. diff --git a/docs/plugins/base_plugin.rst b/docs/plugins/base_plugin.rst index 3e1d5f952..5fa110fe7 100644 --- a/docs/plugins/base_plugin.rst +++ b/docs/plugins/base_plugin.rst @@ -3,8 +3,8 @@ Base Plugin - And here it begins... =================================== -A Plugin in pyLoad is a python file located at one of the subfolders in :file:`module/plugins/`. -All different plugin types inherit from :class:`Base `, which defines basic methods +A Plugin in pyLoad is a python file located at one of the subfolders in :file:`pyload/plugins/`. +All different plugin types inherit from :class:`Base `, which defines basic methods and meta data. You should read this section carefully, because it's the base for all plugin development. It is also a good idea to look at the class diagram [1]_ for all plugin types to get an overview. At last you should look at several already existing plugin to get a more detailed idea of how @@ -14,23 +14,23 @@ Meta Data --------- All important data which must be known by pyLoad is set using class attributes pre- and suffixed with ``__``. -An overview of acceptable values can be found in :class:`Base ` source code. +An overview of acceptable values can be found in :class:`Base ` source code. Unneeded attributes can be left out, except ``__version__``. Nevertheless please fill out most information as you can, when you want to submit your plugin to the public repository. -Additionally :class:`Crypter ` and :class:`Hoster ` +Additionally :class:`Crypter ` and :class:`Hoster ` needs to have a specific regexp [2]_ ``__pattern__``. This will be matched against input url's and if a suited plugin is found it is selected to handle the url. For localization pyLoad supports gettext [3]_, to mark strings for translation surround them with ``_("...")``. -You don't need to subclass :class:`Base ` directly, but the +You don't need to subclass :class:`Base ` directly, but the intermediate type according to your plugin. As an example we choose a hoster plugin, but the same is true for all plugin types. How a basic hoster plugin header could look like:: - from module.plugin.Hoster import Hoster + from pyload.plugin.Hoster import Hoster class MyFileHoster(Hoster): __version__ = "0.1" @@ -81,10 +81,10 @@ ip_check A hoster that checks ip, that can be avoided with reconnect Basic Methods ------------- -All methods can be looked up at :class:`Base `. To note some important ones: +All methods can be looked up at :class:`Base `. To note some important ones: The pyload core instance is accessible at ``self.core`` attribute -and the :class:`Api ` at ``self.core.api`` +and the :class:`Api ` at ``self.core.api`` With ``self.load(...)`` you can load any url and get the result. This method is only available to Hoster and Crypter. For other plugins use ``getURL(...)`` or ``getRequest()``. diff --git a/docs/plugins/crypter_plugin.rst b/docs/plugins/crypter_plugin.rst index 8c54dccb1..b10dd27f9 100644 --- a/docs/plugins/crypter_plugin.rst +++ b/docs/plugins/crypter_plugin.rst @@ -3,12 +3,12 @@ Crypter - Extract links from pages ================================== -We are starting with the simplest plugin, the :class:`Crypter `. +We are starting with the simplest plugin, the :class:`Crypter `. It's job is to take an url as input and generate a new package or links, for example by filtering the urls or loading a page and extracting links from the html code. You need to define the ``__pattern__`` to match -target urls and subclass from :class:`Crypter `. :: +target urls and subclass from :class:`Crypter `. :: - from module.plugin.Crypter import Crypter + from pyload.plugin.Crypter import Crypter class MyFileCrypter(Crypter): __pattern__ = r"mycrypter.com/id/([0-9]+)" @@ -28,7 +28,7 @@ Generating Packages When finished with decrypting just return the urls as list and they will be added to the package. You can also create new Packages if needed by instantiating a :class:`Package` instance, which will look like the following:: - from module.plugin.Crypter import Crypter, Package + from pyload.plugin.Crypter import Crypter, Package class MyFileCrypter(Crypter): @@ -52,12 +52,12 @@ your code you can use:: SimpleCrypter ------------- -For simple crypter services there is the :class:`SimpleCrypter ` class which handles most of the workflow. Only the regexp +For simple crypter services there is the :class:`SimpleCrypter ` class which handles most of the workflow. Only the regexp pattern has to be defined. Exmaple:: - from module.plugins.internal.SimpleCrypter import SimpleCrypter + from pyload.plugins.internal.SimpleCrypter import SimpleCrypter class MyFileCrypter(SimpleCrypter): diff --git a/docs/plugins/hoster_plugin.rst b/docs/plugins/hoster_plugin.rst index 9cd99a1f5..55a973463 100644 --- a/docs/plugins/hoster_plugin.rst +++ b/docs/plugins/hoster_plugin.rst @@ -7,7 +7,7 @@ We head to the next important section, the ``process`` method of your plugin. In fact the ``process`` method is the only functionality your plugin has to provide, but its always a good idea to split up tasks to not produce spaghetti code. An example ``process`` function could look like this :: - from module.plugin.Hoster import Hoster + from pyload.plugin.Hoster import Hoster class MyFileHoster(Hoster): """ @@ -27,7 +27,7 @@ An example ``process`` function could look like this :: # download the file, destination is determined by pyLoad self.download(parsed_url) -You need to know about the :class:`PyFile ` class, since an instance of it is given as a parameter to every pyfile. +You need to know about the :class:`PyFile ` class, since an instance of it is given as a parameter to every pyfile. Some tasks your plugin should handle: check if the file is online, get filename, wait if needed, download the file, etc.. Common Tasks @@ -37,7 +37,7 @@ Some hosters require you to wait a specific time. Just set the time with ``self. ``self.setWait(seconds, True)`` if you want pyLoad to perform a reconnect if needed. To handle captcha input just use ``self.decryptCaptcha(url, ...)``, it will be send to clients -or handled by :class:`Addon ` plugins +or handled by :class:`Addon ` plugins Online status fetching @@ -54,4 +54,4 @@ Testing Examples -------- -The best examples are the already existing plugins in :file:`module/plugins/`. \ No newline at end of file +The best examples are the already existing plugins in :file:`pyload/plugins/`. \ No newline at end of file diff --git a/module/AccountManager.py b/module/AccountManager.py deleted file mode 100644 index 5cdcfb57a..000000000 --- a/module/AccountManager.py +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN, mkaay -############################################################################### - -from threading import Lock -from random import choice - -from module.utils import lock, json - -class AccountManager: - """manages all accounts""" - - def __init__(self, core): - """Constructor""" - - self.core = core - self.lock = Lock() - - self.loadAccounts() - - def loadAccounts(self): - """loads all accounts available""" - - self.accounts = {} - - for plugin, loginname, activated, password, options in self.core.db.loadAccounts(): - # put into options as used in other context - options = json.loads(options) if options else {} - options["activated"] = activated - - self.createAccount(plugin, loginname, password, options) - - - def iterAccounts(self): - """ yields login, account for all accounts""" - for name, data in self.accounts.iteritems(): - for login, account in data.iteritems(): - yield login, account - - def saveAccounts(self): - """save all account information""" - # TODO: multi user - # TODO: activated - - data = [] - for name, plugin in self.accounts.iteritems(): - data.extend( - [(name, acc.loginname, 1 if acc.activated else 0, acc.password, json.dumps(acc.options)) for acc in - plugin.itervalues()]) - self.core.db.saveAccounts(data) - - def createAccount(self, plugin, loginname, password, options): - klass = self.core.pluginManager.loadClass("accounts", plugin) - if not klass: - self.core.log.warning(_("Unknown account plugin %s") % plugin) - return - - if plugin not in self.accounts: - self.accounts[plugin] = {} - - self.core.log.debug("Create account %s:%s" % (plugin, loginname)) - - self.accounts[plugin][loginname] = klass(self, loginname, password, options) - - - def getAccount(self, plugin, user): - return self.accounts[plugin].get(user, None) - - @lock - def updateAccount(self, plugin, user, password=None, options={}): - """add or update account""" - if plugin in self.accounts and user in self.accounts[plugin]: - acc = self.accounts[plugin][user] - updated = acc.update(password, options) - - self.saveAccounts() - if updated: acc.scheduleRefresh(force=True) - else: - self.createAccount(plugin, user, password, options) - self.saveAccounts() - - self.sendChange(plugin, user) - - @lock - def removeAccount(self, plugin, user): - """remove account""" - if plugin in self.accounts and user in self.accounts[plugin]: - del self.accounts[plugin][user] - self.core.db.removeAccount(plugin, user) - self.core.eventManager.dispatchEvent("account:deleted", plugin, user) - else: - self.core.log.debug("Remove non existent account %s %s" % (plugin, user)) - - - @lock - def getAccountForPlugin(self, plugin): - if plugin in self.accounts: - accs = [x for x in self.accounts[plugin].values() if x.isUsable()] - if accs: return choice(accs) - - return None - - @lock - def getAllAccounts(self, refresh=False): - """ Return account info, refresh afterwards if needed - - :param refresh: - :return: - """ - if refresh: - self.core.scheduler.addJob(0, self.core.accountManager.getAllAccounts) - - # load unavailable account info - for p_dict in self.accounts.itervalues(): - for acc in p_dict.itervalues(): - acc.getAccountInfo() - - return self.accounts - - def refreshAllAccounts(self): - """ Force a refresh of every account """ - for p in self.accounts.itervalues(): - for acc in p.itervalues(): - acc.getAccountInfo(True) - - def sendChange(self, plugin, name): - self.core.eventManager.dispatchEvent("account:updated", plugin, name) \ No newline at end of file diff --git a/module/AddonManager.py b/module/AddonManager.py deleted file mode 100644 index 9a8ad44ac..000000000 --- a/module/AddonManager.py +++ /dev/null @@ -1,248 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2013 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -import __builtin__ - -from thread import start_new_thread -from threading import RLock - -from types import MethodType - -from module.threads.AddonThread import AddonThread -from module.PluginManager import literal_eval -from utils import lock, to_string - -class AddonManager: - """ Manages addons, loading, unloading. """ - - def __init__(self, core): - self.core = core - self.config = self.core.config - - __builtin__.addonManager = self #needed to let addons register themselves - - self.log = self.core.log - # TODO: multiuser, addons can store the user itself, probably not needed here - self.plugins = {} - self.methods = {} # dict of names and list of methods usable by rpc - self.events = {} # Contains event that will be registered - - self.lock = RLock() - self.createIndex() - - # manage addons on config change - self.addEvent("config:changed", self.manageAddons) - - @lock - def callInHooks(self, event, eventName, *args): - """ Calls a method in all addons and catch / log errors""" - for plugin in self.plugins.itervalues(): - self.call(plugin, event, *args) - self.dispatchEvent(eventName, *args) - - def call(self, addon, f, *args): - try: - func = getattr(addon, f) - return func(*args) - except Exception, e: - addon.logError(_("Error when executing %s" % f), e) - self.core.print_exc() - - def addRPC(self, plugin, func, doc): - doc = doc.strip() if doc else "" - - if plugin in self.methods: - self.methods[plugin][func] = doc - else: - self.methods[plugin] = {func: doc} - - def callRPC(self, plugin, func, args): - if not args: args = [] - else: - args = literal_eval(args) - - plugin = self.plugins[plugin] - f = getattr(plugin, func) - return f(*args) - - @lock - def createIndex(self): - active = [] - deactive = [] - - for pluginname in self.core.pluginManager.getPlugins("addons"): - try: - # check first for builtin plugin - attrs = self.core.pluginManager.loadAttributes("addons", pluginname) - internal = attrs.get("internal", False) - - if internal or self.core.config.get(pluginname, "activated"): - pluginClass = self.core.pluginManager.loadClass("addons", pluginname) - - if not pluginClass: continue - - plugin = pluginClass(self.core, self) - self.plugins[pluginClass.__name__] = plugin - - # hide internals from printing - if not internal and plugin.isActivated(): - active.append(pluginClass.__name__) - else: - self.log.debug("Loaded internal plugin: %s" % pluginClass.__name__) - else: - deactive.append(pluginname) - - - except: - self.log.warning(_("Failed activating %(name)s") % {"name": pluginname}) - self.core.print_exc() - - self.log.info(_("Activated addons: %s") % ", ".join(sorted(active))) - self.log.info(_("Deactivate addons: %s") % ", ".join(sorted(deactive))) - - def manageAddons(self, plugin, name, value): - # TODO: user - - # check if section was a plugin - if plugin not in self.core.pluginManager.getPlugins("addons"): - return - - if name == "activated" and value: - self.activateAddon(plugin) - elif name == "activated" and not value: - self.deactivateAddon(plugin) - - @lock - def activateAddon(self, plugin): - #check if already loaded - if plugin in self.plugins: - return - - pluginClass = self.core.pluginManager.loadClass("addons", plugin) - - if not pluginClass: return - - self.log.debug("Plugin loaded: %s" % plugin) - - plugin = pluginClass(self.core, self) - self.plugins[pluginClass.__name__] = plugin - - # active the addon in new thread - start_new_thread(plugin.activate, tuple()) - self.registerEvents() # TODO: BUG: events will be destroyed and not re-registered - - @lock - def deactivateAddon(self, plugin): - if plugin not in self.plugins: - return - else: - addon = self.plugins[plugin] - - if addon.__internal__: return - - self.call(addon, "deactivate") - self.log.debug("Plugin deactivated: %s" % plugin) - - #remove periodic call - self.log.debug("Removed callback %s" % self.core.scheduler.removeJob(addon.cb)) - del self.plugins[addon.__name__] - - #remove event listener - for f in dir(addon): - if f.startswith("__") or type(getattr(addon, f)) != MethodType: - continue - self.core.eventManager.removeFromEvents(getattr(addon, f)) - - def activateAddons(self): - self.log.info(_("Activating Plugins...")) - for plugin in self.plugins.itervalues(): - if plugin.isActivated(): - self.call(plugin, "activate") - - self.registerEvents() - - def deactivateAddons(self): - """ Called when core is shutting down """ - self.log.info(_("Deactivating Plugins...")) - for plugin in self.plugins.itervalues(): - self.call(plugin, "deactivate") - - def downloadPreparing(self, pyfile): - self.callInHooks("downloadPreparing", "download:preparing", pyfile) - - def downloadFinished(self, pyfile): - self.callInHooks("downloadFinished", "download:finished", pyfile) - - def downloadFailed(self, pyfile): - self.callInHooks("downloadFailed", "download:failed", pyfile) - - def packageFinished(self, package): - self.callInHooks("packageFinished", "package:finished", package) - - def beforeReconnecting(self, ip): - self.callInHooks("beforeReconnecting", "reconnecting:before", ip) - - def afterReconnecting(self, ip): - self.callInHooks("afterReconnecting", "reconnecting:after", ip) - - @lock - def startThread(self, function, *args, **kwargs): - AddonThread(self.core.threadManager, function, args, kwargs) - - def activePlugins(self): - """ returns all active plugins """ - return [x for x in self.plugins.itervalues() if x.isActivated()] - - def getAllInfo(self): - """returns info stored by addon plugins""" - info = {} - for name, plugin in self.plugins.iteritems(): - if plugin.info: - #copy and convert so str - info[name] = dict( - [(x, to_string(y)) for x, y in plugin.info.iteritems()]) - return info - - def getInfo(self, plugin): - info = {} - if plugin in self.plugins and self.plugins[plugin].info: - info = dict([(x, to_string(y)) - for x, y in self.plugins[plugin].info.iteritems()]) - - return info - - def addEventListener(self, plugin, func, event): - """ add the event to the list """ - if plugin not in self.events: - self.events[plugin] = [] - self.events[plugin].append((func, event)) - - def registerEvents(self): - """ actually register all saved events """ - for name, plugin in self.plugins.iteritems(): - if name in self.events: - for func, event in self.events[name]: - self.addEvent(event, getattr(plugin, func)) - # clean up - del self.events[name] - - def addEvent(self, *args): - self.core.eventManager.addEvent(*args) - - def dispatchEvent(self, *args): - self.core.eventManager.dispatchEvent(*args) - diff --git a/module/Api.py b/module/Api.py deleted file mode 100644 index bfeeff10c..000000000 --- a/module/Api.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2013 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -import re -from types import MethodType - -from remote.apitypes import * - -# contains function names mapped to their permissions -# unlisted functions are for admins only -perm_map = {} - -# decorator only called on init, never initialized, so has no effect on runtime -def RequirePerm(bits): - class _Dec(object): - def __new__(cls, func, *args, **kwargs): - perm_map[func.__name__] = bits - return func - - return _Dec - -urlmatcher = re.compile(r"((https?|ftps?|xdcc|sftp):((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-=\\\.&]*)", re.IGNORECASE) - -stateMap = { - DownloadState.All: frozenset(getattr(DownloadStatus, x) for x in dir(DownloadStatus) if not x.startswith("_")), - DownloadState.Finished: frozenset((DownloadStatus.Finished, DownloadStatus.Skipped)), - DownloadState.Unfinished: None, # set below - DownloadState.Failed: frozenset((DownloadStatus.Failed, DownloadStatus.TempOffline, DownloadStatus.Aborted)), - DownloadState.Unmanaged: None, #TODO -} - -stateMap[DownloadState.Unfinished] = frozenset(stateMap[DownloadState.All].difference(stateMap[DownloadState.Finished])) - -def state_string(state): - return ",".join(str(x) for x in stateMap[state]) - -from datatypes.User import User - -class Api(Iface): - """ - **pyLoads API** - - This is accessible either internal via core.api, websocket backend or json api. - - see Thrift specification file remote/thriftbackend/pyload.thrift\ - for information about data structures and what methods are usable with rpc. - - Most methods requires specific permissions, please look at the source code if you need to know.\ - These can be configured via web interface. - Admin user have all permissions, and are the only ones who can access the methods with no specific permission. - """ - - EXTERNAL = Iface # let the json api know which methods are external - EXTEND = False # only extendable when set too true - - def __init__(self, core): - self.core = core - self.user_apis = {} - - @property - def user(self): - return None #TODO return default user? - - @property - def primaryUID(self): - return self.user.primary if self.user else None - - @classmethod - def initComponents(cls): - # Allow extending the api - # This prevents unintentionally registering of the components, - # but will only work once when they are imported - cls.EXTEND = True - # Import all Api modules, they register themselves. - import module.api - # they will vanish from the namespace afterwards - - - @classmethod - def extend(cls, api): - """Takes all params from api and extends cls with it. - api class can be removed afterwards - - :param api: Class with methods to extend - """ - if cls.EXTEND: - for name, func in api.__dict__.iteritems(): - if name.startswith("_"): continue - setattr(cls, name, MethodType(func, None, cls)) - - return cls.EXTEND - - def withUserContext(self, uid): - """ Returns a proxy version of the api, to call method in user context - - :param uid: user or userData instance or uid - :return: :class:`UserApi` - """ - if isinstance(uid, User): - uid = uid.uid - - if uid not in self.user_apis: - user = self.core.db.getUserData(uid=uid) - if not user: #TODO: anonymous user? - return None - - self.user_apis[uid] = UserApi(self.core, User.fromUserData(self, user)) - - return self.user_apis[uid] - - - ############################# - # Auth+User Information - ############################# - - # TODO - - @RequirePerm(Permission.All) - def login(self, username, password, remoteip=None): - """Login into pyLoad, this **must** be called when using rpc before any methods can be used. - - :param username: - :param password: - :param remoteip: Omit this argument, its only used internal - :return: bool indicating login was successful - """ - return True if self.checkAuth(username, password, remoteip) else False - - def checkAuth(self, username, password, remoteip=None): - """Check authentication and returns details - - :param username: - :param password: - :param remoteip: - :return: dict with info, empty when login is incorrect - """ - self.core.log.info(_("User '%s' tries to log in") % username) - - return self.core.db.checkAuth(username, password) - - def isAuthorized(self, func, user): - """checks if the user is authorized for specific method - - :param func: function name - :param user: `User` - :return: boolean - """ - if user.isAdmin(): - return True - elif func in perm_map and user.hasPermission(perm_map[func]): - return True - else: - return False - - # TODO - @RequirePerm(Permission.All) - def getUserData(self, username, password): - """similar to `checkAuth` but returns UserData thrift type """ - user = self.checkAuth(username, password) - if not user: - raise UserDoesNotExists(username) - - return user.toUserData() - - def getAllUserData(self): - """returns all known user and info""" - return self.core.db.getAllUserData() - - def changePassword(self, username, oldpw, newpw): - """ changes password for specific user """ - return self.core.db.changePassword(username, oldpw, newpw) - - def setUserPermission(self, user, permission, role): - self.core.db.setPermission(user, permission) - self.core.db.setRole(user, role) - - -class UserApi(Api): - """ Proxy object for api that provides all methods in user context """ - - def __init__(self, core, user): - # No need to init super class - self.core = core - self._user = user - - def withUserContext(self, uid): - raise Exception("Not allowed") - - @property - def user(self): - return self._user \ No newline at end of file diff --git a/module/FileManager.py b/module/FileManager.py deleted file mode 100644 index 7b14613f7..000000000 --- a/module/FileManager.py +++ /dev/null @@ -1,582 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -from time import time -from ReadWriteLock import ReadWriteLock - -from module.utils import lock, read_lock - -from Api import PackageStatus, DownloadStatus as DS, TreeCollection, PackageDoesNotExists -from datatypes.PyFile import PyFile -from datatypes.PyPackage import PyPackage, RootPackage - -# invalidates the cache -def invalidate(func): - def new(*args): - args[0].downloadstats = {} - args[0].queuestats = {} - args[0].jobCache = {} - return func(*args) - - return new - -# TODO: needs to be replaced later -OWNER = 0 - -class FileManager: - """Handles all request made to obtain information, - modify status or other request for links or packages""" - - ROOT_PACKAGE = -1 - - def __init__(self, core): - """Constructor""" - self.core = core - self.evm = core.eventManager - - # translations - self.statusMsg = [_("none"), _("offline"), _("online"), _("queued"), _("paused"), - _("finished"), _("skipped"), _("failed"), _("starting"), - _("waiting"), _("downloading"), _("temp. offline"), _("aborted"), - _("decrypting"), _("processing"), _("custom"), _("unknown")] - - self.files = {} # holds instances for files - self.packages = {} # same for packages - - self.jobCache = {} - - # locking the caches, db is already locked implicit - self.lock = ReadWriteLock() - #self.lock._Verbose__verbose = True - - self.downloadstats = {} # cached dl stats - self.queuestats = {} # cached queue stats - - self.db = self.core.db - - def save(self): - """saves all data to backend""" - self.db.commit() - - @read_lock - def syncSave(self): - """saves all data to backend and waits until all data are written""" - for pyfile in self.files.values(): - pyfile.sync() - - for pypack in self.packages.values(): - pypack.sync() - - self.db.syncSave() - - def cachedFiles(self): - return self.files.values() - - def cachedPackages(self): - return self.packages.values() - - def getCollector(self): - pass - - @invalidate - def addLinks(self, data, package): - """Add links, data = (plugin, url) tuple. Internal method should use API.""" - self.db.addLinks(data, package, OWNER) - self.evm.dispatchEvent("package:updated", package) - - - @invalidate - def addPackage(self, name, folder, root, password, site, comment, paused): - """Adds a package to database""" - pid = self.db.addPackage(name, folder, root, password, site, comment, - PackageStatus.Paused if paused else PackageStatus.Ok, OWNER) - p = self.db.getPackageInfo(pid) - - self.evm.dispatchEvent("package:inserted", pid, p.root, p.packageorder) - return pid - - - @lock - def getPackage(self, pid): - """return package instance""" - if pid == self.ROOT_PACKAGE: - return RootPackage(self, OWNER) - elif pid in self.packages: - pack = self.packages[pid] - pack.timestamp = time() - return pack - else: - info = self.db.getPackageInfo(pid, False) - if not info: return None - - pack = PyPackage.fromInfoData(self, info) - self.packages[pid] = pack - - return pack - - @read_lock - def getPackageInfo(self, pid): - """returns dict with package information""" - if pid == self.ROOT_PACKAGE: - pack = RootPackage(self, OWNER).toInfoData() - elif pid in self.packages: - pack = self.packages[pid].toInfoData() - pack.stats = self.db.getStatsForPackage(pid) - else: - pack = self.db.getPackageInfo(pid) - - if not pack: return None - - # todo: what does this todo mean?! - #todo: fill child packs and files - packs = self.db.getAllPackages(root=pid) - if pid in packs: del packs[pid] - pack.pids = packs.keys() - - files = self.db.getAllFiles(package=pid) - pack.fids = files.keys() - - return pack - - @lock - def getFile(self, fid): - """returns pyfile instance""" - if fid in self.files: - return self.files[fid] - else: - info = self.db.getFileInfo(fid) - if not info: return None - - f = PyFile.fromInfoData(self, info) - self.files[fid] = f - return f - - @read_lock - def getFileInfo(self, fid): - """returns dict with file information""" - if fid in self.files: - return self.files[fid].toInfoData() - - return self.db.getFileInfo(fid) - - @read_lock - def getTree(self, pid, full, state, search=None): - """ return a TreeCollection and fill the info data of containing packages. - optional filter only unfnished files - """ - view = TreeCollection(pid) - - # for depth=1, we don't need to retrieve all files/packages - root = pid if not full else None - - packs = self.db.getAllPackages(root) - files = self.db.getAllFiles(package=root, state=state, search=search) - - # updating from cache - for fid, f in self.files.iteritems(): - if fid in files: - files[fid] = f.toInfoData() - - # foreign pid, don't overwrite local pid ! - for fpid, p in self.packages.iteritems(): - if fpid in packs: - # copy the stats data - stats = packs[fpid].stats - packs[fpid] = p.toInfoData() - packs[fpid].stats = stats - - # root package is not in database, create an instance - if pid == self.ROOT_PACKAGE: - view.root = RootPackage(self, OWNER).toInfoData() - packs[self.ROOT_PACKAGE] = view.root - elif pid in packs: - view.root = packs[pid] - else: # package does not exists - return view - - # linear traversal over all data - for fpid, p in packs.iteritems(): - if p.fids is None: p.fids = [] - if p.pids is None: p.pids = [] - - root = packs.get(p.root, None) - if root: - if root.pids is None: root.pids = [] - root.pids.append(fpid) - - for fid, f in files.iteritems(): - p = packs.get(f.package, None) - if p: p.fids.append(fid) - - - # cutting of tree is not good in runtime, only saves bandwidth - # need to remove some entries - if full and pid > -1: - keep = [] - queue = [pid] - while queue: - fpid = queue.pop() - keep.append(fpid) - queue.extend(packs[fpid].pids) - - # now remove unneeded data - for fpid in packs.keys(): - if fpid not in keep: - del packs[fpid] - - for fid, f in files.items(): - if f.package not in keep: - del files[fid] - - #remove root - del packs[pid] - view.files = files - view.packages = packs - - return view - - - @lock - def getJob(self, occ): - """get suitable job""" - - #TODO only accessed by one thread, should not need a lock - #TODO needs to be approved for new database - #TODO clean mess - #TODO improve selection of valid jobs - - if occ in self.jobCache: - if self.jobCache[occ]: - id = self.jobCache[occ].pop() - if id == "empty": - pyfile = None - self.jobCache[occ].append("empty") - else: - pyfile = self.getFile(id) - else: - jobs = self.db.getJob(occ) - jobs.reverse() - if not jobs: - self.jobCache[occ].append("empty") - pyfile = None - else: - self.jobCache[occ].extend(jobs) - pyfile = self.getFile(self.jobCache[occ].pop()) - - else: - self.jobCache = {} #better not caching to much - jobs = self.db.getJob(occ) - jobs.reverse() - self.jobCache[occ] = jobs - - if not jobs: - self.jobCache[occ].append("empty") - pyfile = None - else: - pyfile = self.getFile(self.jobCache[occ].pop()) - - - return pyfile - - def getDownloadStats(self, user=None): - """ return number of downloads """ - if user not in self.downloadstats: - self.downloadstats[user] = self.db.downloadstats(user) - - return self.downloadstats[user] - - def getQueueStats(self, user=None, force=False): - """number of files that have to be processed, failed files will not be included""" - if user not in self.queuestats or force: - self.queuestats[user] = self.db.queuestats(user) - - return self.queuestats[user] - - def scanDownloadFolder(self): - pass - - @lock - @invalidate - def deletePackage(self, pid): - """delete package and all contained links""" - - p = self.getPackage(pid) - if not p: return - - oldorder = p.packageorder - root = p.root - - for pyfile in self.cachedFiles(): - if pyfile.packageid == pid: - pyfile.abortDownload() - - # TODO: delete child packages - # TODO: delete folder - - self.db.deletePackage(pid) - self.releasePackage(pid) - - for pack in self.cachedPackages(): - if pack.root == root and pack.packageorder > oldorder: - pack.packageorder -= 1 - - self.evm.dispatchEvent("package:deleted", pid) - - @lock - @invalidate - def deleteFile(self, fid): - """deletes links""" - - f = self.getFile(fid) - if not f: return - - pid = f.packageid - order = f.fileorder - - if fid in self.core.threadManager.processingIds(): - f.abortDownload() - - # TODO: delete real file - - self.db.deleteFile(fid, f.fileorder, f.packageid) - self.releaseFile(fid) - - for pyfile in self.files.itervalues(): - if pyfile.packageid == pid and pyfile.fileorder > order: - pyfile.fileorder -= 1 - - self.evm.dispatchEvent("file:deleted", fid, pid) - - @lock - def releaseFile(self, fid): - """removes pyfile from cache""" - if fid in self.files: - del self.files[fid] - - @lock - def releasePackage(self, pid): - """removes package from cache""" - if pid in self.packages: - del self.packages[pid] - - def updateFile(self, pyfile): - """updates file""" - self.db.updateFile(pyfile) - - # This event is thrown with pyfile or only fid - self.evm.dispatchEvent("file:updated", pyfile) - - def updatePackage(self, pypack): - """updates a package""" - self.db.updatePackage(pypack) - self.evm.dispatchEvent("package:updated", pypack.pid) - - @invalidate - def updateFileInfo(self, data, pid): - """ updates file info (name, size, status,[ hash,] url)""" - self.db.updateLinkInfo(data) - self.evm.dispatchEvent("package:updated", pid) - - def checkAllLinksFinished(self): - """checks if all files are finished and dispatch event""" - - # TODO: user context? - if not self.db.queuestats()[0]: - self.core.addonManager.dispatchEvent("download:allFinished") - self.core.log.debug("All downloads finished") - return True - - return False - - def checkAllLinksProcessed(self, fid=-1): - """checks if all files was processed and pyload would idle now, needs fid which will be ignored when counting""" - - # reset count so statistic will update (this is called when dl was processed) - self.resetCount() - - # TODO: user context? - if not self.db.processcount(fid): - self.core.addonManager.dispatchEvent("download:allProcessed") - self.core.log.debug("All downloads processed") - return True - - return False - - def checkPackageFinished(self, pyfile): - """ checks if package is finished and calls addonmanager """ - - ids = self.db.getUnfinished(pyfile.packageid) - if not ids or (pyfile.id in ids and len(ids) == 1): - if not pyfile.package().setFinished: - self.core.log.info(_("Package finished: %s") % pyfile.package().name) - self.core.addonManager.packageFinished(pyfile.package()) - pyfile.package().setFinished = True - - def resetCount(self): - self.queuecount = -1 - - @read_lock - @invalidate - def restartPackage(self, pid): - """restart package""" - for pyfile in self.cachedFiles(): - if pyfile.packageid == pid: - self.restartFile(pyfile.id) - - self.db.restartPackage(pid) - - if pid in self.packages: - self.packages[pid].setFinished = False - - self.evm.dispatchEvent("package:updated", pid) - - @read_lock - @invalidate - def restartFile(self, fid): - """ restart file""" - if fid in self.files: - f = self.files[fid] - f.status = DS.Queued - f.name = f.url - f.error = "" - f.abortDownload() - - self.db.restartFile(fid) - self.evm.dispatchEvent("file:updated", fid) - - - @lock - @invalidate - def orderPackage(self, pid, position): - - p = self.getPackageInfo(pid) - self.db.orderPackage(pid, p.root, p.packageorder, position) - - for pack in self.packages.itervalues(): - if pack.root != p.root or pack.packageorder < 0: continue - if pack.pid == pid: - pack.packageorder = position - if p.packageorder > position: - if position <= pack.packageorder < p.packageorder: - pack.packageorder += 1 - elif p.order < position: - if position >= pack.packageorder > p.packageorder: - pack.packageorder -= 1 - - self.db.commit() - - self.evm.dispatchEvent("package:reordered", pid, position, p.root) - - @lock - @invalidate - def orderFiles(self, fids, pid, position): - - files = [self.getFileInfo(fid) for fid in fids] - orders = [f.fileorder for f in files] - if min(orders) + len(files) != max(orders) + 1: - raise Exception("Tried to reorder non continous block of files") - - # minimum fileorder - f = reduce(lambda x,y: x if x.fileorder < y.fileorder else y, files) - order = f.fileorder - - self.db.orderFiles(pid, fids, order, position) - diff = len(fids) - - if f.fileorder > position: - for pyfile in self.files.itervalues(): - if pyfile.packageid != f.package or pyfile.fileorder < 0: continue - if position <= pyfile.fileorder < f.fileorder: - pyfile.fileorder += diff - - for i, fid in enumerate(fids): - if fid in self.files: - self.files[fid].fileorder = position + i - - elif f.fileorder < position: - for pyfile in self.files.itervalues(): - if pyfile.packageid != f.package or pyfile.fileorder < 0: continue - if position >= pyfile.fileorder >= f.fileorder+diff: - pyfile.fileorder -= diff - - for i, fid in enumerate(fids): - if fid in self.files: - self.files[fid].fileorder = position -diff + i + 1 - - self.db.commit() - - self.evm.dispatchEvent("file:reordered", pid) - - @read_lock - @invalidate - def movePackage(self, pid, root): - """ move pid - root """ - - p = self.getPackageInfo(pid) - dest = self.getPackageInfo(root) - if not p: raise PackageDoesNotExists(pid) - if not dest: raise PackageDoesNotExists(root) - - # cantor won't be happy if we put the package in itself - if pid == root or p.root == root: return False - - # TODO move real folders - - # we assume pack is not in use anyway, so we can release it - self.releasePackage(pid) - self.db.movePackage(p.root, p.packageorder, pid, root) - - return True - - @read_lock - @invalidate - def moveFiles(self, fids, pid): - """ move all fids to pid """ - - f = self.getFileInfo(fids[0]) - if not f or f.package == pid: - return False - if not self.getPackageInfo(pid): - raise PackageDoesNotExists(pid) - - # TODO move real files - - self.db.moveFiles(f.package, fids, pid) - - return True - - - @invalidate - def reCheckPackage(self, pid): - """ recheck links in package """ - data = self.db.getPackageData(pid) - - urls = [] - - for pyfile in data.itervalues(): - if pyfile.status not in (DS.NA, DS.Finished, DS.Skipped): - urls.append((pyfile.url, pyfile.pluginname)) - - self.core.threadManager.createInfoThread(urls, pid) - - - @invalidate - def restartFailed(self): - """ restart all failed links """ - # failed should not be in cache anymore, so working on db is sufficient - self.db.restartFailed() diff --git a/module/InitHomeDir.py b/module/InitHomeDir.py deleted file mode 100644 index ccf525576..000000000 --- a/module/InitHomeDir.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN - - This modules inits working directories and global variables, pydir and homedir -""" - -from os import makedirs, path, chdir -from os.path import join -import sys -from sys import argv, platform - -import __builtin__ - -__builtin__.owd = path.abspath("") #original working directory -__builtin__.pypath = path.abspath(path.join(__file__, "..", "..")) - -sys.path.append(join(pypath, "module", "lib")) - -homedir = "" - -if platform == 'nt': - homedir = path.expanduser("~") - if homedir == "~": - import ctypes - - CSIDL_APPDATA = 26 - _SHGetFolderPath = ctypes.windll.shell32.SHGetFolderPathW - _SHGetFolderPath.argtypes = [ctypes.wintypes.HWND, - ctypes.c_int, - ctypes.wintypes.HANDLE, - ctypes.wintypes.DWORD, ctypes.wintypes.LPCWSTR] - - path_buf = ctypes.wintypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH) - result = _SHGetFolderPath(0, CSIDL_APPDATA, 0, 0, path_buf) - homedir = path_buf.value -else: - homedir = path.expanduser("~") - -__builtin__.homedir = homedir - -configdir = None -args = " ".join(argv) -# dirty method to set configdir from commandline arguments -if "--configdir=" in args: - for arg in argv: - if arg.startswith("--configdir="): - configdir = arg.replace('--configdir=', '').strip() - -elif "nosetests" in args: - print "Running in test mode" - configdir = join(pypath, "tests", "config") - -elif path.exists(path.join(pypath, "module", "config", "configdir")): - f = open(path.join(pypath, "module", "config", "configdir"), "rb") - c = f.read().strip() - f.close() - configdir = path.join(pypath, c) - -# default config dir -if not configdir: - if platform in ("posix", "linux2", "darwin"): - configdir = path.join(homedir, ".pyload") - else: - configdir = path.join(homedir, "pyload") - -if not path.exists(configdir): - makedirs(configdir, 0700) - -__builtin__.configdir = configdir -chdir(configdir) - -#print "Using %s as working directory." % configdir diff --git a/module/PluginManager.py b/module/PluginManager.py deleted file mode 100644 index bb1c76bf3..000000000 --- a/module/PluginManager.py +++ /dev/null @@ -1,407 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2013 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN, mkaay -############################################################################### - -import re -import sys - -from os import listdir, makedirs -from os.path import isfile, join, exists, abspath, basename -from sys import version_info -from time import time - -from module.lib.SafeEval import const_eval as literal_eval -from module.plugins.Base import Base - -from new_collections import namedtuple - -#TODO: ignores not updatable - -# ignore these plugin configs, mainly because plugins were wiped out -IGNORE = ( - "FreakshareNet", "SpeedManager", "ArchiveTo", "ShareCx", ('addons', 'UnRar'), - 'EasyShareCom', 'FlyshareCz' - ) - -PluginTuple = namedtuple("PluginTuple", "version re deps category user path") - -class PluginManager: - ROOT = "module.plugins." - LOCALROOT = "localplugins." - TYPES = ("crypter", "hoster", "accounts", "addons", "internal") - - BUILTIN = re.compile(r'__(?P[a-z0-9_]+)__\s*=\s?(True|False|None|[0-9x.]+)', re.I) - SINGLE = re.compile(r'__(?P[a-z0-9_]+)__\s*=\s*(?:r|u|_)?((?:(?[a-z0-9_]+)__\s*=\s*((?:\{|\[|"{3}).*?(?:"""|\}|\]))', re.DOTALL | re.M | re.I) - - NO_MATCH = re.compile(r'^no_match$') - - def __init__(self, core): - self.core = core - - #self.config = self.core.config - self.log = core.log - - self.plugins = {} - self.modules = {} # cached modules - self.history = [] # match history to speedup parsing (type, name) - self.user_context = {} # plugins working with user context - self.createIndex() - - #register for import addon - sys.meta_path.append(self) - - def logDebug(self, type, plugin, msg): - self.log.debug("Plugin %s | %s: %s" % (type, plugin, msg)) - - def createIndex(self): - """create information for all plugins available""" - # add to path, so we can import from userplugins - sys.path.append(abspath("")) - - if not exists("userplugins"): - makedirs("userplugins") - if not exists(join("userplugins", "__init__.py")): - f = open(join("userplugins", "__init__.py"), "wb") - f.close() - - a = time() - for type in self.TYPES: - self.plugins[type] = self.parse(type) - - self.log.debug("Created index of plugins in %.2f ms", (time() - a) * 1000) - - def parse(self, folder, home=None): - """ Analyze and parses all plugins in folder """ - plugins = {} - if home: - pfolder = join("userplugins", folder) - if not exists(pfolder): - makedirs(pfolder) - if not exists(join(pfolder, "__init__.py")): - f = open(join(pfolder, "__init__.py"), "wb") - f.close() - - else: - pfolder = join(pypath, "module", "plugins", folder) - - for f in listdir(pfolder): - if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( - "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): - if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): - continue - elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): - continue - elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): - continue - - # replace suffix and version tag - name = f[:-3] - if name[-1] == ".": name = name[:-4] - - plugin = self.parsePlugin(join(pfolder, f), folder, name, home) - if plugin: - plugins[name] = plugin - - if not home: - temp = self.parse(folder, plugins) - plugins.update(temp) - - return plugins - - def parseAttributes(self, filename, name, folder=""): - """ Parse attribute dict from plugin""" - data = open(filename, "rb") - content = data.read() - data.close() - - attrs = {} - for m in self.BUILTIN.findall(content) + self.SINGLE.findall(content) + self.MULTI.findall(content): - #replace gettext function and eval result - try: - attrs[m[0]] = literal_eval(m[-1].replace("_(", "(")) - except: - self.logDebug(folder, name, "Error when parsing: %s" % m[-1]) - self.core.print_exc() - - if not hasattr(Base, "__%s__" % m[0]): - if m[0] != "type": #TODO remove type from all plugins, its not needed - self.logDebug(folder, name, "Unknown attribute '%s'" % m[0]) - - return attrs - - def parsePlugin(self, filename, folder, name, home=None): - """ Parses a plugin from disk, folder means plugin type in this context. Also sets config. - - :arg home: dict with plugins, of which the found one will be matched against (according version) - :returns PluginTuple""" - - attrs = self.parseAttributes(filename, name, folder) - if not attrs: return - - version = 0 - - if "version" in attrs: - try: - version = float(attrs["version"]) - except ValueError: - self.logDebug(folder, name, "Invalid version %s" % attrs["version"]) - version = 9 #TODO remove when plugins are fixed, causing update loops - else: - self.logDebug(folder, name, "No version attribute") - - # home contains plugins from pyload root - if home and name in home: - if home[name].version >= version: - return - - if name in IGNORE or (folder, name) in IGNORE: - return - - if "pattern" in attrs and attrs["pattern"]: - try: - plugin_re = re.compile(attrs["pattern"], re.I) - except: - self.logDebug(folder, name, "Invalid regexp pattern '%s'" % attrs["pattern"]) - plugin_re = self.NO_MATCH - else: plugin_re = self.NO_MATCH - - deps = attrs.get("dependencies", None) - category = attrs.get("category", None) if folder == "addons" else None - - # create plugin tuple - plugin = PluginTuple(version, plugin_re, deps, category, bool(home), filename) - - # internals have no config - if folder == "internal": - return plugin - - if folder == "addons" and "config" not in attrs and not attrs.get("internal", False): - attrs["config"] = (["activated", "bool", "Activated", False],) - - if "config" in attrs and attrs["config"]: - config = attrs["config"] - desc = attrs.get("description", "") - long_desc = attrs.get("long_description", "") - - if type(config[0]) == tuple: - config = [list(x) for x in config] - else: - config = [list(config)] - - if folder == "addons" and not attrs.get("internal", False): - for item in config: - if item[0] == "activated": break - else: # activated flag missing - config.insert(0, ("activated", "bool", "Activated", False)) - - # Everything that is no addon and user_context=True, is added to dict - if folder != "addons" or attrs.get("user_context", False): - self.user_context[name] = True - - try: - self.core.config.addConfigSection(name, name, desc, long_desc, config) - except: - self.logDebug(folder, name, "Invalid config %s" % config) - - return plugin - - - def parseUrls(self, urls): - """parse plugins for given list of urls, separate to crypter and hoster""" - - res = {"hoster": [], "crypter": []} # tupels of (url, plugin) - - for url in urls: - if type(url) not in (str, unicode, buffer): - self.log.debug("Parsing invalid type %s" % type(url)) - continue - - found = False - - for ptype, name in self.history: - if self.plugins[ptype][name].re.match(url): - res[ptype].append((url, name)) - found = (ptype, name) - break # need to exit this loop first - - if found: # found match - if self.history[0] != found: #update history - self.history.remove(found) - self.history.insert(0, found) - continue - - for ptype in ("crypter", "hoster"): - for name, plugin in self.plugins[ptype].iteritems(): - if plugin.re.match(url): - res[ptype].append((url, name)) - self.history.insert(0, (ptype, name)) - del self.history[10:] # cut down to size of 10 - found = True - break - - if not found: - res["hoster"].append((url, "BasePlugin")) - - return res["hoster"], res["crypter"] - - def getPlugins(self, type): - return self.plugins.get(type, None) - - def findPlugin(self, name, pluginlist=("hoster", "crypter")): - for ptype in pluginlist: - if name in self.plugins[ptype]: - return ptype, self.plugins[ptype][name] - return None, None - - def getPluginModule(self, name): - """ Decprecated: return plugin module from hoster|crypter""" - self.log.debug("Deprecated method: .getPluginModule()") - type, plugin = self.findPlugin(name) - return self.loadModule(type, name) - - def getPluginClass(self, name): - """ return plugin class from hoster|crypter, always the not overwritten one """ - type, plugin = self.findPlugin(name) - return self.loadClass(type, name) - - # MultiHoster will overwrite this - getPlugin = getPluginClass - - def loadAttributes(self, type, name): - plugin = self.plugins[type][name] - return self.parseAttributes(plugin.path, name, type) - - def loadModule(self, type, name): - """ Returns loaded module for plugin - - :param type: plugin type, subfolder of module.plugins - :param name: - """ - plugins = self.plugins[type] - if name in plugins: - if (type, name) in self.modules: return self.modules[(type, name)] - try: - # convert path to python recognizable import - path = basename(plugins[name].path).replace(".pyc", "").replace(".py", "") - module = __import__(self.ROOT + "%s.%s" % (type, path), globals(), locals(), path) - self.modules[(type, name)] = module # cache import, maybe unneeded - return module - except Exception, e: - self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) - self.core.print_exc() - - def loadClass(self, type, name): - """Returns the class of a plugin with the same name""" - module = self.loadModule(type, name) - if module: return getattr(module, name) - - def find_module(self, fullname, path=None): - #redirecting imports if necesarry - if fullname.startswith(self.ROOT) or fullname.startswith(self.LOCALROOT): #separate pyload plugins - if fullname.startswith(self.LOCALROOT): user = 1 - else: user = 0 #used as bool and int - - split = fullname.split(".") - if len(split) != 4 - user: return - type, name = split[2 - user:4 - user] - - if type in self.plugins and name in self.plugins[type]: - #userplugin is a newer version - if not user and self.plugins[type][name].user: - return self - #imported from userdir, but pyloads is newer - if user and not self.plugins[type][name].user: - return self - - def load_module(self, name, replace=True): - if name not in sys.modules: #could be already in modules - if replace: - if self.ROOT in name: - newname = name.replace(self.ROOT, self.LOCALROOT) - else: - newname = name.replace(self.LOCALROOT, self.ROOT) - else: newname = name - - base, plugin = newname.rsplit(".", 1) - - self.log.debug("Redirected import %s -> %s" % (name, newname)) - - module = __import__(newname, globals(), locals(), [plugin]) - #inject under new an old name - sys.modules[name] = module - sys.modules[newname] = module - - return sys.modules[name] - - def reloadPlugins(self, type_plugins): - """ reloads and reindexes plugins """ - if not type_plugins: return False - - self.log.debug("Request reload of plugins: %s" % type_plugins) - - as_dict = {} - for t, n in type_plugins: - if t in as_dict: - as_dict[t].append(n) - else: - as_dict[t] = [n] - - # we do not reload addons or internals, would cause to much side effects - if "addons" in as_dict or "internal" in as_dict: - return False - - for type in as_dict.iterkeys(): - for plugin in as_dict[type]: - if plugin in self.plugins[type]: - if (type, plugin) in self.modules: - self.log.debug("Reloading %s" % plugin) - reload(self.modules[(type, plugin)]) - - # index re-creation - for type in ("crypter", "container", "hoster", "captcha", "accounts"): - self.plugins[type] = self.parse(type) - - if "accounts" in as_dict: #accounts needs to be reloaded - self.core.accountManager.initPlugins() - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - - return True - - def isUserPlugin(self, plugin): - """ A plugin suitable for multiple user """ - return plugin in self.user_context - - def isPluginType(self, plugin, type): - return plugin in self.plugins[type] - - def getCategory(self, plugin): - if plugin in self.plugins["addons"]: - return self.plugins["addons"][plugin].category or "addon" - - def loadIcon(self, name): - """ load icon for single plugin, base64 encoded""" - pass - - def checkDependencies(self, type, name): - """ Check deps for given plugin - - :return: List of unfullfilled dependencies - """ - pass - diff --git a/module/Scheduler.py b/module/Scheduler.py deleted file mode 100644 index 0bc396b69..000000000 --- a/module/Scheduler.py +++ /dev/null @@ -1,141 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" - -from time import time -from heapq import heappop, heappush -from thread import start_new_thread -from threading import Lock - -class AlreadyCalled(Exception): - pass - - -class Deferred(): - def __init__(self): - self.call = [] - self.result = () - - def addCallback(self, f, *cargs, **ckwargs): - self.call.append((f, cargs, ckwargs)) - - def callback(self, *args, **kwargs): - if self.result: - raise AlreadyCalled - self.result = (args, kwargs) - for f, cargs, ckwargs in self.call: - args += tuple(cargs) - kwargs.update(ckwargs) - f(*args ** kwargs) - - -class Scheduler(): - def __init__(self, core): - self.core = core - - self.queue = PriorityQueue() - - def addJob(self, t, call, args=[], kwargs={}, threaded=True): - d = Deferred() - t += time() - j = Job(t, call, args, kwargs, d, threaded) - self.queue.put((t, j)) - return d - - - def removeJob(self, d): - """ - :param d: defered object - :return: if job was deleted - """ - index = -1 - - for i, j in enumerate(self.queue): - if j[1].deferred == d: - index = i - - if index >= 0: - del self.queue[index] - return True - - return False - - def work(self): - while True: - t, j = self.queue.get() - if not j: - break - else: - if t <= time(): - j.start() - else: - self.queue.put((t, j)) - break - - -class Job(): - def __init__(self, time, call, args=[], kwargs={}, deferred=None, threaded=True): - self.time = float(time) - self.call = call - self.args = args - self.kwargs = kwargs - self.deferred = deferred - self.threaded = threaded - - def run(self): - ret = self.call(*self.args, **self.kwargs) - if self.deferred is None: - return - else: - self.deferred.callback(ret) - - def start(self): - if self.threaded: - start_new_thread(self.run, ()) - else: - self.run() - - -class PriorityQueue(): - """ a non blocking priority queue """ - - def __init__(self): - self.queue = [] - self.lock = Lock() - - def __iter__(self): - return iter(self.queue) - - def __delitem__(self, key): - del self.queue[key] - - def put(self, element): - self.lock.acquire() - heappush(self.queue, element) - self.lock.release() - - def get(self): - """ return element or None """ - self.lock.acquire() - try: - el = heappop(self.queue) - return el - except IndexError: - return None, None - finally: - self.lock.release() \ No newline at end of file diff --git a/module/Setup.py b/module/Setup.py deleted file mode 100644 index eef7a88ca..000000000 --- a/module/Setup.py +++ /dev/null @@ -1,418 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -import module.utils.pylgettext as gettext -import os -import sys -import socket -import webbrowser - -from getpass import getpass -from time import time -from sys import exit - -from module.utils.fs import abspath, dirname, exists, join, makedirs -from module.utils import get_console_encoding -from module.web.ServerThread import WebServer - - -class Setup(): - """ - pyLoads initial setup configuration assistant - """ - - def __init__(self, path, config): - self.path = path - self.config = config - self.stdin_encoding = get_console_encoding(sys.stdin.encoding) - self.lang = None - # We will create a timestamp so that the setup will be completed in a specific interval - self.timestamp = time() - - # TODO: probably unneeded - self.yes = "yes" - self.no = "no" - - - def start(self): - web = WebServer(pysetup=self) - web.start() - - error = web.check_error() - if error: #todo errno 44 port already in use - print error - - url = "http://%s:%d/" % (socket.gethostbyname(socket.gethostname()), web.port) - - print "Setup is started" - - opened = webbrowser.open_new_tab(url) - if not opened: - print "Please point your browser to %s" % url - - - self.ask_lang() - - print "" - print _("Would you like to configure pyLoad via Webinterface?") - print _("You need a Browser and a connection to this PC for it.") - print _("Url would be: http://hostname:8000/") - viaweb = self.ask(_("Start initial webinterface for configuration?"), self.yes, bool=True) - if viaweb: - self.start_web() - else: - self.start_cli() - - - - def start_cli(self): - - - print _("Welcome to the pyLoad Configuration Assistent.") - print _("It will check your system and make a basic setup in order to run pyLoad.") - print "" - print _("The value in brackets [] always is the default value,") - print _("in case you don't want to change it or you are unsure what to choose, just hit enter.") - print _( - "Don't forget: You can always rerun this assistent with --setup or -s parameter, when you start pyLoadCore.") - print _("If you have any problems with this assistent hit STRG-C,") - print _("to abort and don't let him start with pyLoadCore automatically anymore.") - print "" - print _("When you are ready for system check, hit enter.") - raw_input() - - #self.get_page_next() - - - if len(avail) < 5: - print _("Features missing: ") - print - - if not self.check_module("Crypto"): - print _("no py-crypto available") - print _("You need this if you want to decrypt container files.") - print "" - - if not ssl: - print _("no SSL available") - print _("This is needed if you want to establish a secure connection to core or webinterface.") - print _("If you only want to access locally to pyLoad ssl is not useful.") - print "" - - if not captcha: - print _("no Captcha Recognition available") - print _("Only needed for some hosters and as freeuser.") - print "" - - if not js: - print _("no JavaScript engine found") - print _("You will need this for some Click'N'Load links. Install Spidermonkey, ossp-js, pyv8 or rhino") - - print _("You can abort the setup now and fix some dependencies if you want.") - - con = self.ask(_("Continue with setup?"), self.yes, bool=True) - - if not con: - return False - - print "" - print _("Do you want to change the config path? Current is %s") % abspath("") - print _( - "If you use pyLoad on a server or the home partition lives on an internal flash it may be a good idea to change it.") - path = self.ask(_("Change config path?"), self.no, bool=True) - if path: - self.conf_path() - #calls exit when changed - - print "" - print _("Do you want to configure login data and basic settings?") - print _("This is recommend for first run.") - con = self.ask(_("Make basic setup?"), self.yes, bool=True) - - if con: - self.conf_basic() - - if ssl: - print "" - print _("Do you want to configure ssl?") - ssl = self.ask(_("Configure ssl?"), self.no, bool=True) - if ssl: - self.conf_ssl() - - if web: - print "" - print _("Do you want to configure webinterface?") - web = self.ask(_("Configure webinterface?"), self.yes, bool=True) - if web: - self.conf_web() - - print "" - print _("Setup finished successfully.") - print _("Hit enter to exit and restart pyLoad") - raw_input() - return True - - - def start_web(self): - print "" - print _("Webinterface running for setup.") - # TODO start browser? - try: - from module.web import ServerThread - ServerThread.setup = self - from module.web import webinterface - webinterface.run_simple() - self.web = True - return True - except Exception, e: - print "Webinterface failed with this error: ", e - print "Falling back to commandline setup." - self.start_cli() - - - def conf_basic(self): - print "" - print _("## Basic Setup ##") - - print "" - print _("The following logindata is valid for CLI, GUI and webinterface.") - - from module.database import DatabaseBackend - - db = DatabaseBackend(None) - db.setup() - username = self.ask(_("Username"), "User") - password = self.ask("", "", password=True) - db.addUser(username, password) - db.shutdown() - - print "" - print _("External clients (GUI, CLI or other) need remote access to work over the network.") - print _("However, if you only want to use the webinterface you may disable it to save ram.") - self.config["remote"]["activated"] = self.ask(_("Enable remote access"), self.yes, bool=True) - - print "" - langs = self.config.getMetaData("general", "language") - self.config["general"]["language"] = self.ask(_("Language"), "en", langs.type.split(";")) - - self.config["general"]["download_folder"] = self.ask(_("Downloadfolder"), "Downloads") - self.config["download"]["max_downloads"] = self.ask(_("Max parallel downloads"), "3") - #print _("You should disable checksum proofing, if you have low hardware requirements.") - #self.config["general"]["checksum"] = self.ask(_("Proof checksum?"), "y", bool=True) - - reconnect = self.ask(_("Use Reconnect?"), self.no, bool=True) - self.config["reconnect"]["activated"] = reconnect - if reconnect: - self.config["reconnect"]["method"] = self.ask(_("Reconnect script location"), "./reconnect.sh") - - - def conf_web(self): - print "" - print _("## Webinterface Setup ##") - - print "" - self.config["webinterface"]["activated"] = self.ask(_("Activate webinterface?"), self.yes, bool=True) - print "" - print _("Listen address, if you use 127.0.0.1 or localhost, the webinterface will only accessible locally.") - self.config["webinterface"]["host"] = self.ask(_("Address"), "0.0.0.0") - self.config["webinterface"]["port"] = self.ask(_("Port"), "8000") - print "" - print _("pyLoad offers several server backends, now following a short explanation.") - print "threaded:", _("Default server, this server offers SSL and is a good alternative to builtin.") - print "fastcgi:", _( - "Can be used by apache, lighttpd, requires you to configure them, which is not too easy job.") - print "lightweight:", _("Very fast alternative written in C, requires libev and linux knowledge.") - print "\t", _("Get it from here: https://github.com/jonashaag/bjoern, compile it") - print "\t", _("and copy bjoern.so to module/lib") - - print - print _( - "Attention: In some rare cases the builtin server is not working, if you notice problems with the webinterface") - print _("come back here and change the builtin server to the threaded one here.") - - self.config["webinterface"]["server"] = self.ask(_("Server"), "threaded", - ["builtin", "threaded", "fastcgi", "lightweight"]) - - def conf_ssl(self): - print "" - print _("## SSL Setup ##") - print "" - print _("Execute these commands from pyLoad config folder to make ssl certificates:") - print "" - print "openssl genrsa -out ssl.key 1024" - print "openssl req -new -key ssl.key -out ssl.csr" - print "openssl req -days 36500 -x509 -key ssl.key -in ssl.csr > ssl.crt " - print "" - print _("If you're done and everything went fine, you can activate ssl now.") - self.config["ssl"]["activated"] = self.ask(_("Activate SSL?"), self.yes, bool=True) - - def set_user(self): - gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) - translation = gettext.translation("setup", join(self.path, "locale"), - languages=[self.config["general"]["language"], "en"], fallback=True) - translation.install(True) - - from module.database import DatabaseBackend - - db = DatabaseBackend(None) - db.setup() - - noaction = True - try: - while True: - print _("Select action") - print _("1 - Create/Edit user") - print _("2 - List users") - print _("3 - Remove user") - print _("4 - Quit") - action = raw_input("[1]/2/3/4: ") - if not action in ("1", "2", "3", "4"): - continue - elif action == "1": - print "" - username = self.ask(_("Username"), "User") - password = self.ask("", "", password=True) - db.addUser(username, password) - noaction = False - elif action == "2": - print "" - print _("Users") - print "-----" - users = db.getAllUserData() - noaction = False - for user in users.itervalues(): - print user.name - print "-----" - print "" - elif action == "3": - print "" - username = self.ask(_("Username"), "") - if username: - db.removeUser(username) - noaction = False - elif action == "4": - db.syncSave() - break - finally: - if not noaction: - db.shutdown() - - def conf_path(self, trans=False): - if trans: - gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) - translation = gettext.translation("setup", join(self.path, "locale"), - languages=[self.config["general"]["language"], "en"], fallback=True) - translation.install(True) - - print _("Setting new configpath, current configuration will not be transferred!") - path = self.ask(_("Configpath"), abspath("")) - try: - path = join(pypath, path) - if not exists(path): - makedirs(path) - f = open(join(pypath, "module", "config", "configdir"), "wb") - f.write(path) - f.close() - print _("Configpath changed, setup will now close, please restart to go on.") - print _("Press Enter to exit.") - raw_input() - exit() - except Exception, e: - print _("Setting config path failed: %s") % str(e) - - - def ask_lang(self): - langs = self.config.getMetaData("general", "language").type.split(";") - self.lang = self.ask(u"Choose your Language / Wähle deine Sprache", "en", langs) - gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) - translation = gettext.translation("setup", join(self.path, "locale"), languages=[self.lang, "en"], fallback=True) - translation.install(True) - - #l10n Input shorthand for yes - self.yes = _("y") - #l10n Input shorthand for no - self.no = _("n") - - def ask(self, qst, default, answers=[], bool=False, password=False): - """ Generate dialog on command line """ - - if answers: - info = "(" - for i, answer in enumerate(answers): - info += (", " if i != 0 else "") + str((answer == default and "[%s]" % answer) or answer) - - info += ")" - elif bool: - if default == self.yes: - info = "([%s]/%s)" % (self.yes, self.no) - else: - info = "(%s/[%s])" % (self.yes, self.no) - else: - info = "[%s]" % default - - if password: - p1 = True - p2 = False - while p1 != p2: - # getpass(_("Password: ")) will crash on systems with broken locales (Win, NAS) - sys.stdout.write(_("Password: ")) - p1 = getpass("") - - if len(p1) < 4: - print _("Password too short. Use at least 4 symbols.") - continue - - sys.stdout.write(_("Password (again): ")) - p2 = getpass("") - - if p1 == p2: - return p1 - else: - print _("Passwords did not match.") - - while True: - input = raw_input(qst + " %s: " % info) - input = input.decode(self.stdin_encoding) - - if input.strip() == "": - input = default - - if bool: - #l10n yes, true,t are inputs for booleans with value true - if input.lower().strip() in [self.yes, _("yes"), _("true"), _("t"), "yes"]: - return True - #l10n no, false,f are inputs for booleans with value false - elif input.lower().strip() in [self.no, _("no"), _("false"), _("f"), "no"]: - return False - else: - print _("Invalid Input") - continue - - if not answers: - return input - - else: - if input in answers: - return input - else: - print _("Invalid Input") - - -if __name__ == "__main__": - test = Setup(join(abspath(dirname(__file__)), ".."), None) - test.start() diff --git a/module/__init__.py b/module/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/module/api/AccountApi.py b/module/api/AccountApi.py deleted file mode 100644 index 981842b5c..000000000 --- a/module/api/AccountApi.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.Api import Api, RequirePerm, Permission - -from ApiComponent import ApiComponent - - -class AccountApi(ApiComponent): - """ All methods to control accounts """ - - @RequirePerm(Permission.Accounts) - def getAccounts(self, refresh): - """Get information about all entered accounts. - - :param refresh: reload account info - :return: list of `AccountInfo` - """ - accs = self.core.accountManager.getAllAccounts(refresh) - accounts = [] - for plugin in accs.itervalues(): - accounts.extend([acc.toInfoData() for acc in plugin.values()]) - - return accounts - - @RequirePerm(Permission.All) - def getAccountTypes(self): - """All available account types. - - :return: string list - """ - return self.core.pluginManager.getPlugins("accounts").keys() - - @RequirePerm(Permission.Accounts) - def updateAccount(self, plugin, login, password): - """Changes pw/options for specific account.""" - # TODO: options - self.core.accountManager.updateAccount(plugin, login, password, {}) - - def updateAccountInfo(self, account): - """ Update account from :class:`AccountInfo` """ - #TODO - - @RequirePerm(Permission.Accounts) - def removeAccount(self, account): - """Remove account from pyload. - - :param account: :class:`ÀccountInfo` instance - """ - self.core.accountManager.removeAccount(account.plugin, account.loginname) - - -if Api.extend(AccountApi): - del AccountApi \ No newline at end of file diff --git a/module/api/AddonApi.py b/module/api/AddonApi.py deleted file mode 100644 index 917c7dc4c..000000000 --- a/module/api/AddonApi.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.Api import Api, RequirePerm, Permission - -from ApiComponent import ApiComponent - -class AddonApi(ApiComponent): - """ Methods to interact with addons """ - - def getAllInfo(self): - """Returns all information stored by addon plugins. Values are always strings - - :return: {"plugin": {"name": value } } - """ - return self.core.addonManager.getAllInfo() - - def getInfoByPlugin(self, plugin): - """Returns information stored by a specific plugin. - - :param plugin: pluginname - :return: dict of attr names mapped to value {"name": value} - """ - return self.core.addonManager.getInfo(plugin) - -if Api.extend(AddonApi): - del AddonApi \ No newline at end of file diff --git a/module/api/ApiComponent.py b/module/api/ApiComponent.py deleted file mode 100644 index 3948086c2..000000000 --- a/module/api/ApiComponent.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.remote.apitypes import Iface - -# Workaround to let code-completion think, this is subclass of Iface -Iface = object -class ApiComponent(Iface): - - __slots__ = [] - - def __init__(self, core, user): - # Only for auto completion, this class can not be instantiated - from pyload import Core - from module.datatypes.User import User - assert isinstance(core, Core) - assert issubclass(ApiComponent, Iface) - self.core = core - assert isinstance(user, User) - self.user = user - self.primaryUID = 0 - # No instantiating! - raise Exception() \ No newline at end of file diff --git a/module/api/CollectorApi.py b/module/api/CollectorApi.py deleted file mode 100644 index eb36f7a21..000000000 --- a/module/api/CollectorApi.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.Api import Api, RequirePerm, Permission - -from ApiComponent import ApiComponent - -class CollectorApi(ApiComponent): - """ Link collector """ - - @RequirePerm(Permission.All) - def getCollector(self): - pass - - @RequirePerm(Permission.Add) - def addToCollector(self, links): - pass - - @RequirePerm(Permission.Add) - def addFromCollector(self, name, new_name): - pass - - @RequirePerm(Permission.Delete) - def deleteCollPack(self, name): - pass - - @RequirePerm(Permission.Add) - def renameCollPack(self, name, new_name): - pass - - @RequirePerm(Permission.Delete) - def deleteCollLink(self, url): - pass - - -if Api.extend(CollectorApi): - del CollectorApi \ No newline at end of file diff --git a/module/api/ConfigApi.py b/module/api/ConfigApi.py deleted file mode 100644 index 527f494ce..000000000 --- a/module/api/ConfigApi.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.Api import Api, RequirePerm, Permission, ConfigHolder, ConfigItem, ConfigInfo -from module.utils import to_string - -from ApiComponent import ApiComponent - -# helper function to create a ConfigHolder -def toConfigHolder(section, config, values): - holder = ConfigHolder(section, config.name, config.description, config.long_desc) - holder.items = [ConfigItem(option, x.name, x.description, x.type, to_string(x.default), - to_string(values.get(option, x.default))) for option, x in - config.config.iteritems()] - return holder - - -class ConfigApi(ApiComponent): - """ Everything related to configuration """ - - def getConfigValue(self, section, option): - """Retrieve config value. - - :param section: name of category, or plugin - :param option: config option - :rtype: str - :return: config value as string - """ - value = self.core.config.get(section, option, self.primaryUID) - return to_string(value) - - def setConfigValue(self, section, option, value): - """Set new config value. - - :param section: - :param option: - :param value: new config value - """ - if option in ("limit_speed", "max_speed"): #not so nice to update the limit - self.core.requestFactory.updateBucket() - - self.core.config.set(section, option, value, self.primaryUID) - - def getConfig(self): - """Retrieves complete config of core. - - :rtype: dict of section -> ConfigHolder - """ - data = {} - for section, config, values in self.core.config.iterCoreSections(): - data[section] = toConfigHolder(section, config, values) - return data - - def getCoreConfig(self): - """ Retrieves core config sections - - :rtype: list of PluginInfo - """ - return [ConfigInfo(section, config.name, config.description, False, False) - for section, config, values in self.core.config.iterCoreSections()] - - @RequirePerm(Permission.Plugins) - def getPluginConfig(self): - """All plugins and addons the current user has configured - - :rtype: list of PluginInfo - """ - # TODO: include addons that are activated by default - # TODO: multi user - # TODO: better plugin / addon activated config - data = [] - active = [x.getName() for x in self.core.addonManager.activePlugins()] - for name, config, values in self.core.config.iterSections(self.primaryUID): - # skip unmodified and inactive addons - if not values and name not in active: continue - - item = ConfigInfo(name, config.name, config.description, - self.core.pluginManager.getCategory(name), - self.core.pluginManager.isUserPlugin(name), - values.get("activated", None if "activated" not in config.config else config.config[ - "activated"].default)) - data.append(item) - - return data - - @RequirePerm(Permission.Plugins) - def getAvailablePlugins(self): - """List of all available plugins, that are configurable - - :rtype: list of PluginInfo - """ - # TODO: filter user_context / addons when not allowed - plugins = [ConfigInfo(name, config.name, config.description, - self.core.pluginManager.getCategory(name), - self.core.pluginManager.isUserPlugin(name)) - for name, config, values in self.core.config.iterSections(self.primaryUID)] - - return plugins - - @RequirePerm(Permission.Plugins) - def loadConfig(self, name): - """Get complete config options for desired section - - :param name: Name of plugin or config section - :rtype: ConfigHolder - """ - # requires at least plugin permissions, but only admin can load core config - config, values = self.core.config.getSection(name, self.primaryUID) - return toConfigHolder(name, config, values) - - - @RequirePerm(Permission.Plugins) - def saveConfig(self, config): - """Used to save a configuration, core config can only be saved by admins - - :param config: :class:`ConfigHolder` - """ - for item in config.items: - self.core.config.set(config.name, item.name, item.value, sync=False, user=self.primaryUID) - # save the changes - self.core.config.saveValues(self.primaryUID, config.name) - - @RequirePerm(Permission.Plugins) - def deleteConfig(self, plugin): - """Deletes modified config - - :param plugin: plugin name - """ - #TODO: delete should deactivate addons? - self.core.config.delete(plugin, self.primaryUID) - - -if Api.extend(ConfigApi): - del ConfigApi \ No newline at end of file diff --git a/module/api/CoreApi.py b/module/api/CoreApi.py deleted file mode 100644 index e5c5e8b41..000000000 --- a/module/api/CoreApi.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.Api import Api, RequirePerm, Permission, ServerStatus, Interaction -from module.utils.fs import join, free_space -from module.utils import compare_time - -from ApiComponent import ApiComponent - -class CoreApi(ApiComponent): - """ This module provides methods for general interaction with the core, like status or progress retrieval """ - - @RequirePerm(Permission.All) - def getServerVersion(self): - """pyLoad Core version """ - return self.core.version - - @RequirePerm(Permission.All) - def getWSAddress(self): - """Gets and address for the websocket based on configuration""" - # TODO SSL (wss) - return "ws://%%s:%d" % self.core.config['remote']['port'] - - @RequirePerm(Permission.All) - def getServerStatus(self): - """Some general information about the current status of pyLoad. - - :return: `ServerStatus` - """ - queue = self.core.files.getQueueStats(self.primaryUID) - total = self.core.files.getDownloadStats(self.primaryUID) - - serverStatus = ServerStatus(0, - total[0], queue[0], - total[1], queue[1], - self.isInteractionWaiting(Interaction.All), - not self.core.threadManager.pause and self.isTimeDownload(), - self.core.threadManager.pause, - self.core.config['reconnect']['activated'] and self.isTimeReconnect()) - - - for pyfile in self.core.threadManager.getActiveDownloads(self.primaryUID): - serverStatus.speed += pyfile.getSpeed() #bytes/s - - return serverStatus - - @RequirePerm(Permission.All) - def getProgressInfo(self): - """ Status of all currently running tasks - - :rtype: list of :class:`ProgressInfo` - """ - return self.core.threadManager.getProgressList(self.primaryUID) - - def pauseServer(self): - """Pause server: It won't start any new downloads, but nothing gets aborted.""" - self.core.threadManager.pause = True - - def unpauseServer(self): - """Unpause server: New Downloads will be started.""" - self.core.threadManager.pause = False - - def togglePause(self): - """Toggle pause state. - - :return: new pause state - """ - self.core.threadManager.pause ^= True - return self.core.threadManager.pause - - def toggleReconnect(self): - """Toggle reconnect activation. - - :return: new reconnect state - """ - self.core.config["reconnect"]["activated"] ^= True - return self.core.config["reconnect"]["activated"] - - def freeSpace(self): - """Available free space at download directory in bytes""" - return free_space(self.core.config["general"]["download_folder"]) - - - def quit(self): - """Clean way to quit pyLoad""" - self.core.do_kill = True - - def restart(self): - """Restart pyload core""" - self.core.do_restart = True - - def getLog(self, offset=0): - """Returns most recent log entries. - - :param offset: line offset - :return: List of log entries - """ - filename = join(self.core.config['log']['log_folder'], 'log.txt') - try: - fh = open(filename, "r") - lines = fh.readlines() - fh.close() - if offset >= len(lines): - return [] - return lines[offset:] - except: - return ['No log available'] - - @RequirePerm(Permission.All) - def isTimeDownload(self): - """Checks if pyload will start new downloads according to time in config. - - :return: bool - """ - start = self.core.config['downloadTime']['start'].split(":") - end = self.core.config['downloadTime']['end'].split(":") - return compare_time(start, end) - - @RequirePerm(Permission.All) - def isTimeReconnect(self): - """Checks if pyload will try to make a reconnect - - :return: bool - """ - start = self.core.config['reconnect']['startTime'].split(":") - end = self.core.config['reconnect']['endTime'].split(":") - return compare_time(start, end) and self.core.config["reconnect"]["activated"] - - -if Api.extend(CoreApi): - del CoreApi \ No newline at end of file diff --git a/module/api/DownloadApi.py b/module/api/DownloadApi.py deleted file mode 100644 index ba49435b3..000000000 --- a/module/api/DownloadApi.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from os.path import isabs - -from module.Api import Api, RequirePerm, Permission -from module.utils.fs import join - -from ApiComponent import ApiComponent - -class DownloadApi(ApiComponent): - """ Component to create, add, delete or modify downloads.""" - - @RequirePerm(Permission.Add) - def generateAndAddPackages(self, links, paused=False): - """Generates and add packages - - :param links: list of urls - :param paused: paused package - :return: list of package ids - """ - return [self.addPackageP(name, urls, "", paused) for name, urls - in self.generatePackages(links).iteritems()] - - @RequirePerm(Permission.Add) - def createPackage(self, name, folder, root, password="", site="", comment="", paused=False): - """Create a new package. - - :param name: display name of the package - :param folder: folder name or relative path, abs path are not allowed - :param root: package id of root package, -1 for top level package - :param password: single pw or list of passwords separated with new line - :param site: arbitrary url to site for more information - :param comment: arbitrary comment - :param paused: No downloads will be started when True - :return: pid of newly created package - """ - - if isabs(folder): - folder = folder.replace("/", "_") - - folder = folder.replace("http://", "").replace(":", "").replace("\\", "_").replace("..", "") - - self.core.log.info(_("Added package %(name)s as folder %(folder)s") % {"name": name, "folder": folder}) - pid = self.core.files.addPackage(name, folder, root, password, site, comment, paused) - - return pid - - - @RequirePerm(Permission.Add) - def addPackage(self, name, links, password=""): - """Convenient method to add a package to the top-level and for adding links. - - :return: package id - """ - return self.addPackageChild(name, links, password, -1, False) - - @RequirePerm(Permission.Add) - def addPackageP(self, name, links, password, paused): - """ Same as above with additional paused attribute. """ - return self.addPackageChild(name, links, password, -1, paused) - - @RequirePerm(Permission.Add) - def addPackageChild(self, name, links, password, root, paused): - """Adds a package, with links to desired package. - - :param root: parents package id - :return: package id of the new package - """ - if self.core.config['general']['folder_per_package']: - folder = name - else: - folder = "" - - pid = self.createPackage(name, folder, root, password) - self.addLinks(pid, links) - - return pid - - @RequirePerm(Permission.Add) - def addLinks(self, pid, links): - """Adds links to specific package. Initiates online status fetching. - - :param pid: package id - :param links: list of urls - """ - hoster, crypter = self.core.pluginManager.parseUrls(links) - - if hoster: - self.core.files.addLinks(hoster, pid) - self.core.threadManager.createInfoThread(hoster, pid) - - self.core.threadManager.createDecryptThread(crypter, pid) - - self.core.log.info((_("Added %d links to package") + " #%d" % pid) % len(hoster)) - self.core.files.save() - - @RequirePerm(Permission.Add) - def uploadContainer(self, filename, data): - """Uploads and adds a container file to pyLoad. - - :param filename: filename, extension is important so it can correctly decrypted - :param data: file content - """ - th = open(join(self.core.config["general"]["download_folder"], "tmp_" + filename), "wb") - th.write(str(data)) - th.close() - - return self.addPackage(th.name, [th.name]) - - @RequirePerm(Permission.Delete) - def deleteFiles(self, fids): - """Deletes several file entries from pyload. - - :param fids: list of file ids - """ - for fid in fids: - self.core.files.deleteFile(fid) - - self.core.files.save() - - @RequirePerm(Permission.Delete) - def deletePackages(self, pids): - """Deletes packages and containing links. - - :param pids: list of package ids - """ - for pid in pids: - self.core.files.deletePackage(pid) - - self.core.files.save() - - - @RequirePerm(Permission.Modify) - def restartPackage(self, pid): - """Restarts a package, resets every containing files. - - :param pid: package id - """ - self.core.files.restartPackage(pid) - - @RequirePerm(Permission.Modify) - def restartFile(self, fid): - """Resets file status, so it will be downloaded again. - - :param fid: file id - """ - self.core.files.restartFile(fid) - - @RequirePerm(Permission.Modify) - def recheckPackage(self, pid): - """Check online status of all files in a package, also a default action when package is added. """ - self.core.files.reCheckPackage(pid) - - @RequirePerm(Permission.Modify) - def restartFailed(self): - """Restarts all failed failes.""" - self.core.files.restartFailed() - - @RequirePerm(Permission.Modify) - def stopAllDownloads(self): - """Aborts all running downloads.""" - - pyfiles = self.core.files.cachedFiles() - for pyfile in pyfiles: - pyfile.abortDownload() - - @RequirePerm(Permission.Modify) - def stopDownloads(self, fids): - """Aborts specific downloads. - - :param fids: list of file ids - :return: - """ - pyfiles = self.core.files.cachedFiles() - for pyfile in pyfiles: - if pyfile.id in fids: - pyfile.abortDownload() - - -if Api.extend(DownloadApi): - del DownloadApi \ No newline at end of file diff --git a/module/api/DownloadPreparingApi.py b/module/api/DownloadPreparingApi.py deleted file mode 100644 index edd5d362c..000000000 --- a/module/api/DownloadPreparingApi.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from itertools import chain - -from module.Api import Api, RequirePerm, Permission, OnlineCheck, LinkStatus, urlmatcher -from module.utils.fs import join -from module.utils.packagetools import parseNames -from module.network.RequestFactory import getURL - -from ApiComponent import ApiComponent - -class DownloadPreparingApi(ApiComponent): - """ All kind of methods to parse links or retrieve online status """ - - @RequirePerm(Permission.Add) - def parseURLs(self, html=None, url=None): - """Parses html content or any arbitrary text for links and returns result of `checkURLs` - - :param html: html source - :return: - """ - urls = [] - - if html: - urls += [x[0] for x in urlmatcher.findall(html)] - - if url: - page = getURL(url) - urls += [x[0] for x in urlmatcher.findall(page)] - - # remove duplicates - return self.checkURLs(set(urls)) - - - @RequirePerm(Permission.Add) - def checkURLs(self, urls): - """ Gets urls and returns pluginname mapped to list of matching urls. - - :param urls: - :return: {plugin: urls} - """ - data, crypter = self.core.pluginManager.parseUrls(urls) - plugins = {} - - for url, plugin in chain(data, crypter): - if plugin in plugins: - plugins[plugin].append(url) - else: - plugins[plugin] = [url] - - return plugins - - @RequirePerm(Permission.Add) - def checkOnlineStatus(self, urls): - """ initiates online status check, will also decrypt files. - - :param urls: - :return: initial set of data as :class:`OnlineCheck` instance containing the result id - """ - data, crypter = self.core.pluginManager.parseUrls(urls) - - # initial result does not contain the crypter links - tmp = [(url, (url, LinkStatus(url, pluginname, "unknown", 3, 0))) for url, pluginname in data] - data = parseNames(tmp) - result = {} - - for k, v in data.iteritems(): - for url, status in v: - status.packagename = k - result[url] = status - - data.update(crypter) # hoster and crypter will be processed - rid = self.core.threadManager.createResultThread(data, False) - - return OnlineCheck(rid, result) - - @RequirePerm(Permission.Add) - def checkOnlineStatusContainer(self, urls, container, data): - """ checks online status of urls and a submitted container file - - :param urls: list of urls - :param container: container file name - :param data: file content - :return: :class:`OnlineCheck` - """ - th = open(join(self.core.config["general"]["download_folder"], "tmp_" + container), "wb") - th.write(str(data)) - th.close() - urls.append(th.name) - return self.checkOnlineStatus(urls) - - @RequirePerm(Permission.Add) - def pollResults(self, rid): - """ Polls the result available for ResultID - - :param rid: `ResultID` - :return: `OnlineCheck`, if rid is -1 then there is no more data available - """ - result = self.core.threadManager.getInfoResult(rid) - - if "ALL_INFO_FETCHED" in result: - del result["ALL_INFO_FETCHED"] - return OnlineCheck(-1, result) - else: - return OnlineCheck(rid, result) - - - @RequirePerm(Permission.Add) - def generatePackages(self, links): - """ Parses links, generates packages names from urls - - :param links: list of urls - :return: package names mapped to urls - """ - result = parseNames((x, x) for x in links) - return result - - -if Api.extend(DownloadPreparingApi): - del DownloadPreparingApi \ No newline at end of file diff --git a/module/api/FileApi.py b/module/api/FileApi.py deleted file mode 100644 index 8a55d9dfd..000000000 --- a/module/api/FileApi.py +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.Api import Api, RequirePerm, Permission, DownloadState, PackageDoesNotExists, FileDoesNotExists -from module.utils import uniqify - -from ApiComponent import ApiComponent - -# TODO: user context -class FileApi(ApiComponent): - """Everything related to available packages or files. Deleting, Modifying and so on.""" - - @RequirePerm(Permission.All) - def getAllFiles(self): - """ same as `getFileTree` for toplevel root and full tree""" - return self.getFileTree(-1, True) - - @RequirePerm(Permission.All) - def getFilteredFiles(self, state): - """ same as `getFilteredFileTree` for toplevel root and full tree""" - return self.getFilteredFileTree(-1, state, True) - - @RequirePerm(Permission.All) - def getFileTree(self, pid, full): - """ Retrieve data for specific package. full=True will retrieve all data available - and can result in greater delays. - - :param pid: package id - :param full: go down the complete tree or only the first layer - :return: :class:`TreeCollection` - """ - return self.core.files.getTree(pid, full, DownloadState.All) - - @RequirePerm(Permission.All) - def getFilteredFileTree(self, pid, full, state): - """ Same as `getFileTree` but only contains files with specific download state. - - :param pid: package id - :param full: go down the complete tree or only the first layer - :param state: :class:`DownloadState`, the attributes used for filtering - :return: :class:`TreeCollection` - """ - return self.core.files.getTree(pid, full, state) - - @RequirePerm(Permission.All) - def getPackageContent(self, pid): - """ Only retrieve content of a specific package. see `getFileTree`""" - return self.getFileTree(pid, False) - - @RequirePerm(Permission.All) - def getPackageInfo(self, pid): - """Returns information about package, without detailed information about containing files - - :param pid: package id - :raises PackageDoesNotExists: - :return: :class:`PackageInfo` - """ - info = self.core.files.getPackageInfo(pid) - if not info: - raise PackageDoesNotExists(pid) - return info - - @RequirePerm(Permission.All) - def getFileInfo(self, fid): - """ Info for specific file - - :param fid: file id - :raises FileDoesNotExists: - :return: :class:`FileInfo` - - """ - info = self.core.files.getFileInfo(fid) - if not info: - raise FileDoesNotExists(fid) - return info - - @RequirePerm(Permission.Download) - def getFilePath(self, fid): - """ Internal method to get the filepath""" - info = self.getFileInfo(fid) - pack = self.core.files.getPackage(info.package) - return pack.getPath(), info.name - - - @RequirePerm(Permission.All) - def findFiles(self, pattern): - return self.core.files.getTree(-1, True, DownloadState.All, pattern) - - @RequirePerm(Permission.All) - def searchSuggestions(self, pattern): - names = self.core.db.getMatchingFilenames(pattern, self.primaryUID) - # TODO: stemming and reducing the names to provide better suggestions - return uniqify(names) - - @RequirePerm(Permission.All) - def findPackages(self, tags): - pass - - @RequirePerm(Permission.Modify) - def updatePackage(self, pack): - """Allows to modify several package attributes. - - :param pid: package id - :param data: :class:`PackageInfo` - """ - pid = pack.pid - p = self.core.files.getPackage(pid) - if not p: raise PackageDoesNotExists(pid) - - #TODO: fix - for key, value in data.iteritems(): - if key == "id": continue - setattr(p, key, value) - - p.sync() - self.core.files.save() - - @RequirePerm(Permission.Modify) - def setPackageFolder(self, pid, path): - pass - - @RequirePerm(Permission.Modify) - def movePackage(self, pid, root): - """ Set a new root for specific package. This will also moves the files on disk\ - and will only work when no file is currently downloading. - - :param pid: package id - :param root: package id of new root - :raises PackageDoesNotExists: When pid or root is missing - :return: False if package can't be moved - """ - return self.core.files.movePackage(pid, root) - - @RequirePerm(Permission.Modify) - def moveFiles(self, fids, pid): - """Move multiple files to another package. This will move the files on disk and\ - only work when files are not downloading. All files needs to be continuous ordered - in the current package. - - :param fids: list of file ids - :param pid: destination package - :return: False if files can't be moved - """ - return self.core.files.moveFiles(fids, pid) - - @RequirePerm(Permission.Modify) - def orderPackage(self, pid, position): - """Set new position for a package. - - :param pid: package id - :param position: new position, 0 for very beginning - """ - self.core.files.orderPackage(pid, position) - - @RequirePerm(Permission.Modify) - def orderFiles(self, fids, pid, position): - """ Set a new position for a bunch of files within a package. - All files have to be in the same package and must be **continuous**\ - in the package. That means no gaps between them. - - :param fids: list of file ids - :param pid: package id of parent package - :param position: new position: 0 for very beginning - """ - self.core.files.orderFiles(fids, pid, position) - - -if Api.extend(FileApi): - del FileApi \ No newline at end of file diff --git a/module/api/UserInteractionApi.py b/module/api/UserInteractionApi.py deleted file mode 100644 index b95b7c468..000000000 --- a/module/api/UserInteractionApi.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.Api import Api, RequirePerm, Permission, Interaction - -from ApiComponent import ApiComponent - -class UserInteractionApi(ApiComponent): - """ Everything needed for user interaction """ - - @RequirePerm(Permission.Interaction) - def isInteractionWaiting(self, mode): - """ Check if task is waiting. - - :param mode: binary or'ed output type - :return: boolean - """ - return self.core.interactionManager.isTaskWaiting(self.primaryUID, mode) - - @RequirePerm(Permission.Interaction) - def getInteractionTasks(self, mode): - """Retrieve task for specific mode. - - :param mode: binary or'ed interaction types which should be retrieved - :rtype list of :class:`InteractionTask` - """ - tasks = self.core.interactionManager.getTasks(self.primaryUID, mode) - # retrieved tasks count as seen - for t in tasks: - t.seen = True - if t.type == Interaction.Notification: - t.setWaiting(self.core.interactionManager.CLIENT_THRESHOLD) - - return tasks - - @RequirePerm(Permission.Interaction) - def setInteractionResult(self, iid, result): - """Set Result for a interaction task. It will be immediately removed from task queue afterwards - - :param iid: interaction id - :param result: result as json string - """ - task = self.core.interactionManager.getTaskByID(iid) - if task and self.primaryUID == task.owner: - task.setResult(result) - - @RequirePerm(Permission.Interaction) - def getAddonHandler(self): - pass - - @RequirePerm(Permission.Interaction) - def callAddonHandler(self, plugin, func, pid_or_fid): - pass - - @RequirePerm(Permission.Download) - def generateDownloadLink(self, fid, timeout): - pass - - -if Api.extend(UserInteractionApi): - del UserInteractionApi \ No newline at end of file diff --git a/module/api/__init__.py b/module/api/__init__.py deleted file mode 100644 index 1348fd26f..000000000 --- a/module/api/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -__all__ = ["CoreApi", "ConfigApi", "DownloadApi", "DownloadPreparingApi", "FileApi", - "CollectorApi", "UserInteractionApi", "AccountApi", "AddonApi"] - -# Import all components -# from .import * -# Above does not work in py 2.5 -for name in __all__: - __import__(__name__ + "." + name) \ No newline at end of file diff --git a/module/cli/AddPackage.py b/module/cli/AddPackage.py deleted file mode 100644 index a73401586..000000000 --- a/module/cli/AddPackage.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -#Copyright (C) 2011 RaNaN -# -#This program is free software; you can redistribute it and/or modify -#it under the terms of the GNU General Public License as published by -#the Free Software Foundation; either version 3 of the License, -#or (at your option) any later version. -# -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#See the GNU General Public License for more details. -# -#You should have received a copy of the GNU General Public License -# along with this program; if not, see . -# -### - -from Handler import Handler -from printer import * - -class AddPackage(Handler): - """ let the user add packages """ - - def init(self): - self.name = "" - self.urls = [] - - def onEnter(self, inp): - if inp == "0": - self.cli.reset() - - if not self.name: - self.name = inp - self.setInput() - elif inp == "END": - #add package - self.client.addPackage(self.name, self.urls, 1) - self.cli.reset() - else: - if inp.strip(): - self.urls.append(inp) - self.setInput() - - def renderBody(self, line): - println(line, white(_("Add Package:"))) - println(line + 1, "") - line += 2 - - if not self.name: - println(line, _("Enter a name for the new package")) - println(line + 1, "") - line += 2 - else: - println(line, _("Package: %s") % self.name) - println(line + 1, _("Parse the links you want to add.")) - println(line + 2, _("Type %s when done.") % mag("END")) - println(line + 3, _("Links added: ") + mag(len(self.urls))) - line += 4 - - println(line, "") - println(line + 1, mag("0.") + _(" back to main menu")) - - return line + 2 \ No newline at end of file diff --git a/module/cli/Handler.py b/module/cli/Handler.py deleted file mode 100644 index 476d09386..000000000 --- a/module/cli/Handler.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -#Copyright (C) 2011 RaNaN -# -#This program is free software; you can redistribute it and/or modify -#it under the terms of the GNU General Public License as published by -#the Free Software Foundation; either version 3 of the License, -#or (at your option) any later version. -# -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#See the GNU General Public License for more details. -# -#You should have received a copy of the GNU General Public License -# along with this program; if not, see . -# -### -class Handler: - def __init__(self, cli): - self.cli = cli - self.init() - - client = property(lambda self: self.cli.client) - input = property(lambda self: self.cli.input) - - def init(self): - pass - - def onChar(self, char): - pass - - def onBackSpace(self): - pass - - def onEnter(self, inp): - pass - - def setInput(self, inp=""): - self.cli.setInput(inp) - - def backspace(self): - self.cli.setInput(self.input[:-1]) - - def renderBody(self, line): - """ gets the line where to render output and should return the line number below its content """ - return line + 1 \ No newline at end of file diff --git a/module/cli/ManageFiles.py b/module/cli/ManageFiles.py deleted file mode 100644 index c133d9959..000000000 --- a/module/cli/ManageFiles.py +++ /dev/null @@ -1,204 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -#Copyright (C) 2011 RaNaN -# -#This program is free software; you can redistribute it and/or modify -#it under the terms of the GNU General Public License as published by -#the Free Software Foundation; either version 3 of the License, -#or (at your option) any later version. -# -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#See the GNU General Public License for more details. -# -#You should have received a copy of the GNU General Public License -# along with this program; if not, see . -# -### - -from itertools import islice -from time import time - -from Handler import Handler -from printer import * - -from module.Api import Destination, PackageData - -class ManageFiles(Handler): - """ possibility to manage queue/collector """ - - def init(self): - self.target = Destination.Queue - self.pos = 0 #position in queue - self.package = -1 #chosen package - self.mode = "" # move/delete/restart - - self.cache = None - self.links = None - self.time = 0 - - def onChar(self, char): - if char in ("m", "d", "r"): - self.mode = char - self.setInput() - elif char == "p": - self.pos = max(0, self.pos - 5) - self.backspace() - elif char == "n": - self.pos += 5 - self.backspace() - - def onBackSpace(self): - if not self.input and self.mode: - self.mode = "" - if not self.input and self.package > -1: - self.package = -1 - - def onEnter(self, input): - if input == "0": - self.cli.reset() - elif self.package < 0 and self.mode: - #mode select - packs = self.parseInput(input) - if self.mode == "m": - [self.client.movePackage((self.target + 1) % 2, x) for x in packs] - elif self.mode == "d": - self.client.deletePackages(packs) - elif self.mode == "r": - [self.client.restartPackage(x) for x in packs] - - elif self.mode: - #edit links - links = self.parseInput(input, False) - - if self.mode == "d": - self.client.deleteFiles(links) - elif self.mode == "r": - map(self.client.restartFile, links) - - else: - #look into package - try: - self.package = int(input) - except: - pass - - self.cache = None - self.links = None - self.pos = 0 - self.mode = "" - self.setInput() - - - def renderBody(self, line): - if self.package < 0: - println(line, white(_("Manage Packages:"))) - else: - println(line, white((_("Manage Links:")))) - line += 1 - - if self.mode: - if self.mode == "m": - println(line, _("What do you want to move?")) - elif self.mode == "d": - println(line, _("What do you want to delete?")) - elif self.mode == "r": - println(line, _("What do you want to restart?")) - - println(line + 1, "Enter a single number, comma separated numbers or ranges. e.g.: 1,2,3 or 1-3.") - line += 2 - else: - println(line, _("Choose what you want to do, or enter package number.")) - println(line + 1, ("%s - %%s, %s - %%s, %s - %%s" % (mag("d"), mag("m"), mag("r"))) % ( - _("delete"), _("move"), _("restart"))) - line += 2 - - if self.package < 0: - #print package info - pack = self.getPackages() - i = 0 - for value in islice(pack, self.pos, self.pos + 5): - try: - println(line, mag(str(value.pid)) + ": " + value.name) - line += 1 - i += 1 - except Exception, e: - pass - for x in range(5 - i): - println(line, "") - line += 1 - else: - #print links info - pack = self.getLinks() - i = 0 - for value in islice(pack.links, self.pos, self.pos + 5): - try: - println(line, mag(value.fid) + ": %s | %s | %s" % ( - value.name, value.statusmsg, value.plugin)) - line += 1 - i += 1 - except Exception, e: - pass - for x in range(5 - i): - println(line, "") - line += 1 - - println(line, mag("p") + _(" - previous") + " | " + mag("n") + _(" - next")) - println(line + 1, mag("0.") + _(" back to main menu")) - - return line + 2 - - - def getPackages(self): - if self.cache and self.time + 2 < time(): - return self.cache - - if self.target == Destination.Queue: - data = self.client.getQueue() - else: - data = self.client.getCollector() - - - self.cache = data - self.time = time() - - return data - - def getLinks(self): - if self.links and self.time + 1 < time(): - return self.links - - try: - data = self.client.getPackageData(self.package) - except: - data = PackageData(links=[]) - - self.links = data - self.time = time() - - return data - - def parseInput(self, inp, package=True): - inp = inp.strip() - if "-" in inp: - l, n, h = inp.partition("-") - l = int(l) - h = int(h) - r = range(l, h + 1) - - ret = [] - if package: - for p in self.cache: - if p.pid in r: - ret.append(p.pid) - else: - for l in self.links.links: - if l.lid in r: - ret.append(l.lid) - - return ret - - else: - return [int(x) for x in inp.split(",")] diff --git a/module/cli/__init__.py b/module/cli/__init__.py deleted file mode 100644 index fa8a09291..000000000 --- a/module/cli/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from AddPackage import AddPackage -from ManageFiles import ManageFiles \ No newline at end of file diff --git a/module/cli/printer.py b/module/cli/printer.py deleted file mode 100644 index c62c1800e..000000000 --- a/module/cli/printer.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -def blue(string): - return "\033[1;34m" + unicode(string) + "\033[0m" - -def green(string): - return "\033[1;32m" + unicode(string) + "\033[0m" - -def yellow(string): - return "\033[1;33m" + unicode(string) + "\033[0m" - -def red(string): - return "\033[1;31m" + unicode(string) + "\033[0m" - -def cyan(string): - return "\033[1;36m" + unicode(string) + "\033[0m" - -def mag(string): - return "\033[1;35m" + unicode(string) + "\033[0m" - -def white(string): - return "\033[1;37m" + unicode(string) + "\033[0m" - -def println(line, content): - print "\033[" + str(line) + ";0H\033[2K" + content \ No newline at end of file diff --git a/module/config/ConfigManager.py b/module/config/ConfigManager.py deleted file mode 100644 index 3290ed4ec..000000000 --- a/module/config/ConfigManager.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from new_collections import OrderedDict - -from module.Api import InvalidConfigSection -from module.utils import from_string, json - -from ConfigParser import ConfigParser - - -def convertKeyError(func): - """ converts KeyError into InvalidConfigSection """ - - def conv(*args, **kwargs): - try: - return func(*args, **kwargs) - except KeyError: - raise InvalidConfigSection(args[1]) - - return conv - - -class ConfigManager(ConfigParser): - """ Manages the core config and configs for addons and single user. - Has similar interface to ConfigParser. """ - - def __init__(self, core, parser): - # No __init__ call to super class is needed! - - self.core = core - self.db = core.db - # The config parser, holding the core config - self.parser = parser - - # similar to parser, separated meta data and values - self.config = OrderedDict() - - # Value cache for multiple user configs - # Values are populated from db on first access - # Entries are saved as (user, section) keys - self.values = {} - # TODO: similar to a cache, could be deleted periodically - - def save(self): - self.parser.save() - - @convertKeyError - def get(self, section, option, user=None): - """get config value, core config only available for admins. - if user is not valid default value will be returned""" - - # Core config loaded from parser, when no user is given or he is admin - if section in self.parser and user is None: - return self.parser.get(section, option) - else: - # We need the id and not the instance - # Will be None for admin user and so the same as internal access - try: - # Check if this config exists - # Configs without meta data can not be loaded! - data = self.config[section].config[option] - return self.loadValues(user, section)[option] - except KeyError: - pass # Returns default value later - - return self.config[section].config[option].default - - def loadValues(self, user, section): - if (user, section) not in self.values: - conf = self.db.loadConfig(section, user) - try: - self.values[user, section] = json.loads(conf) if conf else {} - except ValueError: # Something did go wrong when parsing - self.values[user, section] = {} - self.core.print_exc() - - return self.values[user, section] - - @convertKeyError - def set(self, section, option, value, sync=True, user=None): - """ set config value """ - - changed = False - if section in self.parser and user is None: - changed = self.parser.set(section, option, value, sync) - else: - data = self.config[section].config[option] - value = from_string(value, data.type) - old_value = self.get(section, option) - - # Values will always be saved to db, sync is ignored - if value != old_value: - changed = True - self.values[user, section][option] = value - if sync: self.saveValues(user, section) - - if changed: self.core.evm.dispatchEvent("config:changed", section, option, value) - return changed - - def saveValues(self, user, section): - if section in self.parser and user is None: - self.save() - elif (user, section) in self.values: - self.db.saveConfig(section, json.dumps(self.values[user, section]), user) - - def delete(self, section, user=None): - """ Deletes values saved in db and cached values for given user, NOT meta data - Does not trigger an error when nothing was deleted. """ - if (user, section) in self.values: - del self.values[user, section] - - self.db.deleteConfig(section, user) - self.core.evm.dispatchEvent("config:deleted", section, user) - - def iterCoreSections(self): - return self.parser.iterSections() - - def iterSections(self, user=None): - """ Yields: section, metadata, values """ - values = self.db.loadConfigsForUser(user) - - # Every section needs to be json decoded - for section, data in values.items(): - try: - values[section] = json.loads(data) if data else {} - except ValueError: - values[section] = {} - self.core.print_exc() - - for name, config in self.config.iteritems(): - yield name, config, values[name] if name in values else {} - - def getSection(self, section, user=None): - if section in self.parser and user is None: - return self.parser.getSection(section) - - values = self.loadValues(user, section) - return self.config.get(section), values diff --git a/module/config/ConfigParser.py b/module/config/ConfigParser.py deleted file mode 100644 index bf9192270..000000000 --- a/module/config/ConfigParser.py +++ /dev/null @@ -1,207 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import with_statement -from time import sleep -from os.path import exists -from gettext import gettext -from new_collections import namedtuple, OrderedDict - -from module.utils import from_string -from module.utils.fs import chmod - -from default import make_config - -CONF_VERSION = 2 -SectionTuple = namedtuple("SectionTuple", "name description long_desc config") -ConfigData = namedtuple("ConfigData", "name type description default") - -class ConfigParser: - """ - Holds and manages the configuration + meta data for config read from file. - """ - - CONFIG = "pyload.conf" - - def __init__(self, config=None): - - if config: self.CONFIG = config - - # Meta data information - self.config = OrderedDict() - # The actual config values - self.values = {} - - self.checkVersion() - - self.loadDefault() - self.parseValues(self.CONFIG) - - def loadDefault(self): - make_config(self) - - def checkVersion(self): - """Determines if config needs to be deleted""" - e = None - # workaround conflict, with GUI (which also accesses the config) so try read in 3 times - for i in range(0, 3): - try: - if exists(self.CONFIG): - f = open(self.CONFIG, "rb") - v = f.readline() - f.close() - v = v[v.find(":") + 1:].strip() - - if not v or int(v) < CONF_VERSION: - f = open(self.CONFIG, "wb") - f.write("version: " + str(CONF_VERSION)) - f.close() - print "Old version of %s deleted" % self.CONFIG - else: - f = open(self.CONFIG, "wb") - f.write("version:" + str(CONF_VERSION)) - f.close() - - except Exception, ex: - e = ex - sleep(0.3) - if e: raise e - - - def parseValues(self, filename): - """read config values from file""" - f = open(filename, "rb") - config = f.readlines()[1:] - - # save the current section - section = "" - - for line in config: - line = line.strip() - - # comment line, different variants - if not line or line.startswith("#") or line.startswith("//") or line.startswith(";"): continue - - if line.startswith("["): - section = line.replace("[", "").replace("]", "") - - if section not in self.config: - print "Unrecognized section", section - section = "" - - else: - name, non, value = line.rpartition("=") - name = name.strip() - value = value.strip() - - if not section: - print "Value without section", name - continue - - if name in self.config[section].config: - self.set(section, name, value, sync=False) - else: - print "Unrecognized option", section, name - - - def save(self): - """saves config to filename""" - - configs = [] - f = open(self.CONFIG, "wb") - configs.append(f) - chmod(self.CONFIG, 0600) - f.write("version: %i\n\n" % CONF_VERSION) - - for section, data in self.config.iteritems(): - f.write("[%s]\n" % section) - - for option, data in data.config.iteritems(): - value = self.get(section, option) - if type(value) == unicode: value = value.encode("utf8") - else: value = str(value) - - f.write('%s = %s\n' % (option, value)) - - f.write("\n") - - f.close() - - def __getitem__(self, section): - """provides dictionary like access: c['section']['option']""" - return Section(self, section) - - def __contains__(self, section): - """ checks if parser contains section """ - return section in self.config - - def get(self, section, option): - """get value or default""" - try: - return self.values[section][option] - except KeyError: - return self.config[section].config[option].default - - def set(self, section, option, value, sync=True): - """set value""" - - data = self.config[section].config[option] - value = from_string(value, data.type) - old_value = self.get(section, option) - - # only save when different values - if value != old_value: - if section not in self.values: self.values[section] = {} - self.values[section][option] = value - if sync: - self.save() - return True - - return False - - def getMetaData(self, section, option): - """ get all config data for an option """ - return self.config[section].config[option] - - def iterSections(self): - """ Yields section, config info, values, for all sections """ - - for name, config in self.config.iteritems(): - yield name, config, self.values[name] if name in self.values else {} - - def getSection(self, section): - """ Retrieves single config as tuple (section, values) """ - return self.config[section], self.values[section] if section in self.values else {} - - def addConfigSection(self, section, name, desc, long_desc, config): - """Adds a section to the config. `config` is a list of config tuples as used in plugin api defined as: - The order of the config elements is preserved with OrderedDict - """ - d = OrderedDict() - - for entry in config: - if len(entry) == 5: - conf_name, type, conf_desc, conf_verbose, default = entry - else: # config options without description - conf_name, type, conf_desc, default = entry - conf_verbose = "" - - d[conf_name] = ConfigData(gettext(conf_desc), type, gettext(conf_verbose), from_string(default, type)) - - data = SectionTuple(gettext(name), gettext(desc), gettext(long_desc), d) - self.config[section] = data - -class Section: - """provides dictionary like access for configparser""" - - def __init__(self, parser, section): - """Constructor""" - self.parser = parser - self.section = section - - def __getitem__(self, item): - """getitem""" - return self.parser.get(self.section, item) - - def __setitem__(self, item, value): - """setitem""" - self.parser.set(self.section, item, value) diff --git a/module/config/__init__.py b/module/config/__init__.py deleted file mode 100644 index 4b31e848b..000000000 --- a/module/config/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'christian' diff --git a/module/config/default.py b/module/config/default.py deleted file mode 100644 index 8a2044281..000000000 --- a/module/config/default.py +++ /dev/null @@ -1,107 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Configuration layout for default base config -""" - -#TODO: write tooltips and descriptions -#TODO: use apis config related classes - -def make_config(config): - # Check if gettext is installed - _ = lambda x: x - - config.addConfigSection("remote", _("Remote"), _("Description"), _("Long description"), - [ - ("activated", "bool", _("Activated"), _("Tooltip"), True), - ("port", "int", _("Port"), _("Tooltip"), 7227), - ("listenaddr", "ip", _("Adress"), _("Tooltip"), "0.0.0.0"), - ]) - - config.addConfigSection("log", _("Log"), _("Description"), _("Long description"), - [ - ("log_size", "int", _("Size in kb"), _("Tooltip"), 100), - ("log_folder", "folder", _("Folder"), _("Tooltip"), "Logs"), - ("file_log", "bool", _("File Log"), _("Tooltip"), True), - ("log_count", "int", _("Count"), _("Tooltip"), 5), - ("log_rotate", "bool", _("Log Rotate"), _("Tooltip"), True), - ]) - - config.addConfigSection("permission", _("Permissions"), _("Description"), _("Long description"), - [ - ("group", "str", _("Groupname"), _("Tooltip"), "users"), - ("change_dl", "bool", _("Change Group and User of Downloads"), _("Tooltip"), False), - ("change_file", "bool", _("Change file mode of downloads"), _("Tooltip"), False), - ("user", "str", _("Username"), _("Tooltip"), "user"), - ("file", "str", _("Filemode for Downloads"), _("Tooltip"), "0644"), - ("change_group", "bool", _("Change group of running process"), _("Tooltip"), False), - ("folder", "str", _("Folder Permission mode"), _("Tooltip"), "0755"), - ("change_user", "bool", _("Change user of running process"), _("Tooltip"), False), - ]) - - config.addConfigSection("general", _("General"), _("Description"), _("Long description"), - [ - ("language", "en;de;fr;it;es;nl;sv;ru;pl;cs;sr;pt_BR", _("Language"), _("Tooltip"), "en"), - ("download_folder", "folder", _("Download Folder"), _("Tooltip"), "Downloads"), - ("checksum", "bool", _("Use Checksum"), _("Tooltip"), False), - ("folder_per_package", "bool", _("Create folder for each package"), _("Tooltip"), True), - ("debug_mode", "bool", _("Debug Mode"), _("Tooltip"), False), - ("min_free_space", "int", _("Min Free Space (MB)"), _("Tooltip"), 200), - ("renice", "int", _("CPU Priority"), _("Tooltip"), 0), - ]) - - config.addConfigSection("ssl", _("SSL"), _("Description"), _("Long description"), - [ - ("cert", "file", _("SSL Certificate"), _("Tooltip"), "ssl.crt"), - ("activated", "bool", _("Activated"), _("Tooltip"), False), - ("key", "file", _("SSL Key"), _("Tooltip"), "ssl.key"), - ]) - - config.addConfigSection("webinterface", _("Webinterface"), _("Description"), _("Long description"), - [ - ("template", "str", _("Template"), _("Tooltip"), "default"), - ("activated", "bool", _("Activated"), _("Tooltip"), True), - ("prefix", "str", _("Path Prefix"), _("Tooltip"), ""), - ("server", "auto;threaded;fallback;fastcgi", _("Server"), _("Tooltip"), "auto"), - ("force_server", "str", _("Favor specific server"), _("Tooltip"), ""), - ("host", "ip", _("IP"), _("Tooltip"), "0.0.0.0"), - ("https", "bool", _("Use HTTPS"), _("Tooltip"), False), - ("port", "int", _("Port"), _("Tooltip"), 8001), - ("develop", "str", _("Development mode"), _(""), False), - ]) - - config.addConfigSection("proxy", _("Proxy"), _("Description"), _("Long description"), - [ - ("username", "str", _("Username"), _("Tooltip"), ""), - ("proxy", "bool", _("Use Proxy"), _("Tooltip"), False), - ("address", "str", _("Address"), _("Tooltip"), "localhost"), - ("password", "password", _("Password"), _("Tooltip"), ""), - ("type", "http;socks4;socks5", _("Protocol"), _("Tooltip"), "http"), - ("port", "int", _("Port"), _("Tooltip"), 7070), - ]) - - config.addConfigSection("reconnect", _("Reconnect"), _("Description"), _("Long description"), - [ - ("endTime", "time", _("End"), _("Tooltip"), "0:00"), - ("activated", "bool", _("Use Reconnect"), _("Tooltip"), False), - ("method", "str", _("Method"), _("Tooltip"), "./reconnect.sh"), - ("startTime", "time", _("Start"), _("Tooltip"), "0:00"), - ]) - - config.addConfigSection("download", _("Download"), _("Description"), _("Long description"), - [ - ("max_downloads", "int", _("Max Parallel Downloads"), _("Tooltip"), 3), - ("limit_speed", "bool", _("Limit Download Speed"), _("Tooltip"), False), - ("interface", "str", _("Download interface to bind (ip or Name)"), _("Tooltip"), ""), - ("skip_existing", "bool", _("Skip already existing files"), _("Tooltip"), False), - ("max_speed", "int", _("Max Download Speed in kb/s"), _("Tooltip"), -1), - ("ipv6", "bool", _("Allow IPv6"), _("Tooltip"), False), - ("chunks", "int", _("Max connections for one download"), _("Tooltip"), 3), - ("restart_failed", "bool", _("Restart failed downloads on startup"), _("Tooltip"), False), - ]) - - config.addConfigSection("downloadTime", _("Download Time"), _("Description"), _("Long description"), - [ - ("start", "time", _("Start"), _("Tooltip"), "0:00"), - ("end", "time", _("End"), _("Tooltip"), "0:00"), - ]) diff --git a/module/database/AccountDatabase.py b/module/database/AccountDatabase.py deleted file mode 100644 index 518674951..000000000 --- a/module/database/AccountDatabase.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.database import queue, async -from module.database import DatabaseBackend - - -class AccountMethods: - @queue - def loadAccounts(db): - db.c.execute('SELECT plugin, loginname, activated, password, options FROM accounts;') - return db.c.fetchall() - - @async - def saveAccounts(db, data): - # TODO: owner, shared - - db.c.executemany( - 'INSERT INTO accounts(plugin, loginname, activated, password, options) VALUES(?,?,?,?,?)', data) - - @async - def removeAccount(db, plugin, loginname): - db.c.execute('DELETE FROM accounts WHERE plugin=? AND loginname=?', (plugin, loginname)) - - -DatabaseBackend.registerSub(AccountMethods) \ No newline at end of file diff --git a/module/database/ConfigDatabase.py b/module/database/ConfigDatabase.py deleted file mode 100644 index 554e07132..000000000 --- a/module/database/ConfigDatabase.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.database import DatabaseMethods, queue, async - -class ConfigMethods(DatabaseMethods): - - @async - def saveConfig(self, plugin, config, user=None): - if user is None: user = -1 - self.c.execute('INSERT INTO settings(plugin, config, user) VALUES(?,?,?)', (plugin, config, user)) - - - @queue - def loadConfig(self, plugin, user=None): - if user is None: user = -1 - self.c.execute('SELECT config FROM settings WHERE plugin=? AND user=?', (plugin, user)) - - r = self.c.fetchone() - return r[0] if r else "" - - @async - def deleteConfig(self, plugin, user=None): - if user is None: - self.c.execute('DELETE FROM settings WHERE plugin=?', (plugin, )) - else: - self.c.execute('DELETE FROM settings WHERE plugin=? AND user=?', (plugin, user)) - - @queue - def loadAllConfigs(self): - self.c.execute('SELECT user, plugin, config FROM settings') - configs = {} - for r in self.c: - if r[0] in configs: - configs[r[0]][r[1]] = r[2] - else: - configs[r[0]] = {r[1]: r[2]} - - return configs - - @queue - def loadConfigsForUser(self, user=None): - if user is None: user = -1 - self.c.execute('SELECT plugin, config FROM settings WHERE user=?', (user,)) - configs = {} - for r in self.c: - configs[r[0]] = r[1] - - return configs - - @async - def clearAllConfigs(self): - self.c.execute('DELETE FROM settings') - - -ConfigMethods.register() \ No newline at end of file diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py deleted file mode 100644 index 54d1c54bb..000000000 --- a/module/database/DatabaseBackend.py +++ /dev/null @@ -1,492 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN, mkaay -############################################################################### - -from threading import Thread, Event -from shutil import move - -from Queue import Queue -from traceback import print_exc - -from module.utils.fs import chmod, exists, remove - -try: - from pysqlite2 import dbapi2 as sqlite3 -except: - import sqlite3 - -DB = None -DB_VERSION = 6 - -def set_DB(db): - global DB - DB = db - - -def queue(f): - @staticmethod - def x(*args, **kwargs): - if DB: - return DB.queue(f, *args, **kwargs) - - return x - - -def async(f): - @staticmethod - def x(*args, **kwargs): - if DB: - return DB.async(f, *args, **kwargs) - - return x - - -def inner(f): - @staticmethod - def x(*args, **kwargs): - if DB: - return f(DB, *args, **kwargs) - - return x - - -class DatabaseMethods: - # stubs for autocompletion - core = None - manager = None - conn = None - c = None - - @classmethod - def register(cls): - DatabaseBackend.registerSub(cls) - - -class DatabaseJob(): - def __init__(self, f, *args, **kwargs): - self.done = Event() - - self.f = f - self.args = args - self.kwargs = kwargs - - self.result = None - self.exception = False - - # import inspect - # self.frame = inspect.currentframe() - - def __repr__(self): - from os.path import basename - - frame = self.frame.f_back - output = "" - for i in range(5): - output += "\t%s:%s, %s\n" % (basename(frame.f_code.co_filename), frame.f_lineno, frame.f_code.co_name) - frame = frame.f_back - del frame - del self.frame - - return "DataBase Job %s:%s\n%sResult: %s" % (self.f.__name__, self.args[1:], output, self.result) - - def processJob(self): - try: - self.result = self.f(*self.args, **self.kwargs) - except Exception, e: - print_exc() - try: - print "Database Error @", self.f.__name__, self.args[1:], self.kwargs, e - except: - pass - - self.exception = e - finally: - self.done.set() - - def wait(self): - self.done.wait() - - -class DatabaseBackend(Thread): - subs = [] - - DB_FILE = "pyload.db" - VERSION_FILE = "db.version" - - def __init__(self, core): - Thread.__init__(self) - self.setDaemon(True) - self.core = core - self.manager = None # set later - self.running = Event() - - self.jobs = Queue() - - set_DB(self) - - def setup(self): - """ *MUST* be called before db can be used !""" - self.start() - self.running.wait() - - def init(self): - """main loop, which executes commands""" - - version = self._checkVersion() - - self.conn = sqlite3.connect(self.DB_FILE) - chmod(self.DB_FILE, 0600) - - self.c = self.conn.cursor() - - if version is not None and version < DB_VERSION: - success = self._convertDB(version) - - # delete database - if not success: - self.c.close() - self.conn.close() - - try: - self.manager.core.log.warning(_("Database was deleted due to incompatible version.")) - except: - print "Database was deleted due to incompatible version." - - remove(self.VERSION_FILE) - move(self.DB_FILE, self.DB_FILE + ".backup") - f = open(self.VERSION_FILE, "wb") - f.write(str(DB_VERSION)) - f.close() - - self.conn = sqlite3.connect(self.DB_FILE) - chmod(self.DB_FILE, 0600) - self.c = self.conn.cursor() - - self._createTables() - self.conn.commit() - - - def run(self): - try: - self.init() - finally: - self.running.set() - - while True: - j = self.jobs.get() - if j == "quit": - self.c.close() - self.conn.commit() - self.conn.close() - self.closing.set() - break - j.processJob() - - - def shutdown(self): - self.running.clear() - self.closing = Event() - self.jobs.put("quit") - self.closing.wait(1) - - def _checkVersion(self): - """ get db version""" - if not exists(self.VERSION_FILE): - f = open(self.VERSION_FILE, "wb") - f.write(str(DB_VERSION)) - f.close() - return - - f = open(self.VERSION_FILE, "rb") - v = int(f.read().strip()) - f.close() - - return v - - def _convertDB(self, v): - try: - return getattr(self, "_convertV%i" % v)() - except: - return False - - #--convert scripts start - - def _convertV6(self): - return False - - #--convert scripts end - - def _createTables(self): - """create tables for database""" - - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "packages" (' - '"pid" INTEGER PRIMARY KEY AUTOINCREMENT, ' - '"name" TEXT NOT NULL, ' - '"folder" TEXT DEFAULT "" NOT NULL, ' - '"site" TEXT DEFAULT "" NOT NULL, ' - '"comment" TEXT DEFAULT "" NOT NULL, ' - '"password" TEXT DEFAULT "" NOT NULL, ' - '"added" INTEGER DEFAULT 0 NOT NULL,' # set by trigger - '"status" INTEGER DEFAULT 0 NOT NULL,' - '"tags" TEXT DEFAULT "" NOT NULL,' - '"shared" INTEGER DEFAULT 0 NOT NULL,' - '"packageorder" INTEGER DEFAULT -1 NOT NULL,' #incremented by trigger - '"root" INTEGER DEFAULT -1 NOT NULL, ' - '"owner" INTEGER NOT NULL, ' - 'FOREIGN KEY(owner) REFERENCES users(uid), ' - 'CHECK (root != pid)' - ')' - ) - - self.c.execute( - 'CREATE TRIGGER IF NOT EXISTS "insert_package" AFTER INSERT ON "packages"' - 'BEGIN ' - 'UPDATE packages SET added = strftime("%s", "now"), ' - 'packageorder = (SELECT max(p.packageorder) + 1 FROM packages p WHERE p.root=new.root) ' - 'WHERE rowid = new.rowid;' - 'END' - ) - - self.c.execute( - 'CREATE TRIGGER IF NOT EXISTS "delete_package" AFTER DELETE ON "packages"' - 'BEGIN ' - 'DELETE FROM files WHERE package = old.pid;' - 'UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > old.packageorder AND root=old.pid;' - 'END' - ) - self.c.execute('CREATE INDEX IF NOT EXISTS "package_index" ON packages(root, owner)') - self.c.execute('CREATE INDEX IF NOT EXISTS "package_owner" ON packages(owner)') - - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "files" (' - '"fid" INTEGER PRIMARY KEY AUTOINCREMENT, ' - '"name" TEXT NOT NULL, ' - '"size" INTEGER DEFAULT 0 NOT NULL, ' - '"status" INTEGER DEFAULT 0 NOT NULL, ' - '"media" INTEGER DEFAULT 1 NOT NULL,' - '"added" INTEGER DEFAULT 0 NOT NULL,' - '"fileorder" INTEGER DEFAULT -1 NOT NULL, ' - '"url" TEXT DEFAULT "" NOT NULL, ' - '"plugin" TEXT DEFAULT "" NOT NULL, ' - '"hash" TEXT DEFAULT "" NOT NULL, ' - '"dlstatus" INTEGER DEFAULT 0 NOT NULL, ' - '"error" TEXT DEFAULT "" NOT NULL, ' - '"package" INTEGER NOT NULL, ' - '"owner" INTEGER NOT NULL, ' - 'FOREIGN KEY(owner) REFERENCES users(uid), ' - 'FOREIGN KEY(package) REFERENCES packages(id)' - ')' - ) - self.c.execute('CREATE INDEX IF NOT EXISTS "file_index" ON files(package, owner)') - self.c.execute('CREATE INDEX IF NOT EXISTS "file_owner" ON files(owner)') - - self.c.execute( - 'CREATE TRIGGER IF NOT EXISTS "insert_file" AFTER INSERT ON "files"' - 'BEGIN ' - 'UPDATE files SET added = strftime("%s", "now"), ' - 'fileorder = (SELECT max(f.fileorder) + 1 FROM files f WHERE f.package=new.package) ' - 'WHERE rowid = new.rowid;' - 'END' - ) - - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "collector" (' - '"owner" INTEGER NOT NULL, ' - '"data" TEXT NOT NULL, ' - 'FOREIGN KEY(owner) REFERENCES users(uid), ' - 'PRIMARY KEY(owner) ON CONFLICT REPLACE' - ') ' - ) - - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "storage" (' - '"identifier" TEXT NOT NULL, ' - '"key" TEXT NOT NULL, ' - '"value" TEXT DEFAULT "", ' - 'PRIMARY KEY (identifier, key) ON CONFLICT REPLACE' - ')' - ) - - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "users" (' - '"uid" INTEGER PRIMARY KEY AUTOINCREMENT, ' - '"name" TEXT NOT NULL UNIQUE, ' - '"email" TEXT DEFAULT "" NOT NULL, ' - '"password" TEXT NOT NULL, ' - '"role" INTEGER DEFAULT 0 NOT NULL, ' - '"permission" INTEGER DEFAULT 0 NOT NULL, ' - '"folder" TEXT DEFAULT "" NOT NULL, ' - '"traffic" INTEGER DEFAULT -1 NOT NULL, ' - '"dllimit" INTEGER DEFAULT -1 NOT NULL, ' - '"dlquota" TEXT DEFAULT "" NOT NULL, ' - '"hddquota" INTEGER DEFAULT -1 NOT NULL, ' - '"template" TEXT DEFAULT "default" NOT NULL, ' - '"user" INTEGER DEFAULT -1 NOT NULL, ' # set by trigger to self - 'FOREIGN KEY(user) REFERENCES users(uid)' - ')' - ) - self.c.execute('CREATE INDEX IF NOT EXISTS "username_index" ON users(name)') - - self.c.execute( - 'CREATE TRIGGER IF NOT EXISTS "insert_user" AFTER INSERT ON "users"' - 'BEGIN ' - 'UPDATE users SET user = new.uid, folder=new.name ' - 'WHERE rowid = new.rowid;' - 'END' - ) - - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "settings" (' - '"plugin" TEXT NOT NULL, ' - '"user" INTEGER DEFAULT -1 NOT NULL, ' - '"config" TEXT NOT NULL, ' - 'FOREIGN KEY(user) REFERENCES users(uid), ' - 'PRIMARY KEY (plugin, user) ON CONFLICT REPLACE' - ')' - ) - - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "accounts" (' - '"plugin" TEXT NOT NULL, ' - '"loginname" TEXT NOT NULL, ' - '"owner" INTEGER NOT NULL DEFAULT -1, ' - '"activated" INTEGER DEFAULT 1, ' - '"password" TEXT DEFAULT "", ' - '"shared" INTEGER DEFAULT 0, ' - '"options" TEXT DEFAULT "", ' - 'FOREIGN KEY(owner) REFERENCES users(uid), ' - 'PRIMARY KEY (plugin, loginname, owner) ON CONFLICT REPLACE' - ')' - ) - - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "stats" (' - '"user" INTEGER NOT NULL, ' - '"plugin" TEXT NOT NULL, ' - '"time" INTEGER NOT NULL, ' - '"premium" INTEGER DEFAULT 0 NOT NULL, ' - '"amount" INTEGER DEFAULT 0 NOT NULL, ' - 'FOREIGN KEY(user) REFERENCES users(uid), ' - 'PRIMARY KEY(user, plugin, time)' - ')' - ) - self.c.execute('CREATE INDEX IF NOT EXISTS "stats_time" ON stats(time)') - - #try to lower ids - self.c.execute('SELECT max(fid) FROM files') - fid = self.c.fetchone()[0] - fid = int(fid) if fid else 0 - self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (fid, "files")) - - self.c.execute('SELECT max(pid) FROM packages') - pid = self.c.fetchone()[0] - pid = int(pid) if pid else 0 - self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (pid, "packages")) - - self.c.execute('VACUUM') - - - def createCursor(self): - return self.conn.cursor() - - @async - def commit(self): - self.conn.commit() - - @queue - def syncSave(self): - self.conn.commit() - - @async - def rollback(self): - self.conn.rollback() - - def async(self, f, *args, **kwargs): - args = (self, ) + args - job = DatabaseJob(f, *args, **kwargs) - self.jobs.put(job) - - def queue(self, f, *args, **kwargs): - args = (self, ) + args - job = DatabaseJob(f, *args, **kwargs) - self.jobs.put(job) - # only wait when db is running - if self.running.isSet(): job.wait() - return job.result - - @classmethod - def registerSub(cls, klass): - cls.subs.append(klass) - - @classmethod - def unregisterSub(cls, klass): - cls.subs.remove(klass) - - def __getattr__(self, attr): - for sub in DatabaseBackend.subs: - if hasattr(sub, attr): - return getattr(sub, attr) - raise AttributeError(attr) - -if __name__ == "__main__": - db = DatabaseBackend() - db.setup() - - class Test(): - @queue - def insert(db): - c = db.createCursor() - for i in range(1000): - c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", ("foo", i, "bar")) - - @async - def insert2(db): - c = db.createCursor() - for i in range(1000 * 1000): - c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", ("foo", i, "bar")) - - @queue - def select(db): - c = db.createCursor() - for i in range(10): - res = c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", ("foo", i)) - print res.fetchone() - - @queue - def error(db): - c = db.createCursor() - print "a" - c.execute("SELECT myerror FROM storage WHERE identifier=? AND key=?", ("foo", i)) - print "e" - - db.registerSub(Test) - from time import time - - start = time() - for i in range(100): - db.insert() - end = time() - print end - start - - start = time() - db.insert2() - end = time() - print end - start - - db.error() - diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py deleted file mode 100644 index 023dd1dc5..000000000 --- a/module/database/FileDatabase.py +++ /dev/null @@ -1,448 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -from new_collections import OrderedDict - -from module.Api import DownloadInfo, FileInfo, PackageInfo, PackageStats, DownloadState as DS, state_string -from module.database import DatabaseMethods, queue, async, inner - -zero_stats = PackageStats(0, 0, 0, 0) - - -class FileMethods(DatabaseMethods): - - @queue - def filecount(self): - """returns number of files, currently only used for debugging""" - self.c.execute("SELECT COUNT(*) FROM files") - return self.c.fetchone()[0] - - @queue - def downloadstats(self, user=None): - """ number of downloads and size """ - if user is None: - self.c.execute("SELECT COUNT(*), SUM(f.size) FROM files f WHERE dlstatus != 0") - else: - self.c.execute( - "SELECT COUNT(*), SUM(f.size) FROM files f, packages p WHERE f.package = p.pid AND dlstatus != 0", - user) - - r = self.c.fetchone() - # sum is None when no elements are added - return (r[0], r[1] if r[1] is not None else 0) if r else (0, 0) - - @queue - def queuestats(self, user=None): - """ number and size of files in queue not finished yet""" - # status not in NA, finished, skipped - if user is None: - self.c.execute("SELECT COUNT(*), SUM(f.size) FROM files f WHERE dlstatus NOT IN (0,5,6)") - else: - self.c.execute( - "SELECT COUNT(*), SUM(f.size) FROM files f, package p WHERE f.package = p.pid AND p.owner=? AND dlstatus NOT IN (0,5,6)", - user) - - r = self.c.fetchone() - return (r[0], r[1] if r[1] is not None else 0) if r else (0, 0) - - - # TODO: multi user? - @queue - def processcount(self, fid=-1, user=None): - """ number of files which have to be processed """ - # status in online, queued, starting, waiting, downloading - self.c.execute("SELECT COUNT(*), SUM(size) FROM files WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", (fid, )) - return self.c.fetchone()[0] - - @queue - def processstats(self, user=None): - if user is None: - self.c.execute("SELECT COUNT(*), SUM(size) FROM files WHERE dlstatus IN (2,3,8,9,10)") - else: - self.c.execute( - "SELECT COUNT(*), SUM(f.size) FROM files f, packages p WHERE f.package = p.pid AND dlstatus IN (2,3,8,9,10)", - user) - r = self.c.fetchone() - return (r[0], r[1] if r[1] is not None else 0) if r else (0, 0) - - @queue - def addLink(self, url, name, plugin, package, owner): - # mark file status initially as missing, dlstatus - queued - self.c.execute('INSERT INTO files(url, name, plugin, status, dlstatus, package, owner) VALUES(?,?,?,1,3,?,?)', - (url, name, plugin, package, owner)) - return self.c.lastrowid - - @async - def addLinks(self, links, package, owner): - """ links is a list of tuples (url, plugin)""" - links = [(x[0], x[0], x[1], package, owner) for x in links] - self.c.executemany( - 'INSERT INTO files(url, name, plugin, status, dlstatus, package, owner) VALUES(?,?,?,1,3,?,?)', - links) - - @queue - def addFile(self, name, size, media, package, owner): - # file status - ok, dl status NA - self.c.execute('INSERT INTO files(name, size, media, package, owner) VALUES(?,?,?,?,?)', - (name, size, media, package, owner)) - return self.c.lastrowid - - @queue - def addPackage(self, name, folder, root, password, site, comment, status, owner): - self.c.execute( - 'INSERT INTO packages(name, folder, root, password, site, comment, status, owner) VALUES(?,?,?,?,?,?,?,?)' - , (name, folder, root, password, site, comment, status, owner)) - return self.c.lastrowid - - @async - def deletePackage(self, pid, owner=None): - # order updated by trigger, as well as links deleted - if owner is None: - self.c.execute('DELETE FROM packages WHERE pid=?', (pid,)) - else: - self.c.execute('DELETE FROM packages WHERE pid=? AND owner=?', (pid, owner)) - - @async - def deleteFile(self, fid, order, package, owner=None): - """ To delete a file order and package of it is needed """ - if owner is None: - self.c.execute('DELETE FROM files WHERE fid=?', (fid,)) - self.c.execute('UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=?', - (order, package)) - else: - self.c.execute('DELETE FROM files WHERE fid=? AND owner=?', (fid, owner)) - self.c.execute('UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=? AND owner=?', - (order, package, owner)) - - @async - def saveCollector(self, owner, data): - """ simply save the json string to database """ - self.c.execute("INSERT INTO collector(owner, data) VALUES (?,?)", (owner, data)) - - @queue - def retrieveCollector(self, owner): - """ retrieve the saved string """ - self.c.execute('SELECT data FROM collector WHERE owner=?', (owner,)) - r = self.c.fetchone() - if not r: return None - return r[0] - - @async - def deleteCollector(self, owner): - """ drop saved user collector """ - self.c.execute('DELETE FROM collector WHERE owner=?', (owner,)) - - @queue - def getAllFiles(self, package=None, search=None, state=None, owner=None): - """ Return dict with file information - - :param package: optional package to filter out - :param search: or search string for file name - :param unfinished: filter by dlstatus not finished - :param owner: only specific owner - """ - qry = ('SELECT fid, name, owner, size, status, media, added, fileorder, ' - 'url, plugin, hash, dlstatus, error, package FROM files WHERE ') - - arg = [] - - if state is not None and state != DS.All: - qry += 'dlstatus IN (%s) AND ' % state_string(state) - if owner is not None: - qry += 'owner=? AND ' - arg.append(owner) - - if package is not None: - arg.append(package) - qry += 'package=? AND ' - if search is not None: - search = "%%%s%%" % search.strip("%") - arg.append(search) - qry += "name LIKE ? " - - # make qry valid - if qry.endswith("WHERE "): qry = qry[:-6] - if qry.endswith("AND "): qry = qry[:-4] - - self.c.execute(qry + "ORDER BY package, fileorder", arg) - - data = OrderedDict() - for r in self.c: - f = FileInfo(r[0], r[1], r[13], r[2], r[3], r[4], r[5], r[6], r[7]) - if r[11] > 0: # dl status != NA - f.download = DownloadInfo(r[8], r[9], r[10], r[11], self.manager.statusMsg[r[11]], r[12]) - - data[r[0]] = f - - return data - - @queue - def getMatchingFilenames(self, pattern, owner=None): - """ Return matching file names for pattern, useful for search suggestions """ - qry = 'SELECT name FROM files WHERE name LIKE ?' - args = ["%%%s%%" % pattern.strip("%")] - if owner: - qry += " AND owner=?" - args.append(owner) - - self.c.execute(qry, args) - return [r[0] for r in self.c] - - @queue - def getAllPackages(self, root=None, owner=None, tags=None): - """ Return dict with package information - - :param root: optional root to filter - :param owner: optional user id - :param tags: optional tag list - """ - qry = ( - 'SELECT pid, name, folder, root, owner, site, comment, password, added, tags, status, shared, packageorder ' - 'FROM packages%s ORDER BY root, packageorder') - - if root is None: - stats = self.getPackageStats(owner=owner) - if owner is None: - self.c.execute(qry % "") - else: - self.c.execute(qry % " WHERE owner=?", (owner,)) - else: - stats = self.getPackageStats(root=root, owner=owner) - if owner is None: - self.c.execute(qry % ' WHERE root=? OR pid=?', (root, root)) - else: - self.c.execute(qry % ' WHERE (root=? OR pid=?) AND owner=?', (root, root, owner)) - - data = OrderedDict() - for r in self.c: - data[r[0]] = PackageInfo( - r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9].split(","), r[10], r[11], r[12], - stats.get(r[0], zero_stats) - ) - - return data - - @inner - def getPackageStats(self, pid=None, root=None, owner=None): - qry = ("SELECT p.pid, SUM(f.size) AS sizetotal, COUNT(f.fid) AS linkstotal, sizedone, linksdone " - "FROM packages p JOIN files f ON p.pid = f.package AND f.dlstatus > 0 %(sub)s LEFT OUTER JOIN " - "(SELECT p.pid AS pid, SUM(f.size) AS sizedone, COUNT(f.fid) AS linksdone " - "FROM packages p JOIN files f ON p.pid = f.package %(sub)s AND f.dlstatus in (5,6) GROUP BY p.pid) s ON s.pid = p.pid " - "GROUP BY p.pid") - - # status in (finished, skipped, processing) - - if root is not None: - self.c.execute(qry % {"sub": "AND (p.root=:root OR p.pid=:root)"}, locals()) - elif pid is not None: - self.c.execute(qry % {"sub": "AND p.pid=:pid"}, locals()) - elif owner is not None: - self.c.execute(qry % {"sub": "AND p.owner=:owner"}, locals()) - else: - self.c.execute(qry % {"sub": ""}) - - data = {} - for r in self.c: - data[r[0]] = PackageStats( - r[2] if r[2] else 0, - r[4] if r[4] else 0, - int(r[1]) if r[1] else 0, - int(r[3]) if r[3] else 0, - ) - - return data - - @queue - def getStatsForPackage(self, pid): - return self.getPackageStats(pid=pid)[pid] - - @queue - def getFileInfo(self, fid, force=False): - """get data for specific file, when force is true download info will be appended""" - self.c.execute('SELECT fid, name, owner, size, status, media, added, fileorder, ' - 'url, plugin, hash, dlstatus, error, package FROM files ' - 'WHERE fid=?', (fid,)) - r = self.c.fetchone() - if not r: - return None - else: - f = FileInfo(r[0], r[1], r[13], r[2], r[3], r[4], r[5], r[6], r[7]) - if r[11] > 0 or force: - f.download = DownloadInfo(r[8], r[9], r[10], r[11], self.manager.statusMsg[r[11]], r[12]) - - return f - - @queue - def getPackageInfo(self, pid, stats=True): - """get data for a specific package, optionally with package stats""" - if stats: - stats = self.getPackageStats(pid=pid) - - self.c.execute( - 'SELECT pid, name, folder, root, owner, site, comment, password, added, tags, status, shared, packageorder ' - 'FROM packages WHERE pid=?', (pid,)) - - r = self.c.fetchone() - if not r: - return None - else: - return PackageInfo( - r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9].split(","), r[10], r[11], r[12], - stats.get(r[0], zero_stats) if stats else None - ) - - # TODO: does this need owner? - @async - def updateLinkInfo(self, data): - """ data is list of tuples (name, size, status,[ hash,] url)""" - # status in (NA, Offline, Online, Queued, TempOffline) - if data and len(data[0]) == 4: - self.c.executemany('UPDATE files SET name=?, size=?, dlstatus=? WHERE url=? AND dlstatus IN (0,1,2,3,11)', - data) - else: - self.c.executemany( - 'UPDATE files SET name=?, size=?, dlstatus=?, hash=? WHERE url=? AND dlstatus IN (0,1,2,3,11)', data) - - @async - def updateFile(self, f): - self.c.execute('UPDATE files SET name=?, size=?, status=?,' - 'media=?, url=?, hash=?, dlstatus=?, error=? WHERE fid=?', - (f.name, f.size, f.filestatus, f.media, f.url, - f.hash, f.status, f.error, f.fid)) - - @async - def updatePackage(self, p): - self.c.execute( - 'UPDATE packages SET name=?, folder=?, site=?, comment=?, password=?, tags=?, status=?, shared=? WHERE pid=?', - (p.name, p.folder, p.site, p.comment, p.password, ",".join(p.tags), p.status, p.shared, p.pid)) - - # TODO: most modifying methods needs owner argument to avoid checking beforehand - @async - def orderPackage(self, pid, root, oldorder, order): - if oldorder > order: # package moved upwards - self.c.execute( - 'UPDATE packages SET packageorder=packageorder+1 WHERE packageorder >= ? AND packageorder < ? AND root=? AND packageorder >= 0' - , (order, oldorder, root)) - elif oldorder < order: # moved downwards - self.c.execute( - 'UPDATE packages SET packageorder=packageorder-1 WHERE packageorder <= ? AND packageorder > ? AND root=? AND packageorder >= 0' - , (order, oldorder, root)) - - self.c.execute('UPDATE packages SET packageorder=? WHERE pid=?', (order, pid)) - - @async - def orderFiles(self, pid, fids, oldorder, order): - diff = len(fids) - data = [] - - if oldorder > order: # moved upwards - self.c.execute('UPDATE files SET fileorder=fileorder+? WHERE fileorder >= ? AND fileorder < ? AND package=?' - , (diff, order, oldorder, pid)) - data = [(order + i, fid) for i, fid in enumerate(fids)] - elif oldorder < order: - self.c.execute( - 'UPDATE files SET fileorder=fileorder-? WHERE fileorder <= ? AND fileorder >= ? AND package=?' - , (diff, order, oldorder + diff, pid)) - data = [(order - diff + i + 1, fid) for i, fid in enumerate(fids)] - - self.c.executemany('UPDATE files SET fileorder=? WHERE fid=?', data) - - @async - def moveFiles(self, pid, fids, package): - self.c.execute('SELECT max(fileorder) FROM files WHERE package=?', (package,)) - r = self.c.fetchone() - order = (r[0] if r[0] else 0) + 1 - - self.c.execute('UPDATE files SET fileorder=fileorder-? WHERE fileorder > ? AND package=?', - (len(fids), order, pid)) - - data = [(package, order + i, fid) for i, fid in enumerate(fids)] - self.c.executemany('UPDATE files SET package=?, fileorder=? WHERE fid=?', data) - - @async - def movePackage(self, root, order, pid, dpid): - self.c.execute('SELECT max(packageorder) FROM packages WHERE root=?', (dpid,)) - r = self.c.fetchone() - max = (r[0] if r[0] else 0) + 1 - - self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND root=?', - (order, root)) - - self.c.execute('UPDATE packages SET root=?, packageorder=? WHERE pid=?', (dpid, max, pid)) - - @async - def restartFile(self, fid): - # status -> queued - self.c.execute('UPDATE files SET dlstatus=3, error="" WHERE fid=?', (fid,)) - - @async - def restartPackage(self, pid): - # status -> queued - self.c.execute('UPDATE files SET status=3 WHERE package=?', (pid,)) - - - # TODO: multi user approach - @queue - def getJob(self, occ): - """return pyfile ids, which are suitable for download and don't use a occupied plugin""" - cmd = "(%s)" % ", ".join(["'%s'" % x for x in occ]) - #TODO - - # dlstatus in online, queued | package status = ok - cmd = ("SELECT f.fid FROM files as f INNER JOIN packages as p ON f.package=p.pid " - "WHERE f.plugin NOT IN %s AND f.dlstatus IN (2,3) AND p.status=0 " - "ORDER BY p.packageorder ASC, f.fileorder ASC LIMIT 5") % cmd - - self.c.execute(cmd) - - return [x[0] for x in self.c] - - @queue - def getUnfinished(self, pid): - """return list of max length 3 ids with pyfiles in package not finished or processed""" - - # status in finished, skipped, processing - self.c.execute("SELECT fid FROM files WHERE package=? AND dlstatus NOT IN (5, 6, 14) LIMIT 3", (pid,)) - return [r[0] for r in self.c] - - @queue - def restartFailed(self, owner): - # status=queued, where status in failed, aborted, temp offline - self.c.execute("UPDATE files SET dlstatus=3, error='' WHERE dlstatus IN (7, 11, 12)") - - @queue - def findDuplicates(self, id, folder, filename): - """ checks if filename exists with different id and same package, dlstatus = finished """ - # TODO: also check root of package - self.c.execute( - "SELECT f.plugin FROM files f INNER JOIN packages as p ON f.package=p.pid AND p.folder=? WHERE f.fid!=? AND f.dlstatus=5 AND f.name=?" - , (folder, id, filename)) - return self.c.fetchone() - - @queue - def purgeLinks(self): - # fstatus = missing - self.c.execute("DELETE FROM files WHERE status == 1") - - @queue - def purgeAll(self): # only used for debugging - self.c.execute("DELETE FROM packages") - self.c.execute("DELETE FROM files") - self.c.execute("DELETE FROM collector") - - -FileMethods.register() \ No newline at end of file diff --git a/module/database/StatisticDatabase.py b/module/database/StatisticDatabase.py deleted file mode 100644 index 10619eb5b..000000000 --- a/module/database/StatisticDatabase.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.database import DatabaseMethods, queue, async, inner - -# TODO - -class StatisticMethods(DatabaseMethods): - pass - - - -StatisticMethods.register() \ No newline at end of file diff --git a/module/database/StorageDatabase.py b/module/database/StorageDatabase.py deleted file mode 100644 index ffaf51763..000000000 --- a/module/database/StorageDatabase.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" - -from module.database import DatabaseBackend, queue - -class StorageMethods(): - @queue - def setStorage(db, identifier, key, value): - db.c.execute("SELECT id FROM storage WHERE identifier=? AND key=?", (identifier, key)) - if db.c.fetchone() is not None: - db.c.execute("UPDATE storage SET value=? WHERE identifier=? AND key=?", (value, identifier, key)) - else: - db.c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", (identifier, key, value)) - - @queue - def getStorage(db, identifier, key=None): - if key is not None: - db.c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", (identifier, key)) - row = db.c.fetchone() - if row is not None: - return row[0] - else: - db.c.execute("SELECT key, value FROM storage WHERE identifier=?", (identifier, )) - d = {} - for row in db.c: - d[row[0]] = row[1] - return d - - @queue - def delStorage(db, identifier, key): - db.c.execute("DELETE FROM storage WHERE identifier=? AND key=?", (identifier, key)) - -DatabaseBackend.registerSub(StorageMethods) diff --git a/module/database/UserDatabase.py b/module/database/UserDatabase.py deleted file mode 100644 index 0df94e0eb..000000000 --- a/module/database/UserDatabase.py +++ /dev/null @@ -1,125 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -from hashlib import sha1 -from string import letters, digits -from random import choice - -alphnum = letters+digits - -from module.Api import UserData - -from DatabaseBackend import DatabaseMethods, queue, async - -def random_salt(): - return "".join(choice(alphnum) for x in range(0,5)) - -class UserMethods(DatabaseMethods): - - @queue - def addUser(self, user, password): - salt = random_salt() - h = sha1(salt + password) - password = salt + h.hexdigest() - - self.c.execute('SELECT name FROM users WHERE name=?', (user, )) - if self.c.fetchone() is not None: - self.c.execute('UPDATE users SET password=? WHERE name=?', (password, user)) - else: - self.c.execute('INSERT INTO users (name, password) VALUES (?, ?)', (user, password)) - - @queue - def getUserData(self, name=None, uid=None): - qry = ('SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' - 'hddquota, user, template FROM "users" WHERE ') - - if name is not None: - self.c.execute(qry + "name=?", (name,)) - r = self.c.fetchone() - if r: - return UserData(*r) - - elif uid is not None: - self.c.execute(qry + "uid=?", (uid,)) - r = self.c.fetchone() - if r: - return UserData(*r) - - return None - - @queue - def getAllUserData(self): - self.c.execute('SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' - 'hddquota, user, template FROM "users"') - user = {} - for r in self.c: - user[r[0]] = UserData(*r) - - return user - - - @queue - def checkAuth(self, user, password): - self.c.execute('SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' - 'hddquota, user, template, password FROM "users" WHERE name=?', (user, )) - r = self.c.fetchone() - if not r: - return None - salt = r[-1][:5] - pw = r[-1][5:] - h = sha1(salt + password) - if h.hexdigest() == pw: - return UserData(*r[:-1]) - else: - return None - - @queue #TODO - def changePassword(self, user, oldpw, newpw): - self.c.execute('SELECT rowid, name, password FROM users WHERE name=?', (user, )) - r = self.c.fetchone() - if not r: - return False - - salt = r[2][:5] - pw = r[2][5:] - h = sha1(salt + oldpw) - if h.hexdigest() == pw: - salt = random_salt() - h = sha1(salt + newpw) - password = salt + h.hexdigest() - - self.c.execute("UPDATE users SET password=? WHERE name=?", (password, user)) - return True - - return False - - @async - def setPermission(self, user, perms): - self.c.execute("UPDATE users SET permission=? WHERE name=?", (perms, user)) - - @async - def setRole(self, user, role): - self.c.execute("UPDATE users SET role=? WHERE name=?", (role, user)) - - # TODO update methods - - @async - def removeUser(self, uid=None): - # deletes user and all associated accounts - self.c.execute('DELETE FROM users WHERE user=?', (uid, )) - -UserMethods.register() diff --git a/module/database/__init__.py b/module/database/__init__.py deleted file mode 100644 index d3f97fb53..000000000 --- a/module/database/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from DatabaseBackend import DatabaseMethods, DatabaseBackend, queue, async, inner - -from FileDatabase import FileMethods -from UserDatabase import UserMethods -from StorageDatabase import StorageMethods -from AccountDatabase import AccountMethods -from ConfigDatabase import ConfigMethods -from StatisticDatabase import StatisticMethods \ No newline at end of file diff --git a/module/datatypes/PyFile.py b/module/datatypes/PyFile.py deleted file mode 100644 index 5f2be8769..000000000 --- a/module/datatypes/PyFile.py +++ /dev/null @@ -1,268 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -from time import sleep, time -from ReadWriteLock import ReadWriteLock - -from module.Api import ProgressInfo, DownloadProgress, FileInfo, DownloadInfo, DownloadStatus -from module.utils import lock, read_lock - -statusMap = { - "none": 0, - "offline": 1, - "online": 2, - "queued": 3, - "paused": 4, - "finished": 5, - "skipped": 6, - "failed": 7, - "starting": 8, - "waiting": 9, - "downloading": 10, - "temp. offline": 11, - "aborted": 12, - "decrypting": 13, - "processing": 14, - "custom": 15, - "unknown": 16, -} - - -class PyFile(object): - """ - Represents a file object at runtime - """ - __slots__ = ("m", "fid", "_name", "_size", "filestatus", "media", "added", "fileorder", - "url", "pluginname", "hash", "status", "error", "packageid", "ownerid", - "lock", "plugin", "waitUntil", "abort", "statusname", - "reconnected", "pluginclass") - - @staticmethod - def fromInfoData(m, info): - f = PyFile(m, info.fid, info.name, info.size, info.status, info.media, info.added, info.fileorder, - "", "", "", DownloadStatus.NA, "", info.package, info.owner) - if info.download: - f.url = info.download.url - f.pluginname = info.download.plugin - f.hash = info.download.hash - f.status = info.download.status - f.error = info.download.error - - return f - - def __init__(self, manager, fid, name, size, filestatus, media, added, fileorder, - url, pluginname, hash, status, error, package, owner): - - self.m = manager - - self.fid = int(fid) - self._name = name - self._size = size - self.filestatus = filestatus - self.media = media - self.added = added - self.fileorder = fileorder - self.url = url - self.pluginname = pluginname - self.hash = hash - self.status = status - self.error = error - self.ownerid = owner - self.packageid = package - # database information ends here - - self.lock = ReadWriteLock() - - self.plugin = None - - self.waitUntil = 0 # time() + time to wait - - # status attributes - self.abort = False - self.reconnected = False - self.statusname = None - - - @property - def id(self): - self.m.core.log.debug("Deprecated attr .id, use .fid instead") - return self.fid - - def setSize(self, value): - self._size = int(value) - - # will convert all sizes to ints - size = property(lambda self: self._size, setSize) - - def getName(self): - try: - if self.plugin.req.name: - return self.plugin.req.name - else: - return self._name - except: - return self._name - - def setName(self, name): - """ Only set unicode or utf8 strings as name """ - if type(name) == str: - name = name.decode("utf8") - - self._name = name - - name = property(getName, setName) - - def __repr__(self): - return "" % (self.id, self.name, self.pluginname) - - @lock - def initPlugin(self): - """ inits plugin instance """ - if not self.plugin: - self.pluginclass = self.m.core.pluginManager.getPlugin(self.pluginname) - self.plugin = self.pluginclass(self) - - @read_lock - def hasPlugin(self): - """Thread safe way to determine this file has initialized plugin attribute""" - return hasattr(self, "plugin") and self.plugin - - def package(self): - """ return package instance""" - return self.m.getPackage(self.packageid) - - def setStatus(self, status): - self.status = statusMap[status] - # needs to sync so status is written to database - self.sync() - - def setCustomStatus(self, msg, status="processing"): - self.statusname = msg - self.setStatus(status) - - def getStatusName(self): - if self.status not in (13, 14) or not self.statusname: - return self.m.statusMsg[self.status] - else: - return self.statusname - - def hasStatus(self, status): - return statusMap[status] == self.status - - def sync(self): - """sync PyFile instance with database""" - self.m.updateFile(self) - - @lock - def release(self): - """sync and remove from cache""" - if hasattr(self, "plugin") and self.plugin: - self.plugin.clean() - del self.plugin - - self.m.releaseFile(self.fid) - - - def toInfoData(self): - return FileInfo(self.fid, self.getName(), self.packageid, self.ownerid, self.getSize(), self.filestatus, - self.media, self.added, self.fileorder, DownloadInfo( - self.url, self.pluginname, self.hash, self.status, self.getStatusName(), self.error - ) - ) - - def getPath(self): - pass - - def move(self, pid): - pass - - @read_lock - def abortDownload(self): - """abort pyfile if possible""" - # TODO: abort timeout, currently dead locks - while self.id in self.m.core.threadManager.processingIds(): - self.abort = True - if self.plugin and self.plugin.req: - self.plugin.req.abortDownloads() - sleep(0.1) - - self.abort = False - if self.plugin and self.plugin.req: - self.plugin.req.abortDownloads() - - self.release() - - def finishIfDone(self): - """set status to finish and release file if every thread is finished with it""" - - if self.id in self.m.core.threadManager.processingIds(): - return False - - self.setStatus("finished") - self.release() - self.m.checkAllLinksFinished() - return True - - def checkIfProcessed(self): - self.m.checkAllLinksProcessed(self.id) - - def getSpeed(self): - """ calculates speed """ - try: - return self.plugin.req.speed - except: - return 0 - - def getETA(self): - """ gets established time of arrival / or waiting time""" - try: - if self.status == DownloadStatus.Waiting: - return self.waitUntil - time() - - return self.getBytesLeft() / self.getSpeed() - except: - return 0 - - def getBytesArrived(self): - """ gets bytes arrived """ - try: - return self.plugin.req.arrived - except: - return 0 - - def getBytesLeft(self): - """ gets bytes left """ - try: - return self.plugin.req.size - self.plugin.req.arrived - except: - return 0 - - def getSize(self): - """ get size of download """ - try: - if self.plugin.req.size: - return self.plugin.req.size - else: - return self.size - except: - return self.size - - def getProgressInfo(self): - return ProgressInfo(self.pluginname, self.name, self.getStatusName(), self.getETA(), - self.getBytesArrived(), self.getSize(), - DownloadProgress(self.fid, self.packageid, self.getSpeed(), self.status)) diff --git a/module/datatypes/PyPackage.py b/module/datatypes/PyPackage.py deleted file mode 100644 index 6ba37ee12..000000000 --- a/module/datatypes/PyPackage.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -from time import time - -from module.Api import PackageInfo, PackageStatus -from module.utils.fs import join - -class PyPackage: - """ - Represents a package object at runtime - """ - - @staticmethod - def fromInfoData(m, info): - return PyPackage(m, info.pid, info.name, info.folder, info.root, info.owner, - info.site, info.comment, info.password, info.added, info.tags, info.status, info.shared, info.packageorder) - - def __init__(self, manager, pid, name, folder, root, owner, site, comment, password, added, tags, status, - shared, packageorder): - self.m = manager - - self.pid = pid - self.name = name - self.folder = folder - self.root = root - self.ownerid = owner - self.site = site - self.comment = comment - self.password = password - self.added = added - self.tags = tags - self.status = status - self.shared = shared - self.packageorder = packageorder - self.timestamp = time() - - #: Finish event already fired - self.setFinished = False - - @property - def id(self): - self.m.core.log.debug("Deprecated package attr .id, use .pid instead") - return self.pid - - def isStale(self): - return self.timestamp + 30 * 60 > time() - - def toInfoData(self): - return PackageInfo(self.pid, self.name, self.folder, self.root, self.ownerid, self.site, - self.comment, self.password, self.added, self.tags, self.status, self.shared, self.packageorder - ) - - def getChildren(self): - """get information about contained links""" - return self.m.getPackageData(self.pid)["links"] - - def getPath(self, name=""): - self.timestamp = time() - return join(self.m.getPackage(self.root).getPath(), self.folder, name) - - def sync(self): - """sync with db""" - self.m.updatePackage(self) - - def release(self): - """sync and delete from cache""" - self.sync() - self.m.releasePackage(self.id) - - def delete(self): - self.m.deletePackage(self.id) - - def deleteIfEmpty(self): - """ True if deleted """ - if not len(self.getChildren()): - self.delete() - return True - return False - - def notifyChange(self): - self.m.core.eventManager.dispatchEvent("packageUpdated", self.id) - - -class RootPackage(PyPackage): - def __init__(self, m, owner): - PyPackage.__init__(self, m, -1, "root", "", owner, -2, "", "", "", 0, [], PackageStatus.Ok, False, 0) - - def getPath(self, name=""): - return join(self.m.core.config["general"]["download_folder"], name) - - # no database operations - def sync(self): - pass - - def delete(self): - pass - - def release(self): - pass \ No newline at end of file diff --git a/module/datatypes/User.py b/module/datatypes/User.py deleted file mode 100644 index 141191df4..000000000 --- a/module/datatypes/User.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - - -from module.Api import UserData, Permission, Role -from module.utils import bits_set - -#TODO: activate user -#noinspection PyUnresolvedReferences -class User(UserData): - - @staticmethod - def fromUserData(api, user): - return User(api, user.uid, user.name, user.email, user.role, user.permission, user.folder, - user.traffic, user.dllimit, user.dlquota, user.hddquota, user.user, user.templateName) - - def __init__(self, api, *args, **kwargs): - UserData.__init__(self, *args, **kwargs) - self.api = api - - - def toUserData(self): - # TODO - return UserData() - - def hasPermission(self, perms): - """ Accepts permission bit or name """ - if isinstance(perms, basestring) and hasattr(Permission, perms): - perms = getattr(Permission, perms) - - return bits_set(perms, self.permission) - - def hasRole(self, role): - if isinstance(role, basestring) and hasattr(Role, role): - role = getattr(Role, role) - - return self.role == role - - def isAdmin(self): - return self.hasRole(Role.Admin) - - @property - def primary(self): - """ Primary user id, Internal user handle used for most operations - Secondary user account share id with primary user. Only Admins have no primary id. """ - if self.hasRole(Role.Admin): - return None - return self.user if self.user else self.uid \ No newline at end of file diff --git a/module/datatypes/__init__.py b/module/datatypes/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/module/interaction/EventManager.py b/module/interaction/EventManager.py deleted file mode 100644 index 7d37ca6b9..000000000 --- a/module/interaction/EventManager.py +++ /dev/null @@ -1,84 +0,0 @@ -# -*- coding: utf-8 -*- - -from threading import Lock -from traceback import print_exc - -class EventManager: - """ - Handles all event-related tasks, also stores an event queue for clients, so they can retrieve them later. - - **Known Events:** - Most addon methods exist as events. These are some additional known events. - - ===================== ================ =========================================================== - Name Arguments Description - ===================== ================ =========================================================== - event eventName, *args Called for every event, with eventName and original args - download:preparing fid A download was just queued and will be prepared now. - download:start fid A plugin will immediately start the download afterwards. - download:allProcessed All links were handled, pyLoad would idle afterwards. - download:allFinished All downloads in the queue are finished. - config:changed sec, opt, value The config was changed. - ===================== ================ =========================================================== - - | Notes: - | download:allProcessed is *always* called before download:allFinished. - """ - - def __init__(self, core): - self.core = core - self.log = core.log - - # uuid : list of events - self.clients = {} - self.events = {"event": []} - - self.lock = Lock() - - def getEvents(self, uuid): - """ Get accumulated events for uuid since last call, this also registers a new client """ - if uuid not in self.clients: - self.clients[uuid] = Client() - return self.clients[uuid].get() - - def addEvent(self, event, func): - """Adds an event listener for event name""" - if event in self.events: - if func in self.events[event]: - self.log.debug("Function already registered %s" % func) - else: - self.events[event].append(func) - else: - self.events[event] = [func] - - def removeEvent(self, event, func): - """removes previously added event listener""" - if event in self.events: - self.events[event].remove(func) - - def removeFromEvents(self, func): - """ Removes func from all known events """ - for name, events in self.events.iteritems(): - if func in events: - events.remove(func) - - def dispatchEvent(self, event, *args): - """dispatches event with args""" - for f in self.events["event"]: - try: - f(event, *args) - except Exception, e: - self.log.warning("Error calling event handler %s: %s, %s, %s" - % ("event", f, args, str(e))) - if self.core.debug: - print_exc() - - if event in self.events: - for f in self.events[event]: - try: - f(*args) - except Exception, e: - self.log.warning("Error calling event handler %s: %s, %s, %s" - % (event, f, args, str(e))) - if self.core.debug: - print_exc() \ No newline at end of file diff --git a/module/interaction/InteractionManager.py b/module/interaction/InteractionManager.py deleted file mode 100644 index e4ae05501..000000000 --- a/module/interaction/InteractionManager.py +++ /dev/null @@ -1,166 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" -from threading import Lock -from time import time -from base64 import standard_b64encode - -from new_collections import OrderedDict - -from module.utils import lock, bits_set -from module.Api import Interaction as IA -from module.Api import InputType, Input - -from InteractionTask import InteractionTask - -class InteractionManager: - """ - Class that gives ability to interact with the user. - Arbitrary tasks with predefined output and input types can be set off. - """ - - # number of seconds a client is classified as active - CLIENT_THRESHOLD = 60 - NOTIFICATION_TIMEOUT = 60 * 60 * 30 - MAX_NOTIFICATIONS = 50 - - def __init__(self, core): - self.lock = Lock() - self.core = core - self.tasks = OrderedDict() #task store, for all outgoing tasks - - self.last_clients = {} - self.ids = 0 #uniue interaction ids - - def isClientConnected(self, user): - return self.last_clients.get(user, 0) + self.CLIENT_THRESHOLD > time() - - @lock - def work(self): - # old notifications will be removed - for n in [k for k, v in self.tasks.iteritems() if v.timedOut()]: - del self.tasks[n] - - # keep notifications count limited - n = [k for k,v in self.tasks.iteritems() if v.type == IA.Notification] - n.reverse() - for v in n[:self.MAX_NOTIFICATIONS]: - del self.tasks[v] - - @lock - def createNotification(self, title, content, desc="", plugin="", owner=None): - """ Creates and queues a new Notification - - :param title: short title - :param content: text content - :param desc: short form of the notification - :param plugin: plugin name - :return: :class:`InteractionTask` - """ - task = InteractionTask(self.ids, IA.Notification, Input(InputType.Text, content), "", title, desc, plugin, - owner=owner) - self.ids += 1 - self.queueTask(task) - return task - - @lock - def createQueryTask(self, input, desc, default="", plugin="", owner=None): - # input type was given, create a input widget - if type(input) == int: - input = Input(input) - if not isinstance(input, Input): - raise TypeError("'Input' class expected not '%s'" % type(input)) - - task = InteractionTask(self.ids, IA.Query, input, default, _("Query"), desc, plugin, owner=owner) - self.ids += 1 - self.queueTask(task) - return task - - @lock - def createCaptchaTask(self, img, format, filename, plugin="", type=InputType.Text, owner=None): - """ Createss a new captcha task. - - :param img: image content (not base encoded) - :param format: img format - :param type: :class:`InputType` - :return: - """ - if type == 'textual': - type = InputType.Text - elif type == 'positional': - type = InputType.Click - - input = Input(type, [standard_b64encode(img), format, filename]) - - #todo: title desc plugin - task = InteractionTask(self.ids, IA.Captcha, input, - None, _("Captcha request"), _("Please solve the captcha."), plugin, owner=owner) - - self.ids += 1 - self.queueTask(task) - return task - - @lock - def removeTask(self, task): - if task.iid in self.tasks: - del self.tasks[task.iid] - self.core.evm.dispatchEvent("interaction:deleted", task.iid) - - @lock - def getTaskByID(self, iid): - return self.tasks.get(iid, None) - - @lock - def getTasks(self, user, mode=IA.All): - # update last active clients - self.last_clients[user] = time() - - # filter current mode - tasks = [t for t in self.tasks.itervalues() if mode == IA.All or bits_set(t.type, mode)] - # filter correct user / or shared - tasks = [t for t in tasks if user is None or user == t.owner or t.shared] - - return tasks - - def isTaskWaiting(self, user, mode=IA.All): - tasks = [t for t in self.getTasks(user, mode) if not t.type == IA.Notification or not t.seen] - return len(tasks) > 0 - - def queueTask(self, task): - cli = self.isClientConnected(task.owner) - - # set waiting times based on threshold - if cli: - task.setWaiting(self.CLIENT_THRESHOLD) - else: # TODO: higher threshold after client connects? - task.setWaiting(self.CLIENT_THRESHOLD / 3) - - if task.type == IA.Notification: - task.setWaiting(self.NOTIFICATION_TIMEOUT) # notifications are valid for 30h - - for plugin in self.core.addonManager.activePlugins(): - try: - plugin.newInteractionTask(task) - except: - self.core.print_exc() - - self.tasks[task.iid] = task - self.core.evm.dispatchEvent("interaction:added", task) - - -if __name__ == "__main__": - it = InteractionTask() \ No newline at end of file diff --git a/module/interaction/InteractionTask.py b/module/interaction/InteractionTask.py deleted file mode 100644 index d2877b2b0..000000000 --- a/module/interaction/InteractionTask.py +++ /dev/null @@ -1,100 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" - -from time import time - -from module.Api import InteractionTask as BaseInteractionTask -from module.Api import Interaction, InputType, Input - -#noinspection PyUnresolvedReferences -class InteractionTask(BaseInteractionTask): - """ - General Interaction Task extends ITask defined by api with additional fields and methods. - """ - #: Plugins can put needed data here - storage = None - #: Timestamp when task expires - wait_until = 0 - #: The received result - result = None - #: List of registered handles - handler = None - #: Error Message - error = None - #: Timeout locked - locked = False - #: A task that was retrieved counts as seen - seen = False - #: A task that is relevant to every user - shared = False - #: primary uid of the owner - owner = None - - def __init__(self, *args, **kwargs): - if 'owner' in kwargs: - self.owner = kwargs['owner'] - del kwargs['owner'] - if 'shared' in kwargs: - self.shared = kwargs['shared'] - del kwargs['shared'] - - BaseInteractionTask.__init__(self, *args, **kwargs) - - # additional internal attributes - self.storage = {} - self.handler = [] - self.wait_until = 0 - - def convertResult(self, value): - #TODO: convert based on input/output - return value - - def getResult(self): - return self.result - - def setShared(self): - """ enable shared mode, should not be reversed""" - self.shared = True - - def setResult(self, value): - self.result = self.convertResult(value) - - def setWaiting(self, sec, lock=False): - """ sets waiting in seconds from now, < 0 can be used as infinitive """ - if not self.locked: - if sec < 0: - self.wait_until = -1 - else: - self.wait_until = max(time() + sec, self.wait_until) - - if lock: self.locked = True - - def isWaiting(self): - if self.result or self.error or self.timedOut(): - return False - - return True - - def timedOut(self): - return time() > self.wait_until > -1 - - def correct(self): - [x.taskCorrect(self) for x in self.handler] - - def invalid(self): - [x.taskInvalid(self) for x in self.handler] \ No newline at end of file diff --git a/module/interaction/__init__.py b/module/interaction/__init__.py deleted file mode 100644 index de6d13128..000000000 --- a/module/interaction/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -__author__ = 'christian' - \ No newline at end of file diff --git a/module/lib/Getch.py b/module/lib/Getch.py deleted file mode 100644 index 22b7ea7f8..000000000 --- a/module/lib/Getch.py +++ /dev/null @@ -1,76 +0,0 @@ -class Getch: - """ - Gets a single character from standard input. Does not echo to - the screen. - """ - - def __init__(self): - try: - self.impl = _GetchWindows() - except ImportError: - try: - self.impl = _GetchMacCarbon() - except(AttributeError, ImportError): - self.impl = _GetchUnix() - - def __call__(self): return self.impl() - - -class _GetchUnix: - def __init__(self): - import tty - import sys - - def __call__(self): - import sys - import tty - import termios - - fd = sys.stdin.fileno() - old_settings = termios.tcgetattr(fd) - try: - tty.setraw(sys.stdin.fileno()) - ch = sys.stdin.read(1) - finally: - termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) - return ch - - -class _GetchWindows: - def __init__(self): - import msvcrt - - def __call__(self): - import msvcrt - - return msvcrt.getch() - -class _GetchMacCarbon: - """ - A function which returns the current ASCII key that is down; - if no ASCII key is down, the null string is returned. The - page http://www.mactech.com/macintosh-c/chap02-1.html was - very helpful in figuring out how to do this. - """ - - def __init__(self): - import Carbon - Carbon.Evt #see if it has this (in Unix, it doesn't) - - def __call__(self): - import Carbon - - if Carbon.Evt.EventAvail(0x0008)[0] == 0: # 0x0008 is the keyDownMask - return '' - else: - # - # The event contains the following info: - # (what,msg,when,where,mod)=Carbon.Evt.GetNextEvent(0x0008)[1] - # - # The message (msg) contains the ASCII char which is - # extracted with the 0x000000FF charCodeMask; this - # number is converted to an ASCII character with chr() and - # returned - # - (what, msg, when, where, mod) = Carbon.Evt.GetNextEvent(0x0008)[1] - return chr(msg) \ No newline at end of file diff --git a/module/lib/ReadWriteLock.py b/module/lib/ReadWriteLock.py deleted file mode 100644 index cc82f3d48..000000000 --- a/module/lib/ReadWriteLock.py +++ /dev/null @@ -1,232 +0,0 @@ -# -*- coding: iso-8859-15 -*- -"""locks.py - Read-Write lock thread lock implementation - -See the class documentation for more info. - -Copyright (C) 2007, Heiko Wundram. -Released under the BSD-license. - -http://code.activestate.com/recipes/502283-read-write-lock-class-rlock-like/ -""" - -# Imports -# ------- - -from threading import Condition, Lock, currentThread -from time import time - - -# Read write lock -# --------------- - -class ReadWriteLock(object): - """Read-Write lock class. A read-write lock differs from a standard - threading.RLock() by allowing multiple threads to simultaneously hold a - read lock, while allowing only a single thread to hold a write lock at the - same point of time. - - When a read lock is requested while a write lock is held, the reader - is blocked; when a write lock is requested while another write lock is - held or there are read locks, the writer is blocked. - - Writers are always preferred by this implementation: if there are blocked - threads waiting for a write lock, current readers may request more read - locks (which they eventually should free, as they starve the waiting - writers otherwise), but a new thread requesting a read lock will not - be granted one, and block. This might mean starvation for readers if - two writer threads interweave their calls to acquireWrite() without - leaving a window only for readers. - - In case a current reader requests a write lock, this can and will be - satisfied without giving up the read locks first, but, only one thread - may perform this kind of lock upgrade, as a deadlock would otherwise - occur. After the write lock has been granted, the thread will hold a - full write lock, and not be downgraded after the upgrading call to - acquireWrite() has been match by a corresponding release(). - """ - - def __init__(self): - """Initialize this read-write lock.""" - - # Condition variable, used to signal waiters of a change in object - # state. - self.__condition = Condition(Lock()) - - # Initialize with no writers. - self.__writer = None - self.__upgradewritercount = 0 - self.__pendingwriters = [] - - # Initialize with no readers. - self.__readers = {} - - def acquire(self, blocking=True, timeout=None, shared=False): - if shared: - self.acquireRead(timeout) - else: - self.acquireWrite(timeout) - - def acquireRead(self, timeout=None): - """Acquire a read lock for the current thread, waiting at most - timeout seconds or doing a non-blocking check in case timeout is <= 0. - - In case timeout is None, the call to acquireRead blocks until the - lock request can be serviced. - - In case the timeout expires before the lock could be serviced, a - RuntimeError is thrown.""" - - if timeout is not None: - endtime = time() + timeout - me = currentThread() - self.__condition.acquire() - try: - if self.__writer is me: - # If we are the writer, grant a new read lock, always. - self.__writercount += 1 - return - while True: - if self.__writer is None: - # Only test anything if there is no current writer. - if self.__upgradewritercount or self.__pendingwriters: - if me in self.__readers: - # Only grant a read lock if we already have one - # in case writers are waiting for their turn. - # This means that writers can't easily get starved - # (but see below, readers can). - self.__readers[me] += 1 - return - # No, we aren't a reader (yet), wait for our turn. - else: - # Grant a new read lock, always, in case there are - # no pending writers (and no writer). - self.__readers[me] = self.__readers.get(me, 0) + 1 - return - if timeout is not None: - remaining = endtime - time() - if remaining <= 0: - # Timeout has expired, signal caller of this. - raise RuntimeError("Acquiring read lock timed out") - self.__condition.wait(remaining) - else: - self.__condition.wait() - finally: - self.__condition.release() - - def acquireWrite(self, timeout=None): - """Acquire a write lock for the current thread, waiting at most - timeout seconds or doing a non-blocking check in case timeout is <= 0. - - In case the write lock cannot be serviced due to the deadlock - condition mentioned above, a ValueError is raised. - - In case timeout is None, the call to acquireWrite blocks until the - lock request can be serviced. - - In case the timeout expires before the lock could be serviced, a - RuntimeError is thrown.""" - - if timeout is not None: - endtime = time() + timeout - me, upgradewriter = currentThread(), False - self.__condition.acquire() - try: - if self.__writer is me: - # If we are the writer, grant a new write lock, always. - self.__writercount += 1 - return - elif me in self.__readers: - # If we are a reader, no need to add us to pendingwriters, - # we get the upgradewriter slot. - if self.__upgradewritercount: - # If we are a reader and want to upgrade, and someone - # else also wants to upgrade, there is no way we can do - # this except if one of us releases all his read locks. - # Signal this to user. - raise ValueError( - "Inevitable dead lock, denying write lock" - ) - upgradewriter = True - self.__upgradewritercount = self.__readers.pop(me) - else: - # We aren't a reader, so add us to the pending writers queue - # for synchronization with the readers. - self.__pendingwriters.append(me) - while True: - if not self.__readers and self.__writer is None: - # Only test anything if there are no readers and writers. - if self.__upgradewritercount: - if upgradewriter: - # There is a writer to upgrade, and it's us. Take - # the write lock. - self.__writer = me - self.__writercount = self.__upgradewritercount + 1 - self.__upgradewritercount = 0 - return - # There is a writer to upgrade, but it's not us. - # Always leave the upgrade writer the advance slot, - # because he presumes he'll get a write lock directly - # from a previously held read lock. - elif self.__pendingwriters[0] is me: - # If there are no readers and writers, it's always - # fine for us to take the writer slot, removing us - # from the pending writers queue. - # This might mean starvation for readers, though. - self.__writer = me - self.__writercount = 1 - self.__pendingwriters = self.__pendingwriters[1:] - return - if timeout is not None: - remaining = endtime - time() - if remaining <= 0: - # Timeout has expired, signal caller of this. - if upgradewriter: - # Put us back on the reader queue. No need to - # signal anyone of this change, because no other - # writer could've taken our spot before we got - # here (because of remaining readers), as the test - # for proper conditions is at the start of the - # loop, not at the end. - self.__readers[me] = self.__upgradewritercount - self.__upgradewritercount = 0 - else: - # We were a simple pending writer, just remove us - # from the FIFO list. - self.__pendingwriters.remove(me) - raise RuntimeError("Acquiring write lock timed out") - self.__condition.wait(remaining) - else: - self.__condition.wait() - finally: - self.__condition.release() - - def release(self): - """Release the currently held lock. - - In case the current thread holds no lock, a ValueError is thrown.""" - - me = currentThread() - self.__condition.acquire() - try: - if self.__writer is me: - # We are the writer, take one nesting depth away. - self.__writercount -= 1 - if not self.__writercount: - # No more write locks; take our writer position away and - # notify waiters of the new circumstances. - self.__writer = None - self.__condition.notifyAll() - elif me in self.__readers: - # We are a reader currently, take one nesting depth away. - self.__readers[me] -= 1 - if not self.__readers[me]: - # No more read locks, take our reader position away. - del self.__readers[me] - if not self.__readers: - # No more readers, notify waiters of the new - # circumstances. - self.__condition.notifyAll() - else: - raise ValueError("Trying to release unheld lock") - finally: - self.__condition.release() diff --git a/module/lib/SafeEval.py b/module/lib/SafeEval.py deleted file mode 100644 index 8fc57f261..000000000 --- a/module/lib/SafeEval.py +++ /dev/null @@ -1,47 +0,0 @@ -## {{{ http://code.activestate.com/recipes/286134/ (r3) (modified) -import dis - -_const_codes = map(dis.opmap.__getitem__, [ - 'POP_TOP','ROT_TWO','ROT_THREE','ROT_FOUR','DUP_TOP', - 'BUILD_LIST','BUILD_MAP','BUILD_TUPLE', - 'LOAD_CONST','RETURN_VALUE','STORE_SUBSCR' - ]) - - -_load_names = ['False', 'True', 'null', 'true', 'false'] - -_locals = {'null': None, 'true': True, 'false': False} - -def _get_opcodes(codeobj): - i = 0 - opcodes = [] - s = codeobj.co_code - names = codeobj.co_names - while i < len(s): - code = ord(s[i]) - opcodes.append(code) - if code >= dis.HAVE_ARGUMENT: - i += 3 - else: - i += 1 - return opcodes, names - -def test_expr(expr, allowed_codes): - try: - c = compile(expr, "", "eval") - except: - raise ValueError, "%s is not a valid expression" % expr - codes, names = _get_opcodes(c) - for code in codes: - if code not in allowed_codes: - for n in names: - if n not in _load_names: - raise ValueError, "opcode %s not allowed" % dis.opname[code] - return c - - -def const_eval(expr): - c = test_expr(expr, _const_codes) - return eval(c, None, _locals) - -## end of http://code.activestate.com/recipes/286134/ }}} diff --git a/module/lib/__init__.py b/module/lib/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/module/lib/beaker/__init__.py b/module/lib/beaker/__init__.py deleted file mode 100644 index 792d60054..000000000 --- a/module/lib/beaker/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# diff --git a/module/lib/beaker/cache.py b/module/lib/beaker/cache.py deleted file mode 100644 index 4a96537ff..000000000 --- a/module/lib/beaker/cache.py +++ /dev/null @@ -1,459 +0,0 @@ -"""Cache object - -The Cache object is used to manage a set of cache files and their -associated backend. The backends can be rotated on the fly by -specifying an alternate type when used. - -Advanced users can add new backends in beaker.backends - -""" - -import warnings - -import beaker.container as container -import beaker.util as util -from beaker.exceptions import BeakerException, InvalidCacheBackendError - -import beaker.ext.memcached as memcached -import beaker.ext.database as database -import beaker.ext.sqla as sqla -import beaker.ext.google as google - -# Initialize the basic available backends -clsmap = { - 'memory':container.MemoryNamespaceManager, - 'dbm':container.DBMNamespaceManager, - 'file':container.FileNamespaceManager, - 'ext:memcached':memcached.MemcachedNamespaceManager, - 'ext:database':database.DatabaseNamespaceManager, - 'ext:sqla': sqla.SqlaNamespaceManager, - 'ext:google': google.GoogleNamespaceManager, - } - -# Initialize the cache region dict -cache_regions = {} -cache_managers = {} - -try: - import pkg_resources - - # Load up the additional entry point defined backends - for entry_point in pkg_resources.iter_entry_points('beaker.backends'): - try: - NamespaceManager = entry_point.load() - name = entry_point.name - if name in clsmap: - raise BeakerException("NamespaceManager name conflict,'%s' " - "already loaded" % name) - clsmap[name] = NamespaceManager - except (InvalidCacheBackendError, SyntaxError): - # Ignore invalid backends - pass - except: - import sys - from pkg_resources import DistributionNotFound - # Warn when there's a problem loading a NamespaceManager - if not isinstance(sys.exc_info()[1], DistributionNotFound): - import traceback - from StringIO import StringIO - tb = StringIO() - traceback.print_exc(file=tb) - warnings.warn("Unable to load NamespaceManager entry point: '%s': " - "%s" % (entry_point, tb.getvalue()), RuntimeWarning, - 2) -except ImportError: - pass - - - - -def cache_region(region, *deco_args): - """Decorate a function to cache itself using a cache region - - The region decorator requires arguments if there are more than - 2 of the same named function, in the same module. This is - because the namespace used for the functions cache is based on - the functions name and the module. - - - Example:: - - # Add cache region settings to beaker: - beaker.cache.cache_regions.update(dict_of_config_region_options)) - - @cache_region('short_term', 'some_data') - def populate_things(search_term, limit, offset): - return load_the_data(search_term, limit, offset) - - return load('rabbits', 20, 0) - - .. note:: - - The function being decorated must only be called with - positional arguments. - - """ - cache = [None] - - def decorate(func): - namespace = util.func_namespace(func) - def cached(*args): - reg = cache_regions[region] - if not reg.get('enabled', True): - return func(*args) - - if not cache[0]: - if region not in cache_regions: - raise BeakerException('Cache region not configured: %s' % region) - cache[0] = Cache._get_cache(namespace, reg) - - cache_key = " ".join(map(str, deco_args + args)) - def go(): - return func(*args) - - return cache[0].get_value(cache_key, createfunc=go) - cached._arg_namespace = namespace - cached._arg_region = region - return cached - return decorate - - -def region_invalidate(namespace, region, *args): - """Invalidate a cache region namespace or decorated function - - This function only invalidates cache spaces created with the - cache_region decorator. - - :param namespace: Either the namespace of the result to invalidate, or the - cached function reference - - :param region: The region the function was cached to. If the function was - cached to a single region then this argument can be None - - :param args: Arguments that were used to differentiate the cached - function as well as the arguments passed to the decorated - function - - Example:: - - # Add cache region settings to beaker: - beaker.cache.cache_regions.update(dict_of_config_region_options)) - - def populate_things(invalidate=False): - - @cache_region('short_term', 'some_data') - def load(search_term, limit, offset): - return load_the_data(search_term, limit, offset) - - # If the results should be invalidated first - if invalidate: - region_invalidate(load, None, 'some_data', - 'rabbits', 20, 0) - return load('rabbits', 20, 0) - - """ - if callable(namespace): - if not region: - region = namespace._arg_region - namespace = namespace._arg_namespace - - if not region: - raise BeakerException("Region or callable function " - "namespace is required") - else: - region = cache_regions[region] - - cache = Cache._get_cache(namespace, region) - cache_key = " ".join(str(x) for x in args) - cache.remove_value(cache_key) - - -class Cache(object): - """Front-end to the containment API implementing a data cache. - - :param namespace: the namespace of this Cache - - :param type: type of cache to use - - :param expire: seconds to keep cached data - - :param expiretime: seconds to keep cached data (legacy support) - - :param starttime: time when cache was cache was - - """ - def __init__(self, namespace, type='memory', expiretime=None, - starttime=None, expire=None, **nsargs): - try: - cls = clsmap[type] - if isinstance(cls, InvalidCacheBackendError): - raise cls - except KeyError: - raise TypeError("Unknown cache implementation %r" % type) - - self.namespace = cls(namespace, **nsargs) - self.expiretime = expiretime or expire - self.starttime = starttime - self.nsargs = nsargs - - @classmethod - def _get_cache(cls, namespace, kw): - key = namespace + str(kw) - try: - return cache_managers[key] - except KeyError: - cache_managers[key] = cache = cls(namespace, **kw) - return cache - - def put(self, key, value, **kw): - self._get_value(key, **kw).set_value(value) - set_value = put - - def get(self, key, **kw): - """Retrieve a cached value from the container""" - return self._get_value(key, **kw).get_value() - get_value = get - - def remove_value(self, key, **kw): - mycontainer = self._get_value(key, **kw) - if mycontainer.has_current_value(): - mycontainer.clear_value() - remove = remove_value - - def _get_value(self, key, **kw): - if isinstance(key, unicode): - key = key.encode('ascii', 'backslashreplace') - - if 'type' in kw: - return self._legacy_get_value(key, **kw) - - kw.setdefault('expiretime', self.expiretime) - kw.setdefault('starttime', self.starttime) - - return container.Value(key, self.namespace, **kw) - - @util.deprecated("Specifying a " - "'type' and other namespace configuration with cache.get()/put()/etc. " - "is deprecated. Specify 'type' and other namespace configuration to " - "cache_manager.get_cache() and/or the Cache constructor instead.") - def _legacy_get_value(self, key, type, **kw): - expiretime = kw.pop('expiretime', self.expiretime) - starttime = kw.pop('starttime', None) - createfunc = kw.pop('createfunc', None) - kwargs = self.nsargs.copy() - kwargs.update(kw) - c = Cache(self.namespace.namespace, type=type, **kwargs) - return c._get_value(key, expiretime=expiretime, createfunc=createfunc, - starttime=starttime) - - def clear(self): - """Clear all the values from the namespace""" - self.namespace.remove() - - # dict interface - def __getitem__(self, key): - return self.get(key) - - def __contains__(self, key): - return self._get_value(key).has_current_value() - - def has_key(self, key): - return key in self - - def __delitem__(self, key): - self.remove_value(key) - - def __setitem__(self, key, value): - self.put(key, value) - - -class CacheManager(object): - def __init__(self, **kwargs): - """Initialize a CacheManager object with a set of options - - Options should be parsed with the - :func:`~beaker.util.parse_cache_config_options` function to - ensure only valid options are used. - - """ - self.kwargs = kwargs - self.regions = kwargs.pop('cache_regions', {}) - - # Add these regions to the module global - cache_regions.update(self.regions) - - def get_cache(self, name, **kwargs): - kw = self.kwargs.copy() - kw.update(kwargs) - return Cache._get_cache(name, kw) - - def get_cache_region(self, name, region): - if region not in self.regions: - raise BeakerException('Cache region not configured: %s' % region) - kw = self.regions[region] - return Cache._get_cache(name, kw) - - def region(self, region, *args): - """Decorate a function to cache itself using a cache region - - The region decorator requires arguments if there are more than - 2 of the same named function, in the same module. This is - because the namespace used for the functions cache is based on - the functions name and the module. - - - Example:: - - # Assuming a cache object is available like: - cache = CacheManager(dict_of_config_options) - - - def populate_things(): - - @cache.region('short_term', 'some_data') - def load(search_term, limit, offset): - return load_the_data(search_term, limit, offset) - - return load('rabbits', 20, 0) - - .. note:: - - The function being decorated must only be called with - positional arguments. - - """ - return cache_region(region, *args) - - def region_invalidate(self, namespace, region, *args): - """Invalidate a cache region namespace or decorated function - - This function only invalidates cache spaces created with the - cache_region decorator. - - :param namespace: Either the namespace of the result to invalidate, or the - name of the cached function - - :param region: The region the function was cached to. If the function was - cached to a single region then this argument can be None - - :param args: Arguments that were used to differentiate the cached - function as well as the arguments passed to the decorated - function - - Example:: - - # Assuming a cache object is available like: - cache = CacheManager(dict_of_config_options) - - def populate_things(invalidate=False): - - @cache.region('short_term', 'some_data') - def load(search_term, limit, offset): - return load_the_data(search_term, limit, offset) - - # If the results should be invalidated first - if invalidate: - cache.region_invalidate(load, None, 'some_data', - 'rabbits', 20, 0) - return load('rabbits', 20, 0) - - - """ - return region_invalidate(namespace, region, *args) - if callable(namespace): - if not region: - region = namespace._arg_region - namespace = namespace._arg_namespace - - if not region: - raise BeakerException("Region or callable function " - "namespace is required") - else: - region = self.regions[region] - - cache = self.get_cache(namespace, **region) - cache_key = " ".join(str(x) for x in args) - cache.remove_value(cache_key) - - def cache(self, *args, **kwargs): - """Decorate a function to cache itself with supplied parameters - - :param args: Used to make the key unique for this function, as in region() - above. - - :param kwargs: Parameters to be passed to get_cache(), will override defaults - - Example:: - - # Assuming a cache object is available like: - cache = CacheManager(dict_of_config_options) - - - def populate_things(): - - @cache.cache('mycache', expire=15) - def load(search_term, limit, offset): - return load_the_data(search_term, limit, offset) - - return load('rabbits', 20, 0) - - .. note:: - - The function being decorated must only be called with - positional arguments. - - """ - cache = [None] - key = " ".join(str(x) for x in args) - - def decorate(func): - namespace = util.func_namespace(func) - def cached(*args): - if not cache[0]: - cache[0] = self.get_cache(namespace, **kwargs) - cache_key = key + " " + " ".join(str(x) for x in args) - def go(): - return func(*args) - return cache[0].get_value(cache_key, createfunc=go) - cached._arg_namespace = namespace - return cached - return decorate - - def invalidate(self, func, *args, **kwargs): - """Invalidate a cache decorated function - - This function only invalidates cache spaces created with the - cache decorator. - - :param func: Decorated function to invalidate - - :param args: Used to make the key unique for this function, as in region() - above. - - :param kwargs: Parameters that were passed for use by get_cache(), note that - this is only required if a ``type`` was specified for the - function - - Example:: - - # Assuming a cache object is available like: - cache = CacheManager(dict_of_config_options) - - - def populate_things(invalidate=False): - - @cache.cache('mycache', type="file", expire=15) - def load(search_term, limit, offset): - return load_the_data(search_term, limit, offset) - - # If the results should be invalidated first - if invalidate: - cache.invalidate(load, 'mycache', 'rabbits', 20, 0, type="file") - return load('rabbits', 20, 0) - - """ - namespace = func._arg_namespace - - cache = self.get_cache(namespace, **kwargs) - cache_key = " ".join(str(x) for x in args) - cache.remove_value(cache_key) diff --git a/module/lib/beaker/container.py b/module/lib/beaker/container.py deleted file mode 100644 index 515e97af6..000000000 --- a/module/lib/beaker/container.py +++ /dev/null @@ -1,633 +0,0 @@ -"""Container and Namespace classes""" -import anydbm -import cPickle -import logging -import os -import time - -import beaker.util as util -from beaker.exceptions import CreationAbortedError, MissingCacheParameter -from beaker.synchronization import _threading, file_synchronizer, \ - mutex_synchronizer, NameLock, null_synchronizer - -__all__ = ['Value', 'Container', 'ContainerContext', - 'MemoryContainer', 'DBMContainer', 'NamespaceManager', - 'MemoryNamespaceManager', 'DBMNamespaceManager', 'FileContainer', - 'OpenResourceNamespaceManager', - 'FileNamespaceManager', 'CreationAbortedError'] - - -logger = logging.getLogger('beaker.container') -if logger.isEnabledFor(logging.DEBUG): - debug = logger.debug -else: - def debug(message, *args): - pass - - -class NamespaceManager(object): - """Handles dictionary operations and locking for a namespace of - values. - - The implementation for setting and retrieving the namespace data is - handled by subclasses. - - NamespaceManager may be used alone, or may be privately accessed by - one or more Container objects. Container objects provide per-key - services like expiration times and automatic recreation of values. - - Multiple NamespaceManagers created with a particular name will all - share access to the same underlying datasource and will attempt to - synchronize against a common mutex object. The scope of this - sharing may be within a single process or across multiple - processes, depending on the type of NamespaceManager used. - - The NamespaceManager itself is generally threadsafe, except in the - case of the DBMNamespaceManager in conjunction with the gdbm dbm - implementation. - - """ - - @classmethod - def _init_dependencies(cls): - pass - - def __init__(self, namespace): - self._init_dependencies() - self.namespace = namespace - - def get_creation_lock(self, key): - raise NotImplementedError() - - def do_remove(self): - raise NotImplementedError() - - def acquire_read_lock(self): - pass - - def release_read_lock(self): - pass - - def acquire_write_lock(self, wait=True): - return True - - def release_write_lock(self): - pass - - def has_key(self, key): - return self.__contains__(key) - - def __getitem__(self, key): - raise NotImplementedError() - - def __setitem__(self, key, value): - raise NotImplementedError() - - def set_value(self, key, value, expiretime=None): - """Optional set_value() method called by Value. - - Allows an expiretime to be passed, for namespace - implementations which can prune their collections - using expiretime. - - """ - self[key] = value - - def __contains__(self, key): - raise NotImplementedError() - - def __delitem__(self, key): - raise NotImplementedError() - - def keys(self): - raise NotImplementedError() - - def remove(self): - self.do_remove() - - -class OpenResourceNamespaceManager(NamespaceManager): - """A NamespaceManager where read/write operations require opening/ - closing of a resource which is possibly mutexed. - - """ - def __init__(self, namespace): - NamespaceManager.__init__(self, namespace) - self.access_lock = self.get_access_lock() - self.openers = 0 - self.mutex = _threading.Lock() - - def get_access_lock(self): - raise NotImplementedError() - - def do_open(self, flags): - raise NotImplementedError() - - def do_close(self): - raise NotImplementedError() - - def acquire_read_lock(self): - self.access_lock.acquire_read_lock() - try: - self.open('r', checkcount = True) - except: - self.access_lock.release_read_lock() - raise - - def release_read_lock(self): - try: - self.close(checkcount = True) - finally: - self.access_lock.release_read_lock() - - def acquire_write_lock(self, wait=True): - r = self.access_lock.acquire_write_lock(wait) - try: - if (wait or r): - self.open('c', checkcount = True) - return r - except: - self.access_lock.release_write_lock() - raise - - def release_write_lock(self): - try: - self.close(checkcount=True) - finally: - self.access_lock.release_write_lock() - - def open(self, flags, checkcount=False): - self.mutex.acquire() - try: - if checkcount: - if self.openers == 0: - self.do_open(flags) - self.openers += 1 - else: - self.do_open(flags) - self.openers = 1 - finally: - self.mutex.release() - - def close(self, checkcount=False): - self.mutex.acquire() - try: - if checkcount: - self.openers -= 1 - if self.openers == 0: - self.do_close() - else: - if self.openers > 0: - self.do_close() - self.openers = 0 - finally: - self.mutex.release() - - def remove(self): - self.access_lock.acquire_write_lock() - try: - self.close(checkcount=False) - self.do_remove() - finally: - self.access_lock.release_write_lock() - -class Value(object): - __slots__ = 'key', 'createfunc', 'expiretime', 'expire_argument', 'starttime', 'storedtime',\ - 'namespace' - - def __init__(self, key, namespace, createfunc=None, expiretime=None, starttime=None): - self.key = key - self.createfunc = createfunc - self.expire_argument = expiretime - self.starttime = starttime - self.storedtime = -1 - self.namespace = namespace - - def has_value(self): - """return true if the container has a value stored. - - This is regardless of it being expired or not. - - """ - self.namespace.acquire_read_lock() - try: - return self.namespace.has_key(self.key) - finally: - self.namespace.release_read_lock() - - def can_have_value(self): - return self.has_current_value() or self.createfunc is not None - - def has_current_value(self): - self.namespace.acquire_read_lock() - try: - has_value = self.namespace.has_key(self.key) - if has_value: - try: - stored, expired, value = self._get_value() - return not self._is_expired(stored, expired) - except KeyError: - pass - return False - finally: - self.namespace.release_read_lock() - - def _is_expired(self, storedtime, expiretime): - """Return true if this container's value is expired.""" - return ( - ( - self.starttime is not None and - storedtime < self.starttime - ) - or - ( - expiretime is not None and - time.time() >= expiretime + storedtime - ) - ) - - def get_value(self): - self.namespace.acquire_read_lock() - try: - has_value = self.has_value() - if has_value: - try: - stored, expired, value = self._get_value() - if not self._is_expired(stored, expired): - return value - except KeyError: - # guard against un-mutexed backends raising KeyError - has_value = False - - if not self.createfunc: - raise KeyError(self.key) - finally: - self.namespace.release_read_lock() - - has_createlock = False - creation_lock = self.namespace.get_creation_lock(self.key) - if has_value: - if not creation_lock.acquire(wait=False): - debug("get_value returning old value while new one is created") - return value - else: - debug("lock_creatfunc (didnt wait)") - has_createlock = True - - if not has_createlock: - debug("lock_createfunc (waiting)") - creation_lock.acquire() - debug("lock_createfunc (waited)") - - try: - # see if someone created the value already - self.namespace.acquire_read_lock() - try: - if self.has_value(): - try: - stored, expired, value = self._get_value() - if not self._is_expired(stored, expired): - return value - except KeyError: - # guard against un-mutexed backends raising KeyError - pass - finally: - self.namespace.release_read_lock() - - debug("get_value creating new value") - v = self.createfunc() - self.set_value(v) - return v - finally: - creation_lock.release() - debug("released create lock") - - def _get_value(self): - value = self.namespace[self.key] - try: - stored, expired, value = value - except ValueError: - if not len(value) == 2: - raise - # Old format: upgrade - stored, value = value - expired = self.expire_argument - debug("get_value upgrading time %r expire time %r", stored, self.expire_argument) - self.namespace.release_read_lock() - self.set_value(value, stored) - self.namespace.acquire_read_lock() - except TypeError: - # occurs when the value is None. memcached - # may yank the rug from under us in which case - # that's the result - raise KeyError(self.key) - return stored, expired, value - - def set_value(self, value, storedtime=None): - self.namespace.acquire_write_lock() - try: - if storedtime is None: - storedtime = time.time() - debug("set_value stored time %r expire time %r", storedtime, self.expire_argument) - self.namespace.set_value(self.key, (storedtime, self.expire_argument, value)) - finally: - self.namespace.release_write_lock() - - def clear_value(self): - self.namespace.acquire_write_lock() - try: - debug("clear_value") - if self.namespace.has_key(self.key): - try: - del self.namespace[self.key] - except KeyError: - # guard against un-mutexed backends raising KeyError - pass - self.storedtime = -1 - finally: - self.namespace.release_write_lock() - -class AbstractDictionaryNSManager(NamespaceManager): - """A subclassable NamespaceManager that places data in a dictionary. - - Subclasses should provide a "dictionary" attribute or descriptor - which returns a dict-like object. The dictionary will store keys - that are local to the "namespace" attribute of this manager, so - ensure that the dictionary will not be used by any other namespace. - - e.g.:: - - import collections - cached_data = collections.defaultdict(dict) - - class MyDictionaryManager(AbstractDictionaryNSManager): - def __init__(self, namespace): - AbstractDictionaryNSManager.__init__(self, namespace) - self.dictionary = cached_data[self.namespace] - - The above stores data in a global dictionary called "cached_data", - which is structured as a dictionary of dictionaries, keyed - first on namespace name to a sub-dictionary, then on actual - cache key to value. - - """ - - def get_creation_lock(self, key): - return NameLock( - identifier="memorynamespace/funclock/%s/%s" % (self.namespace, key), - reentrant=True - ) - - def __getitem__(self, key): - return self.dictionary[key] - - def __contains__(self, key): - return self.dictionary.__contains__(key) - - def has_key(self, key): - return self.dictionary.__contains__(key) - - def __setitem__(self, key, value): - self.dictionary[key] = value - - def __delitem__(self, key): - del self.dictionary[key] - - def do_remove(self): - self.dictionary.clear() - - def keys(self): - return self.dictionary.keys() - -class MemoryNamespaceManager(AbstractDictionaryNSManager): - namespaces = util.SyncDict() - - def __init__(self, namespace, **kwargs): - AbstractDictionaryNSManager.__init__(self, namespace) - self.dictionary = MemoryNamespaceManager.namespaces.get(self.namespace, - dict) - -class DBMNamespaceManager(OpenResourceNamespaceManager): - def __init__(self, namespace, dbmmodule=None, data_dir=None, - dbm_dir=None, lock_dir=None, digest_filenames=True, **kwargs): - self.digest_filenames = digest_filenames - - if not dbm_dir and not data_dir: - raise MissingCacheParameter("data_dir or dbm_dir is required") - elif dbm_dir: - self.dbm_dir = dbm_dir - else: - self.dbm_dir = data_dir + "/container_dbm" - util.verify_directory(self.dbm_dir) - - if not lock_dir and not data_dir: - raise MissingCacheParameter("data_dir or lock_dir is required") - elif lock_dir: - self.lock_dir = lock_dir - else: - self.lock_dir = data_dir + "/container_dbm_lock" - util.verify_directory(self.lock_dir) - - self.dbmmodule = dbmmodule or anydbm - - self.dbm = None - OpenResourceNamespaceManager.__init__(self, namespace) - - self.file = util.encoded_path(root= self.dbm_dir, - identifiers=[self.namespace], - extension='.dbm', - digest_filenames=self.digest_filenames) - - debug("data file %s", self.file) - self._checkfile() - - def get_access_lock(self): - return file_synchronizer(identifier=self.namespace, - lock_dir=self.lock_dir) - - def get_creation_lock(self, key): - return file_synchronizer( - identifier = "dbmcontainer/funclock/%s" % self.namespace, - lock_dir=self.lock_dir - ) - - def file_exists(self, file): - if os.access(file, os.F_OK): - return True - else: - for ext in ('db', 'dat', 'pag', 'dir'): - if os.access(file + os.extsep + ext, os.F_OK): - return True - - return False - - def _checkfile(self): - if not self.file_exists(self.file): - g = self.dbmmodule.open(self.file, 'c') - g.close() - - def get_filenames(self): - list = [] - if os.access(self.file, os.F_OK): - list.append(self.file) - - for ext in ('pag', 'dir', 'db', 'dat'): - if os.access(self.file + os.extsep + ext, os.F_OK): - list.append(self.file + os.extsep + ext) - return list - - def do_open(self, flags): - debug("opening dbm file %s", self.file) - try: - self.dbm = self.dbmmodule.open(self.file, flags) - except: - self._checkfile() - self.dbm = self.dbmmodule.open(self.file, flags) - - def do_close(self): - if self.dbm is not None: - debug("closing dbm file %s", self.file) - self.dbm.close() - - def do_remove(self): - for f in self.get_filenames(): - os.remove(f) - - def __getitem__(self, key): - return cPickle.loads(self.dbm[key]) - - def __contains__(self, key): - return self.dbm.has_key(key) - - def __setitem__(self, key, value): - self.dbm[key] = cPickle.dumps(value) - - def __delitem__(self, key): - del self.dbm[key] - - def keys(self): - return self.dbm.keys() - - -class FileNamespaceManager(OpenResourceNamespaceManager): - def __init__(self, namespace, data_dir=None, file_dir=None, lock_dir=None, - digest_filenames=True, **kwargs): - self.digest_filenames = digest_filenames - - if not file_dir and not data_dir: - raise MissingCacheParameter("data_dir or file_dir is required") - elif file_dir: - self.file_dir = file_dir - else: - self.file_dir = data_dir + "/container_file" - util.verify_directory(self.file_dir) - - if not lock_dir and not data_dir: - raise MissingCacheParameter("data_dir or lock_dir is required") - elif lock_dir: - self.lock_dir = lock_dir - else: - self.lock_dir = data_dir + "/container_file_lock" - util.verify_directory(self.lock_dir) - OpenResourceNamespaceManager.__init__(self, namespace) - - self.file = util.encoded_path(root=self.file_dir, - identifiers=[self.namespace], - extension='.cache', - digest_filenames=self.digest_filenames) - self.hash = {} - - debug("data file %s", self.file) - - def get_access_lock(self): - return file_synchronizer(identifier=self.namespace, - lock_dir=self.lock_dir) - - def get_creation_lock(self, key): - return file_synchronizer( - identifier = "filecontainer/funclock/%s" % self.namespace, - lock_dir = self.lock_dir - ) - - def file_exists(self, file): - return os.access(file, os.F_OK) - - def do_open(self, flags): - if self.file_exists(self.file): - fh = open(self.file, 'rb') - try: - self.hash = cPickle.load(fh) - except (IOError, OSError, EOFError, cPickle.PickleError, ValueError): - pass - fh.close() - - self.flags = flags - - def do_close(self): - if self.flags == 'c' or self.flags == 'w': - fh = open(self.file, 'wb') - cPickle.dump(self.hash, fh) - fh.close() - - self.hash = {} - self.flags = None - - def do_remove(self): - try: - os.remove(self.file) - except OSError, err: - # for instance, because we haven't yet used this cache, - # but client code has asked for a clear() operation... - pass - self.hash = {} - - def __getitem__(self, key): - return self.hash[key] - - def __contains__(self, key): - return self.hash.has_key(key) - - def __setitem__(self, key, value): - self.hash[key] = value - - def __delitem__(self, key): - del self.hash[key] - - def keys(self): - return self.hash.keys() - - -#### legacy stuff to support the old "Container" class interface - -namespace_classes = {} - -ContainerContext = dict - -class ContainerMeta(type): - def __init__(cls, classname, bases, dict_): - namespace_classes[cls] = cls.namespace_class - return type.__init__(cls, classname, bases, dict_) - def __call__(self, key, context, namespace, createfunc=None, - expiretime=None, starttime=None, **kwargs): - if namespace in context: - ns = context[namespace] - else: - nscls = namespace_classes[self] - context[namespace] = ns = nscls(namespace, **kwargs) - return Value(key, ns, createfunc=createfunc, - expiretime=expiretime, starttime=starttime) - -class Container(object): - __metaclass__ = ContainerMeta - namespace_class = NamespaceManager - -class FileContainer(Container): - namespace_class = FileNamespaceManager - -class MemoryContainer(Container): - namespace_class = MemoryNamespaceManager - -class DBMContainer(Container): - namespace_class = DBMNamespaceManager - -DbmContainer = DBMContainer diff --git a/module/lib/beaker/converters.py b/module/lib/beaker/converters.py deleted file mode 100644 index f0ad34963..000000000 --- a/module/lib/beaker/converters.py +++ /dev/null @@ -1,26 +0,0 @@ -# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org) -# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php -def asbool(obj): - if isinstance(obj, (str, unicode)): - obj = obj.strip().lower() - if obj in ['true', 'yes', 'on', 'y', 't', '1']: - return True - elif obj in ['false', 'no', 'off', 'n', 'f', '0']: - return False - else: - raise ValueError( - "String is not true/false: %r" % obj) - return bool(obj) - -def aslist(obj, sep=None, strip=True): - if isinstance(obj, (str, unicode)): - lst = obj.split(sep) - if strip: - lst = [v.strip() for v in lst] - return lst - elif isinstance(obj, (list, tuple)): - return obj - elif obj is None: - return [] - else: - return [obj] diff --git a/module/lib/beaker/crypto/__init__.py b/module/lib/beaker/crypto/__init__.py deleted file mode 100644 index 3e26b0c13..000000000 --- a/module/lib/beaker/crypto/__init__.py +++ /dev/null @@ -1,40 +0,0 @@ -from warnings import warn - -from beaker.crypto.pbkdf2 import PBKDF2, strxor -from beaker.crypto.util import hmac, sha1, hmac_sha1, md5 -from beaker import util - -keyLength = None - -if util.jython: - try: - from beaker.crypto.jcecrypto import getKeyLength, aesEncrypt - keyLength = getKeyLength() - except ImportError: - pass -else: - try: - from beaker.crypto.pycrypto import getKeyLength, aesEncrypt, aesDecrypt - keyLength = getKeyLength() - except ImportError: - pass - -if not keyLength: - has_aes = False -else: - has_aes = True - -if has_aes and keyLength < 32: - warn('Crypto implementation only supports key lengths up to %d bits. ' - 'Generated session cookies may be incompatible with other ' - 'environments' % (keyLength * 8)) - - -def generateCryptoKeys(master_key, salt, iterations): - # NB: We XOR parts of the keystream into the randomly-generated parts, just - # in case os.urandom() isn't as random as it should be. Note that if - # os.urandom() returns truly random data, this will have no effect on the - # overall security. - keystream = PBKDF2(master_key, salt, iterations=iterations) - cipher_key = keystream.read(keyLength) - return cipher_key diff --git a/module/lib/beaker/crypto/jcecrypto.py b/module/lib/beaker/crypto/jcecrypto.py deleted file mode 100644 index 4062d513e..000000000 --- a/module/lib/beaker/crypto/jcecrypto.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -Encryption module that uses the Java Cryptography Extensions (JCE). - -Note that in default installations of the Java Runtime Environment, the -maximum key length is limited to 128 bits due to US export -restrictions. This makes the generated keys incompatible with the ones -generated by pycryptopp, which has no such restrictions. To fix this, -download the "Unlimited Strength Jurisdiction Policy Files" from Sun, -which will allow encryption using 256 bit AES keys. -""" -from javax.crypto import Cipher -from javax.crypto.spec import SecretKeySpec, IvParameterSpec - -import jarray - -# Initialization vector filled with zeros -_iv = IvParameterSpec(jarray.zeros(16, 'b')) - -def aesEncrypt(data, key): - cipher = Cipher.getInstance('AES/CTR/NoPadding') - skeySpec = SecretKeySpec(key, 'AES') - cipher.init(Cipher.ENCRYPT_MODE, skeySpec, _iv) - return cipher.doFinal(data).tostring() - -# magic. -aesDecrypt = aesEncrypt - -def getKeyLength(): - maxlen = Cipher.getMaxAllowedKeyLength('AES/CTR/NoPadding') - return min(maxlen, 256) / 8 diff --git a/module/lib/beaker/crypto/pbkdf2.py b/module/lib/beaker/crypto/pbkdf2.py deleted file mode 100644 index 96dc5fbb2..000000000 --- a/module/lib/beaker/crypto/pbkdf2.py +++ /dev/null @@ -1,342 +0,0 @@ -#!/usr/bin/python -# -*- coding: ascii -*- -########################################################################### -# PBKDF2.py - PKCS#5 v2.0 Password-Based Key Derivation -# -# Copyright (C) 2007 Dwayne C. Litzenberger -# All rights reserved. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose and without fee is hereby granted, -# provided that the above copyright notice appear in all copies and that -# both that copyright notice and this permission notice appear in -# supporting documentation. -# -# THE AUTHOR PROVIDES THIS SOFTWARE ``AS IS'' AND ANY EXPRESSED OR -# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Country of origin: Canada -# -########################################################################### -# Sample PBKDF2 usage: -# from Crypto.Cipher import AES -# from PBKDF2 import PBKDF2 -# import os -# -# salt = os.urandom(8) # 64-bit salt -# key = PBKDF2("This passphrase is a secret.", salt).read(32) # 256-bit key -# iv = os.urandom(16) # 128-bit IV -# cipher = AES.new(key, AES.MODE_CBC, iv) -# ... -# -# Sample crypt() usage: -# from PBKDF2 import crypt -# pwhash = crypt("secret") -# alleged_pw = raw_input("Enter password: ") -# if pwhash == crypt(alleged_pw, pwhash): -# print "Password good" -# else: -# print "Invalid password" -# -########################################################################### -# History: -# -# 2007-07-27 Dwayne C. Litzenberger -# - Initial Release (v1.0) -# -# 2007-07-31 Dwayne C. Litzenberger -# - Bugfix release (v1.1) -# - SECURITY: The PyCrypto XOR cipher (used, if available, in the _strxor -# function in the previous release) silently truncates all keys to 64 -# bytes. The way it was used in the previous release, this would only be -# problem if the pseudorandom function that returned values larger than -# 64 bytes (so SHA1, SHA256 and SHA512 are fine), but I don't like -# anything that silently reduces the security margin from what is -# expected. -# -########################################################################### - -__version__ = "1.1" - -from struct import pack -from binascii import b2a_hex -from random import randint - -from base64 import b64encode - -from beaker.crypto.util import hmac as HMAC, hmac_sha1 as SHA1 - -def strxor(a, b): - return "".join([chr(ord(x) ^ ord(y)) for (x, y) in zip(a, b)]) - -class PBKDF2(object): - """PBKDF2.py : PKCS#5 v2.0 Password-Based Key Derivation - - This implementation takes a passphrase and a salt (and optionally an - iteration count, a digest module, and a MAC module) and provides a - file-like object from which an arbitrarily-sized key can be read. - - If the passphrase and/or salt are unicode objects, they are encoded as - UTF-8 before they are processed. - - The idea behind PBKDF2 is to derive a cryptographic key from a - passphrase and a salt. - - PBKDF2 may also be used as a strong salted password hash. The - 'crypt' function is provided for that purpose. - - Remember: Keys generated using PBKDF2 are only as strong as the - passphrases they are derived from. - """ - - def __init__(self, passphrase, salt, iterations=1000, - digestmodule=SHA1, macmodule=HMAC): - if not callable(macmodule): - macmodule = macmodule.new - self.__macmodule = macmodule - self.__digestmodule = digestmodule - self._setup(passphrase, salt, iterations, self._pseudorandom) - - def _pseudorandom(self, key, msg): - """Pseudorandom function. e.g. HMAC-SHA1""" - return self.__macmodule(key=key, msg=msg, - digestmod=self.__digestmodule).digest() - - def read(self, bytes): - """Read the specified number of key bytes.""" - if self.closed: - raise ValueError("file-like object is closed") - - size = len(self.__buf) - blocks = [self.__buf] - i = self.__blockNum - while size < bytes: - i += 1 - if i > 0xffffffff: - # We could return "" here, but - raise OverflowError("derived key too long") - block = self.__f(i) - blocks.append(block) - size += len(block) - buf = "".join(blocks) - retval = buf[:bytes] - self.__buf = buf[bytes:] - self.__blockNum = i - return retval - - def __f(self, i): - # i must fit within 32 bits - assert (1 <= i <= 0xffffffff) - U = self.__prf(self.__passphrase, self.__salt + pack("!L", i)) - result = U - for j in xrange(2, 1+self.__iterations): - U = self.__prf(self.__passphrase, U) - result = strxor(result, U) - return result - - def hexread(self, octets): - """Read the specified number of octets. Return them as hexadecimal. - - Note that len(obj.hexread(n)) == 2*n. - """ - return b2a_hex(self.read(octets)) - - def _setup(self, passphrase, salt, iterations, prf): - # Sanity checks: - - # passphrase and salt must be str or unicode (in the latter - # case, we convert to UTF-8) - if isinstance(passphrase, unicode): - passphrase = passphrase.encode("UTF-8") - if not isinstance(passphrase, str): - raise TypeError("passphrase must be str or unicode") - if isinstance(salt, unicode): - salt = salt.encode("UTF-8") - if not isinstance(salt, str): - raise TypeError("salt must be str or unicode") - - # iterations must be an integer >= 1 - if not isinstance(iterations, (int, long)): - raise TypeError("iterations must be an integer") - if iterations < 1: - raise ValueError("iterations must be at least 1") - - # prf must be callable - if not callable(prf): - raise TypeError("prf must be callable") - - self.__passphrase = passphrase - self.__salt = salt - self.__iterations = iterations - self.__prf = prf - self.__blockNum = 0 - self.__buf = "" - self.closed = False - - def close(self): - """Close the stream.""" - if not self.closed: - del self.__passphrase - del self.__salt - del self.__iterations - del self.__prf - del self.__blockNum - del self.__buf - self.closed = True - -def crypt(word, salt=None, iterations=None): - """PBKDF2-based unix crypt(3) replacement. - - The number of iterations specified in the salt overrides the 'iterations' - parameter. - - The effective hash length is 192 bits. - """ - - # Generate a (pseudo-)random salt if the user hasn't provided one. - if salt is None: - salt = _makesalt() - - # salt must be a string or the us-ascii subset of unicode - if isinstance(salt, unicode): - salt = salt.encode("us-ascii") - if not isinstance(salt, str): - raise TypeError("salt must be a string") - - # word must be a string or unicode (in the latter case, we convert to UTF-8) - if isinstance(word, unicode): - word = word.encode("UTF-8") - if not isinstance(word, str): - raise TypeError("word must be a string or unicode") - - # Try to extract the real salt and iteration count from the salt - if salt.startswith("$p5k2$"): - (iterations, salt, dummy) = salt.split("$")[2:5] - if iterations == "": - iterations = 400 - else: - converted = int(iterations, 16) - if iterations != "%x" % converted: # lowercase hex, minimum digits - raise ValueError("Invalid salt") - iterations = converted - if not (iterations >= 1): - raise ValueError("Invalid salt") - - # Make sure the salt matches the allowed character set - allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789./" - for ch in salt: - if ch not in allowed: - raise ValueError("Illegal character %r in salt" % (ch,)) - - if iterations is None or iterations == 400: - iterations = 400 - salt = "$p5k2$$" + salt - else: - salt = "$p5k2$%x$%s" % (iterations, salt) - rawhash = PBKDF2(word, salt, iterations).read(24) - return salt + "$" + b64encode(rawhash, "./") - -# Add crypt as a static method of the PBKDF2 class -# This makes it easier to do "from PBKDF2 import PBKDF2" and still use -# crypt. -PBKDF2.crypt = staticmethod(crypt) - -def _makesalt(): - """Return a 48-bit pseudorandom salt for crypt(). - - This function is not suitable for generating cryptographic secrets. - """ - binarysalt = "".join([pack("@H", randint(0, 0xffff)) for i in range(3)]) - return b64encode(binarysalt, "./") - -def test_pbkdf2(): - """Module self-test""" - from binascii import a2b_hex - - # - # Test vectors from RFC 3962 - # - - # Test 1 - result = PBKDF2("password", "ATHENA.MIT.EDUraeburn", 1).read(16) - expected = a2b_hex("cdedb5281bb2f801565a1122b2563515") - if result != expected: - raise RuntimeError("self-test failed") - - # Test 2 - result = PBKDF2("password", "ATHENA.MIT.EDUraeburn", 1200).hexread(32) - expected = ("5c08eb61fdf71e4e4ec3cf6ba1f5512b" - "a7e52ddbc5e5142f708a31e2e62b1e13") - if result != expected: - raise RuntimeError("self-test failed") - - # Test 3 - result = PBKDF2("X"*64, "pass phrase equals block size", 1200).hexread(32) - expected = ("139c30c0966bc32ba55fdbf212530ac9" - "c5ec59f1a452f5cc9ad940fea0598ed1") - if result != expected: - raise RuntimeError("self-test failed") - - # Test 4 - result = PBKDF2("X"*65, "pass phrase exceeds block size", 1200).hexread(32) - expected = ("9ccad6d468770cd51b10e6a68721be61" - "1a8b4d282601db3b36be9246915ec82a") - if result != expected: - raise RuntimeError("self-test failed") - - # - # Other test vectors - # - - # Chunked read - f = PBKDF2("kickstart", "workbench", 256) - result = f.read(17) - result += f.read(17) - result += f.read(1) - result += f.read(2) - result += f.read(3) - expected = PBKDF2("kickstart", "workbench", 256).read(40) - if result != expected: - raise RuntimeError("self-test failed") - - # - # crypt() test vectors - # - - # crypt 1 - result = crypt("cloadm", "exec") - expected = '$p5k2$$exec$r1EWMCMk7Rlv3L/RNcFXviDefYa0hlql' - if result != expected: - raise RuntimeError("self-test failed") - - # crypt 2 - result = crypt("gnu", '$p5k2$c$u9HvcT4d$.....') - expected = '$p5k2$c$u9HvcT4d$Sd1gwSVCLZYAuqZ25piRnbBEoAesaa/g' - if result != expected: - raise RuntimeError("self-test failed") - - # crypt 3 - result = crypt("dcl", "tUsch7fU", iterations=13) - expected = "$p5k2$d$tUsch7fU$nqDkaxMDOFBeJsTSfABsyn.PYUXilHwL" - if result != expected: - raise RuntimeError("self-test failed") - - # crypt 4 (unicode) - result = crypt(u'\u0399\u03c9\u03b1\u03bd\u03bd\u03b7\u03c2', - '$p5k2$$KosHgqNo$9mjN8gqjt02hDoP0c2J0ABtLIwtot8cQ') - expected = '$p5k2$$KosHgqNo$9mjN8gqjt02hDoP0c2J0ABtLIwtot8cQ' - if result != expected: - raise RuntimeError("self-test failed") - -if __name__ == '__main__': - test_pbkdf2() - -# vim:set ts=4 sw=4 sts=4 expandtab: diff --git a/module/lib/beaker/crypto/pycrypto.py b/module/lib/beaker/crypto/pycrypto.py deleted file mode 100644 index a3eb4d9db..000000000 --- a/module/lib/beaker/crypto/pycrypto.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Encryption module that uses pycryptopp or pycrypto""" -try: - # Pycryptopp is preferred over Crypto because Crypto has had - # various periods of not being maintained, and pycryptopp uses - # the Crypto++ library which is generally considered the 'gold standard' - # of crypto implementations - from pycryptopp.cipher import aes - - def aesEncrypt(data, key): - cipher = aes.AES(key) - return cipher.process(data) - - # magic. - aesDecrypt = aesEncrypt - -except ImportError: - from Crypto.Cipher import AES - - def aesEncrypt(data, key): - cipher = AES.new(key) - - data = data + (" " * (16 - (len(data) % 16))) - return cipher.encrypt(data) - - def aesDecrypt(data, key): - cipher = AES.new(key) - - return cipher.decrypt(data).rstrip() - -def getKeyLength(): - return 32 diff --git a/module/lib/beaker/crypto/util.py b/module/lib/beaker/crypto/util.py deleted file mode 100644 index d97e8ce6f..000000000 --- a/module/lib/beaker/crypto/util.py +++ /dev/null @@ -1,30 +0,0 @@ -from warnings import warn -from beaker import util - - -try: - # Use PyCrypto (if available) - from Crypto.Hash import HMAC as hmac, SHA as hmac_sha1 - sha1 = hmac_sha1.new - -except ImportError: - - # PyCrypto not available. Use the Python standard library. - import hmac - - # When using the stdlib, we have to make sure the hmac version and sha - # version are compatible - if util.py24: - from sha import sha as sha1 - import sha as hmac_sha1 - else: - # NOTE: We have to use the callable with hashlib (hashlib.sha1), - # otherwise hmac only accepts the sha module object itself - from hashlib import sha1 - hmac_sha1 = sha1 - - -if util.py24: - from md5 import md5 -else: - from hashlib import md5 diff --git a/module/lib/beaker/exceptions.py b/module/lib/beaker/exceptions.py deleted file mode 100644 index cc0eed286..000000000 --- a/module/lib/beaker/exceptions.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Beaker exception classes""" - -class BeakerException(Exception): - pass - - -class CreationAbortedError(Exception): - """Deprecated.""" - - -class InvalidCacheBackendError(BeakerException, ImportError): - pass - - -class MissingCacheParameter(BeakerException): - pass - - -class LockError(BeakerException): - pass - - -class InvalidCryptoBackendError(BeakerException): - pass diff --git a/module/lib/beaker/ext/__init__.py b/module/lib/beaker/ext/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/module/lib/beaker/ext/database.py b/module/lib/beaker/ext/database.py deleted file mode 100644 index 701e6f7d2..000000000 --- a/module/lib/beaker/ext/database.py +++ /dev/null @@ -1,165 +0,0 @@ -import cPickle -import logging -import pickle -from datetime import datetime - -from beaker.container import OpenResourceNamespaceManager, Container -from beaker.exceptions import InvalidCacheBackendError, MissingCacheParameter -from beaker.synchronization import file_synchronizer, null_synchronizer -from beaker.util import verify_directory, SyncDict - -log = logging.getLogger(__name__) - -sa = None -pool = None -types = None - -class DatabaseNamespaceManager(OpenResourceNamespaceManager): - metadatas = SyncDict() - tables = SyncDict() - - @classmethod - def _init_dependencies(cls): - global sa, pool, types - if sa is not None: - return - try: - import sqlalchemy as sa - import sqlalchemy.pool as pool - from sqlalchemy import types - except ImportError: - raise InvalidCacheBackendError("Database cache backend requires " - "the 'sqlalchemy' library") - - def __init__(self, namespace, url=None, sa_opts=None, optimistic=False, - table_name='beaker_cache', data_dir=None, lock_dir=None, - **params): - """Creates a database namespace manager - - ``url`` - SQLAlchemy compliant db url - ``sa_opts`` - A dictionary of SQLAlchemy keyword options to initialize the engine - with. - ``optimistic`` - Use optimistic session locking, note that this will result in an - additional select when updating a cache value to compare version - numbers. - ``table_name`` - The table name to use in the database for the cache. - """ - OpenResourceNamespaceManager.__init__(self, namespace) - - if sa_opts is None: - sa_opts = params - - if lock_dir: - self.lock_dir = lock_dir - elif data_dir: - self.lock_dir = data_dir + "/container_db_lock" - if self.lock_dir: - verify_directory(self.lock_dir) - - # Check to see if the table's been created before - url = url or sa_opts['sa.url'] - table_key = url + table_name - def make_cache(): - # Check to see if we have a connection pool open already - meta_key = url + table_name - def make_meta(): - # SQLAlchemy pops the url, this ensures it sticks around - # later - sa_opts['sa.url'] = url - engine = sa.engine_from_config(sa_opts, 'sa.') - meta = sa.MetaData() - meta.bind = engine - return meta - meta = DatabaseNamespaceManager.metadatas.get(meta_key, make_meta) - # Create the table object and cache it now - cache = sa.Table(table_name, meta, - sa.Column('id', types.Integer, primary_key=True), - sa.Column('namespace', types.String(255), nullable=False), - sa.Column('accessed', types.DateTime, nullable=False), - sa.Column('created', types.DateTime, nullable=False), - sa.Column('data', types.PickleType, nullable=False), - sa.UniqueConstraint('namespace') - ) - cache.create(checkfirst=True) - return cache - self.hash = {} - self._is_new = False - self.loaded = False - self.cache = DatabaseNamespaceManager.tables.get(table_key, make_cache) - - def get_access_lock(self): - return null_synchronizer() - - def get_creation_lock(self, key): - return file_synchronizer( - identifier ="databasecontainer/funclock/%s" % self.namespace, - lock_dir = self.lock_dir) - - def do_open(self, flags): - # If we already loaded the data, don't bother loading it again - if self.loaded: - self.flags = flags - return - - cache = self.cache - result = sa.select([cache.c.data], - cache.c.namespace==self.namespace - ).execute().fetchone() - if not result: - self._is_new = True - self.hash = {} - else: - self._is_new = False - try: - self.hash = result['data'] - except (IOError, OSError, EOFError, cPickle.PickleError, - pickle.PickleError): - log.debug("Couln't load pickle data, creating new storage") - self.hash = {} - self._is_new = True - self.flags = flags - self.loaded = True - - def do_close(self): - if self.flags is not None and (self.flags == 'c' or self.flags == 'w'): - cache = self.cache - if self._is_new: - cache.insert().execute(namespace=self.namespace, data=self.hash, - accessed=datetime.now(), - created=datetime.now()) - self._is_new = False - else: - cache.update(cache.c.namespace==self.namespace).execute( - data=self.hash, accessed=datetime.now()) - self.flags = None - - def do_remove(self): - cache = self.cache - cache.delete(cache.c.namespace==self.namespace).execute() - self.hash = {} - - # We can retain the fact that we did a load attempt, but since the - # file is gone this will be a new namespace should it be saved. - self._is_new = True - - def __getitem__(self, key): - return self.hash[key] - - def __contains__(self, key): - return self.hash.has_key(key) - - def __setitem__(self, key, value): - self.hash[key] = value - - def __delitem__(self, key): - del self.hash[key] - - def keys(self): - return self.hash.keys() - -class DatabaseContainer(Container): - namespace_manager = DatabaseNamespaceManager diff --git a/module/lib/beaker/ext/google.py b/module/lib/beaker/ext/google.py deleted file mode 100644 index dd8380d7f..000000000 --- a/module/lib/beaker/ext/google.py +++ /dev/null @@ -1,120 +0,0 @@ -import cPickle -import logging -from datetime import datetime - -from beaker.container import OpenResourceNamespaceManager, Container -from beaker.exceptions import InvalidCacheBackendError -from beaker.synchronization import null_synchronizer - -log = logging.getLogger(__name__) - -db = None - -class GoogleNamespaceManager(OpenResourceNamespaceManager): - tables = {} - - @classmethod - def _init_dependencies(cls): - global db - if db is not None: - return - try: - db = __import__('google.appengine.ext.db').appengine.ext.db - except ImportError: - raise InvalidCacheBackendError("Datastore cache backend requires the " - "'google.appengine.ext' library") - - def __init__(self, namespace, table_name='beaker_cache', **params): - """Creates a datastore namespace manager""" - OpenResourceNamespaceManager.__init__(self, namespace) - - def make_cache(): - table_dict = dict(created=db.DateTimeProperty(), - accessed=db.DateTimeProperty(), - data=db.BlobProperty()) - table = type(table_name, (db.Model,), table_dict) - return table - self.table_name = table_name - self.cache = GoogleNamespaceManager.tables.setdefault(table_name, make_cache()) - self.hash = {} - self._is_new = False - self.loaded = False - self.log_debug = logging.DEBUG >= log.getEffectiveLevel() - - # Google wants namespaces to start with letters, change the namespace - # to start with a letter - self.namespace = 'p%s' % self.namespace - - def get_access_lock(self): - return null_synchronizer() - - def get_creation_lock(self, key): - # this is weird, should probably be present - return null_synchronizer() - - def do_open(self, flags): - # If we already loaded the data, don't bother loading it again - if self.loaded: - self.flags = flags - return - - item = self.cache.get_by_key_name(self.namespace) - - if not item: - self._is_new = True - self.hash = {} - else: - self._is_new = False - try: - self.hash = cPickle.loads(str(item.data)) - except (IOError, OSError, EOFError, cPickle.PickleError): - if self.log_debug: - log.debug("Couln't load pickle data, creating new storage") - self.hash = {} - self._is_new = True - self.flags = flags - self.loaded = True - - def do_close(self): - if self.flags is not None and (self.flags == 'c' or self.flags == 'w'): - if self._is_new: - item = self.cache(key_name=self.namespace) - item.data = cPickle.dumps(self.hash) - item.created = datetime.now() - item.accessed = datetime.now() - item.put() - self._is_new = False - else: - item = self.cache.get_by_key_name(self.namespace) - item.data = cPickle.dumps(self.hash) - item.accessed = datetime.now() - item.put() - self.flags = None - - def do_remove(self): - item = self.cache.get_by_key_name(self.namespace) - item.delete() - self.hash = {} - - # We can retain the fact that we did a load attempt, but since the - # file is gone this will be a new namespace should it be saved. - self._is_new = True - - def __getitem__(self, key): - return self.hash[key] - - def __contains__(self, key): - return self.hash.has_key(key) - - def __setitem__(self, key, value): - self.hash[key] = value - - def __delitem__(self, key): - del self.hash[key] - - def keys(self): - return self.hash.keys() - - -class GoogleContainer(Container): - namespace_class = GoogleNamespaceManager diff --git a/module/lib/beaker/ext/memcached.py b/module/lib/beaker/ext/memcached.py deleted file mode 100644 index 96516953f..000000000 --- a/module/lib/beaker/ext/memcached.py +++ /dev/null @@ -1,82 +0,0 @@ -from beaker.container import NamespaceManager, Container -from beaker.exceptions import InvalidCacheBackendError, MissingCacheParameter -from beaker.synchronization import file_synchronizer, null_synchronizer -from beaker.util import verify_directory, SyncDict -import warnings - -memcache = None - -class MemcachedNamespaceManager(NamespaceManager): - clients = SyncDict() - - @classmethod - def _init_dependencies(cls): - global memcache - if memcache is not None: - return - try: - import pylibmc as memcache - except ImportError: - try: - import cmemcache as memcache - warnings.warn("cmemcache is known to have serious " - "concurrency issues; consider using 'memcache' or 'pylibmc'") - except ImportError: - try: - import memcache - except ImportError: - raise InvalidCacheBackendError("Memcached cache backend requires either " - "the 'memcache' or 'cmemcache' library") - - def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params): - NamespaceManager.__init__(self, namespace) - - if not url: - raise MissingCacheParameter("url is required") - - if lock_dir: - self.lock_dir = lock_dir - elif data_dir: - self.lock_dir = data_dir + "/container_mcd_lock" - if self.lock_dir: - verify_directory(self.lock_dir) - - self.mc = MemcachedNamespaceManager.clients.get(url, memcache.Client, url.split(';')) - - def get_creation_lock(self, key): - return file_synchronizer( - identifier="memcachedcontainer/funclock/%s" % self.namespace,lock_dir = self.lock_dir) - - def _format_key(self, key): - return self.namespace + '_' + key.replace(' ', '\302\267') - - def __getitem__(self, key): - return self.mc.get(self._format_key(key)) - - def __contains__(self, key): - value = self.mc.get(self._format_key(key)) - return value is not None - - def has_key(self, key): - return key in self - - def set_value(self, key, value, expiretime=None): - if expiretime: - self.mc.set(self._format_key(key), value, time=expiretime) - else: - self.mc.set(self._format_key(key), value) - - def __setitem__(self, key, value): - self.set_value(key, value) - - def __delitem__(self, key): - self.mc.delete(self._format_key(key)) - - def do_remove(self): - self.mc.flush_all() - - def keys(self): - raise NotImplementedError("Memcache caching does not support iteration of all cache keys") - -class MemcachedContainer(Container): - namespace_class = MemcachedNamespaceManager diff --git a/module/lib/beaker/ext/sqla.py b/module/lib/beaker/ext/sqla.py deleted file mode 100644 index 8c79633c1..000000000 --- a/module/lib/beaker/ext/sqla.py +++ /dev/null @@ -1,133 +0,0 @@ -import cPickle -import logging -import pickle -from datetime import datetime - -from beaker.container import OpenResourceNamespaceManager, Container -from beaker.exceptions import InvalidCacheBackendError, MissingCacheParameter -from beaker.synchronization import file_synchronizer, null_synchronizer -from beaker.util import verify_directory, SyncDict - - -log = logging.getLogger(__name__) - -sa = None - -class SqlaNamespaceManager(OpenResourceNamespaceManager): - binds = SyncDict() - tables = SyncDict() - - @classmethod - def _init_dependencies(cls): - global sa - if sa is not None: - return - try: - import sqlalchemy as sa - except ImportError: - raise InvalidCacheBackendError("SQLAlchemy, which is required by " - "this backend, is not installed") - - def __init__(self, namespace, bind, table, data_dir=None, lock_dir=None, - **kwargs): - """Create a namespace manager for use with a database table via - SQLAlchemy. - - ``bind`` - SQLAlchemy ``Engine`` or ``Connection`` object - - ``table`` - SQLAlchemy ``Table`` object in which to store namespace data. - This should usually be something created by ``make_cache_table``. - """ - OpenResourceNamespaceManager.__init__(self, namespace) - - if lock_dir: - self.lock_dir = lock_dir - elif data_dir: - self.lock_dir = data_dir + "/container_db_lock" - if self.lock_dir: - verify_directory(self.lock_dir) - - self.bind = self.__class__.binds.get(str(bind.url), lambda: bind) - self.table = self.__class__.tables.get('%s:%s' % (bind.url, table.name), - lambda: table) - self.hash = {} - self._is_new = False - self.loaded = False - - def get_access_lock(self): - return null_synchronizer() - - def get_creation_lock(self, key): - return file_synchronizer( - identifier ="databasecontainer/funclock/%s" % self.namespace, - lock_dir=self.lock_dir) - - def do_open(self, flags): - if self.loaded: - self.flags = flags - return - select = sa.select([self.table.c.data], - (self.table.c.namespace == self.namespace)) - result = self.bind.execute(select).fetchone() - if not result: - self._is_new = True - self.hash = {} - else: - self._is_new = False - try: - self.hash = result['data'] - except (IOError, OSError, EOFError, cPickle.PickleError, - pickle.PickleError): - log.debug("Couln't load pickle data, creating new storage") - self.hash = {} - self._is_new = True - self.flags = flags - self.loaded = True - - def do_close(self): - if self.flags is not None and (self.flags == 'c' or self.flags == 'w'): - if self._is_new: - insert = self.table.insert() - self.bind.execute(insert, namespace=self.namespace, data=self.hash, - accessed=datetime.now(), created=datetime.now()) - self._is_new = False - else: - update = self.table.update(self.table.c.namespace == self.namespace) - self.bind.execute(update, data=self.hash, accessed=datetime.now()) - self.flags = None - - def do_remove(self): - delete = self.table.delete(self.table.c.namespace == self.namespace) - self.bind.execute(delete) - self.hash = {} - self._is_new = True - - def __getitem__(self, key): - return self.hash[key] - - def __contains__(self, key): - return self.hash.has_key(key) - - def __setitem__(self, key, value): - self.hash[key] = value - - def __delitem__(self, key): - del self.hash[key] - - def keys(self): - return self.hash.keys() - - -class SqlaContainer(Container): - namespace_manager = SqlaNamespaceManager - -def make_cache_table(metadata, table_name='beaker_cache'): - """Return a ``Table`` object suitable for storing cached values for the - namespace manager. Do not create the table.""" - return sa.Table(table_name, metadata, - sa.Column('namespace', sa.String(255), primary_key=True), - sa.Column('accessed', sa.DateTime, nullable=False), - sa.Column('created', sa.DateTime, nullable=False), - sa.Column('data', sa.PickleType, nullable=False)) diff --git a/module/lib/beaker/middleware.py b/module/lib/beaker/middleware.py deleted file mode 100644 index 7ba88b37d..000000000 --- a/module/lib/beaker/middleware.py +++ /dev/null @@ -1,165 +0,0 @@ -import warnings - -try: - from paste.registry import StackedObjectProxy - beaker_session = StackedObjectProxy(name="Beaker Session") - beaker_cache = StackedObjectProxy(name="Cache Manager") -except: - beaker_cache = None - beaker_session = None - -from beaker.cache import CacheManager -from beaker.session import Session, SessionObject -from beaker.util import coerce_cache_params, coerce_session_params, \ - parse_cache_config_options - - -class CacheMiddleware(object): - cache = beaker_cache - - def __init__(self, app, config=None, environ_key='beaker.cache', **kwargs): - """Initialize the Cache Middleware - - The Cache middleware will make a Cache instance available - every request under the ``environ['beaker.cache']`` key by - default. The location in environ can be changed by setting - ``environ_key``. - - ``config`` - dict All settings should be prefixed by 'cache.'. This - method of passing variables is intended for Paste and other - setups that accumulate multiple component settings in a - single dictionary. If config contains *no cache. prefixed - args*, then *all* of the config options will be used to - intialize the Cache objects. - - ``environ_key`` - Location where the Cache instance will keyed in the WSGI - environ - - ``**kwargs`` - All keyword arguments are assumed to be cache settings and - will override any settings found in ``config`` - - """ - self.app = app - config = config or {} - - self.options = {} - - # Update the options with the parsed config - self.options.update(parse_cache_config_options(config)) - - # Add any options from kwargs, but leave out the defaults this - # time - self.options.update( - parse_cache_config_options(kwargs, include_defaults=False)) - - # Assume all keys are intended for cache if none are prefixed with - # 'cache.' - if not self.options and config: - self.options = config - - self.options.update(kwargs) - self.cache_manager = CacheManager(**self.options) - self.environ_key = environ_key - - def __call__(self, environ, start_response): - if environ.get('paste.registry'): - if environ['paste.registry'].reglist: - environ['paste.registry'].register(self.cache, - self.cache_manager) - environ[self.environ_key] = self.cache_manager - return self.app(environ, start_response) - - -class SessionMiddleware(object): - session = beaker_session - - def __init__(self, wrap_app, config=None, environ_key='beaker.session', - **kwargs): - """Initialize the Session Middleware - - The Session middleware will make a lazy session instance - available every request under the ``environ['beaker.session']`` - key by default. The location in environ can be changed by - setting ``environ_key``. - - ``config`` - dict All settings should be prefixed by 'session.'. This - method of passing variables is intended for Paste and other - setups that accumulate multiple component settings in a - single dictionary. If config contains *no cache. prefixed - args*, then *all* of the config options will be used to - intialize the Cache objects. - - ``environ_key`` - Location where the Session instance will keyed in the WSGI - environ - - ``**kwargs`` - All keyword arguments are assumed to be session settings and - will override any settings found in ``config`` - - """ - config = config or {} - - # Load up the default params - self.options = dict(invalidate_corrupt=True, type=None, - data_dir=None, key='beaker.session.id', - timeout=None, secret=None, log_file=None) - - # Pull out any config args meant for beaker session. if there are any - for dct in [config, kwargs]: - for key, val in dct.iteritems(): - if key.startswith('beaker.session.'): - self.options[key[15:]] = val - if key.startswith('session.'): - self.options[key[8:]] = val - if key.startswith('session_'): - warnings.warn('Session options should start with session. ' - 'instead of session_.', DeprecationWarning, 2) - self.options[key[8:]] = val - - # Coerce and validate session params - coerce_session_params(self.options) - - # Assume all keys are intended for cache if none are prefixed with - # 'cache.' - if not self.options and config: - self.options = config - - self.options.update(kwargs) - self.wrap_app = wrap_app - self.environ_key = environ_key - - def __call__(self, environ, start_response): - session = SessionObject(environ, **self.options) - if environ.get('paste.registry'): - if environ['paste.registry'].reglist: - environ['paste.registry'].register(self.session, session) - environ[self.environ_key] = session - environ['beaker.get_session'] = self._get_session - - def session_start_response(status, headers, exc_info = None): - if session.accessed(): - session.persist() - if session.__dict__['_headers']['set_cookie']: - cookie = session.__dict__['_headers']['cookie_out'] - if cookie: - headers.append(('Set-cookie', cookie)) - return start_response(status, headers, exc_info) - return self.wrap_app(environ, session_start_response) - - def _get_session(self): - return Session({}, use_cookies=False, **self.options) - - -def session_filter_factory(global_conf, **kwargs): - def filter(app): - return SessionMiddleware(app, global_conf, **kwargs) - return filter - - -def session_filter_app_factory(app, global_conf, **kwargs): - return SessionMiddleware(app, global_conf, **kwargs) diff --git a/module/lib/beaker/session.py b/module/lib/beaker/session.py deleted file mode 100644 index 7d465530b..000000000 --- a/module/lib/beaker/session.py +++ /dev/null @@ -1,618 +0,0 @@ -import Cookie -import os -import random -import time -from datetime import datetime, timedelta - -from beaker.crypto import hmac as HMAC, hmac_sha1 as SHA1, md5 -from beaker.util import pickle - -from beaker import crypto -from beaker.cache import clsmap -from beaker.exceptions import BeakerException, InvalidCryptoBackendError -from base64 import b64encode, b64decode - - -__all__ = ['SignedCookie', 'Session'] - -getpid = hasattr(os, 'getpid') and os.getpid or (lambda : '') - -class SignedCookie(Cookie.BaseCookie): - """Extends python cookie to give digital signature support""" - def __init__(self, secret, input=None): - self.secret = secret - Cookie.BaseCookie.__init__(self, input) - - def value_decode(self, val): - val = val.strip('"') - sig = HMAC.new(self.secret, val[40:], SHA1).hexdigest() - - # Avoid timing attacks - invalid_bits = 0 - input_sig = val[:40] - if len(sig) != len(input_sig): - return None, val - - for a, b in zip(sig, input_sig): - invalid_bits += a != b - - if invalid_bits: - return None, val - else: - return val[40:], val - - def value_encode(self, val): - sig = HMAC.new(self.secret, val, SHA1).hexdigest() - return str(val), ("%s%s" % (sig, val)) - - -class Session(dict): - """Session object that uses container package for storage. - - ``key`` - The name the cookie should be set to. - ``timeout`` - How long session data is considered valid. This is used - regardless of the cookie being present or not to determine - whether session data is still valid. - ``cookie_domain`` - Domain to use for the cookie. - ``secure`` - Whether or not the cookie should only be sent over SSL. - """ - def __init__(self, request, id=None, invalidate_corrupt=False, - use_cookies=True, type=None, data_dir=None, - key='beaker.session.id', timeout=None, cookie_expires=True, - cookie_domain=None, secret=None, secure=False, - namespace_class=None, **namespace_args): - if not type: - if data_dir: - self.type = 'file' - else: - self.type = 'memory' - else: - self.type = type - - self.namespace_class = namespace_class or clsmap[self.type] - - self.namespace_args = namespace_args - - self.request = request - self.data_dir = data_dir - self.key = key - - self.timeout = timeout - self.use_cookies = use_cookies - self.cookie_expires = cookie_expires - - # Default cookie domain/path - self._domain = cookie_domain - self._path = '/' - self.was_invalidated = False - self.secret = secret - self.secure = secure - self.id = id - self.accessed_dict = {} - - if self.use_cookies: - cookieheader = request.get('cookie', '') - if secret: - try: - self.cookie = SignedCookie(secret, input=cookieheader) - except Cookie.CookieError: - self.cookie = SignedCookie(secret, input=None) - else: - self.cookie = Cookie.SimpleCookie(input=cookieheader) - - if not self.id and self.key in self.cookie: - self.id = self.cookie[self.key].value - - self.is_new = self.id is None - if self.is_new: - self._create_id() - self['_accessed_time'] = self['_creation_time'] = time.time() - else: - try: - self.load() - except: - if invalidate_corrupt: - self.invalidate() - else: - raise - - def _create_id(self): - self.id = md5( - md5("%f%s%f%s" % (time.time(), id({}), random.random(), - getpid())).hexdigest(), - ).hexdigest() - self.is_new = True - self.last_accessed = None - if self.use_cookies: - self.cookie[self.key] = self.id - if self._domain: - self.cookie[self.key]['domain'] = self._domain - if self.secure: - self.cookie[self.key]['secure'] = True - self.cookie[self.key]['path'] = self._path - if self.cookie_expires is not True: - if self.cookie_expires is False: - expires = datetime.fromtimestamp( 0x7FFFFFFF ) - elif isinstance(self.cookie_expires, timedelta): - expires = datetime.today() + self.cookie_expires - elif isinstance(self.cookie_expires, datetime): - expires = self.cookie_expires - else: - raise ValueError("Invalid argument for cookie_expires: %s" - % repr(self.cookie_expires)) - self.cookie[self.key]['expires'] = \ - expires.strftime("%a, %d-%b-%Y %H:%M:%S GMT" ) - self.request['cookie_out'] = self.cookie[self.key].output(header='') - self.request['set_cookie'] = False - - def created(self): - return self['_creation_time'] - created = property(created) - - def _set_domain(self, domain): - self['_domain'] = domain - self.cookie[self.key]['domain'] = domain - self.request['cookie_out'] = self.cookie[self.key].output(header='') - self.request['set_cookie'] = True - - def _get_domain(self): - return self._domain - - domain = property(_get_domain, _set_domain) - - def _set_path(self, path): - self['_path'] = path - self.cookie[self.key]['path'] = path - self.request['cookie_out'] = self.cookie[self.key].output(header='') - self.request['set_cookie'] = True - - def _get_path(self): - return self._path - - path = property(_get_path, _set_path) - - def _delete_cookie(self): - self.request['set_cookie'] = True - self.cookie[self.key] = self.id - if self._domain: - self.cookie[self.key]['domain'] = self._domain - if self.secure: - self.cookie[self.key]['secure'] = True - self.cookie[self.key]['path'] = '/' - expires = datetime.today().replace(year=2003) - self.cookie[self.key]['expires'] = \ - expires.strftime("%a, %d-%b-%Y %H:%M:%S GMT" ) - self.request['cookie_out'] = self.cookie[self.key].output(header='') - self.request['set_cookie'] = True - - def delete(self): - """Deletes the session from the persistent storage, and sends - an expired cookie out""" - if self.use_cookies: - self._delete_cookie() - self.clear() - - def invalidate(self): - """Invalidates this session, creates a new session id, returns - to the is_new state""" - self.clear() - self.was_invalidated = True - self._create_id() - self.load() - - def load(self): - "Loads the data from this session from persistent storage" - self.namespace = self.namespace_class(self.id, - data_dir=self.data_dir, digest_filenames=False, - **self.namespace_args) - now = time.time() - self.request['set_cookie'] = True - - self.namespace.acquire_read_lock() - timed_out = False - try: - self.clear() - try: - session_data = self.namespace['session'] - - # Memcached always returns a key, its None when its not - # present - if session_data is None: - session_data = { - '_creation_time':now, - '_accessed_time':now - } - self.is_new = True - except (KeyError, TypeError): - session_data = { - '_creation_time':now, - '_accessed_time':now - } - self.is_new = True - - if self.timeout is not None and \ - now - session_data['_accessed_time'] > self.timeout: - timed_out= True - else: - # Properly set the last_accessed time, which is different - # than the *currently* _accessed_time - if self.is_new or '_accessed_time' not in session_data: - self.last_accessed = None - else: - self.last_accessed = session_data['_accessed_time'] - - # Update the current _accessed_time - session_data['_accessed_time'] = now - self.update(session_data) - self.accessed_dict = session_data.copy() - finally: - self.namespace.release_read_lock() - if timed_out: - self.invalidate() - - def save(self, accessed_only=False): - """Saves the data for this session to persistent storage - - If accessed_only is True, then only the original data loaded - at the beginning of the request will be saved, with the updated - last accessed time. - - """ - # Look to see if its a new session that was only accessed - # Don't save it under that case - if accessed_only and self.is_new: - return None - - if not hasattr(self, 'namespace'): - self.namespace = self.namespace_class( - self.id, - data_dir=self.data_dir, - digest_filenames=False, - **self.namespace_args) - - self.namespace.acquire_write_lock() - try: - if accessed_only: - data = dict(self.accessed_dict.items()) - else: - data = dict(self.items()) - - # Save the data - if not data and 'session' in self.namespace: - del self.namespace['session'] - else: - self.namespace['session'] = data - finally: - self.namespace.release_write_lock() - if self.is_new: - self.request['set_cookie'] = True - - def revert(self): - """Revert the session to its original state from its first - access in the request""" - self.clear() - self.update(self.accessed_dict) - - # TODO: I think both these methods should be removed. They're from - # the original mod_python code i was ripping off but they really - # have no use here. - def lock(self): - """Locks this session against other processes/threads. This is - automatic when load/save is called. - - ***use with caution*** and always with a corresponding 'unlock' - inside a "finally:" block, as a stray lock typically cannot be - unlocked without shutting down the whole application. - - """ - self.namespace.acquire_write_lock() - - def unlock(self): - """Unlocks this session against other processes/threads. This - is automatic when load/save is called. - - ***use with caution*** and always within a "finally:" block, as - a stray lock typically cannot be unlocked without shutting down - the whole application. - - """ - self.namespace.release_write_lock() - -class CookieSession(Session): - """Pure cookie-based session - - Options recognized when using cookie-based sessions are slightly - more restricted than general sessions. - - ``key`` - The name the cookie should be set to. - ``timeout`` - How long session data is considered valid. This is used - regardless of the cookie being present or not to determine - whether session data is still valid. - ``encrypt_key`` - The key to use for the session encryption, if not provided the - session will not be encrypted. - ``validate_key`` - The key used to sign the encrypted session - ``cookie_domain`` - Domain to use for the cookie. - ``secure`` - Whether or not the cookie should only be sent over SSL. - - """ - def __init__(self, request, key='beaker.session.id', timeout=None, - cookie_expires=True, cookie_domain=None, encrypt_key=None, - validate_key=None, secure=False, **kwargs): - - if not crypto.has_aes and encrypt_key: - raise InvalidCryptoBackendError("No AES library is installed, can't generate " - "encrypted cookie-only Session.") - - self.request = request - self.key = key - self.timeout = timeout - self.cookie_expires = cookie_expires - self.encrypt_key = encrypt_key - self.validate_key = validate_key - self.request['set_cookie'] = False - self.secure = secure - self._domain = cookie_domain - self._path = '/' - - try: - cookieheader = request['cookie'] - except KeyError: - cookieheader = '' - - if validate_key is None: - raise BeakerException("No validate_key specified for Cookie only " - "Session.") - - try: - self.cookie = SignedCookie(validate_key, input=cookieheader) - except Cookie.CookieError: - self.cookie = SignedCookie(validate_key, input=None) - - self['_id'] = self._make_id() - self.is_new = True - - # If we have a cookie, load it - if self.key in self.cookie and self.cookie[self.key].value is not None: - self.is_new = False - try: - self.update(self._decrypt_data()) - except: - pass - if self.timeout is not None and time.time() - \ - self['_accessed_time'] > self.timeout: - self.clear() - self.accessed_dict = self.copy() - self._create_cookie() - - def created(self): - return self['_creation_time'] - created = property(created) - - def id(self): - return self['_id'] - id = property(id) - - def _set_domain(self, domain): - self['_domain'] = domain - self._domain = domain - - def _get_domain(self): - return self._domain - - domain = property(_get_domain, _set_domain) - - def _set_path(self, path): - self['_path'] = path - self._path = path - - def _get_path(self): - return self._path - - path = property(_get_path, _set_path) - - def _encrypt_data(self): - """Serialize, encipher, and base64 the session dict""" - if self.encrypt_key: - nonce = b64encode(os.urandom(40))[:8] - encrypt_key = crypto.generateCryptoKeys(self.encrypt_key, - self.validate_key + nonce, 1) - data = pickle.dumps(self.copy(), 2) - return nonce + b64encode(crypto.aesEncrypt(data, encrypt_key)) - else: - data = pickle.dumps(self.copy(), 2) - return b64encode(data) - - def _decrypt_data(self): - """Bas64, decipher, then un-serialize the data for the session - dict""" - if self.encrypt_key: - nonce = self.cookie[self.key].value[:8] - encrypt_key = crypto.generateCryptoKeys(self.encrypt_key, - self.validate_key + nonce, 1) - payload = b64decode(self.cookie[self.key].value[8:]) - data = crypto.aesDecrypt(payload, encrypt_key) - return pickle.loads(data) - else: - data = b64decode(self.cookie[self.key].value) - return pickle.loads(data) - - def _make_id(self): - return md5(md5( - "%f%s%f%s" % (time.time(), id({}), random.random(), getpid()) - ).hexdigest() - ).hexdigest() - - def save(self, accessed_only=False): - """Saves the data for this session to persistent storage""" - if accessed_only and self.is_new: - return - if accessed_only: - self.clear() - self.update(self.accessed_dict) - self._create_cookie() - - def expire(self): - """Delete the 'expires' attribute on this Session, if any.""" - - self.pop('_expires', None) - - def _create_cookie(self): - if '_creation_time' not in self: - self['_creation_time'] = time.time() - if '_id' not in self: - self['_id'] = self._make_id() - self['_accessed_time'] = time.time() - - if self.cookie_expires is not True: - if self.cookie_expires is False: - expires = datetime.fromtimestamp( 0x7FFFFFFF ) - elif isinstance(self.cookie_expires, timedelta): - expires = datetime.today() + self.cookie_expires - elif isinstance(self.cookie_expires, datetime): - expires = self.cookie_expires - else: - raise ValueError("Invalid argument for cookie_expires: %s" - % repr(self.cookie_expires)) - self['_expires'] = expires - elif '_expires' in self: - expires = self['_expires'] - else: - expires = None - - val = self._encrypt_data() - if len(val) > 4064: - raise BeakerException("Cookie value is too long to store") - - self.cookie[self.key] = val - if '_domain' in self: - self.cookie[self.key]['domain'] = self['_domain'] - elif self._domain: - self.cookie[self.key]['domain'] = self._domain - if self.secure: - self.cookie[self.key]['secure'] = True - - self.cookie[self.key]['path'] = self.get('_path', '/') - - if expires: - self.cookie[self.key]['expires'] = \ - expires.strftime("%a, %d-%b-%Y %H:%M:%S GMT" ) - self.request['cookie_out'] = self.cookie[self.key].output(header='') - self.request['set_cookie'] = True - - def delete(self): - """Delete the cookie, and clear the session""" - # Send a delete cookie request - self._delete_cookie() - self.clear() - - def invalidate(self): - """Clear the contents and start a new session""" - self.delete() - self['_id'] = self._make_id() - - -class SessionObject(object): - """Session proxy/lazy creator - - This object proxies access to the actual session object, so that in - the case that the session hasn't been used before, it will be - setup. This avoid creating and loading the session from persistent - storage unless its actually used during the request. - - """ - def __init__(self, environ, **params): - self.__dict__['_params'] = params - self.__dict__['_environ'] = environ - self.__dict__['_sess'] = None - self.__dict__['_headers'] = [] - - def _session(self): - """Lazy initial creation of session object""" - if self.__dict__['_sess'] is None: - params = self.__dict__['_params'] - environ = self.__dict__['_environ'] - self.__dict__['_headers'] = req = {'cookie_out':None} - req['cookie'] = environ.get('HTTP_COOKIE') - if params.get('type') == 'cookie': - self.__dict__['_sess'] = CookieSession(req, **params) - else: - self.__dict__['_sess'] = Session(req, use_cookies=True, - **params) - return self.__dict__['_sess'] - - def __getattr__(self, attr): - return getattr(self._session(), attr) - - def __setattr__(self, attr, value): - setattr(self._session(), attr, value) - - def __delattr__(self, name): - self._session().__delattr__(name) - - def __getitem__(self, key): - return self._session()[key] - - def __setitem__(self, key, value): - self._session()[key] = value - - def __delitem__(self, key): - self._session().__delitem__(key) - - def __repr__(self): - return self._session().__repr__() - - def __iter__(self): - """Only works for proxying to a dict""" - return iter(self._session().keys()) - - def __contains__(self, key): - return self._session().has_key(key) - - def get_by_id(self, id): - """Loads a session given a session ID""" - params = self.__dict__['_params'] - session = Session({}, use_cookies=False, id=id, **params) - if session.is_new: - return None - return session - - def save(self): - self.__dict__['_dirty'] = True - - def delete(self): - self.__dict__['_dirty'] = True - self._session().delete() - - def persist(self): - """Persist the session to the storage - - If its set to autosave, then the entire session will be saved - regardless of if save() has been called. Otherwise, just the - accessed time will be updated if save() was not called, or - the session will be saved if save() was called. - - """ - if self.__dict__['_params'].get('auto'): - self._session().save() - else: - if self.__dict__.get('_dirty'): - self._session().save() - else: - self._session().save(accessed_only=True) - - def dirty(self): - return self.__dict__.get('_dirty', False) - - def accessed(self): - """Returns whether or not the session has been accessed""" - return self.__dict__['_sess'] is not None diff --git a/module/lib/beaker/synchronization.py b/module/lib/beaker/synchronization.py deleted file mode 100644 index 761303707..000000000 --- a/module/lib/beaker/synchronization.py +++ /dev/null @@ -1,381 +0,0 @@ -"""Synchronization functions. - -File- and mutex-based mutual exclusion synchronizers are provided, -as well as a name-based mutex which locks within an application -based on a string name. - -""" - -import os -import sys -import tempfile - -try: - import threading as _threading -except ImportError: - import dummy_threading as _threading - -# check for fcntl module -try: - sys.getwindowsversion() - has_flock = False -except: - try: - import fcntl - has_flock = True - except ImportError: - has_flock = False - -from beaker import util -from beaker.exceptions import LockError - -__all__ = ["file_synchronizer", "mutex_synchronizer", "null_synchronizer", - "NameLock", "_threading"] - - -class NameLock(object): - """a proxy for an RLock object that is stored in a name based - registry. - - Multiple threads can get a reference to the same RLock based on the - name alone, and synchronize operations related to that name. - - """ - locks = util.WeakValuedRegistry() - - class NLContainer(object): - def __init__(self, reentrant): - if reentrant: - self.lock = _threading.RLock() - else: - self.lock = _threading.Lock() - def __call__(self): - return self.lock - - def __init__(self, identifier = None, reentrant = False): - if identifier is None: - self._lock = NameLock.NLContainer(reentrant) - else: - self._lock = NameLock.locks.get(identifier, NameLock.NLContainer, - reentrant) - - def acquire(self, wait = True): - return self._lock().acquire(wait) - - def release(self): - self._lock().release() - - -_synchronizers = util.WeakValuedRegistry() -def _synchronizer(identifier, cls, **kwargs): - return _synchronizers.sync_get((identifier, cls), cls, identifier, **kwargs) - - -def file_synchronizer(identifier, **kwargs): - if not has_flock or 'lock_dir' not in kwargs: - return mutex_synchronizer(identifier) - else: - return _synchronizer(identifier, FileSynchronizer, **kwargs) - - -def mutex_synchronizer(identifier, **kwargs): - return _synchronizer(identifier, ConditionSynchronizer, **kwargs) - - -class null_synchronizer(object): - def acquire_write_lock(self, wait=True): - return True - def acquire_read_lock(self): - pass - def release_write_lock(self): - pass - def release_read_lock(self): - pass - acquire = acquire_write_lock - release = release_write_lock - - -class SynchronizerImpl(object): - def __init__(self): - self._state = util.ThreadLocal() - - class SyncState(object): - __slots__ = 'reentrantcount', 'writing', 'reading' - - def __init__(self): - self.reentrantcount = 0 - self.writing = False - self.reading = False - - def state(self): - if not self._state.has(): - state = SynchronizerImpl.SyncState() - self._state.put(state) - return state - else: - return self._state.get() - state = property(state) - - def release_read_lock(self): - state = self.state - - if state.writing: - raise LockError("lock is in writing state") - if not state.reading: - raise LockError("lock is not in reading state") - - if state.reentrantcount == 1: - self.do_release_read_lock() - state.reading = False - - state.reentrantcount -= 1 - - def acquire_read_lock(self, wait = True): - state = self.state - - if state.writing: - raise LockError("lock is in writing state") - - if state.reentrantcount == 0: - x = self.do_acquire_read_lock(wait) - if (wait or x): - state.reentrantcount += 1 - state.reading = True - return x - elif state.reading: - state.reentrantcount += 1 - return True - - def release_write_lock(self): - state = self.state - - if state.reading: - raise LockError("lock is in reading state") - if not state.writing: - raise LockError("lock is not in writing state") - - if state.reentrantcount == 1: - self.do_release_write_lock() - state.writing = False - - state.reentrantcount -= 1 - - release = release_write_lock - - def acquire_write_lock(self, wait = True): - state = self.state - - if state.reading: - raise LockError("lock is in reading state") - - if state.reentrantcount == 0: - x = self.do_acquire_write_lock(wait) - if (wait or x): - state.reentrantcount += 1 - state.writing = True - return x - elif state.writing: - state.reentrantcount += 1 - return True - - acquire = acquire_write_lock - - def do_release_read_lock(self): - raise NotImplementedError() - - def do_acquire_read_lock(self): - raise NotImplementedError() - - def do_release_write_lock(self): - raise NotImplementedError() - - def do_acquire_write_lock(self): - raise NotImplementedError() - - -class FileSynchronizer(SynchronizerImpl): - """a synchronizer which locks using flock(). - - Adapted for Python/multithreads from Apache::Session::Lock::File, - http://search.cpan.org/src/CWEST/Apache-Session-1.81/Session/Lock/File.pm - - This module does not unlink temporary files, - because it interferes with proper locking. This can cause - problems on certain systems (Linux) whose file systems (ext2) do not - perform well with lots of files in one directory. To prevent this - you should use a script to clean out old files from your lock directory. - - """ - def __init__(self, identifier, lock_dir): - super(FileSynchronizer, self).__init__() - self._filedescriptor = util.ThreadLocal() - - if lock_dir is None: - lock_dir = tempfile.gettempdir() - else: - lock_dir = lock_dir - - self.filename = util.encoded_path( - lock_dir, - [identifier], - extension='.lock' - ) - - def _filedesc(self): - return self._filedescriptor.get() - _filedesc = property(_filedesc) - - def _open(self, mode): - filedescriptor = self._filedesc - if filedescriptor is None: - filedescriptor = os.open(self.filename, mode) - self._filedescriptor.put(filedescriptor) - return filedescriptor - - def do_acquire_read_lock(self, wait): - filedescriptor = self._open(os.O_CREAT | os.O_RDONLY) - if not wait: - try: - fcntl.flock(filedescriptor, fcntl.LOCK_SH | fcntl.LOCK_NB) - return True - except IOError: - os.close(filedescriptor) - self._filedescriptor.remove() - return False - else: - fcntl.flock(filedescriptor, fcntl.LOCK_SH) - return True - - def do_acquire_write_lock(self, wait): - filedescriptor = self._open(os.O_CREAT | os.O_WRONLY) - if not wait: - try: - fcntl.flock(filedescriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) - return True - except IOError: - os.close(filedescriptor) - self._filedescriptor.remove() - return False - else: - fcntl.flock(filedescriptor, fcntl.LOCK_EX) - return True - - def do_release_read_lock(self): - self._release_all_locks() - - def do_release_write_lock(self): - self._release_all_locks() - - def _release_all_locks(self): - filedescriptor = self._filedesc - if filedescriptor is not None: - fcntl.flock(filedescriptor, fcntl.LOCK_UN) - os.close(filedescriptor) - self._filedescriptor.remove() - - -class ConditionSynchronizer(SynchronizerImpl): - """a synchronizer using a Condition.""" - - def __init__(self, identifier): - super(ConditionSynchronizer, self).__init__() - - # counts how many asynchronous methods are executing - self.async = 0 - - # pointer to thread that is the current sync operation - self.current_sync_operation = None - - # condition object to lock on - self.condition = _threading.Condition(_threading.Lock()) - - def do_acquire_read_lock(self, wait = True): - self.condition.acquire() - try: - # see if a synchronous operation is waiting to start - # or is already running, in which case we wait (or just - # give up and return) - if wait: - while self.current_sync_operation is not None: - self.condition.wait() - else: - if self.current_sync_operation is not None: - return False - - self.async += 1 - finally: - self.condition.release() - - if not wait: - return True - - def do_release_read_lock(self): - self.condition.acquire() - try: - self.async -= 1 - - # check if we are the last asynchronous reader thread - # out the door. - if self.async == 0: - # yes. so if a sync operation is waiting, notifyAll to wake - # it up - if self.current_sync_operation is not None: - self.condition.notifyAll() - elif self.async < 0: - raise LockError("Synchronizer error - too many " - "release_read_locks called") - finally: - self.condition.release() - - def do_acquire_write_lock(self, wait = True): - self.condition.acquire() - try: - # here, we are not a synchronous reader, and after returning, - # assuming waiting or immediate availability, we will be. - - if wait: - # if another sync is working, wait - while self.current_sync_operation is not None: - self.condition.wait() - else: - # if another sync is working, - # we dont want to wait, so forget it - if self.current_sync_operation is not None: - return False - - # establish ourselves as the current sync - # this indicates to other read/write operations - # that they should wait until this is None again - self.current_sync_operation = _threading.currentThread() - - # now wait again for asyncs to finish - if self.async > 0: - if wait: - # wait - self.condition.wait() - else: - # we dont want to wait, so forget it - self.current_sync_operation = None - return False - finally: - self.condition.release() - - if not wait: - return True - - def do_release_write_lock(self): - self.condition.acquire() - try: - if self.current_sync_operation is not _threading.currentThread(): - raise LockError("Synchronizer error - current thread doesnt " - "have the write lock") - - # reset the current sync operation so - # another can get it - self.current_sync_operation = None - - # tell everyone to get ready - self.condition.notifyAll() - finally: - # everyone go !! - self.condition.release() diff --git a/module/lib/beaker/util.py b/module/lib/beaker/util.py deleted file mode 100644 index 04c9617c5..000000000 --- a/module/lib/beaker/util.py +++ /dev/null @@ -1,302 +0,0 @@ -"""Beaker utilities""" - -try: - import thread as _thread - import threading as _threading -except ImportError: - import dummy_thread as _thread - import dummy_threading as _threading - -from datetime import datetime, timedelta -import os -import string -import types -import weakref -import warnings -import sys - -py3k = getattr(sys, 'py3kwarning', False) or sys.version_info >= (3, 0) -py24 = sys.version_info < (2,5) -jython = sys.platform.startswith('java') - -if py3k or jython: - import pickle -else: - import cPickle as pickle - -from beaker.converters import asbool -from threading import local as _tlocal - - -__all__ = ["ThreadLocal", "Registry", "WeakValuedRegistry", "SyncDict", - "encoded_path", "verify_directory"] - - -def verify_directory(dir): - """verifies and creates a directory. tries to - ignore collisions with other threads and processes.""" - - tries = 0 - while not os.access(dir, os.F_OK): - try: - tries += 1 - os.makedirs(dir) - except: - if tries > 5: - raise - - -def deprecated(message): - def wrapper(fn): - def deprecated_method(*args, **kargs): - warnings.warn(message, DeprecationWarning, 2) - return fn(*args, **kargs) - # TODO: use decorator ? functools.wrapper ? - deprecated_method.__name__ = fn.__name__ - deprecated_method.__doc__ = "%s\n\n%s" % (message, fn.__doc__) - return deprecated_method - return wrapper - -class ThreadLocal(object): - """stores a value on a per-thread basis""" - - __slots__ = '_tlocal' - - def __init__(self): - self._tlocal = _tlocal() - - def put(self, value): - self._tlocal.value = value - - def has(self): - return hasattr(self._tlocal, 'value') - - def get(self, default=None): - return getattr(self._tlocal, 'value', default) - - def remove(self): - del self._tlocal.value - -class SyncDict(object): - """ - An efficient/threadsafe singleton map algorithm, a.k.a. - "get a value based on this key, and create if not found or not - valid" paradigm: - - exists && isvalid ? get : create - - Designed to work with weakref dictionaries to expect items - to asynchronously disappear from the dictionary. - - Use python 2.3.3 or greater ! a major bug was just fixed in Nov. - 2003 that was driving me nuts with garbage collection/weakrefs in - this section. - - """ - def __init__(self): - self.mutex = _thread.allocate_lock() - self.dict = {} - - def get(self, key, createfunc, *args, **kwargs): - try: - if self.has_key(key): - return self.dict[key] - else: - return self.sync_get(key, createfunc, *args, **kwargs) - except KeyError: - return self.sync_get(key, createfunc, *args, **kwargs) - - def sync_get(self, key, createfunc, *args, **kwargs): - self.mutex.acquire() - try: - try: - if self.has_key(key): - return self.dict[key] - else: - return self._create(key, createfunc, *args, **kwargs) - except KeyError: - return self._create(key, createfunc, *args, **kwargs) - finally: - self.mutex.release() - - def _create(self, key, createfunc, *args, **kwargs): - self[key] = obj = createfunc(*args, **kwargs) - return obj - - def has_key(self, key): - return self.dict.has_key(key) - - def __contains__(self, key): - return self.dict.__contains__(key) - def __getitem__(self, key): - return self.dict.__getitem__(key) - def __setitem__(self, key, value): - self.dict.__setitem__(key, value) - def __delitem__(self, key): - return self.dict.__delitem__(key) - def clear(self): - self.dict.clear() - - -class WeakValuedRegistry(SyncDict): - def __init__(self): - self.mutex = _threading.RLock() - self.dict = weakref.WeakValueDictionary() - -sha1 = None -def encoded_path(root, identifiers, extension = ".enc", depth = 3, - digest_filenames=True): - - """Generate a unique file-accessible path from the given list of - identifiers starting at the given root directory.""" - ident = "_".join(identifiers) - - global sha1 - if sha1 is None: - from beaker.crypto import sha1 - - if digest_filenames: - if py3k: - ident = sha1(ident.encode('utf-8')).hexdigest() - else: - ident = sha1(ident).hexdigest() - - ident = os.path.basename(ident) - - tokens = [] - for d in range(1, depth): - tokens.append(ident[0:d]) - - dir = os.path.join(root, *tokens) - verify_directory(dir) - - return os.path.join(dir, ident + extension) - - -def verify_options(opt, types, error): - if not isinstance(opt, types): - if not isinstance(types, tuple): - types = (types,) - coerced = False - for typ in types: - try: - if typ in (list, tuple): - opt = [x.strip() for x in opt.split(',')] - else: - if typ == bool: - typ = asbool - opt = typ(opt) - coerced = True - except: - pass - if coerced: - break - if not coerced: - raise Exception(error) - elif isinstance(opt, str) and not opt.strip(): - raise Exception("Empty strings are invalid for: %s" % error) - return opt - - -def verify_rules(params, ruleset): - for key, types, message in ruleset: - if key in params: - params[key] = verify_options(params[key], types, message) - return params - - -def coerce_session_params(params): - rules = [ - ('data_dir', (str, types.NoneType), "data_dir must be a string " - "referring to a directory."), - ('lock_dir', (str, types.NoneType), "lock_dir must be a string referring to a " - "directory."), - ('type', (str, types.NoneType), "Session type must be a string."), - ('cookie_expires', (bool, datetime, timedelta), "Cookie expires was " - "not a boolean, datetime, or timedelta instance."), - ('cookie_domain', (str, types.NoneType), "Cookie domain must be a " - "string."), - ('id', (str,), "Session id must be a string."), - ('key', (str,), "Session key must be a string."), - ('secret', (str, types.NoneType), "Session secret must be a string."), - ('validate_key', (str, types.NoneType), "Session encrypt_key must be " - "a string."), - ('encrypt_key', (str, types.NoneType), "Session validate_key must be " - "a string."), - ('secure', (bool, types.NoneType), "Session secure must be a boolean."), - ('timeout', (int, types.NoneType), "Session timeout must be an " - "integer."), - ('auto', (bool, types.NoneType), "Session is created if accessed."), - ] - return verify_rules(params, rules) - - -def coerce_cache_params(params): - rules = [ - ('data_dir', (str, types.NoneType), "data_dir must be a string " - "referring to a directory."), - ('lock_dir', (str, types.NoneType), "lock_dir must be a string referring to a " - "directory."), - ('type', (str,), "Cache type must be a string."), - ('enabled', (bool, types.NoneType), "enabled must be true/false " - "if present."), - ('expire', (int, types.NoneType), "expire must be an integer representing " - "how many seconds the cache is valid for"), - ('regions', (list, tuple, types.NoneType), "Regions must be a " - "comma seperated list of valid regions") - ] - return verify_rules(params, rules) - - -def parse_cache_config_options(config, include_defaults=True): - """Parse configuration options and validate for use with the - CacheManager""" - - # Load default cache options - if include_defaults: - options= dict(type='memory', data_dir=None, expire=None, - log_file=None) - else: - options = {} - for key, val in config.iteritems(): - if key.startswith('beaker.cache.'): - options[key[13:]] = val - if key.startswith('cache.'): - options[key[6:]] = val - coerce_cache_params(options) - - # Set cache to enabled if not turned off - if 'enabled' not in options: - options['enabled'] = True - - # Configure region dict if regions are available - regions = options.pop('regions', None) - if regions: - region_configs = {} - for region in regions: - # Setup the default cache options - region_options = dict(data_dir=options.get('data_dir'), - lock_dir=options.get('lock_dir'), - type=options.get('type'), - enabled=options['enabled'], - expire=options.get('expire')) - region_len = len(region) + 1 - for key in options.keys(): - if key.startswith('%s.' % region): - region_options[key[region_len:]] = options.pop(key) - coerce_cache_params(region_options) - region_configs[region] = region_options - options['cache_regions'] = region_configs - return options - -def func_namespace(func): - """Generates a unique namespace for a function""" - kls = None - if hasattr(func, 'im_func'): - kls = func.im_class - func = func.im_func - - if kls: - return '%s.%s' % (kls.__module__, kls.__name__) - else: - return '%s.%s' % (func.__module__, func.__name__) diff --git a/module/lib/bottle.py b/module/lib/bottle.py deleted file mode 100644 index b00bda1c9..000000000 --- a/module/lib/bottle.py +++ /dev/null @@ -1,3251 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -Bottle is a fast and simple micro-framework for small web applications. It -offers request dispatching (Routes) with url parameter support, templates, -a built-in HTTP Server and adapters for many third party WSGI/HTTP-server and -template engines - all in a single file and with no dependencies other than the -Python Standard Library. - -Homepage and documentation: http://bottlepy.org/ - -Copyright (c) 2012, Marcel Hellkamp. -License: MIT (see LICENSE for details) -""" - -from __future__ import with_statement - -__author__ = 'Marcel Hellkamp' -__version__ = '0.11.4' -__license__ = 'MIT' - -# The gevent server adapter needs to patch some modules before they are imported -# This is why we parse the commandline parameters here but handle them later -if __name__ == '__main__': - from optparse import OptionParser - _cmd_parser = OptionParser(usage="usage: %prog [options] package.module:app") - _opt = _cmd_parser.add_option - _opt("--version", action="store_true", help="show version number.") - _opt("-b", "--bind", metavar="ADDRESS", help="bind socket to ADDRESS.") - _opt("-s", "--server", default='wsgiref', help="use SERVER as backend.") - _opt("-p", "--plugin", action="append", help="install additional plugin/s.") - _opt("--debug", action="store_true", help="start server in debug mode.") - _opt("--reload", action="store_true", help="auto-reload on file changes.") - _cmd_options, _cmd_args = _cmd_parser.parse_args() - if _cmd_options.server and _cmd_options.server.startswith('gevent'): - import gevent.monkey; gevent.monkey.patch_all() - -import base64, cgi, email.utils, functools, hmac, imp, itertools, mimetypes,\ - os, re, subprocess, sys, tempfile, threading, time, urllib, warnings - -from datetime import date as datedate, datetime, timedelta -from tempfile import TemporaryFile -from traceback import format_exc, print_exc - -try: from json import dumps as json_dumps, loads as json_lds -except ImportError: # pragma: no cover - try: from simplejson import dumps as json_dumps, loads as json_lds - except ImportError: - try: from django.utils.simplejson import dumps as json_dumps, loads as json_lds - except ImportError: - def json_dumps(data): - raise ImportError("JSON support requires Python 2.6 or simplejson.") - json_lds = json_dumps - - - -# We now try to fix 2.5/2.6/3.1/3.2 incompatibilities. -# It ain't pretty but it works... Sorry for the mess. - -py = sys.version_info -py3k = py >= (3,0,0) -py25 = py < (2,6,0) -py31 = (3,1,0) <= py < (3,2,0) - -# Workaround for the missing "as" keyword in py3k. -def _e(): return sys.exc_info()[1] - -# Workaround for the "print is a keyword/function" Python 2/3 dilemma -# and a fallback for mod_wsgi (resticts stdout/err attribute access) -try: - _stdout, _stderr = sys.stdout.write, sys.stderr.write -except IOError: - _stdout = lambda x: sys.stdout.write(x) - _stderr = lambda x: sys.stderr.write(x) - -# Lots of stdlib and builtin differences. -if py3k: - import http.client as httplib - import _thread as thread - from urllib.parse import urljoin, SplitResult as UrlSplitResult - from urllib.parse import urlencode, quote as urlquote, unquote as urlunquote - urlunquote = functools.partial(urlunquote, encoding='latin1') - from http.cookies import SimpleCookie - from collections import MutableMapping as DictMixin - import pickle - from io import BytesIO - basestring = str - unicode = str - json_loads = lambda s: json_lds(touni(s)) - callable = lambda x: hasattr(x, '__call__') - imap = map -else: # 2.x - import httplib - import thread - from urlparse import urljoin, SplitResult as UrlSplitResult - from urllib import urlencode, quote as urlquote, unquote as urlunquote - from Cookie import SimpleCookie - from itertools import imap - import cPickle as pickle - from StringIO import StringIO as BytesIO - if py25: - from UserDict import DictMixin - def next(it): return it.next() - bytes = str - else: # 2.6, 2.7 - from collections import MutableMapping as DictMixin - json_loads = json_lds - -# Some helpers for string/byte handling -def tob(s, enc='utf8'): - return s.encode(enc) if isinstance(s, unicode) else bytes(s) -def touni(s, enc='utf8', err='strict'): - return s.decode(enc, err) if isinstance(s, bytes) else unicode(s) -tonat = touni if py3k else tob - -# 3.2 fixes cgi.FieldStorage to accept bytes (which makes a lot of sense). -# 3.1 needs a workaround. -if py31: - from io import TextIOWrapper - class NCTextIOWrapper(TextIOWrapper): - def close(self): pass # Keep wrapped buffer open. - -# File uploads (which are implemented as empty FiledStorage instances...) -# have a negative truth value. That makes no sense, here is a fix. -class FieldStorage(cgi.FieldStorage): - def __nonzero__(self): return bool(self.list or self.file) - if py3k: __bool__ = __nonzero__ - -# A bug in functools causes it to break if the wrapper is an instance method -def update_wrapper(wrapper, wrapped, *a, **ka): - try: functools.update_wrapper(wrapper, wrapped, *a, **ka) - except AttributeError: pass - - - -# These helpers are used at module level and need to be defined first. -# And yes, I know PEP-8, but sometimes a lower-case classname makes more sense. - -def depr(message): - warnings.warn(message, DeprecationWarning, stacklevel=3) - -def makelist(data): # This is just to handy - if isinstance(data, (tuple, list, set, dict)): return list(data) - elif data: return [data] - else: return [] - - -class DictProperty(object): - ''' Property that maps to a key in a local dict-like attribute. ''' - def __init__(self, attr, key=None, read_only=False): - self.attr, self.key, self.read_only = attr, key, read_only - - def __call__(self, func): - functools.update_wrapper(self, func, updated=[]) - self.getter, self.key = func, self.key or func.__name__ - return self - - def __get__(self, obj, cls): - if obj is None: return self - key, storage = self.key, getattr(obj, self.attr) - if key not in storage: storage[key] = self.getter(obj) - return storage[key] - - def __set__(self, obj, value): - if self.read_only: raise AttributeError("Read-Only property.") - getattr(obj, self.attr)[self.key] = value - - def __delete__(self, obj): - if self.read_only: raise AttributeError("Read-Only property.") - del getattr(obj, self.attr)[self.key] - - -class cached_property(object): - ''' A property that is only computed once per instance and then replaces - itself with an ordinary attribute. Deleting the attribute resets the - property. ''' - - def __init__(self, func): - self.func = func - - def __get__(self, obj, cls): - if obj is None: return self - value = obj.__dict__[self.func.__name__] = self.func(obj) - return value - - -class lazy_attribute(object): - ''' A property that caches itself to the class object. ''' - def __init__(self, func): - functools.update_wrapper(self, func, updated=[]) - self.getter = func - - def __get__(self, obj, cls): - value = self.getter(cls) - setattr(cls, self.__name__, value) - return value - - - - - - -############################################################################### -# Exceptions and Events ######################################################## -############################################################################### - - -class BottleException(Exception): - """ A base class for exceptions used by bottle. """ - pass - - - - - - -############################################################################### -# Routing ###################################################################### -############################################################################### - - -class RouteError(BottleException): - """ This is a base class for all routing related exceptions """ - - -class RouteReset(BottleException): - """ If raised by a plugin or request handler, the route is reset and all - plugins are re-applied. """ - -class RouterUnknownModeError(RouteError): pass - - -class RouteSyntaxError(RouteError): - """ The route parser found something not supported by this router """ - - -class RouteBuildError(RouteError): - """ The route could not been built """ - - -class Router(object): - ''' A Router is an ordered collection of route->target pairs. It is used to - efficiently match WSGI requests against a number of routes and return - the first target that satisfies the request. The target may be anything, - usually a string, ID or callable object. A route consists of a path-rule - and a HTTP method. - - The path-rule is either a static path (e.g. `/contact`) or a dynamic - path that contains wildcards (e.g. `/wiki/`). The wildcard syntax - and details on the matching order are described in docs:`routing`. - ''' - - default_pattern = '[^/]+' - default_filter = 're' - #: Sorry for the mess. It works. Trust me. - rule_syntax = re.compile('(\\\\*)'\ - '(?:(?::([a-zA-Z_][a-zA-Z_0-9]*)?()(?:#(.*?)#)?)'\ - '|(?:<([a-zA-Z_][a-zA-Z_0-9]*)?(?::([a-zA-Z_]*)'\ - '(?::((?:\\\\.|[^\\\\>]+)+)?)?)?>))') - - def __init__(self, strict=False): - self.rules = {} # A {rule: Rule} mapping - self.builder = {} # A rule/name->build_info mapping - self.static = {} # Cache for static routes: {path: {method: target}} - self.dynamic = [] # Cache for dynamic routes. See _compile() - #: If true, static routes are no longer checked first. - self.strict_order = strict - self.filters = {'re': self.re_filter, 'int': self.int_filter, - 'float': self.float_filter, 'path': self.path_filter} - - def re_filter(self, conf): - return conf or self.default_pattern, None, None - - def int_filter(self, conf): - return r'-?\d+', int, lambda x: str(int(x)) - - def float_filter(self, conf): - return r'-?[\d.]+', float, lambda x: str(float(x)) - - def path_filter(self, conf): - return r'.+?', None, None - - def add_filter(self, name, func): - ''' Add a filter. The provided function is called with the configuration - string as parameter and must return a (regexp, to_python, to_url) tuple. - The first element is a string, the last two are callables or None. ''' - self.filters[name] = func - - def parse_rule(self, rule): - ''' Parses a rule into a (name, filter, conf) token stream. If mode is - None, name contains a static rule part. ''' - offset, prefix = 0, '' - for match in self.rule_syntax.finditer(rule): - prefix += rule[offset:match.start()] - g = match.groups() - if len(g[0])%2: # Escaped wildcard - prefix += match.group(0)[len(g[0]):] - offset = match.end() - continue - if prefix: yield prefix, None, None - name, filtr, conf = g[1:4] if not g[2] is None else g[4:7] - if not filtr: filtr = self.default_filter - yield name, filtr, conf or None - offset, prefix = match.end(), '' - if offset <= len(rule) or prefix: - yield prefix+rule[offset:], None, None - - def add(self, rule, method, target, name=None): - ''' Add a new route or replace the target for an existing route. ''' - if rule in self.rules: - self.rules[rule][method] = target - if name: self.builder[name] = self.builder[rule] - return - - target = self.rules[rule] = {method: target} - - # Build pattern and other structures for dynamic routes - anons = 0 # Number of anonymous wildcards - pattern = '' # Regular expression pattern - filters = [] # Lists of wildcard input filters - builder = [] # Data structure for the URL builder - is_static = True - for key, mode, conf in self.parse_rule(rule): - if mode: - is_static = False - mask, in_filter, out_filter = self.filters[mode](conf) - if key: - pattern += '(?P<%s>%s)' % (key, mask) - else: - pattern += '(?:%s)' % mask - key = 'anon%d' % anons; anons += 1 - if in_filter: filters.append((key, in_filter)) - builder.append((key, out_filter or str)) - elif key: - pattern += re.escape(key) - builder.append((None, key)) - self.builder[rule] = builder - if name: self.builder[name] = builder - - if is_static and not self.strict_order: - self.static[self.build(rule)] = target - return - - def fpat_sub(m): - return m.group(0) if len(m.group(1)) % 2 else m.group(1) + '(?:' - flat_pattern = re.sub(r'(\\*)(\(\?P<[^>]*>|\((?!\?))', fpat_sub, pattern) - - try: - re_match = re.compile('^(%s)$' % pattern).match - except re.error: - raise RouteSyntaxError("Could not add Route: %s (%s)" % (rule, _e())) - - def match(path): - """ Return an url-argument dictionary. """ - url_args = re_match(path).groupdict() - for name, wildcard_filter in filters: - try: - url_args[name] = wildcard_filter(url_args[name]) - except ValueError: - raise HTTPError(400, 'Path has wrong format.') - return url_args - - try: - combined = '%s|(^%s$)' % (self.dynamic[-1][0].pattern, flat_pattern) - self.dynamic[-1] = (re.compile(combined), self.dynamic[-1][1]) - self.dynamic[-1][1].append((match, target)) - except (AssertionError, IndexError): # AssertionError: Too many groups - self.dynamic.append((re.compile('(^%s$)' % flat_pattern), - [(match, target)])) - return match - - def build(self, _name, *anons, **query): - ''' Build an URL by filling the wildcards in a rule. ''' - builder = self.builder.get(_name) - if not builder: raise RouteBuildError("No route with that name.", _name) - try: - for i, value in enumerate(anons): query['anon%d'%i] = value - url = ''.join([f(query.pop(n)) if n else f for (n,f) in builder]) - return url if not query else url+'?'+urlencode(query) - except KeyError: - raise RouteBuildError('Missing URL argument: %r' % _e().args[0]) - - def match(self, environ): - ''' Return a (target, url_agrs) tuple or raise HTTPError(400/404/405). ''' - path, targets, urlargs = environ['PATH_INFO'] or '/', None, {} - if path in self.static: - targets = self.static[path] - else: - for combined, rules in self.dynamic: - match = combined.match(path) - if not match: continue - getargs, targets = rules[match.lastindex - 1] - urlargs = getargs(path) if getargs else {} - break - - if not targets: - raise HTTPError(404, "Not found: " + repr(environ['PATH_INFO'])) - method = environ['REQUEST_METHOD'].upper() - if method in targets: - return targets[method], urlargs - if method == 'HEAD' and 'GET' in targets: - return targets['GET'], urlargs - if 'ANY' in targets: - return targets['ANY'], urlargs - allowed = [verb for verb in targets if verb != 'ANY'] - if 'GET' in allowed and 'HEAD' not in allowed: - allowed.append('HEAD') - raise HTTPError(405, "Method not allowed.", Allow=",".join(allowed)) - - -class Route(object): - ''' This class wraps a route callback along with route specific metadata and - configuration and applies Plugins on demand. It is also responsible for - turing an URL path rule into a regular expression usable by the Router. - ''' - - def __init__(self, app, rule, method, callback, name=None, - plugins=None, skiplist=None, **config): - #: The application this route is installed to. - self.app = app - #: The path-rule string (e.g. ``/wiki/:page``). - self.rule = rule - #: The HTTP method as a string (e.g. ``GET``). - self.method = method - #: The original callback with no plugins applied. Useful for introspection. - self.callback = callback - #: The name of the route (if specified) or ``None``. - self.name = name or None - #: A list of route-specific plugins (see :meth:`Bottle.route`). - self.plugins = plugins or [] - #: A list of plugins to not apply to this route (see :meth:`Bottle.route`). - self.skiplist = skiplist or [] - #: Additional keyword arguments passed to the :meth:`Bottle.route` - #: decorator are stored in this dictionary. Used for route-specific - #: plugin configuration and meta-data. - self.config = ConfigDict(config) - - def __call__(self, *a, **ka): - depr("Some APIs changed to return Route() instances instead of"\ - " callables. Make sure to use the Route.call method and not to"\ - " call Route instances directly.") - return self.call(*a, **ka) - - @cached_property - def call(self): - ''' The route callback with all plugins applied. This property is - created on demand and then cached to speed up subsequent requests.''' - return self._make_callback() - - def reset(self): - ''' Forget any cached values. The next time :attr:`call` is accessed, - all plugins are re-applied. ''' - self.__dict__.pop('call', None) - - def prepare(self): - ''' Do all on-demand work immediately (useful for debugging).''' - self.call - - @property - def _context(self): - depr('Switch to Plugin API v2 and access the Route object directly.') - return dict(rule=self.rule, method=self.method, callback=self.callback, - name=self.name, app=self.app, config=self.config, - apply=self.plugins, skip=self.skiplist) - - def all_plugins(self): - ''' Yield all Plugins affecting this route. ''' - unique = set() - for p in reversed(self.app.plugins + self.plugins): - if True in self.skiplist: break - name = getattr(p, 'name', False) - if name and (name in self.skiplist or name in unique): continue - if p in self.skiplist or type(p) in self.skiplist: continue - if name: unique.add(name) - yield p - - def _make_callback(self): - callback = self.callback - for plugin in self.all_plugins(): - try: - if hasattr(plugin, 'apply'): - api = getattr(plugin, 'api', 1) - context = self if api > 1 else self._context - callback = plugin.apply(callback, context) - else: - callback = plugin(callback) - except RouteReset: # Try again with changed configuration. - return self._make_callback() - if not callback is self.callback: - update_wrapper(callback, self.callback) - return callback - - def __repr__(self): - return '<%s %r %r>' % (self.method, self.rule, self.callback) - - - - - - -############################################################################### -# Application Object ########################################################### -############################################################################### - - -class Bottle(object): - """ Each Bottle object represents a single, distinct web application and - consists of routes, callbacks, plugins, resources and configuration. - Instances are callable WSGI applications. - - :param catchall: If true (default), handle all exceptions. Turn off to - let debugging middleware handle exceptions. - """ - - def __init__(self, catchall=True, autojson=True): - #: If true, most exceptions are caught and returned as :exc:`HTTPError` - self.catchall = catchall - - #: A :class:`ResourceManager` for application files - self.resources = ResourceManager() - - #: A :class:`ConfigDict` for app specific configuration. - self.config = ConfigDict() - self.config.autojson = autojson - - self.routes = [] # List of installed :class:`Route` instances. - self.router = Router() # Maps requests to :class:`Route` instances. - self.error_handler = {} - - # Core plugins - self.plugins = [] # List of installed plugins. - self.hooks = HooksPlugin() - self.install(self.hooks) - if self.config.autojson: - self.install(JSONPlugin()) - self.install(TemplatePlugin()) - - - def mount(self, prefix, app, **options): - ''' Mount an application (:class:`Bottle` or plain WSGI) to a specific - URL prefix. Example:: - - root_app.mount('/admin/', admin_app) - - :param prefix: path prefix or `mount-point`. If it ends in a slash, - that slash is mandatory. - :param app: an instance of :class:`Bottle` or a WSGI application. - - All other parameters are passed to the underlying :meth:`route` call. - ''' - if isinstance(app, basestring): - prefix, app = app, prefix - depr('Parameter order of Bottle.mount() changed.') # 0.10 - - segments = [p for p in prefix.split('/') if p] - if not segments: raise ValueError('Empty path prefix.') - path_depth = len(segments) - - def mountpoint_wrapper(): - try: - request.path_shift(path_depth) - rs = BaseResponse([], 200) - def start_response(status, header): - rs.status = status - for name, value in header: rs.add_header(name, value) - return rs.body.append - body = app(request.environ, start_response) - body = itertools.chain(rs.body, body) - return HTTPResponse(body, rs.status_code, **rs.headers) - finally: - request.path_shift(-path_depth) - - options.setdefault('skip', True) - options.setdefault('method', 'ANY') - options.setdefault('mountpoint', {'prefix': prefix, 'target': app}) - options['callback'] = mountpoint_wrapper - - self.route('/%s/<:re:.*>' % '/'.join(segments), **options) - if not prefix.endswith('/'): - self.route('/' + '/'.join(segments), **options) - - def merge(self, routes): - ''' Merge the routes of another :class:`Bottle` application or a list of - :class:`Route` objects into this application. The routes keep their - 'owner', meaning that the :data:`Route.app` attribute is not - changed. ''' - if isinstance(routes, Bottle): - routes = routes.routes - for route in routes: - self.add_route(route) - - def install(self, plugin): - ''' Add a plugin to the list of plugins and prepare it for being - applied to all routes of this application. A plugin may be a simple - decorator or an object that implements the :class:`Plugin` API. - ''' - if hasattr(plugin, 'setup'): plugin.setup(self) - if not callable(plugin) and not hasattr(plugin, 'apply'): - raise TypeError("Plugins must be callable or implement .apply()") - self.plugins.append(plugin) - self.reset() - return plugin - - def uninstall(self, plugin): - ''' Uninstall plugins. Pass an instance to remove a specific plugin, a type - object to remove all plugins that match that type, a string to remove - all plugins with a matching ``name`` attribute or ``True`` to remove all - plugins. Return the list of removed plugins. ''' - removed, remove = [], plugin - for i, plugin in list(enumerate(self.plugins))[::-1]: - if remove is True or remove is plugin or remove is type(plugin) \ - or getattr(plugin, 'name', True) == remove: - removed.append(plugin) - del self.plugins[i] - if hasattr(plugin, 'close'): plugin.close() - if removed: self.reset() - return removed - - def run(self, **kwargs): - ''' Calls :func:`run` with the same parameters. ''' - run(self, **kwargs) - - def reset(self, route=None): - ''' Reset all routes (force plugins to be re-applied) and clear all - caches. If an ID or route object is given, only that specific route - is affected. ''' - if route is None: routes = self.routes - elif isinstance(route, Route): routes = [route] - else: routes = [self.routes[route]] - for route in routes: route.reset() - if DEBUG: - for route in routes: route.prepare() - self.hooks.trigger('app_reset') - - def close(self): - ''' Close the application and all installed plugins. ''' - for plugin in self.plugins: - if hasattr(plugin, 'close'): plugin.close() - self.stopped = True - - def match(self, environ): - """ Search for a matching route and return a (:class:`Route` , urlargs) - tuple. The second value is a dictionary with parameters extracted - from the URL. Raise :exc:`HTTPError` (404/405) on a non-match.""" - return self.router.match(environ) - - def get_url(self, routename, **kargs): - """ Return a string that matches a named route """ - scriptname = request.environ.get('SCRIPT_NAME', '').strip('/') + '/' - location = self.router.build(routename, **kargs).lstrip('/') - return urljoin(urljoin('/', scriptname), location) - - def add_route(self, route): - ''' Add a route object, but do not change the :data:`Route.app` - attribute.''' - self.routes.append(route) - self.router.add(route.rule, route.method, route, name=route.name) - if DEBUG: route.prepare() - - def route(self, path=None, method='GET', callback=None, name=None, - apply=None, skip=None, **config): - """ A decorator to bind a function to a request URL. Example:: - - @app.route('/hello/:name') - def hello(name): - return 'Hello %s' % name - - The ``:name`` part is a wildcard. See :class:`Router` for syntax - details. - - :param path: Request path or a list of paths to listen to. If no - path is specified, it is automatically generated from the - signature of the function. - :param method: HTTP method (`GET`, `POST`, `PUT`, ...) or a list of - methods to listen to. (default: `GET`) - :param callback: An optional shortcut to avoid the decorator - syntax. ``route(..., callback=func)`` equals ``route(...)(func)`` - :param name: The name for this route. (default: None) - :param apply: A decorator or plugin or a list of plugins. These are - applied to the route callback in addition to installed plugins. - :param skip: A list of plugins, plugin classes or names. Matching - plugins are not installed to this route. ``True`` skips all. - - Any additional keyword arguments are stored as route-specific - configuration and passed to plugins (see :meth:`Plugin.apply`). - """ - if callable(path): path, callback = None, path - plugins = makelist(apply) - skiplist = makelist(skip) - def decorator(callback): - # TODO: Documentation and tests - if isinstance(callback, basestring): callback = load(callback) - for rule in makelist(path) or yieldroutes(callback): - for verb in makelist(method): - verb = verb.upper() - route = Route(self, rule, verb, callback, name=name, - plugins=plugins, skiplist=skiplist, **config) - self.add_route(route) - return callback - return decorator(callback) if callback else decorator - - def get(self, path=None, method='GET', **options): - """ Equals :meth:`route`. """ - return self.route(path, method, **options) - - def post(self, path=None, method='POST', **options): - """ Equals :meth:`route` with a ``POST`` method parameter. """ - return self.route(path, method, **options) - - def put(self, path=None, method='PUT', **options): - """ Equals :meth:`route` with a ``PUT`` method parameter. """ - return self.route(path, method, **options) - - def delete(self, path=None, method='DELETE', **options): - """ Equals :meth:`route` with a ``DELETE`` method parameter. """ - return self.route(path, method, **options) - - def error(self, code=500): - """ Decorator: Register an output handler for a HTTP error code""" - def wrapper(handler): - self.error_handler[int(code)] = handler - return handler - return wrapper - - def hook(self, name): - """ Return a decorator that attaches a callback to a hook. Three hooks - are currently implemented: - - - before_request: Executed once before each request - - after_request: Executed once after each request - - app_reset: Called whenever :meth:`reset` is called. - """ - def wrapper(func): - self.hooks.add(name, func) - return func - return wrapper - - def handle(self, path, method='GET'): - """ (deprecated) Execute the first matching route callback and return - the result. :exc:`HTTPResponse` exceptions are caught and returned. - If :attr:`Bottle.catchall` is true, other exceptions are caught as - well and returned as :exc:`HTTPError` instances (500). - """ - depr("This method will change semantics in 0.10. Try to avoid it.") - if isinstance(path, dict): - return self._handle(path) - return self._handle({'PATH_INFO': path, 'REQUEST_METHOD': method.upper()}) - - def default_error_handler(self, res): - return tob(template(ERROR_PAGE_TEMPLATE, e=res)) - - def _handle(self, environ): - try: - environ['bottle.app'] = self - request.bind(environ) - response.bind() - route, args = self.router.match(environ) - environ['route.handle'] = route - environ['bottle.route'] = route - environ['route.url_args'] = args - return route.call(**args) - except HTTPResponse: - return _e() - except RouteReset: - route.reset() - return self._handle(environ) - except (KeyboardInterrupt, SystemExit, MemoryError): - raise - except Exception: - if not self.catchall: raise - stacktrace = format_exc() - environ['wsgi.errors'].write(stacktrace) - return HTTPError(500, "Internal Server Error", _e(), stacktrace) - - def _cast(self, out, peek=None): - """ Try to convert the parameter into something WSGI compatible and set - correct HTTP headers when possible. - Support: False, str, unicode, dict, HTTPResponse, HTTPError, file-like, - iterable of strings and iterable of unicodes - """ - - # Empty output is done here - if not out: - if 'Content-Length' not in response: - response['Content-Length'] = 0 - return [] - # Join lists of byte or unicode strings. Mixed lists are NOT supported - if isinstance(out, (tuple, list))\ - and isinstance(out[0], (bytes, unicode)): - out = out[0][0:0].join(out) # b'abc'[0:0] -> b'' - # Encode unicode strings - if isinstance(out, unicode): - out = out.encode(response.charset) - # Byte Strings are just returned - if isinstance(out, bytes): - if 'Content-Length' not in response: - response['Content-Length'] = len(out) - return [out] - # HTTPError or HTTPException (recursive, because they may wrap anything) - # TODO: Handle these explicitly in handle() or make them iterable. - if isinstance(out, HTTPError): - out.apply(response) - out = self.error_handler.get(out.status_code, self.default_error_handler)(out) - return self._cast(out) - if isinstance(out, HTTPResponse): - out.apply(response) - return self._cast(out.body) - - # File-like objects. - if hasattr(out, 'read'): - if 'wsgi.file_wrapper' in request.environ: - return request.environ['wsgi.file_wrapper'](out) - elif hasattr(out, 'close') or not hasattr(out, '__iter__'): - return WSGIFileWrapper(out) - - # Handle Iterables. We peek into them to detect their inner type. - try: - out = iter(out) - first = next(out) - while not first: - first = next(out) - except StopIteration: - return self._cast('') - except HTTPResponse: - first = _e() - except (KeyboardInterrupt, SystemExit, MemoryError): - raise - except Exception: - if not self.catchall: raise - first = HTTPError(500, 'Unhandled exception', _e(), format_exc()) - - # These are the inner types allowed in iterator or generator objects. - if isinstance(first, HTTPResponse): - return self._cast(first) - if isinstance(first, bytes): - return itertools.chain([first], out) - if isinstance(first, unicode): - return imap(lambda x: x.encode(response.charset), - itertools.chain([first], out)) - return self._cast(HTTPError(500, 'Unsupported response type: %s'\ - % type(first))) - - def wsgi(self, environ, start_response): - """ The bottle WSGI-interface. """ - try: - out = self._cast(self._handle(environ)) - # rfc2616 section 4.3 - if response._status_code in (100, 101, 204, 304)\ - or environ['REQUEST_METHOD'] == 'HEAD': - if hasattr(out, 'close'): out.close() - out = [] - start_response(response._status_line, response.headerlist) - return out - except (KeyboardInterrupt, SystemExit, MemoryError): - raise - except Exception: - if not self.catchall: raise - err = '

Critical error while processing request: %s

' \ - % html_escape(environ.get('PATH_INFO', '/')) - if DEBUG: - err += '

Error:

\n
\n%s\n
\n' \ - '

Traceback:

\n
\n%s\n
\n' \ - % (html_escape(repr(_e())), html_escape(format_exc())) - environ['wsgi.errors'].write(err) - headers = [('Content-Type', 'text/html; charset=UTF-8')] - start_response('500 INTERNAL SERVER ERROR', headers) - return [tob(err)] - - def __call__(self, environ, start_response): - ''' Each instance of :class:'Bottle' is a WSGI application. ''' - return self.wsgi(environ, start_response) - - - - - - -############################################################################### -# HTTP and WSGI Tools ########################################################## -############################################################################### - - -class BaseRequest(object): - """ A wrapper for WSGI environment dictionaries that adds a lot of - convenient access methods and properties. Most of them are read-only. - - Adding new attributes to a request actually adds them to the environ - dictionary (as 'bottle.request.ext.'). This is the recommended - way to store and access request-specific data. - """ - - __slots__ = ('environ') - - #: Maximum size of memory buffer for :attr:`body` in bytes. - MEMFILE_MAX = 102400 - #: Maximum number pr GET or POST parameters per request - MAX_PARAMS = 100 - - def __init__(self, environ=None): - """ Wrap a WSGI environ dictionary. """ - #: The wrapped WSGI environ dictionary. This is the only real attribute. - #: All other attributes actually are read-only properties. - self.environ = {} if environ is None else environ - self.environ['bottle.request'] = self - - @DictProperty('environ', 'bottle.app', read_only=True) - def app(self): - ''' Bottle application handling this request. ''' - raise RuntimeError('This request is not connected to an application.') - - @property - def path(self): - ''' The value of ``PATH_INFO`` with exactly one prefixed slash (to fix - broken clients and avoid the "empty path" edge case). ''' - return '/' + self.environ.get('PATH_INFO','').lstrip('/') - - @property - def method(self): - ''' The ``REQUEST_METHOD`` value as an uppercase string. ''' - return self.environ.get('REQUEST_METHOD', 'GET').upper() - - @DictProperty('environ', 'bottle.request.headers', read_only=True) - def headers(self): - ''' A :class:`WSGIHeaderDict` that provides case-insensitive access to - HTTP request headers. ''' - return WSGIHeaderDict(self.environ) - - def get_header(self, name, default=None): - ''' Return the value of a request header, or a given default value. ''' - return self.headers.get(name, default) - - @DictProperty('environ', 'bottle.request.cookies', read_only=True) - def cookies(self): - """ Cookies parsed into a :class:`FormsDict`. Signed cookies are NOT - decoded. Use :meth:`get_cookie` if you expect signed cookies. """ - cookies = SimpleCookie(self.environ.get('HTTP_COOKIE','')) - cookies = list(cookies.values())[:self.MAX_PARAMS] - return FormsDict((c.key, c.value) for c in cookies) - - def get_cookie(self, key, default=None, secret=None): - """ Return the content of a cookie. To read a `Signed Cookie`, the - `secret` must match the one used to create the cookie (see - :meth:`BaseResponse.set_cookie`). If anything goes wrong (missing - cookie or wrong signature), return a default value. """ - value = self.cookies.get(key) - if secret and value: - dec = cookie_decode(value, secret) # (key, value) tuple or None - return dec[1] if dec and dec[0] == key else default - return value or default - - @DictProperty('environ', 'bottle.request.query', read_only=True) - def query(self): - ''' The :attr:`query_string` parsed into a :class:`FormsDict`. These - values are sometimes called "URL arguments" or "GET parameters", but - not to be confused with "URL wildcards" as they are provided by the - :class:`Router`. ''' - get = self.environ['bottle.get'] = FormsDict() - pairs = _parse_qsl(self.environ.get('QUERY_STRING', '')) - for key, value in pairs[:self.MAX_PARAMS]: - get[key] = value - return get - - @DictProperty('environ', 'bottle.request.forms', read_only=True) - def forms(self): - """ Form values parsed from an `url-encoded` or `multipart/form-data` - encoded POST or PUT request body. The result is retuned as a - :class:`FormsDict`. All keys and values are strings. File uploads - are stored separately in :attr:`files`. """ - forms = FormsDict() - for name, item in self.POST.allitems(): - if not hasattr(item, 'filename'): - forms[name] = item - return forms - - @DictProperty('environ', 'bottle.request.params', read_only=True) - def params(self): - """ A :class:`FormsDict` with the combined values of :attr:`query` and - :attr:`forms`. File uploads are stored in :attr:`files`. """ - params = FormsDict() - for key, value in self.query.allitems(): - params[key] = value - for key, value in self.forms.allitems(): - params[key] = value - return params - - @DictProperty('environ', 'bottle.request.files', read_only=True) - def files(self): - """ File uploads parsed from an `url-encoded` or `multipart/form-data` - encoded POST or PUT request body. The values are instances of - :class:`cgi.FieldStorage`. The most important attributes are: - - filename - The filename, if specified; otherwise None; this is the client - side filename, *not* the file name on which it is stored (that's - a temporary file you don't deal with) - file - The file(-like) object from which you can read the data. - value - The value as a *string*; for file uploads, this transparently - reads the file every time you request the value. Do not do this - on big files. - """ - files = FormsDict() - for name, item in self.POST.allitems(): - if hasattr(item, 'filename'): - files[name] = item - return files - - @DictProperty('environ', 'bottle.request.json', read_only=True) - def json(self): - ''' If the ``Content-Type`` header is ``application/json``, this - property holds the parsed content of the request body. Only requests - smaller than :attr:`MEMFILE_MAX` are processed to avoid memory - exhaustion. ''' - if 'application/json' in self.environ.get('CONTENT_TYPE', '') \ - and 0 < self.content_length < self.MEMFILE_MAX: - return json_loads(self.body.read(self.MEMFILE_MAX)) - return None - - @DictProperty('environ', 'bottle.request.body', read_only=True) - def _body(self): - maxread = max(0, self.content_length) - stream = self.environ['wsgi.input'] - body = BytesIO() if maxread < self.MEMFILE_MAX else TemporaryFile(mode='w+b') - while maxread > 0: - part = stream.read(min(maxread, self.MEMFILE_MAX)) - if not part: break - body.write(part) - maxread -= len(part) - self.environ['wsgi.input'] = body - body.seek(0) - return body - - @property - def body(self): - """ The HTTP request body as a seek-able file-like object. Depending on - :attr:`MEMFILE_MAX`, this is either a temporary file or a - :class:`io.BytesIO` instance. Accessing this property for the first - time reads and replaces the ``wsgi.input`` environ variable. - Subsequent accesses just do a `seek(0)` on the file object. """ - self._body.seek(0) - return self._body - - #: An alias for :attr:`query`. - GET = query - - @DictProperty('environ', 'bottle.request.post', read_only=True) - def POST(self): - """ The values of :attr:`forms` and :attr:`files` combined into a single - :class:`FormsDict`. Values are either strings (form values) or - instances of :class:`cgi.FieldStorage` (file uploads). - """ - post = FormsDict() - # We default to application/x-www-form-urlencoded for everything that - # is not multipart and take the fast path (also: 3.1 workaround) - if not self.content_type.startswith('multipart/'): - maxlen = max(0, min(self.content_length, self.MEMFILE_MAX)) - pairs = _parse_qsl(tonat(self.body.read(maxlen), 'latin1')) - for key, value in pairs[:self.MAX_PARAMS]: - post[key] = value - return post - - safe_env = {'QUERY_STRING':''} # Build a safe environment for cgi - for key in ('REQUEST_METHOD', 'CONTENT_TYPE', 'CONTENT_LENGTH'): - if key in self.environ: safe_env[key] = self.environ[key] - args = dict(fp=self.body, environ=safe_env, keep_blank_values=True) - if py31: - args['fp'] = NCTextIOWrapper(args['fp'], encoding='ISO-8859-1', - newline='\n') - elif py3k: - args['encoding'] = 'ISO-8859-1' - data = FieldStorage(**args) - for item in (data.list or [])[:self.MAX_PARAMS]: - post[item.name] = item if item.filename else item.value - return post - - @property - def COOKIES(self): - ''' Alias for :attr:`cookies` (deprecated). ''' - depr('BaseRequest.COOKIES was renamed to BaseRequest.cookies (lowercase).') - return self.cookies - - @property - def url(self): - """ The full request URI including hostname and scheme. If your app - lives behind a reverse proxy or load balancer and you get confusing - results, make sure that the ``X-Forwarded-Host`` header is set - correctly. """ - return self.urlparts.geturl() - - @DictProperty('environ', 'bottle.request.urlparts', read_only=True) - def urlparts(self): - ''' The :attr:`url` string as an :class:`urlparse.SplitResult` tuple. - The tuple contains (scheme, host, path, query_string and fragment), - but the fragment is always empty because it is not visible to the - server. ''' - env = self.environ - http = env.get('HTTP_X_FORWARDED_PROTO') or env.get('wsgi.url_scheme', 'http') - host = env.get('HTTP_X_FORWARDED_HOST') or env.get('HTTP_HOST') - if not host: - # HTTP 1.1 requires a Host-header. This is for HTTP/1.0 clients. - host = env.get('SERVER_NAME', '127.0.0.1') - port = env.get('SERVER_PORT') - if port and port != ('80' if http == 'http' else '443'): - host += ':' + port - path = urlquote(self.fullpath) - return UrlSplitResult(http, host, path, env.get('QUERY_STRING'), '') - - @property - def fullpath(self): - """ Request path including :attr:`script_name` (if present). """ - return urljoin(self.script_name, self.path.lstrip('/')) - - @property - def query_string(self): - """ The raw :attr:`query` part of the URL (everything in between ``?`` - and ``#``) as a string. """ - return self.environ.get('QUERY_STRING', '') - - @property - def script_name(self): - ''' The initial portion of the URL's `path` that was removed by a higher - level (server or routing middleware) before the application was - called. This script path is returned with leading and tailing - slashes. ''' - script_name = self.environ.get('SCRIPT_NAME', '').strip('/') - return '/' + script_name + '/' if script_name else '/' - - def path_shift(self, shift=1): - ''' Shift path segments from :attr:`path` to :attr:`script_name` and - vice versa. - - :param shift: The number of path segments to shift. May be negative - to change the shift direction. (default: 1) - ''' - script = self.environ.get('SCRIPT_NAME','/') - self['SCRIPT_NAME'], self['PATH_INFO'] = path_shift(script, self.path, shift) - - @property - def content_length(self): - ''' The request body length as an integer. The client is responsible to - set this header. Otherwise, the real length of the body is unknown - and -1 is returned. In this case, :attr:`body` will be empty. ''' - return int(self.environ.get('CONTENT_LENGTH') or -1) - - @property - def content_type(self): - ''' The Content-Type header as a lowercase-string (default: empty). ''' - return self.environ.get('CONTENT_TYPE', '').lower() - - @property - def is_xhr(self): - ''' True if the request was triggered by a XMLHttpRequest. This only - works with JavaScript libraries that support the `X-Requested-With` - header (most of the popular libraries do). ''' - requested_with = self.environ.get('HTTP_X_REQUESTED_WITH','') - return requested_with.lower() == 'xmlhttprequest' - - @property - def is_ajax(self): - ''' Alias for :attr:`is_xhr`. "Ajax" is not the right term. ''' - return self.is_xhr - - @property - def auth(self): - """ HTTP authentication data as a (user, password) tuple. This - implementation currently supports basic (not digest) authentication - only. If the authentication happened at a higher level (e.g. in the - front web-server or a middleware), the password field is None, but - the user field is looked up from the ``REMOTE_USER`` environ - variable. On any errors, None is returned. """ - basic = parse_auth(self.environ.get('HTTP_AUTHORIZATION','')) - if basic: return basic - ruser = self.environ.get('REMOTE_USER') - if ruser: return (ruser, None) - return None - - @property - def remote_route(self): - """ A list of all IPs that were involved in this request, starting with - the client IP and followed by zero or more proxies. This does only - work if all proxies support the ```X-Forwarded-For`` header. Note - that this information can be forged by malicious clients. """ - proxy = self.environ.get('HTTP_X_FORWARDED_FOR') - if proxy: return [ip.strip() for ip in proxy.split(',')] - remote = self.environ.get('REMOTE_ADDR') - return [remote] if remote else [] - - @property - def remote_addr(self): - """ The client IP as a string. Note that this information can be forged - by malicious clients. """ - route = self.remote_route - return route[0] if route else None - - def copy(self): - """ Return a new :class:`Request` with a shallow :attr:`environ` copy. """ - return Request(self.environ.copy()) - - def get(self, value, default=None): return self.environ.get(value, default) - def __getitem__(self, key): return self.environ[key] - def __delitem__(self, key): self[key] = ""; del(self.environ[key]) - def __iter__(self): return iter(self.environ) - def __len__(self): return len(self.environ) - def keys(self): return self.environ.keys() - def __setitem__(self, key, value): - """ Change an environ value and clear all caches that depend on it. """ - - if self.environ.get('bottle.request.readonly'): - raise KeyError('The environ dictionary is read-only.') - - self.environ[key] = value - todelete = () - - if key == 'wsgi.input': - todelete = ('body', 'forms', 'files', 'params', 'post', 'json') - elif key == 'QUERY_STRING': - todelete = ('query', 'params') - elif key.startswith('HTTP_'): - todelete = ('headers', 'cookies') - - for key in todelete: - self.environ.pop('bottle.request.'+key, None) - - def __repr__(self): - return '<%s: %s %s>' % (self.__class__.__name__, self.method, self.url) - - def __getattr__(self, name): - ''' Search in self.environ for additional user defined attributes. ''' - try: - var = self.environ['bottle.request.ext.%s'%name] - return var.__get__(self) if hasattr(var, '__get__') else var - except KeyError: - raise AttributeError('Attribute %r not defined.' % name) - - def __setattr__(self, name, value): - if name == 'environ': return object.__setattr__(self, name, value) - self.environ['bottle.request.ext.%s'%name] = value - - - - -def _hkey(s): - return s.title().replace('_','-') - - -class HeaderProperty(object): - def __init__(self, name, reader=None, writer=str, default=''): - self.name, self.default = name, default - self.reader, self.writer = reader, writer - self.__doc__ = 'Current value of the %r header.' % name.title() - - def __get__(self, obj, cls): - if obj is None: return self - value = obj.headers.get(self.name, self.default) - return self.reader(value) if self.reader else value - - def __set__(self, obj, value): - obj.headers[self.name] = self.writer(value) - - def __delete__(self, obj): - del obj.headers[self.name] - - -class BaseResponse(object): - """ Storage class for a response body as well as headers and cookies. - - This class does support dict-like case-insensitive item-access to - headers, but is NOT a dict. Most notably, iterating over a response - yields parts of the body and not the headers. - """ - - default_status = 200 - default_content_type = 'text/html; charset=UTF-8' - - # Header blacklist for specific response codes - # (rfc2616 section 10.2.3 and 10.3.5) - bad_headers = { - 204: set(('Content-Type',)), - 304: set(('Allow', 'Content-Encoding', 'Content-Language', - 'Content-Length', 'Content-Range', 'Content-Type', - 'Content-Md5', 'Last-Modified'))} - - def __init__(self, body='', status=None, **headers): - self._cookies = None - self._headers = {'Content-Type': [self.default_content_type]} - self.body = body - self.status = status or self.default_status - if headers: - for name, value in headers.items(): - self[name] = value - - def copy(self): - ''' Returns a copy of self. ''' - copy = Response() - copy.status = self.status - copy._headers = dict((k, v[:]) for (k, v) in self._headers.items()) - return copy - - def __iter__(self): - return iter(self.body) - - def close(self): - if hasattr(self.body, 'close'): - self.body.close() - - @property - def status_line(self): - ''' The HTTP status line as a string (e.g. ``404 Not Found``).''' - return self._status_line - - @property - def status_code(self): - ''' The HTTP status code as an integer (e.g. 404).''' - return self._status_code - - def _set_status(self, status): - if isinstance(status, int): - code, status = status, _HTTP_STATUS_LINES.get(status) - elif ' ' in status: - status = status.strip() - code = int(status.split()[0]) - else: - raise ValueError('String status line without a reason phrase.') - if not 100 <= code <= 999: raise ValueError('Status code out of range.') - self._status_code = code - self._status_line = str(status or ('%d Unknown' % code)) - - def _get_status(self): - return self._status_line - - status = property(_get_status, _set_status, None, - ''' A writeable property to change the HTTP response status. It accepts - either a numeric code (100-999) or a string with a custom reason - phrase (e.g. "404 Brain not found"). Both :data:`status_line` and - :data:`status_code` are updated accordingly. The return value is - always a status string. ''') - del _get_status, _set_status - - @property - def headers(self): - ''' An instance of :class:`HeaderDict`, a case-insensitive dict-like - view on the response headers. ''' - hdict = HeaderDict() - hdict.dict = self._headers - return hdict - - def __contains__(self, name): return _hkey(name) in self._headers - def __delitem__(self, name): del self._headers[_hkey(name)] - def __getitem__(self, name): return self._headers[_hkey(name)][-1] - def __setitem__(self, name, value): self._headers[_hkey(name)] = [str(value)] - - def get_header(self, name, default=None): - ''' Return the value of a previously defined header. If there is no - header with that name, return a default value. ''' - return self._headers.get(_hkey(name), [default])[-1] - - def set_header(self, name, value): - ''' Create a new response header, replacing any previously defined - headers with the same name. ''' - self._headers[_hkey(name)] = [str(value)] - - def add_header(self, name, value): - ''' Add an additional response header, not removing duplicates. ''' - self._headers.setdefault(_hkey(name), []).append(str(value)) - - def iter_headers(self): - ''' Yield (header, value) tuples, skipping headers that are not - allowed with the current response status code. ''' - return self.headerlist - - def wsgiheader(self): - depr('The wsgiheader method is deprecated. See headerlist.') #0.10 - return self.headerlist - - @property - def headerlist(self): - ''' WSGI conform list of (header, value) tuples. ''' - out = [] - headers = self._headers.items() - if self._status_code in self.bad_headers: - bad_headers = self.bad_headers[self._status_code] - headers = [h for h in headers if h[0] not in bad_headers] - out += [(name, val) for name, vals in headers for val in vals] - if self._cookies: - for c in self._cookies.values(): - out.append(('Set-Cookie', c.OutputString())) - return out - - content_type = HeaderProperty('Content-Type') - content_length = HeaderProperty('Content-Length', reader=int) - - @property - def charset(self): - """ Return the charset specified in the content-type header (default: utf8). """ - if 'charset=' in self.content_type: - return self.content_type.split('charset=')[-1].split(';')[0].strip() - return 'UTF-8' - - @property - def COOKIES(self): - """ A dict-like SimpleCookie instance. This should not be used directly. - See :meth:`set_cookie`. """ - depr('The COOKIES dict is deprecated. Use `set_cookie()` instead.') # 0.10 - if not self._cookies: - self._cookies = SimpleCookie() - return self._cookies - - def set_cookie(self, name, value, secret=None, **options): - ''' Create a new cookie or replace an old one. If the `secret` parameter is - set, create a `Signed Cookie` (described below). - - :param name: the name of the cookie. - :param value: the value of the cookie. - :param secret: a signature key required for signed cookies. - - Additionally, this method accepts all RFC 2109 attributes that are - supported by :class:`cookie.Morsel`, including: - - :param max_age: maximum age in seconds. (default: None) - :param expires: a datetime object or UNIX timestamp. (default: None) - :param domain: the domain that is allowed to read the cookie. - (default: current domain) - :param path: limits the cookie to a given path (default: current path) - :param secure: limit the cookie to HTTPS connections (default: off). - :param httponly: prevents client-side javascript to read this cookie - (default: off, requires Python 2.6 or newer). - - If neither `expires` nor `max_age` is set (default), the cookie will - expire at the end of the browser session (as soon as the browser - window is closed). - - Signed cookies may store any pickle-able object and are - cryptographically signed to prevent manipulation. Keep in mind that - cookies are limited to 4kb in most browsers. - - Warning: Signed cookies are not encrypted (the client can still see - the content) and not copy-protected (the client can restore an old - cookie). The main intention is to make pickling and unpickling - save, not to store secret information at client side. - ''' - if not self._cookies: - self._cookies = SimpleCookie() - - if secret: - value = touni(cookie_encode((name, value), secret)) - elif not isinstance(value, basestring): - raise TypeError('Secret key missing for non-string Cookie.') - - if len(value) > 4096: raise ValueError('Cookie value to long.') - self._cookies[name] = value - - for key, value in options.items(): - if key == 'max_age': - if isinstance(value, timedelta): - value = value.seconds + value.days * 24 * 3600 - if key == 'expires': - if isinstance(value, (datedate, datetime)): - value = value.timetuple() - elif isinstance(value, (int, float)): - value = time.gmtime(value) - value = time.strftime("%a, %d %b %Y %H:%M:%S GMT", value) - self._cookies[name][key.replace('_', '-')] = value - - def delete_cookie(self, key, **kwargs): - ''' Delete a cookie. Be sure to use the same `domain` and `path` - settings as used to create the cookie. ''' - kwargs['max_age'] = -1 - kwargs['expires'] = 0 - self.set_cookie(key, '', **kwargs) - - def __repr__(self): - out = '' - for name, value in self.headerlist: - out += '%s: %s\n' % (name.title(), value.strip()) - return out - -#: Thread-local storage for :class:`LocalRequest` and :class:`LocalResponse` -#: attributes. -_lctx = threading.local() - -def local_property(name): - def fget(self): - try: - return getattr(_lctx, name) - except AttributeError: - raise RuntimeError("Request context not initialized.") - def fset(self, value): setattr(_lctx, name, value) - def fdel(self): delattr(_lctx, name) - return property(fget, fset, fdel, - 'Thread-local property stored in :data:`_lctx.%s`' % name) - - -class LocalRequest(BaseRequest): - ''' A thread-local subclass of :class:`BaseRequest` with a different - set of attribues for each thread. There is usually only one global - instance of this class (:data:`request`). If accessed during a - request/response cycle, this instance always refers to the *current* - request (even on a multithreaded server). ''' - bind = BaseRequest.__init__ - environ = local_property('request_environ') - - -class LocalResponse(BaseResponse): - ''' A thread-local subclass of :class:`BaseResponse` with a different - set of attribues for each thread. There is usually only one global - instance of this class (:data:`response`). Its attributes are used - to build the HTTP response at the end of the request/response cycle. - ''' - bind = BaseResponse.__init__ - _status_line = local_property('response_status_line') - _status_code = local_property('response_status_code') - _cookies = local_property('response_cookies') - _headers = local_property('response_headers') - body = local_property('response_body') - -Request = BaseRequest -Response = BaseResponse - -class HTTPResponse(Response, BottleException): - def __init__(self, body='', status=None, header=None, **headers): - if header or 'output' in headers: - depr('Call signature changed (for the better)') - if header: headers.update(header) - if 'output' in headers: body = headers.pop('output') - super(HTTPResponse, self).__init__(body, status, **headers) - - def apply(self, response): - response._status_code = self._status_code - response._status_line = self._status_line - response._headers = self._headers - response._cookies = self._cookies - response.body = self.body - - def _output(self, value=None): - depr('Use HTTPResponse.body instead of HTTPResponse.output') - if value is None: return self.body - self.body = value - - output = property(_output, _output, doc='Alias for .body') - -class HTTPError(HTTPResponse): - default_status = 500 - def __init__(self, status=None, body=None, exception=None, traceback=None, header=None, **headers): - self.exception = exception - self.traceback = traceback - super(HTTPError, self).__init__(body, status, header, **headers) - - - - - -############################################################################### -# Plugins ###################################################################### -############################################################################### - -class PluginError(BottleException): pass - -class JSONPlugin(object): - name = 'json' - api = 2 - - def __init__(self, json_dumps=json_dumps): - self.json_dumps = json_dumps - - def apply(self, callback, route): - dumps = self.json_dumps - if not dumps: return callback - def wrapper(*a, **ka): - rv = callback(*a, **ka) - if isinstance(rv, dict): - #Attempt to serialize, raises exception on failure - json_response = dumps(rv) - #Set content type only if serialization succesful - response.content_type = 'application/json' - return json_response - return rv - return wrapper - - -class HooksPlugin(object): - name = 'hooks' - api = 2 - - _names = 'before_request', 'after_request', 'app_reset' - - def __init__(self): - self.hooks = dict((name, []) for name in self._names) - self.app = None - - def _empty(self): - return not (self.hooks['before_request'] or self.hooks['after_request']) - - def setup(self, app): - self.app = app - - def add(self, name, func): - ''' Attach a callback to a hook. ''' - was_empty = self._empty() - self.hooks.setdefault(name, []).append(func) - if self.app and was_empty and not self._empty(): self.app.reset() - - def remove(self, name, func): - ''' Remove a callback from a hook. ''' - was_empty = self._empty() - if name in self.hooks and func in self.hooks[name]: - self.hooks[name].remove(func) - if self.app and not was_empty and self._empty(): self.app.reset() - - def trigger(self, name, *a, **ka): - ''' Trigger a hook and return a list of results. ''' - hooks = self.hooks[name] - if ka.pop('reversed', False): hooks = hooks[::-1] - return [hook(*a, **ka) for hook in hooks] - - def apply(self, callback, route): - if self._empty(): return callback - def wrapper(*a, **ka): - self.trigger('before_request') - rv = callback(*a, **ka) - self.trigger('after_request', reversed=True) - return rv - return wrapper - - -class TemplatePlugin(object): - ''' This plugin applies the :func:`view` decorator to all routes with a - `template` config parameter. If the parameter is a tuple, the second - element must be a dict with additional options (e.g. `template_engine`) - or default variables for the template. ''' - name = 'template' - api = 2 - - def apply(self, callback, route): - conf = route.config.get('template') - if isinstance(conf, (tuple, list)) and len(conf) == 2: - return view(conf[0], **conf[1])(callback) - elif isinstance(conf, str) and 'template_opts' in route.config: - depr('The `template_opts` parameter is deprecated.') #0.9 - return view(conf, **route.config['template_opts'])(callback) - elif isinstance(conf, str): - return view(conf)(callback) - else: - return callback - - -#: Not a plugin, but part of the plugin API. TODO: Find a better place. -class _ImportRedirect(object): - def __init__(self, name, impmask): - ''' Create a virtual package that redirects imports (see PEP 302). ''' - self.name = name - self.impmask = impmask - self.module = sys.modules.setdefault(name, imp.new_module(name)) - self.module.__dict__.update({'__file__': __file__, '__path__': [], - '__all__': [], '__loader__': self}) - sys.meta_path.append(self) - - def find_module(self, fullname, path=None): - if '.' not in fullname: return - packname, modname = fullname.rsplit('.', 1) - if packname != self.name: return - return self - - def load_module(self, fullname): - if fullname in sys.modules: return sys.modules[fullname] - packname, modname = fullname.rsplit('.', 1) - realname = self.impmask % modname - __import__(realname) - module = sys.modules[fullname] = sys.modules[realname] - setattr(self.module, modname, module) - module.__loader__ = self - return module - - - - - - -############################################################################### -# Common Utilities ############################################################# -############################################################################### - - -class MultiDict(DictMixin): - """ This dict stores multiple values per key, but behaves exactly like a - normal dict in that it returns only the newest value for any given key. - There are special methods available to access the full list of values. - """ - - def __init__(self, *a, **k): - self.dict = dict((k, [v]) for (k, v) in dict(*a, **k).items()) - - def __len__(self): return len(self.dict) - def __iter__(self): return iter(self.dict) - def __contains__(self, key): return key in self.dict - def __delitem__(self, key): del self.dict[key] - def __getitem__(self, key): return self.dict[key][-1] - def __setitem__(self, key, value): self.append(key, value) - def keys(self): return self.dict.keys() - - if py3k: - def values(self): return (v[-1] for v in self.dict.values()) - def items(self): return ((k, v[-1]) for k, v in self.dict.items()) - def allitems(self): - return ((k, v) for k, vl in self.dict.items() for v in vl) - iterkeys = keys - itervalues = values - iteritems = items - iterallitems = allitems - - else: - def values(self): return [v[-1] for v in self.dict.values()] - def items(self): return [(k, v[-1]) for k, v in self.dict.items()] - def iterkeys(self): return self.dict.iterkeys() - def itervalues(self): return (v[-1] for v in self.dict.itervalues()) - def iteritems(self): - return ((k, v[-1]) for k, v in self.dict.iteritems()) - def iterallitems(self): - return ((k, v) for k, vl in self.dict.iteritems() for v in vl) - def allitems(self): - return [(k, v) for k, vl in self.dict.iteritems() for v in vl] - - def get(self, key, default=None, index=-1, type=None): - ''' Return the most recent value for a key. - - :param default: The default value to be returned if the key is not - present or the type conversion fails. - :param index: An index for the list of available values. - :param type: If defined, this callable is used to cast the value - into a specific type. Exception are suppressed and result in - the default value to be returned. - ''' - try: - val = self.dict[key][index] - return type(val) if type else val - except Exception: - pass - return default - - def append(self, key, value): - ''' Add a new value to the list of values for this key. ''' - self.dict.setdefault(key, []).append(value) - - def replace(self, key, value): - ''' Replace the list of values with a single value. ''' - self.dict[key] = [value] - - def getall(self, key): - ''' Return a (possibly empty) list of values for a key. ''' - return self.dict.get(key) or [] - - #: Aliases for WTForms to mimic other multi-dict APIs (Django) - getone = get - getlist = getall - - - -class FormsDict(MultiDict): - ''' This :class:`MultiDict` subclass is used to store request form data. - Additionally to the normal dict-like item access methods (which return - unmodified data as native strings), this container also supports - attribute-like access to its values. Attributes are automatically de- - or recoded to match :attr:`input_encoding` (default: 'utf8'). Missing - attributes default to an empty string. ''' - - #: Encoding used for attribute values. - input_encoding = 'utf8' - #: If true (default), unicode strings are first encoded with `latin1` - #: and then decoded to match :attr:`input_encoding`. - recode_unicode = True - - def _fix(self, s, encoding=None): - if isinstance(s, unicode) and self.recode_unicode: # Python 3 WSGI - s = s.encode('latin1') - if isinstance(s, bytes): # Python 2 WSGI - return s.decode(encoding or self.input_encoding) - return s - - def decode(self, encoding=None): - ''' Returns a copy with all keys and values de- or recoded to match - :attr:`input_encoding`. Some libraries (e.g. WTForms) want a - unicode dictionary. ''' - copy = FormsDict() - enc = copy.input_encoding = encoding or self.input_encoding - copy.recode_unicode = False - for key, value in self.allitems(): - copy.append(self._fix(key, enc), self._fix(value, enc)) - return copy - - def getunicode(self, name, default=None, encoding=None): - try: - return self._fix(self[name], encoding) - except (UnicodeError, KeyError): - return default - - def __getattr__(self, name, default=unicode()): - # Without this guard, pickle generates a cryptic TypeError: - if name.startswith('__') and name.endswith('__'): - return super(FormsDict, self).__getattr__(name) - return self.getunicode(name, default=default) - - -class HeaderDict(MultiDict): - """ A case-insensitive version of :class:`MultiDict` that defaults to - replace the old value instead of appending it. """ - - def __init__(self, *a, **ka): - self.dict = {} - if a or ka: self.update(*a, **ka) - - def __contains__(self, key): return _hkey(key) in self.dict - def __delitem__(self, key): del self.dict[_hkey(key)] - def __getitem__(self, key): return self.dict[_hkey(key)][-1] - def __setitem__(self, key, value): self.dict[_hkey(key)] = [str(value)] - def append(self, key, value): - self.dict.setdefault(_hkey(key), []).append(str(value)) - def replace(self, key, value): self.dict[_hkey(key)] = [str(value)] - def getall(self, key): return self.dict.get(_hkey(key)) or [] - def get(self, key, default=None, index=-1): - return MultiDict.get(self, _hkey(key), default, index) - def filter(self, names): - for name in [_hkey(n) for n in names]: - if name in self.dict: - del self.dict[name] - - -class WSGIHeaderDict(DictMixin): - ''' This dict-like class wraps a WSGI environ dict and provides convenient - access to HTTP_* fields. Keys and values are native strings - (2.x bytes or 3.x unicode) and keys are case-insensitive. If the WSGI - environment contains non-native string values, these are de- or encoded - using a lossless 'latin1' character set. - - The API will remain stable even on changes to the relevant PEPs. - Currently PEP 333, 444 and 3333 are supported. (PEP 444 is the only one - that uses non-native strings.) - ''' - #: List of keys that do not have a ``HTTP_`` prefix. - cgikeys = ('CONTENT_TYPE', 'CONTENT_LENGTH') - - def __init__(self, environ): - self.environ = environ - - def _ekey(self, key): - ''' Translate header field name to CGI/WSGI environ key. ''' - key = key.replace('-','_').upper() - if key in self.cgikeys: - return key - return 'HTTP_' + key - - def raw(self, key, default=None): - ''' Return the header value as is (may be bytes or unicode). ''' - return self.environ.get(self._ekey(key), default) - - def __getitem__(self, key): - return tonat(self.environ[self._ekey(key)], 'latin1') - - def __setitem__(self, key, value): - raise TypeError("%s is read-only." % self.__class__) - - def __delitem__(self, key): - raise TypeError("%s is read-only." % self.__class__) - - def __iter__(self): - for key in self.environ: - if key[:5] == 'HTTP_': - yield key[5:].replace('_', '-').title() - elif key in self.cgikeys: - yield key.replace('_', '-').title() - - def keys(self): return [x for x in self] - def __len__(self): return len(self.keys()) - def __contains__(self, key): return self._ekey(key) in self.environ - - -class ConfigDict(dict): - ''' A dict-subclass with some extras: You can access keys like attributes. - Uppercase attributes create new ConfigDicts and act as name-spaces. - Other missing attributes return None. Calling a ConfigDict updates its - values and returns itself. - - >>> cfg = ConfigDict() - >>> cfg.Namespace.value = 5 - >>> cfg.OtherNamespace(a=1, b=2) - >>> cfg - {'Namespace': {'value': 5}, 'OtherNamespace': {'a': 1, 'b': 2}} - ''' - - def __getattr__(self, key): - if key not in self and key[0].isupper(): - self[key] = ConfigDict() - return self.get(key) - - def __setattr__(self, key, value): - if hasattr(dict, key): - raise AttributeError('Read-only attribute.') - if key in self and self[key] and isinstance(self[key], ConfigDict): - raise AttributeError('Non-empty namespace attribute.') - self[key] = value - - def __delattr__(self, key): - if key in self: del self[key] - - def __call__(self, *a, **ka): - for key, value in dict(*a, **ka).items(): setattr(self, key, value) - return self - - -class AppStack(list): - """ A stack-like list. Calling it returns the head of the stack. """ - - def __call__(self): - """ Return the current default application. """ - return self[-1] - - def push(self, value=None): - """ Add a new :class:`Bottle` instance to the stack """ - if not isinstance(value, Bottle): - value = Bottle() - self.append(value) - return value - - -class WSGIFileWrapper(object): - - def __init__(self, fp, buffer_size=1024*64): - self.fp, self.buffer_size = fp, buffer_size - for attr in ('fileno', 'close', 'read', 'readlines', 'tell', 'seek'): - if hasattr(fp, attr): setattr(self, attr, getattr(fp, attr)) - - def __iter__(self): - buff, read = self.buffer_size, self.read - while True: - part = read(buff) - if not part: return - yield part - - -class ResourceManager(object): - ''' This class manages a list of search paths and helps to find and open - application-bound resources (files). - - :param base: default value for :meth:`add_path` calls. - :param opener: callable used to open resources. - :param cachemode: controls which lookups are cached. One of 'all', - 'found' or 'none'. - ''' - - def __init__(self, base='./', opener=open, cachemode='all'): - self.opener = open - self.base = base - self.cachemode = cachemode - - #: A list of search paths. See :meth:`add_path` for details. - self.path = [] - #: A cache for resolved paths. ``res.cache.clear()`` clears the cache. - self.cache = {} - - def add_path(self, path, base=None, index=None, create=False): - ''' Add a new path to the list of search paths. Return False if the - path does not exist. - - :param path: The new search path. Relative paths are turned into - an absolute and normalized form. If the path looks like a file - (not ending in `/`), the filename is stripped off. - :param base: Path used to absolutize relative search paths. - Defaults to :attr:`base` which defaults to ``os.getcwd()``. - :param index: Position within the list of search paths. Defaults - to last index (appends to the list). - - The `base` parameter makes it easy to reference files installed - along with a python module or package:: - - res.add_path('./resources/', __file__) - ''' - base = os.path.abspath(os.path.dirname(base or self.base)) - path = os.path.abspath(os.path.join(base, os.path.dirname(path))) - path += os.sep - if path in self.path: - self.path.remove(path) - if create and not os.path.isdir(path): - os.makedirs(path) - if index is None: - self.path.append(path) - else: - self.path.insert(index, path) - self.cache.clear() - return os.path.exists(path) - - def __iter__(self): - ''' Iterate over all existing files in all registered paths. ''' - search = self.path[:] - while search: - path = search.pop() - if not os.path.isdir(path): continue - for name in os.listdir(path): - full = os.path.join(path, name) - if os.path.isdir(full): search.append(full) - else: yield full - - def lookup(self, name): - ''' Search for a resource and return an absolute file path, or `None`. - - The :attr:`path` list is searched in order. The first match is - returend. Symlinks are followed. The result is cached to speed up - future lookups. ''' - if name not in self.cache or DEBUG: - for path in self.path: - fpath = os.path.join(path, name) - if os.path.isfile(fpath): - if self.cachemode in ('all', 'found'): - self.cache[name] = fpath - return fpath - if self.cachemode == 'all': - self.cache[name] = None - return self.cache[name] - - def open(self, name, mode='r', *args, **kwargs): - ''' Find a resource and return a file object, or raise IOError. ''' - fname = self.lookup(name) - if not fname: raise IOError("Resource %r not found." % name) - return self.opener(name, mode=mode, *args, **kwargs) - - - - - - -############################################################################### -# Application Helper ########################################################### -############################################################################### - - -def abort(code=500, text='Unknown Error: Application stopped.'): - """ Aborts execution and causes a HTTP error. """ - raise HTTPError(code, text) - - -def redirect(url, code=None): - """ Aborts execution and causes a 303 or 302 redirect, depending on - the HTTP protocol version. """ - if code is None: - code = 303 if request.get('SERVER_PROTOCOL') == "HTTP/1.1" else 302 - location = urljoin(request.url, url) - res = HTTPResponse("", status=code, Location=location) - if response._cookies: - res._cookies = response._cookies - raise res - - -def _file_iter_range(fp, offset, bytes, maxread=1024*1024): - ''' Yield chunks from a range in a file. No chunk is bigger than maxread.''' - fp.seek(offset) - while bytes > 0: - part = fp.read(min(bytes, maxread)) - if not part: break - bytes -= len(part) - yield part - - -def static_file(filename, root, mimetype='auto', download=False): - """ Open a file in a safe way and return :exc:`HTTPResponse` with status - code 200, 305, 401 or 404. Set Content-Type, Content-Encoding, - Content-Length and Last-Modified header. Obey If-Modified-Since header - and HEAD requests. - """ - root = os.path.abspath(root) + os.sep - filename = os.path.abspath(os.path.join(root, filename.strip('/\\'))) - headers = dict() - - if not filename.startswith(root): - return HTTPError(403, "Access denied.") - if not os.path.exists(filename) or not os.path.isfile(filename): - return HTTPError(404, "File does not exist.") - if not os.access(filename, os.R_OK): - return HTTPError(403, "You do not have permission to access this file.") - - if mimetype == 'auto': - mimetype, encoding = mimetypes.guess_type(filename) - if mimetype: headers['Content-Type'] = mimetype - if encoding: headers['Content-Encoding'] = encoding - elif mimetype: - headers['Content-Type'] = mimetype - - if download: - download = os.path.basename(filename if download == True else download) - headers['Content-Disposition'] = 'attachment; filename="%s"' % download - - stats = os.stat(filename) - headers['Content-Length'] = clen = stats.st_size - lm = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(stats.st_mtime)) - headers['Last-Modified'] = lm - - ims = request.environ.get('HTTP_IF_MODIFIED_SINCE') - if ims: - ims = parse_date(ims.split(";")[0].strip()) - if ims is not None and ims >= int(stats.st_mtime): - headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) - return HTTPResponse(status=304, **headers) - - body = '' if request.method == 'HEAD' else open(filename, 'rb') - - headers["Accept-Ranges"] = "bytes" - ranges = request.environ.get('HTTP_RANGE') - if 'HTTP_RANGE' in request.environ: - ranges = list(parse_range_header(request.environ['HTTP_RANGE'], clen)) - if not ranges: - return HTTPError(416, "Requested Range Not Satisfiable") - offset, end = ranges[0] - headers["Content-Range"] = "bytes %d-%d/%d" % (offset, end-1, clen) - headers["Content-Length"] = str(end-offset) - if body: body = _file_iter_range(body, offset, end-offset) - return HTTPResponse(body, status=206, **headers) - return HTTPResponse(body, **headers) - - - - - - -############################################################################### -# HTTP Utilities and MISC (TODO) ############################################### -############################################################################### - - -def debug(mode=True): - """ Change the debug level. - There is only one debug level supported at the moment.""" - global DEBUG - DEBUG = bool(mode) - - -def parse_date(ims): - """ Parse rfc1123, rfc850 and asctime timestamps and return UTC epoch. """ - try: - ts = email.utils.parsedate_tz(ims) - return time.mktime(ts[:8] + (0,)) - (ts[9] or 0) - time.timezone - except (TypeError, ValueError, IndexError, OverflowError): - return None - - -def parse_auth(header): - """ Parse rfc2617 HTTP authentication header string (basic) and return (user,pass) tuple or None""" - try: - method, data = header.split(None, 1) - if method.lower() == 'basic': - user, pwd = touni(base64.b64decode(tob(data))).split(':',1) - return user, pwd - except (KeyError, ValueError): - return None - -def parse_range_header(header, maxlen=0): - ''' Yield (start, end) ranges parsed from a HTTP Range header. Skip - unsatisfiable ranges. The end index is non-inclusive.''' - if not header or header[:6] != 'bytes=': return - ranges = [r.split('-', 1) for r in header[6:].split(',') if '-' in r] - for start, end in ranges: - try: - if not start: # bytes=-100 -> last 100 bytes - start, end = max(0, maxlen-int(end)), maxlen - elif not end: # bytes=100- -> all but the first 99 bytes - start, end = int(start), maxlen - else: # bytes=100-200 -> bytes 100-200 (inclusive) - start, end = int(start), min(int(end)+1, maxlen) - if 0 <= start < end <= maxlen: - yield start, end - except ValueError: - pass - -def _parse_qsl(qs): - r = [] - for pair in qs.replace(';','&').split('&'): - if not pair: continue - nv = pair.split('=', 1) - if len(nv) != 2: nv.append('') - key = urlunquote(nv[0].replace('+', ' ')) - value = urlunquote(nv[1].replace('+', ' ')) - r.append((key, value)) - return r - -def _lscmp(a, b): - ''' Compares two strings in a cryptographically safe way: - Runtime is not affected by length of common prefix. ''' - return not sum(0 if x==y else 1 for x, y in zip(a, b)) and len(a) == len(b) - - -def cookie_encode(data, key): - ''' Encode and sign a pickle-able object. Return a (byte) string ''' - msg = base64.b64encode(pickle.dumps(data, -1)) - sig = base64.b64encode(hmac.new(tob(key), msg).digest()) - return tob('!') + sig + tob('?') + msg - - -def cookie_decode(data, key): - ''' Verify and decode an encoded string. Return an object or None.''' - data = tob(data) - if cookie_is_encoded(data): - sig, msg = data.split(tob('?'), 1) - if _lscmp(sig[1:], base64.b64encode(hmac.new(tob(key), msg).digest())): - return pickle.loads(base64.b64decode(msg)) - return None - - -def cookie_is_encoded(data): - ''' Return True if the argument looks like a encoded cookie.''' - return bool(data.startswith(tob('!')) and tob('?') in data) - - -def html_escape(string): - ''' Escape HTML special characters ``&<>`` and quotes ``'"``. ''' - return string.replace('&','&').replace('<','<').replace('>','>')\ - .replace('"','"').replace("'",''') - - -def html_quote(string): - ''' Escape and quote a string to be used as an HTTP attribute.''' - return '"%s"' % html_escape(string).replace('\n','%#10;')\ - .replace('\r',' ').replace('\t',' ') - - -def yieldroutes(func): - """ Return a generator for routes that match the signature (name, args) - of the func parameter. This may yield more than one route if the function - takes optional keyword arguments. The output is best described by example:: - - a() -> '/a' - b(x, y) -> '/b/:x/:y' - c(x, y=5) -> '/c/:x' and '/c/:x/:y' - d(x=5, y=6) -> '/d' and '/d/:x' and '/d/:x/:y' - """ - import inspect # Expensive module. Only import if necessary. - path = '/' + func.__name__.replace('__','/').lstrip('/') - spec = inspect.getargspec(func) - argc = len(spec[0]) - len(spec[3] or []) - path += ('/:%s' * argc) % tuple(spec[0][:argc]) - yield path - for arg in spec[0][argc:]: - path += '/:%s' % arg - yield path - - -def path_shift(script_name, path_info, shift=1): - ''' Shift path fragments from PATH_INFO to SCRIPT_NAME and vice versa. - - :return: The modified paths. - :param script_name: The SCRIPT_NAME path. - :param script_name: The PATH_INFO path. - :param shift: The number of path fragments to shift. May be negative to - change the shift direction. (default: 1) - ''' - if shift == 0: return script_name, path_info - pathlist = path_info.strip('/').split('/') - scriptlist = script_name.strip('/').split('/') - if pathlist and pathlist[0] == '': pathlist = [] - if scriptlist and scriptlist[0] == '': scriptlist = [] - if shift > 0 and shift <= len(pathlist): - moved = pathlist[:shift] - scriptlist = scriptlist + moved - pathlist = pathlist[shift:] - elif shift < 0 and shift >= -len(scriptlist): - moved = scriptlist[shift:] - pathlist = moved + pathlist - scriptlist = scriptlist[:shift] - else: - empty = 'SCRIPT_NAME' if shift < 0 else 'PATH_INFO' - raise AssertionError("Cannot shift. Nothing left from %s" % empty) - new_script_name = '/' + '/'.join(scriptlist) - new_path_info = '/' + '/'.join(pathlist) - if path_info.endswith('/') and pathlist: new_path_info += '/' - return new_script_name, new_path_info - - -def validate(**vkargs): - """ - Validates and manipulates keyword arguments by user defined callables. - Handles ValueError and missing arguments by raising HTTPError(403). - """ - depr('Use route wildcard filters instead.') - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kargs): - for key, value in vkargs.items(): - if key not in kargs: - abort(403, 'Missing parameter: %s' % key) - try: - kargs[key] = value(kargs[key]) - except ValueError: - abort(403, 'Wrong parameter format for: %s' % key) - return func(*args, **kargs) - return wrapper - return decorator - - -def auth_basic(check, realm="private", text="Access denied"): - ''' Callback decorator to require HTTP auth (basic). - TODO: Add route(check_auth=...) parameter. ''' - def decorator(func): - def wrapper(*a, **ka): - user, password = request.auth or (None, None) - if user is None or not check(user, password): - response.headers['WWW-Authenticate'] = 'Basic realm="%s"' % realm - return HTTPError(401, text) - return func(*a, **ka) - return wrapper - return decorator - - -# Shortcuts for common Bottle methods. -# They all refer to the current default application. - -def make_default_app_wrapper(name): - ''' Return a callable that relays calls to the current default app. ''' - @functools.wraps(getattr(Bottle, name)) - def wrapper(*a, **ka): - return getattr(app(), name)(*a, **ka) - return wrapper - -route = make_default_app_wrapper('route') -get = make_default_app_wrapper('get') -post = make_default_app_wrapper('post') -put = make_default_app_wrapper('put') -delete = make_default_app_wrapper('delete') -error = make_default_app_wrapper('error') -mount = make_default_app_wrapper('mount') -hook = make_default_app_wrapper('hook') -install = make_default_app_wrapper('install') -uninstall = make_default_app_wrapper('uninstall') -url = make_default_app_wrapper('get_url') - - - - - - - -############################################################################### -# Server Adapter ############################################################### -############################################################################### - - -class ServerAdapter(object): - quiet = False - def __init__(self, host='127.0.0.1', port=8080, **config): - self.options = config - self.host = host - self.port = int(port) - - def run(self, handler): # pragma: no cover - pass - - def __repr__(self): - args = ', '.join(['%s=%s'%(k,repr(v)) for k, v in self.options.items()]) - return "%s(%s)" % (self.__class__.__name__, args) - - -class CGIServer(ServerAdapter): - quiet = True - def run(self, handler): # pragma: no cover - from wsgiref.handlers import CGIHandler - def fixed_environ(environ, start_response): - environ.setdefault('PATH_INFO', '') - return handler(environ, start_response) - CGIHandler().run(fixed_environ) - - -class FlupFCGIServer(ServerAdapter): - def run(self, handler): # pragma: no cover - import flup.server.fcgi - self.options.setdefault('bindAddress', (self.host, self.port)) - flup.server.fcgi.WSGIServer(handler, **self.options).run() - - -class WSGIRefServer(ServerAdapter): - def run(self, handler): # pragma: no cover - from wsgiref.simple_server import make_server, WSGIRequestHandler - if self.quiet: - class QuietHandler(WSGIRequestHandler): - def log_request(*args, **kw): pass - self.options['handler_class'] = QuietHandler - srv = make_server(self.host, self.port, handler, **self.options) - srv.serve_forever() - - -class CherryPyServer(ServerAdapter): - def run(self, handler): # pragma: no cover - from cherrypy import wsgiserver - server = wsgiserver.CherryPyWSGIServer((self.host, self.port), handler) - try: - server.start() - finally: - server.stop() - - -class WaitressServer(ServerAdapter): - def run(self, handler): - from waitress import serve - serve(handler, host=self.host, port=self.port) - - -class PasteServer(ServerAdapter): - def run(self, handler): # pragma: no cover - from paste import httpserver - if not self.quiet: - from paste.translogger import TransLogger - handler = TransLogger(handler) - httpserver.serve(handler, host=self.host, port=str(self.port), - **self.options) - - -class MeinheldServer(ServerAdapter): - def run(self, handler): - from meinheld import server - server.listen((self.host, self.port)) - server.run(handler) - - -class FapwsServer(ServerAdapter): - """ Extremely fast webserver using libev. See http://www.fapws.org/ """ - def run(self, handler): # pragma: no cover - import fapws._evwsgi as evwsgi - from fapws import base, config - port = self.port - if float(config.SERVER_IDENT[-2:]) > 0.4: - # fapws3 silently changed its API in 0.5 - port = str(port) - evwsgi.start(self.host, port) - # fapws3 never releases the GIL. Complain upstream. I tried. No luck. - if 'BOTTLE_CHILD' in os.environ and not self.quiet: - _stderr("WARNING: Auto-reloading does not work with Fapws3.\n") - _stderr(" (Fapws3 breaks python thread support)\n") - evwsgi.set_base_module(base) - def app(environ, start_response): - environ['wsgi.multiprocess'] = False - return handler(environ, start_response) - evwsgi.wsgi_cb(('', app)) - evwsgi.run() - - -class TornadoServer(ServerAdapter): - """ The super hyped asynchronous server by facebook. Untested. """ - def run(self, handler): # pragma: no cover - import tornado.wsgi, tornado.httpserver, tornado.ioloop - container = tornado.wsgi.WSGIContainer(handler) - server = tornado.httpserver.HTTPServer(container) - server.listen(port=self.port) - tornado.ioloop.IOLoop.instance().start() - - -class AppEngineServer(ServerAdapter): - """ Adapter for Google App Engine. """ - quiet = True - def run(self, handler): - from google.appengine.ext.webapp import util - # A main() function in the handler script enables 'App Caching'. - # Lets makes sure it is there. This _really_ improves performance. - module = sys.modules.get('__main__') - if module and not hasattr(module, 'main'): - module.main = lambda: util.run_wsgi_app(handler) - util.run_wsgi_app(handler) - - -class TwistedServer(ServerAdapter): - """ Untested. """ - def run(self, handler): - from twisted.web import server, wsgi - from twisted.python.threadpool import ThreadPool - from twisted.internet import reactor - thread_pool = ThreadPool() - thread_pool.start() - reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop) - factory = server.Site(wsgi.WSGIResource(reactor, thread_pool, handler)) - reactor.listenTCP(self.port, factory, interface=self.host) - reactor.run() - - -class DieselServer(ServerAdapter): - """ Untested. """ - def run(self, handler): - from diesel.protocols.wsgi import WSGIApplication - app = WSGIApplication(handler, port=self.port) - app.run() - - -class GeventServer(ServerAdapter): - """ Untested. Options: - - * `fast` (default: False) uses libevent's http server, but has some - issues: No streaming, no pipelining, no SSL. - """ - def run(self, handler): - from gevent import wsgi, pywsgi, local - if not isinstance(_lctx, local.local): - msg = "Bottle requires gevent.monkey.patch_all() (before import)" - raise RuntimeError(msg) - if not self.options.get('fast'): wsgi = pywsgi - log = None if self.quiet else 'default' - wsgi.WSGIServer((self.host, self.port), handler, log=log).serve_forever() - - -class GunicornServer(ServerAdapter): - """ Untested. See http://gunicorn.org/configure.html for options. """ - def run(self, handler): - from gunicorn.app.base import Application - - config = {'bind': "%s:%d" % (self.host, int(self.port))} - config.update(self.options) - - class GunicornApplication(Application): - def init(self, parser, opts, args): - return config - - def load(self): - return handler - - GunicornApplication().run() - - -class EventletServer(ServerAdapter): - """ Untested """ - def run(self, handler): - from eventlet import wsgi, listen - try: - wsgi.server(listen((self.host, self.port)), handler, - log_output=(not self.quiet)) - except TypeError: - # Fallback, if we have old version of eventlet - wsgi.server(listen((self.host, self.port)), handler) - - -class RocketServer(ServerAdapter): - """ Untested. """ - def run(self, handler): - from rocket import Rocket - server = Rocket((self.host, self.port), 'wsgi', { 'wsgi_app' : handler }) - server.start() - - -class BjoernServer(ServerAdapter): - """ Fast server written in C: https://github.com/jonashaag/bjoern """ - def run(self, handler): - from bjoern import run - run(handler, self.host, self.port) - - -class AutoServer(ServerAdapter): - """ Untested. """ - adapters = [WaitressServer, PasteServer, TwistedServer, CherryPyServer, WSGIRefServer] - def run(self, handler): - for sa in self.adapters: - try: - return sa(self.host, self.port, **self.options).run(handler) - except ImportError: - pass - -server_names = { - 'cgi': CGIServer, - 'flup': FlupFCGIServer, - 'wsgiref': WSGIRefServer, - 'waitress': WaitressServer, - 'cherrypy': CherryPyServer, - 'paste': PasteServer, - 'fapws3': FapwsServer, - 'tornado': TornadoServer, - 'gae': AppEngineServer, - 'twisted': TwistedServer, - 'diesel': DieselServer, - 'meinheld': MeinheldServer, - 'gunicorn': GunicornServer, - 'eventlet': EventletServer, - 'gevent': GeventServer, - 'rocket': RocketServer, - 'bjoern' : BjoernServer, - 'auto': AutoServer, -} - - - - - - -############################################################################### -# Application Control ########################################################## -############################################################################### - - -def load(target, **namespace): - """ Import a module or fetch an object from a module. - - * ``package.module`` returns `module` as a module object. - * ``pack.mod:name`` returns the module variable `name` from `pack.mod`. - * ``pack.mod:func()`` calls `pack.mod.func()` and returns the result. - - The last form accepts not only function calls, but any type of - expression. Keyword arguments passed to this function are available as - local variables. Example: ``import_string('re:compile(x)', x='[a-z]')`` - """ - module, target = target.split(":", 1) if ':' in target else (target, None) - if module not in sys.modules: __import__(module) - if not target: return sys.modules[module] - if target.isalnum(): return getattr(sys.modules[module], target) - package_name = module.split('.')[0] - namespace[package_name] = sys.modules[package_name] - return eval('%s.%s' % (module, target), namespace) - - -def load_app(target): - """ Load a bottle application from a module and make sure that the import - does not affect the current default application, but returns a separate - application object. See :func:`load` for the target parameter. """ - global NORUN; NORUN, nr_old = True, NORUN - try: - tmp = default_app.push() # Create a new "default application" - rv = load(target) # Import the target module - return rv if callable(rv) else tmp - finally: - default_app.remove(tmp) # Remove the temporary added default application - NORUN = nr_old - -_debug = debug -def run(app=None, server='wsgiref', host='127.0.0.1', port=8080, - interval=1, reloader=False, quiet=False, plugins=None, - debug=False, **kargs): - """ Start a server instance. This method blocks until the server terminates. - - :param app: WSGI application or target string supported by - :func:`load_app`. (default: :func:`default_app`) - :param server: Server adapter to use. See :data:`server_names` keys - for valid names or pass a :class:`ServerAdapter` subclass. - (default: `wsgiref`) - :param host: Server address to bind to. Pass ``0.0.0.0`` to listens on - all interfaces including the external one. (default: 127.0.0.1) - :param port: Server port to bind to. Values below 1024 require root - privileges. (default: 8080) - :param reloader: Start auto-reloading server? (default: False) - :param interval: Auto-reloader interval in seconds (default: 1) - :param quiet: Suppress output to stdout and stderr? (default: False) - :param options: Options passed to the server adapter. - """ - if NORUN: return - if reloader and not os.environ.get('BOTTLE_CHILD'): - try: - lockfile = None - fd, lockfile = tempfile.mkstemp(prefix='bottle.', suffix='.lock') - os.close(fd) # We only need this file to exist. We never write to it - while os.path.exists(lockfile): - args = [sys.executable] + sys.argv - environ = os.environ.copy() - environ['BOTTLE_CHILD'] = 'true' - environ['BOTTLE_LOCKFILE'] = lockfile - p = subprocess.Popen(args, env=environ) - while p.poll() is None: # Busy wait... - os.utime(lockfile, None) # I am alive! - time.sleep(interval) - if p.poll() != 3: - if os.path.exists(lockfile): os.unlink(lockfile) - sys.exit(p.poll()) - except KeyboardInterrupt: - pass - finally: - if os.path.exists(lockfile): - os.unlink(lockfile) - return - - try: - _debug(debug) - app = app or default_app() - if isinstance(app, basestring): - app = load_app(app) - if not callable(app): - raise ValueError("Application is not callable: %r" % app) - - for plugin in plugins or []: - app.install(plugin) - - if server in server_names: - server = server_names.get(server) - if isinstance(server, basestring): - server = load(server) - if isinstance(server, type): - server = server(host=host, port=port, **kargs) - if not isinstance(server, ServerAdapter): - raise ValueError("Unknown or unsupported server: %r" % server) - - server.quiet = server.quiet or quiet - if not server.quiet: - _stderr("Bottle v%s server starting up (using %s)...\n" % (__version__, repr(server))) - _stderr("Listening on http://%s:%d/\n" % (server.host, server.port)) - _stderr("Hit Ctrl-C to quit.\n\n") - - if reloader: - lockfile = os.environ.get('BOTTLE_LOCKFILE') - bgcheck = FileCheckerThread(lockfile, interval) - with bgcheck: - server.run(app) - if bgcheck.status == 'reload': - sys.exit(3) - else: - server.run(app) - except KeyboardInterrupt: - pass - except (SystemExit, MemoryError): - raise - except: - if not reloader: raise - if not getattr(server, 'quiet', quiet): - print_exc() - time.sleep(interval) - sys.exit(3) - - - -class FileCheckerThread(threading.Thread): - ''' Interrupt main-thread as soon as a changed module file is detected, - the lockfile gets deleted or gets to old. ''' - - def __init__(self, lockfile, interval): - threading.Thread.__init__(self) - self.lockfile, self.interval = lockfile, interval - #: Is one of 'reload', 'error' or 'exit' - self.status = None - - def run(self): - exists = os.path.exists - mtime = lambda path: os.stat(path).st_mtime - files = dict() - - for module in list(sys.modules.values()): - path = getattr(module, '__file__', '') - if path[-4:] in ('.pyo', '.pyc'): path = path[:-1] - if path and exists(path): files[path] = mtime(path) - - while not self.status: - if not exists(self.lockfile)\ - or mtime(self.lockfile) < time.time() - self.interval - 5: - self.status = 'error' - thread.interrupt_main() - for path, lmtime in list(files.items()): - if not exists(path) or mtime(path) > lmtime: - self.status = 'reload' - thread.interrupt_main() - break - time.sleep(self.interval) - - def __enter__(self): - self.start() - - def __exit__(self, exc_type, exc_val, exc_tb): - if not self.status: self.status = 'exit' # silent exit - self.join() - return exc_type is not None and issubclass(exc_type, KeyboardInterrupt) - - - - - -############################################################################### -# Template Adapters ############################################################ -############################################################################### - - -class TemplateError(HTTPError): - def __init__(self, message): - HTTPError.__init__(self, 500, message) - - -class BaseTemplate(object): - """ Base class and minimal API for template adapters """ - extensions = ['tpl','html','thtml','stpl'] - settings = {} #used in prepare() - defaults = {} #used in render() - - def __init__(self, source=None, name=None, lookup=[], encoding='utf8', **settings): - """ Create a new template. - If the source parameter (str or buffer) is missing, the name argument - is used to guess a template filename. Subclasses can assume that - self.source and/or self.filename are set. Both are strings. - The lookup, encoding and settings parameters are stored as instance - variables. - The lookup parameter stores a list containing directory paths. - The encoding parameter should be used to decode byte strings or files. - The settings parameter contains a dict for engine-specific settings. - """ - self.name = name - self.source = source.read() if hasattr(source, 'read') else source - self.filename = source.filename if hasattr(source, 'filename') else None - self.lookup = [os.path.abspath(x) for x in lookup] - self.encoding = encoding - self.settings = self.settings.copy() # Copy from class variable - self.settings.update(settings) # Apply - if not self.source and self.name: - self.filename = self.search(self.name, self.lookup) - if not self.filename: - raise TemplateError('Template %s not found.' % repr(name)) - if not self.source and not self.filename: - raise TemplateError('No template specified.') - self.prepare(**self.settings) - - @classmethod - def search(cls, name, lookup=[]): - """ Search name in all directories specified in lookup. - First without, then with common extensions. Return first hit. """ - if not lookup: - depr('The template lookup path list should not be empty.') - lookup = ['.'] - - if os.path.isabs(name) and os.path.isfile(name): - depr('Absolute template path names are deprecated.') - return os.path.abspath(name) - - for spath in lookup: - spath = os.path.abspath(spath) + os.sep - fname = os.path.abspath(os.path.join(spath, name)) - if not fname.startswith(spath): continue - if os.path.isfile(fname): return fname - for ext in cls.extensions: - if os.path.isfile('%s.%s' % (fname, ext)): - return '%s.%s' % (fname, ext) - - @classmethod - def global_config(cls, key, *args): - ''' This reads or sets the global settings stored in class.settings. ''' - if args: - cls.settings = cls.settings.copy() # Make settings local to class - cls.settings[key] = args[0] - else: - return cls.settings[key] - - def prepare(self, **options): - """ Run preparations (parsing, caching, ...). - It should be possible to call this again to refresh a template or to - update settings. - """ - raise NotImplementedError - - def render(self, *args, **kwargs): - """ Render the template with the specified local variables and return - a single byte or unicode string. If it is a byte string, the encoding - must match self.encoding. This method must be thread-safe! - Local variables may be provided in dictionaries (*args) - or directly, as keywords (**kwargs). - """ - raise NotImplementedError - - -class MakoTemplate(BaseTemplate): - def prepare(self, **options): - from mako.template import Template - from mako.lookup import TemplateLookup - options.update({'input_encoding':self.encoding}) - options.setdefault('format_exceptions', bool(DEBUG)) - lookup = TemplateLookup(directories=self.lookup, **options) - if self.source: - self.tpl = Template(self.source, lookup=lookup, **options) - else: - self.tpl = Template(uri=self.name, filename=self.filename, lookup=lookup, **options) - - def render(self, *args, **kwargs): - for dictarg in args: kwargs.update(dictarg) - _defaults = self.defaults.copy() - _defaults.update(kwargs) - return self.tpl.render(**_defaults) - - -class CheetahTemplate(BaseTemplate): - def prepare(self, **options): - from Cheetah.Template import Template - self.context = threading.local() - self.context.vars = {} - options['searchList'] = [self.context.vars] - if self.source: - self.tpl = Template(source=self.source, **options) - else: - self.tpl = Template(file=self.filename, **options) - - def render(self, *args, **kwargs): - for dictarg in args: kwargs.update(dictarg) - self.context.vars.update(self.defaults) - self.context.vars.update(kwargs) - out = str(self.tpl) - self.context.vars.clear() - return out - - -class Jinja2Template(BaseTemplate): - def prepare(self, filters=None, tests=None, **kwargs): - from jinja2 import Environment, FunctionLoader - if 'prefix' in kwargs: # TODO: to be removed after a while - raise RuntimeError('The keyword argument `prefix` has been removed. ' - 'Use the full jinja2 environment name line_statement_prefix instead.') - self.env = Environment(loader=FunctionLoader(self.loader), **kwargs) - if filters: self.env.filters.update(filters) - if tests: self.env.tests.update(tests) - if self.source: - self.tpl = self.env.from_string(self.source) - else: - self.tpl = self.env.get_template(self.filename) - - def render(self, *args, **kwargs): - for dictarg in args: kwargs.update(dictarg) - _defaults = self.defaults.copy() - _defaults.update(kwargs) - return self.tpl.render(**_defaults) - - def loader(self, name): - fname = self.search(name, self.lookup) - if not fname: return - with open(fname, "rb") as f: - return f.read().decode(self.encoding) - - -class SimpleTALTemplate(BaseTemplate): - ''' Deprecated, do not use. ''' - def prepare(self, **options): - depr('The SimpleTAL template handler is deprecated'\ - ' and will be removed in 0.12') - from simpletal import simpleTAL - if self.source: - self.tpl = simpleTAL.compileHTMLTemplate(self.source) - else: - with open(self.filename, 'rb') as fp: - self.tpl = simpleTAL.compileHTMLTemplate(tonat(fp.read())) - - def render(self, *args, **kwargs): - from simpletal import simpleTALES - for dictarg in args: kwargs.update(dictarg) - context = simpleTALES.Context() - for k,v in self.defaults.items(): - context.addGlobal(k, v) - for k,v in kwargs.items(): - context.addGlobal(k, v) - output = StringIO() - self.tpl.expand(context, output) - return output.getvalue() - - -class SimpleTemplate(BaseTemplate): - blocks = ('if', 'elif', 'else', 'try', 'except', 'finally', 'for', 'while', - 'with', 'def', 'class') - dedent_blocks = ('elif', 'else', 'except', 'finally') - - @lazy_attribute - def re_pytokens(cls): - ''' This matches comments and all kinds of quoted strings but does - NOT match comments (#...) within quoted strings. (trust me) ''' - return re.compile(r''' - (''(?!')|""(?!")|'{6}|"{6} # Empty strings (all 4 types) - |'(?:[^\\']|\\.)+?' # Single quotes (') - |"(?:[^\\"]|\\.)+?" # Double quotes (") - |'{3}(?:[^\\]|\\.|\n)+?'{3} # Triple-quoted strings (') - |"{3}(?:[^\\]|\\.|\n)+?"{3} # Triple-quoted strings (") - |\#.* # Comments - )''', re.VERBOSE) - - def prepare(self, escape_func=html_escape, noescape=False, **kwargs): - self.cache = {} - enc = self.encoding - self._str = lambda x: touni(x, enc) - self._escape = lambda x: escape_func(touni(x, enc)) - if noescape: - self._str, self._escape = self._escape, self._str - - @classmethod - def split_comment(cls, code): - """ Removes comments (#...) from python code. """ - if '#' not in code: return code - #: Remove comments only (leave quoted strings as they are) - subf = lambda m: '' if m.group(0)[0]=='#' else m.group(0) - return re.sub(cls.re_pytokens, subf, code) - - @cached_property - def co(self): - return compile(self.code, self.filename or '', 'exec') - - @cached_property - def code(self): - stack = [] # Current Code indentation - lineno = 0 # Current line of code - ptrbuffer = [] # Buffer for printable strings and token tuple instances - codebuffer = [] # Buffer for generated python code - multiline = dedent = oneline = False - template = self.source or open(self.filename, 'rb').read() - - def yield_tokens(line): - for i, part in enumerate(re.split(r'\{\{(.*?)\}\}', line)): - if i % 2: - if part.startswith('!'): yield 'RAW', part[1:] - else: yield 'CMD', part - else: yield 'TXT', part - - def flush(): # Flush the ptrbuffer - if not ptrbuffer: return - cline = '' - for line in ptrbuffer: - for token, value in line: - if token == 'TXT': cline += repr(value) - elif token == 'RAW': cline += '_str(%s)' % value - elif token == 'CMD': cline += '_escape(%s)' % value - cline += ', ' - cline = cline[:-2] + '\\\n' - cline = cline[:-2] - if cline[:-1].endswith('\\\\\\\\\\n'): - cline = cline[:-7] + cline[-1] # 'nobr\\\\\n' --> 'nobr' - cline = '_printlist([' + cline + '])' - del ptrbuffer[:] # Do this before calling code() again - code(cline) - - def code(stmt): - for line in stmt.splitlines(): - codebuffer.append(' ' * len(stack) + line.strip()) - - for line in template.splitlines(True): - lineno += 1 - line = touni(line, self.encoding) - sline = line.lstrip() - if lineno <= 2: - m = re.match(r"%\s*#.*coding[:=]\s*([-\w.]+)", sline) - if m: self.encoding = m.group(1) - if m: line = line.replace('coding','coding (removed)') - if sline and sline[0] == '%' and sline[:2] != '%%': - line = line.split('%',1)[1].lstrip() # Full line following the % - cline = self.split_comment(line).strip() - cmd = re.split(r'[^a-zA-Z0-9_]', cline)[0] - flush() # You are actually reading this? Good luck, it's a mess :) - if cmd in self.blocks or multiline: - cmd = multiline or cmd - dedent = cmd in self.dedent_blocks # "else:" - if dedent and not oneline and not multiline: - cmd = stack.pop() - code(line) - oneline = not cline.endswith(':') # "if 1: pass" - multiline = cmd if cline.endswith('\\') else False - if not oneline and not multiline: - stack.append(cmd) - elif cmd == 'end' and stack: - code('#end(%s) %s' % (stack.pop(), line.strip()[3:])) - elif cmd == 'include': - p = cline.split(None, 2)[1:] - if len(p) == 2: - code("_=_include(%s, _stdout, %s)" % (repr(p[0]), p[1])) - elif p: - code("_=_include(%s, _stdout)" % repr(p[0])) - else: # Empty %include -> reverse of %rebase - code("_printlist(_base)") - elif cmd == 'rebase': - p = cline.split(None, 2)[1:] - if len(p) == 2: - code("globals()['_rebase']=(%s, dict(%s))" % (repr(p[0]), p[1])) - elif p: - code("globals()['_rebase']=(%s, {})" % repr(p[0])) - else: - code(line) - else: # Line starting with text (not '%') or '%%' (escaped) - if line.strip().startswith('%%'): - line = line.replace('%%', '%', 1) - ptrbuffer.append(yield_tokens(line)) - flush() - return '\n'.join(codebuffer) + '\n' - - def subtemplate(self, _name, _stdout, *args, **kwargs): - for dictarg in args: kwargs.update(dictarg) - if _name not in self.cache: - self.cache[_name] = self.__class__(name=_name, lookup=self.lookup) - return self.cache[_name].execute(_stdout, kwargs) - - def execute(self, _stdout, *args, **kwargs): - for dictarg in args: kwargs.update(dictarg) - env = self.defaults.copy() - env.update({'_stdout': _stdout, '_printlist': _stdout.extend, - '_include': self.subtemplate, '_str': self._str, - '_escape': self._escape, 'get': env.get, - 'setdefault': env.setdefault, 'defined': env.__contains__}) - env.update(kwargs) - eval(self.co, env) - if '_rebase' in env: - subtpl, rargs = env['_rebase'] - rargs['_base'] = _stdout[:] #copy stdout - del _stdout[:] # clear stdout - return self.subtemplate(subtpl,_stdout,rargs) - return env - - def render(self, *args, **kwargs): - """ Render the template using keyword arguments as local variables. """ - for dictarg in args: kwargs.update(dictarg) - stdout = [] - self.execute(stdout, kwargs) - return ''.join(stdout) - - -def template(*args, **kwargs): - ''' - Get a rendered template as a string iterator. - You can use a name, a filename or a template string as first parameter. - Template rendering arguments can be passed as dictionaries - or directly (as keyword arguments). - ''' - tpl = args[0] if args else None - adapter = kwargs.pop('template_adapter', SimpleTemplate) - lookup = kwargs.pop('template_lookup', TEMPLATE_PATH) - tplid = (id(lookup), tpl) - if tplid not in TEMPLATES or DEBUG: - settings = kwargs.pop('template_settings', {}) - if isinstance(tpl, adapter): - TEMPLATES[tplid] = tpl - if settings: TEMPLATES[tplid].prepare(**settings) - elif "\n" in tpl or "{" in tpl or "%" in tpl or '$' in tpl: - TEMPLATES[tplid] = adapter(source=tpl, lookup=lookup, **settings) - else: - TEMPLATES[tplid] = adapter(name=tpl, lookup=lookup, **settings) - if not TEMPLATES[tplid]: - abort(500, 'Template (%s) not found' % tpl) - for dictarg in args[1:]: kwargs.update(dictarg) - return TEMPLATES[tplid].render(kwargs) - -mako_template = functools.partial(template, template_adapter=MakoTemplate) -cheetah_template = functools.partial(template, template_adapter=CheetahTemplate) -jinja2_template = functools.partial(template, template_adapter=Jinja2Template) -simpletal_template = functools.partial(template, template_adapter=SimpleTALTemplate) - - -def view(tpl_name, **defaults): - ''' Decorator: renders a template for a handler. - The handler can control its behavior like that: - - - return a dict of template vars to fill out the template - - return something other than a dict and the view decorator will not - process the template, but return the handler result as is. - This includes returning a HTTPResponse(dict) to get, - for instance, JSON with autojson or other castfilters. - ''' - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - result = func(*args, **kwargs) - if isinstance(result, (dict, DictMixin)): - tplvars = defaults.copy() - tplvars.update(result) - return template(tpl_name, **tplvars) - return result - return wrapper - return decorator - -mako_view = functools.partial(view, template_adapter=MakoTemplate) -cheetah_view = functools.partial(view, template_adapter=CheetahTemplate) -jinja2_view = functools.partial(view, template_adapter=Jinja2Template) -simpletal_view = functools.partial(view, template_adapter=SimpleTALTemplate) - - - - - - -############################################################################### -# Constants and Globals ######################################################## -############################################################################### - - -TEMPLATE_PATH = ['./', './views/'] -TEMPLATES = {} -DEBUG = False -NORUN = False # If set, run() does nothing. Used by load_app() - -#: A dict to map HTTP status codes (e.g. 404) to phrases (e.g. 'Not Found') -HTTP_CODES = httplib.responses -HTTP_CODES[418] = "I'm a teapot" # RFC 2324 -HTTP_CODES[428] = "Precondition Required" -HTTP_CODES[429] = "Too Many Requests" -HTTP_CODES[431] = "Request Header Fields Too Large" -HTTP_CODES[511] = "Network Authentication Required" -_HTTP_STATUS_LINES = dict((k, '%d %s'%(k,v)) for (k,v) in HTTP_CODES.items()) - -#: The default template used for error pages. Override with @error() -ERROR_PAGE_TEMPLATE = """ -%%try: - %%from %s import DEBUG, HTTP_CODES, request, touni - - - - Error: {{e.status}} - - - -

Error: {{e.status}}

-

Sorry, the requested URL {{repr(request.url)}} - caused an error:

- 
{{e.body}}
- %%if DEBUG and e.exception: -

Exception:

- 
{{repr(e.exception)}}
- %%end - %%if DEBUG and e.traceback: -

Traceback:

- 
{{e.traceback}}
- %%end - - -%%except ImportError: - ImportError: Could not generate the error page. Please add bottle to - the import path. -%%end -""" % __name__ - -#: A thread-safe instance of :class:`LocalRequest`. If accessed from within a -#: request callback, this instance always refers to the *current* request -#: (even on a multithreaded server). -request = LocalRequest() - -#: A thread-safe instance of :class:`LocalResponse`. It is used to change the -#: HTTP response for the *current* request. -response = LocalResponse() - -#: A thread-safe namespace. Not used by Bottle. -local = threading.local() - -# Initialize app stack (create first empty Bottle app) -# BC: 0.6.4 and needed for run() -app = default_app = AppStack() -app.push() - -#: A virtual package that redirects import statements. -#: Example: ``import bottle.ext.sqlite`` actually imports `bottle_sqlite`. -ext = _ImportRedirect('bottle.ext' if __name__ == '__main__' else __name__+".ext", 'bottle_%s').module - -if __name__ == '__main__': - opt, args, parser = _cmd_options, _cmd_args, _cmd_parser - if opt.version: - _stdout('Bottle %s\n'%__version__) - sys.exit(0) - if not args: - parser.print_help() - _stderr('\nError: No application specified.\n') - sys.exit(1) - - sys.path.insert(0, '.') - sys.modules.setdefault('bottle', sys.modules['__main__']) - - host, port = (opt.bind or 'localhost'), 8080 - if ':' in host: - host, port = host.rsplit(':', 1) - - run(args[0], host=host, port=port, server=opt.server, - reloader=opt.reload, plugins=opt.plugin, debug=opt.debug) - - - - -# THE END diff --git a/module/lib/forwarder.py b/module/lib/forwarder.py deleted file mode 100644 index eacb33c2b..000000000 --- a/module/lib/forwarder.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" - -from sys import argv -from sys import exit - -import socket -import thread - -from traceback import print_exc - -class Forwarder(): - - def __init__(self, extip,extport=9666): - print "Start portforwarding to %s:%s" % (extip, extport) - proxy(extip, extport, 9666) - - -def proxy(*settings): - while True: - server(*settings) - -def server(*settings): - try: - dock_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - dock_socket.bind(("127.0.0.1", settings[2])) - dock_socket.listen(5) - while True: - client_socket = dock_socket.accept()[0] - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.connect((settings[0], settings[1])) - thread.start_new_thread(forward, (client_socket, server_socket)) - thread.start_new_thread(forward, (server_socket, client_socket)) - except Exception: - print_exc() - - -def forward(source, destination): - string = ' ' - while string: - string = source.recv(1024) - if string: - destination.sendall(string) - else: - #source.shutdown(socket.SHUT_RD) - destination.shutdown(socket.SHUT_WR) - -if __name__ == "__main__": - args = argv[1:] - if not args: - print "Usage: forwarder.py " - exit() - if len(args) == 1: - args.append(9666) - - f = Forwarder(args[0], int(args[1])) - \ No newline at end of file diff --git a/module/lib/hg_tool.py b/module/lib/hg_tool.py deleted file mode 100644 index cd97833df..000000000 --- a/module/lib/hg_tool.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from subprocess import Popen, PIPE -from time import time, gmtime, strftime - -aliases = {"zoidber": "zoidberg", "zoidberg10": "zoidberg", "webmaster": "dhmh", "mast3rranan": "ranan", - "ranan2": "ranan"} -exclude = ["locale/*", "module/lib/*"] -date_format = "%Y-%m-%d" -line_re = re.compile(r" (\d+) \**", re.I) - -def add_exclude_flags(args): - for dir in exclude: - args.extend(["-X", dir]) - -# remove small percentages -def wipe(data, perc=1): - s = (sum(data.values()) * perc) / 100 - for k, v in data.items(): - if v < s: del data[k] - - return data - -# remove aliases -def de_alias(data): - for k, v in aliases.iteritems(): - if k not in data: continue - alias = aliases[k] - - if alias in data: data[alias] += data[k] - else: data[alias] = data[k] - - del data[k] - - return data - - -def output(data): - s = float(sum(data.values())) - print "Total Lines: %d" % s - for k, v in data.iteritems(): - print "%15s: %.1f%% | %d" % (k, (v * 100) / s, v) - print - - -def file_list(): - args = ["hg", "status", "-A"] - add_exclude_flags(args) - p = Popen(args, stdout=PIPE) - out, err = p.communicate() - return [x.split()[1] for x in out.splitlines() if x.split()[0] in "CMA"] - - -def hg_annotate(path): - args = ["hg", "annotate", "-u", path] - p = Popen(args, stdout=PIPE) - out, err = p.communicate() - - data = {} - - for line in out.splitlines(): - author, non, line = line.partition(":") - - # probably binary file - if author == path: return {} - - author = author.strip().lower() - if not line.strip(): continue # don't count blank lines - - if author in data: data[author] += 1 - else: data[author] = 1 - - return de_alias(data) - - -def hg_churn(days=None): - args = ["hg", "churn"] - if days: - args.append("-d") - t = time() - 60 * 60 * 24 * days - args.append("%s to %s" % (strftime(date_format, gmtime(t)), strftime(date_format))) - - add_exclude_flags(args) - p = Popen(args, stdout=PIPE) - out, err = p.communicate() - - data = {} - - for line in out.splitlines(): - m = line_re.search(line) - author = line.split()[0] - lines = int(m.group(1)) - - if "@" in author: - author, n, email = author.partition("@") - - author = author.strip().lower() - - if author in data: data[author] += lines - else: data[author] = lines - - return de_alias(data) - - -def complete_annotate(): - files = file_list() - data = {} - for f in files: - tmp = hg_annotate(f) - for k, v in tmp.iteritems(): - if k in data: data[k] += v - else: data[k] = v - - return data - - -if __name__ == "__main__": - for d in (30, 90, 180): - c = wipe(hg_churn(d)) - print "Changes in %d days:" % d - output(c) - - c = wipe(hg_churn()) - print "Total changes:" - output(c) - - print "Current source code version:" - data = wipe(complete_annotate()) - output(data) - - diff --git a/module/lib/mod_pywebsocket/COPYING b/module/lib/mod_pywebsocket/COPYING deleted file mode 100644 index 989d02e4c..000000000 --- a/module/lib/mod_pywebsocket/COPYING +++ /dev/null @@ -1,28 +0,0 @@ -Copyright 2012, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/module/lib/mod_pywebsocket/__init__.py b/module/lib/mod_pywebsocket/__init__.py deleted file mode 100644 index 454ae0c45..000000000 --- a/module/lib/mod_pywebsocket/__init__.py +++ /dev/null @@ -1,197 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""WebSocket extension for Apache HTTP Server. - -mod_pywebsocket is a WebSocket extension for Apache HTTP Server -intended for testing or experimental purposes. mod_python is required. - - -Installation -============ - -0. Prepare an Apache HTTP Server for which mod_python is enabled. - -1. Specify the following Apache HTTP Server directives to suit your - configuration. - - If mod_pywebsocket is not in the Python path, specify the following. - is the directory where mod_pywebsocket is installed. - - PythonPath "sys.path+['']" - - Always specify the following. is the directory where - user-written WebSocket handlers are placed. - - PythonOption mod_pywebsocket.handler_root - PythonHeaderParserHandler mod_pywebsocket.headerparserhandler - - To limit the search for WebSocket handlers to a directory - under , configure as follows: - - PythonOption mod_pywebsocket.handler_scan - - is useful in saving scan time when - contains many non-WebSocket handler files. - - If you want to allow handlers whose canonical path is not under the root - directory (i.e. symbolic link is in root directory but its target is not), - configure as follows: - - PythonOption mod_pywebsocket.allow_handlers_outside_root_dir On - - Example snippet of httpd.conf: - (mod_pywebsocket is in /websock_lib, WebSocket handlers are in - /websock_handlers, port is 80 for ws, 443 for wss.) - - - PythonPath "sys.path+['/websock_lib']" - PythonOption mod_pywebsocket.handler_root /websock_handlers - PythonHeaderParserHandler mod_pywebsocket.headerparserhandler - - -2. Tune Apache parameters for serving WebSocket. We'd like to note that at - least TimeOut directive from core features and RequestReadTimeout - directive from mod_reqtimeout should be modified not to kill connections - in only a few seconds of idle time. - -3. Verify installation. You can use example/console.html to poke the server. - - -Writing WebSocket handlers -========================== - -When a WebSocket request comes in, the resource name -specified in the handshake is considered as if it is a file path under - and the handler defined in -/_wsh.py is invoked. - -For example, if the resource name is /example/chat, the handler defined in -/example/chat_wsh.py is invoked. - -A WebSocket handler is composed of the following three functions: - - web_socket_do_extra_handshake(request) - web_socket_transfer_data(request) - web_socket_passive_closing_handshake(request) - -where: - request: mod_python request. - -web_socket_do_extra_handshake is called during the handshake after the -headers are successfully parsed and WebSocket properties (ws_location, -ws_origin, and ws_resource) are added to request. A handler -can reject the request by raising an exception. - -A request object has the following properties that you can use during the -extra handshake (web_socket_do_extra_handshake): -- ws_resource -- ws_origin -- ws_version -- ws_location (HyBi 00 only) -- ws_extensions (HyBi 06 and later) -- ws_deflate (HyBi 06 and later) -- ws_protocol -- ws_requested_protocols (HyBi 06 and later) - -The last two are a bit tricky. See the next subsection. - - -Subprotocol Negotiation ------------------------ - -For HyBi 06 and later, ws_protocol is always set to None when -web_socket_do_extra_handshake is called. If ws_requested_protocols is not -None, you must choose one subprotocol from this list and set it to -ws_protocol. - -For HyBi 00, when web_socket_do_extra_handshake is called, -ws_protocol is set to the value given by the client in -Sec-WebSocket-Protocol header or None if -such header was not found in the opening handshake request. Finish extra -handshake with ws_protocol untouched to accept the request subprotocol. -Then, Sec-WebSocket-Protocol header will be sent to -the client in response with the same value as requested. Raise an exception -in web_socket_do_extra_handshake to reject the requested subprotocol. - - -Data Transfer -------------- - -web_socket_transfer_data is called after the handshake completed -successfully. A handler can receive/send messages from/to the client -using request. mod_pywebsocket.msgutil module provides utilities -for data transfer. - -You can receive a message by the following statement. - - message = request.ws_stream.receive_message() - -This call blocks until any complete text frame arrives, and the payload data -of the incoming frame will be stored into message. When you're using IETF -HyBi 00 or later protocol, receive_message() will return None on receiving -client-initiated closing handshake. When any error occurs, receive_message() -will raise some exception. - -You can send a message by the following statement. - - request.ws_stream.send_message(message) - - -Closing Connection ------------------- - -Executing the following statement or just return-ing from -web_socket_transfer_data cause connection close. - - request.ws_stream.close_connection() - -close_connection will wait -for closing handshake acknowledgement coming from the client. When it -couldn't receive a valid acknowledgement, raises an exception. - -web_socket_passive_closing_handshake is called after the server receives -incoming closing frame from the client peer immediately. You can specify -code and reason by return values. They are sent as a outgoing closing frame -from the server. A request object has the following properties that you can -use in web_socket_passive_closing_handshake. -- ws_close_code -- ws_close_reason - - -Threading ---------- - -A WebSocket handler must be thread-safe if the server (Apache or -standalone.py) is configured to use threads. -""" - - -# vi:sts=4 sw=4 et tw=72 diff --git a/module/lib/mod_pywebsocket/_stream_base.py b/module/lib/mod_pywebsocket/_stream_base.py deleted file mode 100644 index 60fb33d2c..000000000 --- a/module/lib/mod_pywebsocket/_stream_base.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""Base stream class. -""" - - -# Note: request.connection.write/read are used in this module, even though -# mod_python document says that they should be used only in connection -# handlers. Unfortunately, we have no other options. For example, -# request.write/read are not suitable because they don't allow direct raw bytes -# writing/reading. - - -from mod_pywebsocket import util - - -# Exceptions - - -class ConnectionTerminatedException(Exception): - """This exception will be raised when a connection is terminated - unexpectedly. - """ - - pass - - -class InvalidFrameException(ConnectionTerminatedException): - """This exception will be raised when we received an invalid frame we - cannot parse. - """ - - pass - - -class BadOperationException(Exception): - """This exception will be raised when send_message() is called on - server-terminated connection or receive_message() is called on - client-terminated connection. - """ - - pass - - -class UnsupportedFrameException(Exception): - """This exception will be raised when we receive a frame with flag, opcode - we cannot handle. Handlers can just catch and ignore this exception and - call receive_message() again to continue processing the next frame. - """ - - pass - - -class InvalidUTF8Exception(Exception): - """This exception will be raised when we receive a text frame which - contains invalid UTF-8 strings. - """ - - pass - - -class StreamBase(object): - """Base stream class.""" - - def __init__(self, request): - """Construct an instance. - - Args: - request: mod_python request. - """ - - self._logger = util.get_class_logger(self) - - self._request = request - - def _read(self, length): - """Reads length bytes from connection. In case we catch any exception, - prepends remote address to the exception message and raise again. - - Raises: - ConnectionTerminatedException: when read returns empty string. - """ - - bytes = self._request.connection.read(length) - if not bytes: - raise ConnectionTerminatedException( - 'Receiving %d byte failed. Peer (%r) closed connection' % - (length, (self._request.connection.remote_addr,))) - return bytes - - def _write(self, bytes): - """Writes given bytes to connection. In case we catch any exception, - prepends remote address to the exception message and raise again. - """ - - try: - self._request.connection.write(bytes) - except Exception, e: - util.prepend_message_to_exception( - 'Failed to send message to %r: ' % - (self._request.connection.remote_addr,), - e) - raise - - def receive_bytes(self, length): - """Receives multiple bytes. Retries read when we couldn't receive the - specified amount. - - Raises: - ConnectionTerminatedException: when read returns empty string. - """ - - bytes = [] - while length > 0: - new_bytes = self._read(length) - bytes.append(new_bytes) - length -= len(new_bytes) - return ''.join(bytes) - - def _read_until(self, delim_char): - """Reads bytes until we encounter delim_char. The result will not - contain delim_char. - - Raises: - ConnectionTerminatedException: when read returns empty string. - """ - - bytes = [] - while True: - ch = self._read(1) - if ch == delim_char: - break - bytes.append(ch) - return ''.join(bytes) - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/_stream_hixie75.py b/module/lib/mod_pywebsocket/_stream_hixie75.py deleted file mode 100644 index 94cf5b31b..000000000 --- a/module/lib/mod_pywebsocket/_stream_hixie75.py +++ /dev/null @@ -1,229 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""This file provides a class for parsing/building frames of the WebSocket -protocol version HyBi 00 and Hixie 75. - -Specification: -- HyBi 00 http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-00 -- Hixie 75 http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-75 -""" - - -from mod_pywebsocket import common -from mod_pywebsocket._stream_base import BadOperationException -from mod_pywebsocket._stream_base import ConnectionTerminatedException -from mod_pywebsocket._stream_base import InvalidFrameException -from mod_pywebsocket._stream_base import StreamBase -from mod_pywebsocket._stream_base import UnsupportedFrameException -from mod_pywebsocket import util - - -class StreamHixie75(StreamBase): - """A class for parsing/building frames of the WebSocket protocol version - HyBi 00 and Hixie 75. - """ - - def __init__(self, request, enable_closing_handshake=False): - """Construct an instance. - - Args: - request: mod_python request. - enable_closing_handshake: to let StreamHixie75 perform closing - handshake as specified in HyBi 00, set - this option to True. - """ - - StreamBase.__init__(self, request) - - self._logger = util.get_class_logger(self) - - self._enable_closing_handshake = enable_closing_handshake - - self._request.client_terminated = False - self._request.server_terminated = False - - def send_message(self, message, end=True, binary=False): - """Send message. - - Args: - message: unicode string to send. - binary: not used in hixie75. - - Raises: - BadOperationException: when called on a server-terminated - connection. - """ - - if not end: - raise BadOperationException( - 'StreamHixie75 doesn\'t support send_message with end=False') - - if binary: - raise BadOperationException( - 'StreamHixie75 doesn\'t support send_message with binary=True') - - if self._request.server_terminated: - raise BadOperationException( - 'Requested send_message after sending out a closing handshake') - - self._write(''.join(['\x00', message.encode('utf-8'), '\xff'])) - - def _read_payload_length_hixie75(self): - """Reads a length header in a Hixie75 version frame with length. - - Raises: - ConnectionTerminatedException: when read returns empty string. - """ - - length = 0 - while True: - b_str = self._read(1) - b = ord(b_str) - length = length * 128 + (b & 0x7f) - if (b & 0x80) == 0: - break - return length - - def receive_message(self): - """Receive a WebSocket frame and return its payload an unicode string. - - Returns: - payload unicode string in a WebSocket frame. - - Raises: - ConnectionTerminatedException: when read returns empty - string. - BadOperationException: when called on a client-terminated - connection. - """ - - if self._request.client_terminated: - raise BadOperationException( - 'Requested receive_message after receiving a closing ' - 'handshake') - - while True: - # Read 1 byte. - # mp_conn.read will block if no bytes are available. - # Timeout is controlled by TimeOut directive of Apache. - frame_type_str = self.receive_bytes(1) - frame_type = ord(frame_type_str) - if (frame_type & 0x80) == 0x80: - # The payload length is specified in the frame. - # Read and discard. - length = self._read_payload_length_hixie75() - if length > 0: - _ = self.receive_bytes(length) - # 5.3 3. 12. if /type/ is 0xFF and /length/ is 0, then set the - # /client terminated/ flag and abort these steps. - if not self._enable_closing_handshake: - continue - - if frame_type == 0xFF and length == 0: - self._request.client_terminated = True - - if self._request.server_terminated: - self._logger.debug( - 'Received ack for server-initiated closing ' - 'handshake') - return None - - self._logger.debug( - 'Received client-initiated closing handshake') - - self._send_closing_handshake() - self._logger.debug( - 'Sent ack for client-initiated closing handshake') - return None - else: - # The payload is delimited with \xff. - bytes = self._read_until('\xff') - # The WebSocket protocol section 4.4 specifies that invalid - # characters must be replaced with U+fffd REPLACEMENT - # CHARACTER. - message = bytes.decode('utf-8', 'replace') - if frame_type == 0x00: - return message - # Discard data of other types. - - def _send_closing_handshake(self): - if not self._enable_closing_handshake: - raise BadOperationException( - 'Closing handshake is not supported in Hixie 75 protocol') - - self._request.server_terminated = True - - # 5.3 the server may decide to terminate the WebSocket connection by - # running through the following steps: - # 1. send a 0xFF byte and a 0x00 byte to the client to indicate the - # start of the closing handshake. - self._write('\xff\x00') - - def close_connection(self, unused_code='', unused_reason=''): - """Closes a WebSocket connection. - - Raises: - ConnectionTerminatedException: when closing handshake was - not successfull. - """ - - if self._request.server_terminated: - self._logger.debug( - 'Requested close_connection but server is already terminated') - return - - if not self._enable_closing_handshake: - self._request.server_terminated = True - self._logger.debug('Connection closed') - return - - self._send_closing_handshake() - self._logger.debug('Sent server-initiated closing handshake') - - # TODO(ukai): 2. wait until the /client terminated/ flag has been set, - # or until a server-defined timeout expires. - # - # For now, we expect receiving closing handshake right after sending - # out closing handshake, and if we couldn't receive non-handshake - # frame, we take it as ConnectionTerminatedException. - message = self.receive_message() - if message is not None: - raise ConnectionTerminatedException( - 'Didn\'t receive valid ack for closing handshake') - # TODO: 3. close the WebSocket connection. - # note: mod_python Connection (mp_conn) doesn't have close method. - - def send_ping(self, body): - raise BadOperationException( - 'StreamHixie75 doesn\'t support send_ping') - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/_stream_hybi.py b/module/lib/mod_pywebsocket/_stream_hybi.py deleted file mode 100644 index bd158fa6b..000000000 --- a/module/lib/mod_pywebsocket/_stream_hybi.py +++ /dev/null @@ -1,915 +0,0 @@ -# Copyright 2012, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""This file provides classes and helper functions for parsing/building frames -of the WebSocket protocol (RFC 6455). - -Specification: -http://tools.ietf.org/html/rfc6455 -""" - - -from collections import deque -import logging -import os -import struct -import time - -from mod_pywebsocket import common -from mod_pywebsocket import util -from mod_pywebsocket._stream_base import BadOperationException -from mod_pywebsocket._stream_base import ConnectionTerminatedException -from mod_pywebsocket._stream_base import InvalidFrameException -from mod_pywebsocket._stream_base import InvalidUTF8Exception -from mod_pywebsocket._stream_base import StreamBase -from mod_pywebsocket._stream_base import UnsupportedFrameException - - -_NOOP_MASKER = util.NoopMasker() - - -class Frame(object): - - def __init__(self, fin=1, rsv1=0, rsv2=0, rsv3=0, - opcode=None, payload=''): - self.fin = fin - self.rsv1 = rsv1 - self.rsv2 = rsv2 - self.rsv3 = rsv3 - self.opcode = opcode - self.payload = payload - - -# Helper functions made public to be used for writing unittests for WebSocket -# clients. - - -def create_length_header(length, mask): - """Creates a length header. - - Args: - length: Frame length. Must be less than 2^63. - mask: Mask bit. Must be boolean. - - Raises: - ValueError: when bad data is given. - """ - - if mask: - mask_bit = 1 << 7 - else: - mask_bit = 0 - - if length < 0: - raise ValueError('length must be non negative integer') - elif length <= 125: - return chr(mask_bit | length) - elif length < (1 << 16): - return chr(mask_bit | 126) + struct.pack('!H', length) - elif length < (1 << 63): - return chr(mask_bit | 127) + struct.pack('!Q', length) - else: - raise ValueError('Payload is too big for one frame') - - -def create_header(opcode, payload_length, fin, rsv1, rsv2, rsv3, mask): - """Creates a frame header. - - Raises: - Exception: when bad data is given. - """ - - if opcode < 0 or 0xf < opcode: - raise ValueError('Opcode out of range') - - if payload_length < 0 or (1 << 63) <= payload_length: - raise ValueError('payload_length out of range') - - if (fin | rsv1 | rsv2 | rsv3) & ~1: - raise ValueError('FIN bit and Reserved bit parameter must be 0 or 1') - - header = '' - - first_byte = ((fin << 7) - | (rsv1 << 6) | (rsv2 << 5) | (rsv3 << 4) - | opcode) - header += chr(first_byte) - header += create_length_header(payload_length, mask) - - return header - - -def _build_frame(header, body, mask): - if not mask: - return header + body - - masking_nonce = os.urandom(4) - masker = util.RepeatedXorMasker(masking_nonce) - - return header + masking_nonce + masker.mask(body) - - -def _filter_and_format_frame_object(frame, mask, frame_filters): - for frame_filter in frame_filters: - frame_filter.filter(frame) - - header = create_header( - frame.opcode, len(frame.payload), frame.fin, - frame.rsv1, frame.rsv2, frame.rsv3, mask) - return _build_frame(header, frame.payload, mask) - - -def create_binary_frame( - message, opcode=common.OPCODE_BINARY, fin=1, mask=False, frame_filters=[]): - """Creates a simple binary frame with no extension, reserved bit.""" - - frame = Frame(fin=fin, opcode=opcode, payload=message) - return _filter_and_format_frame_object(frame, mask, frame_filters) - - -def create_text_frame( - message, opcode=common.OPCODE_TEXT, fin=1, mask=False, frame_filters=[]): - """Creates a simple text frame with no extension, reserved bit.""" - - encoded_message = message.encode('utf-8') - return create_binary_frame(encoded_message, opcode, fin, mask, - frame_filters) - - -def parse_frame(receive_bytes, logger=None, - ws_version=common.VERSION_HYBI_LATEST, - unmask_receive=True): - """Parses a frame. Returns a tuple containing each header field and - payload. - - Args: - receive_bytes: a function that reads frame data from a stream or - something similar. The function takes length of the bytes to be - read. The function must raise ConnectionTerminatedException if - there is not enough data to be read. - logger: a logging object. - ws_version: the version of WebSocket protocol. - unmask_receive: unmask received frames. When received unmasked - frame, raises InvalidFrameException. - - Raises: - ConnectionTerminatedException: when receive_bytes raises it. - InvalidFrameException: when the frame contains invalid data. - """ - - if not logger: - logger = logging.getLogger() - - logger.log(common.LOGLEVEL_FINE, 'Receive the first 2 octets of a frame') - - received = receive_bytes(2) - - first_byte = ord(received[0]) - fin = (first_byte >> 7) & 1 - rsv1 = (first_byte >> 6) & 1 - rsv2 = (first_byte >> 5) & 1 - rsv3 = (first_byte >> 4) & 1 - opcode = first_byte & 0xf - - second_byte = ord(received[1]) - mask = (second_byte >> 7) & 1 - payload_length = second_byte & 0x7f - - logger.log(common.LOGLEVEL_FINE, - 'FIN=%s, RSV1=%s, RSV2=%s, RSV3=%s, opcode=%s, ' - 'Mask=%s, Payload_length=%s', - fin, rsv1, rsv2, rsv3, opcode, mask, payload_length) - - if (mask == 1) != unmask_receive: - raise InvalidFrameException( - 'Mask bit on the received frame did\'nt match masking ' - 'configuration for received frames') - - # The HyBi and later specs disallow putting a value in 0x0-0xFFFF - # into the 8-octet extended payload length field (or 0x0-0xFD in - # 2-octet field). - valid_length_encoding = True - length_encoding_bytes = 1 - if payload_length == 127: - logger.log(common.LOGLEVEL_FINE, - 'Receive 8-octet extended payload length') - - extended_payload_length = receive_bytes(8) - payload_length = struct.unpack( - '!Q', extended_payload_length)[0] - if payload_length > 0x7FFFFFFFFFFFFFFF: - raise InvalidFrameException( - 'Extended payload length >= 2^63') - if ws_version >= 13 and payload_length < 0x10000: - valid_length_encoding = False - length_encoding_bytes = 8 - - logger.log(common.LOGLEVEL_FINE, - 'Decoded_payload_length=%s', payload_length) - elif payload_length == 126: - logger.log(common.LOGLEVEL_FINE, - 'Receive 2-octet extended payload length') - - extended_payload_length = receive_bytes(2) - payload_length = struct.unpack( - '!H', extended_payload_length)[0] - if ws_version >= 13 and payload_length < 126: - valid_length_encoding = False - length_encoding_bytes = 2 - - logger.log(common.LOGLEVEL_FINE, - 'Decoded_payload_length=%s', payload_length) - - if not valid_length_encoding: - logger.warning( - 'Payload length is not encoded using the minimal number of ' - 'bytes (%d is encoded using %d bytes)', - payload_length, - length_encoding_bytes) - - if mask == 1: - logger.log(common.LOGLEVEL_FINE, 'Receive mask') - - masking_nonce = receive_bytes(4) - masker = util.RepeatedXorMasker(masking_nonce) - - logger.log(common.LOGLEVEL_FINE, 'Mask=%r', masking_nonce) - else: - masker = _NOOP_MASKER - - logger.log(common.LOGLEVEL_FINE, 'Receive payload data') - if logger.isEnabledFor(common.LOGLEVEL_FINE): - receive_start = time.time() - - raw_payload_bytes = receive_bytes(payload_length) - - if logger.isEnabledFor(common.LOGLEVEL_FINE): - logger.log( - common.LOGLEVEL_FINE, - 'Done receiving payload data at %s MB/s', - payload_length / (time.time() - receive_start) / 1000 / 1000) - logger.log(common.LOGLEVEL_FINE, 'Unmask payload data') - - if logger.isEnabledFor(common.LOGLEVEL_FINE): - unmask_start = time.time() - - bytes = masker.mask(raw_payload_bytes) - - if logger.isEnabledFor(common.LOGLEVEL_FINE): - logger.log( - common.LOGLEVEL_FINE, - 'Done unmasking payload data at %s MB/s', - payload_length / (time.time() - unmask_start) / 1000 / 1000) - - return opcode, bytes, fin, rsv1, rsv2, rsv3 - - -class FragmentedFrameBuilder(object): - """A stateful class to send a message as fragments.""" - - def __init__(self, mask, frame_filters=[], encode_utf8=True): - """Constructs an instance.""" - - self._mask = mask - self._frame_filters = frame_filters - # This is for skipping UTF-8 encoding when building text type frames - # from compressed data. - self._encode_utf8 = encode_utf8 - - self._started = False - - # Hold opcode of the first frame in messages to verify types of other - # frames in the message are all the same. - self._opcode = common.OPCODE_TEXT - - def build(self, payload_data, end, binary): - if binary: - frame_type = common.OPCODE_BINARY - else: - frame_type = common.OPCODE_TEXT - if self._started: - if self._opcode != frame_type: - raise ValueError('Message types are different in frames for ' - 'the same message') - opcode = common.OPCODE_CONTINUATION - else: - opcode = frame_type - self._opcode = frame_type - - if end: - self._started = False - fin = 1 - else: - self._started = True - fin = 0 - - if binary or not self._encode_utf8: - return create_binary_frame( - payload_data, opcode, fin, self._mask, self._frame_filters) - else: - return create_text_frame( - payload_data, opcode, fin, self._mask, self._frame_filters) - - -def _create_control_frame(opcode, body, mask, frame_filters): - frame = Frame(opcode=opcode, payload=body) - - for frame_filter in frame_filters: - frame_filter.filter(frame) - - if len(frame.payload) > 125: - raise BadOperationException( - 'Payload data size of control frames must be 125 bytes or less') - - header = create_header( - frame.opcode, len(frame.payload), frame.fin, - frame.rsv1, frame.rsv2, frame.rsv3, mask) - return _build_frame(header, frame.payload, mask) - - -def create_ping_frame(body, mask=False, frame_filters=[]): - return _create_control_frame(common.OPCODE_PING, body, mask, frame_filters) - - -def create_pong_frame(body, mask=False, frame_filters=[]): - return _create_control_frame(common.OPCODE_PONG, body, mask, frame_filters) - - -def create_close_frame(body, mask=False, frame_filters=[]): - return _create_control_frame( - common.OPCODE_CLOSE, body, mask, frame_filters) - - -def create_closing_handshake_body(code, reason): - body = '' - if code is not None: - if (code > common.STATUS_USER_PRIVATE_MAX or - code < common.STATUS_NORMAL_CLOSURE): - raise BadOperationException('Status code is out of range') - if (code == common.STATUS_NO_STATUS_RECEIVED or - code == common.STATUS_ABNORMAL_CLOSURE or - code == common.STATUS_TLS_HANDSHAKE): - raise BadOperationException('Status code is reserved pseudo ' - 'code') - encoded_reason = reason.encode('utf-8') - body = struct.pack('!H', code) + encoded_reason - return body - - -class StreamOptions(object): - """Holds option values to configure Stream objects.""" - - def __init__(self): - """Constructs StreamOptions.""" - - # Enables deflate-stream extension. - self.deflate_stream = False - - # Filters applied to frames. - self.outgoing_frame_filters = [] - self.incoming_frame_filters = [] - - # Filters applied to messages. Control frames are not affected by them. - self.outgoing_message_filters = [] - self.incoming_message_filters = [] - - self.encode_text_message_to_utf8 = True - self.mask_send = False - self.unmask_receive = True - # RFC6455 disallows fragmented control frames, but mux extension - # relaxes the restriction. - self.allow_fragmented_control_frame = False - - -class Stream(StreamBase): - """A class for parsing/building frames of the WebSocket protocol - (RFC 6455). - """ - - def __init__(self, request, options): - """Constructs an instance. - - Args: - request: mod_python request. - """ - - StreamBase.__init__(self, request) - - self._logger = util.get_class_logger(self) - - self._options = options - - if self._options.deflate_stream: - self._logger.debug('Setup filter for deflate-stream') - self._request = util.DeflateRequest(self._request) - - self._request.client_terminated = False - self._request.server_terminated = False - - # Holds body of received fragments. - self._received_fragments = [] - # Holds the opcode of the first fragment. - self._original_opcode = None - - self._writer = FragmentedFrameBuilder( - self._options.mask_send, self._options.outgoing_frame_filters, - self._options.encode_text_message_to_utf8) - - self._ping_queue = deque() - - def _receive_frame(self): - """Receives a frame and return data in the frame as a tuple containing - each header field and payload separately. - - Raises: - ConnectionTerminatedException: when read returns empty - string. - InvalidFrameException: when the frame contains invalid data. - """ - - def _receive_bytes(length): - return self.receive_bytes(length) - - return parse_frame(receive_bytes=_receive_bytes, - logger=self._logger, - ws_version=self._request.ws_version, - unmask_receive=self._options.unmask_receive) - - def _receive_frame_as_frame_object(self): - opcode, bytes, fin, rsv1, rsv2, rsv3 = self._receive_frame() - - return Frame(fin=fin, rsv1=rsv1, rsv2=rsv2, rsv3=rsv3, - opcode=opcode, payload=bytes) - - def receive_filtered_frame(self): - """Receives a frame and applies frame filters and message filters. - The frame to be received must satisfy following conditions: - - The frame is not fragmented. - - The opcode of the frame is TEXT or BINARY. - - DO NOT USE this method except for testing purpose. - """ - - frame = self._receive_frame_as_frame_object() - if not frame.fin: - raise InvalidFrameException( - 'Segmented frames must not be received via ' - 'receive_filtered_frame()') - if (frame.opcode != common.OPCODE_TEXT and - frame.opcode != common.OPCODE_BINARY): - raise InvalidFrameException( - 'Control frames must not be received via ' - 'receive_filtered_frame()') - - for frame_filter in self._options.incoming_frame_filters: - frame_filter.filter(frame) - for message_filter in self._options.incoming_message_filters: - frame.payload = message_filter.filter(frame.payload) - return frame - - def send_message(self, message, end=True, binary=False): - """Send message. - - Args: - message: text in unicode or binary in str to send. - binary: send message as binary frame. - - Raises: - BadOperationException: when called on a server-terminated - connection or called with inconsistent message type or - binary parameter. - """ - - if self._request.server_terminated: - raise BadOperationException( - 'Requested send_message after sending out a closing handshake') - - if binary and isinstance(message, unicode): - raise BadOperationException( - 'Message for binary frame must be instance of str') - - for message_filter in self._options.outgoing_message_filters: - message = message_filter.filter(message, end, binary) - - try: - # Set this to any positive integer to limit maximum size of data in - # payload data of each frame. - MAX_PAYLOAD_DATA_SIZE = -1 - - if MAX_PAYLOAD_DATA_SIZE <= 0: - self._write(self._writer.build(message, end, binary)) - return - - bytes_written = 0 - while True: - end_for_this_frame = end - bytes_to_write = len(message) - bytes_written - if (MAX_PAYLOAD_DATA_SIZE > 0 and - bytes_to_write > MAX_PAYLOAD_DATA_SIZE): - end_for_this_frame = False - bytes_to_write = MAX_PAYLOAD_DATA_SIZE - - frame = self._writer.build( - message[bytes_written:bytes_written + bytes_to_write], - end_for_this_frame, - binary) - self._write(frame) - - bytes_written += bytes_to_write - - # This if must be placed here (the end of while block) so that - # at least one frame is sent. - if len(message) <= bytes_written: - break - except ValueError, e: - raise BadOperationException(e) - - def _get_message_from_frame(self, frame): - """Gets a message from frame. If the message is composed of fragmented - frames and the frame is not the last fragmented frame, this method - returns None. The whole message will be returned when the last - fragmented frame is passed to this method. - - Raises: - InvalidFrameException: when the frame doesn't match defragmentation - context, or the frame contains invalid data. - """ - - if frame.opcode == common.OPCODE_CONTINUATION: - if not self._received_fragments: - if frame.fin: - raise InvalidFrameException( - 'Received a termination frame but fragmentation ' - 'not started') - else: - raise InvalidFrameException( - 'Received an intermediate frame but ' - 'fragmentation not started') - - if frame.fin: - # End of fragmentation frame - self._received_fragments.append(frame.payload) - message = ''.join(self._received_fragments) - self._received_fragments = [] - return message - else: - # Intermediate frame - self._received_fragments.append(frame.payload) - return None - else: - if self._received_fragments: - if frame.fin: - raise InvalidFrameException( - 'Received an unfragmented frame without ' - 'terminating existing fragmentation') - else: - raise InvalidFrameException( - 'New fragmentation started without terminating ' - 'existing fragmentation') - - if frame.fin: - # Unfragmented frame - - self._original_opcode = frame.opcode - return frame.payload - else: - # Start of fragmentation frame - - if (not self._options.allow_fragmented_control_frame and - common.is_control_opcode(frame.opcode)): - raise InvalidFrameException( - 'Control frames must not be fragmented') - - self._original_opcode = frame.opcode - self._received_fragments.append(frame.payload) - return None - - def _process_close_message(self, message): - """Processes close message. - - Args: - message: close message. - - Raises: - InvalidFrameException: when the message is invalid. - """ - - self._request.client_terminated = True - - # Status code is optional. We can have status reason only if we - # have status code. Status reason can be empty string. So, - # allowed cases are - # - no application data: no code no reason - # - 2 octet of application data: has code but no reason - # - 3 or more octet of application data: both code and reason - if len(message) == 0: - self._logger.debug('Received close frame (empty body)') - self._request.ws_close_code = ( - common.STATUS_NO_STATUS_RECEIVED) - elif len(message) == 1: - raise InvalidFrameException( - 'If a close frame has status code, the length of ' - 'status code must be 2 octet') - elif len(message) >= 2: - self._request.ws_close_code = struct.unpack( - '!H', message[0:2])[0] - self._request.ws_close_reason = message[2:].decode( - 'utf-8', 'replace') - self._logger.debug( - 'Received close frame (code=%d, reason=%r)', - self._request.ws_close_code, - self._request.ws_close_reason) - - # Drain junk data after the close frame if necessary. - self._drain_received_data() - - if self._request.server_terminated: - self._logger.debug( - 'Received ack for server-initiated closing handshake') - return - - self._logger.debug( - 'Received client-initiated closing handshake') - - code = common.STATUS_NORMAL_CLOSURE - reason = '' - if hasattr(self._request, '_dispatcher'): - dispatcher = self._request._dispatcher - code, reason = dispatcher.passive_closing_handshake( - self._request) - if code is None and reason is not None and len(reason) > 0: - self._logger.warning( - 'Handler specified reason despite code being None') - reason = '' - if reason is None: - reason = '' - self._send_closing_handshake(code, reason) - self._logger.debug( - 'Sent ack for client-initiated closing handshake ' - '(code=%r, reason=%r)', code, reason) - - def _process_ping_message(self, message): - """Processes ping message. - - Args: - message: ping message. - """ - - try: - handler = self._request.on_ping_handler - if handler: - handler(self._request, message) - return - except AttributeError, e: - pass - self._send_pong(message) - - def _process_pong_message(self, message): - """Processes pong message. - - Args: - message: pong message. - """ - - # TODO(tyoshino): Add ping timeout handling. - - inflight_pings = deque() - - while True: - try: - expected_body = self._ping_queue.popleft() - if expected_body == message: - # inflight_pings contains pings ignored by the - # other peer. Just forget them. - self._logger.debug( - 'Ping %r is acked (%d pings were ignored)', - expected_body, len(inflight_pings)) - break - else: - inflight_pings.append(expected_body) - except IndexError, e: - # The received pong was unsolicited pong. Keep the - # ping queue as is. - self._ping_queue = inflight_pings - self._logger.debug('Received a unsolicited pong') - break - - try: - handler = self._request.on_pong_handler - if handler: - handler(self._request, message) - except AttributeError, e: - pass - - def receive_message(self): - """Receive a WebSocket frame and return its payload as a text in - unicode or a binary in str. - - Returns: - payload data of the frame - - as unicode instance if received text frame - - as str instance if received binary frame - or None iff received closing handshake. - Raises: - BadOperationException: when called on a client-terminated - connection. - ConnectionTerminatedException: when read returns empty - string. - InvalidFrameException: when the frame contains invalid - data. - UnsupportedFrameException: when the received frame has - flags, opcode we cannot handle. You can ignore this - exception and continue receiving the next frame. - """ - - if self._request.client_terminated: - raise BadOperationException( - 'Requested receive_message after receiving a closing ' - 'handshake') - - while True: - # mp_conn.read will block if no bytes are available. - # Timeout is controlled by TimeOut directive of Apache. - - frame = self._receive_frame_as_frame_object() - - # Check the constraint on the payload size for control frames - # before extension processes the frame. - # See also http://tools.ietf.org/html/rfc6455#section-5.5 - if (common.is_control_opcode(frame.opcode) and - len(frame.payload) > 125): - raise InvalidFrameException( - 'Payload data size of control frames must be 125 bytes or ' - 'less') - - for frame_filter in self._options.incoming_frame_filters: - frame_filter.filter(frame) - - if frame.rsv1 or frame.rsv2 or frame.rsv3: - raise UnsupportedFrameException( - 'Unsupported flag is set (rsv = %d%d%d)' % - (frame.rsv1, frame.rsv2, frame.rsv3)) - - message = self._get_message_from_frame(frame) - if message is None: - continue - - for message_filter in self._options.incoming_message_filters: - message = message_filter.filter(message) - - if self._original_opcode == common.OPCODE_TEXT: - # The WebSocket protocol section 4.4 specifies that invalid - # characters must be replaced with U+fffd REPLACEMENT - # CHARACTER. - try: - return message.decode('utf-8') - except UnicodeDecodeError, e: - raise InvalidUTF8Exception(e) - elif self._original_opcode == common.OPCODE_BINARY: - return message - elif self._original_opcode == common.OPCODE_CLOSE: - self._process_close_message(message) - return None - elif self._original_opcode == common.OPCODE_PING: - self._process_ping_message(message) - elif self._original_opcode == common.OPCODE_PONG: - self._process_pong_message(message) - else: - raise UnsupportedFrameException( - 'Opcode %d is not supported' % self._original_opcode) - - def _send_closing_handshake(self, code, reason): - body = create_closing_handshake_body(code, reason) - frame = create_close_frame( - body, mask=self._options.mask_send, - frame_filters=self._options.outgoing_frame_filters) - - self._request.server_terminated = True - - self._write(frame) - - def close_connection(self, code=common.STATUS_NORMAL_CLOSURE, reason=''): - """Closes a WebSocket connection. - - Args: - code: Status code for close frame. If code is None, a close - frame with empty body will be sent. - reason: string representing close reason. - Raises: - BadOperationException: when reason is specified with code None - or reason is not an instance of both str and unicode. - """ - - if self._request.server_terminated: - self._logger.debug( - 'Requested close_connection but server is already terminated') - return - - if code is None: - if reason is not None and len(reason) > 0: - raise BadOperationException( - 'close reason must not be specified if code is None') - reason = '' - else: - if not isinstance(reason, str) and not isinstance(reason, unicode): - raise BadOperationException( - 'close reason must be an instance of str or unicode') - - self._send_closing_handshake(code, reason) - self._logger.debug( - 'Sent server-initiated closing handshake (code=%r, reason=%r)', - code, reason) - - if (code == common.STATUS_GOING_AWAY or - code == common.STATUS_PROTOCOL_ERROR): - # It doesn't make sense to wait for a close frame if the reason is - # protocol error or that the server is going away. For some of - # other reasons, it might not make sense to wait for a close frame, - # but it's not clear, yet. - return - - # TODO(ukai): 2. wait until the /client terminated/ flag has been set, - # or until a server-defined timeout expires. - # - # For now, we expect receiving closing handshake right after sending - # out closing handshake. - message = self.receive_message() - if message is not None: - raise ConnectionTerminatedException( - 'Didn\'t receive valid ack for closing handshake') - # TODO: 3. close the WebSocket connection. - # note: mod_python Connection (mp_conn) doesn't have close method. - - def send_ping(self, body=''): - frame = create_ping_frame( - body, - self._options.mask_send, - self._options.outgoing_frame_filters) - self._write(frame) - - self._ping_queue.append(body) - - def _send_pong(self, body): - frame = create_pong_frame( - body, - self._options.mask_send, - self._options.outgoing_frame_filters) - self._write(frame) - - def get_last_received_opcode(self): - """Returns the opcode of the WebSocket message which the last received - frame belongs to. The return value is valid iff immediately after - receive_message call. - """ - - return self._original_opcode - - def _drain_received_data(self): - """Drains unread data in the receive buffer to avoid sending out TCP - RST packet. This is because when deflate-stream is enabled, some - DEFLATE block for flushing data may follow a close frame. If any data - remains in the receive buffer of a socket when the socket is closed, - it sends out TCP RST packet to the other peer. - - Since mod_python's mp_conn object doesn't support non-blocking read, - we perform this only when pywebsocket is running in standalone mode. - """ - - # If self._options.deflate_stream is true, self._request is - # DeflateRequest, so we can get wrapped request object by - # self._request._request. - # - # Only _StandaloneRequest has _drain_received_data method. - if (self._options.deflate_stream and - ('_drain_received_data' in dir(self._request._request))): - self._request._request._drain_received_data() - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/common.py b/module/lib/mod_pywebsocket/common.py deleted file mode 100644 index 2388379c0..000000000 --- a/module/lib/mod_pywebsocket/common.py +++ /dev/null @@ -1,307 +0,0 @@ -# Copyright 2012, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""This file must not depend on any module specific to the WebSocket protocol. -""" - - -from mod_pywebsocket import http_header_util - - -# Additional log level definitions. -LOGLEVEL_FINE = 9 - -# Constants indicating WebSocket protocol version. -VERSION_HIXIE75 = -1 -VERSION_HYBI00 = 0 -VERSION_HYBI01 = 1 -VERSION_HYBI02 = 2 -VERSION_HYBI03 = 2 -VERSION_HYBI04 = 4 -VERSION_HYBI05 = 5 -VERSION_HYBI06 = 6 -VERSION_HYBI07 = 7 -VERSION_HYBI08 = 8 -VERSION_HYBI09 = 8 -VERSION_HYBI10 = 8 -VERSION_HYBI11 = 8 -VERSION_HYBI12 = 8 -VERSION_HYBI13 = 13 -VERSION_HYBI14 = 13 -VERSION_HYBI15 = 13 -VERSION_HYBI16 = 13 -VERSION_HYBI17 = 13 - -# Constants indicating WebSocket protocol latest version. -VERSION_HYBI_LATEST = VERSION_HYBI13 - -# Port numbers -DEFAULT_WEB_SOCKET_PORT = 80 -DEFAULT_WEB_SOCKET_SECURE_PORT = 443 - -# Schemes -WEB_SOCKET_SCHEME = 'ws' -WEB_SOCKET_SECURE_SCHEME = 'wss' - -# Frame opcodes defined in the spec. -OPCODE_CONTINUATION = 0x0 -OPCODE_TEXT = 0x1 -OPCODE_BINARY = 0x2 -OPCODE_CLOSE = 0x8 -OPCODE_PING = 0x9 -OPCODE_PONG = 0xa - -# UUIDs used by HyBi 04 and later opening handshake and frame masking. -WEBSOCKET_ACCEPT_UUID = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11' - -# Opening handshake header names and expected values. -UPGRADE_HEADER = 'Upgrade' -WEBSOCKET_UPGRADE_TYPE = 'websocket' -WEBSOCKET_UPGRADE_TYPE_HIXIE75 = 'WebSocket' -CONNECTION_HEADER = 'Connection' -UPGRADE_CONNECTION_TYPE = 'Upgrade' -HOST_HEADER = 'Host' -ORIGIN_HEADER = 'Origin' -SEC_WEBSOCKET_ORIGIN_HEADER = 'Sec-WebSocket-Origin' -SEC_WEBSOCKET_KEY_HEADER = 'Sec-WebSocket-Key' -SEC_WEBSOCKET_ACCEPT_HEADER = 'Sec-WebSocket-Accept' -SEC_WEBSOCKET_VERSION_HEADER = 'Sec-WebSocket-Version' -SEC_WEBSOCKET_PROTOCOL_HEADER = 'Sec-WebSocket-Protocol' -SEC_WEBSOCKET_EXTENSIONS_HEADER = 'Sec-WebSocket-Extensions' -SEC_WEBSOCKET_DRAFT_HEADER = 'Sec-WebSocket-Draft' -SEC_WEBSOCKET_KEY1_HEADER = 'Sec-WebSocket-Key1' -SEC_WEBSOCKET_KEY2_HEADER = 'Sec-WebSocket-Key2' -SEC_WEBSOCKET_LOCATION_HEADER = 'Sec-WebSocket-Location' - -# Extensions -DEFLATE_STREAM_EXTENSION = 'deflate-stream' -DEFLATE_FRAME_EXTENSION = 'deflate-frame' -PERFRAME_COMPRESSION_EXTENSION = 'perframe-compress' -PERMESSAGE_COMPRESSION_EXTENSION = 'permessage-compress' -X_WEBKIT_DEFLATE_FRAME_EXTENSION = 'x-webkit-deflate-frame' -X_WEBKIT_PERMESSAGE_COMPRESSION_EXTENSION = 'x-webkit-permessage-compress' -MUX_EXTENSION = 'mux_DO_NOT_USE' - -# Status codes -# Code STATUS_NO_STATUS_RECEIVED, STATUS_ABNORMAL_CLOSURE, and -# STATUS_TLS_HANDSHAKE are pseudo codes to indicate specific error cases. -# Could not be used for codes in actual closing frames. -# Application level errors must use codes in the range -# STATUS_USER_REGISTERED_BASE to STATUS_USER_PRIVATE_MAX. The codes in the -# range STATUS_USER_REGISTERED_BASE to STATUS_USER_REGISTERED_MAX are managed -# by IANA. Usually application must define user protocol level errors in the -# range STATUS_USER_PRIVATE_BASE to STATUS_USER_PRIVATE_MAX. -STATUS_NORMAL_CLOSURE = 1000 -STATUS_GOING_AWAY = 1001 -STATUS_PROTOCOL_ERROR = 1002 -STATUS_UNSUPPORTED_DATA = 1003 -STATUS_NO_STATUS_RECEIVED = 1005 -STATUS_ABNORMAL_CLOSURE = 1006 -STATUS_INVALID_FRAME_PAYLOAD_DATA = 1007 -STATUS_POLICY_VIOLATION = 1008 -STATUS_MESSAGE_TOO_BIG = 1009 -STATUS_MANDATORY_EXTENSION = 1010 -STATUS_INTERNAL_ENDPOINT_ERROR = 1011 -STATUS_TLS_HANDSHAKE = 1015 -STATUS_USER_REGISTERED_BASE = 3000 -STATUS_USER_REGISTERED_MAX = 3999 -STATUS_USER_PRIVATE_BASE = 4000 -STATUS_USER_PRIVATE_MAX = 4999 -# Following definitions are aliases to keep compatibility. Applications must -# not use these obsoleted definitions anymore. -STATUS_NORMAL = STATUS_NORMAL_CLOSURE -STATUS_UNSUPPORTED = STATUS_UNSUPPORTED_DATA -STATUS_CODE_NOT_AVAILABLE = STATUS_NO_STATUS_RECEIVED -STATUS_ABNORMAL_CLOSE = STATUS_ABNORMAL_CLOSURE -STATUS_INVALID_FRAME_PAYLOAD = STATUS_INVALID_FRAME_PAYLOAD_DATA -STATUS_MANDATORY_EXT = STATUS_MANDATORY_EXTENSION - -# HTTP status codes -HTTP_STATUS_BAD_REQUEST = 400 -HTTP_STATUS_FORBIDDEN = 403 -HTTP_STATUS_NOT_FOUND = 404 - - -def is_control_opcode(opcode): - return (opcode >> 3) == 1 - - -class ExtensionParameter(object): - """Holds information about an extension which is exchanged on extension - negotiation in opening handshake. - """ - - def __init__(self, name): - self._name = name - # TODO(tyoshino): Change the data structure to more efficient one such - # as dict when the spec changes to say like - # - Parameter names must be unique - # - The order of parameters is not significant - self._parameters = [] - - def name(self): - return self._name - - def add_parameter(self, name, value): - self._parameters.append((name, value)) - - def get_parameters(self): - return self._parameters - - def get_parameter_names(self): - return [name for name, unused_value in self._parameters] - - def has_parameter(self, name): - for param_name, param_value in self._parameters: - if param_name == name: - return True - return False - - def get_parameter_value(self, name): - for param_name, param_value in self._parameters: - if param_name == name: - return param_value - - -class ExtensionParsingException(Exception): - def __init__(self, name): - super(ExtensionParsingException, self).__init__(name) - - -def _parse_extension_param(state, definition, allow_quoted_string): - param_name = http_header_util.consume_token(state) - - if param_name is None: - raise ExtensionParsingException('No valid parameter name found') - - http_header_util.consume_lwses(state) - - if not http_header_util.consume_string(state, '='): - definition.add_parameter(param_name, None) - return - - http_header_util.consume_lwses(state) - - if allow_quoted_string: - # TODO(toyoshim): Add code to validate that parsed param_value is token - param_value = http_header_util.consume_token_or_quoted_string(state) - else: - param_value = http_header_util.consume_token(state) - if param_value is None: - raise ExtensionParsingException( - 'No valid parameter value found on the right-hand side of ' - 'parameter %r' % param_name) - - definition.add_parameter(param_name, param_value) - - -def _parse_extension(state, allow_quoted_string): - extension_token = http_header_util.consume_token(state) - if extension_token is None: - return None - - extension = ExtensionParameter(extension_token) - - while True: - http_header_util.consume_lwses(state) - - if not http_header_util.consume_string(state, ';'): - break - - http_header_util.consume_lwses(state) - - try: - _parse_extension_param(state, extension, allow_quoted_string) - except ExtensionParsingException, e: - raise ExtensionParsingException( - 'Failed to parse parameter for %r (%r)' % - (extension_token, e)) - - return extension - - -def parse_extensions(data, allow_quoted_string=False): - """Parses Sec-WebSocket-Extensions header value returns a list of - ExtensionParameter objects. - - Leading LWSes must be trimmed. - """ - - state = http_header_util.ParsingState(data) - - extension_list = [] - while True: - extension = _parse_extension(state, allow_quoted_string) - if extension is not None: - extension_list.append(extension) - - http_header_util.consume_lwses(state) - - if http_header_util.peek(state) is None: - break - - if not http_header_util.consume_string(state, ','): - raise ExtensionParsingException( - 'Failed to parse Sec-WebSocket-Extensions header: ' - 'Expected a comma but found %r' % - http_header_util.peek(state)) - - http_header_util.consume_lwses(state) - - if len(extension_list) == 0: - raise ExtensionParsingException( - 'No valid extension entry found') - - return extension_list - - -def format_extension(extension): - """Formats an ExtensionParameter object.""" - - formatted_params = [extension.name()] - for param_name, param_value in extension.get_parameters(): - if param_value is None: - formatted_params.append(param_name) - else: - quoted_value = http_header_util.quote_if_necessary(param_value) - formatted_params.append('%s=%s' % (param_name, quoted_value)) - return '; '.join(formatted_params) - - -def format_extensions(extension_list): - """Formats a list of ExtensionParameter objects.""" - - formatted_extension_list = [] - for extension in extension_list: - formatted_extension_list.append(format_extension(extension)) - return ', '.join(formatted_extension_list) - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/dispatch.py b/module/lib/mod_pywebsocket/dispatch.py deleted file mode 100644 index 25905f180..000000000 --- a/module/lib/mod_pywebsocket/dispatch.py +++ /dev/null @@ -1,387 +0,0 @@ -# Copyright 2012, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""Dispatch WebSocket request. -""" - - -import logging -import os -import re - -from mod_pywebsocket import common -from mod_pywebsocket import handshake -from mod_pywebsocket import msgutil -from mod_pywebsocket import mux -from mod_pywebsocket import stream -from mod_pywebsocket import util - - -_SOURCE_PATH_PATTERN = re.compile(r'(?i)_wsh\.py$') -_SOURCE_SUFFIX = '_wsh.py' -_DO_EXTRA_HANDSHAKE_HANDLER_NAME = 'web_socket_do_extra_handshake' -_TRANSFER_DATA_HANDLER_NAME = 'web_socket_transfer_data' -_PASSIVE_CLOSING_HANDSHAKE_HANDLER_NAME = ( - 'web_socket_passive_closing_handshake') - - -class DispatchException(Exception): - """Exception in dispatching WebSocket request.""" - - def __init__(self, name, status=common.HTTP_STATUS_NOT_FOUND): - super(DispatchException, self).__init__(name) - self.status = status - - -def _default_passive_closing_handshake_handler(request): - """Default web_socket_passive_closing_handshake handler.""" - - return common.STATUS_NORMAL_CLOSURE, '' - - -def _normalize_path(path): - """Normalize path. - - Args: - path: the path to normalize. - - Path is converted to the absolute path. - The input path can use either '\\' or '/' as the separator. - The normalized path always uses '/' regardless of the platform. - """ - - path = path.replace('\\', os.path.sep) - path = os.path.realpath(path) - path = path.replace('\\', '/') - return path - - -def _create_path_to_resource_converter(base_dir): - """Returns a function that converts the path of a WebSocket handler source - file to a resource string by removing the path to the base directory from - its head, removing _SOURCE_SUFFIX from its tail, and replacing path - separators in it with '/'. - - Args: - base_dir: the path to the base directory. - """ - - base_dir = _normalize_path(base_dir) - - base_len = len(base_dir) - suffix_len = len(_SOURCE_SUFFIX) - - def converter(path): - if not path.endswith(_SOURCE_SUFFIX): - return None - # _normalize_path must not be used because resolving symlink breaks - # following path check. - path = path.replace('\\', '/') - if not path.startswith(base_dir): - return None - return path[base_len:-suffix_len] - - return converter - - -def _enumerate_handler_file_paths(directory): - """Returns a generator that enumerates WebSocket Handler source file names - in the given directory. - """ - - for root, unused_dirs, files in os.walk(directory): - for base in files: - path = os.path.join(root, base) - if _SOURCE_PATH_PATTERN.search(path): - yield path - - -class _HandlerSuite(object): - """A handler suite holder class.""" - - def __init__(self, do_extra_handshake, transfer_data, - passive_closing_handshake): - self.do_extra_handshake = do_extra_handshake - self.transfer_data = transfer_data - self.passive_closing_handshake = passive_closing_handshake - - -def _source_handler_file(handler_definition): - """Source a handler definition string. - - Args: - handler_definition: a string containing Python statements that define - handler functions. - """ - - global_dic = {} - try: - exec handler_definition in global_dic - except Exception: - raise DispatchException('Error in sourcing handler:' + - util.get_stack_trace()) - passive_closing_handshake_handler = None - try: - passive_closing_handshake_handler = _extract_handler( - global_dic, _PASSIVE_CLOSING_HANDSHAKE_HANDLER_NAME) - except Exception: - passive_closing_handshake_handler = ( - _default_passive_closing_handshake_handler) - return _HandlerSuite( - _extract_handler(global_dic, _DO_EXTRA_HANDSHAKE_HANDLER_NAME), - _extract_handler(global_dic, _TRANSFER_DATA_HANDLER_NAME), - passive_closing_handshake_handler) - - -def _extract_handler(dic, name): - """Extracts a callable with the specified name from the given dictionary - dic. - """ - - if name not in dic: - raise DispatchException('%s is not defined.' % name) - handler = dic[name] - if not callable(handler): - raise DispatchException('%s is not callable.' % name) - return handler - - -class Dispatcher(object): - """Dispatches WebSocket requests. - - This class maintains a map from resource name to handlers. - """ - - def __init__( - self, root_dir, scan_dir=None, - allow_handlers_outside_root_dir=True): - """Construct an instance. - - Args: - root_dir: The directory where handler definition files are - placed. - scan_dir: The directory where handler definition files are - searched. scan_dir must be a directory under root_dir, - including root_dir itself. If scan_dir is None, - root_dir is used as scan_dir. scan_dir can be useful - in saving scan time when root_dir contains many - subdirectories. - allow_handlers_outside_root_dir: Scans handler files even if their - canonical path is not under root_dir. - """ - - self._logger = util.get_class_logger(self) - - self._handler_suite_map = {} - self._source_warnings = [] - if scan_dir is None: - scan_dir = root_dir - if not os.path.realpath(scan_dir).startswith( - os.path.realpath(root_dir)): - raise DispatchException('scan_dir:%s must be a directory under ' - 'root_dir:%s.' % (scan_dir, root_dir)) - self._source_handler_files_in_dir( - root_dir, scan_dir, allow_handlers_outside_root_dir) - - def add_resource_path_alias(self, - alias_resource_path, existing_resource_path): - """Add resource path alias. - - Once added, request to alias_resource_path would be handled by - handler registered for existing_resource_path. - - Args: - alias_resource_path: alias resource path - existing_resource_path: existing resource path - """ - try: - handler_suite = self._handler_suite_map[existing_resource_path] - self._handler_suite_map[alias_resource_path] = handler_suite - except KeyError: - raise DispatchException('No handler for: %r' % - existing_resource_path) - - def source_warnings(self): - """Return warnings in sourcing handlers.""" - - return self._source_warnings - - def do_extra_handshake(self, request): - """Do extra checking in WebSocket handshake. - - Select a handler based on request.uri and call its - web_socket_do_extra_handshake function. - - Args: - request: mod_python request. - - Raises: - DispatchException: when handler was not found - AbortedByUserException: when user handler abort connection - HandshakeException: when opening handshake failed - """ - - handler_suite = self.get_handler_suite(request.ws_resource) - if handler_suite is None: - raise DispatchException('No handler for: %r' % request.ws_resource) - do_extra_handshake_ = handler_suite.do_extra_handshake - try: - do_extra_handshake_(request) - except handshake.AbortedByUserException, e: - raise - except Exception, e: - util.prepend_message_to_exception( - '%s raised exception for %s: ' % ( - _DO_EXTRA_HANDSHAKE_HANDLER_NAME, - request.ws_resource), - e) - raise handshake.HandshakeException(e, common.HTTP_STATUS_FORBIDDEN) - - def transfer_data(self, request): - """Let a handler transfer_data with a WebSocket client. - - Select a handler based on request.ws_resource and call its - web_socket_transfer_data function. - - Args: - request: mod_python request. - - Raises: - DispatchException: when handler was not found - AbortedByUserException: when user handler abort connection - """ - - # TODO(tyoshino): Terminate underlying TCP connection if possible. - try: - if mux.use_mux(request): - mux.start(request, self) - else: - handler_suite = self.get_handler_suite(request.ws_resource) - if handler_suite is None: - raise DispatchException('No handler for: %r' % - request.ws_resource) - transfer_data_ = handler_suite.transfer_data - transfer_data_(request) - - if not request.server_terminated: - request.ws_stream.close_connection() - # Catch non-critical exceptions the handler didn't handle. - except handshake.AbortedByUserException, e: - self._logger.debug('%s', e) - raise - except msgutil.BadOperationException, e: - self._logger.debug('%s', e) - request.ws_stream.close_connection(common.STATUS_ABNORMAL_CLOSURE) - except msgutil.InvalidFrameException, e: - # InvalidFrameException must be caught before - # ConnectionTerminatedException that catches InvalidFrameException. - self._logger.debug('%s', e) - request.ws_stream.close_connection(common.STATUS_PROTOCOL_ERROR) - except msgutil.UnsupportedFrameException, e: - self._logger.debug('%s', e) - request.ws_stream.close_connection(common.STATUS_UNSUPPORTED_DATA) - except stream.InvalidUTF8Exception, e: - self._logger.debug('%s', e) - request.ws_stream.close_connection( - common.STATUS_INVALID_FRAME_PAYLOAD_DATA) - except msgutil.ConnectionTerminatedException, e: - self._logger.debug('%s', e) - except Exception, e: - util.prepend_message_to_exception( - '%s raised exception for %s: ' % ( - _TRANSFER_DATA_HANDLER_NAME, request.ws_resource), - e) - raise - - def passive_closing_handshake(self, request): - """Prepare code and reason for responding client initiated closing - handshake. - """ - - handler_suite = self.get_handler_suite(request.ws_resource) - if handler_suite is None: - return _default_passive_closing_handshake_handler(request) - return handler_suite.passive_closing_handshake(request) - - def get_handler_suite(self, resource): - """Retrieves two handlers (one for extra handshake processing, and one - for data transfer) for the given request as a HandlerSuite object. - """ - - fragment = None - if '#' in resource: - resource, fragment = resource.split('#', 1) - if '?' in resource: - resource = resource.split('?', 1)[0] - handler_suite = self._handler_suite_map.get(resource) - if handler_suite and fragment: - raise DispatchException('Fragment identifiers MUST NOT be used on ' - 'WebSocket URIs', - common.HTTP_STATUS_BAD_REQUEST) - return handler_suite - - def _source_handler_files_in_dir( - self, root_dir, scan_dir, allow_handlers_outside_root_dir): - """Source all the handler source files in the scan_dir directory. - - The resource path is determined relative to root_dir. - """ - - # We build a map from resource to handler code assuming that there's - # only one path from root_dir to scan_dir and it can be obtained by - # comparing realpath of them. - - # Here we cannot use abspath. See - # https://bugs.webkit.org/show_bug.cgi?id=31603 - - convert = _create_path_to_resource_converter(root_dir) - scan_realpath = os.path.realpath(scan_dir) - root_realpath = os.path.realpath(root_dir) - for path in _enumerate_handler_file_paths(scan_realpath): - if (not allow_handlers_outside_root_dir and - (not os.path.realpath(path).startswith(root_realpath))): - self._logger.debug( - 'Canonical path of %s is not under root directory' % - path) - continue - try: - handler_suite = _source_handler_file(open(path).read()) - except DispatchException, e: - self._source_warnings.append('%s: %s' % (path, e)) - continue - resource = convert(path) - if resource is None: - self._logger.debug( - 'Path to resource conversion on %s failed' % path) - else: - self._handler_suite_map[convert(path)] = handler_suite - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/extensions.py b/module/lib/mod_pywebsocket/extensions.py deleted file mode 100644 index 03dbf9ee1..000000000 --- a/module/lib/mod_pywebsocket/extensions.py +++ /dev/null @@ -1,727 +0,0 @@ -# Copyright 2012, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -from mod_pywebsocket import common -from mod_pywebsocket import util -from mod_pywebsocket.http_header_util import quote_if_necessary - - -_available_processors = {} - - -class ExtensionProcessorInterface(object): - - def name(self): - return None - - def get_extension_response(self): - return None - - def setup_stream_options(self, stream_options): - pass - - -class DeflateStreamExtensionProcessor(ExtensionProcessorInterface): - """WebSocket DEFLATE stream extension processor. - - Specification: - Section 9.2.1 in - http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-10 - """ - - def __init__(self, request): - self._logger = util.get_class_logger(self) - - self._request = request - - def name(self): - return common.DEFLATE_STREAM_EXTENSION - - def get_extension_response(self): - if len(self._request.get_parameter_names()) != 0: - return None - - self._logger.debug( - 'Enable %s extension', common.DEFLATE_STREAM_EXTENSION) - - return common.ExtensionParameter(common.DEFLATE_STREAM_EXTENSION) - - def setup_stream_options(self, stream_options): - stream_options.deflate_stream = True - - -_available_processors[common.DEFLATE_STREAM_EXTENSION] = ( - DeflateStreamExtensionProcessor) - - -def _log_compression_ratio(logger, original_bytes, total_original_bytes, - filtered_bytes, total_filtered_bytes): - # Print inf when ratio is not available. - ratio = float('inf') - average_ratio = float('inf') - if original_bytes != 0: - ratio = float(filtered_bytes) / original_bytes - if total_original_bytes != 0: - average_ratio = ( - float(total_filtered_bytes) / total_original_bytes) - logger.debug('Outgoing compress ratio: %f (average: %f)' % - (ratio, average_ratio)) - - -def _log_decompression_ratio(logger, received_bytes, total_received_bytes, - filtered_bytes, total_filtered_bytes): - # Print inf when ratio is not available. - ratio = float('inf') - average_ratio = float('inf') - if received_bytes != 0: - ratio = float(received_bytes) / filtered_bytes - if total_filtered_bytes != 0: - average_ratio = ( - float(total_received_bytes) / total_filtered_bytes) - logger.debug('Incoming compress ratio: %f (average: %f)' % - (ratio, average_ratio)) - - -class DeflateFrameExtensionProcessor(ExtensionProcessorInterface): - """WebSocket Per-frame DEFLATE extension processor. - - Specification: - http://tools.ietf.org/html/draft-tyoshino-hybi-websocket-perframe-deflate - """ - - _WINDOW_BITS_PARAM = 'max_window_bits' - _NO_CONTEXT_TAKEOVER_PARAM = 'no_context_takeover' - - def __init__(self, request): - self._logger = util.get_class_logger(self) - - self._request = request - - self._response_window_bits = None - self._response_no_context_takeover = False - self._bfinal = False - - # Counters for statistics. - - # Total number of outgoing bytes supplied to this filter. - self._total_outgoing_payload_bytes = 0 - # Total number of bytes sent to the network after applying this filter. - self._total_filtered_outgoing_payload_bytes = 0 - - # Total number of bytes received from the network. - self._total_incoming_payload_bytes = 0 - # Total number of incoming bytes obtained after applying this filter. - self._total_filtered_incoming_payload_bytes = 0 - - def name(self): - return common.DEFLATE_FRAME_EXTENSION - - def get_extension_response(self): - # Any unknown parameter will be just ignored. - - window_bits = self._request.get_parameter_value( - self._WINDOW_BITS_PARAM) - no_context_takeover = self._request.has_parameter( - self._NO_CONTEXT_TAKEOVER_PARAM) - if (no_context_takeover and - self._request.get_parameter_value( - self._NO_CONTEXT_TAKEOVER_PARAM) is not None): - return None - - if window_bits is not None: - try: - window_bits = int(window_bits) - except ValueError, e: - return None - if window_bits < 8 or window_bits > 15: - return None - - self._deflater = util._RFC1979Deflater( - window_bits, no_context_takeover) - - self._inflater = util._RFC1979Inflater() - - self._compress_outgoing = True - - response = common.ExtensionParameter(self._request.name()) - - if self._response_window_bits is not None: - response.add_parameter( - self._WINDOW_BITS_PARAM, str(self._response_window_bits)) - if self._response_no_context_takeover: - response.add_parameter( - self._NO_CONTEXT_TAKEOVER_PARAM, None) - - self._logger.debug( - 'Enable %s extension (' - 'request: window_bits=%s; no_context_takeover=%r, ' - 'response: window_wbits=%s; no_context_takeover=%r)' % - (self._request.name(), - window_bits, - no_context_takeover, - self._response_window_bits, - self._response_no_context_takeover)) - - return response - - def setup_stream_options(self, stream_options): - - class _OutgoingFilter(object): - - def __init__(self, parent): - self._parent = parent - - def filter(self, frame): - self._parent._outgoing_filter(frame) - - class _IncomingFilter(object): - - def __init__(self, parent): - self._parent = parent - - def filter(self, frame): - self._parent._incoming_filter(frame) - - stream_options.outgoing_frame_filters.append( - _OutgoingFilter(self)) - stream_options.incoming_frame_filters.insert( - 0, _IncomingFilter(self)) - - def set_response_window_bits(self, value): - self._response_window_bits = value - - def set_response_no_context_takeover(self, value): - self._response_no_context_takeover = value - - def set_bfinal(self, value): - self._bfinal = value - - def enable_outgoing_compression(self): - self._compress_outgoing = True - - def disable_outgoing_compression(self): - self._compress_outgoing = False - - def _outgoing_filter(self, frame): - """Transform outgoing frames. This method is called only by - an _OutgoingFilter instance. - """ - - original_payload_size = len(frame.payload) - self._total_outgoing_payload_bytes += original_payload_size - - if (not self._compress_outgoing or - common.is_control_opcode(frame.opcode)): - self._total_filtered_outgoing_payload_bytes += ( - original_payload_size) - return - - frame.payload = self._deflater.filter( - frame.payload, bfinal=self._bfinal) - frame.rsv1 = 1 - - filtered_payload_size = len(frame.payload) - self._total_filtered_outgoing_payload_bytes += filtered_payload_size - - _log_compression_ratio(self._logger, original_payload_size, - self._total_outgoing_payload_bytes, - filtered_payload_size, - self._total_filtered_outgoing_payload_bytes) - - def _incoming_filter(self, frame): - """Transform incoming frames. This method is called only by - an _IncomingFilter instance. - """ - - received_payload_size = len(frame.payload) - self._total_incoming_payload_bytes += received_payload_size - - if frame.rsv1 != 1 or common.is_control_opcode(frame.opcode): - self._total_filtered_incoming_payload_bytes += ( - received_payload_size) - return - - frame.payload = self._inflater.filter(frame.payload) - frame.rsv1 = 0 - - filtered_payload_size = len(frame.payload) - self._total_filtered_incoming_payload_bytes += filtered_payload_size - - _log_decompression_ratio(self._logger, received_payload_size, - self._total_incoming_payload_bytes, - filtered_payload_size, - self._total_filtered_incoming_payload_bytes) - - -_available_processors[common.DEFLATE_FRAME_EXTENSION] = ( - DeflateFrameExtensionProcessor) - - -# Adding vendor-prefixed deflate-frame extension. -# TODO(bashi): Remove this after WebKit stops using vendor prefix. -_available_processors[common.X_WEBKIT_DEFLATE_FRAME_EXTENSION] = ( - DeflateFrameExtensionProcessor) - - -def _parse_compression_method(data): - """Parses the value of "method" extension parameter.""" - - return common.parse_extensions(data, allow_quoted_string=True) - - -def _create_accepted_method_desc(method_name, method_params): - """Creates accepted-method-desc from given method name and parameters""" - - extension = common.ExtensionParameter(method_name) - for name, value in method_params: - extension.add_parameter(name, value) - return common.format_extension(extension) - - -class CompressionExtensionProcessorBase(ExtensionProcessorInterface): - """Base class for Per-frame and Per-message compression extension.""" - - _METHOD_PARAM = 'method' - - def __init__(self, request): - self._logger = util.get_class_logger(self) - self._request = request - self._compression_method_name = None - self._compression_processor = None - self._compression_processor_hook = None - - def name(self): - return '' - - def _lookup_compression_processor(self, method_desc): - return None - - def _get_compression_processor_response(self): - """Looks up the compression processor based on the self._request and - returns the compression processor's response. - """ - - method_list = self._request.get_parameter_value(self._METHOD_PARAM) - if method_list is None: - return None - methods = _parse_compression_method(method_list) - if methods is None: - return None - comression_processor = None - # The current implementation tries only the first method that matches - # supported algorithm. Following methods aren't tried even if the - # first one is rejected. - # TODO(bashi): Need to clarify this behavior. - for method_desc in methods: - compression_processor = self._lookup_compression_processor( - method_desc) - if compression_processor is not None: - self._compression_method_name = method_desc.name() - break - if compression_processor is None: - return None - - if self._compression_processor_hook: - self._compression_processor_hook(compression_processor) - - processor_response = compression_processor.get_extension_response() - if processor_response is None: - return None - self._compression_processor = compression_processor - return processor_response - - def get_extension_response(self): - processor_response = self._get_compression_processor_response() - if processor_response is None: - return None - - response = common.ExtensionParameter(self._request.name()) - accepted_method_desc = _create_accepted_method_desc( - self._compression_method_name, - processor_response.get_parameters()) - response.add_parameter(self._METHOD_PARAM, accepted_method_desc) - self._logger.debug( - 'Enable %s extension (method: %s)' % - (self._request.name(), self._compression_method_name)) - return response - - def setup_stream_options(self, stream_options): - if self._compression_processor is None: - return - self._compression_processor.setup_stream_options(stream_options) - - def set_compression_processor_hook(self, hook): - self._compression_processor_hook = hook - - def get_compression_processor(self): - return self._compression_processor - - -class PerFrameCompressionExtensionProcessor(CompressionExtensionProcessorBase): - """WebSocket Per-frame compression extension processor. - - Specification: - http://tools.ietf.org/html/draft-ietf-hybi-websocket-perframe-compression - """ - - _DEFLATE_METHOD = 'deflate' - - def __init__(self, request): - CompressionExtensionProcessorBase.__init__(self, request) - - def name(self): - return common.PERFRAME_COMPRESSION_EXTENSION - - def _lookup_compression_processor(self, method_desc): - if method_desc.name() == self._DEFLATE_METHOD: - return DeflateFrameExtensionProcessor(method_desc) - return None - - -_available_processors[common.PERFRAME_COMPRESSION_EXTENSION] = ( - PerFrameCompressionExtensionProcessor) - - -class DeflateMessageProcessor(ExtensionProcessorInterface): - """Per-message deflate processor.""" - - _S2C_MAX_WINDOW_BITS_PARAM = 's2c_max_window_bits' - _S2C_NO_CONTEXT_TAKEOVER_PARAM = 's2c_no_context_takeover' - _C2S_MAX_WINDOW_BITS_PARAM = 'c2s_max_window_bits' - _C2S_NO_CONTEXT_TAKEOVER_PARAM = 'c2s_no_context_takeover' - - def __init__(self, request): - self._request = request - self._logger = util.get_class_logger(self) - - self._c2s_max_window_bits = None - self._c2s_no_context_takeover = False - self._bfinal = False - - self._compress_outgoing_enabled = False - - # True if a message is fragmented and compression is ongoing. - self._compress_ongoing = False - - # Counters for statistics. - - # Total number of outgoing bytes supplied to this filter. - self._total_outgoing_payload_bytes = 0 - # Total number of bytes sent to the network after applying this filter. - self._total_filtered_outgoing_payload_bytes = 0 - - # Total number of bytes received from the network. - self._total_incoming_payload_bytes = 0 - # Total number of incoming bytes obtained after applying this filter. - self._total_filtered_incoming_payload_bytes = 0 - - def name(self): - return 'deflate' - - def get_extension_response(self): - # Any unknown parameter will be just ignored. - - s2c_max_window_bits = self._request.get_parameter_value( - self._S2C_MAX_WINDOW_BITS_PARAM) - if s2c_max_window_bits is not None: - try: - s2c_max_window_bits = int(s2c_max_window_bits) - except ValueError, e: - return None - if s2c_max_window_bits < 8 or s2c_max_window_bits > 15: - return None - - s2c_no_context_takeover = self._request.has_parameter( - self._S2C_NO_CONTEXT_TAKEOVER_PARAM) - if (s2c_no_context_takeover and - self._request.get_parameter_value( - self._S2C_NO_CONTEXT_TAKEOVER_PARAM) is not None): - return None - - self._deflater = util._RFC1979Deflater( - s2c_max_window_bits, s2c_no_context_takeover) - - self._inflater = util._RFC1979Inflater() - - self._compress_outgoing_enabled = True - - response = common.ExtensionParameter(self._request.name()) - - if s2c_max_window_bits is not None: - response.add_parameter( - self._S2C_MAX_WINDOW_BITS_PARAM, str(s2c_max_window_bits)) - - if s2c_no_context_takeover: - response.add_parameter( - self._S2C_NO_CONTEXT_TAKEOVER_PARAM, None) - - if self._c2s_max_window_bits is not None: - response.add_parameter( - self._C2S_MAX_WINDOW_BITS_PARAM, - str(self._c2s_max_window_bits)) - if self._c2s_no_context_takeover: - response.add_parameter( - self._C2S_NO_CONTEXT_TAKEOVER_PARAM, None) - - self._logger.debug( - 'Enable %s extension (' - 'request: s2c_max_window_bits=%s; s2c_no_context_takeover=%r, ' - 'response: c2s_max_window_bits=%s; c2s_no_context_takeover=%r)' % - (self._request.name(), - s2c_max_window_bits, - s2c_no_context_takeover, - self._c2s_max_window_bits, - self._c2s_no_context_takeover)) - - return response - - def setup_stream_options(self, stream_options): - class _OutgoingMessageFilter(object): - - def __init__(self, parent): - self._parent = parent - - def filter(self, message, end=True, binary=False): - return self._parent._process_outgoing_message( - message, end, binary) - - class _IncomingMessageFilter(object): - - def __init__(self, parent): - self._parent = parent - self._decompress_next_message = False - - def decompress_next_message(self): - self._decompress_next_message = True - - def filter(self, message): - message = self._parent._process_incoming_message( - message, self._decompress_next_message) - self._decompress_next_message = False - return message - - self._outgoing_message_filter = _OutgoingMessageFilter(self) - self._incoming_message_filter = _IncomingMessageFilter(self) - stream_options.outgoing_message_filters.append( - self._outgoing_message_filter) - stream_options.incoming_message_filters.append( - self._incoming_message_filter) - - class _OutgoingFrameFilter(object): - - def __init__(self, parent): - self._parent = parent - self._set_compression_bit = False - - def set_compression_bit(self): - self._set_compression_bit = True - - def filter(self, frame): - self._parent._process_outgoing_frame( - frame, self._set_compression_bit) - self._set_compression_bit = False - - class _IncomingFrameFilter(object): - - def __init__(self, parent): - self._parent = parent - - def filter(self, frame): - self._parent._process_incoming_frame(frame) - - self._outgoing_frame_filter = _OutgoingFrameFilter(self) - self._incoming_frame_filter = _IncomingFrameFilter(self) - stream_options.outgoing_frame_filters.append( - self._outgoing_frame_filter) - stream_options.incoming_frame_filters.append( - self._incoming_frame_filter) - - stream_options.encode_text_message_to_utf8 = False - - def set_c2s_max_window_bits(self, value): - self._c2s_max_window_bits = value - - def set_c2s_no_context_takeover(self, value): - self._c2s_no_context_takeover = value - - def set_bfinal(self, value): - self._bfinal = value - - def enable_outgoing_compression(self): - self._compress_outgoing_enabled = True - - def disable_outgoing_compression(self): - self._compress_outgoing_enabled = False - - def _process_incoming_message(self, message, decompress): - if not decompress: - return message - - received_payload_size = len(message) - self._total_incoming_payload_bytes += received_payload_size - - message = self._inflater.filter(message) - - filtered_payload_size = len(message) - self._total_filtered_incoming_payload_bytes += filtered_payload_size - - _log_decompression_ratio(self._logger, received_payload_size, - self._total_incoming_payload_bytes, - filtered_payload_size, - self._total_filtered_incoming_payload_bytes) - - return message - - def _process_outgoing_message(self, message, end, binary): - if not binary: - message = message.encode('utf-8') - - if not self._compress_outgoing_enabled: - return message - - original_payload_size = len(message) - self._total_outgoing_payload_bytes += original_payload_size - - message = self._deflater.filter( - message, flush=end, bfinal=self._bfinal) - - filtered_payload_size = len(message) - self._total_filtered_outgoing_payload_bytes += filtered_payload_size - - _log_compression_ratio(self._logger, original_payload_size, - self._total_outgoing_payload_bytes, - filtered_payload_size, - self._total_filtered_outgoing_payload_bytes) - - if not self._compress_ongoing: - self._outgoing_frame_filter.set_compression_bit() - self._compress_ongoing = not end - return message - - def _process_incoming_frame(self, frame): - if frame.rsv1 == 1 and not common.is_control_opcode(frame.opcode): - self._incoming_message_filter.decompress_next_message() - frame.rsv1 = 0 - - def _process_outgoing_frame(self, frame, compression_bit): - if (not compression_bit or - common.is_control_opcode(frame.opcode)): - return - - frame.rsv1 = 1 - - -class PerMessageCompressionExtensionProcessor( - CompressionExtensionProcessorBase): - """WebSocket Per-message compression extension processor. - - Specification: - http://tools.ietf.org/html/draft-ietf-hybi-permessage-compression - """ - - _DEFLATE_METHOD = 'deflate' - - def __init__(self, request): - CompressionExtensionProcessorBase.__init__(self, request) - - def name(self): - return common.PERMESSAGE_COMPRESSION_EXTENSION - - def _lookup_compression_processor(self, method_desc): - if method_desc.name() == self._DEFLATE_METHOD: - return DeflateMessageProcessor(method_desc) - return None - - -_available_processors[common.PERMESSAGE_COMPRESSION_EXTENSION] = ( - PerMessageCompressionExtensionProcessor) - - -# Adding vendor-prefixed permessage-compress extension. -# TODO(bashi): Remove this after WebKit stops using vendor prefix. -_available_processors[common.X_WEBKIT_PERMESSAGE_COMPRESSION_EXTENSION] = ( - PerMessageCompressionExtensionProcessor) - - -class MuxExtensionProcessor(ExtensionProcessorInterface): - """WebSocket multiplexing extension processor.""" - - _QUOTA_PARAM = 'quota' - - def __init__(self, request): - self._request = request - - def name(self): - return common.MUX_EXTENSION - - def get_extension_response(self, ws_request, - logical_channel_extensions): - # Mux extension cannot be used after extensions that depend on - # frame boundary, extension data field, or any reserved bits - # which are attributed to each frame. - for extension in logical_channel_extensions: - name = extension.name() - if (name == common.PERFRAME_COMPRESSION_EXTENSION or - name == common.DEFLATE_FRAME_EXTENSION or - name == common.X_WEBKIT_DEFLATE_FRAME_EXTENSION): - return None - - quota = self._request.get_parameter_value(self._QUOTA_PARAM) - if quota is None: - ws_request.mux_quota = 0 - else: - try: - quota = int(quota) - except ValueError, e: - return None - if quota < 0 or quota >= 2 ** 32: - return None - ws_request.mux_quota = quota - - ws_request.mux = True - ws_request.mux_extensions = logical_channel_extensions - return common.ExtensionParameter(common.MUX_EXTENSION) - - def setup_stream_options(self, stream_options): - pass - - -_available_processors[common.MUX_EXTENSION] = MuxExtensionProcessor - - -def get_extension_processor(extension_request): - global _available_processors - processor_class = _available_processors.get(extension_request.name()) - if processor_class is None: - return None - return processor_class(extension_request) - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/handshake/__init__.py b/module/lib/mod_pywebsocket/handshake/__init__.py deleted file mode 100644 index 194f6b395..000000000 --- a/module/lib/mod_pywebsocket/handshake/__init__.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""WebSocket opening handshake processor. This class try to apply available -opening handshake processors for each protocol version until a connection is -successfully established. -""" - - -import logging - -from mod_pywebsocket import common -from mod_pywebsocket.handshake import hybi00 -from mod_pywebsocket.handshake import hybi -# Export AbortedByUserException, HandshakeException, and VersionException -# symbol from this module. -from mod_pywebsocket.handshake._base import AbortedByUserException -from mod_pywebsocket.handshake._base import HandshakeException -from mod_pywebsocket.handshake._base import VersionException - - -_LOGGER = logging.getLogger(__name__) - - -def do_handshake(request, dispatcher, allowDraft75=False, strict=False): - """Performs WebSocket handshake. - - Args: - request: mod_python request. - dispatcher: Dispatcher (dispatch.Dispatcher). - allowDraft75: obsolete argument. ignored. - strict: obsolete argument. ignored. - - Handshaker will add attributes such as ws_resource in performing - handshake. - """ - - _LOGGER.debug('Client\'s opening handshake resource: %r', request.uri) - # To print mimetools.Message as escaped one-line string, we converts - # headers_in to dict object. Without conversion, if we use %r, it just - # prints the type and address, and if we use %s, it prints the original - # header string as multiple lines. - # - # Both mimetools.Message and MpTable_Type of mod_python can be - # converted to dict. - # - # mimetools.Message.__str__ returns the original header string. - # dict(mimetools.Message object) returns the map from header names to - # header values. While MpTable_Type doesn't have such __str__ but just - # __repr__ which formats itself as well as dictionary object. - _LOGGER.debug( - 'Client\'s opening handshake headers: %r', dict(request.headers_in)) - - handshakers = [] - handshakers.append( - ('RFC 6455', hybi.Handshaker(request, dispatcher))) - handshakers.append( - ('HyBi 00', hybi00.Handshaker(request, dispatcher))) - - for name, handshaker in handshakers: - _LOGGER.debug('Trying protocol version %s', name) - try: - handshaker.do_handshake() - _LOGGER.info('Established (%s protocol)', name) - return - except HandshakeException, e: - _LOGGER.debug( - 'Failed to complete opening handshake as %s protocol: %r', - name, e) - if e.status: - raise e - except AbortedByUserException, e: - raise - except VersionException, e: - raise - - # TODO(toyoshim): Add a test to cover the case all handshakers fail. - raise HandshakeException( - 'Failed to complete opening handshake for all available protocols', - status=common.HTTP_STATUS_BAD_REQUEST) - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/handshake/_base.py b/module/lib/mod_pywebsocket/handshake/_base.py deleted file mode 100644 index e5c94ca90..000000000 --- a/module/lib/mod_pywebsocket/handshake/_base.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright 2012, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""Common functions and exceptions used by WebSocket opening handshake -processors. -""" - - -from mod_pywebsocket import common -from mod_pywebsocket import http_header_util - - -class AbortedByUserException(Exception): - """Exception for aborting a connection intentionally. - - If this exception is raised in do_extra_handshake handler, the connection - will be abandoned. No other WebSocket or HTTP(S) handler will be invoked. - - If this exception is raised in transfer_data_handler, the connection will - be closed without closing handshake. No other WebSocket or HTTP(S) handler - will be invoked. - """ - - pass - - -class HandshakeException(Exception): - """This exception will be raised when an error occurred while processing - WebSocket initial handshake. - """ - - def __init__(self, name, status=None): - super(HandshakeException, self).__init__(name) - self.status = status - - -class VersionException(Exception): - """This exception will be raised when a version of client request does not - match with version the server supports. - """ - - def __init__(self, name, supported_versions=''): - """Construct an instance. - - Args: - supported_version: a str object to show supported hybi versions. - (e.g. '8, 13') - """ - super(VersionException, self).__init__(name) - self.supported_versions = supported_versions - - -def get_default_port(is_secure): - if is_secure: - return common.DEFAULT_WEB_SOCKET_SECURE_PORT - else: - return common.DEFAULT_WEB_SOCKET_PORT - - -def validate_subprotocol(subprotocol, hixie): - """Validate a value in the Sec-WebSocket-Protocol field. - - See - - RFC 6455: Section 4.1., 4.2.2., and 4.3. - - HyBi 00: Section 4.1. Opening handshake - - Args: - hixie: if True, checks if characters in subprotocol are in range - between U+0020 and U+007E. It's required by HyBi 00 but not by - RFC 6455. - """ - - if not subprotocol: - raise HandshakeException('Invalid subprotocol name: empty') - if hixie: - # Parameter should be in the range U+0020 to U+007E. - for c in subprotocol: - if not 0x20 <= ord(c) <= 0x7e: - raise HandshakeException( - 'Illegal character in subprotocol name: %r' % c) - else: - # Parameter should be encoded HTTP token. - state = http_header_util.ParsingState(subprotocol) - token = http_header_util.consume_token(state) - rest = http_header_util.peek(state) - # If |rest| is not None, |subprotocol| is not one token or invalid. If - # |rest| is None, |token| must not be None because |subprotocol| is - # concatenation of |token| and |rest| and is not None. - if rest is not None: - raise HandshakeException('Invalid non-token string in subprotocol ' - 'name: %r' % rest) - - -def parse_host_header(request): - fields = request.headers_in['Host'].split(':', 1) - if len(fields) == 1: - return fields[0], get_default_port(request.is_https()) - try: - return fields[0], int(fields[1]) - except ValueError, e: - raise HandshakeException('Invalid port number format: %r' % e) - - -def format_header(name, value): - return '%s: %s\r\n' % (name, value) - - -def build_location(request): - """Build WebSocket location for request.""" - location_parts = [] - if request.is_https(): - location_parts.append(common.WEB_SOCKET_SECURE_SCHEME) - else: - location_parts.append(common.WEB_SOCKET_SCHEME) - location_parts.append('://') - host, port = parse_host_header(request) - connection_port = request.connection.local_addr[1] - if port != connection_port: - raise HandshakeException('Header/connection port mismatch: %d/%d' % - (port, connection_port)) - location_parts.append(host) - if (port != get_default_port(request.is_https())): - location_parts.append(':') - location_parts.append(str(port)) - location_parts.append(request.uri) - return ''.join(location_parts) - - -def get_mandatory_header(request, key): - value = request.headers_in.get(key) - if value is None: - raise HandshakeException('Header %s is not defined' % key) - return value - - -def validate_mandatory_header(request, key, expected_value, fail_status=None): - value = get_mandatory_header(request, key) - - if value.lower() != expected_value.lower(): - raise HandshakeException( - 'Expected %r for header %s but found %r (case-insensitive)' % - (expected_value, key, value), status=fail_status) - - -def check_request_line(request): - # 5.1 1. The three character UTF-8 string "GET". - # 5.1 2. A UTF-8-encoded U+0020 SPACE character (0x20 byte). - if request.method != 'GET': - raise HandshakeException('Method is not GET: %r' % request.method) - - if request.protocol != 'HTTP/1.1': - raise HandshakeException('Version is not HTTP/1.1: %r' % - request.protocol) - - -def check_header_lines(request, mandatory_headers): - check_request_line(request) - - # The expected field names, and the meaning of their corresponding - # values, are as follows. - # |Upgrade| and |Connection| - for key, expected_value in mandatory_headers: - validate_mandatory_header(request, key, expected_value) - - -def parse_token_list(data): - """Parses a header value which follows 1#token and returns parsed elements - as a list of strings. - - Leading LWSes must be trimmed. - """ - - state = http_header_util.ParsingState(data) - - token_list = [] - - while True: - token = http_header_util.consume_token(state) - if token is not None: - token_list.append(token) - - http_header_util.consume_lwses(state) - - if http_header_util.peek(state) is None: - break - - if not http_header_util.consume_string(state, ','): - raise HandshakeException( - 'Expected a comma but found %r' % http_header_util.peek(state)) - - http_header_util.consume_lwses(state) - - if len(token_list) == 0: - raise HandshakeException('No valid token found') - - return token_list - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/handshake/hybi.py b/module/lib/mod_pywebsocket/handshake/hybi.py deleted file mode 100644 index fc0e2a096..000000000 --- a/module/lib/mod_pywebsocket/handshake/hybi.py +++ /dev/null @@ -1,404 +0,0 @@ -# Copyright 2012, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""This file provides the opening handshake processor for the WebSocket -protocol (RFC 6455). - -Specification: -http://tools.ietf.org/html/rfc6455 -""" - - -# Note: request.connection.write is used in this module, even though mod_python -# document says that it should be used only in connection handlers. -# Unfortunately, we have no other options. For example, request.write is not -# suitable because it doesn't allow direct raw bytes writing. - - -import base64 -import logging -import os -import re - -from mod_pywebsocket import common -from mod_pywebsocket.extensions import get_extension_processor -from mod_pywebsocket.handshake._base import check_request_line -from mod_pywebsocket.handshake._base import format_header -from mod_pywebsocket.handshake._base import get_mandatory_header -from mod_pywebsocket.handshake._base import HandshakeException -from mod_pywebsocket.handshake._base import parse_token_list -from mod_pywebsocket.handshake._base import validate_mandatory_header -from mod_pywebsocket.handshake._base import validate_subprotocol -from mod_pywebsocket.handshake._base import VersionException -from mod_pywebsocket.stream import Stream -from mod_pywebsocket.stream import StreamOptions -from mod_pywebsocket import util - - -# Used to validate the value in the Sec-WebSocket-Key header strictly. RFC 4648 -# disallows non-zero padding, so the character right before == must be any of -# A, Q, g and w. -_SEC_WEBSOCKET_KEY_REGEX = re.compile('^[+/0-9A-Za-z]{21}[AQgw]==$') - -# Defining aliases for values used frequently. -_VERSION_HYBI08 = common.VERSION_HYBI08 -_VERSION_HYBI08_STRING = str(_VERSION_HYBI08) -_VERSION_LATEST = common.VERSION_HYBI_LATEST -_VERSION_LATEST_STRING = str(_VERSION_LATEST) -_SUPPORTED_VERSIONS = [ - _VERSION_LATEST, - _VERSION_HYBI08, -] - - -def compute_accept(key): - """Computes value for the Sec-WebSocket-Accept header from value of the - Sec-WebSocket-Key header. - """ - - accept_binary = util.sha1_hash( - key + common.WEBSOCKET_ACCEPT_UUID).digest() - accept = base64.b64encode(accept_binary) - - return (accept, accept_binary) - - -class Handshaker(object): - """Opening handshake processor for the WebSocket protocol (RFC 6455).""" - - def __init__(self, request, dispatcher): - """Construct an instance. - - Args: - request: mod_python request. - dispatcher: Dispatcher (dispatch.Dispatcher). - - Handshaker will add attributes such as ws_resource during handshake. - """ - - self._logger = util.get_class_logger(self) - - self._request = request - self._dispatcher = dispatcher - - def _validate_connection_header(self): - connection = get_mandatory_header( - self._request, common.CONNECTION_HEADER) - - try: - connection_tokens = parse_token_list(connection) - except HandshakeException, e: - raise HandshakeException( - 'Failed to parse %s: %s' % (common.CONNECTION_HEADER, e)) - - connection_is_valid = False - for token in connection_tokens: - if token.lower() == common.UPGRADE_CONNECTION_TYPE.lower(): - connection_is_valid = True - break - if not connection_is_valid: - raise HandshakeException( - '%s header doesn\'t contain "%s"' % - (common.CONNECTION_HEADER, common.UPGRADE_CONNECTION_TYPE)) - - def do_handshake(self): - self._request.ws_close_code = None - self._request.ws_close_reason = None - - # Parsing. - - check_request_line(self._request) - - validate_mandatory_header( - self._request, - common.UPGRADE_HEADER, - common.WEBSOCKET_UPGRADE_TYPE) - - self._validate_connection_header() - - self._request.ws_resource = self._request.uri - - unused_host = get_mandatory_header(self._request, common.HOST_HEADER) - - self._request.ws_version = self._check_version() - - # This handshake must be based on latest hybi. We are responsible to - # fallback to HTTP on handshake failure as latest hybi handshake - # specifies. - try: - self._get_origin() - self._set_protocol() - self._parse_extensions() - - # Key validation, response generation. - - key = self._get_key() - (accept, accept_binary) = compute_accept(key) - self._logger.debug( - '%s: %r (%s)', - common.SEC_WEBSOCKET_ACCEPT_HEADER, - accept, - util.hexify(accept_binary)) - - self._logger.debug('Protocol version is RFC 6455') - - # Setup extension processors. - - processors = [] - if self._request.ws_requested_extensions is not None: - for extension_request in self._request.ws_requested_extensions: - processor = get_extension_processor(extension_request) - # Unknown extension requests are just ignored. - if processor is not None: - processors.append(processor) - self._request.ws_extension_processors = processors - - # Extra handshake handler may modify/remove processors. - self._dispatcher.do_extra_handshake(self._request) - processors = filter(lambda processor: processor is not None, - self._request.ws_extension_processors) - - accepted_extensions = [] - - # We need to take care of mux extension here. Extensions that - # are placed before mux should be applied to logical channels. - mux_index = -1 - for i, processor in enumerate(processors): - if processor.name() == common.MUX_EXTENSION: - mux_index = i - break - if mux_index >= 0: - mux_processor = processors[mux_index] - logical_channel_processors = processors[:mux_index] - processors = processors[mux_index+1:] - - for processor in logical_channel_processors: - extension_response = processor.get_extension_response() - if extension_response is None: - # Rejected. - continue - accepted_extensions.append(extension_response) - # Pass a shallow copy of accepted_extensions as extensions for - # logical channels. - mux_response = mux_processor.get_extension_response( - self._request, accepted_extensions[:]) - if mux_response is not None: - accepted_extensions.append(mux_response) - - stream_options = StreamOptions() - - # When there is mux extension, here, |processors| contain only - # prosessors for extensions placed after mux. - for processor in processors: - - extension_response = processor.get_extension_response() - if extension_response is None: - # Rejected. - continue - - accepted_extensions.append(extension_response) - - processor.setup_stream_options(stream_options) - - if len(accepted_extensions) > 0: - self._request.ws_extensions = accepted_extensions - self._logger.debug( - 'Extensions accepted: %r', - map(common.ExtensionParameter.name, accepted_extensions)) - else: - self._request.ws_extensions = None - - self._request.ws_stream = self._create_stream(stream_options) - - if self._request.ws_requested_protocols is not None: - if self._request.ws_protocol is None: - raise HandshakeException( - 'do_extra_handshake must choose one subprotocol from ' - 'ws_requested_protocols and set it to ws_protocol') - validate_subprotocol(self._request.ws_protocol, hixie=False) - - self._logger.debug( - 'Subprotocol accepted: %r', - self._request.ws_protocol) - else: - if self._request.ws_protocol is not None: - raise HandshakeException( - 'ws_protocol must be None when the client didn\'t ' - 'request any subprotocol') - - self._send_handshake(accept) - except HandshakeException, e: - if not e.status: - # Fallback to 400 bad request by default. - e.status = common.HTTP_STATUS_BAD_REQUEST - raise e - - def _get_origin(self): - if self._request.ws_version is _VERSION_HYBI08: - origin_header = common.SEC_WEBSOCKET_ORIGIN_HEADER - else: - origin_header = common.ORIGIN_HEADER - origin = self._request.headers_in.get(origin_header) - if origin is None: - self._logger.debug('Client request does not have origin header') - self._request.ws_origin = origin - - def _check_version(self): - version = get_mandatory_header(self._request, - common.SEC_WEBSOCKET_VERSION_HEADER) - if version == _VERSION_HYBI08_STRING: - return _VERSION_HYBI08 - if version == _VERSION_LATEST_STRING: - return _VERSION_LATEST - - if version.find(',') >= 0: - raise HandshakeException( - 'Multiple versions (%r) are not allowed for header %s' % - (version, common.SEC_WEBSOCKET_VERSION_HEADER), - status=common.HTTP_STATUS_BAD_REQUEST) - raise VersionException( - 'Unsupported version %r for header %s' % - (version, common.SEC_WEBSOCKET_VERSION_HEADER), - supported_versions=', '.join(map(str, _SUPPORTED_VERSIONS))) - - def _set_protocol(self): - self._request.ws_protocol = None - - protocol_header = self._request.headers_in.get( - common.SEC_WEBSOCKET_PROTOCOL_HEADER) - - if protocol_header is None: - self._request.ws_requested_protocols = None - return - - self._request.ws_requested_protocols = parse_token_list( - protocol_header) - self._logger.debug('Subprotocols requested: %r', - self._request.ws_requested_protocols) - - def _parse_extensions(self): - extensions_header = self._request.headers_in.get( - common.SEC_WEBSOCKET_EXTENSIONS_HEADER) - if not extensions_header: - self._request.ws_requested_extensions = None - return - - if self._request.ws_version is common.VERSION_HYBI08: - allow_quoted_string=False - else: - allow_quoted_string=True - try: - self._request.ws_requested_extensions = common.parse_extensions( - extensions_header, allow_quoted_string=allow_quoted_string) - except common.ExtensionParsingException, e: - raise HandshakeException( - 'Failed to parse Sec-WebSocket-Extensions header: %r' % e) - - self._logger.debug( - 'Extensions requested: %r', - map(common.ExtensionParameter.name, - self._request.ws_requested_extensions)) - - def _validate_key(self, key): - if key.find(',') >= 0: - raise HandshakeException('Request has multiple %s header lines or ' - 'contains illegal character \',\': %r' % - (common.SEC_WEBSOCKET_KEY_HEADER, key)) - - # Validate - key_is_valid = False - try: - # Validate key by quick regex match before parsing by base64 - # module. Because base64 module skips invalid characters, we have - # to do this in advance to make this server strictly reject illegal - # keys. - if _SEC_WEBSOCKET_KEY_REGEX.match(key): - decoded_key = base64.b64decode(key) - if len(decoded_key) == 16: - key_is_valid = True - except TypeError, e: - pass - - if not key_is_valid: - raise HandshakeException( - 'Illegal value for header %s: %r' % - (common.SEC_WEBSOCKET_KEY_HEADER, key)) - - return decoded_key - - def _get_key(self): - key = get_mandatory_header( - self._request, common.SEC_WEBSOCKET_KEY_HEADER) - - decoded_key = self._validate_key(key) - - self._logger.debug( - '%s: %r (%s)', - common.SEC_WEBSOCKET_KEY_HEADER, - key, - util.hexify(decoded_key)) - - return key - - def _create_stream(self, stream_options): - return Stream(self._request, stream_options) - - def _create_handshake_response(self, accept): - response = [] - - response.append('HTTP/1.1 101 Switching Protocols\r\n') - - response.append(format_header( - common.UPGRADE_HEADER, common.WEBSOCKET_UPGRADE_TYPE)) - response.append(format_header( - common.CONNECTION_HEADER, common.UPGRADE_CONNECTION_TYPE)) - response.append(format_header( - common.SEC_WEBSOCKET_ACCEPT_HEADER, accept)) - if self._request.ws_protocol is not None: - response.append(format_header( - common.SEC_WEBSOCKET_PROTOCOL_HEADER, - self._request.ws_protocol)) - if (self._request.ws_extensions is not None and - len(self._request.ws_extensions) != 0): - response.append(format_header( - common.SEC_WEBSOCKET_EXTENSIONS_HEADER, - common.format_extensions(self._request.ws_extensions))) - response.append('\r\n') - - return ''.join(response) - - def _send_handshake(self, accept): - raw_response = self._create_handshake_response(accept) - self._request.connection.write(raw_response) - self._logger.debug('Sent server\'s opening handshake: %r', - raw_response) - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/handshake/hybi00.py b/module/lib/mod_pywebsocket/handshake/hybi00.py deleted file mode 100644 index cc6f8dc43..000000000 --- a/module/lib/mod_pywebsocket/handshake/hybi00.py +++ /dev/null @@ -1,242 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""This file provides the opening handshake processor for the WebSocket -protocol version HyBi 00. - -Specification: -http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-00 -""" - - -# Note: request.connection.write/read are used in this module, even though -# mod_python document says that they should be used only in connection -# handlers. Unfortunately, we have no other options. For example, -# request.write/read are not suitable because they don't allow direct raw bytes -# writing/reading. - - -import logging -import re -import struct - -from mod_pywebsocket import common -from mod_pywebsocket.stream import StreamHixie75 -from mod_pywebsocket import util -from mod_pywebsocket.handshake._base import HandshakeException -from mod_pywebsocket.handshake._base import build_location -from mod_pywebsocket.handshake._base import check_header_lines -from mod_pywebsocket.handshake._base import format_header -from mod_pywebsocket.handshake._base import get_mandatory_header -from mod_pywebsocket.handshake._base import validate_subprotocol - - -_MANDATORY_HEADERS = [ - # key, expected value or None - [common.UPGRADE_HEADER, common.WEBSOCKET_UPGRADE_TYPE_HIXIE75], - [common.CONNECTION_HEADER, common.UPGRADE_CONNECTION_TYPE], -] - - -class Handshaker(object): - """Opening handshake processor for the WebSocket protocol version HyBi 00. - """ - - def __init__(self, request, dispatcher): - """Construct an instance. - - Args: - request: mod_python request. - dispatcher: Dispatcher (dispatch.Dispatcher). - - Handshaker will add attributes such as ws_resource in performing - handshake. - """ - - self._logger = util.get_class_logger(self) - - self._request = request - self._dispatcher = dispatcher - - def do_handshake(self): - """Perform WebSocket Handshake. - - On _request, we set - ws_resource, ws_protocol, ws_location, ws_origin, ws_challenge, - ws_challenge_md5: WebSocket handshake information. - ws_stream: Frame generation/parsing class. - ws_version: Protocol version. - - Raises: - HandshakeException: when any error happened in parsing the opening - handshake request. - """ - - # 5.1 Reading the client's opening handshake. - # dispatcher sets it in self._request. - check_header_lines(self._request, _MANDATORY_HEADERS) - self._set_resource() - self._set_subprotocol() - self._set_location() - self._set_origin() - self._set_challenge_response() - self._set_protocol_version() - - self._dispatcher.do_extra_handshake(self._request) - - self._send_handshake() - - def _set_resource(self): - self._request.ws_resource = self._request.uri - - def _set_subprotocol(self): - # |Sec-WebSocket-Protocol| - subprotocol = self._request.headers_in.get( - common.SEC_WEBSOCKET_PROTOCOL_HEADER) - if subprotocol is not None: - validate_subprotocol(subprotocol, hixie=True) - self._request.ws_protocol = subprotocol - - def _set_location(self): - # |Host| - host = self._request.headers_in.get(common.HOST_HEADER) - if host is not None: - self._request.ws_location = build_location(self._request) - # TODO(ukai): check host is this host. - - def _set_origin(self): - # |Origin| - origin = self._request.headers_in.get(common.ORIGIN_HEADER) - if origin is not None: - self._request.ws_origin = origin - - def _set_protocol_version(self): - # |Sec-WebSocket-Draft| - draft = self._request.headers_in.get(common.SEC_WEBSOCKET_DRAFT_HEADER) - if draft is not None and draft != '0': - raise HandshakeException('Illegal value for %s: %s' % - (common.SEC_WEBSOCKET_DRAFT_HEADER, - draft)) - - self._logger.debug('Protocol version is HyBi 00') - self._request.ws_version = common.VERSION_HYBI00 - self._request.ws_stream = StreamHixie75(self._request, True) - - def _set_challenge_response(self): - # 5.2 4-8. - self._request.ws_challenge = self._get_challenge() - # 5.2 9. let /response/ be the MD5 finterprint of /challenge/ - self._request.ws_challenge_md5 = util.md5_hash( - self._request.ws_challenge).digest() - self._logger.debug( - 'Challenge: %r (%s)', - self._request.ws_challenge, - util.hexify(self._request.ws_challenge)) - self._logger.debug( - 'Challenge response: %r (%s)', - self._request.ws_challenge_md5, - util.hexify(self._request.ws_challenge_md5)) - - def _get_key_value(self, key_field): - key_value = get_mandatory_header(self._request, key_field) - - self._logger.debug('%s: %r', key_field, key_value) - - # 5.2 4. let /key-number_n/ be the digits (characters in the range - # U+0030 DIGIT ZERO (0) to U+0039 DIGIT NINE (9)) in /key_n/, - # interpreted as a base ten integer, ignoring all other characters - # in /key_n/. - try: - key_number = int(re.sub("\\D", "", key_value)) - except: - raise HandshakeException('%s field contains no digit' % key_field) - # 5.2 5. let /spaces_n/ be the number of U+0020 SPACE characters - # in /key_n/. - spaces = re.subn(" ", "", key_value)[1] - if spaces == 0: - raise HandshakeException('%s field contains no space' % key_field) - - self._logger.debug( - '%s: Key-number is %d and number of spaces is %d', - key_field, key_number, spaces) - - # 5.2 6. if /key-number_n/ is not an integral multiple of /spaces_n/ - # then abort the WebSocket connection. - if key_number % spaces != 0: - raise HandshakeException( - '%s: Key-number (%d) is not an integral multiple of spaces ' - '(%d)' % (key_field, key_number, spaces)) - # 5.2 7. let /part_n/ be /key-number_n/ divided by /spaces_n/. - part = key_number / spaces - self._logger.debug('%s: Part is %d', key_field, part) - return part - - def _get_challenge(self): - # 5.2 4-7. - key1 = self._get_key_value(common.SEC_WEBSOCKET_KEY1_HEADER) - key2 = self._get_key_value(common.SEC_WEBSOCKET_KEY2_HEADER) - # 5.2 8. let /challenge/ be the concatenation of /part_1/, - challenge = '' - challenge += struct.pack('!I', key1) # network byteorder int - challenge += struct.pack('!I', key2) # network byteorder int - challenge += self._request.connection.read(8) - return challenge - - def _send_handshake(self): - response = [] - - # 5.2 10. send the following line. - response.append('HTTP/1.1 101 WebSocket Protocol Handshake\r\n') - - # 5.2 11. send the following fields to the client. - response.append(format_header( - common.UPGRADE_HEADER, common.WEBSOCKET_UPGRADE_TYPE_HIXIE75)) - response.append(format_header( - common.CONNECTION_HEADER, common.UPGRADE_CONNECTION_TYPE)) - response.append(format_header( - common.SEC_WEBSOCKET_LOCATION_HEADER, self._request.ws_location)) - response.append(format_header( - common.SEC_WEBSOCKET_ORIGIN_HEADER, self._request.ws_origin)) - if self._request.ws_protocol: - response.append(format_header( - common.SEC_WEBSOCKET_PROTOCOL_HEADER, - self._request.ws_protocol)) - # 5.2 12. send two bytes 0x0D 0x0A. - response.append('\r\n') - # 5.2 13. send /response/ - response.append(self._request.ws_challenge_md5) - - raw_response = ''.join(response) - self._request.connection.write(raw_response) - self._logger.debug('Sent server\'s opening handshake: %r', - raw_response) - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/headerparserhandler.py b/module/lib/mod_pywebsocket/headerparserhandler.py deleted file mode 100644 index 2cc62de04..000000000 --- a/module/lib/mod_pywebsocket/headerparserhandler.py +++ /dev/null @@ -1,244 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""PythonHeaderParserHandler for mod_pywebsocket. - -Apache HTTP Server and mod_python must be configured such that this -function is called to handle WebSocket request. -""" - - -import logging - -from mod_python import apache - -from mod_pywebsocket import common -from mod_pywebsocket import dispatch -from mod_pywebsocket import handshake -from mod_pywebsocket import util - - -# PythonOption to specify the handler root directory. -_PYOPT_HANDLER_ROOT = 'mod_pywebsocket.handler_root' - -# PythonOption to specify the handler scan directory. -# This must be a directory under the root directory. -# The default is the root directory. -_PYOPT_HANDLER_SCAN = 'mod_pywebsocket.handler_scan' - -# PythonOption to allow handlers whose canonical path is -# not under the root directory. It's disallowed by default. -# Set this option with value of 'yes' to allow. -_PYOPT_ALLOW_HANDLERS_OUTSIDE_ROOT = ( - 'mod_pywebsocket.allow_handlers_outside_root_dir') -# Map from values to their meanings. 'Yes' and 'No' are allowed just for -# compatibility. -_PYOPT_ALLOW_HANDLERS_OUTSIDE_ROOT_DEFINITION = { - 'off': False, 'no': False, 'on': True, 'yes': True} - -# (Obsolete option. Ignored.) -# PythonOption to specify to allow handshake defined in Hixie 75 version -# protocol. The default is None (Off) -_PYOPT_ALLOW_DRAFT75 = 'mod_pywebsocket.allow_draft75' -# Map from values to their meanings. -_PYOPT_ALLOW_DRAFT75_DEFINITION = {'off': False, 'on': True} - - -class ApacheLogHandler(logging.Handler): - """Wrapper logging.Handler to emit log message to apache's error.log.""" - - _LEVELS = { - logging.DEBUG: apache.APLOG_DEBUG, - logging.INFO: apache.APLOG_INFO, - logging.WARNING: apache.APLOG_WARNING, - logging.ERROR: apache.APLOG_ERR, - logging.CRITICAL: apache.APLOG_CRIT, - } - - def __init__(self, request=None): - logging.Handler.__init__(self) - self._log_error = apache.log_error - if request is not None: - self._log_error = request.log_error - - # Time and level will be printed by Apache. - self._formatter = logging.Formatter('%(name)s: %(message)s') - - def emit(self, record): - apache_level = apache.APLOG_DEBUG - if record.levelno in ApacheLogHandler._LEVELS: - apache_level = ApacheLogHandler._LEVELS[record.levelno] - - msg = self._formatter.format(record) - - # "server" parameter must be passed to have "level" parameter work. - # If only "level" parameter is passed, nothing shows up on Apache's - # log. However, at this point, we cannot get the server object of the - # virtual host which will process WebSocket requests. The only server - # object we can get here is apache.main_server. But Wherever (server - # configuration context or virtual host context) we put - # PythonHeaderParserHandler directive, apache.main_server just points - # the main server instance (not any of virtual server instance). Then, - # Apache follows LogLevel directive in the server configuration context - # to filter logs. So, we need to specify LogLevel in the server - # configuration context. Even if we specify "LogLevel debug" in the - # virtual host context which actually handles WebSocket connections, - # DEBUG level logs never show up unless "LogLevel debug" is specified - # in the server configuration context. - # - # TODO(tyoshino): Provide logging methods on request object. When - # request is mp_request object (when used together with Apache), the - # methods call request.log_error indirectly. When request is - # _StandaloneRequest, the methods call Python's logging facility which - # we create in standalone.py. - self._log_error(msg, apache_level, apache.main_server) - - -def _configure_logging(): - logger = logging.getLogger() - # Logs are filtered by Apache based on LogLevel directive in Apache - # configuration file. We must just pass logs for all levels to - # ApacheLogHandler. - logger.setLevel(logging.DEBUG) - logger.addHandler(ApacheLogHandler()) - - -_configure_logging() - -_LOGGER = logging.getLogger(__name__) - - -def _parse_option(name, value, definition): - if value is None: - return False - - meaning = definition.get(value.lower()) - if meaning is None: - raise Exception('Invalid value for PythonOption %s: %r' % - (name, value)) - return meaning - - -def _create_dispatcher(): - _LOGGER.info('Initializing Dispatcher') - - options = apache.main_server.get_options() - - handler_root = options.get(_PYOPT_HANDLER_ROOT, None) - if not handler_root: - raise Exception('PythonOption %s is not defined' % _PYOPT_HANDLER_ROOT, - apache.APLOG_ERR) - - handler_scan = options.get(_PYOPT_HANDLER_SCAN, handler_root) - - allow_handlers_outside_root = _parse_option( - _PYOPT_ALLOW_HANDLERS_OUTSIDE_ROOT, - options.get(_PYOPT_ALLOW_HANDLERS_OUTSIDE_ROOT), - _PYOPT_ALLOW_HANDLERS_OUTSIDE_ROOT_DEFINITION) - - dispatcher = dispatch.Dispatcher( - handler_root, handler_scan, allow_handlers_outside_root) - - for warning in dispatcher.source_warnings(): - apache.log_error('mod_pywebsocket: %s' % warning, apache.APLOG_WARNING) - - return dispatcher - - -# Initialize -_dispatcher = _create_dispatcher() - - -def headerparserhandler(request): - """Handle request. - - Args: - request: mod_python request. - - This function is named headerparserhandler because it is the default - name for a PythonHeaderParserHandler. - """ - - handshake_is_done = False - try: - # Fallback to default http handler for request paths for which - # we don't have request handlers. - if not _dispatcher.get_handler_suite(request.uri): - request.log_error('No handler for resource: %r' % request.uri, - apache.APLOG_INFO) - request.log_error('Fallback to Apache', apache.APLOG_INFO) - return apache.DECLINED - except dispatch.DispatchException, e: - request.log_error('mod_pywebsocket: %s' % e, apache.APLOG_INFO) - if not handshake_is_done: - return e.status - - try: - allow_draft75 = _parse_option( - _PYOPT_ALLOW_DRAFT75, - apache.main_server.get_options().get(_PYOPT_ALLOW_DRAFT75), - _PYOPT_ALLOW_DRAFT75_DEFINITION) - - try: - handshake.do_handshake( - request, _dispatcher, allowDraft75=allow_draft75) - except handshake.VersionException, e: - request.log_error('mod_pywebsocket: %s' % e, apache.APLOG_INFO) - request.err_headers_out.add(common.SEC_WEBSOCKET_VERSION_HEADER, - e.supported_versions) - return apache.HTTP_BAD_REQUEST - except handshake.HandshakeException, e: - # Handshake for ws/wss failed. - # Send http response with error status. - request.log_error('mod_pywebsocket: %s' % e, apache.APLOG_INFO) - return e.status - - handshake_is_done = True - request._dispatcher = _dispatcher - _dispatcher.transfer_data(request) - except handshake.AbortedByUserException, e: - request.log_error('mod_pywebsocket: %s' % e, apache.APLOG_INFO) - except Exception, e: - # DispatchException can also be thrown if something is wrong in - # pywebsocket code. It's caught here, then. - - request.log_error('mod_pywebsocket: %s\n%s' % - (e, util.get_stack_trace()), - apache.APLOG_ERR) - # Unknown exceptions before handshake mean Apache must handle its - # request with another handler. - if not handshake_is_done: - return apache.DECLINED - # Set assbackwards to suppress response header generation by Apache. - request.assbackwards = 1 - return apache.DONE # Return DONE such that no other handlers are invoked. - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/http_header_util.py b/module/lib/mod_pywebsocket/http_header_util.py deleted file mode 100644 index b77465393..000000000 --- a/module/lib/mod_pywebsocket/http_header_util.py +++ /dev/null @@ -1,263 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""Utilities for parsing and formatting headers that follow the grammar defined -in HTTP RFC http://www.ietf.org/rfc/rfc2616.txt. -""" - - -import urlparse - - -_SEPARATORS = '()<>@,;:\\"/[]?={} \t' - - -def _is_char(c): - """Returns true iff c is in CHAR as specified in HTTP RFC.""" - - return ord(c) <= 127 - - -def _is_ctl(c): - """Returns true iff c is in CTL as specified in HTTP RFC.""" - - return ord(c) <= 31 or ord(c) == 127 - - -class ParsingState(object): - - def __init__(self, data): - self.data = data - self.head = 0 - - -def peek(state, pos=0): - """Peeks the character at pos from the head of data.""" - - if state.head + pos >= len(state.data): - return None - - return state.data[state.head + pos] - - -def consume(state, amount=1): - """Consumes specified amount of bytes from the head and returns the - consumed bytes. If there's not enough bytes to consume, returns None. - """ - - if state.head + amount > len(state.data): - return None - - result = state.data[state.head:state.head + amount] - state.head = state.head + amount - return result - - -def consume_string(state, expected): - """Given a parsing state and a expected string, consumes the string from - the head. Returns True if consumed successfully. Otherwise, returns - False. - """ - - pos = 0 - - for c in expected: - if c != peek(state, pos): - return False - pos += 1 - - consume(state, pos) - return True - - -def consume_lws(state): - """Consumes a LWS from the head. Returns True if any LWS is consumed. - Otherwise, returns False. - - LWS = [CRLF] 1*( SP | HT ) - """ - - original_head = state.head - - consume_string(state, '\r\n') - - pos = 0 - - while True: - c = peek(state, pos) - if c == ' ' or c == '\t': - pos += 1 - else: - if pos == 0: - state.head = original_head - return False - else: - consume(state, pos) - return True - - -def consume_lwses(state): - """Consumes *LWS from the head.""" - - while consume_lws(state): - pass - - -def consume_token(state): - """Consumes a token from the head. Returns the token or None if no token - was found. - """ - - pos = 0 - - while True: - c = peek(state, pos) - if c is None or c in _SEPARATORS or _is_ctl(c) or not _is_char(c): - if pos == 0: - return None - - return consume(state, pos) - else: - pos += 1 - - -def consume_token_or_quoted_string(state): - """Consumes a token or a quoted-string, and returns the token or unquoted - string. If no token or quoted-string was found, returns None. - """ - - original_head = state.head - - if not consume_string(state, '"'): - return consume_token(state) - - result = [] - - expect_quoted_pair = False - - while True: - if not expect_quoted_pair and consume_lws(state): - result.append(' ') - continue - - c = consume(state) - if c is None: - # quoted-string is not enclosed with double quotation - state.head = original_head - return None - elif expect_quoted_pair: - expect_quoted_pair = False - if _is_char(c): - result.append(c) - else: - # Non CHAR character found in quoted-pair - state.head = original_head - return None - elif c == '\\': - expect_quoted_pair = True - elif c == '"': - return ''.join(result) - elif _is_ctl(c): - # Invalid character %r found in qdtext - state.head = original_head - return None - else: - result.append(c) - - -def quote_if_necessary(s): - """Quotes arbitrary string into quoted-string.""" - - quote = False - if s == '': - return '""' - - result = [] - for c in s: - if c == '"' or c in _SEPARATORS or _is_ctl(c) or not _is_char(c): - quote = True - - if c == '"' or _is_ctl(c): - result.append('\\' + c) - else: - result.append(c) - - if quote: - return '"' + ''.join(result) + '"' - else: - return ''.join(result) - - -def parse_uri(uri): - """Parse absolute URI then return host, port and resource.""" - - parsed = urlparse.urlsplit(uri) - if parsed.scheme != 'wss' and parsed.scheme != 'ws': - # |uri| must be a relative URI. - # TODO(toyoshim): Should validate |uri|. - return None, None, uri - - if parsed.hostname is None: - return None, None, None - - port = None - try: - port = parsed.port - except ValueError, e: - # port property cause ValueError on invalid null port description like - # 'ws://host:/path'. - return None, None, None - - if port is None: - if parsed.scheme == 'ws': - port = 80 - else: - port = 443 - - path = parsed.path - if not path: - path += '/' - if parsed.query: - path += '?' + parsed.query - if parsed.fragment: - path += '#' + parsed.fragment - - return parsed.hostname, port, path - - -try: - urlparse.uses_netloc.index('ws') -except ValueError, e: - # urlparse in Python2.5.1 doesn't have 'ws' and 'wss' entries. - urlparse.uses_netloc.append('ws') - urlparse.uses_netloc.append('wss') - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/memorizingfile.py b/module/lib/mod_pywebsocket/memorizingfile.py deleted file mode 100644 index 4d4cd9585..000000000 --- a/module/lib/mod_pywebsocket/memorizingfile.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""Memorizing file. - -A memorizing file wraps a file and memorizes lines read by readline. -""" - - -import sys - - -class MemorizingFile(object): - """MemorizingFile wraps a file and memorizes lines read by readline. - - Note that data read by other methods are not memorized. This behavior - is good enough for memorizing lines SimpleHTTPServer reads before - the control reaches WebSocketRequestHandler. - """ - - def __init__(self, file_, max_memorized_lines=sys.maxint): - """Construct an instance. - - Args: - file_: the file object to wrap. - max_memorized_lines: the maximum number of lines to memorize. - Only the first max_memorized_lines are memorized. - Default: sys.maxint. - """ - - self._file = file_ - self._memorized_lines = [] - self._max_memorized_lines = max_memorized_lines - self._buffered = False - self._buffered_line = None - - def __getattribute__(self, name): - if name in ('_file', '_memorized_lines', '_max_memorized_lines', - '_buffered', '_buffered_line', 'readline', - 'get_memorized_lines'): - return object.__getattribute__(self, name) - return self._file.__getattribute__(name) - - def readline(self, size=-1): - """Override file.readline and memorize the line read. - - Note that even if size is specified and smaller than actual size, - the whole line will be read out from underlying file object by - subsequent readline calls. - """ - - if self._buffered: - line = self._buffered_line - self._buffered = False - else: - line = self._file.readline() - if line and len(self._memorized_lines) < self._max_memorized_lines: - self._memorized_lines.append(line) - if size >= 0 and size < len(line): - self._buffered = True - self._buffered_line = line[size:] - return line[:size] - return line - - def get_memorized_lines(self): - """Get lines memorized so far.""" - return self._memorized_lines - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/msgutil.py b/module/lib/mod_pywebsocket/msgutil.py deleted file mode 100644 index 4c1a0114b..000000000 --- a/module/lib/mod_pywebsocket/msgutil.py +++ /dev/null @@ -1,219 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""Message related utilities. - -Note: request.connection.write/read are used in this module, even though -mod_python document says that they should be used only in connection -handlers. Unfortunately, we have no other options. For example, -request.write/read are not suitable because they don't allow direct raw -bytes writing/reading. -""" - - -import Queue -import threading - - -# Export Exception symbols from msgutil for backward compatibility -from mod_pywebsocket._stream_base import ConnectionTerminatedException -from mod_pywebsocket._stream_base import InvalidFrameException -from mod_pywebsocket._stream_base import BadOperationException -from mod_pywebsocket._stream_base import UnsupportedFrameException - - -# An API for handler to send/receive WebSocket messages. -def close_connection(request): - """Close connection. - - Args: - request: mod_python request. - """ - request.ws_stream.close_connection() - - -def send_message(request, payload_data, end=True, binary=False): - """Send a message (or part of a message). - - Args: - request: mod_python request. - payload_data: unicode text or str binary to send. - end: True to terminate a message. - False to send payload_data as part of a message that is to be - terminated by next or later send_message call with end=True. - binary: send payload_data as binary frame(s). - Raises: - BadOperationException: when server already terminated. - """ - request.ws_stream.send_message(payload_data, end, binary) - - -def receive_message(request): - """Receive a WebSocket frame and return its payload as a text in - unicode or a binary in str. - - Args: - request: mod_python request. - Raises: - InvalidFrameException: when client send invalid frame. - UnsupportedFrameException: when client send unsupported frame e.g. some - of reserved bit is set but no extension can - recognize it. - InvalidUTF8Exception: when client send a text frame containing any - invalid UTF-8 string. - ConnectionTerminatedException: when the connection is closed - unexpectedly. - BadOperationException: when client already terminated. - """ - return request.ws_stream.receive_message() - - -def send_ping(request, body=''): - request.ws_stream.send_ping(body) - - -class MessageReceiver(threading.Thread): - """This class receives messages from the client. - - This class provides three ways to receive messages: blocking, - non-blocking, and via callback. Callback has the highest precedence. - - Note: This class should not be used with the standalone server for wss - because pyOpenSSL used by the server raises a fatal error if the socket - is accessed from multiple threads. - """ - - def __init__(self, request, onmessage=None): - """Construct an instance. - - Args: - request: mod_python request. - onmessage: a function to be called when a message is received. - May be None. If not None, the function is called on - another thread. In that case, MessageReceiver.receive - and MessageReceiver.receive_nowait are useless - because they will never return any messages. - """ - - threading.Thread.__init__(self) - self._request = request - self._queue = Queue.Queue() - self._onmessage = onmessage - self._stop_requested = False - self.setDaemon(True) - self.start() - - def run(self): - try: - while not self._stop_requested: - message = receive_message(self._request) - if self._onmessage: - self._onmessage(message) - else: - self._queue.put(message) - finally: - close_connection(self._request) - - def receive(self): - """ Receive a message from the channel, blocking. - - Returns: - message as a unicode string. - """ - return self._queue.get() - - def receive_nowait(self): - """ Receive a message from the channel, non-blocking. - - Returns: - message as a unicode string if available. None otherwise. - """ - try: - message = self._queue.get_nowait() - except Queue.Empty: - message = None - return message - - def stop(self): - """Request to stop this instance. - - The instance will be stopped after receiving the next message. - This method may not be very useful, but there is no clean way - in Python to forcefully stop a running thread. - """ - self._stop_requested = True - - -class MessageSender(threading.Thread): - """This class sends messages to the client. - - This class provides both synchronous and asynchronous ways to send - messages. - - Note: This class should not be used with the standalone server for wss - because pyOpenSSL used by the server raises a fatal error if the socket - is accessed from multiple threads. - """ - - def __init__(self, request): - """Construct an instance. - - Args: - request: mod_python request. - """ - threading.Thread.__init__(self) - self._request = request - self._queue = Queue.Queue() - self.setDaemon(True) - self.start() - - def run(self): - while True: - message, condition = self._queue.get() - condition.acquire() - send_message(self._request, message) - condition.notify() - condition.release() - - def send(self, message): - """Send a message, blocking.""" - - condition = threading.Condition() - condition.acquire() - self._queue.put((message, condition)) - condition.wait() - - def send_nowait(self, message): - """Send a message, non-blocking.""" - - self._queue.put((message, threading.Condition())) - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/mux.py b/module/lib/mod_pywebsocket/mux.py deleted file mode 100644 index f0bdd2461..000000000 --- a/module/lib/mod_pywebsocket/mux.py +++ /dev/null @@ -1,1636 +0,0 @@ -# Copyright 2012, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""This file provides classes and helper functions for multiplexing extension. - -Specification: -http://tools.ietf.org/html/draft-ietf-hybi-websocket-multiplexing-06 -""" - - -import collections -import copy -import email -import email.parser -import logging -import math -import struct -import threading -import traceback - -from mod_pywebsocket import common -from mod_pywebsocket import handshake -from mod_pywebsocket import util -from mod_pywebsocket._stream_base import BadOperationException -from mod_pywebsocket._stream_base import ConnectionTerminatedException -from mod_pywebsocket._stream_hybi import Frame -from mod_pywebsocket._stream_hybi import Stream -from mod_pywebsocket._stream_hybi import StreamOptions -from mod_pywebsocket._stream_hybi import create_binary_frame -from mod_pywebsocket._stream_hybi import create_closing_handshake_body -from mod_pywebsocket._stream_hybi import create_header -from mod_pywebsocket._stream_hybi import create_length_header -from mod_pywebsocket._stream_hybi import parse_frame -from mod_pywebsocket.handshake import hybi - - -_CONTROL_CHANNEL_ID = 0 -_DEFAULT_CHANNEL_ID = 1 - -_MUX_OPCODE_ADD_CHANNEL_REQUEST = 0 -_MUX_OPCODE_ADD_CHANNEL_RESPONSE = 1 -_MUX_OPCODE_FLOW_CONTROL = 2 -_MUX_OPCODE_DROP_CHANNEL = 3 -_MUX_OPCODE_NEW_CHANNEL_SLOT = 4 - -_MAX_CHANNEL_ID = 2 ** 29 - 1 - -_INITIAL_NUMBER_OF_CHANNEL_SLOTS = 64 -_INITIAL_QUOTA_FOR_CLIENT = 8 * 1024 - -_HANDSHAKE_ENCODING_IDENTITY = 0 -_HANDSHAKE_ENCODING_DELTA = 1 - -# We need only these status code for now. -_HTTP_BAD_RESPONSE_MESSAGES = { - common.HTTP_STATUS_BAD_REQUEST: 'Bad Request', -} - -# DropChannel reason code -# TODO(bashi): Define all reason code defined in -05 draft. -_DROP_CODE_NORMAL_CLOSURE = 1000 - -_DROP_CODE_INVALID_ENCAPSULATING_MESSAGE = 2001 -_DROP_CODE_CHANNEL_ID_TRUNCATED = 2002 -_DROP_CODE_ENCAPSULATED_FRAME_IS_TRUNCATED = 2003 -_DROP_CODE_UNKNOWN_MUX_OPCODE = 2004 -_DROP_CODE_INVALID_MUX_CONTROL_BLOCK = 2005 -_DROP_CODE_CHANNEL_ALREADY_EXISTS = 2006 -_DROP_CODE_NEW_CHANNEL_SLOT_VIOLATION = 2007 - -_DROP_CODE_UNKNOWN_REQUEST_ENCODING = 3002 -_DROP_CODE_SEND_QUOTA_VIOLATION = 3005 -_DROP_CODE_ACKNOWLEDGED = 3008 - - -class MuxUnexpectedException(Exception): - """Exception in handling multiplexing extension.""" - pass - - -# Temporary -class MuxNotImplementedException(Exception): - """Raised when a flow enters unimplemented code path.""" - pass - - -class LogicalConnectionClosedException(Exception): - """Raised when logical connection is gracefully closed.""" - pass - - -class PhysicalConnectionError(Exception): - """Raised when there is a physical connection error.""" - def __init__(self, drop_code, message=''): - super(PhysicalConnectionError, self).__init__( - 'code=%d, message=%r' % (drop_code, message)) - self.drop_code = drop_code - self.message = message - - -class LogicalChannelError(Exception): - """Raised when there is a logical channel error.""" - def __init__(self, channel_id, drop_code, message=''): - super(LogicalChannelError, self).__init__( - 'channel_id=%d, code=%d, message=%r' % ( - channel_id, drop_code, message)) - self.channel_id = channel_id - self.drop_code = drop_code - self.message = message - - -def _encode_channel_id(channel_id): - if channel_id < 0: - raise ValueError('Channel id %d must not be negative' % channel_id) - - if channel_id < 2 ** 7: - return chr(channel_id) - if channel_id < 2 ** 14: - return struct.pack('!H', 0x8000 + channel_id) - if channel_id < 2 ** 21: - first = chr(0xc0 + (channel_id >> 16)) - return first + struct.pack('!H', channel_id & 0xffff) - if channel_id < 2 ** 29: - return struct.pack('!L', 0xe0000000 + channel_id) - - raise ValueError('Channel id %d is too large' % channel_id) - - -def _encode_number(number): - return create_length_header(number, False) - - -def _create_add_channel_response(channel_id, encoded_handshake, - encoding=0, rejected=False, - outer_frame_mask=False): - if encoding != 0 and encoding != 1: - raise ValueError('Invalid encoding %d' % encoding) - - first_byte = ((_MUX_OPCODE_ADD_CHANNEL_RESPONSE << 5) | - (rejected << 4) | encoding) - block = (chr(first_byte) + - _encode_channel_id(channel_id) + - _encode_number(len(encoded_handshake)) + - encoded_handshake) - payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block - return create_binary_frame(payload, mask=outer_frame_mask) - - -def _create_drop_channel(channel_id, code=None, message='', - outer_frame_mask=False): - if len(message) > 0 and code is None: - raise ValueError('Code must be specified if message is specified') - - first_byte = _MUX_OPCODE_DROP_CHANNEL << 5 - block = chr(first_byte) + _encode_channel_id(channel_id) - if code is None: - block += _encode_number(0) # Reason size - else: - reason = struct.pack('!H', code) + message - reason_size = _encode_number(len(reason)) - block += reason_size + reason - - payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block - return create_binary_frame(payload, mask=outer_frame_mask) - - -def _create_flow_control(channel_id, replenished_quota, - outer_frame_mask=False): - first_byte = _MUX_OPCODE_FLOW_CONTROL << 5 - block = (chr(first_byte) + - _encode_channel_id(channel_id) + - _encode_number(replenished_quota)) - payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block - return create_binary_frame(payload, mask=outer_frame_mask) - - -def _create_new_channel_slot(slots, send_quota, outer_frame_mask=False): - if slots < 0 or send_quota < 0: - raise ValueError('slots and send_quota must be non-negative.') - first_byte = _MUX_OPCODE_NEW_CHANNEL_SLOT << 5 - block = (chr(first_byte) + - _encode_number(slots) + - _encode_number(send_quota)) - payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block - return create_binary_frame(payload, mask=outer_frame_mask) - - -def _create_fallback_new_channel_slot(outer_frame_mask=False): - first_byte = (_MUX_OPCODE_NEW_CHANNEL_SLOT << 5) | 1 # Set the F flag - block = (chr(first_byte) + _encode_number(0) + _encode_number(0)) - payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block - return create_binary_frame(payload, mask=outer_frame_mask) - - -def _parse_request_text(request_text): - request_line, header_lines = request_text.split('\r\n', 1) - - words = request_line.split(' ') - if len(words) != 3: - raise ValueError('Bad Request-Line syntax %r' % request_line) - [command, path, version] = words - if version != 'HTTP/1.1': - raise ValueError('Bad request version %r' % version) - - # email.parser.Parser() parses RFC 2822 (RFC 822) style headers. - # RFC 6455 refers RFC 2616 for handshake parsing, and RFC 2616 refers - # RFC 822. - headers = email.parser.Parser().parsestr(header_lines) - return command, path, version, headers - - -class _ControlBlock(object): - """A structure that holds parsing result of multiplexing control block. - Control block specific attributes will be added by _MuxFramePayloadParser. - (e.g. encoded_handshake will be added for AddChannelRequest and - AddChannelResponse) - """ - - def __init__(self, opcode): - self.opcode = opcode - - -class _MuxFramePayloadParser(object): - """A class that parses multiplexed frame payload.""" - - def __init__(self, payload): - self._data = payload - self._read_position = 0 - self._logger = util.get_class_logger(self) - - def read_channel_id(self): - """Reads channel id. - - Raises: - ValueError: when the payload doesn't contain - valid channel id. - """ - - remaining_length = len(self._data) - self._read_position - pos = self._read_position - if remaining_length == 0: - raise ValueError('Invalid channel id format') - - channel_id = ord(self._data[pos]) - channel_id_length = 1 - if channel_id & 0xe0 == 0xe0: - if remaining_length < 4: - raise ValueError('Invalid channel id format') - channel_id = struct.unpack('!L', - self._data[pos:pos+4])[0] & 0x1fffffff - channel_id_length = 4 - elif channel_id & 0xc0 == 0xc0: - if remaining_length < 3: - raise ValueError('Invalid channel id format') - channel_id = (((channel_id & 0x1f) << 16) + - struct.unpack('!H', self._data[pos+1:pos+3])[0]) - channel_id_length = 3 - elif channel_id & 0x80 == 0x80: - if remaining_length < 2: - raise ValueError('Invalid channel id format') - channel_id = struct.unpack('!H', - self._data[pos:pos+2])[0] & 0x3fff - channel_id_length = 2 - self._read_position += channel_id_length - - return channel_id - - def read_inner_frame(self): - """Reads an inner frame. - - Raises: - PhysicalConnectionError: when the inner frame is invalid. - """ - - if len(self._data) == self._read_position: - raise PhysicalConnectionError( - _DROP_CODE_ENCAPSULATED_FRAME_IS_TRUNCATED) - - bits = ord(self._data[self._read_position]) - self._read_position += 1 - fin = (bits & 0x80) == 0x80 - rsv1 = (bits & 0x40) == 0x40 - rsv2 = (bits & 0x20) == 0x20 - rsv3 = (bits & 0x10) == 0x10 - opcode = bits & 0xf - payload = self.remaining_data() - # Consume rest of the message which is payload data of the original - # frame. - self._read_position = len(self._data) - return fin, rsv1, rsv2, rsv3, opcode, payload - - def _read_number(self): - if self._read_position + 1 > len(self._data): - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Cannot read the first byte of number field') - - number = ord(self._data[self._read_position]) - if number & 0x80 == 0x80: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'The most significant bit of the first byte of number should ' - 'be unset') - self._read_position += 1 - pos = self._read_position - if number == 127: - if pos + 8 > len(self._data): - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Invalid number field') - self._read_position += 8 - number = struct.unpack('!Q', self._data[pos:pos+8])[0] - if number > 0x7FFFFFFFFFFFFFFF: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Encoded number >= 2^63') - if number <= 0xFFFF: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - '%d should not be encoded by 9 bytes encoding' % number) - return number - if number == 126: - if pos + 2 > len(self._data): - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Invalid number field') - self._read_position += 2 - number = struct.unpack('!H', self._data[pos:pos+2])[0] - if number <= 125: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - '%d should not be encoded by 3 bytes encoding' % number) - return number - - def _read_size_and_contents(self): - """Reads data that consists of followings: - - the size of the contents encoded the same way as payload length - of the WebSocket Protocol with 1 bit padding at the head. - - the contents. - """ - - size = self._read_number() - pos = self._read_position - if pos + size > len(self._data): - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Cannot read %d bytes data' % size) - - self._read_position += size - return self._data[pos:pos+size] - - def _read_add_channel_request(self, first_byte, control_block): - reserved = (first_byte >> 2) & 0x7 - if reserved != 0: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Reserved bits must be unset') - - # Invalid encoding will be handled by MuxHandler. - encoding = first_byte & 0x3 - try: - control_block.channel_id = self.read_channel_id() - except ValueError, e: - raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) - control_block.encoding = encoding - encoded_handshake = self._read_size_and_contents() - control_block.encoded_handshake = encoded_handshake - return control_block - - def _read_add_channel_response(self, first_byte, control_block): - reserved = (first_byte >> 2) & 0x3 - if reserved != 0: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Reserved bits must be unset') - - control_block.accepted = (first_byte >> 4) & 1 - control_block.encoding = first_byte & 0x3 - try: - control_block.channel_id = self.read_channel_id() - except ValueError, e: - raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) - control_block.encoded_handshake = self._read_size_and_contents() - return control_block - - def _read_flow_control(self, first_byte, control_block): - reserved = first_byte & 0x1f - if reserved != 0: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Reserved bits must be unset') - - try: - control_block.channel_id = self.read_channel_id() - except ValueError, e: - raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) - control_block.send_quota = self._read_number() - return control_block - - def _read_drop_channel(self, first_byte, control_block): - reserved = first_byte & 0x1f - if reserved != 0: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Reserved bits must be unset') - - try: - control_block.channel_id = self.read_channel_id() - except ValueError, e: - raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) - reason = self._read_size_and_contents() - if len(reason) == 0: - control_block.drop_code = None - control_block.drop_message = '' - elif len(reason) >= 2: - control_block.drop_code = struct.unpack('!H', reason[:2])[0] - control_block.drop_message = reason[2:] - else: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Received DropChannel that conains only 1-byte reason') - return control_block - - def _read_new_channel_slot(self, first_byte, control_block): - reserved = first_byte & 0x1e - if reserved != 0: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Reserved bits must be unset') - control_block.fallback = first_byte & 1 - control_block.slots = self._read_number() - control_block.send_quota = self._read_number() - return control_block - - def read_control_blocks(self): - """Reads control block(s). - - Raises: - PhysicalConnectionError: when the payload contains invalid control - block(s). - StopIteration: when no control blocks left. - """ - - while self._read_position < len(self._data): - first_byte = ord(self._data[self._read_position]) - self._read_position += 1 - opcode = (first_byte >> 5) & 0x7 - control_block = _ControlBlock(opcode=opcode) - if opcode == _MUX_OPCODE_ADD_CHANNEL_REQUEST: - yield self._read_add_channel_request(first_byte, control_block) - elif opcode == _MUX_OPCODE_ADD_CHANNEL_RESPONSE: - yield self._read_add_channel_response( - first_byte, control_block) - elif opcode == _MUX_OPCODE_FLOW_CONTROL: - yield self._read_flow_control(first_byte, control_block) - elif opcode == _MUX_OPCODE_DROP_CHANNEL: - yield self._read_drop_channel(first_byte, control_block) - elif opcode == _MUX_OPCODE_NEW_CHANNEL_SLOT: - yield self._read_new_channel_slot(first_byte, control_block) - else: - raise PhysicalConnectionError( - _DROP_CODE_UNKNOWN_MUX_OPCODE, - 'Invalid opcode %d' % opcode) - - assert self._read_position == len(self._data) - raise StopIteration - - def remaining_data(self): - """Returns remaining data.""" - - return self._data[self._read_position:] - - -class _LogicalRequest(object): - """Mimics mod_python request.""" - - def __init__(self, channel_id, command, path, protocol, headers, - connection): - """Constructs an instance. - - Args: - channel_id: the channel id of the logical channel. - command: HTTP request command. - path: HTTP request path. - headers: HTTP headers. - connection: _LogicalConnection instance. - """ - - self.channel_id = channel_id - self.method = command - self.uri = path - self.protocol = protocol - self.headers_in = headers - self.connection = connection - self.server_terminated = False - self.client_terminated = False - - def is_https(self): - """Mimics request.is_https(). Returns False because this method is - used only by old protocols (hixie and hybi00). - """ - - return False - - -class _LogicalConnection(object): - """Mimics mod_python mp_conn.""" - - # For details, see the comment of set_read_state(). - STATE_ACTIVE = 1 - STATE_GRACEFULLY_CLOSED = 2 - STATE_TERMINATED = 3 - - def __init__(self, mux_handler, channel_id): - """Constructs an instance. - - Args: - mux_handler: _MuxHandler instance. - channel_id: channel id of this connection. - """ - - self._mux_handler = mux_handler - self._channel_id = channel_id - self._incoming_data = '' - self._write_condition = threading.Condition() - self._waiting_write_completion = False - self._read_condition = threading.Condition() - self._read_state = self.STATE_ACTIVE - - def get_local_addr(self): - """Getter to mimic mp_conn.local_addr.""" - - return self._mux_handler.physical_connection.get_local_addr() - local_addr = property(get_local_addr) - - def get_remote_addr(self): - """Getter to mimic mp_conn.remote_addr.""" - - return self._mux_handler.physical_connection.get_remote_addr() - remote_addr = property(get_remote_addr) - - def get_memorized_lines(self): - """Gets memorized lines. Not supported.""" - - raise MuxUnexpectedException('_LogicalConnection does not support ' - 'get_memorized_lines') - - def write(self, data): - """Writes data. mux_handler sends data asynchronously. The caller will - be suspended until write done. - - Args: - data: data to be written. - - Raises: - MuxUnexpectedException: when called before finishing the previous - write. - """ - - try: - self._write_condition.acquire() - if self._waiting_write_completion: - raise MuxUnexpectedException( - 'Logical connection %d is already waiting the completion ' - 'of write' % self._channel_id) - - self._waiting_write_completion = True - self._mux_handler.send_data(self._channel_id, data) - self._write_condition.wait() - finally: - self._write_condition.release() - - def write_control_data(self, data): - """Writes data via the control channel. Don't wait finishing write - because this method can be called by mux dispatcher. - - Args: - data: data to be written. - """ - - self._mux_handler.send_control_data(data) - - def notify_write_done(self): - """Called when sending data is completed.""" - - try: - self._write_condition.acquire() - if not self._waiting_write_completion: - raise MuxUnexpectedException( - 'Invalid call of notify_write_done for logical connection' - ' %d' % self._channel_id) - self._waiting_write_completion = False - self._write_condition.notify() - finally: - self._write_condition.release() - - def append_frame_data(self, frame_data): - """Appends incoming frame data. Called when mux_handler dispatches - frame data to the corresponding application. - - Args: - frame_data: incoming frame data. - """ - - self._read_condition.acquire() - self._incoming_data += frame_data - self._read_condition.notify() - self._read_condition.release() - - def read(self, length): - """Reads data. Blocks until enough data has arrived via physical - connection. - - Args: - length: length of data to be read. - Raises: - LogicalConnectionClosedException: when closing handshake for this - logical channel has been received. - ConnectionTerminatedException: when the physical connection has - closed, or an error is caused on the reader thread. - """ - - self._read_condition.acquire() - while (self._read_state == self.STATE_ACTIVE and - len(self._incoming_data) < length): - self._read_condition.wait() - - try: - if self._read_state == self.STATE_GRACEFULLY_CLOSED: - raise LogicalConnectionClosedException( - 'Logical channel %d has closed.' % self._channel_id) - elif self._read_state == self.STATE_TERMINATED: - raise ConnectionTerminatedException( - 'Receiving %d byte failed. Logical channel (%d) closed' % - (length, self._channel_id)) - - value = self._incoming_data[:length] - self._incoming_data = self._incoming_data[length:] - finally: - self._read_condition.release() - - return value - - def set_read_state(self, new_state): - """Sets the state of this connection. Called when an event for this - connection has occurred. - - Args: - new_state: state to be set. new_state must be one of followings: - - STATE_GRACEFULLY_CLOSED: when closing handshake for this - connection has been received. - - STATE_TERMINATED: when the physical connection has closed or - DropChannel of this connection has received. - """ - - self._read_condition.acquire() - self._read_state = new_state - self._read_condition.notify() - self._read_condition.release() - - -class _LogicalStream(Stream): - """Mimics the Stream class. This class interprets multiplexed WebSocket - frames. - """ - - def __init__(self, request, send_quota, receive_quota): - """Constructs an instance. - - Args: - request: _LogicalRequest instance. - send_quota: Initial send quota. - receive_quota: Initial receive quota. - """ - - # TODO(bashi): Support frame filters. - stream_options = StreamOptions() - # Physical stream is responsible for masking. - stream_options.unmask_receive = False - # Control frames can be fragmented on logical channel. - stream_options.allow_fragmented_control_frame = True - Stream.__init__(self, request, stream_options) - self._send_quota = send_quota - self._send_quota_condition = threading.Condition() - self._receive_quota = receive_quota - self._write_inner_frame_semaphore = threading.Semaphore() - - def _create_inner_frame(self, opcode, payload, end=True): - # TODO(bashi): Support extensions that use reserved bits. - first_byte = (end << 7) | opcode - return (_encode_channel_id(self._request.channel_id) + - chr(first_byte) + payload) - - def _write_inner_frame(self, opcode, payload, end=True): - payload_length = len(payload) - write_position = 0 - - try: - # An inner frame will be fragmented if there is no enough send - # quota. This semaphore ensures that fragmented inner frames are - # sent in order on the logical channel. - # Note that frames that come from other logical channels or - # multiplexing control blocks can be inserted between fragmented - # inner frames on the physical channel. - self._write_inner_frame_semaphore.acquire() - while write_position < payload_length: - try: - self._send_quota_condition.acquire() - while self._send_quota == 0: - self._logger.debug( - 'No quota. Waiting FlowControl message for %d.' % - self._request.channel_id) - self._send_quota_condition.wait() - - remaining = payload_length - write_position - write_length = min(self._send_quota, remaining) - inner_frame_end = ( - end and - (write_position + write_length == payload_length)) - - inner_frame = self._create_inner_frame( - opcode, - payload[write_position:write_position+write_length], - inner_frame_end) - frame_data = self._writer.build( - inner_frame, end=True, binary=True) - self._send_quota -= write_length - self._logger.debug('Consumed quota=%d, remaining=%d' % - (write_length, self._send_quota)) - finally: - self._send_quota_condition.release() - - # Writing data will block the worker so we need to release - # _send_quota_condition before writing. - self._logger.debug('Sending inner frame: %r' % frame_data) - self._request.connection.write(frame_data) - write_position += write_length - - opcode = common.OPCODE_CONTINUATION - - except ValueError, e: - raise BadOperationException(e) - finally: - self._write_inner_frame_semaphore.release() - - def replenish_send_quota(self, send_quota): - """Replenish send quota.""" - - self._send_quota_condition.acquire() - self._send_quota += send_quota - self._logger.debug('Replenished send quota for channel id %d: %d' % - (self._request.channel_id, self._send_quota)) - self._send_quota_condition.notify() - self._send_quota_condition.release() - - def consume_receive_quota(self, amount): - """Consumes receive quota. Returns False on failure.""" - - if self._receive_quota < amount: - self._logger.debug('Violate quota on channel id %d: %d < %d' % - (self._request.channel_id, - self._receive_quota, amount)) - return False - self._receive_quota -= amount - return True - - def send_message(self, message, end=True, binary=False): - """Override Stream.send_message.""" - - if self._request.server_terminated: - raise BadOperationException( - 'Requested send_message after sending out a closing handshake') - - if binary and isinstance(message, unicode): - raise BadOperationException( - 'Message for binary frame must be instance of str') - - if binary: - opcode = common.OPCODE_BINARY - else: - opcode = common.OPCODE_TEXT - message = message.encode('utf-8') - - self._write_inner_frame(opcode, message, end) - - def _receive_frame(self): - """Overrides Stream._receive_frame. - - In addition to call Stream._receive_frame, this method adds the amount - of payload to receiving quota and sends FlowControl to the client. - We need to do it here because Stream.receive_message() handles - control frames internally. - """ - - opcode, payload, fin, rsv1, rsv2, rsv3 = Stream._receive_frame(self) - amount = len(payload) - self._receive_quota += amount - frame_data = _create_flow_control(self._request.channel_id, - amount) - self._logger.debug('Sending flow control for %d, replenished=%d' % - (self._request.channel_id, amount)) - self._request.connection.write_control_data(frame_data) - return opcode, payload, fin, rsv1, rsv2, rsv3 - - def receive_message(self): - """Overrides Stream.receive_message.""" - - # Just call Stream.receive_message(), but catch - # LogicalConnectionClosedException, which is raised when the logical - # connection has closed gracefully. - try: - return Stream.receive_message(self) - except LogicalConnectionClosedException, e: - self._logger.debug('%s', e) - return None - - def _send_closing_handshake(self, code, reason): - """Overrides Stream._send_closing_handshake.""" - - body = create_closing_handshake_body(code, reason) - self._logger.debug('Sending closing handshake for %d: (%r, %r)' % - (self._request.channel_id, code, reason)) - self._write_inner_frame(common.OPCODE_CLOSE, body, end=True) - - self._request.server_terminated = True - - def send_ping(self, body=''): - """Overrides Stream.send_ping""" - - self._logger.debug('Sending ping on logical channel %d: %r' % - (self._request.channel_id, body)) - self._write_inner_frame(common.OPCODE_PING, body, end=True) - - self._ping_queue.append(body) - - def _send_pong(self, body): - """Overrides Stream._send_pong""" - - self._logger.debug('Sending pong on logical channel %d: %r' % - (self._request.channel_id, body)) - self._write_inner_frame(common.OPCODE_PONG, body, end=True) - - def close_connection(self, code=common.STATUS_NORMAL_CLOSURE, reason=''): - """Overrides Stream.close_connection.""" - - # TODO(bashi): Implement - self._logger.debug('Closing logical connection %d' % - self._request.channel_id) - self._request.server_terminated = True - - def _drain_received_data(self): - """Overrides Stream._drain_received_data. Nothing need to be done for - logical channel. - """ - - pass - - -class _OutgoingData(object): - """A structure that holds data to be sent via physical connection and - origin of the data. - """ - - def __init__(self, channel_id, data): - self.channel_id = channel_id - self.data = data - - -class _PhysicalConnectionWriter(threading.Thread): - """A thread that is responsible for writing data to physical connection. - - TODO(bashi): Make sure there is no thread-safety problem when the reader - thread reads data from the same socket at a time. - """ - - def __init__(self, mux_handler): - """Constructs an instance. - - Args: - mux_handler: _MuxHandler instance. - """ - - threading.Thread.__init__(self) - self._logger = util.get_class_logger(self) - self._mux_handler = mux_handler - self.setDaemon(True) - self._stop_requested = False - self._deque = collections.deque() - self._deque_condition = threading.Condition() - - def put_outgoing_data(self, data): - """Puts outgoing data. - - Args: - data: _OutgoingData instance. - - Raises: - BadOperationException: when the thread has been requested to - terminate. - """ - - try: - self._deque_condition.acquire() - if self._stop_requested: - raise BadOperationException('Cannot write data anymore') - - self._deque.append(data) - self._deque_condition.notify() - finally: - self._deque_condition.release() - - def _write_data(self, outgoing_data): - try: - self._mux_handler.physical_connection.write(outgoing_data.data) - except Exception, e: - util.prepend_message_to_exception( - 'Failed to send message to %r: ' % - (self._mux_handler.physical_connection.remote_addr,), e) - raise - - # TODO(bashi): It would be better to block the thread that sends - # control data as well. - if outgoing_data.channel_id != _CONTROL_CHANNEL_ID: - self._mux_handler.notify_write_done(outgoing_data.channel_id) - - def run(self): - self._deque_condition.acquire() - while not self._stop_requested: - if len(self._deque) == 0: - self._deque_condition.wait() - continue - - outgoing_data = self._deque.popleft() - self._deque_condition.release() - self._write_data(outgoing_data) - self._deque_condition.acquire() - - # Flush deque - try: - while len(self._deque) > 0: - outgoing_data = self._deque.popleft() - self._write_data(outgoing_data) - finally: - self._deque_condition.release() - - def stop(self): - """Stops the writer thread.""" - - self._deque_condition.acquire() - self._stop_requested = True - self._deque_condition.notify() - self._deque_condition.release() - - -class _PhysicalConnectionReader(threading.Thread): - """A thread that is responsible for reading data from physical connection. - """ - - def __init__(self, mux_handler): - """Constructs an instance. - - Args: - mux_handler: _MuxHandler instance. - """ - - threading.Thread.__init__(self) - self._logger = util.get_class_logger(self) - self._mux_handler = mux_handler - self.setDaemon(True) - - def run(self): - while True: - try: - physical_stream = self._mux_handler.physical_stream - message = physical_stream.receive_message() - if message is None: - break - # Below happens only when a data message is received. - opcode = physical_stream.get_last_received_opcode() - if opcode != common.OPCODE_BINARY: - self._mux_handler.fail_physical_connection( - _DROP_CODE_INVALID_ENCAPSULATING_MESSAGE, - 'Received a text message on physical connection') - break - - except ConnectionTerminatedException, e: - self._logger.debug('%s', e) - break - - try: - self._mux_handler.dispatch_message(message) - except PhysicalConnectionError, e: - self._mux_handler.fail_physical_connection( - e.drop_code, e.message) - break - except LogicalChannelError, e: - self._mux_handler.fail_logical_channel( - e.channel_id, e.drop_code, e.message) - except Exception, e: - self._logger.debug(traceback.format_exc()) - break - - self._mux_handler.notify_reader_done() - - -class _Worker(threading.Thread): - """A thread that is responsible for running the corresponding application - handler. - """ - - def __init__(self, mux_handler, request): - """Constructs an instance. - - Args: - mux_handler: _MuxHandler instance. - request: _LogicalRequest instance. - """ - - threading.Thread.__init__(self) - self._logger = util.get_class_logger(self) - self._mux_handler = mux_handler - self._request = request - self.setDaemon(True) - - def run(self): - self._logger.debug('Logical channel worker started. (id=%d)' % - self._request.channel_id) - try: - # Non-critical exceptions will be handled by dispatcher. - self._mux_handler.dispatcher.transfer_data(self._request) - finally: - self._mux_handler.notify_worker_done(self._request.channel_id) - - -class _MuxHandshaker(hybi.Handshaker): - """Opening handshake processor for multiplexing.""" - - _DUMMY_WEBSOCKET_KEY = 'dGhlIHNhbXBsZSBub25jZQ==' - - def __init__(self, request, dispatcher, send_quota, receive_quota): - """Constructs an instance. - Args: - request: _LogicalRequest instance. - dispatcher: Dispatcher instance (dispatch.Dispatcher). - send_quota: Initial send quota. - receive_quota: Initial receive quota. - """ - - hybi.Handshaker.__init__(self, request, dispatcher) - self._send_quota = send_quota - self._receive_quota = receive_quota - - # Append headers which should not be included in handshake field of - # AddChannelRequest. - # TODO(bashi): Make sure whether we should raise exception when - # these headers are included already. - request.headers_in[common.UPGRADE_HEADER] = ( - common.WEBSOCKET_UPGRADE_TYPE) - request.headers_in[common.CONNECTION_HEADER] = ( - common.UPGRADE_CONNECTION_TYPE) - request.headers_in[common.SEC_WEBSOCKET_VERSION_HEADER] = ( - str(common.VERSION_HYBI_LATEST)) - request.headers_in[common.SEC_WEBSOCKET_KEY_HEADER] = ( - self._DUMMY_WEBSOCKET_KEY) - - def _create_stream(self, stream_options): - """Override hybi.Handshaker._create_stream.""" - - self._logger.debug('Creating logical stream for %d' % - self._request.channel_id) - return _LogicalStream(self._request, self._send_quota, - self._receive_quota) - - def _create_handshake_response(self, accept): - """Override hybi._create_handshake_response.""" - - response = [] - - response.append('HTTP/1.1 101 Switching Protocols\r\n') - - # Upgrade, Connection and Sec-WebSocket-Accept should be excluded. - if self._request.ws_protocol is not None: - response.append('%s: %s\r\n' % ( - common.SEC_WEBSOCKET_PROTOCOL_HEADER, - self._request.ws_protocol)) - if (self._request.ws_extensions is not None and - len(self._request.ws_extensions) != 0): - response.append('%s: %s\r\n' % ( - common.SEC_WEBSOCKET_EXTENSIONS_HEADER, - common.format_extensions(self._request.ws_extensions))) - response.append('\r\n') - - return ''.join(response) - - def _send_handshake(self, accept): - """Override hybi.Handshaker._send_handshake.""" - - # Don't send handshake response for the default channel - if self._request.channel_id == _DEFAULT_CHANNEL_ID: - return - - handshake_response = self._create_handshake_response(accept) - frame_data = _create_add_channel_response( - self._request.channel_id, - handshake_response) - self._logger.debug('Sending handshake response for %d: %r' % - (self._request.channel_id, frame_data)) - self._request.connection.write_control_data(frame_data) - - -class _LogicalChannelData(object): - """A structure that holds information about logical channel. - """ - - def __init__(self, request, worker): - self.request = request - self.worker = worker - self.drop_code = _DROP_CODE_NORMAL_CLOSURE - self.drop_message = '' - - -class _HandshakeDeltaBase(object): - """A class that holds information for delta-encoded handshake.""" - - def __init__(self, headers): - self._headers = headers - - def create_headers(self, delta=None): - """Creates request headers for an AddChannelRequest that has - delta-encoded handshake. - - Args: - delta: headers should be overridden. - """ - - headers = copy.copy(self._headers) - if delta: - for key, value in delta.items(): - # The spec requires that a header with an empty value is - # removed from the delta base. - if len(value) == 0 and headers.has_key(key): - del headers[key] - else: - headers[key] = value - # TODO(bashi): Support extensions - headers['Sec-WebSocket-Extensions'] = '' - return headers - - -class _MuxHandler(object): - """Multiplexing handler. When a handler starts, it launches three - threads; the reader thread, the writer thread, and a worker thread. - - The reader thread reads data from the physical stream, i.e., the - ws_stream object of the underlying websocket connection. The reader - thread interprets multiplexed frames and dispatches them to logical - channels. Methods of this class are mostly called by the reader thread. - - The writer thread sends multiplexed frames which are created by - logical channels via the physical connection. - - The worker thread launched at the starting point handles the - "Implicitly Opened Connection". If multiplexing handler receives - an AddChannelRequest and accepts it, the handler will launch a new worker - thread and dispatch the request to it. - """ - - def __init__(self, request, dispatcher): - """Constructs an instance. - - Args: - request: mod_python request of the physical connection. - dispatcher: Dispatcher instance (dispatch.Dispatcher). - """ - - self.original_request = request - self.dispatcher = dispatcher - self.physical_connection = request.connection - self.physical_stream = request.ws_stream - self._logger = util.get_class_logger(self) - self._logical_channels = {} - self._logical_channels_condition = threading.Condition() - # Holds client's initial quota - self._channel_slots = collections.deque() - self._handshake_base = None - self._worker_done_notify_received = False - self._reader = None - self._writer = None - - def start(self): - """Starts the handler. - - Raises: - MuxUnexpectedException: when the handler already started, or when - opening handshake of the default channel fails. - """ - - if self._reader or self._writer: - raise MuxUnexpectedException('MuxHandler already started') - - self._reader = _PhysicalConnectionReader(self) - self._writer = _PhysicalConnectionWriter(self) - self._reader.start() - self._writer.start() - - # Create "Implicitly Opened Connection". - logical_connection = _LogicalConnection(self, _DEFAULT_CHANNEL_ID) - self._handshake_base = _HandshakeDeltaBase( - self.original_request.headers_in) - logical_request = _LogicalRequest( - _DEFAULT_CHANNEL_ID, - self.original_request.method, - self.original_request.uri, - self.original_request.protocol, - self._handshake_base.create_headers(), - logical_connection) - # Client's send quota for the implicitly opened connection is zero, - # but we will send FlowControl later so set the initial quota to - # _INITIAL_QUOTA_FOR_CLIENT. - self._channel_slots.append(_INITIAL_QUOTA_FOR_CLIENT) - if not self._do_handshake_for_logical_request( - logical_request, send_quota=self.original_request.mux_quota): - raise MuxUnexpectedException( - 'Failed handshake on the default channel id') - self._add_logical_channel(logical_request) - - # Send FlowControl for the implicitly opened connection. - frame_data = _create_flow_control(_DEFAULT_CHANNEL_ID, - _INITIAL_QUOTA_FOR_CLIENT) - logical_request.connection.write_control_data(frame_data) - - def add_channel_slots(self, slots, send_quota): - """Adds channel slots. - - Args: - slots: number of slots to be added. - send_quota: initial send quota for slots. - """ - - self._channel_slots.extend([send_quota] * slots) - # Send NewChannelSlot to client. - frame_data = _create_new_channel_slot(slots, send_quota) - self.send_control_data(frame_data) - - def wait_until_done(self, timeout=None): - """Waits until all workers are done. Returns False when timeout has - occurred. Returns True on success. - - Args: - timeout: timeout in sec. - """ - - self._logical_channels_condition.acquire() - try: - while len(self._logical_channels) > 0: - self._logger.debug('Waiting workers(%d)...' % - len(self._logical_channels)) - self._worker_done_notify_received = False - self._logical_channels_condition.wait(timeout) - if not self._worker_done_notify_received: - self._logger.debug('Waiting worker(s) timed out') - return False - - finally: - self._logical_channels_condition.release() - - # Flush pending outgoing data - self._writer.stop() - self._writer.join() - - return True - - def notify_write_done(self, channel_id): - """Called by the writer thread when a write operation has done. - - Args: - channel_id: objective channel id. - """ - - try: - self._logical_channels_condition.acquire() - if channel_id in self._logical_channels: - channel_data = self._logical_channels[channel_id] - channel_data.request.connection.notify_write_done() - else: - self._logger.debug('Seems that logical channel for %d has gone' - % channel_id) - finally: - self._logical_channels_condition.release() - - def send_control_data(self, data): - """Sends data via the control channel. - - Args: - data: data to be sent. - """ - - self._writer.put_outgoing_data(_OutgoingData( - channel_id=_CONTROL_CHANNEL_ID, data=data)) - - def send_data(self, channel_id, data): - """Sends data via given logical channel. This method is called by - worker threads. - - Args: - channel_id: objective channel id. - data: data to be sent. - """ - - self._writer.put_outgoing_data(_OutgoingData( - channel_id=channel_id, data=data)) - - def _send_drop_channel(self, channel_id, code=None, message=''): - frame_data = _create_drop_channel(channel_id, code, message) - self._logger.debug( - 'Sending drop channel for channel id %d' % channel_id) - self.send_control_data(frame_data) - - def _send_error_add_channel_response(self, channel_id, status=None): - if status is None: - status = common.HTTP_STATUS_BAD_REQUEST - - if status in _HTTP_BAD_RESPONSE_MESSAGES: - message = _HTTP_BAD_RESPONSE_MESSAGES[status] - else: - self._logger.debug('Response message for %d is not found' % status) - message = '???' - - response = 'HTTP/1.1 %d %s\r\n\r\n' % (status, message) - frame_data = _create_add_channel_response(channel_id, - encoded_handshake=response, - encoding=0, rejected=True) - self.send_control_data(frame_data) - - def _create_logical_request(self, block): - if block.channel_id == _CONTROL_CHANNEL_ID: - # TODO(bashi): Raise PhysicalConnectionError with code 2006 - # instead of MuxUnexpectedException. - raise MuxUnexpectedException( - 'Received the control channel id (0) as objective channel ' - 'id for AddChannel') - - if block.encoding > _HANDSHAKE_ENCODING_DELTA: - raise PhysicalConnectionError( - _DROP_CODE_UNKNOWN_REQUEST_ENCODING) - - method, path, version, headers = _parse_request_text( - block.encoded_handshake) - if block.encoding == _HANDSHAKE_ENCODING_DELTA: - headers = self._handshake_base.create_headers(headers) - - connection = _LogicalConnection(self, block.channel_id) - request = _LogicalRequest(block.channel_id, method, path, version, - headers, connection) - return request - - def _do_handshake_for_logical_request(self, request, send_quota=0): - try: - receive_quota = self._channel_slots.popleft() - except IndexError: - raise LogicalChannelError( - request.channel_id, _DROP_CODE_NEW_CHANNEL_SLOT_VIOLATION) - - handshaker = _MuxHandshaker(request, self.dispatcher, - send_quota, receive_quota) - try: - handshaker.do_handshake() - except handshake.VersionException, e: - self._logger.info('%s', e) - self._send_error_add_channel_response( - request.channel_id, status=common.HTTP_STATUS_BAD_REQUEST) - return False - except handshake.HandshakeException, e: - # TODO(bashi): Should we _Fail the Logical Channel_ with 3001 - # instead? - self._logger.info('%s', e) - self._send_error_add_channel_response(request.channel_id, - status=e.status) - return False - except handshake.AbortedByUserException, e: - self._logger.info('%s', e) - self._send_error_add_channel_response(request.channel_id) - return False - - return True - - def _add_logical_channel(self, logical_request): - try: - self._logical_channels_condition.acquire() - if logical_request.channel_id in self._logical_channels: - self._logger.debug('Channel id %d already exists' % - logical_request.channel_id) - raise PhysicalConnectionError( - _DROP_CODE_CHANNEL_ALREADY_EXISTS, - 'Channel id %d already exists' % - logical_request.channel_id) - worker = _Worker(self, logical_request) - channel_data = _LogicalChannelData(logical_request, worker) - self._logical_channels[logical_request.channel_id] = channel_data - worker.start() - finally: - self._logical_channels_condition.release() - - def _process_add_channel_request(self, block): - try: - logical_request = self._create_logical_request(block) - except ValueError, e: - self._logger.debug('Failed to create logical request: %r' % e) - self._send_error_add_channel_response( - block.channel_id, status=common.HTTP_STATUS_BAD_REQUEST) - return - if self._do_handshake_for_logical_request(logical_request): - if block.encoding == _HANDSHAKE_ENCODING_IDENTITY: - # Update handshake base. - # TODO(bashi): Make sure this is the right place to update - # handshake base. - self._handshake_base = _HandshakeDeltaBase( - logical_request.headers_in) - self._add_logical_channel(logical_request) - else: - self._send_error_add_channel_response( - block.channel_id, status=common.HTTP_STATUS_BAD_REQUEST) - - def _process_flow_control(self, block): - try: - self._logical_channels_condition.acquire() - if not block.channel_id in self._logical_channels: - return - channel_data = self._logical_channels[block.channel_id] - channel_data.request.ws_stream.replenish_send_quota( - block.send_quota) - finally: - self._logical_channels_condition.release() - - def _process_drop_channel(self, block): - self._logger.debug( - 'DropChannel received for %d: code=%r, reason=%r' % - (block.channel_id, block.drop_code, block.drop_message)) - try: - self._logical_channels_condition.acquire() - if not block.channel_id in self._logical_channels: - return - channel_data = self._logical_channels[block.channel_id] - channel_data.drop_code = _DROP_CODE_ACKNOWLEDGED - # Close the logical channel - channel_data.request.connection.set_read_state( - _LogicalConnection.STATE_TERMINATED) - finally: - self._logical_channels_condition.release() - - def _process_control_blocks(self, parser): - for control_block in parser.read_control_blocks(): - opcode = control_block.opcode - self._logger.debug('control block received, opcode: %d' % opcode) - if opcode == _MUX_OPCODE_ADD_CHANNEL_REQUEST: - self._process_add_channel_request(control_block) - elif opcode == _MUX_OPCODE_ADD_CHANNEL_RESPONSE: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Received AddChannelResponse') - elif opcode == _MUX_OPCODE_FLOW_CONTROL: - self._process_flow_control(control_block) - elif opcode == _MUX_OPCODE_DROP_CHANNEL: - self._process_drop_channel(control_block) - elif opcode == _MUX_OPCODE_NEW_CHANNEL_SLOT: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Received NewChannelSlot') - else: - raise MuxUnexpectedException( - 'Unexpected opcode %r' % opcode) - - def _process_logical_frame(self, channel_id, parser): - self._logger.debug('Received a frame. channel id=%d' % channel_id) - try: - self._logical_channels_condition.acquire() - if not channel_id in self._logical_channels: - # We must ignore the message for an inactive channel. - return - channel_data = self._logical_channels[channel_id] - fin, rsv1, rsv2, rsv3, opcode, payload = parser.read_inner_frame() - if not channel_data.request.ws_stream.consume_receive_quota( - len(payload)): - # The client violates quota. Close logical channel. - raise LogicalChannelError( - channel_id, _DROP_CODE_SEND_QUOTA_VIOLATION) - header = create_header(opcode, len(payload), fin, rsv1, rsv2, rsv3, - mask=False) - frame_data = header + payload - channel_data.request.connection.append_frame_data(frame_data) - finally: - self._logical_channels_condition.release() - - def dispatch_message(self, message): - """Dispatches message. The reader thread calls this method. - - Args: - message: a message that contains encapsulated frame. - Raises: - PhysicalConnectionError: if the message contains physical - connection level errors. - LogicalChannelError: if the message contains logical channel - level errors. - """ - - parser = _MuxFramePayloadParser(message) - try: - channel_id = parser.read_channel_id() - except ValueError, e: - raise PhysicalConnectionError(_DROP_CODE_CHANNEL_ID_TRUNCATED) - if channel_id == _CONTROL_CHANNEL_ID: - self._process_control_blocks(parser) - else: - self._process_logical_frame(channel_id, parser) - - def notify_worker_done(self, channel_id): - """Called when a worker has finished. - - Args: - channel_id: channel id corresponded with the worker. - """ - - self._logger.debug('Worker for channel id %d terminated' % channel_id) - try: - self._logical_channels_condition.acquire() - if not channel_id in self._logical_channels: - raise MuxUnexpectedException( - 'Channel id %d not found' % channel_id) - channel_data = self._logical_channels.pop(channel_id) - finally: - self._worker_done_notify_received = True - self._logical_channels_condition.notify() - self._logical_channels_condition.release() - - if not channel_data.request.server_terminated: - self._send_drop_channel( - channel_id, code=channel_data.drop_code, - message=channel_data.drop_message) - - def notify_reader_done(self): - """This method is called by the reader thread when the reader has - finished. - """ - - # Terminate all logical connections - self._logger.debug('termiating all logical connections...') - self._logical_channels_condition.acquire() - for channel_data in self._logical_channels.values(): - try: - channel_data.request.connection.set_read_state( - _LogicalConnection.STATE_TERMINATED) - except Exception: - pass - self._logical_channels_condition.release() - - def fail_physical_connection(self, code, message): - """Fail the physical connection. - - Args: - code: drop reason code. - message: drop message. - """ - - self._logger.debug('Failing the physical connection...') - self._send_drop_channel(_CONTROL_CHANNEL_ID, code, message) - self.physical_stream.close_connection( - common.STATUS_INTERNAL_ENDPOINT_ERROR) - - def fail_logical_channel(self, channel_id, code, message): - """Fail a logical channel. - - Args: - channel_id: channel id. - code: drop reason code. - message: drop message. - """ - - self._logger.debug('Failing logical channel %d...' % channel_id) - try: - self._logical_channels_condition.acquire() - if channel_id in self._logical_channels: - channel_data = self._logical_channels[channel_id] - # Close the logical channel. notify_worker_done() will be - # called later and it will send DropChannel. - channel_data.drop_code = code - channel_data.drop_message = message - channel_data.request.connection.set_read_state( - _LogicalConnection.STATE_TERMINATED) - else: - self._send_drop_channel(channel_id, code, message) - finally: - self._logical_channels_condition.release() - - -def use_mux(request): - return hasattr(request, 'mux') and request.mux - - -def start(request, dispatcher): - mux_handler = _MuxHandler(request, dispatcher) - mux_handler.start() - - mux_handler.add_channel_slots(_INITIAL_NUMBER_OF_CHANNEL_SLOTS, - _INITIAL_QUOTA_FOR_CLIENT) - - mux_handler.wait_until_done() - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/standalone.py b/module/lib/mod_pywebsocket/standalone.py deleted file mode 100755 index 07a33d9c9..000000000 --- a/module/lib/mod_pywebsocket/standalone.py +++ /dev/null @@ -1,998 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2012, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""Standalone WebSocket server. - -Use this file to launch pywebsocket without Apache HTTP Server. - - -BASIC USAGE - -Go to the src directory and run - - $ python mod_pywebsocket/standalone.py [-p ] - [-w ] - [-d ] - - is the port number to use for ws:// connection. - - is the path to the root directory of HTML files. - - is the path to the root directory of WebSocket handlers. -If not specified, will be used. See __init__.py (or -run $ pydoc mod_pywebsocket) for how to write WebSocket handlers. - -For more detail and other options, run - - $ python mod_pywebsocket/standalone.py --help - -or see _build_option_parser method below. - -For trouble shooting, adding "--log_level debug" might help you. - - -TRY DEMO - -Go to the src directory and run - - $ python standalone.py -d example - -to launch pywebsocket with the sample handler and html on port 80. Open -http://localhost/console.html, click the connect button, type something into -the text box next to the send button and click the send button. If everything -is working, you'll see the message you typed echoed by the server. - - -SUPPORTING TLS - -To support TLS, run standalone.py with -t, -k, and -c options. - - -SUPPORTING CLIENT AUTHENTICATION - -To support client authentication with TLS, run standalone.py with -t, -k, -c, -and --tls-client-auth, and --tls-client-ca options. - -E.g., $./standalone.py -d ../example -p 10443 -t -c ../test/cert/cert.pem -k -../test/cert/key.pem --tls-client-auth --tls-client-ca=../test/cert/cacert.pem - - -CONFIGURATION FILE - -You can also write a configuration file and use it by specifying the path to -the configuration file by --config option. Please write a configuration file -following the documentation of the Python ConfigParser library. Name of each -entry must be the long version argument name. E.g. to set log level to debug, -add the following line: - -log_level=debug - -For options which doesn't take value, please add some fake value. E.g. for ---tls option, add the following line: - -tls=True - -Note that tls will be enabled even if you write tls=False as the value part is -fake. - -When both a command line argument and a configuration file entry are set for -the same configuration item, the command line value will override one in the -configuration file. - - -THREADING - -This server is derived from SocketServer.ThreadingMixIn. Hence a thread is -used for each request. - - -SECURITY WARNING - -This uses CGIHTTPServer and CGIHTTPServer is not secure. -It may execute arbitrary Python code or external programs. It should not be -used outside a firewall. -""" - -import BaseHTTPServer -import CGIHTTPServer -import SimpleHTTPServer -import SocketServer -import ConfigParser -import base64 -import httplib -import logging -import logging.handlers -import optparse -import os -import re -import select -import socket -import sys -import threading -import time - -_HAS_SSL = False -_HAS_OPEN_SSL = False -try: - import ssl - _HAS_SSL = True -except ImportError: - try: - import OpenSSL.SSL - _HAS_OPEN_SSL = True - except ImportError: - pass - -from mod_pywebsocket import common -from mod_pywebsocket import dispatch -from mod_pywebsocket import handshake -from mod_pywebsocket import http_header_util -from mod_pywebsocket import memorizingfile -from mod_pywebsocket import util - - -_DEFAULT_LOG_MAX_BYTES = 1024 * 256 -_DEFAULT_LOG_BACKUP_COUNT = 5 - -_DEFAULT_REQUEST_QUEUE_SIZE = 128 - -# 1024 is practically large enough to contain WebSocket handshake lines. -_MAX_MEMORIZED_LINES = 1024 - - -class _StandaloneConnection(object): - """Mimic mod_python mp_conn.""" - - def __init__(self, request_handler): - """Construct an instance. - - Args: - request_handler: A WebSocketRequestHandler instance. - """ - - self._request_handler = request_handler - - def get_local_addr(self): - """Getter to mimic mp_conn.local_addr.""" - - return (self._request_handler.server.server_name, - self._request_handler.server.server_port) - local_addr = property(get_local_addr) - - def get_remote_addr(self): - """Getter to mimic mp_conn.remote_addr. - - Setting the property in __init__ won't work because the request - handler is not initialized yet there.""" - - return self._request_handler.client_address - remote_addr = property(get_remote_addr) - - def write(self, data): - """Mimic mp_conn.write().""" - - return self._request_handler.wfile.write(data) - - def read(self, length): - """Mimic mp_conn.read().""" - - return self._request_handler.rfile.read(length) - - def get_memorized_lines(self): - """Get memorized lines.""" - - return self._request_handler.rfile.get_memorized_lines() - - -class _StandaloneRequest(object): - """Mimic mod_python request.""" - - def __init__(self, request_handler, use_tls): - """Construct an instance. - - Args: - request_handler: A WebSocketRequestHandler instance. - """ - - self._logger = util.get_class_logger(self) - - self._request_handler = request_handler - self.connection = _StandaloneConnection(request_handler) - self._use_tls = use_tls - self.headers_in = request_handler.headers - - def get_uri(self): - """Getter to mimic request.uri.""" - - return self._request_handler.path - uri = property(get_uri) - - def get_method(self): - """Getter to mimic request.method.""" - - return self._request_handler.command - method = property(get_method) - - def get_protocol(self): - """Getter to mimic request.protocol.""" - - return self._request_handler.request_version - protocol = property(get_protocol) - - def is_https(self): - """Mimic request.is_https().""" - - return self._use_tls - - def _drain_received_data(self): - """Don't use this method from WebSocket handler. Drains unread data - in the receive buffer. - """ - - raw_socket = self._request_handler.connection - drained_data = util.drain_received_data(raw_socket) - - if drained_data: - self._logger.debug( - 'Drained data following close frame: %r', drained_data) - - -class _StandaloneSSLConnection(object): - """A wrapper class for OpenSSL.SSL.Connection to provide makefile method - which is not supported by the class. - """ - - def __init__(self, connection): - self._connection = connection - - def __getattribute__(self, name): - if name in ('_connection', 'makefile'): - return object.__getattribute__(self, name) - return self._connection.__getattribute__(name) - - def __setattr__(self, name, value): - if name in ('_connection', 'makefile'): - return object.__setattr__(self, name, value) - return self._connection.__setattr__(name, value) - - def makefile(self, mode='r', bufsize=-1): - return socket._fileobject(self._connection, mode, bufsize) - - -def _alias_handlers(dispatcher, websock_handlers_map_file): - """Set aliases specified in websock_handler_map_file in dispatcher. - - Args: - dispatcher: dispatch.Dispatcher instance - websock_handler_map_file: alias map file - """ - - fp = open(websock_handlers_map_file) - try: - for line in fp: - if line[0] == '#' or line.isspace(): - continue - m = re.match('(\S+)\s+(\S+)', line) - if not m: - logging.warning('Wrong format in map file:' + line) - continue - try: - dispatcher.add_resource_path_alias( - m.group(1), m.group(2)) - except dispatch.DispatchException, e: - logging.error(str(e)) - finally: - fp.close() - - -class WebSocketServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): - """HTTPServer specialized for WebSocket.""" - - # Overrides SocketServer.ThreadingMixIn.daemon_threads - daemon_threads = True - # Overrides BaseHTTPServer.HTTPServer.allow_reuse_address - allow_reuse_address = True - - def __init__(self, options): - """Override SocketServer.TCPServer.__init__ to set SSL enabled - socket object to self.socket before server_bind and server_activate, - if necessary. - """ - - # Share a Dispatcher among request handlers to save time for - # instantiation. Dispatcher can be shared because it is thread-safe. - options.dispatcher = dispatch.Dispatcher( - options.websock_handlers, - options.scan_dir, - options.allow_handlers_outside_root_dir) - if options.websock_handlers_map_file: - _alias_handlers(options.dispatcher, - options.websock_handlers_map_file) - warnings = options.dispatcher.source_warnings() - if warnings: - for warning in warnings: - logging.warning('mod_pywebsocket: %s' % warning) - - self._logger = util.get_class_logger(self) - - self.request_queue_size = options.request_queue_size - self.__ws_is_shut_down = threading.Event() - self.__ws_serving = False - - SocketServer.BaseServer.__init__( - self, (options.server_host, options.port), WebSocketRequestHandler) - - # Expose the options object to allow handler objects access it. We name - # it with websocket_ prefix to avoid conflict. - self.websocket_server_options = options - - self._create_sockets() - self.server_bind() - self.server_activate() - - def _create_sockets(self): - self.server_name, self.server_port = self.server_address - self._sockets = [] - if not self.server_name: - # On platforms that doesn't support IPv6, the first bind fails. - # On platforms that supports IPv6 - # - If it binds both IPv4 and IPv6 on call with AF_INET6, the - # first bind succeeds and the second fails (we'll see 'Address - # already in use' error). - # - If it binds only IPv6 on call with AF_INET6, both call are - # expected to succeed to listen both protocol. - addrinfo_array = [ - (socket.AF_INET6, socket.SOCK_STREAM, '', '', ''), - (socket.AF_INET, socket.SOCK_STREAM, '', '', '')] - else: - addrinfo_array = socket.getaddrinfo(self.server_name, - self.server_port, - socket.AF_UNSPEC, - socket.SOCK_STREAM, - socket.IPPROTO_TCP) - for addrinfo in addrinfo_array: - self._logger.info('Create socket on: %r', addrinfo) - family, socktype, proto, canonname, sockaddr = addrinfo - try: - socket_ = socket.socket(family, socktype) - except Exception, e: - self._logger.info('Skip by failure: %r', e) - continue - if self.websocket_server_options.use_tls: - if _HAS_SSL: - if self.websocket_server_options.tls_client_auth: - client_cert_ = ssl.CERT_REQUIRED - else: - client_cert_ = ssl.CERT_NONE - socket_ = ssl.wrap_socket(socket_, - keyfile=self.websocket_server_options.private_key, - certfile=self.websocket_server_options.certificate, - ssl_version=ssl.PROTOCOL_SSLv23, - ca_certs=self.websocket_server_options.tls_client_ca, - cert_reqs=client_cert_) - if _HAS_OPEN_SSL: - ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) - ctx.use_privatekey_file( - self.websocket_server_options.private_key) - ctx.use_certificate_file( - self.websocket_server_options.certificate) - socket_ = OpenSSL.SSL.Connection(ctx, socket_) - self._sockets.append((socket_, addrinfo)) - - def server_bind(self): - """Override SocketServer.TCPServer.server_bind to enable multiple - sockets bind. - """ - - failed_sockets = [] - - for socketinfo in self._sockets: - socket_, addrinfo = socketinfo - self._logger.info('Bind on: %r', addrinfo) - if self.allow_reuse_address: - socket_.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - try: - socket_.bind(self.server_address) - except Exception, e: - self._logger.info('Skip by failure: %r', e) - socket_.close() - failed_sockets.append(socketinfo) - if self.server_address[1] == 0: - # The operating system assigns the actual port number for port - # number 0. This case, the second and later sockets should use - # the same port number. Also self.server_port is rewritten - # because it is exported, and will be used by external code. - self.server_address = ( - self.server_name, socket_.getsockname()[1]) - self.server_port = self.server_address[1] - self._logger.info('Port %r is assigned', self.server_port) - - for socketinfo in failed_sockets: - self._sockets.remove(socketinfo) - - def server_activate(self): - """Override SocketServer.TCPServer.server_activate to enable multiple - sockets listen. - """ - - failed_sockets = [] - - for socketinfo in self._sockets: - socket_, addrinfo = socketinfo - self._logger.info('Listen on: %r', addrinfo) - try: - socket_.listen(self.request_queue_size) - except Exception, e: - self._logger.info('Skip by failure: %r', e) - socket_.close() - failed_sockets.append(socketinfo) - - for socketinfo in failed_sockets: - self._sockets.remove(socketinfo) - - if len(self._sockets) == 0: - self._logger.critical( - 'No sockets activated. Use info log level to see the reason.') - - def server_close(self): - """Override SocketServer.TCPServer.server_close to enable multiple - sockets close. - """ - - for socketinfo in self._sockets: - socket_, addrinfo = socketinfo - self._logger.info('Close on: %r', addrinfo) - socket_.close() - - def fileno(self): - """Override SocketServer.TCPServer.fileno.""" - - self._logger.critical('Not supported: fileno') - return self._sockets[0][0].fileno() - - def handle_error(self, rquest, client_address): - """Override SocketServer.handle_error.""" - - self._logger.error( - 'Exception in processing request from: %r\n%s', - client_address, - util.get_stack_trace()) - # Note: client_address is a tuple. - - def get_request(self): - """Override TCPServer.get_request to wrap OpenSSL.SSL.Connection - object with _StandaloneSSLConnection to provide makefile method. We - cannot substitute OpenSSL.SSL.Connection.makefile since it's readonly - attribute. - """ - - accepted_socket, client_address = self.socket.accept() - if self.websocket_server_options.use_tls and _HAS_OPEN_SSL: - accepted_socket = _StandaloneSSLConnection(accepted_socket) - return accepted_socket, client_address - - def serve_forever(self, poll_interval=0.5): - """Override SocketServer.BaseServer.serve_forever.""" - - self.__ws_serving = True - self.__ws_is_shut_down.clear() - handle_request = self.handle_request - if hasattr(self, '_handle_request_noblock'): - handle_request = self._handle_request_noblock - else: - self._logger.warning('Fallback to blocking request handler') - try: - while self.__ws_serving: - r, w, e = select.select( - [socket_[0] for socket_ in self._sockets], - [], [], poll_interval) - for socket_ in r: - self.socket = socket_ - handle_request() - self.socket = None - finally: - self.__ws_is_shut_down.set() - - def shutdown(self): - """Override SocketServer.BaseServer.shutdown.""" - - self.__ws_serving = False - self.__ws_is_shut_down.wait() - - -class WebSocketRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler): - """CGIHTTPRequestHandler specialized for WebSocket.""" - - # Use httplib.HTTPMessage instead of mimetools.Message. - MessageClass = httplib.HTTPMessage - - def setup(self): - """Override SocketServer.StreamRequestHandler.setup to wrap rfile - with MemorizingFile. - - This method will be called by BaseRequestHandler's constructor - before calling BaseHTTPRequestHandler.handle. - BaseHTTPRequestHandler.handle will call - BaseHTTPRequestHandler.handle_one_request and it will call - WebSocketRequestHandler.parse_request. - """ - - # Call superclass's setup to prepare rfile, wfile, etc. See setup - # definition on the root class SocketServer.StreamRequestHandler to - # understand what this does. - CGIHTTPServer.CGIHTTPRequestHandler.setup(self) - - self.rfile = memorizingfile.MemorizingFile( - self.rfile, - max_memorized_lines=_MAX_MEMORIZED_LINES) - - def __init__(self, request, client_address, server): - self._logger = util.get_class_logger(self) - - self._options = server.websocket_server_options - - # Overrides CGIHTTPServerRequestHandler.cgi_directories. - self.cgi_directories = self._options.cgi_directories - # Replace CGIHTTPRequestHandler.is_executable method. - if self._options.is_executable_method is not None: - self.is_executable = self._options.is_executable_method - - # This actually calls BaseRequestHandler.__init__. - CGIHTTPServer.CGIHTTPRequestHandler.__init__( - self, request, client_address, server) - - def parse_request(self): - """Override BaseHTTPServer.BaseHTTPRequestHandler.parse_request. - - Return True to continue processing for HTTP(S), False otherwise. - - See BaseHTTPRequestHandler.handle_one_request method which calls - this method to understand how the return value will be handled. - """ - - # We hook parse_request method, but also call the original - # CGIHTTPRequestHandler.parse_request since when we return False, - # CGIHTTPRequestHandler.handle_one_request continues processing and - # it needs variables set by CGIHTTPRequestHandler.parse_request. - # - # Variables set by this method will be also used by WebSocket request - # handling (self.path, self.command, self.requestline, etc. See also - # how _StandaloneRequest's members are implemented using these - # attributes). - if not CGIHTTPServer.CGIHTTPRequestHandler.parse_request(self): - return False - - if self._options.use_basic_auth: - auth = self.headers.getheader('Authorization') - if auth != self._options.basic_auth_credential: - self.send_response(401) - self.send_header('WWW-Authenticate', - 'Basic realm="Pywebsocket"') - self.end_headers() - self._logger.info('Request basic authentication') - return True - - host, port, resource = http_header_util.parse_uri(self.path) - if resource is None: - self._logger.info('Invalid URI: %r', self.path) - self._logger.info('Fallback to CGIHTTPRequestHandler') - return True - server_options = self.server.websocket_server_options - if host is not None: - validation_host = server_options.validation_host - if validation_host is not None and host != validation_host: - self._logger.info('Invalid host: %r (expected: %r)', - host, - validation_host) - self._logger.info('Fallback to CGIHTTPRequestHandler') - return True - if port is not None: - validation_port = server_options.validation_port - if validation_port is not None and port != validation_port: - self._logger.info('Invalid port: %r (expected: %r)', - port, - validation_port) - self._logger.info('Fallback to CGIHTTPRequestHandler') - return True - self.path = resource - - request = _StandaloneRequest(self, self._options.use_tls) - - try: - # Fallback to default http handler for request paths for which - # we don't have request handlers. - if not self._options.dispatcher.get_handler_suite(self.path): - self._logger.info('No handler for resource: %r', - self.path) - self._logger.info('Fallback to CGIHTTPRequestHandler') - return True - except dispatch.DispatchException, e: - self._logger.info('%s', e) - self.send_error(e.status) - return False - - # If any Exceptions without except clause setup (including - # DispatchException) is raised below this point, it will be caught - # and logged by WebSocketServer. - - try: - try: - handshake.do_handshake( - request, - self._options.dispatcher, - allowDraft75=self._options.allow_draft75, - strict=self._options.strict) - except handshake.VersionException, e: - self._logger.info('%s', e) - self.send_response(common.HTTP_STATUS_BAD_REQUEST) - self.send_header(common.SEC_WEBSOCKET_VERSION_HEADER, - e.supported_versions) - self.end_headers() - return False - except handshake.HandshakeException, e: - # Handshake for ws(s) failed. - self._logger.info('%s', e) - self.send_error(e.status) - return False - - request._dispatcher = self._options.dispatcher - self._options.dispatcher.transfer_data(request) - except handshake.AbortedByUserException, e: - self._logger.info('%s', e) - return False - - def log_request(self, code='-', size='-'): - """Override BaseHTTPServer.log_request.""" - - self._logger.info('"%s" %s %s', - self.requestline, str(code), str(size)) - - def log_error(self, *args): - """Override BaseHTTPServer.log_error.""" - - # Despite the name, this method is for warnings than for errors. - # For example, HTTP status code is logged by this method. - self._logger.warning('%s - %s', - self.address_string(), - args[0] % args[1:]) - - def is_cgi(self): - """Test whether self.path corresponds to a CGI script. - - Add extra check that self.path doesn't contains .. - Also check if the file is a executable file or not. - If the file is not executable, it is handled as static file or dir - rather than a CGI script. - """ - - if CGIHTTPServer.CGIHTTPRequestHandler.is_cgi(self): - if '..' in self.path: - return False - # strip query parameter from request path - resource_name = self.path.split('?', 2)[0] - # convert resource_name into real path name in filesystem. - scriptfile = self.translate_path(resource_name) - if not os.path.isfile(scriptfile): - return False - if not self.is_executable(scriptfile): - return False - return True - return False - - -def _get_logger_from_class(c): - return logging.getLogger('%s.%s' % (c.__module__, c.__name__)) - - -def _configure_logging(options): - logging.addLevelName(common.LOGLEVEL_FINE, 'FINE') - - logger = logging.getLogger() - logger.setLevel(logging.getLevelName(options.log_level.upper())) - if options.log_file: - handler = logging.handlers.RotatingFileHandler( - options.log_file, 'a', options.log_max, options.log_count) - else: - handler = logging.StreamHandler() - formatter = logging.Formatter( - '[%(asctime)s] [%(levelname)s] %(name)s: %(message)s') - handler.setFormatter(formatter) - logger.addHandler(handler) - - deflate_log_level_name = logging.getLevelName( - options.deflate_log_level.upper()) - _get_logger_from_class(util._Deflater).setLevel( - deflate_log_level_name) - _get_logger_from_class(util._Inflater).setLevel( - deflate_log_level_name) - - -def _build_option_parser(): - parser = optparse.OptionParser() - - parser.add_option('--config', dest='config_file', type='string', - default=None, - help=('Path to configuration file. See the file comment ' - 'at the top of this file for the configuration ' - 'file format')) - parser.add_option('-H', '--server-host', '--server_host', - dest='server_host', - default='', - help='server hostname to listen to') - parser.add_option('-V', '--validation-host', '--validation_host', - dest='validation_host', - default=None, - help='server hostname to validate in absolute path.') - parser.add_option('-p', '--port', dest='port', type='int', - default=common.DEFAULT_WEB_SOCKET_PORT, - help='port to listen to') - parser.add_option('-P', '--validation-port', '--validation_port', - dest='validation_port', type='int', - default=None, - help='server port to validate in absolute path.') - parser.add_option('-w', '--websock-handlers', '--websock_handlers', - dest='websock_handlers', - default='.', - help=('The root directory of WebSocket handler files. ' - 'If the path is relative, --document-root is used ' - 'as the base.')) - parser.add_option('-m', '--websock-handlers-map-file', - '--websock_handlers_map_file', - dest='websock_handlers_map_file', - default=None, - help=('WebSocket handlers map file. ' - 'Each line consists of alias_resource_path and ' - 'existing_resource_path, separated by spaces.')) - parser.add_option('-s', '--scan-dir', '--scan_dir', dest='scan_dir', - default=None, - help=('Must be a directory under --websock-handlers. ' - 'Only handlers under this directory are scanned ' - 'and registered to the server. ' - 'Useful for saving scan time when the handler ' - 'root directory contains lots of files that are ' - 'not handler file or are handler files but you ' - 'don\'t want them to be registered. ')) - parser.add_option('--allow-handlers-outside-root-dir', - '--allow_handlers_outside_root_dir', - dest='allow_handlers_outside_root_dir', - action='store_true', - default=False, - help=('Scans WebSocket handlers even if their canonical ' - 'path is not under --websock-handlers.')) - parser.add_option('-d', '--document-root', '--document_root', - dest='document_root', default='.', - help='Document root directory.') - parser.add_option('-x', '--cgi-paths', '--cgi_paths', dest='cgi_paths', - default=None, - help=('CGI paths relative to document_root.' - 'Comma-separated. (e.g -x /cgi,/htbin) ' - 'Files under document_root/cgi_path are handled ' - 'as CGI programs. Must be executable.')) - parser.add_option('-t', '--tls', dest='use_tls', action='store_true', - default=False, help='use TLS (wss://)') - parser.add_option('-k', '--private-key', '--private_key', - dest='private_key', - default='', help='TLS private key file.') - parser.add_option('-c', '--certificate', dest='certificate', - default='', help='TLS certificate file.') - parser.add_option('--tls-client-auth', dest='tls_client_auth', - action='store_true', default=False, - help='Requires TLS client auth on every connection.') - parser.add_option('--tls-client-ca', dest='tls_client_ca', default='', - help=('Specifies a pem file which contains a set of ' - 'concatenated CA certificates which are used to ' - 'validate certificates passed from clients')) - parser.add_option('--basic-auth', dest='use_basic_auth', - action='store_true', default=False, - help='Requires Basic authentication.') - parser.add_option('--basic-auth-credential', - dest='basic_auth_credential', default='test:test', - help='Specifies the credential of basic authentication ' - 'by username:password pair (e.g. test:test).') - parser.add_option('-l', '--log-file', '--log_file', dest='log_file', - default='', help='Log file.') - # Custom log level: - # - FINE: Prints status of each frame processing step - parser.add_option('--log-level', '--log_level', type='choice', - dest='log_level', default='warn', - choices=['fine', - 'debug', 'info', 'warning', 'warn', 'error', - 'critical'], - help='Log level.') - parser.add_option('--deflate-log-level', '--deflate_log_level', - type='choice', - dest='deflate_log_level', default='warn', - choices=['debug', 'info', 'warning', 'warn', 'error', - 'critical'], - help='Log level for _Deflater and _Inflater.') - parser.add_option('--thread-monitor-interval-in-sec', - '--thread_monitor_interval_in_sec', - dest='thread_monitor_interval_in_sec', - type='int', default=-1, - help=('If positive integer is specified, run a thread ' - 'monitor to show the status of server threads ' - 'periodically in the specified inteval in ' - 'second. If non-positive integer is specified, ' - 'disable the thread monitor.')) - parser.add_option('--log-max', '--log_max', dest='log_max', type='int', - default=_DEFAULT_LOG_MAX_BYTES, - help='Log maximum bytes') - parser.add_option('--log-count', '--log_count', dest='log_count', - type='int', default=_DEFAULT_LOG_BACKUP_COUNT, - help='Log backup count') - parser.add_option('--allow-draft75', dest='allow_draft75', - action='store_true', default=False, - help='Obsolete option. Ignored.') - parser.add_option('--strict', dest='strict', action='store_true', - default=False, help='Obsolete option. Ignored.') - parser.add_option('-q', '--queue', dest='request_queue_size', type='int', - default=_DEFAULT_REQUEST_QUEUE_SIZE, - help='request queue size') - - return parser - - -class ThreadMonitor(threading.Thread): - daemon = True - - def __init__(self, interval_in_sec): - threading.Thread.__init__(self, name='ThreadMonitor') - - self._logger = util.get_class_logger(self) - - self._interval_in_sec = interval_in_sec - - def run(self): - while True: - thread_name_list = [] - for thread in threading.enumerate(): - thread_name_list.append(thread.name) - self._logger.info( - "%d active threads: %s", - threading.active_count(), - ', '.join(thread_name_list)) - time.sleep(self._interval_in_sec) - - -def _parse_args_and_config(args): - parser = _build_option_parser() - - # First, parse options without configuration file. - temporary_options, temporary_args = parser.parse_args(args=args) - if temporary_args: - logging.critical( - 'Unrecognized positional arguments: %r', temporary_args) - sys.exit(1) - - if temporary_options.config_file: - try: - config_fp = open(temporary_options.config_file, 'r') - except IOError, e: - logging.critical( - 'Failed to open configuration file %r: %r', - temporary_options.config_file, - e) - sys.exit(1) - - config_parser = ConfigParser.SafeConfigParser() - config_parser.readfp(config_fp) - config_fp.close() - - args_from_config = [] - for name, value in config_parser.items('pywebsocket'): - args_from_config.append('--' + name) - args_from_config.append(value) - if args is None: - args = args_from_config - else: - args = args_from_config + args - return parser.parse_args(args=args) - else: - return temporary_options, temporary_args - - -def _main(args=None): - """You can call this function from your own program, but please note that - this function has some side-effects that might affect your program. For - example, util.wrap_popen3_for_win use in this method replaces implementation - of os.popen3. - """ - - options, args = _parse_args_and_config(args=args) - - os.chdir(options.document_root) - - _configure_logging(options) - - # TODO(tyoshino): Clean up initialization of CGI related values. Move some - # of code here to WebSocketRequestHandler class if it's better. - options.cgi_directories = [] - options.is_executable_method = None - if options.cgi_paths: - options.cgi_directories = options.cgi_paths.split(',') - if sys.platform in ('cygwin', 'win32'): - cygwin_path = None - # For Win32 Python, it is expected that CYGWIN_PATH - # is set to a directory of cygwin binaries. - # For example, websocket_server.py in Chromium sets CYGWIN_PATH to - # full path of third_party/cygwin/bin. - if 'CYGWIN_PATH' in os.environ: - cygwin_path = os.environ['CYGWIN_PATH'] - util.wrap_popen3_for_win(cygwin_path) - - def __check_script(scriptpath): - return util.get_script_interp(scriptpath, cygwin_path) - - options.is_executable_method = __check_script - - if options.use_tls: - if not (_HAS_SSL or _HAS_OPEN_SSL): - logging.critical('TLS support requires ssl or pyOpenSSL module.') - sys.exit(1) - if not options.private_key or not options.certificate: - logging.critical( - 'To use TLS, specify private_key and certificate.') - sys.exit(1) - - if options.tls_client_auth: - if not options.use_tls: - logging.critical('TLS must be enabled for client authentication.') - sys.exit(1) - if not _HAS_SSL: - logging.critical('Client authentication requires ssl module.') - - if not options.scan_dir: - options.scan_dir = options.websock_handlers - - if options.use_basic_auth: - options.basic_auth_credential = 'Basic ' + base64.b64encode( - options.basic_auth_credential) - - try: - if options.thread_monitor_interval_in_sec > 0: - # Run a thread monitor to show the status of server threads for - # debugging. - ThreadMonitor(options.thread_monitor_interval_in_sec).start() - - server = WebSocketServer(options) - server.serve_forever() - except Exception, e: - logging.critical('mod_pywebsocket: %s' % e) - logging.critical('mod_pywebsocket: %s' % util.get_stack_trace()) - sys.exit(1) - - -if __name__ == '__main__': - _main(sys.argv[1:]) - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/stream.py b/module/lib/mod_pywebsocket/stream.py deleted file mode 100644 index edc533279..000000000 --- a/module/lib/mod_pywebsocket/stream.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""This file exports public symbols. -""" - - -from mod_pywebsocket._stream_base import BadOperationException -from mod_pywebsocket._stream_base import ConnectionTerminatedException -from mod_pywebsocket._stream_base import InvalidFrameException -from mod_pywebsocket._stream_base import InvalidUTF8Exception -from mod_pywebsocket._stream_base import UnsupportedFrameException -from mod_pywebsocket._stream_hixie75 import StreamHixie75 -from mod_pywebsocket._stream_hybi import Frame -from mod_pywebsocket._stream_hybi import Stream -from mod_pywebsocket._stream_hybi import StreamOptions - -# These methods are intended to be used by WebSocket client developers to have -# their implementations receive broken data in tests. -from mod_pywebsocket._stream_hybi import create_close_frame -from mod_pywebsocket._stream_hybi import create_header -from mod_pywebsocket._stream_hybi import create_length_header -from mod_pywebsocket._stream_hybi import create_ping_frame -from mod_pywebsocket._stream_hybi import create_pong_frame -from mod_pywebsocket._stream_hybi import create_binary_frame -from mod_pywebsocket._stream_hybi import create_text_frame -from mod_pywebsocket._stream_hybi import create_closing_handshake_body - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/util.py b/module/lib/mod_pywebsocket/util.py deleted file mode 100644 index 7bb0b5d9e..000000000 --- a/module/lib/mod_pywebsocket/util.py +++ /dev/null @@ -1,515 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""WebSocket utilities. -""" - - -import array -import errno - -# Import hash classes from a module available and recommended for each Python -# version and re-export those symbol. Use sha and md5 module in Python 2.4, and -# hashlib module in Python 2.6. -try: - import hashlib - md5_hash = hashlib.md5 - sha1_hash = hashlib.sha1 -except ImportError: - import md5 - import sha - md5_hash = md5.md5 - sha1_hash = sha.sha - -import StringIO -import logging -import os -import re -import socket -import traceback -import zlib - - -def get_stack_trace(): - """Get the current stack trace as string. - - This is needed to support Python 2.3. - TODO: Remove this when we only support Python 2.4 and above. - Use traceback.format_exc instead. - """ - - out = StringIO.StringIO() - traceback.print_exc(file=out) - return out.getvalue() - - -def prepend_message_to_exception(message, exc): - """Prepend message to the exception.""" - - exc.args = (message + str(exc),) - return - - -def __translate_interp(interp, cygwin_path): - """Translate interp program path for Win32 python to run cygwin program - (e.g. perl). Note that it doesn't support path that contains space, - which is typically true for Unix, where #!-script is written. - For Win32 python, cygwin_path is a directory of cygwin binaries. - - Args: - interp: interp command line - cygwin_path: directory name of cygwin binary, or None - Returns: - translated interp command line. - """ - if not cygwin_path: - return interp - m = re.match('^[^ ]*/([^ ]+)( .*)?', interp) - if m: - cmd = os.path.join(cygwin_path, m.group(1)) - return cmd + m.group(2) - return interp - - -def get_script_interp(script_path, cygwin_path=None): - """Gets #!-interpreter command line from the script. - - It also fixes command path. When Cygwin Python is used, e.g. in WebKit, - it could run "/usr/bin/perl -wT hello.pl". - When Win32 Python is used, e.g. in Chromium, it couldn't. So, fix - "/usr/bin/perl" to "\perl.exe". - - Args: - script_path: pathname of the script - cygwin_path: directory name of cygwin binary, or None - Returns: - #!-interpreter command line, or None if it is not #!-script. - """ - fp = open(script_path) - line = fp.readline() - fp.close() - m = re.match('^#!(.*)', line) - if m: - return __translate_interp(m.group(1), cygwin_path) - return None - - -def wrap_popen3_for_win(cygwin_path): - """Wrap popen3 to support #!-script on Windows. - - Args: - cygwin_path: path for cygwin binary if command path is needed to be - translated. None if no translation required. - """ - - __orig_popen3 = os.popen3 - - def __wrap_popen3(cmd, mode='t', bufsize=-1): - cmdline = cmd.split(' ') - interp = get_script_interp(cmdline[0], cygwin_path) - if interp: - cmd = interp + ' ' + cmd - return __orig_popen3(cmd, mode, bufsize) - - os.popen3 = __wrap_popen3 - - -def hexify(s): - return ' '.join(map(lambda x: '%02x' % ord(x), s)) - - -def get_class_logger(o): - return logging.getLogger( - '%s.%s' % (o.__class__.__module__, o.__class__.__name__)) - - -class NoopMasker(object): - """A masking object that has the same interface as RepeatedXorMasker but - just returns the string passed in without making any change. - """ - - def __init__(self): - pass - - def mask(self, s): - return s - - -class RepeatedXorMasker(object): - """A masking object that applies XOR on the string given to mask method - with the masking bytes given to the constructor repeatedly. This object - remembers the position in the masking bytes the last mask method call - ended and resumes from that point on the next mask method call. - """ - - def __init__(self, mask): - self._mask = map(ord, mask) - self._mask_size = len(self._mask) - self._count = 0 - - def mask(self, s): - result = array.array('B') - result.fromstring(s) - # Use temporary local variables to eliminate the cost to access - # attributes - count = self._count - mask = self._mask - mask_size = self._mask_size - for i in xrange(len(result)): - result[i] ^= mask[count] - count = (count + 1) % mask_size - self._count = count - - return result.tostring() - - -class DeflateRequest(object): - """A wrapper class for request object to intercept send and recv to perform - deflate compression and decompression transparently. - """ - - def __init__(self, request): - self._request = request - self.connection = DeflateConnection(request.connection) - - def __getattribute__(self, name): - if name in ('_request', 'connection'): - return object.__getattribute__(self, name) - return self._request.__getattribute__(name) - - def __setattr__(self, name, value): - if name in ('_request', 'connection'): - return object.__setattr__(self, name, value) - return self._request.__setattr__(name, value) - - -# By making wbits option negative, we can suppress CMF/FLG (2 octet) and -# ADLER32 (4 octet) fields of zlib so that we can use zlib module just as -# deflate library. DICTID won't be added as far as we don't set dictionary. -# LZ77 window of 32K will be used for both compression and decompression. -# For decompression, we can just use 32K to cover any windows size. For -# compression, we use 32K so receivers must use 32K. -# -# Compression level is Z_DEFAULT_COMPRESSION. We don't have to match level -# to decode. -# -# See zconf.h, deflate.cc, inflate.cc of zlib library, and zlibmodule.c of -# Python. See also RFC1950 (ZLIB 3.3). - - -class _Deflater(object): - - def __init__(self, window_bits): - self._logger = get_class_logger(self) - - self._compress = zlib.compressobj( - zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -window_bits) - - def compress(self, bytes): - compressed_bytes = self._compress.compress(bytes) - self._logger.debug('Compress input %r', bytes) - self._logger.debug('Compress result %r', compressed_bytes) - return compressed_bytes - - def compress_and_flush(self, bytes): - compressed_bytes = self._compress.compress(bytes) - compressed_bytes += self._compress.flush(zlib.Z_SYNC_FLUSH) - self._logger.debug('Compress input %r', bytes) - self._logger.debug('Compress result %r', compressed_bytes) - return compressed_bytes - - def compress_and_finish(self, bytes): - compressed_bytes = self._compress.compress(bytes) - compressed_bytes += self._compress.flush(zlib.Z_FINISH) - self._logger.debug('Compress input %r', bytes) - self._logger.debug('Compress result %r', compressed_bytes) - return compressed_bytes - -class _Inflater(object): - - def __init__(self): - self._logger = get_class_logger(self) - - self._unconsumed = '' - - self.reset() - - def decompress(self, size): - if not (size == -1 or size > 0): - raise Exception('size must be -1 or positive') - - data = '' - - while True: - if size == -1: - data += self._decompress.decompress(self._unconsumed) - # See Python bug http://bugs.python.org/issue12050 to - # understand why the same code cannot be used for updating - # self._unconsumed for here and else block. - self._unconsumed = '' - else: - data += self._decompress.decompress( - self._unconsumed, size - len(data)) - self._unconsumed = self._decompress.unconsumed_tail - if self._decompress.unused_data: - # Encountered a last block (i.e. a block with BFINAL = 1) and - # found a new stream (unused_data). We cannot use the same - # zlib.Decompress object for the new stream. Create a new - # Decompress object to decompress the new one. - # - # It's fine to ignore unconsumed_tail if unused_data is not - # empty. - self._unconsumed = self._decompress.unused_data - self.reset() - if size >= 0 and len(data) == size: - # data is filled. Don't call decompress again. - break - else: - # Re-invoke Decompress.decompress to try to decompress all - # available bytes before invoking read which blocks until - # any new byte is available. - continue - else: - # Here, since unused_data is empty, even if unconsumed_tail is - # not empty, bytes of requested length are already in data. We - # don't have to "continue" here. - break - - if data: - self._logger.debug('Decompressed %r', data) - return data - - def append(self, data): - self._logger.debug('Appended %r', data) - self._unconsumed += data - - def reset(self): - self._logger.debug('Reset') - self._decompress = zlib.decompressobj(-zlib.MAX_WBITS) - - -# Compresses/decompresses given octets using the method introduced in RFC1979. - - -class _RFC1979Deflater(object): - """A compressor class that applies DEFLATE to given byte sequence and - flushes using the algorithm described in the RFC1979 section 2.1. - """ - - def __init__(self, window_bits, no_context_takeover): - self._deflater = None - if window_bits is None: - window_bits = zlib.MAX_WBITS - self._window_bits = window_bits - self._no_context_takeover = no_context_takeover - - def filter(self, bytes, flush=True, bfinal=False): - if self._deflater is None or (self._no_context_takeover and flush): - self._deflater = _Deflater(self._window_bits) - - if bfinal: - result = self._deflater.compress_and_finish(bytes) - # Add a padding block with BFINAL = 0 and BTYPE = 0. - result = result + chr(0) - self._deflater = None - return result - if flush: - # Strip last 4 octets which is LEN and NLEN field of a - # non-compressed block added for Z_SYNC_FLUSH. - return self._deflater.compress_and_flush(bytes)[:-4] - return self._deflater.compress(bytes) - -class _RFC1979Inflater(object): - """A decompressor class for byte sequence compressed and flushed following - the algorithm described in the RFC1979 section 2.1. - """ - - def __init__(self): - self._inflater = _Inflater() - - def filter(self, bytes): - # Restore stripped LEN and NLEN field of a non-compressed block added - # for Z_SYNC_FLUSH. - self._inflater.append(bytes + '\x00\x00\xff\xff') - return self._inflater.decompress(-1) - - -class DeflateSocket(object): - """A wrapper class for socket object to intercept send and recv to perform - deflate compression and decompression transparently. - """ - - # Size of the buffer passed to recv to receive compressed data. - _RECV_SIZE = 4096 - - def __init__(self, socket): - self._socket = socket - - self._logger = get_class_logger(self) - - self._deflater = _Deflater(zlib.MAX_WBITS) - self._inflater = _Inflater() - - def recv(self, size): - """Receives data from the socket specified on the construction up - to the specified size. Once any data is available, returns it even - if it's smaller than the specified size. - """ - - # TODO(tyoshino): Allow call with size=0. It should block until any - # decompressed data is available. - if size <= 0: - raise Exception('Non-positive size passed') - while True: - data = self._inflater.decompress(size) - if len(data) != 0: - return data - - read_data = self._socket.recv(DeflateSocket._RECV_SIZE) - if not read_data: - return '' - self._inflater.append(read_data) - - def sendall(self, bytes): - self.send(bytes) - - def send(self, bytes): - self._socket.sendall(self._deflater.compress_and_flush(bytes)) - return len(bytes) - - -class DeflateConnection(object): - """A wrapper class for request object to intercept write and read to - perform deflate compression and decompression transparently. - """ - - def __init__(self, connection): - self._connection = connection - - self._logger = get_class_logger(self) - - self._deflater = _Deflater(zlib.MAX_WBITS) - self._inflater = _Inflater() - - def get_remote_addr(self): - return self._connection.remote_addr - remote_addr = property(get_remote_addr) - - def put_bytes(self, bytes): - self.write(bytes) - - def read(self, size=-1): - """Reads at most size bytes. Blocks until there's at least one byte - available. - """ - - # TODO(tyoshino): Allow call with size=0. - if not (size == -1 or size > 0): - raise Exception('size must be -1 or positive') - - data = '' - while True: - if size == -1: - data += self._inflater.decompress(-1) - else: - data += self._inflater.decompress(size - len(data)) - - if size >= 0 and len(data) != 0: - break - - # TODO(tyoshino): Make this read efficient by some workaround. - # - # In 3.0.3 and prior of mod_python, read blocks until length bytes - # was read. We don't know the exact size to read while using - # deflate, so read byte-by-byte. - # - # _StandaloneRequest.read that ultimately performs - # socket._fileobject.read also blocks until length bytes was read - read_data = self._connection.read(1) - if not read_data: - break - self._inflater.append(read_data) - return data - - def write(self, bytes): - self._connection.write(self._deflater.compress_and_flush(bytes)) - - -def _is_ewouldblock_errno(error_number): - """Returns True iff error_number indicates that receive operation would - block. To make this portable, we check availability of errno and then - compare them. - """ - - for error_name in ['WSAEWOULDBLOCK', 'EWOULDBLOCK', 'EAGAIN']: - if (error_name in dir(errno) and - error_number == getattr(errno, error_name)): - return True - return False - - -def drain_received_data(raw_socket): - # Set the socket non-blocking. - original_timeout = raw_socket.gettimeout() - raw_socket.settimeout(0.0) - - drained_data = [] - - # Drain until the socket is closed or no data is immediately - # available for read. - while True: - try: - data = raw_socket.recv(1) - if not data: - break - drained_data.append(data) - except socket.error, e: - # e can be either a pair (errno, string) or just a string (or - # something else) telling what went wrong. We suppress only - # the errors that indicates that the socket blocks. Those - # exceptions can be parsed as a pair (errno, string). - try: - error_number, message = e - except: - # Failed to parse socket.error. - raise e - - if _is_ewouldblock_errno(error_number): - break - else: - raise e - - # Rollback timeout value. - raw_socket.settimeout(original_timeout) - - return ''.join(drained_data) - - -# vi:sts=4 sw=4 et diff --git a/module/lib/new_collections.py b/module/lib/new_collections.py deleted file mode 100644 index 12d05b4b9..000000000 --- a/module/lib/new_collections.py +++ /dev/null @@ -1,375 +0,0 @@ -## {{{ http://code.activestate.com/recipes/576693/ (r9) -# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. -# Passes Python2.7's test suite and incorporates all the latest updates. - -try: - from thread import get_ident as _get_ident -except ImportError: - from dummy_thread import get_ident as _get_ident - -try: - from _abcoll import KeysView, ValuesView, ItemsView -except ImportError: - pass - - -class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as for regular dictionaries. - - # The internal self.__map dictionary maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. - - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. Signature is the same as for - regular dictionaries, but keyword arguments are not recommended - because their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link which goes at the end of the linked - # list, and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which is - # then removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, key = self.__map.pop(key) - link_prev[1] = link_next - link_next[0] = link_prev - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - root = self.__root - curr = root[1] - while curr is not root: - yield curr[2] - curr = curr[1] - - def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - root = self.__root - curr = root[0] - while curr is not root: - yield curr[2] - curr = curr[0] - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - try: - for node in self.__map.itervalues(): - del node[:] - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - except AttributeError: - pass - dict.clear(self) - - def popitem(self, last=True): - '''od.popitem() -> (k, v), return and remove a (key, value) pair. - Pairs are returned in LIFO order if last is true or FIFO order if false. - - ''' - if not self: - raise KeyError('dictionary is empty') - root = self.__root - if last: - link = root[0] - link_prev = link[0] - link_prev[1] = root - root[0] = link_prev - else: - link = root[1] - link_next = link[1] - root[1] = link_next - link_next[0] = root - key = link[2] - del self.__map[key] - value = dict.pop(self, key) - return key, value - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) items in od' - for k in self: - yield (k, self[k]) - - def update(*args, **kwds): - '''od.update(E, **F) -> None. Update od from dict/iterable E and F. - - If E is a dict instance, does: for k in E: od[k] = E[k] - If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] - Or if E is an iterable of items, does: for k, v in E: od[k] = v - In either case, this is followed by: for k, v in F.items(): od[k] = v - - ''' - if len(args) > 2: - raise TypeError('update() takes at most 2 positional ' - 'arguments (%d given)' % (len(args),)) - elif not args: - raise TypeError('update() takes at least 1 argument (0 given)') - self = args[0] - # Make progressively weaker assumptions about "other" - other = () - if len(args) == 2: - other = args[1] - if isinstance(other, dict): - for key in other: - self[key] = other[key] - elif hasattr(other, 'keys'): - for key in other.keys(): - self[key] = other[key] - else: - for key, value in other: - self[key] = value - for key, value in kwds.items(): - self[key] = value - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default - - def __repr__(self, _repr_running={}): - 'od.__repr__() <==> repr(od)' - call_key = id(self), _get_ident() - if call_key in _repr_running: - return '...' - _repr_running[call_key] = 1 - try: - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - finally: - del _repr_running[call_key] - - def __reduce__(self): - 'Return state information for pickling' - items = [[k, self[k]] for k in self] - inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def copy(self): - 'od.copy() -> a shallow copy of od' - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S - and values equal to v (which defaults to None). - - ''' - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive - while comparison to a regular mapping is order-insensitive. - - ''' - if isinstance(other, OrderedDict): - return len(self)==len(other) and self.items() == other.items() - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other - - # -- the following methods are only used in Python 2.7 -- - - def viewkeys(self): - "od.viewkeys() -> a set-like object providing a view on od's keys" - return KeysView(self) - - def viewvalues(self): - "od.viewvalues() -> an object providing a view on od's values" - return ValuesView(self) - - def viewitems(self): - "od.viewitems() -> a set-like object providing a view on od's items" - return ItemsView(self) -## end of http://code.activestate.com/recipes/576693/ }}} - -## {{{ http://code.activestate.com/recipes/500261/ (r15) -from operator import itemgetter as _itemgetter -from keyword import iskeyword as _iskeyword -import sys as _sys - -def namedtuple(typename, field_names, verbose=False, rename=False): - """Returns a new subclass of tuple with named fields. - - >>> Point = namedtuple('Point', 'x y') - >>> Point.__doc__ # docstring for the new class - 'Point(x, y)' - >>> p = Point(11, y=22) # instantiate with positional args or keywords - >>> p[0] + p[1] # indexable like a plain tuple - 33 - >>> x, y = p # unpack like a regular tuple - >>> x, y - (11, 22) - >>> p.x + p.y # fields also accessable by name - 33 - >>> d = p._asdict() # convert to a dictionary - >>> d['x'] - 11 - >>> Point(**d) # convert from a dictionary - Point(x=11, y=22) - >>> p._replace(x=100) # _replace() is like str.replace() but targets named fields - Point(x=100, y=22) - - """ - - # Parse and validate the field names. Validation serves two purposes, - # generating informative error messages and preventing template injection attacks. - if isinstance(field_names, basestring): - field_names = field_names.replace(',', ' ').split() # names separated by whitespace and/or commas - field_names = tuple(map(str, field_names)) - if rename: - names = list(field_names) - seen = set() - for i, name in enumerate(names): - if (not min(c.isalnum() or c=='_' for c in name) or _iskeyword(name) - or not name or name[0].isdigit() or name.startswith('_') - or name in seen): - names[i] = '_%d' % i - seen.add(name) - field_names = tuple(names) - for name in (typename,) + field_names: - if not min(c.isalnum() or c=='_' for c in name): - raise ValueError('Type names and field names can only contain alphanumeric characters and underscores: %r' % name) - if _iskeyword(name): - raise ValueError('Type names and field names cannot be a keyword: %r' % name) - if name[0].isdigit(): - raise ValueError('Type names and field names cannot start with a number: %r' % name) - seen_names = set() - for name in field_names: - if name.startswith('_') and not rename: - raise ValueError('Field names cannot start with an underscore: %r' % name) - if name in seen_names: - raise ValueError('Encountered duplicate field name: %r' % name) - seen_names.add(name) - - # Create and fill-in the class template - numfields = len(field_names) - argtxt = repr(field_names).replace("'", "")[1:-1] # tuple repr without parens or quotes - reprtxt = ', '.join('%s=%%r' % name for name in field_names) - template = '''class %(typename)s(tuple): - '%(typename)s(%(argtxt)s)' \n - __slots__ = () \n - _fields = %(field_names)r \n - def __new__(_cls, %(argtxt)s): - return _tuple.__new__(_cls, (%(argtxt)s)) \n - @classmethod - def _make(cls, iterable, new=tuple.__new__, len=len): - 'Make a new %(typename)s object from a sequence or iterable' - result = new(cls, iterable) - if len(result) != %(numfields)d: - raise TypeError('Expected %(numfields)d arguments, got %%d' %% len(result)) - return result \n - def __repr__(self): - return '%(typename)s(%(reprtxt)s)' %% self \n - def _asdict(self): - 'Return a new dict which maps field names to their values' - return dict(zip(self._fields, self)) \n - def _replace(_self, **kwds): - 'Return a new %(typename)s object replacing specified fields with new values' - result = _self._make(map(kwds.pop, %(field_names)r, _self)) - if kwds: - raise ValueError('Got unexpected field names: %%r' %% kwds.keys()) - return result \n - def __getnewargs__(self): - return tuple(self) \n\n''' % locals() - for i, name in enumerate(field_names): - template += ' %s = _property(_itemgetter(%d))\n' % (name, i) - if verbose: - print template - - # Execute the template string in a temporary namespace - namespace = dict(_itemgetter=_itemgetter, __name__='namedtuple_%s' % typename, - _property=property, _tuple=tuple) - try: - exec template in namespace - except SyntaxError, e: - raise SyntaxError(e.message + ':\n' + template) - result = namespace[typename] - - # For pickling to work, the __module__ variable needs to be set to the frame - # where the named tuple is created. Bypass this step in enviroments where - # sys._getframe is not defined (Jython for example) or sys._getframe is not - # defined for arguments greater than 0 (IronPython). - try: - result.__module__ = _sys._getframe(1).f_globals.get('__name__', '__main__') - except (AttributeError, ValueError): - pass - - return result -## end of http://code.activestate.com/recipes/500261/ }}} diff --git a/module/lib/rename_process.py b/module/lib/rename_process.py deleted file mode 100644 index 2527cef39..000000000 --- a/module/lib/rename_process.py +++ /dev/null @@ -1,14 +0,0 @@ -import sys - -def renameProcess(new_name): - """ Renames the process calling the function to the given name. """ - if sys.platform != 'linux2': - return False - try: - from ctypes import CDLL - libc = CDLL('libc.so.6') - libc.prctl(15, new_name, 0, 0, 0) - return True - except Exception, e: - #print "Rename process failed", e - return False diff --git a/module/lib/simplejson/__init__.py b/module/lib/simplejson/__init__.py deleted file mode 100644 index ef5c0db48..000000000 --- a/module/lib/simplejson/__init__.py +++ /dev/null @@ -1,466 +0,0 @@ -r"""JSON (JavaScript Object Notation) is a subset of -JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data -interchange format. - -:mod:`simplejson` exposes an API familiar to users of the standard library -:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained -version of the :mod:`json` library contained in Python 2.6, but maintains -compatibility with Python 2.4 and Python 2.5 and (currently) has -significant performance advantages, even without using the optional C -extension for speedups. - -Encoding basic Python object hierarchies:: - - >>> import simplejson as json - >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) - '["foo", {"bar": ["baz", null, 1.0, 2]}]' - >>> print json.dumps("\"foo\bar") - "\"foo\bar" - >>> print json.dumps(u'\u1234') - "\u1234" - >>> print json.dumps('\\') - "\\" - >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True) - {"a": 0, "b": 0, "c": 0} - >>> from StringIO import StringIO - >>> io = StringIO() - >>> json.dump(['streaming API'], io) - >>> io.getvalue() - '["streaming API"]' - -Compact encoding:: - - >>> import simplejson as json - >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':')) - '[1,2,3,{"4":5,"6":7}]' - -Pretty printing:: - - >>> import simplejson as json - >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=' ') - >>> print '\n'.join([l.rstrip() for l in s.splitlines()]) - { - "4": 5, - "6": 7 - } - -Decoding JSON:: - - >>> import simplejson as json - >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] - >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj - True - >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' - True - >>> from StringIO import StringIO - >>> io = StringIO('["streaming API"]') - >>> json.load(io)[0] == 'streaming API' - True - -Specializing JSON object decoding:: - - >>> import simplejson as json - >>> def as_complex(dct): - ... if '__complex__' in dct: - ... return complex(dct['real'], dct['imag']) - ... return dct - ... - >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', - ... object_hook=as_complex) - (1+2j) - >>> from decimal import Decimal - >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1') - True - -Specializing JSON object encoding:: - - >>> import simplejson as json - >>> def encode_complex(obj): - ... if isinstance(obj, complex): - ... return [obj.real, obj.imag] - ... raise TypeError(repr(o) + " is not JSON serializable") - ... - >>> json.dumps(2 + 1j, default=encode_complex) - '[2.0, 1.0]' - >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) - '[2.0, 1.0]' - >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) - '[2.0, 1.0]' - - -Using simplejson.tool from the shell to validate and pretty-print:: - - $ echo '{"json":"obj"}' | python -m simplejson.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m simplejson.tool - Expecting property name: line 1 column 2 (char 2) -""" -__version__ = '2.2.1' -__all__ = [ - 'dump', 'dumps', 'load', 'loads', - 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', - 'OrderedDict', -] - -__author__ = 'Bob Ippolito ' - -from decimal import Decimal - -from decoder import JSONDecoder, JSONDecodeError -from encoder import JSONEncoder -def _import_OrderedDict(): - import collections - try: - return collections.OrderedDict - except AttributeError: - import ordered_dict - return ordered_dict.OrderedDict -OrderedDict = _import_OrderedDict() - -def _import_c_make_encoder(): - try: - from simplejson._speedups import make_encoder - return make_encoder - except ImportError: - return None - -_default_encoder = JSONEncoder( - skipkeys=False, - ensure_ascii=True, - check_circular=True, - allow_nan=True, - indent=None, - separators=None, - encoding='utf-8', - default=None, - use_decimal=True, - namedtuple_as_object=True, - tuple_as_array=True, -) - -def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=True, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, use_decimal=True, - namedtuple_as_object=True, tuple_as_array=True, - **kw): - """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a - ``.write()``-supporting file-like object). - - If ``skipkeys`` is true then ``dict`` keys that are not basic types - (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If ``ensure_ascii`` is false, then the some chunks written to ``fp`` - may be ``unicode`` instances, subject to normal Python ``str`` to - ``unicode`` coercion rules. Unless ``fp.write()`` explicitly - understands ``unicode`` (as in ``codecs.getwriter()``) this is likely - to cause an error. - - If ``check_circular`` is false, then the circular reference check - for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). - - If ``allow_nan`` is false, then it will be a ``ValueError`` to - serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) - in strict compliance of the JSON specification, instead of using the - JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). - - If *indent* is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If ``separators`` is an ``(item_separator, dict_separator)`` tuple - then it will be used instead of the default ``(', ', ': ')`` separators. - ``(',', ':')`` is the most compact JSON representation. - - ``encoding`` is the character encoding for str instances, default is UTF-8. - - ``default(obj)`` is a function that should return a serializable version - of obj or raise TypeError. The default simply raises TypeError. - - If *use_decimal* is true (default: ``True``) then decimal.Decimal - will be natively serialized to JSON with full precision. - - If *namedtuple_as_object* is true (default: ``True``), - :class:`tuple` subclasses with ``_asdict()`` methods will be encoded - as JSON objects. - - If *tuple_as_array* is true (default: ``True``), - :class:`tuple` (and subclasses) will be encoded as JSON arrays. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and use_decimal - and namedtuple_as_object and tuple_as_array and not kw): - iterable = _default_encoder.iterencode(obj) - else: - if cls is None: - cls = JSONEncoder - iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, - default=default, use_decimal=use_decimal, - namedtuple_as_object=namedtuple_as_object, - tuple_as_array=tuple_as_array, - **kw).iterencode(obj) - # could accelerate with writelines in some versions of Python, at - # a debuggability cost - for chunk in iterable: - fp.write(chunk) - - -def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=True, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, use_decimal=True, - namedtuple_as_object=True, - tuple_as_array=True, - **kw): - """Serialize ``obj`` to a JSON formatted ``str``. - - If ``skipkeys`` is false then ``dict`` keys that are not basic types - (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If ``ensure_ascii`` is false, then the return value will be a - ``unicode`` instance subject to normal Python ``str`` to ``unicode`` - coercion rules instead of being escaped to an ASCII ``str``. - - If ``check_circular`` is false, then the circular reference check - for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). - - If ``allow_nan`` is false, then it will be a ``ValueError`` to - serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in - strict compliance of the JSON specification, instead of using the - JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). - - If ``indent`` is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If ``separators`` is an ``(item_separator, dict_separator)`` tuple - then it will be used instead of the default ``(', ', ': ')`` separators. - ``(',', ':')`` is the most compact JSON representation. - - ``encoding`` is the character encoding for str instances, default is UTF-8. - - ``default(obj)`` is a function that should return a serializable version - of obj or raise TypeError. The default simply raises TypeError. - - If *use_decimal* is true (default: ``True``) then decimal.Decimal - will be natively serialized to JSON with full precision. - - If *namedtuple_as_object* is true (default: ``True``), - :class:`tuple` subclasses with ``_asdict()`` methods will be encoded - as JSON objects. - - If *tuple_as_array* is true (default: ``True``), - :class:`tuple` (and subclasses) will be encoded as JSON arrays. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and use_decimal - and namedtuple_as_object and tuple_as_array and not kw): - return _default_encoder.encode(obj) - if cls is None: - cls = JSONEncoder - return cls( - skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, default=default, - use_decimal=use_decimal, - namedtuple_as_object=namedtuple_as_object, - tuple_as_array=tuple_as_array, - **kw).encode(obj) - - -_default_decoder = JSONDecoder(encoding=None, object_hook=None, - object_pairs_hook=None) - - -def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, - use_decimal=False, namedtuple_as_object=True, tuple_as_array=True, - **kw): - """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing - a JSON document) to a Python object. - - *encoding* determines the encoding used to interpret any - :class:`str` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as :class:`unicode`. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *parse_constant*, if specified, will be called with one of the - following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This - can be used to raise an exception if invalid JSON numbers are - encountered. - - If *use_decimal* is true (default: ``False``) then it implies - parse_float=decimal.Decimal for parity with ``dump``. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. - - """ - return loads(fp.read(), - encoding=encoding, cls=cls, object_hook=object_hook, - parse_float=parse_float, parse_int=parse_int, - parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, - use_decimal=use_decimal, **kw) - - -def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, - use_decimal=False, **kw): - """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON - document) to a Python object. - - *encoding* determines the encoding used to interpret any - :class:`str` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as :class:`unicode`. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *parse_constant*, if specified, will be called with one of the - following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This - can be used to raise an exception if invalid JSON numbers are - encountered. - - If *use_decimal* is true (default: ``False``) then it implies - parse_float=decimal.Decimal for parity with ``dump``. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. - - """ - if (cls is None and encoding is None and object_hook is None and - parse_int is None and parse_float is None and - parse_constant is None and object_pairs_hook is None - and not use_decimal and not kw): - return _default_decoder.decode(s) - if cls is None: - cls = JSONDecoder - if object_hook is not None: - kw['object_hook'] = object_hook - if object_pairs_hook is not None: - kw['object_pairs_hook'] = object_pairs_hook - if parse_float is not None: - kw['parse_float'] = parse_float - if parse_int is not None: - kw['parse_int'] = parse_int - if parse_constant is not None: - kw['parse_constant'] = parse_constant - if use_decimal: - if parse_float is not None: - raise TypeError("use_decimal=True implies parse_float=Decimal") - kw['parse_float'] = Decimal - return cls(encoding=encoding, **kw).decode(s) - - -def _toggle_speedups(enabled): - import simplejson.decoder as dec - import simplejson.encoder as enc - import simplejson.scanner as scan - c_make_encoder = _import_c_make_encoder() - if enabled: - dec.scanstring = dec.c_scanstring or dec.py_scanstring - enc.c_make_encoder = c_make_encoder - enc.encode_basestring_ascii = (enc.c_encode_basestring_ascii or - enc.py_encode_basestring_ascii) - scan.make_scanner = scan.c_make_scanner or scan.py_make_scanner - else: - dec.scanstring = dec.py_scanstring - enc.c_make_encoder = None - enc.encode_basestring_ascii = enc.py_encode_basestring_ascii - scan.make_scanner = scan.py_make_scanner - dec.make_scanner = scan.make_scanner - global _default_decoder - _default_decoder = JSONDecoder( - encoding=None, - object_hook=None, - object_pairs_hook=None, - ) - global _default_encoder - _default_encoder = JSONEncoder( - skipkeys=False, - ensure_ascii=True, - check_circular=True, - allow_nan=True, - indent=None, - separators=None, - encoding='utf-8', - default=None, - ) diff --git a/module/lib/simplejson/decoder.py b/module/lib/simplejson/decoder.py deleted file mode 100644 index e5496d6e7..000000000 --- a/module/lib/simplejson/decoder.py +++ /dev/null @@ -1,421 +0,0 @@ -"""Implementation of JSONDecoder -""" -import re -import sys -import struct - -from simplejson.scanner import make_scanner -def _import_c_scanstring(): - try: - from simplejson._speedups import scanstring - return scanstring - except ImportError: - return None -c_scanstring = _import_c_scanstring() - -__all__ = ['JSONDecoder'] - -FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL - -def _floatconstants(): - _BYTES = '7FF80000000000007FF0000000000000'.decode('hex') - # The struct module in Python 2.4 would get frexp() out of range here - # when an endian is specified in the format string. Fixed in Python 2.5+ - if sys.byteorder != 'big': - _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1] - nan, inf = struct.unpack('dd', _BYTES) - return nan, inf, -inf - -NaN, PosInf, NegInf = _floatconstants() - - -class JSONDecodeError(ValueError): - """Subclass of ValueError with the following additional properties: - - msg: The unformatted error message - doc: The JSON document being parsed - pos: The start index of doc where parsing failed - end: The end index of doc where parsing failed (may be None) - lineno: The line corresponding to pos - colno: The column corresponding to pos - endlineno: The line corresponding to end (may be None) - endcolno: The column corresponding to end (may be None) - - """ - def __init__(self, msg, doc, pos, end=None): - ValueError.__init__(self, errmsg(msg, doc, pos, end=end)) - self.msg = msg - self.doc = doc - self.pos = pos - self.end = end - self.lineno, self.colno = linecol(doc, pos) - if end is not None: - self.endlineno, self.endcolno = linecol(doc, end) - else: - self.endlineno, self.endcolno = None, None - - -def linecol(doc, pos): - lineno = doc.count('\n', 0, pos) + 1 - if lineno == 1: - colno = pos - else: - colno = pos - doc.rindex('\n', 0, pos) - return lineno, colno - - -def errmsg(msg, doc, pos, end=None): - # Note that this function is called from _speedups - lineno, colno = linecol(doc, pos) - if end is None: - #fmt = '{0}: line {1} column {2} (char {3})' - #return fmt.format(msg, lineno, colno, pos) - fmt = '%s: line %d column %d (char %d)' - return fmt % (msg, lineno, colno, pos) - endlineno, endcolno = linecol(doc, end) - #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})' - #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end) - fmt = '%s: line %d column %d - line %d column %d (char %d - %d)' - return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end) - - -_CONSTANTS = { - '-Infinity': NegInf, - 'Infinity': PosInf, - 'NaN': NaN, -} - -STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) -BACKSLASH = { - '"': u'"', '\\': u'\\', '/': u'/', - 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', -} - -DEFAULT_ENCODING = "utf-8" - -def py_scanstring(s, end, encoding=None, strict=True, - _b=BACKSLASH, _m=STRINGCHUNK.match): - """Scan the string s for a JSON string. End is the index of the - character in s after the quote that started the JSON string. - Unescapes all valid JSON string escape sequences and raises ValueError - on attempt to decode an invalid string. If strict is False then literal - control characters are allowed in the string. - - Returns a tuple of the decoded string and the index of the character in s - after the end quote.""" - if encoding is None: - encoding = DEFAULT_ENCODING - chunks = [] - _append = chunks.append - begin = end - 1 - while 1: - chunk = _m(s, end) - if chunk is None: - raise JSONDecodeError( - "Unterminated string starting at", s, begin) - end = chunk.end() - content, terminator = chunk.groups() - # Content is contains zero or more unescaped string characters - if content: - if not isinstance(content, unicode): - content = unicode(content, encoding) - _append(content) - # Terminator is the end of string, a literal control character, - # or a backslash denoting that an escape sequence follows - if terminator == '"': - break - elif terminator != '\\': - if strict: - msg = "Invalid control character %r at" % (terminator,) - #msg = "Invalid control character {0!r} at".format(terminator) - raise JSONDecodeError(msg, s, end) - else: - _append(terminator) - continue - try: - esc = s[end] - except IndexError: - raise JSONDecodeError( - "Unterminated string starting at", s, begin) - # If not a unicode escape sequence, must be in the lookup table - if esc != 'u': - try: - char = _b[esc] - except KeyError: - msg = "Invalid \\escape: " + repr(esc) - raise JSONDecodeError(msg, s, end) - end += 1 - else: - # Unicode escape sequence - esc = s[end + 1:end + 5] - next_end = end + 5 - if len(esc) != 4: - msg = "Invalid \\uXXXX escape" - raise JSONDecodeError(msg, s, end) - uni = int(esc, 16) - # Check for surrogate pair on UCS-4 systems - if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535: - msg = "Invalid \\uXXXX\\uXXXX surrogate pair" - if not s[end + 5:end + 7] == '\\u': - raise JSONDecodeError(msg, s, end) - esc2 = s[end + 7:end + 11] - if len(esc2) != 4: - raise JSONDecodeError(msg, s, end) - uni2 = int(esc2, 16) - uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00)) - next_end += 6 - char = unichr(uni) - end = next_end - # Append the unescaped character - _append(char) - return u''.join(chunks), end - - -# Use speedup if available -scanstring = c_scanstring or py_scanstring - -WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) -WHITESPACE_STR = ' \t\n\r' - -def JSONObject((s, end), encoding, strict, scan_once, object_hook, - object_pairs_hook, memo=None, - _w=WHITESPACE.match, _ws=WHITESPACE_STR): - # Backwards compatibility - if memo is None: - memo = {} - memo_get = memo.setdefault - pairs = [] - # Use a slice to prevent IndexError from being raised, the following - # check will raise a more specific ValueError if the string is empty - nextchar = s[end:end + 1] - # Normally we expect nextchar == '"' - if nextchar != '"': - if nextchar in _ws: - end = _w(s, end).end() - nextchar = s[end:end + 1] - # Trivial empty object - if nextchar == '}': - if object_pairs_hook is not None: - result = object_pairs_hook(pairs) - return result, end + 1 - pairs = {} - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end + 1 - elif nextchar != '"': - raise JSONDecodeError("Expecting property name", s, end) - end += 1 - while True: - key, end = scanstring(s, end, encoding, strict) - key = memo_get(key, key) - - # To skip some function call overhead we optimize the fast paths where - # the JSON key separator is ": " or just ":". - if s[end:end + 1] != ':': - end = _w(s, end).end() - if s[end:end + 1] != ':': - raise JSONDecodeError("Expecting : delimiter", s, end) - - end += 1 - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - try: - value, end = scan_once(s, end) - except StopIteration: - raise JSONDecodeError("Expecting object", s, end) - pairs.append((key, value)) - - try: - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - end += 1 - - if nextchar == '}': - break - elif nextchar != ',': - raise JSONDecodeError("Expecting , delimiter", s, end - 1) - - try: - nextchar = s[end] - if nextchar in _ws: - end += 1 - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - - end += 1 - if nextchar != '"': - raise JSONDecodeError("Expecting property name", s, end - 1) - - if object_pairs_hook is not None: - result = object_pairs_hook(pairs) - return result, end - pairs = dict(pairs) - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end - -def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): - values = [] - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - # Look-ahead for trivial empty array - if nextchar == ']': - return values, end + 1 - _append = values.append - while True: - try: - value, end = scan_once(s, end) - except StopIteration: - raise JSONDecodeError("Expecting object", s, end) - _append(value) - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - end += 1 - if nextchar == ']': - break - elif nextchar != ',': - raise JSONDecodeError("Expecting , delimiter", s, end) - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - return values, end - -class JSONDecoder(object): - """Simple JSON decoder - - Performs the following translations in decoding by default: - - +---------------+-------------------+ - | JSON | Python | - +===============+===================+ - | object | dict | - +---------------+-------------------+ - | array | list | - +---------------+-------------------+ - | string | unicode | - +---------------+-------------------+ - | number (int) | int, long | - +---------------+-------------------+ - | number (real) | float | - +---------------+-------------------+ - | true | True | - +---------------+-------------------+ - | false | False | - +---------------+-------------------+ - | null | None | - +---------------+-------------------+ - - It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as - their corresponding ``float`` values, which is outside the JSON spec. - - """ - - def __init__(self, encoding=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, strict=True, - object_pairs_hook=None): - """ - *encoding* determines the encoding used to interpret any - :class:`str` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as :class:`unicode`. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *parse_constant*, if specified, will be called with one of the - following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This - can be used to raise an exception if invalid JSON numbers are - encountered. - - *strict* controls the parser's behavior when it encounters an - invalid control character in a string. The default setting of - ``True`` means that unescaped control characters are parse errors, if - ``False`` then control characters will be allowed in strings. - - """ - self.encoding = encoding - self.object_hook = object_hook - self.object_pairs_hook = object_pairs_hook - self.parse_float = parse_float or float - self.parse_int = parse_int or int - self.parse_constant = parse_constant or _CONSTANTS.__getitem__ - self.strict = strict - self.parse_object = JSONObject - self.parse_array = JSONArray - self.parse_string = scanstring - self.memo = {} - self.scan_once = make_scanner(self) - - def decode(self, s, _w=WHITESPACE.match): - """Return the Python representation of ``s`` (a ``str`` or ``unicode`` - instance containing a JSON document) - - """ - obj, end = self.raw_decode(s, idx=_w(s, 0).end()) - end = _w(s, end).end() - if end != len(s): - raise JSONDecodeError("Extra data", s, end, len(s)) - return obj - - def raw_decode(self, s, idx=0): - """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` - beginning with a JSON document) and return a 2-tuple of the Python - representation and the index in ``s`` where the document ended. - - This can be used to decode a JSON document from a string that may - have extraneous data at the end. - - """ - try: - obj, end = self.scan_once(s, idx) - except StopIteration: - raise JSONDecodeError("No JSON object could be decoded", s, idx) - return obj, end diff --git a/module/lib/simplejson/encoder.py b/module/lib/simplejson/encoder.py deleted file mode 100644 index 5ec7440f1..000000000 --- a/module/lib/simplejson/encoder.py +++ /dev/null @@ -1,534 +0,0 @@ -"""Implementation of JSONEncoder -""" -import re -from decimal import Decimal - -def _import_speedups(): - try: - from simplejson import _speedups - return _speedups.encode_basestring_ascii, _speedups.make_encoder - except ImportError: - return None, None -c_encode_basestring_ascii, c_make_encoder = _import_speedups() - -from simplejson.decoder import PosInf - -ESCAPE = re.compile(ur'[\x00-\x1f\\"\b\f\n\r\t\u2028\u2029]') -ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') -HAS_UTF8 = re.compile(r'[\x80-\xff]') -ESCAPE_DCT = { - '\\': '\\\\', - '"': '\\"', - '\b': '\\b', - '\f': '\\f', - '\n': '\\n', - '\r': '\\r', - '\t': '\\t', - u'\u2028': '\\u2028', - u'\u2029': '\\u2029', -} -for i in range(0x20): - #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) - ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) - -FLOAT_REPR = repr - -def encode_basestring(s): - """Return a JSON representation of a Python string - - """ - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = s.decode('utf-8') - def replace(match): - return ESCAPE_DCT[match.group(0)] - return u'"' + ESCAPE.sub(replace, s) + u'"' - - -def py_encode_basestring_ascii(s): - """Return an ASCII-only JSON representation of a Python string - - """ - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = s.decode('utf-8') - def replace(match): - s = match.group(0) - try: - return ESCAPE_DCT[s] - except KeyError: - n = ord(s) - if n < 0x10000: - #return '\\u{0:04x}'.format(n) - return '\\u%04x' % (n,) - else: - # surrogate pair - n -= 0x10000 - s1 = 0xd800 | ((n >> 10) & 0x3ff) - s2 = 0xdc00 | (n & 0x3ff) - #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) - return '\\u%04x\\u%04x' % (s1, s2) - return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' - - -encode_basestring_ascii = ( - c_encode_basestring_ascii or py_encode_basestring_ascii) - -class JSONEncoder(object): - """Extensible JSON encoder for Python data structures. - - Supports the following objects and types by default: - - +-------------------+---------------+ - | Python | JSON | - +===================+===============+ - | dict, namedtuple | object | - +-------------------+---------------+ - | list, tuple | array | - +-------------------+---------------+ - | str, unicode | string | - +-------------------+---------------+ - | int, long, float | number | - +-------------------+---------------+ - | True | true | - +-------------------+---------------+ - | False | false | - +-------------------+---------------+ - | None | null | - +-------------------+---------------+ - - To extend this to recognize other objects, subclass and implement a - ``.default()`` method with another method that returns a serializable - object for ``o`` if possible, otherwise it should call the superclass - implementation (to raise ``TypeError``). - - """ - item_separator = ', ' - key_separator = ': ' - def __init__(self, skipkeys=False, ensure_ascii=True, - check_circular=True, allow_nan=True, sort_keys=False, - indent=None, separators=None, encoding='utf-8', default=None, - use_decimal=True, namedtuple_as_object=True, - tuple_as_array=True): - """Constructor for JSONEncoder, with sensible defaults. - - If skipkeys is false, then it is a TypeError to attempt - encoding of keys that are not str, int, long, float or None. If - skipkeys is True, such items are simply skipped. - - If ensure_ascii is true, the output is guaranteed to be str - objects with all incoming unicode characters escaped. If - ensure_ascii is false, the output will be unicode object. - - If check_circular is true, then lists, dicts, and custom encoded - objects will be checked for circular references during encoding to - prevent an infinite recursion (which would cause an OverflowError). - Otherwise, no such check takes place. - - If allow_nan is true, then NaN, Infinity, and -Infinity will be - encoded as such. This behavior is not JSON specification compliant, - but is consistent with most JavaScript based encoders and decoders. - Otherwise, it will be a ValueError to encode such floats. - - If sort_keys is true, then the output of dictionaries will be - sorted by key; this is useful for regression tests to ensure - that JSON serializations can be compared on a day-to-day basis. - - If indent is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If specified, separators should be a (item_separator, key_separator) - tuple. The default is (', ', ': '). To get the most compact JSON - representation you should specify (',', ':') to eliminate whitespace. - - If specified, default is a function that gets called for objects - that can't otherwise be serialized. It should return a JSON encodable - version of the object or raise a ``TypeError``. - - If encoding is not None, then all input strings will be - transformed into unicode using that encoding prior to JSON-encoding. - The default is UTF-8. - - If use_decimal is true (not the default), ``decimal.Decimal`` will - be supported directly by the encoder. For the inverse, decode JSON - with ``parse_float=decimal.Decimal``. - - If namedtuple_as_object is true (the default), tuple subclasses with - ``_asdict()`` methods will be encoded as JSON objects. - - If tuple_as_array is true (the default), tuple (and subclasses) will - be encoded as JSON arrays. - """ - - self.skipkeys = skipkeys - self.ensure_ascii = ensure_ascii - self.check_circular = check_circular - self.allow_nan = allow_nan - self.sort_keys = sort_keys - self.use_decimal = use_decimal - self.namedtuple_as_object = namedtuple_as_object - self.tuple_as_array = tuple_as_array - if isinstance(indent, (int, long)): - indent = ' ' * indent - self.indent = indent - if separators is not None: - self.item_separator, self.key_separator = separators - elif indent is not None: - self.item_separator = ',' - if default is not None: - self.default = default - self.encoding = encoding - - def default(self, o): - """Implement this method in a subclass such that it returns - a serializable object for ``o``, or calls the base implementation - (to raise a ``TypeError``). - - For example, to support arbitrary iterators, you could - implement default like this:: - - def default(self, o): - try: - iterable = iter(o) - except TypeError: - pass - else: - return list(iterable) - return JSONEncoder.default(self, o) - - """ - raise TypeError(repr(o) + " is not JSON serializable") - - def encode(self, o): - """Return a JSON string representation of a Python data structure. - - >>> from simplejson import JSONEncoder - >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) - '{"foo": ["bar", "baz"]}' - - """ - # This is for extremely simple cases and benchmarks. - if isinstance(o, basestring): - if isinstance(o, str): - _encoding = self.encoding - if (_encoding is not None - and not (_encoding == 'utf-8')): - o = o.decode(_encoding) - if self.ensure_ascii: - return encode_basestring_ascii(o) - else: - return encode_basestring(o) - # This doesn't pass the iterator directly to ''.join() because the - # exceptions aren't as detailed. The list call should be roughly - # equivalent to the PySequence_Fast that ''.join() would do. - chunks = self.iterencode(o, _one_shot=True) - if not isinstance(chunks, (list, tuple)): - chunks = list(chunks) - if self.ensure_ascii: - return ''.join(chunks) - else: - return u''.join(chunks) - - def iterencode(self, o, _one_shot=False): - """Encode the given object and yield each string - representation as available. - - For example:: - - for chunk in JSONEncoder().iterencode(bigobject): - mysocket.write(chunk) - - """ - if self.check_circular: - markers = {} - else: - markers = None - if self.ensure_ascii: - _encoder = encode_basestring_ascii - else: - _encoder = encode_basestring - if self.encoding != 'utf-8': - def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): - if isinstance(o, str): - o = o.decode(_encoding) - return _orig_encoder(o) - - def floatstr(o, allow_nan=self.allow_nan, - _repr=FLOAT_REPR, _inf=PosInf, _neginf=-PosInf): - # Check for specials. Note that this type of test is processor - # and/or platform-specific, so do tests which don't depend on - # the internals. - - if o != o: - text = 'NaN' - elif o == _inf: - text = 'Infinity' - elif o == _neginf: - text = '-Infinity' - else: - return _repr(o) - - if not allow_nan: - raise ValueError( - "Out of range float values are not JSON compliant: " + - repr(o)) - - return text - - - key_memo = {} - if (_one_shot and c_make_encoder is not None - and self.indent is None): - _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, self.allow_nan, key_memo, self.use_decimal, - self.namedtuple_as_object, self.tuple_as_array) - else: - _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, _one_shot, self.use_decimal, - self.namedtuple_as_object, self.tuple_as_array) - try: - return _iterencode(o, 0) - finally: - key_memo.clear() - - -class JSONEncoderForHTML(JSONEncoder): - """An encoder that produces JSON safe to embed in HTML. - - To embed JSON content in, say, a script tag on a web page, the - characters &, < and > should be escaped. They cannot be escaped - with the usual entities (e.g. &) because they are not expanded - within |;var)" - SHOW_PATTERN = r"title='(.*?)'" - SERIE_PATTERN = r".*Serie.*" - - def decrypt(self, pyfile): - src = self.req.load(str(pyfile.url)) - - pattern = re.compile(self.VALUES_PATTERN, re.DOTALL) - data = json_loads(re.search(pattern, src).group(1)) - - # Get package info - links = [] - Showname = re.search(self.SHOW_PATTERN, src) - if Showname: - Showname = Showname.group(1).decode("utf-8") - else: - Showname = self.pyfile.package().name - - if re.search(self.SERIE_PATTERN, src): - for Season in data: - self.logDebug("Season " + Season) - for Episode in data[Season]: - self.logDebug("Episode " + Episode) - links.extend(self.getpreferred(data[Season][Episode])) - if self.getConfig("changeNameS") == "Episode": - self.packages.append((data[Season][Episode]['info']['name'].split("»")[0], links, data[Season][Episode]['info']['name'].split("»")[0])) - links = [] - - if self.getConfig("changeNameS") == "Season": - self.packages.append((Showname + " Season " + Season, links, Showname + " Season " + Season)) - links = [] - - if self.getConfig("changeNameS") == "Show": - if links == []: - self.fail('Could not extract any links (Out of Date?)') - else: - self.packages.append((Showname, links, Showname)) - - elif self.getConfig("changeNameS") == "Packagename": - if links == []: - self.fail('Could not extract any links (Out of Date?)') - else: - self.core.files.addLinks(links, self.pyfile.package().id) - else: - for Movie in data: - links.extend(self.getpreferred(data[Movie])) - if self.getConfig("changeName") == "Movie": - if links == []: - self.fail('Could not extract any links (Out of Date?)') - else: - self.packages.append((Showname, links, Showname)) - - elif self.getConfig("changeName") == "Packagename": - if links == []: - self.fail('Could not extract any links (Out of Date?)') - else: - self.core.files.addLinks(links, self.pyfile.package().id) - - #selects the preferred hoster, after that selects any hoster (ignoring the one to ignore) - #selects only one Hoster - def getpreferred(self, hosterslist): - hosterlist = {} - if 'u' in hosterslist: - hosterlist.update(hosterslist['u']) - if ('d' in hosterslist): - hosterlist.update(hosterslist['d']) - if self.getConfig("useStreams") and 's' in hosterslist: - hosterlist.update(hosterslist['s']) - - result = [] - preferredList = self.getConfig("hosterList").strip().lower().replace('|',',').replace('.','').replace(';',',').split(',') - if self.getConfig("randomPreferred") == True: - random.shuffle(preferredList) - for preferred in preferredList: - for Hoster in hosterlist: - if preferred == Hoster.split('<')[0].strip().lower().replace('.',''): - for Part in hosterlist[Hoster]: - self.logDebug("selected " + Part[3]) - result.append(str(Part[3])) - return result - - ignorelist = self.getConfig("ignoreList").strip().lower().replace('|',',').replace('.','').replace(';',',').split(',') - if self.getConfig('hosterListMode') == "all": - for Hoster in hosterlist: - if Hoster.split('<')[0].strip().lower().replace('.','') not in ignorelist: - for Part in hosterlist[Hoster]: - self.logDebug("selected " + Part[3]) - result.append(str(Part[3])) - return result - return result - - - diff --git a/module/plugins/crypter/CCF.py b/module/plugins/crypter/CCF.py deleted file mode 100644 index ab7ff1099..000000000 --- a/module/plugins/crypter/CCF.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from urllib2 import build_opener - -from module.plugins.Crypter import Crypter -from module.lib.MultipartPostHandler import MultipartPostHandler - -from os import makedirs -from os.path import exists, join - -class CCF(Crypter): - __name__ = "CCF" - __version__ = "0.2" - __pattern__ = r"(?!http://).*\.ccf$" - __description__ = """CCF Container Convert Plugin""" - __author_name__ = ("Willnix") - __author_mail__ = ("Willnix@pyload.org") - - def decrypt(self, pyfile): - - infile = pyfile.url.replace("\n", "") - - opener = build_opener(MultipartPostHandler) - params = {"src": "ccf", - "filename": "test.ccf", - "upload": open(infile, "rb")} - tempdlc_content = opener.open('http://service.jdownloader.net/dlcrypt/getDLC.php', params).read() - - download_folder = self.config['general']['download_folder'] - location = download_folder #join(download_folder, self.pyfile.package().folder.decode(sys.getfilesystemencoding())) - if not exists(location): - makedirs(location) - - tempdlc_name = join(location, "tmp_%s.dlc" % pyfile.name) - tempdlc = open(tempdlc_name, "w") - tempdlc.write(re.search(r'(.*)', tempdlc_content, re.DOTALL).group(1)) - tempdlc.close() - - self.packages.append((tempdlc_name, [tempdlc_name], tempdlc_name)) - diff --git a/module/plugins/crypter/CrockoComFolder.py b/module/plugins/crypter/CrockoComFolder.py deleted file mode 100644 index d727ec7ab..000000000 --- a/module/plugins/crypter/CrockoComFolder.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.internal.SimpleCrypter import SimpleCrypter - -class CrockoComFolder(SimpleCrypter): - __name__ = "CrockoComFolder" - __type__ = "crypter" - __pattern__ = r"http://(www\.)?crocko.com/f/.*" - __version__ = "0.01" - __description__ = """Crocko.com Folder Plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - LINK_PATTERN = r'download' \ No newline at end of file diff --git a/module/plugins/crypter/CryptItCom.py b/module/plugins/crypter/CryptItCom.py deleted file mode 100644 index 4935758c7..000000000 --- a/module/plugins/crypter/CryptItCom.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- - - -import re - -from random import randint - -from module.plugins.Crypter import Crypter - - -class CryptItCom(Crypter): - __name__ = "CryptItCom" - __type__ = "container" - __pattern__ = r"http://[\w\.]*?crypt-it\.com/(s|e|d|c)/[\w]+" - __version__ = "0.1" - __description__ = """Crypt.It.com Container Plugin""" - __author_name__ = ("jeix") - __author_mail__ = ("jeix@hasnomail.de") - - def file_exists(self): - html = self.load(self.pyfile.url) - if r'
Was ist Crypt-It
' in html: - return False - return True - - def decrypt(self, pyfile): - if not self.file_exists(): - self.offline() - - # @TODO parse name and password - repl_pattern = r"/(s|e|d|c)/" - url = re.sub(repl_pattern, r"/d/", self.pyfile.url) - - pyfile.name = "tmp_cryptit_%s.ccf" % randint(0,1000) - location = self.download(url) - - self.packages.append(["Crypt-it Package", [location], "Crypt-it Package"]) - \ No newline at end of file diff --git a/module/plugins/crypter/CzshareComFolder.py b/module/plugins/crypter/CzshareComFolder.py deleted file mode 100644 index c240c6a70..000000000 --- a/module/plugins/crypter/CzshareComFolder.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter - -class CzshareComFolder(Crypter): - __name__ = "CzshareComFolder" - __type__ = "crypter" - __pattern__ = r"http://(\w*\.)*czshare\.(com|cz)/folders/.*" - __version__ = "0.1" - __description__ = """Czshare.com Folder Plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FOLDER_PATTERN = r'\s*\s*(.*?)
' - LINK_PATTERN = r'info' - #NEXT_PAGE_PATTERN = r'' - - def decrypt(self, pyfile): - html = self.load(self.pyfile.url) - - new_links = [] - found = re.search(self.FOLDER_PATTERN, html, re.DOTALL) - if found is None: self.fail("Parse error (FOLDER)") - new_links.extend(re.findall(self.LINK_PATTERN, found.group(1))) - - if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) - else: - self.fail('Could not extract any links') \ No newline at end of file diff --git a/module/plugins/crypter/DDLMusicOrg.py b/module/plugins/crypter/DDLMusicOrg.py deleted file mode 100644 index f7cc996d0..000000000 --- a/module/plugins/crypter/DDLMusicOrg.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from time import sleep - -from module.plugins.Crypter import Crypter - -class DDLMusicOrg(Crypter): - __name__ = "DDLMusicOrg" - __type__ = "container" - __pattern__ = r"http://[\w\.]*?ddl-music\.org/captcha/ddlm_cr\d\.php\?\d+\?\d+" - __version__ = "0.3" - __description__ = """ddl-music.org Container Plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def setup(self): - self.multiDL = False - - def decrypt(self, pyfile): - html = self.req.load(self.pyfile.url, cookies=True) - - if re.search(r"Wer dies nicht rechnen kann", html) is not None: - self.offline() - - math = re.search(r"(\d+) ([\+-]) (\d+) =\s+", htmlwithlink) - if m: - self.packages.append((self.pyfile.package().name, [m.group(1)], self.pyfile.package().folder)) - else: - self.retry() diff --git a/module/plugins/crypter/DataHuFolder.py b/module/plugins/crypter/DataHuFolder.py deleted file mode 100644 index f710f60d7..000000000 --- a/module/plugins/crypter/DataHuFolder.py +++ /dev/null @@ -1,55 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################ -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU Affero General Public License as # -# published by the Free Software Foundation, either version 3 of the # -# License, or (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU Affero General Public License for more details. # -# # -# You should have received a copy of the GNU Affero General Public License # -# along with this program. If not, see . # -############################################################################ - -import re - -from module.plugins.internal.SimpleCrypter import SimpleCrypter - - -class DataHuFolder(SimpleCrypter): - __name__ = "DataHuFolder" - __type__ = "crypter" - __pattern__ = r"http://(www\.)?data.hu/dir/\w+" - __version__ = "0.03" - __description__ = """Data.hu Folder Plugin""" - __author_name__ = ("crash", "stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - LINK_PATTERN = r"\1" - TITLE_PATTERN = ur'(?P<title>.+) Let\xf6lt\xe9se' - - def decrypt(self, pyfile): - self.html = self.load(pyfile.url, decode=True) - - if u'K\xe9rlek add meg a jelsz\xf3t' in self.html: # Password protected - password = self.getPassword() - if password is '': - self.fail("No password specified, please set right password on Add package form and retry") - self.logDebug('The folder is password protected', 'Using password: ' + password) - self.html = self.load(pyfile.url, post={'mappa_pass': password}, decode=True) - if u'Hib\xe1s jelsz\xf3' in self.html: # Wrong password - self.fail("Incorrect password, please set right password on Add package form and retry") - - package_name, folder_name = self.getPackageNameAndFolder() - - package_links = re.findall(self.LINK_PATTERN, self.html) - self.logDebug('Package has %d links' % len(package_links)) - - if package_links: - self.packages = [(package_name, package_links, folder_name)] - else: - self.fail('Could not extract any links') diff --git a/module/plugins/crypter/DdlstorageComFolder.py b/module/plugins/crypter/DdlstorageComFolder.py deleted file mode 100644 index d536032c6..000000000 --- a/module/plugins/crypter/DdlstorageComFolder.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter -from module.plugins.hoster.MediafireCom import checkHTMLHeader -from module.common.json_layer import json_loads - -class DdlstorageComFolder(Crypter): - __name__ = "DdlstorageComFolder" - __type__ = "crypter" - __pattern__ = r"http://(?:\w*\.)*?ddlstorage.com/folder/\w{10}" - __version__ = "0.01" - __description__ = """DDLStorage.com Folder Plugin""" - __author_name__ = ("godofdream") - __author_mail__ = ("soilfiction@gmail.com") - - FILE_URL_PATTERN = '' - - def decrypt(self, pyfile): - new_links = [] - # load and parse html - html = self.load(pyfile.url) - found = re.findall(self.FILE_URL_PATTERN, html) - self.logDebug(found) - for link in found: - # file page - new_links.append("http://www.ddlstorage.com/%s" % link) - - if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) - else: - self.fail('Could not extract any links') diff --git a/module/plugins/crypter/DepositfilesComFolder.py b/module/plugins/crypter/DepositfilesComFolder.py deleted file mode 100644 index 9023b238f..000000000 --- a/module/plugins/crypter/DepositfilesComFolder.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.internal.SimpleCrypter import SimpleCrypter - -class DepositfilesComFolder(SimpleCrypter): - __name__ = "DepositfilesComFolder" - __type__ = "crypter" - __pattern__ = r"http://(www\.)?depositfiles.com/folders/\w+" - __version__ = "0.01" - __description__ = """Depositfiles.com Folder Plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - LINK_PATTERN = r'
]*>\s*' \ No newline at end of file diff --git a/module/plugins/crypter/Dereferer.py b/module/plugins/crypter/Dereferer.py deleted file mode 100644 index 584835e18..000000000 --- a/module/plugins/crypter/Dereferer.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . -""" - -import re -import urllib - -from module.plugins.Crypter import Crypter - -class Dereferer(Crypter): - __name__ = "Dereferer" - __type__ = "crypter" - __pattern__ = r'https?://([^/]+)/.*?(?P(ht|f)tps?(://|%3A%2F%2F).*)' - __version__ = "0.1" - __description__ = """Crypter for dereferers""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - def decrypt(self, pyfile): - link = re.match(self.__pattern__, self.pyfile.url).group('url') - self.core.files.addLinks([ urllib.unquote(link).rstrip('+') ], self.pyfile.package().id) diff --git a/module/plugins/crypter/DontKnowMe.py b/module/plugins/crypter/DontKnowMe.py deleted file mode 100644 index dfa72df47..000000000 --- a/module/plugins/crypter/DontKnowMe.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -import urllib - -from module.plugins.Crypter import Crypter - -class DontKnowMe(Crypter): - __name__ = "DontKnowMe" - __type__ = "crypter" - __pattern__ = r"http://dontknow.me/at/\?.+$" - __version__ = "0.1" - __description__ = """DontKnowMe""" - __author_name__ = ("selaux") - __author_mail__ = ("") - - LINK_PATTERN = r"http://dontknow.me/at/\?(.+)$" - - def decrypt(self, pyfile): - link = re.findall(self.LINK_PATTERN, self.pyfile.url)[0] - self.core.files.addLinks([ urllib.unquote(link) ], self.pyfile.package().id) diff --git a/module/plugins/crypter/DownloadVimeoCom.py b/module/plugins/crypter/DownloadVimeoCom.py deleted file mode 100644 index 88310915b..000000000 --- a/module/plugins/crypter/DownloadVimeoCom.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -import HTMLParser -from module.plugins.Crypter import Crypter - -class DownloadVimeoCom(Crypter): - __name__ = 'DownloadVimeoCom' - __type__ = 'crypter' - __pattern__ = r'(?:http://vimeo\.com/\d*|http://smotri\.com/video/view/\?id=.*)' - ## The download from dailymotion failed with a 403 - __version__ = '0.1' - __description__ = """Video Download Plugin based on downloadvimeo.com""" - __author_name__ = ('4Christopher') - __author_mail__ = ('4Christopher@gmx.de') - BASE_URL = 'http://downloadvimeo.com' - - def decrypt(self, pyfile): - self.package = pyfile.package() - html = self.load('%s/generate?url=%s' % (self.BASE_URL, pyfile.url)) - h = HTMLParser.HTMLParser() - try: - f = re.search(r'cmd quality="(?P[^"]+?)">\s*?(?P[^<]*?)', html) - except: - self.logDebug('Failed to find the URL') - else: - url = h.unescape(f.group('URL')) - self.logDebug('Quality: %s, URL: %s' % (f.group('quality'), url)) - self.packages.append((self.package.name, [url], self.package.folder)) diff --git a/module/plugins/crypter/DuckCryptInfo.py b/module/plugins/crypter/DuckCryptInfo.py deleted file mode 100644 index 4886d24db..000000000 --- a/module/plugins/crypter/DuckCryptInfo.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.lib.BeautifulSoup import BeautifulSoup -from module.plugins.Crypter import Crypter - -class DuckCryptInfo(Crypter): - __name__ = "DuckCryptInfo" - __type__ = "container" - __pattern__ = r"http://(?:www\.)?duckcrypt.info/(folder|wait|link)/(\w+)/?(\w*)" - __version__ = "0.02" - __description__ = """DuckCrypt.Info Container Plugin""" - __author_name__ = ("godofdream") - __author_mail__ = ("soilfiction@gmail.com") - - TIMER_PATTERN = r'(.*)' - - def decrypt(self, pyfile): - url = pyfile.url - # seems we don't need to wait - #src = self.req.load(str(url)) - #found = re.search(self.TIMER_PATTERN, src) - #if found: - # self.logDebug("Sleeping for" % found.group(1)) - # self.setWait(int(found.group(1)) ,False) - found = re.search(self.__pattern__, url) - if not found: - self.fail('Weird error in link') - if str(found.group(1)) == "link": - self.handleLink(url) - else: - self.handleFolder(found) - - - - def handleFolder(self, found): - src = self.load("http://duckcrypt.info/ajax/auth.php?hash=" + str(found.group(2))) - found = re.search(self.__pattern__, src) - self.logDebug("Redirectet to " + str(found.group(0))) - src = self.load(str(found.group(0))) - soup = BeautifulSoup(src) - cryptlinks = soup.findAll("div", attrs={"class": "folderbox"}) - self.logDebug("Redirectet to " + str(cryptlinks)) - if not cryptlinks: - self.fail('no links found - (Plugin out of date?)') - for clink in cryptlinks: - if clink.find("a"): - self.handleLink(clink.find("a")['href']) - - def handleLink(self, url): - src = self.load(url) - soup = BeautifulSoup(src) - link = soup.find("iframe")["src"] - if not link: - self.logDebug('no links found - (Plugin out of date?)') - else: - self.core.files.addLinks([link], self.pyfile.package().id) - diff --git a/module/plugins/crypter/EasybytezComFolder.py b/module/plugins/crypter/EasybytezComFolder.py deleted file mode 100644 index 1b887e421..000000000 --- a/module/plugins/crypter/EasybytezComFolder.py +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################ -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU Affero General Public License as # -# published by the Free Software Foundation, either version 3 of the # -# License, or (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU Affero General Public License for more details. # -# # -# You should have received a copy of the GNU Affero General Public License # -# along with this program. If not, see . # -############################################################################ - -import re - -from module.plugins.internal.SimpleCrypter import SimpleCrypter - - -class EasybytezComFolder(SimpleCrypter): - __name__ = "EasybytezComFolder" - __type__ = "crypter" - __pattern__ = r"https?://(www\.)?easybytez\.com/users/\w+/\w+" - __version__ = "0.01" - __description__ = """Easybytez Crypter Plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - LINK_PATTERN = r'
.+
' - TITLE_PATTERN = r'Files of (?P<title>.+) folder' - PAGES_PATTERN = r"(\d+)Next »
\(\d+ total\)
" - - def decrypt(self, pyfile): - self.html = self.load(pyfile.url, decode=True) - - package_name, folder_name = self.getPackageNameAndFolder() - - package_links = re.findall(self.LINK_PATTERN, self.html) - - pages = re.search(self.PAGES_PATTERN, self.html) - if pages: - pages = int(pages.group(1)) - else: - pages = 1 - - p = 2 - while p <= pages: - self.html = self.load(pyfile.url, get={'page': p}, decode=True) - package_links += re.findall(self.LINK_PATTERN, self.html) - p += 1 - - self.logDebug('Package has %d links' % len(package_links)) - - if package_links: - self.packages = [(package_name, package_links, folder_name)] - else: - self.fail('Could not extract any links') \ No newline at end of file diff --git a/module/plugins/crypter/EmbeduploadCom.py b/module/plugins/crypter/EmbeduploadCom.py deleted file mode 100644 index 8fd70882f..000000000 --- a/module/plugins/crypter/EmbeduploadCom.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter -from module.network.HTTPRequest import BadHeader - -class EmbeduploadCom(Crypter): - __name__ = "EmbeduploadCom" - __type__ = "crypter" - __pattern__ = r"http://(www\.)?embedupload.com/\?d=.*" - __version__ = "0.02" - __description__ = """EmbedUpload.com crypter""" - __config__ = [("preferedHoster", "str", "Prefered hoster list (bar-separated) ", "embedupload"), - ("ignoredHoster", "str", "Ignored hoster list (bar-separated) ", "")] - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - LINK_PATTERN = r'
]*>\s*' - - def decrypt(self, pyfile): - self.html = self.load(self.pyfile.url, decode=True) - tmp_links = [] - new_links = [] - - found = re.findall(self.LINK_PATTERN, self.html) - if found: - prefered_set = set(self.getConfig("preferedHoster").split('|')) - prefered_set = map(lambda s: s.lower().split('.')[0], prefered_set) - print "PF", prefered_set - tmp_links.extend([x[1] for x in found if x[0] in prefered_set]) - self.getLocation(tmp_links, new_links) - - if not new_links: - ignored_set = set(self.getConfig("ignoredHoster").split('|')) - ignored_set = map(lambda s: s.lower().split('.')[0], ignored_set) - print "IG", ignored_set - tmp_links.extend([x[1] for x in found if x[0] not in ignored_set]) - self.getLocation(tmp_links, new_links) - - if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) - else: - self.fail('Could not extract any links') - - def getLocation(self, tmp_links, new_links): - for link in tmp_links: - try: - header = self.load(link, just_header = True) - if "location" in header: - new_links.append(header['location']) - except BadHeader: - pass - - \ No newline at end of file diff --git a/module/plugins/crypter/FilebeerInfoFolder.py b/module/plugins/crypter/FilebeerInfoFolder.py deleted file mode 100644 index f45144f14..000000000 --- a/module/plugins/crypter/FilebeerInfoFolder.py +++ /dev/null @@ -1,35 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter - -class FilebeerInfoFolder(Crypter): - __name__ = "FilebeerInfoFolder" - __type__ = "crypter" - __pattern__ = r"http://(?:www\.)?filebeer\.info/(\d+~f).*" - __version__ = "0.01" - __description__ = """Filebeer.info Folder Plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - LINK_PATTERN = r'' - PAGE_COUNT_PATTERN = r'

\s*Total Pages (\d+)' - - def decrypt(self, pyfile): - pyfile.url = re.sub(self.__pattern__, r'http://filebeer.info/\1?page=1', pyfile.url) - html = self.load(pyfile.url) - - page_count = int(re.search(self.PAGE_COUNT_PATTERN, html).group(1)) - new_links = [] - - for i in range(1, page_count + 1): - self.logInfo("Fetching links from page %i" % i) - new_links.extend(re.findall(self.LINK_PATTERN, html)) - - if i < page_count: - html = self.load("%s?page=%d" % (pyfile.url, i+1)) - - if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) - else: - self.fail('Could not extract any links') \ No newline at end of file diff --git a/module/plugins/crypter/FilefactoryComFolder.py b/module/plugins/crypter/FilefactoryComFolder.py deleted file mode 100644 index 32793b491..000000000 --- a/module/plugins/crypter/FilefactoryComFolder.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter - -class FilefactoryComFolder(Crypter): - __name__ = "FilefactoryComFolder" - __type__ = "crypter" - __pattern__ = r"(http://(www\.)?filefactory\.com/f/\w+).*" - __version__ = "0.1" - __description__ = """Filefactory.com Folder Plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FOLDER_PATTERN = r'(.*?)
' - LINK_PATTERN = r'' - PAGINATOR_PATTERN = r'

\s*\s*
    (.*?)
\s*
' - NEXT_PAGE_PATTERN = r'
  • .*?
    • \s*
  • ' - - def decrypt(self, pyfile): - url_base = re.search(self.__pattern__, self.pyfile.url).group(1) - html = self.load(url_base) - - new_links = [] - for i in range(1,100): - self.logInfo("Fetching links from page %i" % i) - found = re.search(self.FOLDER_PATTERN, html, re.DOTALL) - if found is None: self.fail("Parse error (FOLDER)") - - new_links.extend(re.findall(self.LINK_PATTERN, found.group(1))) - - try: - paginator = re.search(self.PAGINATOR_PATTERN, html, re.DOTALL).group(1) - next_page = re.search(self.NEXT_PAGE_PATTERN, paginator).group(1) - html = self.load("%s/%s" % (url_base, next_page)) - except Exception, e: - break - else: - self.logInfo("Limit of 99 pages reached, aborting") - - if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) - else: - self.fail('Could not extract any links') \ No newline at end of file diff --git a/module/plugins/crypter/FileserveComFolder.py b/module/plugins/crypter/FileserveComFolder.py deleted file mode 100644 index 9fe806971..000000000 --- a/module/plugins/crypter/FileserveComFolder.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from module.plugins.Crypter import Crypter - -class FileserveComFolder(Crypter): - __name__ = "FileserveComFolder" - __type__ = "crypter" - __pattern__ = r"http://(?:www\.)?fileserve.com/list/\w+" - __version__ = "0.11" - __description__ = """FileServeCom.com Folder Plugin""" - __author_name__ = ("fionnc") - __author_mail__ = ("fionnc@gmail.com") - - FOLDER_PATTERN = r'(.*?)
    ' - LINK_PATTERN = r'    ' - - def decrypt(self, pyfile): - html = self.load(self.pyfile.url) - - new_links = [] - - folder = re.search(self.FOLDER_PATTERN, html, re.DOTALL) - if folder is None: self.fail("Parse error (FOLDER)") - - new_links.extend(re.findall(self.LINK_PATTERN, folder.group(1))) - - if new_links: - self.core.files.addLinks(map(lambda s:"http://fileserve.com%s" % s, new_links), self.pyfile.package().id) - else: - self.fail('Could not extract any links') \ No newline at end of file diff --git a/module/plugins/crypter/FourChanOrg.py b/module/plugins/crypter/FourChanOrg.py deleted file mode 100644 index 5c96e723d..000000000 --- a/module/plugins/crypter/FourChanOrg.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re - -from module.plugins.Crypter import Crypter - -class FourChanOrg(Crypter): - # Based on 4chandl by Roland Beermann - # https://gist.github.com/enkore/3492599 - __name__ = "FourChanOrg" - __type__ = "container" - __version__ = "0.3" - __pattern__ = r"http://boards\.4chan.org/\w+/res/(\d+)" - __description__ = "Downloader for entire 4chan threads" - - def decrypt(self, pyfile): - pagehtml = self.load(pyfile.url) - - images = set(re.findall(r'(images\.4chan\.org/[^/]*/src/[^"<]*)', pagehtml)) - urls = [] - for image in images: - urls.append("http://" + image) - - self.core.files.addLinks(urls, self.pyfile.package().id) diff --git a/module/plugins/crypter/FshareVnFolder.py b/module/plugins/crypter/FshareVnFolder.py deleted file mode 100644 index 2515e7edd..000000000 --- a/module/plugins/crypter/FshareVnFolder.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.internal.SimpleCrypter import SimpleCrypter - -class FshareVnFolder(SimpleCrypter): - __name__ = "FshareVnFolder" - __type__ = "crypter" - __pattern__ = r"http://(www\.)?fshare.vn/folder/.*" - __version__ = "0.01" - __description__ = """Fshare.vn Folder Plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - LINK_PATTERN = r'
  • ' \ No newline at end of file diff --git a/module/plugins/crypter/GooGl.py b/module/plugins/crypter/GooGl.py deleted file mode 100644 index bcb1d7494..000000000 --- a/module/plugins/crypter/GooGl.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################ -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU Affero General Public License as # -# published by the Free Software Foundation, either version 3 of the # -# License, or (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU Affero General Public License for more details. # -# # -# You should have received a copy of the GNU Affero General Public License # -# along with this program. If not, see . # -############################################################################ - -from module.plugins.Crypter import Crypter -from module.common.json_layer import json_loads - - -class GooGl(Crypter): - __name__ = "GooGl" - __type__ = "crypter" - __pattern__ = r"https?://(www\.)?goo\.gl/\w+" - __version__ = "0.01" - __description__ = """Goo.gl Crypter Plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - API_URL = 'https://www.googleapis.com/urlshortener/v1/url' - - def decrypt(self, pyfile): - rep = self.load(self.API_URL, get={'shortUrl': pyfile.url}) - self.logDebug('JSON data: ' + rep) - rep = json_loads(rep) - - if 'longUrl' in rep: - self.core.files.addLinks([rep['longUrl']], self.pyfile.package().id) - else: - self.fail('Unable to expand shortened link') diff --git a/module/plugins/crypter/HoerbuchIn.py b/module/plugins/crypter/HoerbuchIn.py deleted file mode 100644 index 6f23b2eb9..000000000 --- a/module/plugins/crypter/HoerbuchIn.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re - -from module.plugins.Crypter import Crypter -from module.lib.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup - -class HoerbuchIn(Crypter): - __name__ = "HoerbuchIn" - __type__ = "container" - __pattern__ = r"http://(www\.)?hoerbuch\.in/(wp/horbucher/\d+/.+/|tp/out.php\?.+|protection/folder_\d+\.html)" - __version__ = "0.7" - __description__ = """Hoerbuch.in Container Plugin""" - __author_name__ = ("spoob", "mkaay") - __author_mail__ = ("spoob@pyload.org", "mkaay@mkaay.de") - - article = re.compile("http://(www\.)?hoerbuch\.in/wp/horbucher/\d+/.+/") - protection = re.compile("http://(www\.)?hoerbuch\.in/protection/folder_\d+.html") - - def decrypt(self, pyfile): - self.pyfile = pyfile - - if self.article.match(self.pyfile.url): - src = self.load(self.pyfile.url) - soup = BeautifulSoup(src, convertEntities=BeautifulStoneSoup.HTML_ENTITIES) - - abookname = soup.find("a", attrs={"rel": "bookmark"}).text - for a in soup.findAll("a", attrs={"href": self.protection}): - package = "%s (%s)" % (abookname, a.previousSibling.previousSibling.text[:-1]) - links = self.decryptFolder(a["href"]) - - self.packages.append((package, links, self.pyfile.package().folder)) - else: - links = self.decryptFolder(self.pyfile.url) - - self.packages.append((self.pyfile.package().name, links, self.pyfile.package().folder)) - - def decryptFolder(self, url): - m = self.protection.search(url) - if not m: - self.fail("Bad URL") - url = m.group(0) - - self.pyfile.url = url - src = self.req.load(url, post={"viewed": "adpg"}) - - links = [] - pattern = re.compile(r'
    ", src) - - if not len(links) > 0: - self.retry() - - self.correctCaptcha() - - cleanedLinks = [] - for link in links: - if link.startswith("http://dontknow.me/at/?"): - cleanedLink = urllib.unquote(link[23:]) - else: - cleanedLink = link - self.logDebug("Link: %s" % cleanedLink) - cleanedLinks.append(cleanedLink) - - self.logDebug("Decrypted %d links" % len(links)) - - self.pyfile.package().password = "iload.to" - self.packages.append((self.pyfile.package().name, cleanedLinks, self.pyfile.package().folder)) \ No newline at end of file diff --git a/module/plugins/crypter/LetitbitNetFolder.py b/module/plugins/crypter/LetitbitNetFolder.py deleted file mode 100644 index 68aad9dd7..000000000 --- a/module/plugins/crypter/LetitbitNetFolder.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter - - -class LetitbitNetFolder(Crypter): - __name__ = "LetitbitNetFolder" - __type__ = "crypter" - __pattern__ = r"http://(?:www\.)?letitbit.net/folder/\w+" - __version__ = "0.1" - __description__ = """Letitbit.net Folder Plugin""" - __author_name__ = ("DHMH", "z00nx") - __author_mail__ = ("webmaster@pcProfil.de", "z00nx0@gmail.com") - - FOLDER_PATTERN = r'(.*)
    ' - LINK_PATTERN = r'    ' - - def decrypt(self, pyfile): - html = self.load(self.pyfile.url) - - new_links = [] - - folder = re.search(self.FOLDER_PATTERN, html, re.DOTALL) - if folder is None: - self.fail("Parse error (FOLDER)") - - new_links.extend(re.findall(self.LINK_PATTERN, folder.group(0))) - - if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) - else: - self.fail('Could not extract any links') diff --git a/module/plugins/crypter/LinkList.py b/module/plugins/crypter/LinkList.py deleted file mode 100644 index ebfa373eb..000000000 --- a/module/plugins/crypter/LinkList.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.plugins.Crypter import Crypter, Package - -class LinkList(Crypter): - __name__ = "LinkList" - __version__ = "0.11" - __pattern__ = r".+\.txt$" - __description__ = """Read Link Lists in txt format""" - __author_name__ = ("spoob", "jeix") - __author_mail__ = ("spoob@pyload.org", "jeix@hasnomail.com") - - # method declaration is needed here - def decryptURL(self, url): - return Crypter.decryptURL(self, url) - - def decryptFile(self, content): - links = content.splitlines() - - curPack = "default" - packages = {curPack:[]} - - for link in links: - link = link.strip() - if not link: continue - - if link.startswith(";"): - continue - if link.startswith("[") and link.endswith("]"): - # new package - curPack = link[1:-1] - packages[curPack] = [] - continue - packages[curPack].append(link) - - # empty packages fix - delete = [] - - for key,value in packages.iteritems(): - if not value: - delete.append(key) - - for key in delete: - del packages[key] - - urls = [] - - for name, links in packages.iteritems(): - if name == "default": - urls.extend(links) - else: - urls.append(Package(name, links)) - - return urls \ No newline at end of file diff --git a/module/plugins/crypter/LinkSaveIn.py b/module/plugins/crypter/LinkSaveIn.py deleted file mode 100644 index 30cc61055..000000000 --- a/module/plugins/crypter/LinkSaveIn.py +++ /dev/null @@ -1,227 +0,0 @@ -# -*- coding: utf-8 -*- - -# -# v2.01 - hagg -# * cnl2 and web links are skipped if JS is not available (instead of failing the package) -# * only best available link source is used (priority: cnl2>rsdf>ccf>dlc>web -# - -from Crypto.Cipher import AES -from module.plugins.Crypter import Crypter -from module.unescape import unescape -import base64 -import binascii -import re - -class LinkSaveIn(Crypter): - __name__ = "LinkSaveIn" - __type__ = "crypter" - __pattern__ = r"http://(www\.)?linksave.in/(?P\w+)$" - __version__ = "2.01" - __description__ = """LinkSave.in Crypter Plugin""" - __author_name__ = ("fragonib") - __author_mail__ = ("fragonib[AT]yahoo[DOT]es") - - # Constants - _JK_KEY_ = "jk" - _CRYPTED_KEY_ = "crypted" - HOSTER_DOMAIN = "linksave.in" - - def setup(self): - self.html = None - self.fileid = None - self.captcha = False - self.package = None - self.preferred_sources = ['cnl2', 'rsdf', 'ccf', 'dlc', 'web'] - - def decrypt(self, pyfile): - - # Init - self.package = pyfile.package() - self.fileid = re.match(self.__pattern__, pyfile.url).group('id') - self.req.cj.setCookie(self.HOSTER_DOMAIN, "Linksave_Language", "english") - - # Request package - self.html = self.load(self.pyfile.url) - if not self.isOnline(): - self.offline() - - # Check for protection - if self.isPasswordProtected(): - self.unlockPasswordProtection() - self.handleErrors() - - if self.isCaptchaProtected(): - self.captcha = True - self.unlockCaptchaProtection() - self.handleErrors() - - # Get package name and folder - (package_name, folder_name) = self.getPackageInfo() - - # Extract package links - package_links = [] - for type_ in self.preferred_sources: - package_links.extend(self.handleLinkSource(type_)) - if package_links: # use only first source which provides links - break - package_links = set(package_links) - - # Pack - if package_links: - self.packages = [(package_name, package_links, folder_name)] - else: - self.fail('Could not extract any links') - - def isOnline(self): - if "Error 404 - Folder not found!" in self.html: - self.logDebug("File not found") - return False - return True - - def isPasswordProtected(self): - if re.search(r'''Captcha:" in self.html: - self.logDebug("Links are captcha protected") - return True - return False - - def unlockPasswordProtection(self): - password = self.getPassword() - self.logDebug("Submitting password [%s] for protected links" % password) - post = {"id": self.fileid, "besucherpasswort": password, 'login': 'submit'} - self.html = self.load(self.pyfile.url, post=post) - - def unlockCaptchaProtection(self): - captcha_hash = re.search(r'name="hash" value="([^"]+)', self.html).group(1) - captcha_url = re.search(r'src=".(/captcha/cap.php\?hsh=[^"]+)', self.html).group(1) - captcha_code = self.decryptCaptcha("http://linksave.in" + captcha_url, forceUser=True) - self.html = self.load(self.pyfile.url, post={"id": self.fileid, "hash": captcha_hash, "code": captcha_code}) - - def getPackageInfo(self): - name = self.pyfile.package().name - folder = self.pyfile.package().folder - self.logDebug("Defaulting to pyfile name [%s] and folder [%s] for package" % (name, folder)) - return name, folder - - def handleErrors(self): - if "The visitorpassword you have entered is wrong" in self.html: - self.logDebug("Incorrect password, please set right password on 'Edit package' form and retry") - self.fail("Incorrect password, please set right password on 'Edit package' form and retry") - - if self.captcha: - if "Wrong code. Please retry" in self.html: - self.logDebug("Invalid captcha, retrying") - self.invalidCaptcha() - self.retry() - else: - self.correctCaptcha() - - def handleLinkSource(self, type_): - if type_ == 'cnl2': - return self.handleCNL2() - elif type_ in ('rsdf', 'ccf', 'dlc'): - return self.handleContainer(type_) - elif type_ == 'web': - return self.handleWebLinks() - else: - self.fail('unknown source type "%s" (this is probably a bug)' % type_) - - def handleWebLinks(self): - package_links = [] - self.logDebug("Search for Web links") - if not self.js: - self.logDebug("no JS -> skip Web links") - else: - #@TODO: Gather paginated web links - pattern = r'(.*)', response)[-1] - jseval = self.js.eval("document = { write: function(e) { return e; } }; %s" % jscode) - dlLink = re.search(r'http://linksave\.in/dl-\w+', jseval).group(0) - self.logDebug("JsEngine returns value [%s] for redirection link" % dlLink) - response = self.load(dlLink) - link = unescape(re.search(r'') - for pattern in patterns: - rexpr = re.compile(pattern, re.DOTALL) - content = re.sub(rexpr, "", content) - return content - - def removeContainers(self,package_links): - tmp_package_links = package_links[:] - for link in tmp_package_links: - self.logDebug(link) - if ".dlc" in link or ".ccf" in link or ".rsdf" in link: - self.logDebug("Removing [%s] from package_links" % link) - package_links.remove(link) - - if len(package_links) > 0: - return package_links - else: - return tmp_package_links - - def isOnline(self): - if "Your folder does not exist" in self.cleanedHtml: - self.logDebug("File not found") - return False - return True - - def isProtected(self): - if re.search(r'''''', self.cleanedHtml): - self.logDebug("Links are protected") - return True - return False - - def getPackageInfo(self): - title_re = r'

    (?P[^<]+).*?</span>.*?</h2>' - m = re.findall(title_re, self.html, re.DOTALL) - if m is not None: - title = m[-1].strip() - name = folder = title - self.logDebug("Found name [%s] and folder [%s] in package info" % (name, folder)) - else: - name = self.package.name - folder = self.package.folder - self.logDebug("Package info not found, defaulting to pyfile name [%s] and folder [%s]" % (name, folder)) - return name, folder - - def unlockProtection(self): - - postData = {} - - form = re.search(r'''<form\ name="protected"(.*?)</form>''', self.cleanedHtml, re.DOTALL).group(1) - - # Submit package password - if "password" in form: - password = self.getPassword() - self.logDebug("Submitting password [%s] for protected links" % password) - postData['password'] = password - - # Resolve anicaptcha - if "anicaptcha" in form: - self.captcha = True - self.logDebug("Captcha protected, resolving captcha") - captchaUri = re.search(r'src="(/temp/anicaptcha/[^"]+)', form).group(1) - captcha = self.decryptCaptcha("http://ncrypt.in" + captchaUri) - self.logDebug("Captcha resolved [%s]" % captcha) - postData['captcha'] = captcha - - # Resolve recaptcha - if "recaptcha" in form: - self.captcha = True - id = re.search(r'\?k=(.*?)"', form).group(1) - self.logDebug("Resolving ReCaptcha with key [%s]" % id) - recaptcha = ReCaptcha(self) - challenge, code = recaptcha.challenge(id) - postData['recaptcha_challenge_field'] = challenge - postData['recaptcha_response_field'] = code - - # Resolve circlecaptcha - if "circlecaptcha" in form: - self.captcha = True - self.logDebug("Captcha protected") - captcha_img_url = "http://ncrypt.in/classes/captcha/circlecaptcha.php" - coords = self.decryptCaptcha(captcha_img_url, forceUser=True, imgtype="png", result_type='positional') - self.logDebug("Captcha resolved, coords [%s]" % str(coords)) - self.captcha_post_url = self.pyfile.url - - postData['circle.x'] = coords[0] - postData['circle.y'] = coords[1] - - - # Unlock protection - postData['submit_protected'] = 'Continue to folder ' - return self.load(self.pyfile.url, post=postData) - - def handleErrors(self): - - if "This password is invalid!" in self.cleanedHtml: - self.logDebug("Incorrect password, please set right password on 'Edit package' form and retry") - self.fail("Incorrect password, please set right password on 'Edit package' form and retry") - - if self.captcha: - if "The securitycheck was wrong!" in self.cleanedHtml: - self.logDebug("Invalid captcha, retrying") - self.invalidCaptcha() - self.retry() - else: - self.correctCaptcha() - - def handleWebLinks(self): - package_links = [] - self.logDebug("Handling Web links") - - pattern = r"(http://ncrypt\.in/link-.*?=)" - links = re.findall(pattern, self.html) - self.logDebug("Decrypting %d Web links" % len(links)) - for i, link in enumerate(links): - self.logDebug("Decrypting Web link %d, %s" % (i+1, link)) - try: - url = link.replace("link-", "frame-") - link = self.load(url, just_header=True)['location'] - package_links.append(link) - except Exception, detail: - self.logDebug("Error decrypting Web link %s, %s" % (link, detail)) - return package_links - - def handleContainers(self): - package_links = [] - self.logDebug("Handling Container links") - - pattern = r"/container/(rsdf|dlc|ccf)/([a-z0-9]+)" - containersLinks = re.findall(pattern, self.html) - self.logDebug("Decrypting %d Container links" % len(containersLinks)) - for containerLink in containersLinks: - link = "http://ncrypt.in/container/%s/%s.%s" % (containerLink[0], containerLink[1], containerLink[0]) - package_links.append(link) - return package_links - - def handleCNL2(self): - package_links = [] - self.logDebug("Handling CNL2 links") - - if 'cnl2_output' in self.cleanedHtml: - try: - (vcrypted, vjk) = self._getCipherParams() - for (crypted, jk) in zip(vcrypted, vjk): - package_links.extend(self._getLinks(crypted, jk)) - except: - self.fail("Unable to decrypt CNL2 links") - return package_links - - def _getCipherParams(self): - - pattern = r'<input.*?name="%s".*?value="(.*?)"' - - # Get jk - jk_re = pattern % NCryptIn._JK_KEY_ - vjk = re.findall(jk_re, self.html) - - # Get crypted - crypted_re = pattern % NCryptIn._CRYPTED_KEY_ - vcrypted = re.findall(crypted_re, self.html) - - # Log and return - self.logDebug("Detected %d crypted blocks" % len(vcrypted)) - return vcrypted, vjk - - def _getLinks(self, crypted, jk): - - # Get key - jreturn = self.js.eval("%s f()" % jk) - self.logDebug("JsEngine returns value [%s]" % jreturn) - key = binascii.unhexlify(jreturn) - - # Decode crypted - crypted = base64.standard_b64decode(crypted) - - # Decrypt - Key = key - IV = key - obj = AES.new(Key, AES.MODE_CBC, IV) - text = obj.decrypt(crypted) - - # Extract links - text = text.replace("\x00", "").replace("\r", "") - links = text.split("\n") - links = filter(lambda x: x != "", links) - - # Log and return - self.logDebug("Block has %d links" % len(links)) - return links diff --git a/module/plugins/crypter/NetfolderIn.py b/module/plugins/crypter/NetfolderIn.py deleted file mode 100644 index c5c602c27..000000000 --- a/module/plugins/crypter/NetfolderIn.py +++ /dev/null @@ -1,71 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from module.plugins.internal.SimpleCrypter import SimpleCrypter - - -class NetfolderIn(SimpleCrypter): - __name__ = "NetfolderIn" - __type__ = "crypter" - __pattern__ = r"http://(?:www\.)?netfolder.in/((?P<id1>\w+)/\w+|folder.php\?folder_id=(?P<id2>\w+))" - __version__ = "0.6" - __description__ = """NetFolder Crypter Plugin""" - __author_name__ = ("RaNaN", "fragonib") - __author_mail__ = ("RaNaN@pyload.org", "fragonib[AT]yahoo[DOT]es") - - TITLE_PATTERN = r'<div class="Text">Inhalt des Ordners <span(.*)>(?P<title>.+)</span></div>' - - def decrypt(self, pyfile): - # Request package - self.html = self.load(pyfile.url) - - # Check for password protection - if self.isPasswordProtected(): - self.html = self.submitPassword() - if self.html is None: - self.fail("Incorrect password, please set right password on Add package form and retry") - - # Get package name and folder - (package_name, folder_name) = self.getPackageNameAndFolder() - - # Get package links - package_links = self.getLinks() - - # Set package - self.packages = [(package_name, package_links, folder_name)] - - def isPasswordProtected(self): - - if '<input type="password" name="password"' in self.html: - self.logDebug("Links are password protected") - return True - return False - - def submitPassword(self): - # Gather data - try: - m = re.match(self.__pattern__, self.pyfile.url) - id = max(m.group('id1'), m.group('id2')) - except AttributeError: - self.logDebug("Unable to get package id from url [%s]" % self.pyfile.url) - return - url = "http://netfolder.in/folder.php?folder_id=" + id - password = self.getPassword() - - # Submit package password - post = {'password': password, 'save': 'Absenden'} - self.logDebug("Submitting password [%s] for protected links with id [%s]" % (password, id)) - html = self.load(url, {}, post) - - # Check for invalid password - if '<div class="InPage_Error">' in html: - self.logDebug("Incorrect password, please set right password on Edit package form and retry") - return None - - return html - - def getLinks(self): - links = re.search(r'name="list" value="(.*?)"', self.html).group(1).split(",") - self.logDebug("Package has %d links" % len(links)) - return links diff --git a/module/plugins/crypter/OneKhDe.py b/module/plugins/crypter/OneKhDe.py deleted file mode 100644 index c77203187..000000000 --- a/module/plugins/crypter/OneKhDe.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re - -from module.unescape import unescape -from module.plugins.Crypter import Crypter - -class OneKhDe(Crypter): - __name__ = "OneKhDe" - __type__ = "container" - __pattern__ = r"http://(www\.)?1kh.de/f/" - __version__ = "0.1" - __description__ = """1kh.de Container Plugin""" - __author_name__ = ("spoob") - __author_mail__ = ("spoob@pyload.org") - - def __init__(self, parent): - Crypter.__init__(self, parent) - self.parent = parent - self.html = None - - def file_exists(self): - """ returns True or False - """ - return True - - def proceed(self, url, location): - url = self.parent.url - self.html = self.req.load(url) - temp_links = [] - link_ids = re.findall(r"<a id=\"DownloadLink_(\d*)\" href=\"http://1kh.de/", self.html) - for id in link_ids: - new_link = unescape(re.search("width=\"100%\" src=\"(.*)\"></iframe>", self.req.load("http://1kh.de/l/" + id)).group(1)) - temp_links.append(new_link) - self.links = temp_links diff --git a/module/plugins/crypter/OronComFolder.py b/module/plugins/crypter/OronComFolder.py deleted file mode 100755 index 726371966..000000000 --- a/module/plugins/crypter/OronComFolder.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from module.plugins.Crypter import Crypter - -class OronComFolder(Crypter): - __name__ = "OronComFolder" - __type__ = "crypter" - __pattern__ = r"http://(?:www\.)?oron.com/folder/\w+" - __version__ = "0.2" - __description__ = """Oron.com Folder Plugin""" - __author_name__ = ("DHMH") - __author_mail__ = ("webmaster@pcProfil.de") - - FOLDER_PATTERN = r'<table(?:.*)class="tbl"(?:.*)>(?:.*)<table(?:.*)class="tbl2"(?:.*)>(?P<body>.*)</table>(?:.*)</table>' - LINK_PATTERN = r'<a href="([^"]+)" target="_blank">' - - def decryptURL(self, url): - html = self.load(url) - - new_links = [] - - if 'No such folder exist' in html: - # Don't fail because if there's more than a folder for this package - # and only one of them fails, no urls at all will be added. - self.logWarning("Folder does not exist") - return new_links - - folder = re.search(self.FOLDER_PATTERN, html, re.DOTALL) - if folder is None: - # Don't fail because if there's more than a folder for this package - # and only one of them fails, no urls at all will be added. - self.logWarning("Parse error (FOLDER)") - return new_links - - new_links.extend(re.findall(self.LINK_PATTERN, folder.group(0))) - - if new_links: - self.logDebug("Found %d new links" % len(new_links)) - return new_links - else: - # Don't fail because if there's more than a folder for this package - # and only one of them fails, no urls at all will be added. - self.logWarning('Could not extract any links') - return new_links diff --git a/module/plugins/crypter/QuickshareCzFolder.py b/module/plugins/crypter/QuickshareCzFolder.py deleted file mode 100644 index 6cb049935..000000000 --- a/module/plugins/crypter/QuickshareCzFolder.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter - -class QuickshareCzFolder(Crypter): - __name__ = "QuickshareCzFolder" - __type__ = "crypter" - __pattern__ = r"http://(www\.)?quickshare.cz/slozka-\d+.*" - __version__ = "0.1" - __description__ = """Quickshare.cz Folder Plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - - FOLDER_PATTERN = r'<textarea[^>]*>(.*?)</textarea>' - LINK_PATTERN = r'(http://www.quickshare.cz/\S+)' - - def decrypt(self, pyfile): - html = self.load(self.pyfile.url) - - new_links = [] - found = re.search(self.FOLDER_PATTERN, html, re.DOTALL) - if found is None: self.fail("Parse error (FOLDER)") - new_links.extend(re.findall(self.LINK_PATTERN, found.group(1))) - - if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) - else: - self.fail('Could not extract any links') \ No newline at end of file diff --git a/module/plugins/crypter/RSDF.py b/module/plugins/crypter/RSDF.py deleted file mode 100644 index cbc9864b1..000000000 --- a/module/plugins/crypter/RSDF.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import base64 -import binascii -import re - -from module.plugins.Crypter import Crypter - -class RSDF(Crypter): - __name__ = "RSDF" - __version__ = "0.21" - __pattern__ = r".*\.rsdf" - __description__ = """RSDF Container Decode Plugin""" - __author_name__ = ("RaNaN", "spoob") - __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org") - - - def decrypt(self, pyfile): - - from Crypto.Cipher import AES - - infile = pyfile.url.replace("\n", "") - Key = binascii.unhexlify('8C35192D964DC3182C6F84F3252239EB4A320D2500000000') - - IV = binascii.unhexlify('FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF') - IV_Cipher = AES.new(Key, AES.MODE_ECB) - IV = IV_Cipher.encrypt(IV) - - obj = AES.new(Key, AES.MODE_CFB, IV) - - rsdf = open(infile, 'r') - - data = rsdf.read() - rsdf.close() - - if re.search(r"<title>404 - Not Found", data) is None: - data = binascii.unhexlify(''.join(data.split())) - data = data.splitlines() - - links = [] - for link in data: - link = base64.b64decode(link) - link = obj.decrypt(link) - decryptedUrl = link.replace('CCF: ', '') - links.append(decryptedUrl) - - self.log.debug("%s: adding package %s with %d links" % (self.__name__,pyfile.package().name,len(links))) - self.packages.append((pyfile.package().name, links)) diff --git a/module/plugins/crypter/RSLayerCom.py b/module/plugins/crypter/RSLayerCom.py deleted file mode 100644 index 6e4266f2e..000000000 --- a/module/plugins/crypter/RSLayerCom.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re - -from module.plugins.Crypter import Crypter -from module.lib.BeautifulSoup import BeautifulSoup -from module.unescape import unescape - -class RSLayerCom(Crypter): - __name__ = "RSLayerCom" - __type__ = "container" - __pattern__ = r"http://(www\.)?rs-layer.com/directory-" - __config__ = [] - __version__ = "0.2" - __description__ = """RS-Layer.com Container Plugin""" - __author_name__ = ("hzpz") - __author_mail__ = ("none") - - def decrypt(self, pyfile): - url = pyfile.url - src = self.req.load(str(url)) - - soup = BeautifulSoup(src) - captchaTag = soup.find("img", attrs={"id": "captcha_image"}) - if captchaTag: - captchaUrl = "http://rs-layer.com/" + captchaTag["src"] - self.logDebug("Captcha URL: %s" % captchaUrl) - result = self.decryptCaptcha(str(captchaUrl), imgtype="png") - captchaInput = soup.find("input", attrs={"id": "captcha"}) - self.req.lastUrl = url - src = self.req.load(str(url), post={'captcha_input': result, 'image_name': captchaTag["src"]}) - - link_ids = re.findall(r"onclick=\"getFile\(\'([0-9]{7}-.{8})\'\);changeBackgroundColor", src) - - if not len(link_ids) > 0: - self.retry() - - self.correctCaptcha() - - links = [] - for id in link_ids: - self.logDebug("ID: %s" % id) - new_link = unescape(re.search(r"' - - - def setup(self): - self.fileid = None - self.package = None - self.password = None - self.html = None - self.captcha = False - - def decrypt(self, pyfile): - - # Init - self.initPackage(pyfile) - - # Request package - self.requestPackage() - - # Check for online - if not self.isOnline(): - self.offline() - - # Check for protection - if self.isPasswordProtected(): - self.unlockPasswordProtection() - self.handleErrors() - - if self.isCaptchaProtected(): - self.captcha = True - self.unlockCaptchaProtection() - self.handleErrors() - - # Get package name and folder - (package_name, folder_name) = self.getPackageInfo() - - # Extract package links - package_links = [] - for sources in self.PREFERRED_LINK_SOURCES: - package_links.extend(self.handleLinkSource(sources)) - if package_links: # use only first source which provides links - break - package_links = set(package_links) - - # Pack - if package_links: - self.packages = [(package_name, package_links, folder_name)] - else: - self.fail('Could not extract any links') - - def initPackage(self, pyfile): - self.fileid = re.match(self.__pattern__, pyfile.url).group('id') - self.package = pyfile.package() - self.password = self.getPassword() - self.url = pyfile.url - - def requestPackage(self): - self.html = self.load(self.url, decode = True) - - def isOnline(self): - if self.OFFLINE_TOKEN in self.html: - self.logDebug("File not found") - return False - return True - - def isPasswordProtected(self): - if self.PASSWORD_TOKEN in self.html: - self.logDebug("Links are password protected") - return True - - def isCaptchaProtected(self): - if self.CAPTCHA_TOKEN in self.html: - self.logDebug("Links are captcha protected") - return True - return False - - def unlockPasswordProtection(self): - self.logDebug("Submitting password [%s] for protected links" % self.password) - passwd_url = self.PASSWORD_SUBMIT_URL + "?id=%s" % self.fileid - passwd_data = { 'id': self.fileid, 'password': self.password, 'pw': 'submit' } - self.html = self.load(passwd_url, post=passwd_data, decode=True) - - def unlockCaptchaProtection(self): - self.logDebug("Request user positional captcha resolving") - captcha_img_url = self.CAPTCHA_IMG_URL + "?id=%s" % self.fileid - coords = self.decryptCaptcha(captcha_img_url, forceUser=True, imgtype="png", result_type='positional') - self.logDebug("Captcha resolved, coords [%s]" % str(coords)) - captcha_post_url = self.CAPTCHA_SUBMIT_URL + "?id=%s" % self.fileid - captcha_post_data = { 'button.x': coords[0], 'button.y': coords[1], 'captcha': 'submit' } - self.html = self.load(captcha_post_url, post=captcha_post_data, decode=True) - - def getPackageInfo(self): - name = folder = None - - # Try to get info from web - m = re.search(self.FILE_TITLE_REGEX, self.html) - if m is not None: - title = m.group(1).strip() - if not self.FILE_NOTITLE in title: - name = folder = title - self.logDebug("Found name [%s] and folder [%s] in package info" % (name, folder)) - - # Fallback to defaults - if not name or not folder: - name = self.package.name - folder = self.package.folder - self.logDebug("Package info not found, defaulting to pyfile name [%s] and folder [%s]" % (name, folder)) - - # Return package info - return name, folder - - def handleErrors(self): - if self.PASSWORD_ERROR_ROKEN in self.html: - msg = "Incorrect password, please set right password on 'Edit package' form and retry" - self.logDebug(msg) - self.fail(msg) - - if self.captcha: - if self.CAPTCHA_ERROR_ROKEN in self.html: - self.logDebug("Invalid captcha, retrying") - self.invalidCaptcha() - self.retry() - else: - self.correctCaptcha() - - def handleLinkSource(self, source): - if source == 'cnl2': - return self.handleCNL2Links() - elif source == 'dlc': - return self.handleDLCLinks() - elif source == 'web': - return self.handleWEBLinks() - else: - self.fail('Unknown source [%s] (this is probably a bug)' % source) - - def handleCNL2Links(self): - self.logDebug("Search for CNL2 links") - package_links = [] - m = re.search(self.CNL2_FORM_REGEX, self.html, re.DOTALL) - if m is not None: - cnl2_form = m.group(1) - try: - (vcrypted, vjk) = self._getCipherParams(cnl2_form) - for (crypted, jk) in zip(vcrypted, vjk): - package_links.extend(self._getLinks(crypted, jk)) - except: - self.logDebug("Unable to decrypt CNL2 links") - return package_links - - def handleDLCLinks(self): - self.logDebug('Search for DLC links') - package_links = [] - m = re.search(self.DLC_LINK_REGEX, self.html) - if m is not None: - container_url = self.DLC_DOWNLOAD_URL + "?id=%s&dlc=1" % self.fileid - self.logDebug("Downloading DLC container link [%s]" % container_url) - try: - dlc = self.load(container_url) - dlc_filename = self.fileid + ".dlc" - dlc_filepath = os.path.join(self.config["general"]["download_folder"], dlc_filename) - f = open(dlc_filepath, "wb") - f.write(dlc) - f.close() - package_links.append(dlc_filepath) - except: - self.logDebug("Unable to download DLC container") - return package_links - - def handleWEBLinks(self): - self.logDebug("Search for WEB links") - package_links = [] - fw_params = re.findall(self.WEB_FORWARD_REGEX, self.html) - self.logDebug("Decrypting %d Web links" % len(fw_params)) - for index, fw_param in enumerate(fw_params): - try: - fw_url = self.WEB_FORWARD_URL + "?%s" % fw_param - self.logDebug("Decrypting Web link %d, %s" % (index+1, fw_url)) - fw_response = self.load(fw_url, decode=True) - dl_link = re.search(self.WEB_LINK_REGEX, fw_response).group('link') - package_links.append(dl_link) - except Exception, detail: - self.logDebug("Error decrypting Web link %s, %s" % (index, detail)) - self.setWait(4) - self.wait() - return package_links - - def _getCipherParams(self, cnl2_form): - - # Get jk - jk_re = self.CNL2_FORMINPUT_REGEX % self.CNL2_JK_KEY - vjk = re.findall(jk_re, cnl2_form, re.IGNORECASE) - - # Get crypted - crypted_re = self.CNL2_FORMINPUT_REGEX % RelinkUs.CNL2_CRYPTED_KEY - vcrypted = re.findall(crypted_re, cnl2_form, re.IGNORECASE) - - # Log and return - self.logDebug("Detected %d crypted blocks" % len(vcrypted)) - return vcrypted, vjk - - def _getLinks(self, crypted, jk): - - # Get key - jreturn = self.js.eval("%s f()" % jk) - self.logDebug("JsEngine returns value [%s]" % jreturn) - key = binascii.unhexlify(jreturn) - - # Decode crypted - crypted = base64.standard_b64decode(crypted) - - # Decrypt - Key = key - IV = key - obj = AES.new(Key, AES.MODE_CBC, IV) - text = obj.decrypt(crypted) - - # Extract links - text = text.replace("\x00", "").replace("\r", "") - links = text.split("\n") - links = filter(lambda x: x != "", links) - - # Log and return - self.logDebug("Package has %d links" % len(links)) - return links diff --git a/module/plugins/crypter/SecuredIn.py b/module/plugins/crypter/SecuredIn.py deleted file mode 100644 index e41896c5f..000000000 --- a/module/plugins/crypter/SecuredIn.py +++ /dev/null @@ -1,334 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from module.plugins.Crypter import Crypter -from module.lib.BeautifulSoup import BeautifulSoup - -from math import ceil - -class SecuredIn(Crypter): - __name__ = "SecuredIn" - __type__ = "container" - __pattern__ = r"http://[\w\.]*?secured\.in/download-[\d]+-[\w]{8}\.html" - __version__ = "0.1" - __description__ = """secured.in Container Plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def __init__(self, parent): - Crypter.__init__(self, parent) - self.parent = parent - self.html = None - self.multi_dl = False - - def file_exists(self): - return True - - def proceed(self, url, location): - links = [] - ajaxUrl = "http://secured.in/ajax-handler.php" - src = self.req.load(url, cookies=True) - soup = BeautifulSoup(src) - img = soup.find("img", attrs={"id":"captcha_img"}) - for i in range(3): - form = soup.find("form", attrs={"id":"frm_captcha"}) - captchaHash = form.find("input", attrs={"id":"captcha_hash"})["value"] - captchaUrl = "http://secured.in/%s" % img["src"] - captchaData = self.req.load(str(captchaUrl), cookies=True) - result = self.waitForCaptcha(captchaData, "jpg") - src = self.req.load(url, cookies=True, post={"captcha_key":result, "captcha_hash":captchaHash}) - soup = BeautifulSoup(src) - img = soup.find("img", attrs={"id":"captcha_img"}) - if not img: - files = soup.findAll("tr", attrs={"id":re.compile("file-\d+")}) - dlIDPattern = re.compile("accessDownload\(\d, \d+, '(.*?)', \d\)") - cypher = self.Cypher() - for cfile in files: - m = dlIDPattern.search(cfile["onclick"]) - if m: - crypted = self.req.load(ajaxUrl, cookies=True, post={"cmd":"download", "download_id":m.group(1)}) - cypher.reset() - link = cypher.cypher(crypted) - links.append(link) - break - self.links = links - - class Cypher(): - def __init__(self): - self.reset() - - def reset(self): - self.iatwbfrd = [ - 0xd1310ba6, 0x98dfb5ac, 0x2ffd72db, 0xd01adfb7, 0xb8e1afed, 0x6a267e96, 0xba7c9045, 0xf12c7f99, 0x24a19947, 0xb3916cf7, 0x0801f2e2, - 0x858efc16, 0x636920d8, 0x71574e69, 0xa458fea3, 0xf4933d7e, 0x0d95748f, 0x728eb658, 0x718bcd58, 0x82154aee, 0x7b54a41d, 0xc25a59b5, - 0x9c30d539, 0x2af26013, 0xc5d1b023, 0x286085f0, 0xca417918, 0xb8db38ef, 0x8e79dcb0, 0x603a180e, 0x6c9e0e8b, 0xb01e8a3e, 0xd71577c1, - 0xbd314b27, 0x78af2fda, 0x55605c60, 0xe65525f3, 0xaa55ab94, 0x57489862, 0x63e81440, 0x55ca396a, 0x2aab10b6, 0xb4cc5c34, 0x1141e8ce, - 0xa15486af, 0x7c72e993, 0xb3ee1411, 0x636fbc2a, 0x2ba9c55d, 0x741831f6, 0xce5c3e16, 0x9b87931e, 0xafd6ba33, 0x6c24cf5c, 0x7a325381, - 0x28958677, 0x3b8f4898, 0x6b4bb9af, 0xc4bfe81b, 0x66282193, 0x61d809cc, 0xfb21a991, 0x487cac60, 0x5dec8032, 0xef845d5d, 0xe98575b1, - 0xdc262302, 0xeb651b88, 0x23893e81, 0xd396acc5, 0x0f6d6ff3, 0x83f44239, 0x2e0b4482, 0xa4842004, 0x69c8f04a, 0x9e1f9b5e, 0x21c66842, - 0xf6e96c9a, 0x670c9c61, 0xabd388f0, 0x6a51a0d2, 0xd8542f68, 0x960fa728, 0xab5133a3, 0x6eef0b6c, 0x137a3be4, 0xba3bf050, 0x7efb2a98, - 0xa1f1651d, 0x39af0176, 0x66ca593e, 0x82430e88, 0x8cee8619, 0x456f9fb4, 0x7d84a5c3, 0x3b8b5ebe, 0xe06f75d8, 0x85c12073, 0x401a449f, - 0x56c16aa6, 0x4ed3aa62, 0x363f7706, 0x1bfedf72, 0x429b023d, 0x37d0d724, 0xd00a1248, 0xdb0fead3, 0x49f1c09b, 0x075372c9, 0x80991b7b, - 0x25d479d8, 0xf6e8def7, 0xe3fe501a, 0xb6794c3b, 0x976ce0bd, 0x04c006ba, 0xc1a94fb6, 0x409f60c4, 0x5e5c9ec2, 0x196a2463, 0x68fb6faf, - 0x3e6c53b5, 0x1339b2eb, 0x3b52ec6f, 0x6dfc511f, 0x9b30952c, 0xcc814544, 0xaf5ebd09, 0xbee3d004, 0xde334afd, 0x660f2807, 0x192e4bb3, - 0xc0cba857, 0x45c8740f, 0xd20b5f39, 0xb9d3fbdb, 0x5579c0bd, 0x1a60320a, 0xd6a100c6, 0x402c7279, 0x679f25fe, 0xfb1fa3cc, 0x8ea5e9f8, - 0xdb3222f8, 0x3c7516df, 0xfd616b15, 0x2f501ec8, 0xad0552ab, 0x323db5fa, 0xfd238760, 0x53317b48, 0x3e00df82, 0x9e5c57bb, 0xca6f8ca0, - 0x1a87562e, 0xdf1769db, 0xd542a8f6, 0x287effc3, 0xac6732c6, 0x8c4f5573, 0x695b27b0, 0xbbca58c8, 0xe1ffa35d, 0xb8f011a0, 0x10fa3d98, - 0xfd2183b8, 0x4afcb56c, 0x2dd1d35b, 0x9a53e479, 0xb6f84565, 0xd28e49bc, 0x4bfb9790, 0xe1ddf2da, 0xa4cb7e33, 0x62fb1341, 0xcee4c6e8, - 0xef20cada, 0x36774c01, 0xd07e9efe, 0x2bf11fb4, 0x95dbda4d, 0xae909198, 0xeaad8e71, 0x6b93d5a0, 0xd08ed1d0, 0xafc725e0, 0x8e3c5b2f, - 0x8e7594b7, 0x8ff6e2fb, 0xf2122b64, 0x8888b812, 0x900df01c, 0x4fad5ea0, 0x688fc31c, 0xd1cff191, 0xb3a8c1ad, 0x2f2f2218, 0xbe0e1777, - 0xea752dfe, 0x8b021fa1, 0xe5a0cc0f, 0xb56f74e8, 0x18acf3d6, 0xce89e299, 0xb4a84fe0, 0xfd13e0b7, 0x7cc43b81, 0xd2ada8d9, 0x165fa266, - 0x80957705, 0x93cc7314, 0x211a1477, 0xe6ad2065, 0x77b5fa86, 0xc75442f5, 0xfb9d35cf, 0xebcdaf0c, 0x7b3e89a0, 0xd6411bd3, 0xae1e7e49, - 0x00250e2d, 0x2071b35e, 0x226800bb, 0x57b8e0af, 0x2464369b, 0xf009b91e, 0x5563911d, 0x59dfa6aa, 0x78c14389, 0xd95a537f, 0x207d5ba2, - 0x02e5b9c5, 0x83260376, 0x6295cfa9, 0x11c81968, 0x4e734a41, 0xb3472dca, 0x7b14a94a, 0x1b510052, 0x9a532915, 0xd60f573f, 0xbc9bc6e4, - 0x2b60a476, 0x81e67400, 0x08ba6fb5, 0x571be91f, 0xf296ec6b, 0x2a0dd915, 0xb6636521, 0xe7b9f9b6, 0xff34052e, 0xc5855664, 0x53b02d5d, - 0xa99f8fa1, 0x08ba4799, 0x6e85076a - ] - - self.olkemfjq = [ - 0x243f6a88, 0x85a308d3, 0x13198a2e, 0x03707344, 0xa4093822, 0x299f31d0, 0x082efa98, 0xec4e6c89, 0x452821e6, 0x38d01377, 0xbe5466cf, - 0x34e90c6c, 0xc0ac29b7, 0xc97c50dd, 0x3f84d5b5, 0xb5470917, 0x9216d5d9, 0x8979fb1b - ] - - self.oqlaoymh = 0 - self.oqmykrna = 0 - self.pqmyzkid = 0 - self.pldmjnde = 0 - self.ldiwkqly = 0 - - self.plkodnyq = [ - 0x3a39ce37, 0xd3faf5cf, 0xabc27737, 0x5ac52d1b, 0x5cb0679e, 0x4fa33742, 0xd3822740, 0x99bc9bbe, 0xd5118e9d, 0xbf0f7315, 0xd62d1c7e, - 0xc700c47b, 0xb78c1b6b, 0x21a19045, 0xb26eb1be, 0x6a366eb4, 0x5748ab2f, 0xbc946e79, 0xc6a376d2, 0x6549c2c8, 0x530ff8ee, 0x468dde7d, - 0xd5730a1d, 0x4cd04dc6, 0x2939bbdb, 0xa9ba4650, 0xac9526e8, 0xbe5ee304, 0xa1fad5f0, 0x6a2d519a, 0x63ef8ce2, 0x9a86ee22, 0xc089c2b8, - 0x43242ef6, 0xa51e03aa, 0x9cf2d0a4, 0x83c061ba, 0x9be96a4d, 0x8fe51550, 0xba645bd6, 0x2826a2f9, 0xa73a3ae1, 0x4ba99586, 0xef5562e9, - 0xc72fefd3, 0xf752f7da, 0x3f046f69, 0x77fa0a59, 0x80e4a915, 0x87b08601, 0x9b09e6ad, 0x3b3ee593, 0xe990fd5a, 0x9e34d797, 0x2cf0b7d9, - 0x022b8b51, 0x96d5ac3a, 0x017da67d, 0xd1cf3ed6, 0x7c7d2d28, 0x1f9f25cf, 0xadf2b89b, 0x5ad6b472, 0x5a88f54c, 0xe029ac71, 0xe019a5e6, - 0x47b0acfd, 0xed93fa9b, 0xe8d3c48d, 0x283b57cc, 0xf8d56629, 0x79132e28, 0x785f0191, 0xed756055, 0xf7960e44, 0xe3d35e8c, 0x15056dd4, - 0x88f46dba, 0x03a16125, 0x0564f0bd, 0xc3eb9e15, 0x3c9057a2, 0x97271aec, 0xa93a072a, 0x1b3f6d9b, 0x1e6321f5, 0xf59c66fb, 0x26dcf319, - 0x7533d928, 0xb155fdf5, 0x03563482, 0x8aba3cbb, 0x28517711, 0xc20ad9f8, 0xabcc5167, 0xccad925f, 0x4de81751, 0x3830dc8e, 0x379d5862, - 0x9320f991, 0xea7a90c2, 0xfb3e7bce, 0x5121ce64, 0x774fbe32, 0xa8b6e37e, 0xc3293d46, 0x48de5369, 0x6413e680, 0xa2ae0810, 0xdd6db224, - 0x69852dfd, 0x09072166, 0xb39a460a, 0x6445c0dd, 0x586cdecf, 0x1c20c8ae, 0x5bbef7dd, 0x1b588d40, 0xccd2017f, 0x6bb4e3bb, 0xdda26a7e, - 0x3a59ff45, 0x3e350a44, 0xbcb4cdd5, 0x72eacea8, 0xfa6484bb, 0x8d6612ae, 0xbf3c6f47, 0xd29be463, 0x542f5d9e, 0xaec2771b, 0xf64e6370, - 0x740e0d8d, 0xe75b1357, 0xf8721671, 0xaf537d5d, 0x4040cb08, 0x4eb4e2cc, 0x34d2466a, 0x0115af84, 0xe1b00428, 0x95983a1d, 0x06b89fb4, - 0xce6ea048, 0x6f3f3b82, 0x3520ab82, 0x011a1d4b, 0x277227f8, 0x611560b1, 0xe7933fdc, 0xbb3a792b, 0x344525bd, 0xa08839e1, 0x51ce794b, - 0x2f32c9b7, 0xa01fbac9, 0xe01cc87e, 0xbcc7d1f6, 0xcf0111c3, 0xa1e8aac7, 0x1a908749, 0xd44fbd9a, 0xd0dadecb, 0xd50ada38, 0x0339c32a, - 0xc6913667, 0x8df9317c, 0xe0b12b4f, 0xf79e59b7, 0x43f5bb3a, 0xf2d519ff, 0x27d9459c, 0xbf97222c, 0x15e6fc2a, 0x0f91fc71, 0x9b941525, - 0xfae59361, 0xceb69ceb, 0xc2a86459, 0x12baa8d1, 0xb6c1075e, 0xe3056a0c, 0x10d25065, 0xcb03a442, 0xe0ec6e0e, 0x1698db3b, 0x4c98a0be, - 0x3278e964, 0x9f1f9532, 0xe0d392df, 0xd3a0342b, 0x8971f21e, 0x1b0a7441, 0x4ba3348c, 0xc5be7120, 0xc37632d8, 0xdf359f8d, 0x9b992f2e, - 0xe60b6f47, 0x0fe3f11d, 0xe54cda54, 0x1edad891, 0xce6279cf, 0xcd3e7e6f, 0x1618b166, 0xfd2c1d05, 0x848fd2c5, 0xf6fb2299, 0xf523f357, - 0xa6327623, 0x93a83531, 0x56cccd02, 0xacf08162, 0x5a75ebb5, 0x6e163697, 0x88d273cc, 0xde966292, 0x81b949d0, 0x4c50901b, 0x71c65614, - 0xe6c6c7bd, 0x327a140a, 0x45e1d006, 0xc3f27b9a, 0xc9aa53fd, 0x62a80f00, 0xbb25bfe2, 0x35bdd2f6, 0x71126905, 0xb2040222, 0xb6cbcf7c, - 0xcd769c2b, 0x53113ec0, 0x1640e3d3, 0x38abbd60, 0x2547adf0, 0xba38209c, 0xf746ce76, 0x77afa1c5, 0x20756060, 0x85cbfe4e, 0x8ae88dd8, - 0x7aaaf9b0, 0x4cf9aa7e, 0x1948c25c, 0x02fb8a8c, 0x01c36ae4, 0xd6ebe1f9, 0x90d4f869, 0xa65cdea0, 0x3f09252d, 0xc208e69f, 0xb74e6132, - 0xce77e25b, 0x578fdfe3, 0x3ac372e6 - ] - - self.pnjzokye = None - - self.thdlpsmy = [ - 0xe93d5a68, 0x948140f7, 0xf64c261c, 0x94692934, 0x411520f7, 0x7602d4f7, 0xbcf46b2e, 0xd4a20068, 0xd4082471, 0x3320f46a, 0x43b7d4b7, - 0x500061af, 0x1e39f62e, 0x97244546, 0x14214f74, 0xbf8b8840, 0x4d95fc1d, 0x96b591af, 0x70f4ddd3, 0x66a02f45, 0xbfbc09ec, 0x03bd9785, - 0x7fac6dd0, 0x31cb8504, 0x96eb27b3, 0x55fd3941, 0xda2547e6, 0xabca0a9a, 0x28507825, 0x530429f4, 0x0a2c86da, 0xe9b66dfb, 0x68dc1462, - 0xd7486900, 0x680ec0a4, 0x27a18dee, 0x4f3ffea2, 0xe887ad8c, 0xb58ce006, 0x7af4d6b6, 0xaace1e7c, 0xd3375fec, 0xce78a399, 0x406b2a42, - 0x20fe9e35, 0xd9f385b9, 0xee39d7ab, 0x3b124e8b, 0x1dc9faf7, 0x4b6d1856, 0x26a36631, 0xeae397b2, 0x3a6efa74, 0xdd5b4332, 0x6841e7f7, - 0xca7820fb, 0xfb0af54e, 0xd8feb397, 0x454056ac, 0xba489527, 0x55533a3a, 0x20838d87, 0xfe6ba9b7, 0xd096954b, 0x55a867bc, 0xa1159a58, - 0xcca92963, 0x99e1db33, 0xa62a4a56, 0x3f3125f9, 0x5ef47e1c, 0x9029317c, 0xfdf8e802, 0x04272f70, 0x80bb155c, 0x05282ce3, 0x95c11548, - 0xe4c66d22, 0x48c1133f, 0xc70f86dc, 0x07f9c9ee, 0x41041f0f, 0x404779a4, 0x5d886e17, 0x325f51eb, 0xd59bc0d1, 0xf2bcc18f, 0x41113564, - 0x257b7834, 0x602a9c60, 0xdff8e8a3, 0x1f636c1b, 0x0e12b4c2, 0x02e1329e, 0xaf664fd1, 0xcad18115, 0x6b2395e0, 0x333e92e1, 0x3b240b62, - 0xeebeb922, 0x85b2a20e, 0xe6ba0d99, 0xde720c8c, 0x2da2f728, 0xd0127845, 0x95b794fd, 0x647d0862, 0xe7ccf5f0, 0x5449a36f, 0x877d48fa, - 0xc39dfd27, 0xf33e8d1e, 0x0a476341, 0x992eff74, 0x3a6f6eab, 0xf4f8fd37, 0xa812dc60, 0xa1ebddf8, 0x991be14c, 0xdb6e6b0d, 0xc67b5510, - 0x6d672c37, 0x2765d43b, 0xdcd0e804, 0xf1290dc7, 0xcc00ffa3, 0xb5390f92, 0x690fed0b, 0x667b9ffb, 0xcedb7d9c, 0xa091cf0b, 0xd9155ea3, - 0xbb132f88, 0x515bad24, 0x7b9479bf, 0x763bd6eb, 0x37392eb3, 0xcc115979, 0x8026e297, 0xf42e312d, 0x6842ada7, 0xc66a2b3b, 0x12754ccc, - 0x782ef11c, 0x6a124237, 0xb79251e7, 0x06a1bbe6, 0x4bfb6350, 0x1a6b1018, 0x11caedfa, 0x3d25bdd8, 0xe2e1c3c9, 0x44421659, 0x0a121386, - 0xd90cec6e, 0xd5abea2a, 0x64af674e, 0xda86a85f, 0xbebfe988, 0x64e4c3fe, 0x9dbc8057, 0xf0f7c086, 0x60787bf8, 0x6003604d, 0xd1fd8346, - 0xf6381fb0, 0x7745ae04, 0xd736fccc, 0x83426b33, 0xf01eab71, 0xb0804187, 0x3c005e5f, 0x77a057be, 0xbde8ae24, 0x55464299, 0xbf582e61, - 0x4e58f48f, 0xf2ddfda2, 0xf474ef38, 0x8789bdc2, 0x5366f9c3, 0xc8b38e74, 0xb475f255, 0x46fcd9b9, 0x7aeb2661, 0x8b1ddf84, 0x846a0e79, - 0x915f95e2, 0x466e598e, 0x20b45770, 0x8cd55591, 0xc902de4c, 0xb90bace1, 0xbb8205d0, 0x11a86248, 0x7574a99e, 0xb77f19b6, 0xe0a9dc09, - 0x662d09a1, 0xc4324633, 0xe85a1f02, 0x09f0be8c, 0x4a99a025, 0x1d6efe10, 0x1ab93d1d, 0x0ba5a4df, 0xa186f20f, 0x2868f169, 0xdcb7da83, - 0x573906fe, 0xa1e2ce9b, 0x4fcd7f52, 0x50115e01, 0xa70683fa, 0xa002b5c4, 0x0de6d027, 0x9af88c27, 0x773f8641, 0xc3604c06, 0x61a806b5, - 0xf0177a28, 0xc0f586e0, 0x006058aa, 0x30dc7d62, 0x11e69ed7, 0x2338ea63, 0x53c2dd94, 0xc2c21634, 0xbbcbee56, 0x90bcb6de, 0xebfc7da1, - 0xce591d76, 0x6f05e409, 0x4b7c0188, 0x39720a3d, 0x7c927c24, 0x86e3725f, 0x724d9db9, 0x1ac15bb4, 0xd39eb8fc, 0xed545578, 0x08fca5b5, - 0xd83d7cd3, 0x4dad0fc4, 0x1e50ef5e, 0xb161e6f8, 0xa28514d9, 0x6c51133c, 0x6fd5c7e7, 0x56e14ec4, 0x362abfce, 0xddc6c837, 0xd79a3234, - 0x92638212, 0x670efa8e, 0x406000e0 - ] - - self.ybghjtik = [ - 0x4b7a70e9, 0xb5b32944, 0xdb75092e, 0xc4192623, 0xad6ea6b0, 0x49a7df7d, 0x9cee60b8, 0x8fedb266, 0xecaa8c71, 0x699a17ff, 0x5664526c, - 0xc2b19ee1, 0x193602a5, 0x75094c29, 0xa0591340, 0xe4183a3e, 0x3f54989a, 0x5b429d65, 0x6b8fe4d6, 0x99f73fd6, 0xa1d29c07, 0xefe830f5, - 0x4d2d38e6, 0xf0255dc1, 0x4cdd2086, 0x8470eb26, 0x6382e9c6, 0x021ecc5e, 0x09686b3f, 0x3ebaefc9, 0x3c971814, 0x6b6a70a1, 0x687f3584, - 0x52a0e286, 0xb79c5305, 0xaa500737, 0x3e07841c, 0x7fdeae5c, 0x8e7d44ec, 0x5716f2b8, 0xb03ada37, 0xf0500c0d, 0xf01c1f04, 0x0200b3ff, - 0xae0cf51a, 0x3cb574b2, 0x25837a58, 0xdc0921bd, 0xd19113f9, 0x7ca92ff6, 0x94324773, 0x22f54701, 0x3ae5e581, 0x37c2dadc, 0xc8b57634, - 0x9af3dda7, 0xa9446146, 0x0fd0030e, 0xecc8c73e, 0xa4751e41, 0xe238cd99, 0x3bea0e2f, 0x3280bba1, 0x183eb331, 0x4e548b38, 0x4f6db908, - 0x6f420d03, 0xf60a04bf, 0x2cb81290, 0x24977c79, 0x5679b072, 0xbcaf89af, 0xde9a771f, 0xd9930810, 0xb38bae12, 0xdccf3f2e, 0x5512721f, - 0x2e6b7124, 0x501adde6, 0x9f84cd87, 0x7a584718, 0x7408da17, 0xbc9f9abc, 0xe94b7d8c, 0xec7aec3a, 0xdb851dfa, 0x63094366, 0xc464c3d2, - 0xef1c1847, 0x3215d908, 0xdd433b37, 0x24c2ba16, 0x12a14d43, 0x2a65c451, 0x50940002, 0x133ae4dd, 0x71dff89e, 0x10314e55, 0x81ac77d6, - 0x5f11199b, 0x043556f1, 0xd7a3c76b, 0x3c11183b, 0x5924a509, 0xf28fe6ed, 0x97f1fbfa, 0x9ebabf2c, 0x1e153c6e, 0x86e34570, 0xeae96fb1, - 0x860e5e0a, 0x5a3e2ab3, 0x771fe71c, 0x4e3d06fa, 0x2965dcb9, 0x99e71d0f, 0x803e89d6, 0x5266c825, 0x2e4cc978, 0x9c10b36a, 0xc6150eba, - 0x94e2ea78, 0xa5fc3c53, 0x1e0a2df4, 0xf2f74ea7, 0x361d2b3d, 0x1939260f, 0x19c27960, 0x5223a708, 0xf71312b6, 0xebadfe6e, 0xeac31f66, - 0xe3bc4595, 0xa67bc883, 0xb17f37d1, 0x018cff28, 0xc332ddef, 0xbe6c5aa5, 0x65582185, 0x68ab9802, 0xeecea50f, 0xdb2f953b, 0x2aef7dad, - 0x5b6e2f84, 0x1521b628, 0x29076170, 0xecdd4775, 0x619f1510, 0x13cca830, 0xeb61bd96, 0x0334fe1e, 0xaa0363cf, 0xb5735c90, 0x4c70a239, - 0xd59e9e0b, 0xcbaade14, 0xeecc86bc, 0x60622ca7, 0x9cab5cab, 0xb2f3846e, 0x648b1eaf, 0x19bdf0ca, 0xa02369b9, 0x655abb50, 0x40685a32, - 0x3c2ab4b3, 0x319ee9d5, 0xc021b8f7, 0x9b540b19, 0x875fa099, 0x95f7997e, 0x623d7da8, 0xf837889a, 0x97e32d77, 0x11ed935f, 0x16681281, - 0x0e358829, 0xc7e61fd6, 0x96dedfa1, 0x7858ba99, 0x57f584a5, 0x1b227263, 0x9b83c3ff, 0x1ac24696, 0xcdb30aeb, 0x532e3054, 0x8fd948e4, - 0x6dbc3128, 0x58ebf2ef, 0x34c6ffea, 0xfe28ed61, 0xee7c3c73, 0x5d4a14d9, 0xe864b7e3, 0x42105d14, 0x203e13e0, 0x45eee2b6, 0xa3aaabea, - 0xdb6c4f15, 0xfacb4fd0, 0xc742f442, 0xef6abbb5, 0x654f3b1d, 0x41cd2105, 0xd81e799e, 0x86854dc7, 0xe44b476a, 0x3d816250, 0xcf62a1f2, - 0x5b8d2646, 0xfc8883a0, 0xc1c7b6a3, 0x7f1524c3, 0x69cb7492, 0x47848a0b, 0x5692b285, 0x095bbf00, 0xad19489d, 0x1462b174, 0x23820e00, - 0x58428d2a, 0x0c55f5ea, 0x1dadf43e, 0x233f7061, 0x3372f092, 0x8d937e41, 0xd65fecf1, 0x6c223bdb, 0x7cde3759, 0xcbee7460, 0x4085f2a7, - 0xce77326e, 0xa6078084, 0x19f8509e, 0xe8efd855, 0x61d99735, 0xa969a7aa, 0xc50c06c2, 0x5a04abfc, 0x800bcadc, 0x9e447a2e, 0xc3453484, - 0xfdd56705, 0x0e1e9ec9, 0xdb73dbd3, 0x105588cd, 0x675fda79, 0xe3674340, 0xc5c43465, 0x713e38d8, 0x3d28f89e, 0xf16dff20, 0x153e21e7, - 0x8fb03d4a, 0xe6e39f2b, 0xdb83adf7 - ] - - def cypher(self, code): - return self.lskdqpyr(code, "") - - def lskdqpyr(self, alokfmth, yoaksjdh): - if self.pnjzokye is None or self.pnjzokye.lower() == yoaksjdh: - self.yoliukev(yoaksjdh) - self.pnjzokye = yoaksjdh - alokfmth = self.plaomtje(alokfmth) - ykaiumgp = "" - alokijuh = len(alokfmth) - lokimyas = self.ylomiktb(alokfmth[0:8]) - palsiuzt = lokimyas[0] - tzghbndf = lokimyas[1] - awsedrft = [None, None] - for kiujzhqa in range(8, alokijuh, 8): - lokimyas = self.ylomiktb(alokfmth[kiujzhqa:kiujzhqa+8]) - awsedrft[0] = lokimyas[0] - awsedrft[1] = lokimyas[1] - lokimyas = self.okaqnhlp(lokimyas[0], lokimyas[1]) - lokimyas[0] ^= palsiuzt - lokimyas[1] ^= tzghbndf - palsiuzt = awsedrft[0] - tzghbndf = awsedrft[1] - ykaiumgp += self.ykijmtkd(lokimyas) - return ykaiumgp - - def okaqnhlp(self, lahgrnvp, trenlpys): - ujhaqylw = 0 - for yalmhopr in range(17, 1, -1): - lahgrnvp ^= self.ldiwkqly[yalmhopr] - trenlpys ^= (self.oqlaoymh[lahgrnvp >> 24 & 0xff] + self.oqmykrna[lahgrnvp >> 16 & 0xff] ^ self.pqmyzkid[lahgrnvp >> 8 & 0xff]) + self.pldmjnde[lahgrnvp & 0xff] - ujhaqylw = lahgrnvp - lahgrnvp = trenlpys - trenlpys = ujhaqylw - ujhaqylw = lahgrnvp - lahgrnvp = trenlpys - trenlpys = ujhaqylw - trenlpys ^= self.ldiwkqly[1] - lahgrnvp ^= self.ldiwkqly[0] - return [lahgrnvp, trenlpys] - - def plaomtje(self, yoiumqpy): - qkailkzt = "" - xoliuzem = 0 - lyomiujt = 0 - yploemju = -1 - for i in range(0, len(yoiumqpy)): - yploamzu = ord(yoiumqpy[i]) - if ord('A') <= yploamzu <= ord('Z'): - xoliuzem = ord(yoiumqpy[i]) - 65 - elif ord('a') <= yploamzu <= ord('z'): - xoliuzem = ord(yoiumqpy[i]) - 97 + 26 - elif ord('0') <= yploamzu <= ord('9'): - xoliuzem = ord(yoiumqpy[i]) - 48 + 52 - elif yploamzu == ord('+'): - xoliuzem = 62 - elif yploamzu == ord('/'): - xoliuzem = 63 - else: - continue - yploemju += 1 - - lxkdmizj = 0 - switch = yploemju % 4 - if switch == 0: - lyomiujt = xoliuzem - continue - elif switch == 1: - lxkdmizj = lyomiujt << 2 | xoliuzem >> 4 - lyomiujt = xoliuzem & 0x0F - elif switch == 2: - lxkdmizj = lyomiujt << 4 | xoliuzem >> 2 - lyomiujt = xoliuzem & 0x03 - elif switch == 3: - lxkdmizj = lyomiujt << 6 | xoliuzem >> 0 - lyomiujt = xoliuzem & 0x00 - qkailkzt += unichr(lxkdmizj) - return qkailkzt - - def qmyjuila(self, oqlamykt, yalkionj): - dolizmvw = 0 - for iumswkya in range(0, 16): - oqlamykt ^= self.ldiwkqly[iumswkya] - yalkionj ^= (self.oqlaoymh[oqlamykt >> 24 & 0xff] + self.oqmykrna[oqlamykt >> 16 & 0xff] ^ self.pqmyzkid[oqlamykt >> 8 & 0xff]) + self.pldmjnde[oqlamykt & 0xff] - dolizmvw = oqlamykt - oqlamykt = yalkionj - yalkionj = dolizmvw - dolizmvw = oqlamykt - oqlamykt = yalkionj - yalkionj = dolizmvw - yalkionj ^= self.ldiwkqly[16] - oqlamykt ^= self.ldiwkqly[17] - return [oqlamykt, yalkionj] - - def ykijmtkd(self, yoirlkqw): - loipamyu = len(yoirlkqw) - yoirlkqwchar = [] - for ymujtnbq in range(0, loipamyu): - yoir = [yoirlkqw[ymujtnbq] >> 24 & 0xff, yoirlkqw[ymujtnbq] >> 16 & 0xff, yoirlkqw[ymujtnbq] >> 8 & 0xff, yoirlkqw[ymujtnbq] & 0xff] - for c in yoir: - yoirlkqwchar.append(chr(c)) - return "".join(yoirlkqwchar) - - def ylomiktb(self, lofiuzmq): - plokimqw = int(ceil(len(lofiuzmq) / 4.0)) - lopkisdq = [] - for ypoqlktz in range(0, plokimqw): - lopkisdq.append(ord(lofiuzmq[(ypoqlktz << 2) + 3]) + (ord(lofiuzmq[(ypoqlktz << 2) + 2]) << 8) + (ord(lofiuzmq[(ypoqlktz << 2) + 1]) << 16) + (ord(lofiuzmq[(ypoqlktz << 2)]) << 24)) - return lopkisdq - - def yoliukev(self, kaiumylq): - self.oqlaoymh = self.iatwbfrd - self.oqmykrna = self.ybghjtik - self.pqmyzkid = self.thdlpsmy - self.pldmjnde = self.plkodnyq - - yaqpolft = [0 for i in range(len(kaiumylq))] - - yaqwsedr = 0 - btzqwsay = 0 - while yaqwsedr < len(kaiumylq): - wlqoakmy = 0 - for lopiuztr in range(0, 4): - wlqoakmy = wlqoakmy << 8 | ord(kaiumylq[yaqwsedr % len(kaiumylq)]) - yaqwsedr += 1 - yaqpolft[btzqwsay] = wlqoakmy - btzqwsay += 1 - self.ldiwkqly = [] - for btzqwsay in range(0, 18): - self.ldiwkqly.append(self.olkemfjq[btzqwsay]) - yalopiuq = [0, 0] - for btzqwsay in range(0, 18, 2): - yalopiuq = self.qmyjuila(yalopiuq[0], yalopiuq[1]) - self.ldiwkqly[btzqwsay] = yalopiuq[0] - self.ldiwkqly[btzqwsay + 1] = yalopiuq[1] - for btzqwsay in range(0, 256, 2): - yalopiuq = self.qmyjuila(yalopiuq[0], yalopiuq[1]) - self.oqlaoymh[btzqwsay] = yalopiuq[0] - self.oqlaoymh[btzqwsay + 1] = yalopiuq[1] - for btzqwsay in range(0, 256, 2): - yalopiuq = self.qmyjuila(yalopiuq[0], yalopiuq[1]) - self.oqmykrna[btzqwsay] = yalopiuq[0] - self.oqmykrna[btzqwsay + 1] = yalopiuq[1] - for btzqwsay in range(0, 256, 2): - yalopiuq = self.qmyjuila(yalopiuq[0], yalopiuq[1]) - self.pqmyzkid[btzqwsay] = yalopiuq[0] - self.pqmyzkid[btzqwsay + 1] = yalopiuq[1] - for btzqwsay in range(0, 256, 2): - yalopiuq = self.qmyjuila(yalopiuq[0], yalopiuq[1]) - self.pldmjnde[btzqwsay] = yalopiuq[0] - self.pldmjnde[btzqwsay + 1] = yalopiuq[1] - diff --git a/module/plugins/crypter/SerienjunkiesOrg.py b/module/plugins/crypter/SerienjunkiesOrg.py deleted file mode 100644 index 3fcc12e36..000000000 --- a/module/plugins/crypter/SerienjunkiesOrg.py +++ /dev/null @@ -1,321 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from time import sleep -import random -from module.plugins.Crypter import Crypter -from module.lib.BeautifulSoup import BeautifulSoup -from module.unescape import unescape - -class SerienjunkiesOrg(Crypter): - __name__ = "SerienjunkiesOrg" - __type__ = "container" - __pattern__ = r"http://.*?(serienjunkies.org|dokujunkies.org)/.*?" - __version__ = "0.38" - __config__ = [ - ("changeNameSJ", "Packagename;Show;Season;Format;Episode", "Take SJ.org name", "Show"), - ("changeNameDJ", "Packagename;Show;Format;Episode", "Take DJ.org name", "Show"), - ("randomPreferred", "bool", "Randomize Preferred-List", False), - ("hosterListMode", "OnlyOne;OnlyPreferred(One);OnlyPreferred(All);All", "Use for hosters (if supported)", "All"), - ("hosterList", "str", "Preferred Hoster list (comma separated)", "RapidshareCom,UploadedTo,NetloadIn,FilefactoryCom,FreakshareNet,FilebaseTo,HotfileCom,DepositfilesCom,EasyshareCom,KickloadCom"), - ("ignoreList", "str", "Ignored Hoster list (comma separated)", "MegauploadCom") - ] - __description__ = """serienjunkies.org Container Plugin""" - __author_name__ = ("mkaay", "godofdream") - __author_mail__ = ("mkaay@mkaay.de", "soilfiction@gmail.com") - - - def setup(self): - self.multiDL = False - - def getSJSrc(self, url): - src = self.req.load(str(url)) - if "This website is not available in your country" in src: - self.fail("Not available in your country") - if not src.find("Enter Serienjunkies") == -1: - sleep(1) - src = self.req.load(str(url)) - return src - - def handleShow(self, url): - src = self.getSJSrc(url) - soup = BeautifulSoup(src) - packageName = self.pyfile.package().name - if self.getConfig("changeNameSJ") == "Show": - found = unescape(soup.find("h2").find("a").string.split(' –')[0]) - if found: - packageName = found - - nav = soup.find("div", attrs={"id": "scb"}) - - package_links = [] - for a in nav.findAll("a"): - if self.getConfig("changeNameSJ") == "Show": - package_links.append(a["href"]) - else: - package_links.append(a["href"] + "#hasName") - if self.getConfig("changeNameSJ") == "Show": - self.packages.append((packageName, package_links, packageName)) - else: - self.core.files.addLinks(package_links, self.pyfile.package().id) - - - def handleSeason(self, url): - src = self.getSJSrc(url) - soup = BeautifulSoup(src) - post = soup.find("div", attrs={"class": "post-content"}) - ps = post.findAll("p") - - seasonName = unescape(soup.find("a", attrs={"rel": "bookmark"}).string).replace("–", "-") - groups = {} - gid = -1 - for p in ps: - if re.search("Sprache|Format", str(p)): - var = p.findAll("strong") - opts = {"Sprache": "", "Format": ""} - for v in var: - n = unescape(v.string).strip() - n = re.sub(r"^([:]?)(.*?)([:]?)$", r'\2', n) - if n.strip() not in opts: - continue - val = v.nextSibling - if not val: - continue - val = val.replace("|", "").strip() - val = re.sub(r"^([:]?)(.*?)([:]?)$", r'\2', val) - opts[n.strip()] = val.strip() - gid += 1 - groups[gid] = {} - groups[gid]["ep"] = {} - groups[gid]["opts"] = opts - elif re.search("Download:", str(p)): - parts = str(p).split("
    ") - if re.search("", parts[0]): - ename = re.search('(.*?)',parts[0]).group(1).strip().decode("utf-8").replace("–", "-") - groups[gid]["ep"][ename] = {} - parts.remove(parts[0]) - for part in parts: - hostername = re.search(" \| ([-a-zA-Z0-9]+\.\w+)",part) - if hostername: - hostername = hostername.group(1) - groups[gid]["ep"][ename][hostername] = [] - links = re.findall('href="(.*?)"',part) - for link in links: - groups[gid]["ep"][ename][hostername].append(link + "#hasName") - - links = [] - for g in groups.values(): - for ename in g["ep"]: - links.extend(self.getpreferred(g["ep"][ename])) - if self.getConfig("changeNameSJ") == "Episode": - self.packages.append((ename, links, ename)) - links = [] - package = "%s (%s, %s)" % (seasonName, g["opts"]["Format"], g["opts"]["Sprache"]) - if self.getConfig("changeNameSJ") == "Format": - self.packages.append((package, links, package)) - links = [] - if (self.getConfig("changeNameSJ") == "Packagename") or re.search("#hasName", url): - self.core.files.addLinks(links, self.pyfile.package().id) - elif (self.getConfig("changeNameSJ") == "Season") or not re.search("#hasName", url): - self.packages.append((seasonName, links, seasonName)) - - def handleEpisode(self, url): - src = self.getSJSrc(url) - if not src.find( - "Du hast das Download-Limit überschritten! Bitte versuche es später nocheinmal.") == -1: - self.fail(_("Downloadlimit reached")) - else: - soup = BeautifulSoup(src) - form = soup.find("form") - h1 = soup.find("h1") - - if h1.get("class") == "wrap": - captchaTag = soup.find(attrs={"src": re.compile("^/secure/")}) - if not captchaTag: - sleep(5) - self.retry() - - captchaUrl = "http://download.serienjunkies.org" + captchaTag["src"] - result = self.decryptCaptcha(str(captchaUrl), imgtype="png") - sinp = form.find(attrs={"name": "s"}) - - self.req.lastURL = str(url) - sj = self.load(str(url), post={'s': sinp["value"], 'c': result, 'action': "Download"}) - - soup = BeautifulSoup(sj) - rawLinks = soup.findAll(attrs={"action": re.compile("^http://download.serienjunkies.org/")}) - - if not len(rawLinks) > 0: - sleep(1) - self.retry() - return - - self.correctCaptcha() - - links = [] - for link in rawLinks: - frameUrl = link["action"].replace("/go-", "/frame/go-") - links.append(self.handleFrame(frameUrl)) - if re.search("#hasName", url) or ((self.getConfig("changeNameSJ") == "Packagename") and (self.getConfig("changeNameDJ") == "Packagename")): - self.core.files.addLinks(links, self.pyfile.package().id) - else: - if h1.text[2] == "_": - eName = h1.text[3:] - else: - eName = h1.text - self.packages.append((eName, links, eName)) - - - def handleOldStyleLink(self, url): - sj = self.req.load(str(url)) - soup = BeautifulSoup(sj) - form = soup.find("form", attrs={"action": re.compile("^http://serienjunkies.org")}) - captchaTag = form.find(attrs={"src": re.compile("^/safe/secure/")}) - captchaUrl = "http://serienjunkies.org" + captchaTag["src"] - result = self.decryptCaptcha(str(captchaUrl)) - url = form["action"] - sinp = form.find(attrs={"name": "s"}) - - self.req.load(str(url), post={'s': sinp["value"], 'c': result, 'dl.start': "Download"}, cookies=False, - just_header=True) - decrypted = self.req.lastEffectiveURL - if decrypted == str(url): - self.retry() - self.core.files.addLinks([decrypted], self.pyfile.package().id) - - def handleFrame(self, url): - self.req.load(str(url)) - return self.req.lastEffectiveURL - - def handleShowDJ(self, url): - src = self.getSJSrc(url) - soup = BeautifulSoup(src) - post = soup.find("div", attrs={"id": "page_post"}) - ps = post.findAll("p") - found = unescape(soup.find("h2").find("a").string.split(' –')[0]) - if found: - seasonName = found - - groups = {} - gid = -1 - for p in ps: - if re.search("Sprache|Format", str(p)): - var = p.findAll("strong") - opts = {"Sprache": "", "Format": ""} - for v in var: - n = unescape(v.string).strip() - n = re.sub(r"^([:]?)(.*?)([:]?)$", r'\2', n) - if n.strip() not in opts: - continue - val = v.nextSibling - if not val: - continue - val = val.replace("|", "").strip() - val = re.sub(r"^([:]?)(.*?)([:]?)$", r'\2', val) - opts[n.strip()] = val.strip() - gid += 1 - groups[gid] = {} - groups[gid]["ep"] = {} - groups[gid]["opts"] = opts - elif re.search("Download:", str(p)): - parts = str(p).split("
    ") - if re.search("", parts[0]): - ename = re.search('(.*?)',parts[0]).group(1).strip().decode("utf-8").replace("–", "-") - groups[gid]["ep"][ename] = {} - parts.remove(parts[0]) - for part in parts: - hostername = re.search(" \| ([-a-zA-Z0-9]+\.\w+)",part) - if hostername: - hostername = hostername.group(1) - groups[gid]["ep"][ename][hostername] = [] - links = re.findall('href="(.*?)"',part) - for link in links: - groups[gid]["ep"][ename][hostername].append(link + "#hasName") - - links = [] - for g in groups.values(): - for ename in g["ep"]: - links.extend(self.getpreferred(g["ep"][ename])) - if self.getConfig("changeNameDJ") == "Episode": - self.packages.append((ename, links, ename)) - links = [] - package = "%s (%s, %s)" % (seasonName, g["opts"]["Format"], g["opts"]["Sprache"]) - if self.getConfig("changeNameDJ") == "Format": - self.packages.append((package, links, package)) - links = [] - if (self.getConfig("changeNameDJ") == "Packagename") or re.search("#hasName", url): - self.core.files.addLinks(links, self.pyfile.package().id) - elif (self.getConfig("changeNameDJ") == "Show") or not re.search("#hasName", url): - self.packages.append((seasonName, links, seasonName)) - - - - - - - - def handleCategoryDJ(self, url): - package_links = [] - src = self.getSJSrc(url) - soup = BeautifulSoup(src) - content = soup.find("div", attrs={"id": "content"}) - for a in content.findAll("a", attrs={"rel": "bookmark"}): - package_links.append(a["href"]) - self.core.files.addLinks(package_links, self.pyfile.package().id) - - def decrypt(self, pyfile): - showPattern = re.compile("^http://serienjunkies.org/serie/(.*)/$") - seasonPattern = re.compile("^http://serienjunkies.org/.*?/(.*)/$") - episodePattern = re.compile("^http://download.serienjunkies.org/f-.*?.html(#hasName)?$") - oldStyleLink = re.compile("^http://serienjunkies.org/safe/(.*)$") - categoryPatternDJ = re.compile("^http://dokujunkies.org/.*?(.*)$") - showPatternDJ = re.compile("^http://dokujunkies.org/.*?/(.*)\.html(#hasName)?$") - framePattern = re.compile("^http://download.(serienjunkies.org|dokujunkies.org)/frame/go-.*?/$") - url = pyfile.url - if framePattern.match(url): - self.packages.append((self.pyfile.package().name, [self.handleFrame(url)], self.pyfile.package().name)) - elif episodePattern.match(url): - self.handleEpisode(url) - elif oldStyleLink.match(url): - self.handleOldStyleLink(url) - elif showPattern.match(url): - self.handleShow(url) - elif showPatternDJ.match(url): - self.handleShowDJ(url) - elif seasonPattern.match(url): - self.handleSeason(url) - elif categoryPatternDJ.match(url): - self.handleCategoryDJ(url) - - #selects the preferred hoster, after that selects any hoster (ignoring the one to ignore) - def getpreferred(self, hosterlist): - - result = [] - preferredList = self.getConfig("hosterList").strip().lower().replace('|',',').replace('.','').replace(';',',').split(',') - if (self.getConfig("randomPreferred") == True) and (self.getConfig("hosterListMode") in ["OnlyOne","OnlyPreferred(One)"]) : - random.shuffle(preferredList) - # we don't want hosters be read two times - hosterlist2 = hosterlist.copy() - - for preferred in preferredList: - for Hoster in hosterlist: - if preferred == Hoster.lower().replace('.',''): - for Part in hosterlist[Hoster]: - self.logDebug("selected " + Part) - result.append(str(Part)) - del(hosterlist2[Hoster]) - if (self.getConfig("hosterListMode") in ["OnlyOne","OnlyPreferred(One)"]): - return result - - - ignorelist = self.getConfig("ignoreList").strip().lower().replace('|',',').replace('.','').replace(';',',').split(',') - if self.getConfig('hosterListMode') in ["OnlyOne","All"]: - for Hoster in hosterlist2: - if Hoster.strip().lower().replace('.','') not in ignorelist: - for Part in hosterlist2[Hoster]: - self.logDebug("selected2 " + Part) - result.append(str(Part)) - - if self.getConfig('hosterListMode') == "OnlyOne": - return result - return result diff --git a/module/plugins/crypter/ShareLinksBiz.py b/module/plugins/crypter/ShareLinksBiz.py deleted file mode 100644 index 1ffa5d41a..000000000 --- a/module/plugins/crypter/ShareLinksBiz.py +++ /dev/null @@ -1,269 +0,0 @@ -# -*- coding: utf-8 -*- - -from Crypto.Cipher import AES -from module.plugins.Crypter import Crypter -from module.plugins.ReCaptcha import ReCaptcha -import base64 -import binascii -import re - - -class ShareLinksBiz(Crypter): - __name__ = "ShareLinksBiz" - __type__ = "crypter" - __pattern__ = r"(?Phttp://[\w\.]*?(share-links|s2l)\.biz)/(?P_?[0-9a-z]+)(/.*)?" - __version__ = "1.12" - __description__ = """Share-Links.biz Crypter""" - __author_name__ = ("fragonib") - __author_mail__ = ("fragonib[AT]yahoo[DOT]es") - - - def setup(self): - self.baseUrl = None - self.fileId = None - self.package = None - self.html = None - self.captcha = False - - def decrypt(self, pyfile): - - # Init - self.initFile(pyfile) - - # Request package - url = self.baseUrl + '/' + self.fileId - self.html = self.load(url, decode=True) - - # Unblock server (load all images) - self.unblockServer() - - # Check for protection - if self.isPasswordProtected(): - self.unlockPasswordProtection() - self.handleErrors() - - if self.isCaptchaProtected(): - self.captcha = True - self.unlockCaptchaProtection() - self.handleErrors() - - # Extract package links - package_links = [] - package_links.extend(self.handleWebLinks()) - package_links.extend(self.handleContainers()) - package_links.extend(self.handleCNL2()) - package_links = set(package_links) - - # Get package info - package_name, package_folder = self.getPackageInfo() - - # Pack - self.packages = [(package_name, package_links, package_folder)] - - def initFile(self, pyfile): - url = pyfile.url - if 's2l.biz' in url: - url = self.load(url, just_header=True)['location'] - self.baseUrl = re.search(self.__pattern__, url).group(1) - self.fileId = re.match(self.__pattern__, url).group('id') - self.package = pyfile.package() - - def isOnline(self): - if "No usable content was found" in self.html: - self.logDebug("File not found") - return False - return True - - def isPasswordProtected(self): - if re.search(r'''''', self.html): - self.logDebug("Links are protected") - return True - return False - - def isCaptchaProtected(self): - if '=x1 and x<=x2) and (y>=y1 and y<=y2): - return href - - def handleErrors(self): - if "The inserted password was wrong" in self.html: - self.logDebug("Incorrect password, please set right password on 'Edit package' form and retry") - self.fail("Incorrect password, please set right password on 'Edit package' form and retry") - - if self.captcha: - if "Your choice was wrong" in self.html: - self.logDebug("Invalid captcha, retrying") - self.invalidCaptcha() - self.setWait(5) - self.wait() - self.retry() - else: - self.correctCaptcha() - - def getPackageInfo(self): - name = folder = None - - # Extract from web package header - title_re = r'

    (.*)

    ' - m = re.search(title_re, self.html, re.DOTALL) - if m is not None: - title = m.group(1).strip() - if 'unnamed' not in title: - name = folder = title - self.logDebug("Found name [%s] and folder [%s] in package info" % (name, folder)) - - # Fallback to defaults - if not name or not folder: - name = self.package.name - folder = self.package.folder - self.logDebug("Package info not found, defaulting to pyfile name [%s] and folder [%s]" % (name, folder)) - - # Return package info - return name, folder - - def handleWebLinks(self): - package_links = [] - self.logDebug("Handling Web links") - - #@TODO: Gather paginated web links - pattern = r"javascript:_get\('(.*?)', \d+, ''\)" - ids = re.findall(pattern, self.html) - self.logDebug("Decrypting %d Web links" % len(ids)) - for i, id in enumerate(ids): - try: - self.logDebug("Decrypting Web link %d, [%s]" % (i+1, id)) - dwLink = self.baseUrl + "/get/lnk/" + id - response = self.load(dwLink) - code = re.search(r'frm/(\d+)', response).group(1) - fwLink = self.baseUrl + "/get/frm/" + code - response = self.load(fwLink) - jscode = re.search(r'', response, re.DOTALL).group(1) - jscode = self.js.eval("f = %s" % jscode) - jslauncher = "window=''; parent={frames:{Main:{location:{href:''}}},location:''}; %s; parent.frames.Main.location.href" - dlLink = self.js.eval(jslauncher % jscode) - self.logDebug("JsEngine returns value [%s] for redirection link" % dlLink) - package_links.append(dlLink) - except Exception, detail: - self.logDebug("Error decrypting Web link [%s], %s" % (id, detail)) - return package_links - - def handleContainers(self): - package_links = [] - self.logDebug("Handling Container links") - - pattern = r"javascript:_get\('(.*?)', 0, '(rsdf|ccf|dlc)'\)" - containersLinks = re.findall(pattern, self.html) - self.logDebug("Decrypting %d Container links" % len(containersLinks)) - for containerLink in containersLinks: - link = "%s/get/%s/%s" % (self.baseUrl, containerLink[1], containerLink[0]) - package_links.append(link) - return package_links - - def handleCNL2(self): - package_links = [] - self.logDebug("Handling CNL2 links") - - if '/lib/cnl2/ClicknLoad.swf' in self.html: - try: - (crypted, jk) = self._getCipherParams() - package_links.extend(self._getLinks(crypted, jk)) - except: - self.fail("Unable to decrypt CNL2 links") - return package_links - - def _getCipherParams(self): - - # Request CNL2 - code = re.search(r'ClicknLoad.swf\?code=(.*?)"', self.html).group(1) - url = "%s/get/cnl2/%s" % (self.baseUrl, code) - response = self.load(url) - params = response.split(";;") - - # Get jk - strlist = list(base64.standard_b64decode(params[1])) - strlist.reverse() - jk = ''.join(strlist) - - # Get crypted - strlist = list(base64.standard_b64decode(params[2])) - strlist.reverse() - crypted = ''.join(strlist) - - # Log and return - return crypted, jk - - def _getLinks(self, crypted, jk): - - # Get key - jreturn = self.js.eval("%s f()" % jk) - self.logDebug("JsEngine returns value [%s]" % jreturn) - key = binascii.unhexlify(jreturn) - - # Decode crypted - crypted = base64.standard_b64decode(crypted) - - # Decrypt - Key = key - IV = key - obj = AES.new(Key, AES.MODE_CBC, IV) - text = obj.decrypt(crypted) - - # Extract links - text = text.replace("\x00", "").replace("\r", "") - links = text.split("\n") - links = filter(lambda x: x != "", links) - - # Log and return - self.logDebug("Block has %d links" % len(links)) - return links \ No newline at end of file diff --git a/module/plugins/crypter/ShareRapidComFolder.py b/module/plugins/crypter/ShareRapidComFolder.py deleted file mode 100644 index cb7f37525..000000000 --- a/module/plugins/crypter/ShareRapidComFolder.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.internal.SimpleCrypter import SimpleCrypter - -class ShareRapidComFolder(SimpleCrypter): - __name__ = "ShareRapidComFolder" - __type__ = "crypter" - __pattern__ = r"http://(?:www\.)?((share(-?rapid\.(biz|com|cz|info|eu|net|org|pl|sk)|-(central|credit|free|net)\.cz|-ms\.net)|(s-?rapid|rapids)\.(cz|sk))|(e-stahuj|mediatack|premium-rapidshare|rapidshare-premium|qiuck)\.cz|kadzet\.com|stahuj-zdarma\.eu|strelci\.net|universal-share\.com)/(slozka/.+)" - __version__ = "0.01" - __description__ = """Share-Rapid.com Folder Plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - LINK_PATTERN = r']*>    ' \ No newline at end of file diff --git a/module/plugins/crypter/SpeedLoadOrgFolder.py b/module/plugins/crypter/SpeedLoadOrgFolder.py deleted file mode 100644 index f85ede6f3..000000000 --- a/module/plugins/crypter/SpeedLoadOrgFolder.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################ -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU Affero General Public License as # -# published by the Free Software Foundation, either version 3 of the # -# License, or (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU Affero General Public License for more details. # -# # -# You should have received a copy of the GNU Affero General Public License # -# along with this program. If not, see . # -############################################################################ - -from module.plugins.internal.SimpleCrypter import SimpleCrypter - -class SpeedLoadOrgFolder(SimpleCrypter): - __name__ = "SpeedLoadOrgFolder" - __type__ = "crypter" - __pattern__ = r"http://(www\.)?speedload\.org/(\d+~f$|folder/\d+/)" - __version__ = "0.2" - __description__ = """Speedload Crypter Plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - LINK_PATTERN = r'
    Files of: (?P[^<]+) folder' diff --git a/module/plugins/crypter/StealthTo.py b/module/plugins/crypter/StealthTo.py deleted file mode 100644 index cf7a79e9b..000000000 --- a/module/plugins/crypter/StealthTo.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re - -from module.plugins.Crypter import Crypter - -class StealthTo(Crypter): - __name__ = "StealthTo" - __type__ = "container" - __pattern__ = r"http://(www\.)?stealth.to/folder/" - __version__ = "0.1" - __description__ = """Stealth.to Container Plugin""" - __author_name__ = ("spoob") - __author_mail__ = ("spoob@pyload.org") - - def __init__(self, parent): - Crypter.__init__(self, parent) - self.parent = parent - self.html = None - - def file_exists(self): - """ returns True or False - """ - return True - - def proceed(self, url, location): - url = self.parent.url - self.html = self.req.load(url, cookies=True) - temp_links = [] - ids = [] - ats = [] # authenticity_token - inputs = re.findall(r"(<(input|form)[^>]+)", self.html) - for input in inputs: - if re.search(r"name=\"authenticity_token\"",input[0]): - ats.append(re.search(r"value=\"([^\"]+)", input[0]).group(1)) - if re.search(r"name=\"id\"",input[0]): - ids.append(re.search(r"value=\"([^\"]+)", input[0]).group(1)) - - for i in range(0, len(ids)): - self.req.load(url + "/web", post={"authenticity_token": ats[i], "id": str(ids[i]), "link": ("download_" + str(ids[i]))}, cookies=True) - new_html = self.req.load(url + "/web", post={"authenticity_token": ats[i], "id": str(ids[i]), "link": "1"}, cookies=True) - temp_links.append(re.search(r"iframe src=\"(.*)\" frameborder", new_html).group(1)) - - self.links = temp_links diff --git a/module/plugins/crypter/TrailerzoneInfo.py b/module/plugins/crypter/TrailerzoneInfo.py deleted file mode 100644 index 43a4fcce5..000000000 --- a/module/plugins/crypter/TrailerzoneInfo.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter - -class TrailerzoneInfo(Crypter): - __name__ = "TrailerzoneInfo" - __type__ = "crypter" - __pattern__ = r"http://(www\.)?trailerzone.info/.*?" - __version__ = "0.02" - __description__ = """TrailerZone.info Crypter Plugin""" - __author_name__ = ("godofdream") - __author_mail__ = ("soilfiction@gmail.com") - - JS_KEY_PATTERN = r"' - - def process(self, pyfile): - self.html = self.load(pyfile.url, decode = True) - self.getFileInfo() - - # parse js variables - self.jsvars = dict((x, y.strip("'")) for x,y in re.findall(r"var (\w+) = ([0-9.]+|'[^']*')", self.html)) - self.logDebug(self.jsvars) - pyfile.name = self.jsvars['ID3'] - - # determine download type - free or premium - if self.premium: - if 'UU_prihlasen' in self.jsvars: - if self.jsvars['UU_prihlasen'] == '0': - self.logWarning('User not logged in') - self.relogin(self.user) - self.retry() - elif float(self.jsvars['UU_kredit']) < float(self.jsvars['kredit_odecet']): - self.logWarning('Not enough credit left') - self.premium = False - - if self.premium: - self.handlePremium() - else: - self.handleFree() - - check = self.checkDownload({"err": re.compile(r"\AChyba!")}, max_size=100) - if check == "err": - self.fail("File not found or plugin defect") - - def handleFree(self): - # get download url - download_url = '%s/download.php' % self.jsvars['server'] - data = dict((x, self.jsvars[x]) for x in self.jsvars if x in ('ID1', 'ID2', 'ID3', 'ID4')) - self.logDebug("FREE URL1:" + download_url, data) - - self.req.http.c.setopt(FOLLOWLOCATION, 0) - self.load(download_url, post=data) - self.header = self.req.http.header - self.req.http.c.setopt(FOLLOWLOCATION, 1) - - found = re.search("Location\s*:\s*(.*)", self.header, re.I) - if not found: self.fail('File not found') - download_url = found.group(1) - self.logDebug("FREE URL2:" + download_url) - - # check errors - found = re.search(r'/chyba/(\d+)', download_url) - if found: - if found.group(1) == '1': - self.retry(max_tries=60, wait_time=120, reason="This IP is already downloading") - elif found.group(1) == '2': - self.retry(max_tries=60, wait_time=60, reason="No free slots available") - else: - self.fail('Error %d' % found.group(1)) - - # download file - self.download(download_url) - - def handlePremium(self): - download_url = '%s/download_premium.php' % self.jsvars['server'] - data = dict((x, self.jsvars[x]) for x in self.jsvars if x in ('ID1', 'ID2', 'ID4', 'ID5')) - self.logDebug("PREMIUM URL:" + download_url, data) - self.download(download_url, get=data) - -getInfo = create_getInfo(QuickshareCz) diff --git a/module/plugins/hoster/RapidgatorNet.py b/module/plugins/hoster/RapidgatorNet.py deleted file mode 100644 index d2ca77e0f..000000000 --- a/module/plugins/hoster/RapidgatorNet.py +++ /dev/null @@ -1,192 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: zoidberg -""" - -import re -from pycurl import HTTPHEADER -from random import random - -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.plugins.internal.CaptchaService import ReCaptcha, SolveMedia, AdsCaptcha -from module.common.json_layer import json_loads -from module.network.HTTPRequest import BadHeader - -class RapidgatorNet(SimpleHoster): - __name__ = "RapidgatorNet" - __type__ = "hoster" - __pattern__ = r"http://(?:www\.)?(rapidgator.net)/file/(\w+)" - __version__ = "0.17" - __description__ = """rapidgator.net""" - __author_name__ = ("zoidberg", "chrox", "stickell") - - API_URL = 'http://rapidgator.net/api/file' - - FILE_INFO_PATTERN = r'Downloading:(\s*<[^>]*>)*\s*(?P.*?)(\s*<[^>]*>)*\s*File size:\s*(?P.*?)' - FILE_OFFLINE_PATTERN = r'File not found' - - JSVARS_PATTERN = r"\s+var\s*(startTimerUrl|getDownloadUrl|captchaUrl|fid|secs)\s*=\s*'?(.*?)'?;" - DOWNLOAD_LINK_PATTERN = r"return '(http[^']+)';\s*}\s*}\s*}\);" - RECAPTCHA_KEY_PATTERN = r'"http://api.recaptcha.net/challenge?k=(.*?)"' - ADSCAPTCHA_SRC_PATTERN = r'(http://api.adscaptcha.com/Get.aspx[^"\']*)' - SOLVEMEDIA_PATTERN = r'http:\/\/api\.solvemedia\.com\/papi\/challenge\.script\?k=(.*?)"' - - def setup(self): - self.resumeDownload = False - self.multiDL = False - self.sid = None - self.chunkLimit = 1 - self.req.setOption("timeout", 120) - - def process(self, pyfile): - if self.account: - self.sid = self.account.getAccountData(self.user).get('SID', None) - - if self.sid: - self.handlePremium() - else: - self.handleFree() - - def getAPIResponse(self, cmd): - try: - json = self.load('%s/%s' % (self.API_URL, cmd), - get = {'sid': self.sid, - 'url': self.pyfile.url}, decode = True) - self.logDebug('API:%s' % cmd, json, "SID: %s" % self.sid) - json = json_loads(json) - status = json['response_status'] - msg = json['response_details'] - except BadHeader, e: - self.logError('API:%s' % cmd, e, "SID: %s" % self.sid) - status = e.code - msg = e - - if status == 200: - return json['response'] - elif status == 423: - self.account.empty(self.user) - self.retry() - else: - self.account.relogin(self.user) - self.retry(wait_time=60) - - def handlePremium(self): - #self.logDebug("ACCOUNT_DATA", self.account.getAccountData(self.user)) - self.api_data = self.getAPIResponse('info') - self.api_data['md5'] = self.api_data['hash'] - self.pyfile.name = self.api_data['filename'] - self.pyfile.size = self.api_data['size'] - url = self.getAPIResponse('download')['url'] - self.multiDL = True - self.download(url) - - def handleFree(self): - self.html = self.load(self.pyfile.url, decode = True) - self.getFileInfo() - - if "You can download files up to 500 MB in free mode" in self.html \ - or "This file can be downloaded by premium only" in self.html: - self.fail("Premium account needed for download") - - self.checkWait() - - jsvars = dict(re.findall(self.JSVARS_PATTERN, self.html)) - self.logDebug(jsvars) - - self.req.http.lastURL = self.pyfile.url - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) - - url = "http://rapidgator.net%s?fid=%s" % (jsvars.get('startTimerUrl', '/download/AjaxStartTimer'), jsvars["fid"]) - jsvars.update(self.getJsonResponse(url)) - - self.setWait(int(jsvars.get('secs', 30)) + 1, False) - self.wait() - - url = "http://rapidgator.net%s?sid=%s" % (jsvars.get('getDownloadUrl', '/download/AjaxGetDownload'), jsvars["sid"]) - jsvars.update(self.getJsonResponse(url)) - - self.req.http.lastURL = self.pyfile.url - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With:"]) - - url = "http://rapidgator.net%s" % jsvars.get('captchaUrl', '/download/captcha') - self.html = self.load(url) - found = re.search(self.ADSCAPTCHA_SRC_PATTERN, self.html) - if found: - captcha_key = found.group(1) - captcha = AdsCaptcha(self) - else: - found = re.search(self.RECAPTCHA_KEY_PATTERN, self.html) - if found: - captcha_key = found.group(1) - captcha = ReCaptcha(self) - - else: - found = re.search(self.SOLVEMEDIA_PATTERN, self.html) - if found: - captcha_key = found.group(1) - captcha = SolveMedia(self) - else: - self.parseError("Captcha") - - for i in range(5): - self.checkWait() - captcha_challenge, captcha_response = captcha.challenge(captcha_key) - - self.html = self.load(url, post={ - "DownloadCaptchaForm[captcha]": "", - "adcopy_challenge": captcha_challenge, - "adcopy_response": captcha_response - }) - - if 'The verification code is incorrect' in self.html: - self.invalidCaptcha() - else: - self.correctCaptcha() - break - else: - self.fail("No valid captcha solution received") - - found = re.search(self.DOWNLOAD_LINK_PATTERN, self.html) - if not found: - self.parseError("download link") - download_url = found.group(1) - self.logDebug(download_url) - self.download(download_url) - - def checkWait(self): - found = re.search(r"(?:Delay between downloads must be not less than|Try again in)\s*(\d+)\s*(hour|min)", self.html) - if found: - wait_time = int(found.group(1)) * {"hour": 60, "min": 1}[found.group(2)] - else: - found = re.search(r"You have reached your (daily|hourly) downloads limit", self.html) - if found: - wait_time = 60 - else: - return - - self.logDebug("Waiting %d minutes" % wait_time) - self.setWait(wait_time * 60, True) - self.wait() - self.retry(max_tries = 24) - - def getJsonResponse(self, url): - response = self.load(url, decode = True) - if not response.startswith('{'): - self.retry() - self.logDebug(url, response) - return json_loads(response) - -getInfo = create_getInfo(RapidgatorNet) diff --git a/module/plugins/hoster/RapidshareCom.py b/module/plugins/hoster/RapidshareCom.py deleted file mode 100644 index 6aacd684e..000000000 --- a/module/plugins/hoster/RapidshareCom.py +++ /dev/null @@ -1,225 +0,0 @@ - -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# v1.36 -# * fixed call checkfiles subroutine -# v1.35 -# * fixed rs-urls in handleFree(..) and freeWait(..) -# * removed getInfo(..) function as it was not used anywhere (in this file) -# * removed some (old?) comment blocks - -import re - -from module.network.RequestFactory import getURL -from module.plugins.Hoster import Hoster - -def getInfo(urls): - ids = "" - names = "" - - p = re.compile(RapidshareCom.__pattern__) - - for url in urls: - r = p.search(url) - if r.group("name"): - ids+= ","+r.group("id") - names+= ","+r.group("name") - elif r.group("name_new"): - ids+= ","+r.group("id_new") - names+= ","+r.group("name_new") - - url = "http://api.rapidshare.com/cgi-bin/rsapi.cgi?sub=checkfiles&files=%s&filenames=%s" % (ids[1:], names[1:]) - - api = getURL(url) - result = [] - i = 0 - for res in api.split(): - tmp = res.split(",") - if tmp[4] in ("0", "4", "5"): status = 1 - elif tmp[4] == "1": status = 2 - else: status = 3 - - result.append( (tmp[1], tmp[2], status, urls[i]) ) - i += 1 - - yield result - - -class RapidshareCom(Hoster): - __name__ = "RapidshareCom" - __type__ = "hoster" - __pattern__ = r"https?://[\w\.]*?rapidshare.com/(?:files/(?P\d*?)/(?P[^?]+)|#!download\|(?:\w+)\|(?P\d+)\|(?P[^|]+))" - __version__ = "1.38" - __description__ = """Rapidshare.com Download Hoster""" - __config__ = [("server", "Cogent;Deutsche Telekom;Level(3);Level(3) #2;GlobalCrossing;Level(3) #3;Teleglobe;GlobalCrossing #2;TeliaSonera #2;Teleglobe #2;TeliaSonera #3;TeliaSonera", "Preferred Server", "None")] - __author_name__ = ("spoob", "RaNaN", "mkaay") - __author_mail__ = ("spoob@pyload.org", "ranan@pyload.org", "mkaay@mkaay.de") - - def setup(self): - self.html = None - self.no_download = True - self.api_data = None - self.offset = 0 - self.dl_dict = {} - - self.id = None - self.name = None - - self.chunkLimit = -1 if self.premium else 1 - self.multiDL = self.resumeDownload = self.premium - - def process(self, pyfile): - self.url = self.pyfile.url - self.prepare() - - def prepare(self): - m = re.search(self.__pattern__, self.url) - - if m.group("name"): - self.id = m.group("id") - self.name = m.group("name") - else: - self.id = m.group("id_new") - self.name = m.group("name_new") - - self.download_api_data() - if self.api_data["status"] == "1": - self.pyfile.name = self.get_file_name() - - if self.premium: - self.handlePremium() - else: - self.handleFree() - - elif self.api_data["status"] == "2": - self.log.info(_("Rapidshare: Traffic Share (direct download)")) - self.pyfile.name = self.get_file_name() - - self.download(self.pyfile.url, get={"directstart":1}) - - elif self.api_data["status"] in ("0","4","5"): - self.offline() - elif self.api_data["status"] == "3": - self.tempOffline() - else: - self.fail("Unknown response code.") - - def handleFree(self): - - while self.no_download: - self.dl_dict = self.freeWait() - - #tmp = "#!download|%(server)s|%(id)s|%(name)s|%(size)s" - download = "http://%(host)s/cgi-bin/rsapi.cgi?sub=download&editparentlocation=0&bin=1&fileid=%(id)s&filename=%(name)s&dlauth=%(auth)s" % self.dl_dict - - self.log.debug("RS API Request: %s" % download) - self.download(download, ref=False) - - check = self.checkDownload({"ip" : "You need RapidPro to download more files from your IP address", - "auth" : "Download auth invalid"}) - if check == "ip": - self.setWait(60) - self.log.info(_("Already downloading from this ip address, waiting 60 seconds")) - self.wait() - self.handleFree() - elif check == "auth": - self.log.info(_("Invalid Auth Code, download will be restarted")) - self.offset += 5 - self.handleFree() - - def handlePremium(self): - info = self.account.getAccountInfo(True) - self.log.debug("%s: Use Premium Account" % self.__name__) - url = self.api_data["mirror"] - self.download(url, get={"directstart":1}) - - - def download_api_data(self, force=False): - """ - http://images.rapidshare.com/apidoc.txt - """ - if self.api_data and not force: - return - api_url_base = "http://api.rapidshare.com/cgi-bin/rsapi.cgi" - api_param_file = {"sub": "checkfiles", "incmd5": "1", "files": self.id, "filenames": self.name} - src = self.load(api_url_base, cookies=False, get=api_param_file).strip() - self.log.debug("RS INFO API: %s" % src) - if src.startswith("ERROR"): - return - fields = src.split(",") - """ - status codes: - 0=File not found - 1=File OK (Anonymous downloading) - 3=Server down - 4=File marked as illegal - 5=Anonymous file locked, because it has more than 10 downloads already - 50+n=File OK (TrafficShare direct download type "n" without any logging.) - 100+n=File OK (TrafficShare direct download type "n" with logging. Read our privacy policy to see what is logged.) - """ - self.api_data = {"fileid": fields[0], "filename": fields[1], "size": int(fields[2]), "serverid": fields[3], - "status": fields[4], "shorthost": fields[5], "checksum": fields[6].strip().lower()} - - if int(self.api_data["status"]) > 100: - self.api_data["status"] = str(int(self.api_data["status"]) - 100) - elif int(self.api_data["status"]) > 50: - self.api_data["status"] = str(int(self.api_data["status"]) - 50) - - self.api_data["mirror"] = "http://rs%(serverid)s%(shorthost)s.rapidshare.com/files/%(fileid)s/%(filename)s" % self.api_data - - def freeWait(self): - """downloads html with the important information - """ - self.no_download = True - - id = self.id - name = self.name - - prepare = "https://api.rapidshare.com/cgi-bin/rsapi.cgi?sub=download&fileid=%(id)s&filename=%(name)s&try=1&cbf=RSAPIDispatcher&cbid=1" % {"name": name, "id" : id} - - self.log.debug("RS API Request: %s" % prepare) - result = self.load(prepare, ref=False) - self.log.debug("RS API Result: %s" % result) - - between_wait = re.search("You need to wait (\d+) seconds", result) - - if "You need RapidPro to download more files from your IP address" in result: - self.setWait(60) - self.log.info(_("Already downloading from this ip address, waiting 60 seconds")) - self.wait() - elif "Too many users downloading from this server right now" in result or "All free download slots are full" in result: - self.setWait(120) - self.log.info(_("RapidShareCom: No free slots")) - self.wait() - elif "This file is too big to download it for free" in result: - self.fail(_("You need a premium account for this file")) - elif "Filename invalid." in result: - self.fail(_("Filename reported invalid")) - elif between_wait: - self.setWait(int(between_wait.group(1))) - self.wantReconnect = True - self.wait() - else: - self.no_download = False - - tmp, info = result.split(":") - data = info.split(",") - - dl_dict = {"id": id, - "name": name, - "host": data[0], - "auth": data[1], - "server": self.api_data["serverid"], - "size": self.api_data["size"] - } - self.setWait(int(data[2])+2+self.offset) - self.wait() - - return dl_dict - - - def get_file_name(self): - if self.api_data["filename"]: - return self.api_data["filename"] - return self.url.split("/")[-1] \ No newline at end of file diff --git a/module/plugins/hoster/RarefileNet.py b/module/plugins/hoster/RarefileNet.py deleted file mode 100644 index a0f5930b5..000000000 --- a/module/plugins/hoster/RarefileNet.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo -from module.utils import html_unescape - - -class RarefileNet(XFileSharingPro): - __name__ = "RarefileNet" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*rarefile.net/\w{12}" - __version__ = "0.03" - __description__ = """Rarefile.net hoster plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r'(?P.*?)' - FILE_SIZE_PATTERN = r'Size : (?P.+?) ' - DIRECT_LINK_PATTERN = r'(?P=link)' - HOSTER_NAME = "rarefile.net" - - def setup(self): - self.resumeDownload = self.multiDL = self.premium - - def handleCaptcha(self, inputs): - captcha_div = re.search(r'Enter code.*?(.*?)
    ', self.html, re.S).group(1) - self.logDebug(captcha_div) - numerals = re.findall('(\d)', html_unescape(captcha_div)) - inputs['code'] = "".join([a[1] for a in sorted(numerals, key = lambda num: int(num[0]))]) - self.logDebug("CAPTCHA", inputs['code'], numerals) - return 3 - -getInfo = create_getInfo(RarefileNet) diff --git a/module/plugins/hoster/RealdebridCom.py b/module/plugins/hoster/RealdebridCom.py deleted file mode 100644 index 3c796232e..000000000 --- a/module/plugins/hoster/RealdebridCom.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from time import time -from urllib import quote, unquote -from random import randrange - -from module.utils import parseFileSize, remove_chars -from module.common.json_layer import json_loads -from module.plugins.Hoster import Hoster - -class RealdebridCom(Hoster): - __name__ = "RealdebridCom" - __version__ = "0.49" - __type__ = "hoster" - - __pattern__ = r"https?://.*real-debrid\..*" - __description__ = """Real-Debrid.com hoster plugin""" - __author_name__ = ("Devirex, Hazzard") - __author_mail__ = ("naibaf_11@yahoo.de") - - def getFilename(self, url): - try: - name = unquote(url.rsplit("/", 1)[1]) - except IndexError: - name = "Unknown_Filename..." - if not name or name.endswith(".."): #incomplete filename, append random stuff - name += "%s.tmp" % randrange(100,999) - return name - - def init(self): - self.tries = 0 - self.chunkLimit = 3 - self.resumeDownload = True - - - def process(self, pyfile): - if not self.account: - self.logError(_("Please enter your Real-debrid account or deactivate this plugin")) - self.fail("No Real-debrid account provided") - - self.log.debug("Real-Debrid: Old URL: %s" % pyfile.url) - if re.match(self.__pattern__, pyfile.url): - new_url = pyfile.url - else: - password = self.getPassword().splitlines() - if not password: password = "" - else: password = password[0] - - url = "http://real-debrid.com/ajax/unrestrict.php?lang=en&link=%s&password=%s&time=%s" % (quote(pyfile.url, ""), password, int(time()*1000)) - page = self.load(url) - data = json_loads(page) - - self.logDebug("Returned Data: %s" % data) - - if data["error"] != 0: - if data["message"] == "Your file is unavailable on the hoster.": - self.offline() - else: - self.logWarning(data["message"]) - self.tempOffline() - else: - if self.pyfile.name is not None and self.pyfile.name.endswith('.tmp') and data["file_name"]: - self.pyfile.name = data["file_name"] - self.pyfile.size = parseFileSize(data["file_size"]) - new_url = data['generated_links'][0][-1] - - if self.getConfig("https"): - new_url = new_url.replace("http://", "https://") - else: - new_url = new_url.replace("https://", "http://") - - self.log.debug("Real-Debrid: New URL: %s" % new_url) - - if pyfile.name.startswith("http") or pyfile.name.startswith("Unknown") or pyfile.name.endswith('..'): - #only use when name wasnt already set - pyfile.name = self.getFilename(new_url) - - self.download(new_url, disposition=True) - - check = self.checkDownload( - {"error": "An error occured while processing your request"}) - - if check == "error": - #usual this download can safely be retried - self.retry(reason="An error occured while generating link.", wait_time=60) - diff --git a/module/plugins/hoster/RedtubeCom.py b/module/plugins/hoster/RedtubeCom.py deleted file mode 100644 index 9ffafd905..000000000 --- a/module/plugins/hoster/RedtubeCom.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from module.plugins.Hoster import Hoster -from module.unescape import unescape - -class RedtubeCom(Hoster): - __name__ = "RedtubeCom" - __type__ = "hoster" - __pattern__ = r'http://[\w\.]*?redtube\.com/\d+' - __version__ = "0.2" - __description__ = """Redtube.com Download Hoster""" - __author_name__ = ("jeix") - __author_mail__ = ("jeix@hasnomail.de") - - def process(self, pyfile): - self.download_html() - if not self.file_exists(): - self.offline() - - pyfile.name = self.get_file_name() - self.download(self.get_file_url()) - - def download_html(self): - url = self.pyfile.url - self.html = self.load(url) - - def get_file_url(self): - """ returns the absolute downloadable filepath - """ - if self.html is None: - self.download_html() - - file_url = unescape(re.search(r'hashlink=(http.*?)"', self.html).group(1)) - - return file_url - - def get_file_name(self): - if self.html is None: - self.download_html() - - name = re.search('(.*?)- RedTube - Free Porn Videos', self.html).group(1).strip() + ".flv" - return name - - def file_exists(self): - """ returns True or False - """ - if self.html is None: - self.download_html() - - if re.search(r'This video has been removed.', self.html) is not None: - return False - else: - return True - diff --git a/module/plugins/hoster/RehostTo.py b/module/plugins/hoster/RehostTo.py deleted file mode 100644 index 141dcb8c8..000000000 --- a/module/plugins/hoster/RehostTo.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from urllib import quote, unquote -from module.plugins.Hoster import Hoster - -class RehostTo(Hoster): - __name__ = "RehostTo" - __version__ = "0.11" - __type__ = "hoster" - __pattern__ = r"https?://.*rehost.to\..*" - __description__ = """rehost.com hoster plugin""" - __author_name__ = ("RaNaN") - __author_mail__ = ("RaNaN@pyload.org") - - def getFilename(self, url): - return unquote(url.rsplit("/", 1)[1]) - - def setup(self): - self.chunkLimit = 1 - self.resumeDownload = True - - def process(self, pyfile): - if not self.account: - self.log.error(_("Please enter your rehost.to account or deactivate this plugin")) - self.fail("No rehost.to account provided") - - data = self.account.getAccountInfo(self.user) - long_ses = data["long_ses"] - - self.log.debug("Rehost.to: Old URL: %s" % pyfile.url) - new_url = "http://rehost.to/process_download.php?user=cookie&pass=%s&dl=%s" % (long_ses, quote(pyfile.url, "")) - - #raise timeout to 2min - self.req.setOption("timeout", 120) - - self.download(new_url, disposition=True) \ No newline at end of file diff --git a/module/plugins/hoster/ReloadCc.py b/module/plugins/hoster/ReloadCc.py deleted file mode 100644 index 7dc6d9bb6..000000000 --- a/module/plugins/hoster/ReloadCc.py +++ /dev/null @@ -1,103 +0,0 @@ -from module.plugins.Hoster import Hoster - -from module.common.json_layer import json_loads - -from module.network.HTTPRequest import BadHeader - -class ReloadCc(Hoster): - __name__ = "ReloadCc" - __version__ = "0.4" - __type__ = "hoster" - __description__ = """Reload.Cc hoster plugin""" - - # Since we want to allow the user to specify the list of hoster to use we let MultiHoster.coreReady create the regex patterns for us using getHosters in our ReloadCc hook. - __pattern__ = None - - __author_name__ = ("Reload Team") - __author_mail__ = ("hello@reload.cc") - - def process(self, pyfile): - # Check account - if not self.account or not self.account.canUse(): - self.logError("Please enter a valid reload.cc account or deactivate this plugin") - self.fail("No valid reload.cc account provided") - - # In some cases hostsers do not supply us with a filename at download, so we are going to set a fall back filename (e.g. for freakshare or xfileshare) - self.pyfile.name = self.pyfile.name.split('/').pop() # Remove everthing before last slash - - # Correction for automatic assigned filename: Removing html at end if needed - suffix_to_remove = ["html", "htm", "php", "php3", "asp", "shtm", "shtml", "cfml", "cfm"] - temp = self.pyfile.name.split('.') - if temp.pop() in suffix_to_remove: - self.pyfile.name = ".".join(temp) - - # Get account data - (user, data) = self.account.selectAccount() - - query_params = dict( - via='pyload', - v=1, - user=user, - uri=self.pyfile.url - ) - - try: - query_params.update(dict(hash=self.account.infos[user]['pwdhash'])) - except Exception: - query_params.update(dict(pwd=data['password'])) - - try: - answer = self.load("http://api.reload.cc/dl", get=query_params) - except BadHeader, e: - if e.code == 400: - self.fail("The URI is not supported by Reload.cc.") - elif e.code == 401: - self.fail("Wrong username or password") - elif e.code == 402: - self.fail("Your account is inactive. A payment is required for downloading!") - elif e.code == 403: - self.fail("Your account is disabled. Please contact the Reload.cc support!") - elif e.code == 409: - self.logWarning("The hoster seems to be a limited hoster and you've used your daily traffic for this hoster: %s" % self.pyfile.url) - # Wait for 6 hours and retry up to 4 times => one day - self.retry(max_retries=4, wait_time=(3600 * 6), reason="Limited hoster traffic limit exceeded") - elif e.code == 429: - self.retry(max_retries=5, wait_time=120, reason="Too many concurrent connections") # Too many connections, wait 2 minutes and try again - elif e.code == 503: - self.retry(wait_time=600, reason="Reload.cc is currently in maintenance mode! Please check again later.") # Retry in 10 minutes - else: - self.fail("Internal error within Reload.cc. Please contact the Reload.cc support for further information.") - return - - data = json_loads(answer) - - # Check status and decide what to do - status = data.get('status', None) - if status == "ok": - conn_limit = data.get('msg', 0) - # API says these connections are limited - # Make sure this limit is used - the download will fail if not - if conn_limit > 0: - try: - self.limitDL = int(conn_limit) - except ValueError: - self.limitDL = 1 - else: - self.limitDL = 0 - - try: - self.download(data['link'], disposition=True) - except BadHeader, e: - if e.code == 404: - self.fail("File Not Found") - elif e.code == 412: - self.fail("File access password is wrong") - elif e.code == 417: - self.fail("Password required for file access") - elif e.code == 429: - self.retry(max_retries=5, wait_time=120, reason="Too many concurrent connections") # Too many connections, wait 2 minutes and try again - else: - self.fail("Internal error within Reload.cc. Please contact the Reload.cc support for further information.") - return - else: - self.fail("Internal error within Reload.cc. Please contact the Reload.cc support for further information.") diff --git a/module/plugins/hoster/RyushareCom.py b/module/plugins/hoster/RyushareCom.py deleted file mode 100644 index 7bfe4e8fe..000000000 --- a/module/plugins/hoster/RyushareCom.py +++ /dev/null @@ -1,55 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo -import re - - -class RyushareCom(XFileSharingPro): - __name__ = "RyushareCom" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*?ryushare.com/\w{11,}" - __version__ = "0.11" - __description__ = """ryushare.com hoster plugin""" - __author_name__ = ("zoidberg", "stickell") - __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it") - - HOSTER_NAME = "ryushare.com" - - WAIT_PATTERN = r'(?:You have to|Please) wait (?:(?P\d+) minutes, )?(?:)?(?P\d+)(?:)? seconds' - DIRECT_LINK_PATTERN = r'Click here to download' - - def setup(self): - self.resumeDownload = self.multiDL = True - if not self.premium: - self.limitDL = 2 - # Up to 3 chunks allowed in free downloads. Unknown for premium - self.chunkLimit = 3 - - def getDownloadLink(self): - self.html = self.load(self.pyfile.url) - action, inputs = self.parseHtmlForm(input_names={"op": re.compile("^download")}) - if inputs.has_key('method_premium'): - del inputs['method_premium'] - - self.html = self.load(self.pyfile.url, post = inputs) - action, inputs = self.parseHtmlForm('F1') - - for i in xrange(10): - self.logInfo('Attempt to detect direct link #%d' % i) - - # Wait - if 'You have reached the download-limit!!!' in self.html: - self.setWait(3600, True) - else: - m = re.search(self.WAIT_PATTERN, self.html).groupdict('0') - waittime = int(m['min']) * 60 + int(m['sec']) - self.setWait(waittime) - self.wait() - - self.html = self.load(self.pyfile.url, post = inputs) - if 'Click here to download' in self.html: - m = re.search(self.DIRECT_LINK_PATTERN, self.html) - return m.group(1) - - self.parseError('No direct link within 10 retries') - -getInfo = create_getInfo(RyushareCom) diff --git a/module/plugins/hoster/SecureUploadEu.py b/module/plugins/hoster/SecureUploadEu.py deleted file mode 100644 index b9a900d96..000000000 --- a/module/plugins/hoster/SecureUploadEu.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - - -class SecureUploadEu(XFileSharingPro): - __name__ = "SecureUploadEu" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?secureupload\.eu/(\w){12}(/\w+)" - __version__ = "0.01" - __description__ = """SecureUpload.eu hoster plugin""" - __author_name__ = ("z00nx") - __author_mail__ = ("z00nx0@gmail.com") - - HOSTER_NAME = "secureupload.eu" - FILE_INFO_PATTERN = '

    Downloading (?P[^<]+) \((?P[^<]+)\)

    ' - FILE_OFFLINE_PATTERN = 'The file was removed|File Not Found' - -getInfo = create_getInfo(SecureUploadEu) diff --git a/module/plugins/hoster/SendmywayCom.py b/module/plugins/hoster/SendmywayCom.py deleted file mode 100644 index fcbac850a..000000000 --- a/module/plugins/hoster/SendmywayCom.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - - -class SendmywayCom(XFileSharingPro): - __name__ = "SendmywayCom" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*?sendmyway.com/\w{12}" - __version__ = "0.01" - __description__ = """SendMyWay hoster plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r'

    <.*?>\s*(?P.+)' - FILE_SIZE_PATTERN = r'\((?P\d+) bytes\)' - HOSTER_NAME = "sendmyway.com" - -getInfo = create_getInfo(SendmywayCom) diff --git a/module/plugins/hoster/SendspaceCom.py b/module/plugins/hoster/SendspaceCom.py deleted file mode 100644 index 22abaff56..000000000 --- a/module/plugins/hoster/SendspaceCom.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - -class SendspaceCom(SimpleHoster): - __name__ = "SendspaceCom" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?sendspace.com/file/.*" - __version__ = "0.13" - __description__ = """sendspace.com plugin - free only""" - __author_name__ = ("zoidberg") - - DOWNLOAD_URL_PATTERN = r'\s*<(?:b|strong)>(?P[^<]+)\s*File Size:\s*(?P[0-9.]+)(?P[kKMG])i?B\s*

    ' - FILE_OFFLINE_PATTERN = r'
    Sorry, the file you requested is not available.
    ' - CAPTCHA_PATTERN = r'' - USER_CAPTCHA_PATTERN = r'' - - def handleFree(self): - params = {} - for i in range(3): - found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) - if found: - if params.has_key('captcha_hash'): self.correctCaptcha() - download_url = found.group(1) - break - - found = re.search(self.CAPTCHA_PATTERN, self.html) - if found: - if params.has_key('captcha_hash'): self.invalidCaptcha() - captcha_url1 = "http://www.sendspace.com/" + found.group(1) - found = re.search(self.USER_CAPTCHA_PATTERN, self.html) - captcha_url2 = "http://www.sendspace.com/" + found.group(1) - params = {'captcha_hash' : found.group(2), - 'captcha_submit': 'Verify', - 'captcha_answer': self.decryptCaptcha(captcha_url1) + " " + self.decryptCaptcha(captcha_url2) - } - else: - params = {'download': "Regular Download"} - - self.logDebug(params) - self.html = self.load(self.pyfile.url, post = params) - else: - self.fail("Download link not found") - - self.logDebug("Download URL: %s" % download_url) - self.download(download_url) - -create_getInfo(SendspaceCom) \ No newline at end of file diff --git a/module/plugins/hoster/Share4webCom.py b/module/plugins/hoster/Share4webCom.py deleted file mode 100644 index ef9c2acf8..000000000 --- a/module/plugins/hoster/Share4webCom.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.hoster.UnibytesCom import UnibytesCom -from module.plugins.internal.SimpleHoster import create_getInfo - -class Share4webCom(UnibytesCom): - __name__ = "Share4webCom" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?share4web\.com/get/\w+" - __version__ = "0.1" - __description__ = """Share4web.com""" - __author_name__ = ("zoidberg") - - DOMAIN = 'http://www.share4web.com' - -getInfo = create_getInfo(UnibytesCom) \ No newline at end of file diff --git a/module/plugins/hoster/Share76Com.py b/module/plugins/hoster/Share76Com.py deleted file mode 100644 index db850cb73..000000000 --- a/module/plugins/hoster/Share76Com.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - -class Share76Com(XFileSharingPro): - __name__ = "Share76Com" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*?share76.com/\w{12}" - __version__ = "0.03" - __description__ = """share76.com hoster plugin""" - __author_name__ = ("me") - - FILE_INFO_PATTERN = r'

    \s*File:\s*]*>(?P[^>]+)\s*\[]*>(?P[0-9.]+) (?P[kKMG])i?B\]

    ' - HOSTER_NAME = "share76.com" - - def setup(self): - self.resumeDownload = self.multiDL = self.premium - self.chunkLimit = 1 - -getInfo = create_getInfo(Share76Com) diff --git a/module/plugins/hoster/ShareFilesCo.py b/module/plugins/hoster/ShareFilesCo.py deleted file mode 100644 index ee44b0a1f..000000000 --- a/module/plugins/hoster/ShareFilesCo.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo -import re - -class ShareFilesCo(XFileSharingPro): - __name__ = "ShareFilesCo" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?sharefiles\.co/\w{12}" - __version__ = "0.01" - __description__ = """Sharefiles.co hoster plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - HOSTER_NAME = "sharefiles.co" - - def startDownload(self, link): - link = link.strip() - if link.startswith('http://adf.ly'): - link = re.sub('http://adf.ly/\d+/', '', link) - if self.captcha: self.correctCaptcha() - self.logDebug('DIRECT LINK: %s' % link) - self.download(link) - -getInfo = create_getInfo(ShareFilesCo) diff --git a/module/plugins/hoster/ShareRapidCom.py b/module/plugins/hoster/ShareRapidCom.py deleted file mode 100644 index 5a08fed1f..000000000 --- a/module/plugins/hoster/ShareRapidCom.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from pycurl import HTTPHEADER -from module.network.RequestFactory import getRequest, getURL -from module.network.HTTPRequest import BadHeader -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo -from module.common.json_layer import json_loads - -def checkFile(url): - response = getURL("http://share-rapid.com/checkfiles.php", post = {"files": url}, decode = True) - info = json_loads(response) - - if "error" in info: - if info['error'] == False: - info['name'] = info['filename'] - info['status'] = 2 - elif info['msg'] == "Not found": - info['status'] = 1 #offline - elif info['msg'] == "Service Unavailable": - info['status'] = 6 #temp.offline - - return info - -def getInfo(urls): - for url in urls: - info = checkFile(url) - if "filename" in info: - yield info['name'], info['size'], info['status'], url - else: - file_info = (url, 0, 3, url) - h = getRequest() - try: - h.c.setopt(HTTPHEADER, ["Accept: text/html"]) - html = h.load(url, cookies = True, decode = True) - file_info = parseFileInfo(ShareRapidCom, url, html) - finally: - h.close() - yield file_info - -class ShareRapidCom(SimpleHoster): - __name__ = "ShareRapidCom" - __type__ = "hoster" - __pattern__ = r"http://(?:www\.)?((share(-?rapid\.(biz|com|cz|info|eu|net|org|pl|sk)|-(central|credit|free|net)\.cz|-ms\.net)|(s-?rapid|rapids)\.(cz|sk))|(e-stahuj|mediatack|premium-rapidshare|rapidshare-premium|qiuck)\.cz|kadzet\.com|stahuj-zdarma\.eu|strelci\.net|universal-share\.com)/stahuj/(\w+)" - __version__ = "0.52" - __description__ = """Share-rapid.com plugin - premium only""" - __author_name__ = ("MikyWoW", "zoidberg") - __author_mail__ = ("MikyWoW@seznam.cz", "zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r']*>]*>(?:]*>)?(?P[^<]+)' - FILE_SIZE_PATTERN = r'Velikost:\s*\s*(?P[0-9.]+) (?P[kKMG])i?B' - FILE_OFFLINE_PATTERN = ur'Nastala chyba 404|Soubor byl smazán' - - DOWNLOAD_URL_PATTERN = r'    ([^<]+)' - ERR_LOGIN_PATTERN = ur'
    Stahování je přístupné pouze přihlášeným uživatelům' - ERR_CREDIT_PATTERN = ur'
    Stahování zdarma je možné jen přes náš' - - FILE_URL_REPLACEMENTS = [(__pattern__, r'http://share-rapid.com/stahuj/\1')] - - def setup(self): - self.chunkLimit = 1 - self.resumeDownload = True - - def process(self, pyfile): - if not self.account: self.fail("User not logged in") - - self.info = checkFile(pyfile.url) - self.logDebug(self.info) - - pyfile.status = self.info['status'] - - if pyfile.status == 2: - pyfile.name = self.info['name'] - pyfile.size = self.info['size'] - elif pyfile.status == 1: - self.offline() - elif pyfile.status == 6: - self.tempOffline() - else: - self.fail("Unexpected file status") - - url = "http://share-rapid.com/stahuj/%s" % self.info['filepath'] - try: - self.html = self.load(url, decode=True) - except BadHeader, e: - self.account.relogin(self.user) - self.retry(3, 0, str(e)) - - found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) - if found is not None: - link = found.group(1) - self.logDebug("Premium link: %s" % link) - - self.check_data = {"size": pyfile.size} - self.download(link) - else: - if re.search(self.ERR_LOGIN_PATTERN, self.html): - self.relogin(self.user) - self.retry(3,0,"User login failed") - elif re.search(self.ERR_CREDIT_PATTERN, self.html): - self.fail("Not enough credit left") - else: - self.fail("Download link not found") diff --git a/module/plugins/hoster/SharebeesCom.py b/module/plugins/hoster/SharebeesCom.py deleted file mode 100644 index f5bacc5b0..000000000 --- a/module/plugins/hoster/SharebeesCom.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - - -class SharebeesCom(XFileSharingPro): - __name__ = "SharebeesCom" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*?sharebees.com/\w{12}" - __version__ = "0.01" - __description__ = """ShareBees hoster plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r'

    <.*?>\s*(?P.+)' - FILE_SIZE_PATTERN = r'\((?P\d+) bytes\)' - FORM_PATTERN = 'F1' - HOSTER_NAME = "sharebees.com" - -getInfo = create_getInfo(SharebeesCom) diff --git a/module/plugins/hoster/ShareonlineBiz.py b/module/plugins/hoster/ShareonlineBiz.py deleted file mode 100644 index e1867168b..000000000 --- a/module/plugins/hoster/ShareonlineBiz.py +++ /dev/null @@ -1,203 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from base64 import b64decode -import hashlib -import random -from time import time, sleep - -from module.plugins.Hoster import Hoster -from module.network.RequestFactory import getURL -from module.plugins.Plugin import chunks -from module.plugins.ReCaptcha import ReCaptcha as _ReCaptcha - -def getInfo(urls): - api_url_base = "http://api.share-online.biz/linkcheck.php" - - for chunk in chunks(urls, 90): - api_param_file = {"links": "\n".join(x.replace("http://www.share-online.biz/dl/","").rstrip("/") for x in chunk)} #api only supports old style links - src = getURL(api_url_base, post=api_param_file, decode=True) - result = [] - for i, res in enumerate(src.split("\n")): - if not res: - continue - fields = res.split(";") - - if fields[1] == "OK": - status = 2 - elif fields[1] in ("DELETED", "NOT FOUND"): - status = 1 - else: - status = 3 - - result.append((fields[2], int(fields[3]), status, chunk[i])) - yield result - -#suppress ocr plugin -class ReCaptcha(_ReCaptcha): - def result(self, server, challenge): - return self.plugin.decryptCaptcha("%simage"%server, get={"c":challenge}, cookies=True, forceUser=True, imgtype="jpg") - -class ShareonlineBiz(Hoster): - __name__ = "ShareonlineBiz" - __type__ = "hoster" - __pattern__ = r"http://[\w\.]*?(share\-online\.biz|egoshare\.com)/(download.php\?id\=|dl/)[\w]+" - __version__ = "0.36" - __description__ = """Shareonline.biz Download Hoster""" - __author_name__ = ("spoob", "mkaay", "zoidberg") - __author_mail__ = ("spoob@pyload.org", "mkaay@mkaay.de", "zoidberg@mujmail.cz") - - ERROR_INFO_PATTERN = r'

    Information:

    \s*
    \s*(.*?)' - - def setup(self): - # range request not working? - # api supports resume, only one chunk - # website isn't supporting resuming in first place - self.file_id = re.search(r"(id\=|/dl/)([a-zA-Z0-9]+)", self.pyfile.url).group(2) - self.pyfile.url = "http://www.share-online.biz/dl/" + self.file_id - - self.resumeDownload = self.premium - self.multiDL = False - #self.chunkLimit = 1 - - self.check_data = None - - def process(self, pyfile): - if self.premium: - self.handleAPIPremium() - #web-download fallback removed - didn't work anyway - else: - self.handleFree() - - """ - check = self.checkDownload({"failure": re.compile(self.ERROR_INFO_PATTERN)}) - if check == "failure": - try: - self.retry(reason = self.lastCheck.group(1).decode("utf8")) - except: - self.retry(reason = "Unknown error") - """ - - if self.api_data: - self.check_data = {"size": int(self.api_data['size']), "md5": self.api_data['md5']} - - def downloadAPIData(self): - api_url_base = "http://api.share-online.biz/linkcheck.php?md5=1" - api_param_file = {"links": self.pyfile.url.replace("http://www.share-online.biz/dl/","")} #api only supports old style links - src = self.load(api_url_base, cookies=False, post=api_param_file, decode=True) - - fields = src.split(";") - self.api_data = {"fileid": fields[0], - "status": fields[1]} - if not self.api_data["status"] == "OK": - self.offline() - self.api_data["filename"] = fields[2] - self.api_data["size"] = fields[3] # in bytes - self.api_data["md5"] = fields[4].strip().lower().replace("\n\n", "") # md5 - - def handleFree(self): - self.downloadAPIData() - self.pyfile.name = self.api_data["filename"] - self.pyfile.size = int(self.api_data["size"]) - - self.html = self.load(self.pyfile.url, cookies = True) #refer, stuff - self.setWait(3) - self.wait() - - self.html = self.load("%s/free/" % self.pyfile.url, post={"dl_free":"1", "choice": "free"}, decode = True) - self.checkErrors() - - found = re.search(r'var wait=(\d+);', self.html) - - recaptcha = ReCaptcha(self) - for i in range(5): - challenge, response = recaptcha.challenge("6LdatrsSAAAAAHZrB70txiV5p-8Iv8BtVxlTtjKX") - self.setWait(int(found.group(1)) if found else 30) - response = self.load("%s/free/captcha/%d" % (self.pyfile.url, int(time() * 1000)), post = { - 'dl_free': '1', - 'recaptcha_challenge_field': challenge, - 'recaptcha_response_field': response}) - - if not response == '0': - break - - else: self.fail("No valid captcha solution received") - - download_url = response.decode("base64") - self.logDebug(download_url) - if not download_url.startswith("http://"): - self.parseError("download url") - - self.wait() - self.download(download_url) - # check download - check = self.checkDownload({ - "cookie": re.compile(r'
    Share-Online") - }) - if check == "cookie": - self.retry(5, 60, "Cookie failure") - elif check == "fail": - self.retry(5, 300, "Download failed") - - def checkErrors(self): - found = re.search(r"/failure/(.*?)/1", self.req.lastEffectiveURL) - if found: - err = found.group(1) - found = re.search(self.ERROR_INFO_PATTERN, self.html) - msg = found.group(1) if found else "" - self.logError(err, msg or "Unknown error occurred") - - if err in ('freelimit', 'size', 'proxy'): - self.fail(msg or "Premium account needed") - if err in ('invalid'): - self.fail(msg or "File not available") - elif err in ('server'): - self.setWait(600, False) - elif err in ('expired'): - self.setWait(30, False) - else: - self.setWait(300, True) - - self.wait() - self.retry(max_tries=25, reason = msg) - - def handleAPIPremium(self): #should be working better - self.account.getAccountInfo(self.user, True) - src = self.load("http://api.share-online.biz/account.php", - {"username": self.user, "password": self.account.accounts[self.user]["password"], "act": "download", "lid": self.file_id}) - - self.api_data = dlinfo = {} - for line in src.splitlines(): - key, value = line.split(": ") - dlinfo[key.lower()] = value - - self.logDebug(dlinfo) - if not dlinfo["status"] == "online": - self.offline() - - self.pyfile.name = dlinfo["name"] - self.pyfile.size = int(dlinfo["size"]) - - dlLink = dlinfo["url"] - if dlLink == "server_under_maintenance": - self.tempoffline() - else: - self.multiDL = True - self.download(dlLink) - - def checksum(self, local_file): - if self.api_data and "md5" in self.api_data and self.api_data["md5"]: - h = hashlib.md5() - f = open(local_file, "rb") - h.update(f.read()) - f.close() - hexd = h.hexdigest() - if hexd == self.api_data["md5"]: - return True, 0 - else: - return False, 1 - else: - self.logWarning("MD5 checksum missing") - return True, 5 diff --git a/module/plugins/hoster/ShareplaceCom.py b/module/plugins/hoster/ShareplaceCom.py deleted file mode 100644 index 7f0dee0e5..000000000 --- a/module/plugins/hoster/ShareplaceCom.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -import urllib -from module.plugins.Hoster import Hoster - -class ShareplaceCom(Hoster): - __name__ = "ShareplaceCom" - __type__ = "hoster" - __pattern__ = r"(http://)?(www\.)?shareplace\.(com|org)/\?[a-zA-Z0-9]+" - __version__ = "0.1" - __description__ = """Shareplace.com Download Hoster""" - __author_name__ = ("ACCakut, based on YourfilesTo by jeix and skydancer") - __author_mail__ = ("none") - - def setup(self): - self.html = None - self.multiDL = True - - def process(self,pyfile): - self.pyfile = pyfile - self.prepare() - self.download(self.get_file_url()) - - def prepare(self): - if not self.file_exists(): - self.offline() - - self.pyfile.name = self.get_file_name() - - wait_time = self.get_waiting_time() - self.setWait(wait_time) - self.log.debug("%s: Waiting %d seconds." % (self.__name__,wait_time)) - self.wait() - - def get_waiting_time(self): - if self.html is None: - self.download_html() - - #var zzipitime = 15; - m = re.search(r'var zzipitime = (\d+);', self.html) - if m: - sec = int(m.group(1)) - else: - sec = 0 - - return sec - - def download_html(self): - url = re.sub("shareplace.com\/\?", "shareplace.com//index1.php/?a=", self.pyfile.url) - self.html = self.load(url, decode=True) - - def get_file_url(self): - """ returns the absolute downloadable filepath - """ - url = re.search(r"var beer = '(.*?)';", self.html) - if url: - url = url.group(1) - url = urllib.unquote(url.replace("http://http:/", "").replace("vvvvvvvvv", "").replace("lllllllll", "").replace("teletubbies", "")) - self.logDebug("URL: %s" % url) - return url - else: - self.fail("absolute filepath could not be found. offline? ") - - def get_file_name(self): - if self.html is None: - self.download_html() - - return re.search("\s*(.*?)\s*", self.html).group(1) - - def file_exists(self): - """ returns True or False - """ - if self.html is None: - self.download_html() - - if re.search(r"HTTP Status 404", self.html) is not None: - return False - else: - return True - - - diff --git a/module/plugins/hoster/ShragleCom.py b/module/plugins/hoster/ShragleCom.py deleted file mode 100644 index 99f9f2366..000000000 --- a/module/plugins/hoster/ShragleCom.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from pycurl import FOLLOWLOCATION - -from module.plugins.Hoster import Hoster -from module.plugins.internal.SimpleHoster import parseHtmlForm -from module.plugins.ReCaptcha import ReCaptcha -from module.network.RequestFactory import getURL - -API_KEY = "078e5ca290d728fd874121030efb4a0d" - -def parseFileInfo(self, url): - file_id = re.match(self.__pattern__, url).group('ID') - - data = getURL( - "http://www.cloudnator.com/api.php?key=%s&action=getStatus&fileID=%s" % (API_KEY, file_id), - decode = True - ).split() - - if len(data) == 4: - name, size, md5, status = data - size = int(size) - - if hasattr(self, "check_data"): - self.checkdata = {"size": size, "md5": md5} - - return name, size, 2 if status == "0" else 1, url - else: - return url, 0, 1, url - -def getInfo(urls): - for url in urls: - file_info = parseFileInfo(ShragleCom, url) - yield file_info - -class ShragleCom(Hoster): - __name__ = "ShragleCom" - __type__ = "hoster" - __pattern__ = r"http://(?:www.)?(cloudnator|shragle).com/files/(?P.*?)/" - __version__ = "0.21" - __description__ = """Cloudnator.com (Shragle.com) Download PLugin""" - __author_name__ = ("RaNaN", "zoidberg") - __author_mail__ = ("RaNaN@pyload.org", "zoidberg@mujmail.cz") - - def setup(self): - self.html = None - self.multiDL = False - self.check_data = None - - def process(self, pyfile): - #get file status and info - self.pyfile.name, self.pyfile.size, status = parseFileInfo(self, pyfile.url)[:3] - if status != 2: - self.offline() - - self.handleFree() - - def handleFree(self): - self.html = self.load(self.pyfile.url) - - #get wait time - found = re.search('\s*var\sdownloadWait\s=\s(\d+);', self.html) - self.setWait(int(found.group(1)) if found else 30) - - #parse download form - action, inputs = parseHtmlForm('id="download', self.html) - - #solve captcha - found = re.search('recaptcha/api/(?:challenge|noscript)?k=(.+?)', self.html) - captcha_key = found.group(1) if found else "6LdEFb0SAAAAAAwM70vnYo2AkiVkCx-xmfniatHz" - - recaptcha = ReCaptcha(self) - - inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(captcha_key) - self.wait() - - #validate - self.req.http.c.setopt(FOLLOWLOCATION, 0) - self.html = self.load(action, post = inputs) - - found = re.search(r"Location\s*:\s*(\S*)", self.req.http.header, re.I) - if found: - self.correctCaptcha() - download_url = found.group(1) - else: - if "Sicherheitscode falsch" in self.html: - self.invalidCaptcha() - self.retry(max_tries = 5, reason = "Invalid captcha") - else: - self.fail("Invalid session") - - #download - self.req.http.c.setopt(FOLLOWLOCATION, 1) - self.download(download_url) - - check = self.checkDownload({ - "ip_blocked": re.compile(r'
    \w+)" - __version__ = "1.01" - __description__ = """Speedload.org hoster plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - FILE_NAME_PATTERN = r'Filename:(?P[^<]+)' - FILE_SIZE_PATTERN = r'Size:[\w. ]+\((?P\d+) bytes\)' - - HOSTER_NAME = "speedload.org" - - def handlePremium(self): - self.download(self.pyfile.url, post = self.getPostParameters()) - -getInfo = create_getInfo(SpeedLoadOrg) diff --git a/module/plugins/hoster/SpeedfileCz.py b/module/plugins/hoster/SpeedfileCz.py deleted file mode 100644 index bfd316dfa..000000000 --- a/module/plugins/hoster/SpeedfileCz.py +++ /dev/null @@ -1,65 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - -class SpeedfileCz(SimpleHoster): - __name__ = "SpeedFileCz" - __type__ = "hoster" - __pattern__ = r"http://speedfile.cz/.*" - __version__ = "0.31" - __description__ = """speedfile.cz""" - __author_name__ = ("zoidberg") - - FILE_NAME_PATTERN = r'' - FILE_SIZE_PATTERN = r'(?P[0-9.]+) (?P[kKMG])i?B' - URL_PATTERN = r'' - FILE_OFFLINE_PATTERN = r'Speedfile \| 404' - WAIT_PATTERN = r'"requestedAt":(\d+),"allowedAt":(\d+),"adUri"' - - def setup(self): - self.multiDL = False - - def process(self, pyfile): - self.html = self.load(pyfile.url, decode=True) - - if re.search(self.FILE_OFFLINE_PATTERN, self.html): - self.offline() - - found = re.search(self.FILE_NAME_PATTERN, self.html) - if found is None: - self.fail("Parse error (NAME)") - pyfile.name = found.group(1) - - found = re.search(self.URL_PATTERN, self.html) - if found is None: - self.fail("Parse error (URL)") - download_url = "http://speedfile.cz/" + found.group(1) - - self.html = self.load(download_url) - self.logDebug(self.html) - found = re.search(self.WAIT_PATTERN, self.html) - if found is None: - self.fail("Parse error (WAIT)") - self.setWait(int(found.group(2)) - int(found.group(1))) - self.wait() - - self.download(download_url) - -create_getInfo(SpeedfileCz) \ No newline at end of file diff --git a/module/plugins/hoster/StreamCz.py b/module/plugins/hoster/StreamCz.py deleted file mode 100644 index ca1033502..000000000 --- a/module/plugins/hoster/StreamCz.py +++ /dev/null @@ -1,76 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.Hoster import Hoster -from module.network.RequestFactory import getURL - -def getInfo(urls): - result = [] - - for url in urls: - - html = getURL(url) - if re.search(StreamCz.FILE_OFFLINE_PATTERN, html): - # File offline - result.append((url, 0, 1, url)) - else: - result.append((url, 0, 2, url)) - yield result - -class StreamCz(Hoster): - __name__ = "StreamCz" - __type__ = "hoster" - __pattern__ = r"http://www.stream.cz/[^/]+/\d+.*" - __version__ = "0.1" - __description__ = """stream.cz""" - __author_name__ = ("zoidberg") - - FILE_OFFLINE_PATTERN = r'<h1 class="commonTitle">Str.nku nebylo mo.n. nal.zt \(404\)</h1>' - FILE_NAME_PATTERN = r'<link rel="video_src" href="http://www.stream.cz/\w+/(\d+)-([^"]+)" />' - CDN_PATTERN = r'<param name="flashvars" value="[^"]*&id=(?P<ID>\d+)(?:&cdnLQ=(?P<cdnLQ>\d*))?(?:&cdnHQ=(?P<cdnHQ>\d*))?(?:&cdnHD=(?P<cdnHD>\d*))?&' - - def setup(self): - self.multiDL = True - self.resumeDownload = True - - def process(self, pyfile): - - self.html = self.load(pyfile.url, decode=True) - - if re.search(self.FILE_OFFLINE_PATTERN, self.html): - self.offline() - - found = re.search(self.CDN_PATTERN, self.html) - if found is None: self.fail("Parse error (CDN)") - cdn = found.groupdict() - self.logDebug(cdn) - for cdnkey in ("cdnHD", "cdnHQ", "cdnLQ"): - if cdn.has_key(cdnkey) and cdn[cdnkey] > '': - cdnid = cdn[cdnkey] - break - else: - self.fail("Stream URL not found") - - found = re.search(self.FILE_NAME_PATTERN, self.html) - if found is None: self.fail("Parse error (NAME)") - pyfile.name = "%s-%s.%s.mp4" % (found.group(2), found.group(1), cdnkey[-2:]) - - download_url = "http://cdn-dispatcher.stream.cz/?id=" + cdnid - self.logInfo("STREAM (%s): %s" % (cdnkey[-2:], download_url)) - self.download(download_url) diff --git a/module/plugins/hoster/TurbobitNet.py b/module/plugins/hoster/TurbobitNet.py deleted file mode 100644 index b429d5510..000000000 --- a/module/plugins/hoster/TurbobitNet.py +++ /dev/null @@ -1,170 +0,0 @@ -# -*- coding: utf-8 -*- -""" - Copyright (C) 2012 pyLoad team - Copyright (C) 2012 JD-Team support@jdownloader.org - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -import random -from urllib import quote -from binascii import hexlify, unhexlify -from Crypto.Cipher import ARC4 -import time - -from module.network.RequestFactory import getURL -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, timestamp -from module.plugins.ReCaptcha import ReCaptcha - -from pycurl import HTTPHEADER - -class TurbobitNet(SimpleHoster): - __name__ = "TurbobitNet" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)?(turbobit.net|unextfiles.com)/(?:download/free/)?(?P<ID>\w+).*" - __version__ = "0.08" - __description__ = """Turbobit.net plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_INFO_PATTERN = r"<span class='file-icon1[^>]*>(?P<N>[^<]+)</span>\s*\((?P<S>[^\)]+)\)\s*</h1>" #long filenames are shortened - FILE_NAME_PATTERN = r'<meta name="keywords" content="\s+(?P<N>[^,]+)' #full name but missing on page2 - FILE_OFFLINE_PATTERN = r'<h2>File Not Found</h2>|html\(\'File was not found' - FILE_URL_REPLACEMENTS = [(r"http://(?:\w*\.)?(turbobit.net|unextfiles.com)/(?:download/free/)?(?P<ID>\w+).*", "http://turbobit.net/\g<ID>.html")] - SH_COOKIES = [("turbobit.net", "user_lang", "en")] - - CAPTCHA_KEY_PATTERN = r'src="http://api\.recaptcha\.net/challenge\?k=([^"]+)"' - DOWNLOAD_URL_PATTERN = r'(?P<url>/download/redirect/[^"\']+)' - LIMIT_WAIT_PATTERN = r'<div id="time-limit-text">\s*.*?<span id=\'timeout\'>(\d+)</span>' - CAPTCHA_SRC_PATTERN = r'<img alt="Captcha" src="(.*?)"' - - def handleFree(self): - self.url = "http://turbobit.net/download/free/%s" % self.file_info['ID'] - self.html = self.load(self.url) - - rtUpdate = self.getRtUpdate() - - self.solveCaptcha() - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) - self.url = self.getDownloadUrl(rtUpdate) - - self.wait() - self.html = self.load(self.url) - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With:"]) - self.downloadFile() - - def solveCaptcha(self): - for i in range(5): - found = re.search(self.LIMIT_WAIT_PATTERN, self.html) - if found: - wait_time = int(found.group(1)) - self.setWait(wait_time, wait_time > 60) - self.wait() - self.retry() - - action, inputs = self.parseHtmlForm("action='#'") - if not inputs: self.parseError("captcha form") - self.logDebug(inputs) - - if inputs['captcha_type'] == 'recaptcha': - recaptcha = ReCaptcha(self) - found = re.search(self.CAPTCHA_KEY_PATTERN, self.html) - captcha_key = found.group(1) if found else '6LcTGLoSAAAAAHCWY9TTIrQfjUlxu6kZlTYP50_c' - inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(captcha_key) - else: - found = re.search(self.CAPTCHA_SRC_PATTERN, self.html) - if not found: self.parseError('captcha') - captcha_url = found.group(1) - inputs['captcha_response'] = self.decryptCaptcha(captcha_url) - - self.logDebug(inputs) - self.html = self.load(self.url, post = inputs) - - if not "<div class='download-timer-header'>" in self.html: - self.invalidCaptcha() - else: - self.correctCaptcha() - break - else: self.fail("Invalid captcha") - - def getRtUpdate(self): - rtUpdate = self.getStorage("rtUpdate") - if not rtUpdate: - if self.getStorage("version") != self.__version__ or int(self.getStorage("timestamp", 0)) + 86400000 < timestamp(): - # that's right, we are even using jdownloader updates - rtUpdate = getURL("http://update0.jdownloader.org/pluginstuff/tbupdate.js") - rtUpdate = self.decrypt(rtUpdate.splitlines()[1]) - # but we still need to fix the syntax to work with other engines than rhino - rtUpdate = re.sub(r'for each\(var (\w+) in(\[[^\]]+\])\)\{',r'zza=\2;for(var zzi=0;zzi<zza.length;zzi++){\1=zza[zzi];',rtUpdate) - rtUpdate = re.sub(r"for\((\w+)=",r"for(var \1=", rtUpdate) - - self.logDebug("rtUpdate") - self.setStorage("rtUpdate", rtUpdate) - self.setStorage("timestamp", timestamp()) - self.setStorage("version", self.__version__) - else: - self.logError("Unable to download, wait for update...") - self.tempOffline() - - return rtUpdate - - def getDownloadUrl(self, rtUpdate): - self.req.http.lastURL = self.url - - found = re.search("(/\w+/timeout\.js\?\w+=)([^\"\'<>]+)", self.html) - url = "http://turbobit.net%s%s" % (found.groups() if found else ('/files/timeout.js?ver=', ''.join(random.choice('0123456789ABCDEF') for x in range(32)))) - fun = self.load(url) - - self.setWait(65, False) - - for b in [1,3]: - self.jscode = "var id = \'%s\';var b = %d;var inn = \'%s\';%sout" % (self.file_info['ID'], b, quote(fun), rtUpdate) - - try: - out = self.js.eval(self.jscode) - self.logDebug("URL", self.js.engine, out) - if out.startswith('/download/'): - return "http://turbobit.net%s" % out.strip() - except Exception, e: - self.logError(e) - else: - if self.retries >= 2: - # retry with updated js - self.delStorage("rtUpdate") - self.retry() - - def decrypt(self, data): - cipher = ARC4.new(hexlify('E\x15\xa1\x9e\xa3M\xa0\xc6\xa0\x84\xb6H\x83\xa8o\xa0')) - return unhexlify(cipher.encrypt(unhexlify(data))) - - def getLocalTimeString(self): - lt = time.localtime() - tz = time.altzone if lt.tm_isdst else time.timezone - return "%s GMT%+03d%02d" % (time.strftime("%a %b %d %Y %H:%M:%S", lt), -tz // 3600, tz % 3600) - - def handlePremium(self): - self.logDebug("Premium download as user %s" % self.user) - self.downloadFile() - - def downloadFile(self): - found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) - if not found: self.parseError("download link") - self.url = "http://turbobit.net" + found.group('url') - self.logDebug(self.url) - self.download(self.url) - -getInfo = create_getInfo(TurbobitNet) diff --git a/module/plugins/hoster/TurbouploadCom.py b/module/plugins/hoster/TurbouploadCom.py deleted file mode 100644 index 6e81c6319..000000000 --- a/module/plugins/hoster/TurbouploadCom.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.DeadHoster import DeadHoster as EasybytezCom, create_getInfo -#from module.plugins.internal.SimpleHoster import create_getInfo -#from module.plugins.hoster.EasybytezCom import EasybytezCom - -class TurbouploadCom(EasybytezCom): - __name__ = "TurbouploadCom" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)?turboupload.com/(\w+).*" - __version__ = "0.02" - __description__ = """turboupload.com""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - # shares code with EasybytezCom - - DIRECT_LINK_PATTERN = r'<a href="(http://turboupload.com/files/[^"]+)">\1</a>' - - def handleFree(self): - self.html = self.load(self.pyfile.url, post = self.getPostParameters(), ref = True, cookies = True) - found = re.search(self.DIRECT_LINK_PATTERN, self.html) - if not found: self.parseError('Download Link') - url = found.group(1) - self.logDebug('URL: ' + url) - self.download(url) - -getInfo = create_getInfo(TurbouploadCom) \ No newline at end of file diff --git a/module/plugins/hoster/TusfilesNet.py b/module/plugins/hoster/TusfilesNet.py deleted file mode 100644 index 517df8561..000000000 --- a/module/plugins/hoster/TusfilesNet.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - -class TusfilesNet(XFileSharingPro): - __name__ = "TusfilesNet" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?tusfiles\.net/\w{12}" - __version__ = "0.01" - __description__ = """Tusfiles.net hoster plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - FILE_INFO_PATTERN = r'<li>(?P<N>[^<]+)</li>\s+<li><b>Size:</b> <small>(?P<S>[\d.]+) (?P<U>\w+)</small></li>' - FILE_OFFLINE_PATTERN = r'The file you were looking for could not be found' - - HOSTER_NAME = "tusfiles.net" - -getInfo = create_getInfo(TusfilesNet) diff --git a/module/plugins/hoster/TwoSharedCom.py b/module/plugins/hoster/TwoSharedCom.py deleted file mode 100644 index 8401e0cb0..000000000 --- a/module/plugins/hoster/TwoSharedCom.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -import re - -class TwoSharedCom(SimpleHoster): - __name__ = "TwoSharedCom" - __type__ = "hoster" - __pattern__ = r"http://[\w\.]*?2shared.com/(account/)?(download|get|file|document|photo|video|audio)/.*" - __version__ = "0.11" - __description__ = """2Shared Download Hoster""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r'<h1>(?P<N>.*)</h1>' - FILE_SIZE_PATTERN = r'<span class="dtitle">File size:</span>\s*(?P<S>[0-9,.]+) (?P<U>[kKMG])i?B' - FILE_OFFLINE_PATTERN = r'The file link that you requested is not valid\.|This file was deleted\.' - DOWNLOAD_URL_PATTERN = r"window.location ='([^']+)';" - - def setup(self): - self.resumeDownload = self.multiDL = True - - def handleFree(self): - found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) - if not found: self.parseError('Download link') - link = found.group(1) - self.logDebug("Download URL %s" % link) - - self.download(link) - -getInfo = create_getInfo(TwoSharedCom) - \ No newline at end of file diff --git a/module/plugins/hoster/UlozTo.py b/module/plugins/hoster/UlozTo.py deleted file mode 100644 index 5c38fdaad..000000000 --- a/module/plugins/hoster/UlozTo.py +++ /dev/null @@ -1,156 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - -def convertDecimalPrefix(m): - # decimal prefixes used in filesize and traffic - return ("%%.%df" % {'k':3,'M':6,'G':9}[m.group(2)] % float(m.group(1))).replace('.','') - -class UlozTo(SimpleHoster): - __name__ = "UlozTo" - __type__ = "hoster" - __pattern__ = r"http://(\w*\.)?(uloz\.to|ulozto\.(cz|sk|net)|bagruj.cz|zachowajto.pl)/(?:live/)?(?P<id>\w+/[^/?]*)" - __version__ = "0.92" - __description__ = """uloz.to""" - __author_name__ = ("zoidberg") - - FILE_NAME_PATTERN = r'<a href="#download" class="jsShowDownload">(?P<N>[^<]+)</a>' - FILE_SIZE_PATTERN = r'<span id="fileSize">.*?(?P<S>[0-9.]+\s[kMG]?B)</span>' - FILE_INFO_PATTERN = r'<p>File <strong>(?P<N>[^<]+)</strong> is password protected</p>' - FILE_OFFLINE_PATTERN = r'<title>404 - Page not found|

    File (has been deleted|was banned)

    ' - FILE_SIZE_REPLACEMENTS = [('([0-9.]+)\s([kMG])B', convertDecimalPrefix)] - FILE_URL_REPLACEMENTS = [(r"(?<=http://)([^/]+)", "www.ulozto.net")] - - PASSWD_PATTERN = r'
    ' - VIPLINK_PATTERN = r'' - FREE_URL_PATTERN = r'
    \s*
  • Error rewriting the text.
    • '), - "offline": re.compile(self.FILE_OFFLINE_PATTERN), - "passwd": self.PASSWD_PATTERN, - "server_error": 'src="http://img.ulozto.cz/error403/vykricnik.jpg"', #paralell dl, server overload etc. - "not_found": "Ulož.to" - }) - - if check == "wrong_captcha": - self.delStorage("captcha_id") - self.delStorage("captcha_text") - self.invalidCaptcha() - self.retry(reason="Wrong captcha code") - elif check == "offline": - self.offline() - elif check == "passwd": - self.fail("Wrong password") - elif check == "server_error": - self.logError("Server error, try downloading later") - self.multiDL = False - self.setWait(3600, True) - self.wait() - self.retry() - elif check == "not_found": - self.fail("Server error - file not downloadable") - -getInfo = create_getInfo(UlozTo) diff --git a/module/plugins/hoster/UloziskoSk.py b/module/plugins/hoster/UloziskoSk.py deleted file mode 100644 index c607e7a5b..000000000 --- a/module/plugins/hoster/UloziskoSk.py +++ /dev/null @@ -1,75 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, PluginParseError - -class UloziskoSk(SimpleHoster): - __name__ = "UloziskoSk" - __type__ = "hoster" - __pattern__ = r"http://(\w*\.)?ulozisko.sk/.*" - __version__ = "0.23" - __description__ = """Ulozisko.sk""" - __author_name__ = ("zoidberg") - - URL_PATTERN = r'' - ID_PATTERN = r'' - FILE_NAME_PATTERN = r'
    (?P[^<]+)
    ' - FILE_SIZE_PATTERN = ur'Veľkosť súboru: (?P[0-9.]+) (?P[kKMG])i?B
    ' - CAPTCHA_PATTERN = r'' - FILE_OFFLINE_PATTERN = ur'Zadaný súbor neexistuje z jedného z nasledujúcich dôvodov:' - IMG_PATTERN = ur'PRE ZVÄČŠENIE KLIKNITE NA OBRÁZOK
    ' - - def process(self, pyfile): - self.html = self.load(pyfile.url, decode=True) - self.getFileInfo() - - found = re.search(self.IMG_PATTERN, self.html) - if found: - url = "http://ulozisko.sk" + found.group(1) - self.download(url) - else: - self.handleFree() - - def handleFree(self): - found = re.search(self.URL_PATTERN, self.html) - if found is None: raise PluginParseError('URL') - parsed_url = 'http://www.ulozisko.sk' + found.group(1) - - found = re.search(self.ID_PATTERN, self.html) - if found is None: raise PluginParseError('ID') - id = found.group(1) - - self.logDebug('URL:' + parsed_url + ' ID:' + id) - - found = re.search(self.CAPTCHA_PATTERN, self.html) - if found is None: raise PluginParseError('CAPTCHA') - captcha_url = 'http://www.ulozisko.sk' + found.group(1) - - captcha = self.decryptCaptcha(captcha_url, cookies=True) - - self.logDebug('CAPTCHA_URL:' + captcha_url + ' CAPTCHA:' + captcha) - - self.download(parsed_url, post={ - "antispam": captcha, - "id": id, - "name": self.pyfile.name, - "but": "++++STIAHNI+S%DABOR++++" - }) - -getInfo = create_getInfo(UloziskoSk) diff --git a/module/plugins/hoster/UnibytesCom.py b/module/plugins/hoster/UnibytesCom.py deleted file mode 100644 index 3c8552271..000000000 --- a/module/plugins/hoster/UnibytesCom.py +++ /dev/null @@ -1,80 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: zoidberg -""" - -import re -from pycurl import FOLLOWLOCATION -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - -class UnibytesCom(SimpleHoster): - __name__ = "UnibytesCom" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?unibytes\.com/[a-zA-Z0-9-._ ]{11}B" - __version__ = "0.1" - __description__ = """UniBytes.com""" - __author_name__ = ("zoidberg") - - FILE_INFO_PATTERN = r']*?id="fileName"[^>]*>(?P[^>]+)\s*\((?P\d.*?)\)' - DOMAIN = 'http://www.unibytes.com' - - WAIT_PATTERN = r'Wait for (\d+) sec' - DOWNLOAD_LINK_PATTERN = r'Download' - - def handleFree(self): - action, post_data = self.parseHtmlForm('id="startForm"') - self.req.http.c.setopt(FOLLOWLOCATION, 0) - - for i in range(8): - self.logDebug(action, post_data) - self.html = self.load(self.DOMAIN + action, post = post_data) - - found = re.search(r'location:\s*(\S+)', self.req.http.header, re.I) - if found: - url = found.group(1) - break - - if '>Somebody else is already downloading using your IP-address<' in self.html: - self.setWait(600, True) - self.wait() - self.retry() - - if post_data['step'] == 'last': - found = re.search(self.DOWNLOAD_LINK_PATTERN, self.html) - if found: - url = found.group(1) - self.correctCaptcha() - break - else: - self.invalidCaptcha() - - last_step = post_data['step'] - action, post_data = self.parseHtmlForm('id="stepForm"') - - if last_step == 'timer': - found = re.search(self.WAIT_PATTERN, self.html) - self.setWait(int(found.group(1)) if found else 60, False) - self.wait() - elif last_step in ('captcha', 'last'): - post_data['captcha'] = self.decryptCaptcha(self.DOMAIN + '/captcha.jpg') - else: - self.fail("No valid captcha code entered") - - self.logDebug('Download link: ' + url) - self.req.http.c.setopt(FOLLOWLOCATION, 1) - self.download(url) - -getInfo = create_getInfo(UnibytesCom) \ No newline at end of file diff --git a/module/plugins/hoster/UploadStationCom.py b/module/plugins/hoster/UploadStationCom.py deleted file mode 100644 index d24682e4d..000000000 --- a/module/plugins/hoster/UploadStationCom.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.FileserveCom import FileserveCom, checkFile -from module.plugins.Plugin import chunks - -class UploadStationCom(FileserveCom): - __name__ = "UploadStationCom" - __type__ = "hoster" - __pattern__ = r"http://(?:www\.)?uploadstation\.com/file/(?P[A-Za-z0-9]+)" - __version__ = "0.51" - __description__ = """UploadStation.Com File Download Hoster""" - __author_name__ = ("fragonib", "zoidberg") - __author_mail__ = ("fragonib[AT]yahoo[DOT]es", "zoidberg@mujmail.cz") - - URLS = ['http://www.uploadstation.com/file/', 'http://www.uploadstation.com/check-links.php', 'http://www.uploadstation.com/checkReCaptcha.php'] - LINKCHECK_TR = r'
    (.*?)\t{9}
    ' - LINKCHECK_TD = r'
    (?:<[^>]*>| )*([^<]*)' - - LONG_WAIT_PATTERN = r'

    You have to wait (\d+) (\w+) to download the next file\.

    ' - -def getInfo(urls): - for chunk in chunks(urls, 100): yield checkFile(UploadStationCom, chunk) \ No newline at end of file diff --git a/module/plugins/hoster/UploadedTo.py b/module/plugins/hoster/UploadedTo.py deleted file mode 100644 index 5aa08891e..000000000 --- a/module/plugins/hoster/UploadedTo.py +++ /dev/null @@ -1,238 +0,0 @@ -# -*- coding: utf-8 -*- - -# Test links (random.bin): -# http://ul.to/044yug9o -# http://ul.to/gzfhd0xs - -import re -from time import sleep - -from module.utils import html_unescape, parseFileSize, chunks - -from module.plugins.Hoster import Hoster -from module.network.RequestFactory import getURL -from module.plugins.ReCaptcha import ReCaptcha - -key = "bGhGMkllZXByd2VEZnU5Y2NXbHhYVlZ5cEE1bkEzRUw=".decode('base64') - - -def getID(url): - """ returns id from file url""" - m = re.match(UploadedTo.__pattern__, url) - return m.group('ID') - - -def getAPIData(urls): - post = {"apikey": key} - - idMap = {} - - for i, url in enumerate(urls): - id = getID(url) - post["id_%s" % i] = id - idMap[id] = url - - for i in xrange(5): - api = unicode(getURL("http://uploaded.net/api/filemultiple", post=post, decode=False), 'iso-8859-1') - if api != "can't find request": - break - else: - sleep(3) - - result = {} - - if api: - for line in api.splitlines(): - data = line.split(",", 4) - if data[1] in idMap: - result[data[1]] = (data[0], data[2], data[4], data[3], idMap[data[1]]) - - return result - - -def parseFileInfo(self, url='', html=''): - if not html and hasattr(self, "html"): html = self.html - name, size, status, found, fileid = url, 0, 3, None, None - - if re.search(self.FILE_OFFLINE_PATTERN, html): - # File offline - status = 1 - else: - found = re.search(self.FILE_INFO_PATTERN, html) - if found: - name, fileid = html_unescape(found.group('N')), found.group('ID') - size = parseFileSize(found.group('S')) - status = 2 - - return name, size, status, fileid - - -def getInfo(urls): - for chunk in chunks(urls, 80): - result = [] - - api = getAPIData(chunk) - - for data in api.itervalues(): - if data[0] == "online": - result.append((html_unescape(data[2]), data[1], 2, data[4])) - - elif data[0] == "offline": - result.append((data[4], 0, 1, data[4])) - - yield result - - -class UploadedTo(Hoster): - __name__ = "UploadedTo" - __type__ = "hoster" - __pattern__ = r"https?://[\w\.-]*?(uploaded\.(to|net)|ul\.to)(/file/|/?\?id=|.*?&id=|/)(?P\w+)" - __version__ = "0.70" - __description__ = """Uploaded.net Download Hoster""" - __author_name__ = ("spoob", "mkaay", "zoidberg", "netpok", "stickell") - __author_mail__ = ("spoob@pyload.org", "mkaay@mkaay.de", "zoidberg@mujmail.cz", "netpok@gmail.com", "l.stickell@yahoo.it") - - FILE_INFO_PATTERN = r'(?P[^<]+)  \s*]*>(?P[^<]+)' - FILE_OFFLINE_PATTERN = r'Error: 404' - DL_LIMIT_PATTERN = "You have reached the max. number of possible free downloads for this hour" - - def setup(self): - self.html = None - self.multiDL = False - self.resumeDownload = False - self.url = False - self.chunkLimit = 1 # critical problems with more chunks - if self.account: - self.premium = self.account.getAccountInfo(self.user)["premium"] - if self.premium: - self.multiDL = True - self.resumeDownload = True - - self.fileID = getID(self.pyfile.url) - self.pyfile.url = "http://uploaded.net/file/%s" % self.fileID - - def process(self, pyfile): - self.load("http://uploaded.net/language/en", just_header=True) - - api = getAPIData([pyfile.url]) - - # TODO: fallback to parse from site, because api sometimes delivers wrong status codes - - if not api: - self.logWarning("No response for API call") - - self.html = unicode(self.load(pyfile.url, decode=False), 'iso-8859-1') - name, size, status, self.fileID = parseFileInfo(self) - self.logDebug(name, size, status, self.fileID) - if status == 1: - self.offline() - elif status == 2: - pyfile.name, pyfile.size = name, size - else: - self.fail('Parse error - file info') - elif api == 'Access denied': - self.fail(_("API key invalid")) - - else: - if self.fileID not in api: - self.offline() - - self.data = api[self.fileID] - if self.data[0] != "online": - self.offline() - - pyfile.name = html_unescape(self.data[2]) - - # self.pyfile.name = self.get_file_name() - - if self.premium: - self.handlePremium() - else: - self.handleFree() - - - def handlePremium(self): - info = self.account.getAccountInfo(self.user, True) - self.log.debug("%(name)s: Use Premium Account (%(left)sGB left)" % {"name": self.__name__, - "left": info["trafficleft"] / 1024 / 1024}) - if int(self.data[1]) / 1024 > info["trafficleft"]: - self.log.info(_("%s: Not enough traffic left" % self.__name__)) - self.account.empty(self.user) - self.resetAccount() - self.fail(_("Traffic exceeded")) - - header = self.load("http://uploaded.net/file/%s" % self.fileID, just_header=True) - if "location" in header: - #Direct download - print "Direct Download: " + header['location'] - self.download(header['location']) - else: - #Indirect download - self.html = self.load("http://uploaded.net/file/%s" % self.fileID) - found = re.search(r'
    (\d+) seconds", self.html) - if not found: - self.fail("File not downloadable for free users") - self.setWait(int(found.group(1))) - - js = self.load("http://uploaded.net/js/download.js", decode=True) - - challengeId = re.search(r'Recaptcha\.create\("([^"]+)', js) - - url = "http://uploaded.net/io/ticket/captcha/%s" % self.fileID - downloadURL = "" - - for i in range(5): - #self.req.lastURL = str(self.url) - re_captcha = ReCaptcha(self) - challenge, result = re_captcha.challenge(challengeId.group(1)) - options = {"recaptcha_challenge_field": challenge, "recaptcha_response_field": result} - self.wait() - - result = self.load(url, post=options) - self.logDebug("result: %s" % result) - - if "limit-size" in result: - self.fail("File too big for free download") - elif "limit-slot" in result: # Temporary restriction so just wait a bit - self.setWait(30 * 60, True) - self.wait() - self.retry() - elif "limit-parallel" in result: - self.fail("Cannot download in parallel") - elif self.DL_LIMIT_PATTERN in result: # limit-dl - self.setWait(3 * 60 * 60, True) - self.wait() - self.retry() - elif 'err:"captcha"' in result: - self.logError("ul.net captcha is disabled") - self.invalidCaptcha() - elif "type:'download'" in result: - self.correctCaptcha() - downloadURL = re.search("url:'([^']+)", result).group(1) - break - else: - self.fail("Unknown error '%s'") - - if not downloadURL: - self.fail("No Download url retrieved/all captcha attempts failed") - - self.download(downloadURL, disposition=True) - check = self.checkDownload({"limit-dl": self.DL_LIMIT_PATTERN}) - if check == "limit-dl": - self.setWait(3 * 60 * 60, True) - self.wait() - self.retry() diff --git a/module/plugins/hoster/UploadheroCom.py b/module/plugins/hoster/UploadheroCom.py deleted file mode 100644 index 502f849af..000000000 --- a/module/plugins/hoster/UploadheroCom.py +++ /dev/null @@ -1,84 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - -class UploadheroCom(SimpleHoster): - __name__ = "UploadheroCom" - __type__ = "hoster" - __pattern__ = r"http://(?:www\.)?uploadhero\.com?/dl/\w+" - __version__ = "0.13" - __description__ = """UploadHero.com plugin""" - __author_name__ = ("mcmyst", "zoidberg") - __author_mail__ = ("mcmyst@hotmail.fr", "zoidberg@mujmail.cz") - - SH_COOKIES = [("http://uploadhero.com", "lang", "en")] - FILE_NAME_PATTERN = r'
    (?P.*?)
    ' - FILE_SIZE_PATTERN = r'Taille du fichier : (?P.*?)' - FILE_OFFLINE_PATTERN = r'

    |

    Le lien du fichier ci-dessus n\'existe plus.' - - DOWNLOAD_URL_PATTERN = r'(\d+).*\s*(\d+)' - - CAPTCHA_PATTERN = r'"(/captchadl\.php\?[a-z0-9]+)"' - FREE_URL_PATTERN = r'var magicomfg = \'"/]+)"' - - def handleFree(self): - self.checkErrors() - - found = re.search(self.CAPTCHA_PATTERN, self.html) - if not found: self.parseError("Captcha URL") - captcha_url = "http://uploadhero.com" + found.group(1) - - for i in range(5): - captcha = self.decryptCaptcha(captcha_url) - self.html = self.load(self.pyfile.url, get = {"code": captcha}) - found = re.search(self.FREE_URL_PATTERN, self.html) - if found: - self.correctCaptcha() - download_url = found.group(1) or found.group(2) - break - else: - self.invalidCaptcha() - else: - self.fail("No valid captcha code entered") - - self.download(download_url) - - def handlePremium(self): - self.log.debug("%s: Use Premium Account" % self.__name__) - self.html = self.load(self.pyfile.url) - link = re.search(self.DOWNLOAD_URL_PATTERN, self.html).group(1) - self.log.debug("Downloading link : '%s'" % link) - self.download(link) - - def checkErrors(self): - found = re.search(self.IP_BLOCKED_PATTERN, self.html) - if found: - self.html = self.load("http://uploadhero.com%s" % found.group(1)) - - found = re.search(self.IP_WAIT_PATTERN, self.html) - wait_time = (int(found.group(1)) * 60 + int(found.group(2))) if found else 300 - self.setWait(wait_time, True) - self.wait() - self.retry() - -getInfo = create_getInfo(UploadheroCom) diff --git a/module/plugins/hoster/UploadingCom.py b/module/plugins/hoster/UploadingCom.py deleted file mode 100644 index 4a157a787..000000000 --- a/module/plugins/hoster/UploadingCom.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: jeix -""" - -import re -from pycurl import HTTPHEADER -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, timestamp -from module.common.json_layer import json_loads - -class UploadingCom(SimpleHoster): - __name__ = "UploadingCom" - __type__ = "hoster" - __pattern__ = r"http://(?:www\.)?uploading\.com/files/(?:get/)?(?P[\w\d]+)" - __version__ = "0.32" - __description__ = """Uploading.Com File Download Hoster""" - __author_name__ = ("jeix", "mkaay", "zoidberg") - __author_mail__ = ("jeix@hasnomail.de", "mkaay@mkaay.de", "zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r'Download (?P<N>.*?) for free on uploading.com' - FILE_SIZE_PATTERN = r'File size: (?P.*?)' - FILE_OFFLINE_PATTERN = r'The requested file is not found' - - def process(self, pyfile): - # set lang to english - self.req.cj.setCookie("uploading.com", "lang", "1") - self.req.cj.setCookie("uploading.com", "language", "1") - self.req.cj.setCookie("uploading.com", "setlang", "en") - self.req.cj.setCookie("uploading.com", "_lang", "en") - - if not "/get/" in self.pyfile.url: - self.pyfile.url = self.pyfile.url.replace("/files", "/files/get") - - self.html = self.load(pyfile.url, decode = True) - self.file_info = self.getFileInfo() - - if self.premium: - self.handlePremium() - else: - self.handleFree() - - def handlePremium(self): - postData = {'action': 'get_link', - 'code': self.file_info['ID'], - 'pass': 'undefined'} - - self.html = self.load('http://uploading.com/files/get/?JsHttpRequest=%d-xml' % timestamp(), post=postData) - url = re.search(r'"link"\s*:\s*"(.*?)"', self.html) - if url: - url = url.group(1).replace("\\/", "/") - self.download(url) - - raise Exception("Plugin defect.") - - def handleFree(self): - found = re.search('

    ((Daily )?Download Limit)

    ', self.html) - if found: - self.pyfile.error = found.group(1) - self.logWarning(self.pyfile.error) - self.retry(max_tries=6, wait_time = 21600 if found.group(2) else 900, reason = self.pyfile.error) - - ajax_url = "http://uploading.com/files/get/?ajax" - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) - self.req.http.lastURL = self.pyfile.url - - response = json_loads(self.load(ajax_url, post = {'action': 'second_page', 'code': self.file_info['ID']})) - if 'answer' in response and 'wait_time' in response['answer']: - wait_time = int(response['answer']['wait_time']) - self.log.info("%s: Waiting %d seconds." % (self.__name__, wait_time)) - self.setWait(wait_time) - self.wait() - else: - self.pluginParseError("AJAX/WAIT") - - response = json_loads(self.load(ajax_url, post = {'action': 'get_link', 'code': self.file_info['ID'], 'pass': 'false'})) - if 'answer' in response and 'link' in response['answer']: - url = response['answer']['link'] - else: - self.pluginParseError("AJAX/URL") - - self.html = self.load(url) - found = re.search(r'\s*Download File\s*]*>(?P[^>]+)\s*[^\(]*\((?P[^\)]+)\)' - FILE_OFFLINE_PATTERN = r'
    File Not Found
    ' - HOSTER_NAME = "uptobox.com" - - def setup(self): - self.resumeDownload = self.multiDL = self.premium - self.chunkLimit = 1 - -getInfo = create_getInfo(UptoboxCom) \ No newline at end of file diff --git a/module/plugins/hoster/VeehdCom.py b/module/plugins/hoster/VeehdCom.py deleted file mode 100644 index 23048b831..000000000 --- a/module/plugins/hoster/VeehdCom.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import re -from module.plugins.Hoster import Hoster - -class VeehdCom(Hoster): - __name__ = 'VeehdCom' - __type__ = 'hoster' - __pattern__ = r'http://veehd\.com/video/\d+_\S+' - __config__ = [ - ('filename_spaces', 'bool', "Allow spaces in filename", 'False'), - ('replacement_char', 'str', "Filename replacement character", '_'), - ] - __version__ = '0.2' - __description__ = """Veehd.com Download Hoster""" - __author_name__ = ('cat') - __author_mail__ = ('cat@pyload') - - def _debug(self, msg): - self.log.debug('[%s] %s' % (self.__name__, msg)) - - def setup(self): - self.html = None - self.multiDL = True - self.req.canContinue = True - - def process(self, pyfile): - self.download_html() - if not self.file_exists(): - self.offline() - - pyfile.name = self.get_file_name() - self.download(self.get_file_url()) - - def download_html(self): - url = self.pyfile.url - self._debug("Requesting page: %s" % (repr(url),)) - self.html = self.load(url) - - def file_exists(self): - if self.html is None: - self.download_html() - - if 'Veehd' in self.html: - return False - return True - - def get_file_name(self): - if self.html is None: - self.download_html() - - match = re.search(r']*>([^<]+) on Veehd', self.html) - if not match: - self.fail("video title not found") - name = match.group(1) - - # replace unwanted characters in filename - if self.getConf('filename_spaces'): - pattern = '[^0-9A-Za-z\.\ ]+' - else: - pattern = '[^0-9A-Za-z\.]+' - - name = re.sub(pattern, self.getConf('replacement_char'), - name) - return name + '.avi' - - def get_file_url(self): - """ returns the absolute downloadable filepath - """ - if self.html is None: - self.download_html() - - match = re.search(r'. - - @author: zoidberg -""" - -#similar to coolshare.cz (down) - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.network.HTTPRequest import BadHeader -from module.utils import html_unescape - -class WarserverCz(SimpleHoster): - __name__ = "WarserverCz" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)?warserver.cz/stahnout/(?P\d+)/.+" - __version__ = "0.12" - __description__ = """Warserver.cz""" - __author_name__ = ("zoidberg") - - FILE_NAME_PATTERN = r'(?P[^<]+)' - FILE_SIZE_PATTERN = r'
  • Velikost: (?P[^<]+)' - FILE_OFFLINE_PATTERN = r'

    Soubor nenalezen

    ' - - PREMIUM_URL_PATTERN = r'href="(http://[^/]+/dwn-premium.php.*?)"' - DOMAIN = "http://csd01.coolshare.cz" - - DOMAIN = "http://s01.warserver.cz" - - def handleFree(self): - try: - self.download("%s/dwn-free.php?fid=%s" % (self.DOMAIN, self.file_info['ID'])) - except BadHeader, e: - self.logError(e) - if e.code == 403: - self.longWait(60,60) - else: raise - self.checkDownloadedFile() - - def handlePremium(self): - found = re.search(self.PREMIUM_URL_PATTERN, self.html) - if not found: self.parseError("Premium URL") - url = html_unescape(found.group(1)) - self.logDebug("Premium URL: " + url) - if not url.startswith("http://"): self.resetAccount() - self.download(url) - self.checkDownloadedFile() - - def checkDownloadedFile(self): - check = self.checkDownload({ - "offline": ">404 Not Found<" - }) - - if check == "offline": - self.offline() - -getInfo = create_getInfo(WarserverCz) \ No newline at end of file diff --git a/module/plugins/hoster/WebshareCz.py b/module/plugins/hoster/WebshareCz.py deleted file mode 100644 index 195e65a93..000000000 --- a/module/plugins/hoster/WebshareCz.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.network.HTTPRequest import BadHeader - -class WebshareCz(SimpleHoster): - __name__ = "WebshareCz" - __type__ = "hoster" - __pattern__ = r"http://(\w+\.)?webshare.cz/(stahnout/)?(?P\w{10})-.+" - __version__ = "0.12" - __description__ = """WebShare.cz""" - __author_name__ = ("zoidberg") - - FILE_NAME_PATTERN = r'

    Stahujete soubor:

    \s*
    (?P[^<]+)
    ' - FILE_SIZE_PATTERN = r'

    Velikost souboru je:

    \s*
    (?P[^<]+)
    ' - FILE_OFFLINE_PATTERN = r'

    Soubor ".*?" nebyl nalezen.

    ' - - DOWNLOAD_LINK_PATTERN = r'id="download_link" href="(?P.*?)"' - - def setup(self): - self.multiDL = True - - def handleFree(self): - url_a = re.search(r"(var l.*)", self.html).group(1) - url_b = re.search(r"(var keyStr.*)", self.html).group(1) - url = self.js.eval("%s\n%s\ndec(l)" % (url_a, url_b)) - - self.logDebug('Download link: ' + url) - self.download(url) - -getInfo = create_getInfo(WebshareCz) \ No newline at end of file diff --git a/module/plugins/hoster/WrzucTo.py b/module/plugins/hoster/WrzucTo.py deleted file mode 100644 index 4a5e89f22..000000000 --- a/module/plugins/hoster/WrzucTo.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from pycurl import HTTPHEADER - -class WrzucTo(SimpleHoster): - __name__ = "WrzucTo" - __type__ = "hoster" - __pattern__ = r"http://(?:\w+\.)*?wrzuc\.to/([a-zA-Z0-9]+(\.wt|\.html)|(\w+/?linki/[a-zA-Z0-9]+))" - __version__ = "0.01" - __description__ = """Wrzuc.to plugin - free only""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - SH_COOKIES = [("http://www.wrzuc.to", "language", "en")] - FILE_SIZE_PATTERN = r'class="info">\s*\s*(?P.*?)' - FILE_NAME_PATTERN = r'id="file_info">\s*(?P.*?)' - - def setup(self): - self.multiDL = True - - def handleFree(self): - data = dict(re.findall(r'(md5|file): "(.*?)"', self.html)) - if len(data) != 2: self.parseError('File ID') - - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) - self.req.http.lastURL = self.pyfile.url - self.load("http://www.wrzuc.to/ajax/server/prepair", post = {"md5": data['md5']}) - - self.req.http.lastURL = self.pyfile.url - self.html = self.load("http://www.wrzuc.to/ajax/server/download_link", post = {"file": data['file']}) - - data.update(re.findall(r'"(download_link|server_id)":"(.*?)"', self.html)) - if len(data) != 4: self.parseError('Download URL') - - download_url = "http://%s.wrzuc.to/pobierz/%s" % (data['server_id'], data['download_link']) - self.logDebug("Download URL: %s" % download_url) - self.download(download_url) - -getInfo = create_getInfo(WrzucTo) - diff --git a/module/plugins/hoster/WuploadCom.py b/module/plugins/hoster/WuploadCom.py deleted file mode 100644 index 1a0eb442b..000000000 --- a/module/plugins/hoster/WuploadCom.py +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -import string -from urllib import unquote - -from module.plugins.Hoster import Hoster -from module.plugins.ReCaptcha import ReCaptcha -from module.plugins.Plugin import chunks - -from module.network.RequestFactory import getURL -from module.common.json_layer import json_loads - - -def getInfo(urls): - for chunk in chunks(urls, 20): - result = [] - ids = dict() - for url in chunk: - id = getId(url) - if id: - ids[id] = url - else: - result.append((None, 0, 1, url)) - - if len(ids) > 0: - check_url = "http://api.wupload.com/link?method=getInfo&format=json&ids=" + ",".join(ids.keys()) - response = json_loads(getURL(check_url).decode("utf8", "ignore")) - for item in response["FSApi_Link"]["getInfo"]["response"]["links"]: - if item["status"] != "AVAILABLE": - result.append((None, 0, 1, ids[str(item["id"])])) - else: - result.append((unquote(item["filename"]), item["size"], 2, ids[str(item["id"])])) - yield result - - -def getId(url): - match = re.search(WuploadCom.FILE_ID_PATTERN, url) - if match: - return string.replace(match.group("id"), "/", "-") - else: - return None - - -class WuploadCom(Hoster): - __name__ = "WuploadCom" - __type__ = "hoster" - __pattern__ = r"http://[\w\.]*?wupload\..*?/file/(([a-z][0-9]+/)?[0-9]+)(/.*)?" - __version__ = "0.20" - __description__ = """Wupload com""" - __author_name__ = ("jeix", "paulking") - __author_mail__ = ("jeix@hasnomail.de", "") - - API_ADDRESS = "http://api.wupload.com" - URL_DOMAIN_PATTERN = r'(?P.*?)(?P.wupload\..+?)(?P/.*)' - FILE_ID_PATTERN = r'/file/(?P([a-z][0-9]+/)?[0-9]+)(/.*)?' - FILE_LINK_PATTERN = r'

    Download Link

    \s*]*>([^<]+)' - OVR_KILL_LINK_PATTERN = r'

    Delete Link

    \s*]*>([^<]+)' - CAPTCHA_URL_PATTERN = r'(http://[^"\']+?/captchas?/[^"\']+)' - RECAPTCHA_URL_PATTERN = r'http://[^"\']+?recaptcha[^"\']+?\?k=([^"\']+)"' - CAPTCHA_DIV_PATTERN = r'Enter code.*?(.*?)

    Download Now' - WAIT_TIME_PATTERN = r'countDownDelay = (?P\d+)' - WAIT_TM_PATTERN = r"name='tm' value='(.*?)' />" - WAIT_TM_HASH_PATTERN = r"name='tm_hash' value='(.*?)' />" - CAPTCHA_TYPE1_PATTERN = r'Recaptcha.create\("(.*?)",' - CAPTCHA_TYPE2_PATTERN = r'id="recaptcha_image"> 300: - self.wantReconnect = True - - self.setWait(wait) - self.logDebug("Waiting %d seconds." % wait) - self.wait() - - tm = re.search(self.WAIT_TM_PATTERN, self.html) - tm_hash = re.search(self.WAIT_TM_HASH_PATTERN, self.html) - - if tm and tm_hash: - tm = tm.group(1) - tm_hash = tm_hash.group(1) - self.html = self.load(url, post={"tm": tm, "tm_hash": tm_hash}) - self.handleErrors() - break - else: - self.html = self.load(url) - self.handleErrors() - waitSearch = re.search(self.WAIT_TIME_PATTERN, self.html) - - def handleErrors(self): - if "This file is available for premium users only." in self.html: - self.fail("need premium account for file") - - if "The file that you're trying to download is larger than" in self.html: - self.fail("need premium account for file") - - if "Free users may only download 1 file at a time" in self.html: - self.fail("only 1 file at a time for free users") - - if "Free user can not download files" in self.html: - self.fail("need premium account for file") - - if "Download session in progress" in self.html: - self.fail("already downloading") - - if "This file is password protected" in self.html: - self.fail("This file is password protected") - - if "An Error Occurred" in self.html: - self.fail("A server error occured.") - - if "This file was deleted" in self.html: - self.offline() diff --git a/module/plugins/hoster/X7To.py b/module/plugins/hoster/X7To.py deleted file mode 100644 index 79adf2a3f..000000000 --- a/module/plugins/hoster/X7To.py +++ /dev/null @@ -1,93 +0,0 @@ -# -*- coding: utf-8 -*- -import re - -from module.plugins.Hoster import Hoster - -from module.network.RequestFactory import getURL - -def getInfo(urls): - yield [(url, 0, 1, url) for url in urls] - - -class X7To(Hoster): - __name__ = "X7To" - __type__ = "hoster" - __pattern__ = r"http://(?:www.)?x7.to/" - __version__ = "0.3" - __description__ = """X7.To File Download Hoster""" - __author_name__ = ("ernieb") - __author_mail__ = ("ernieb") - - FILE_INFO_PATTERN=r'', self.html) - if overLimit: - self.logDebug("over limit, falling back to free") - self.handleFree() - else: - realurl = re.search(r'', self.html) - if realurl: - realurl = realurl.group(1) - self.logDebug("premium url found %s" % realurl) - else: - self.logDebug("premium link not found") - self.download(realurl) - - def handleFree(self): - # find file id - file_id = re.search(r"var dlID = '(.*?)'", self.html) - if not file_id: - self.fail("Free download id not found") - - file_url = "http://x7.to/james/ticket/dl/" + file_id.group(1) - self.logDebug("download id %s" % file_id.group(1)) - - self.html = self.load(file_url, ref=False, decode=True) - - # deal with errors - if "limit-dl" in self.html: - self.logDebug("Limit reached ... waiting") - self.setWait(900,True) - self.wait() - self.retry() - - if "limit-parallel" in self.html: - self.fail("Cannot download in parallel") - - # no waiting required, go to download - waitCheck = re.search(r"wait:(\d*),", self.html) - if waitCheck: - waitCheck = int(waitCheck.group(1)) - self.setWait(waitCheck) - self.wait() - - urlCheck = re.search(r"url:'(.*?)'", self.html) - url = None - if urlCheck: - url = urlCheck.group(1) - self.logDebug("free url found %s" % url) - - if url: - try: - self.download(url) - except: - self.logDebug("downloading url failed: %s" % url) - else: - self.fail("Free download url found") diff --git a/module/plugins/hoster/XFileSharingPro.py b/module/plugins/hoster/XFileSharingPro.py deleted file mode 100644 index 1120a2a8b..000000000 --- a/module/plugins/hoster/XFileSharingPro.py +++ /dev/null @@ -1,314 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: zoidberg -""" - -import re -from random import random -from urllib import unquote -from urlparse import urlparse -from pycurl import FOLLOWLOCATION, LOW_SPEED_TIME -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, PluginParseError -from module.plugins.ReCaptcha import ReCaptcha -from module.plugins.internal.CaptchaService import SolveMedia, AdsCaptcha -from module.utils import html_unescape - -class XFileSharingPro(SimpleHoster): - """ - Common base for XFileSharingPro hosters like EasybytezCom, CramitIn, FiledinoCom... - Some hosters may work straight away when added to __pattern__ - However, most of them will NOT work because they are either down or running a customized version - """ - __name__ = "XFileSharingPro" - __type__ = "hoster" - __pattern__ = r"^unmatchable$" - __version__ = "0.17" - __description__ = """XFileSharingPro common hoster base""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r'[^<]+ \((?P[^<]+)\)' - FILE_INFO_PATTERN = r'Filename:(?P[^<]+)\s*.*?\((?P[^<]+)\)' - FILE_OFFLINE_PATTERN = r'<(b|h[1-6])>File Not Found' - - WAIT_PATTERN = r'.*?>(\d+)' - LONG_WAIT_PATTERN = r'(?P\d+(?=\s*hour))?.*?(?P\d+(?=\s*minute))?.*?(?P\d+(?=\s*second))?' - OVR_DOWNLOAD_LINK_PATTERN = r'

    • ' - SOLVEMEDIA_PATTERN = r'http:\/\/api\.solvemedia\.com\/papi\/challenge\.script\?k=(.*?)"' - ERROR_PATTERN = r'class=["\']err["\'][^>]*>(.*?)"," ",self.errmsg)) - - if 'wait' in self.errmsg: - wait_time = sum([int(v) * {"hour": 3600, "minute": 60, "second": 1}[u] for v, u in re.findall('(\d+)\s*(hour|minute|second)?', self.errmsg)]) - self.setWait(wait_time, True) - self.wait() - elif 'captcha' in self.errmsg: - self.invalidCaptcha() - elif 'premium' in self.errmsg and 'require' in self.errmsg: - self.fail("File can be downloaded by premium users only") - elif 'limit' in self.errmsg: - self.setWait(3600, True) - self.wait() - self.retry(25) - elif 'countdown' in self.errmsg or 'Expired session' in self.errmsg: - self.retry(3) - elif 'maintenance' in self.errmsg: - self.tempOffline() - elif 'download files up to' in self.errmsg: - self.fail("File too large for free download") - else: - self.fail(self.errmsg) - - else: - self.errmsg = None - - return self.errmsg - - def getPostParameters(self): - for i in range(3): - if not self.errmsg: self.checkErrors() - - if hasattr(self,"FORM_PATTERN"): - action, inputs = self.parseHtmlForm(self.FORM_PATTERN) - else: - action, inputs = self.parseHtmlForm(input_names={"op": re.compile("^download")}) - - if not inputs: - action, inputs = self.parseHtmlForm('F1') - if not inputs: - if self.errmsg: - self.retry() - else: - self.parseError("Form not found") - - self.logDebug(self.HOSTER_NAME, inputs) - - if 'op' in inputs and inputs['op'] in ('download2', 'download3'): - if "password" in inputs: - if self.passwords: - inputs['password'] = self.passwords.pop(0) - else: - self.fail("No or invalid passport") - - if not self.premium: - found = re.search(self.WAIT_PATTERN, self.html) - if found: - wait_time = int(found.group(1)) + 1 - self.setWait(wait_time, False) - else: - wait_time = 0 - - self.captcha = self.handleCaptcha(inputs) - - if wait_time: self.wait() - - self.errmsg = None - return inputs - - else: - inputs['referer'] = self.pyfile.url - - if self.premium: - inputs['method_premium'] = "Premium Download" - if 'method_free' in inputs: del inputs['method_free'] - else: - inputs['method_free'] = "Free Download" - if 'method_premium' in inputs: del inputs['method_premium'] - - self.html = self.load(self.pyfile.url, post = inputs, ref = True) - self.errmsg = None - - else: self.parseError('FORM: %s' % (inputs['op'] if 'op' in inputs else 'UNKNOWN')) - - def handleCaptcha(self, inputs): - found = re.search(self.RECAPTCHA_URL_PATTERN, self.html) - if found: - recaptcha_key = unquote(found.group(1)) - self.logDebug("RECAPTCHA KEY: %s" % recaptcha_key) - recaptcha = ReCaptcha(self) - inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(recaptcha_key) - return 1 - else: - found = re.search(self.CAPTCHA_URL_PATTERN, self.html) - if found: - captcha_url = found.group(1) - inputs['code'] = self.decryptCaptcha(captcha_url) - return 2 - else: - found = re.search(self.CAPTCHA_DIV_PATTERN, self.html, re.S) - if found: - captcha_div = found.group(1) - self.logDebug(captcha_div) - numerals = re.findall('(\d)', html_unescape(captcha_div)) - inputs['code'] = "".join([a[1] for a in sorted(numerals, key = lambda num: int(num[0]))]) - self.logDebug("CAPTCHA", inputs['code'], numerals) - return 3 - else: - found = re.search(self.SOLVEMEDIA_PATTERN, self.html) - if found: - captcha_key = found.group(1) - captcha = SolveMedia(self) - inputs['adcopy_challenge'], inputs['adcopy_response'] = captcha.challenge(captcha_key) - return 4 - return 0 - -getInfo = create_getInfo(XFileSharingPro) diff --git a/module/plugins/hoster/XHamsterCom.py b/module/plugins/hoster/XHamsterCom.py deleted file mode 100644 index 0779a78e6..000000000 --- a/module/plugins/hoster/XHamsterCom.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from module.plugins.Hoster import Hoster -from urllib import unquote -from module.common.json_layer import json_loads - -def clean_json(json_expr): - json_expr = re.sub('[\n\r]', '', json_expr) - json_expr = re.sub(' +', '', json_expr) - json_expr = re.sub('\'','"', json_expr) - - return json_expr - -class XHamsterCom(Hoster): - __name__ = "XHamsterCom" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?xhamster\.com/movies/.+" - __version__ = "0.1" - __config__ = [("type", ".mp4;.flv", "Preferred type", ".mp4")] - __description__ = """XHamster.com Video Download Hoster""" - - def setup(self): - self.html = None - - def process(self, pyfile): - self.pyfile = pyfile - - if not self.file_exists(): - self.offline() - - if self.getConfig("type"): - self.desired_fmt = self.getConf("type") - - self.pyfile.name = self.get_file_name() + self.desired_fmt - self.download(self.get_file_url()) - - def download_html(self): - url = self.pyfile.url - self.html = self.load(url) - - def get_file_url(self): - """ returns the absolute downloadable filepath - """ - if self.html is None: - self.download_html() - - flashvar_pattern = re.compile('flashvars = ({.*?});', re.DOTALL) - json_flashvar=flashvar_pattern.search(self.html) - - if json_flashvar is None: - self.fail("Parse error (flashvars)") - - j = clean_json(json_flashvar.group(1)) - flashvars = json_loads(j) - - if flashvars["srv"]: - srv_url = flashvars["srv"] + '/' - else: - self.fail("Parse error (srv_url)") - - if flashvars["url_mode"]: - url_mode = flashvars["url_mode"] - else: - self.fail("Parse error (url_mode)") - - - if self.desired_fmt == ".mp4": - file_url = re.search(r"(.*?) - xHamster\.com" - file_name = re.search(file_name_pattern, self.html) - if file_name is None: - file_name_pattern = r"

    (.*)

    " - file_name = re.search(file_name_pattern, self.html) - if file_name is None: - file_name_pattern = r"http://[www.]+xhamster\.com/movies/.*/(.*?)\.html?" - file_name = re.search(file_name_pattern, self.pyfile.url) - if file_name is None: - file_name_pattern = r"
    (.*)
    " - file_name = re.search(file_name_pattern, self.html) - if file_name is None: - return "Unknown" - - return file_name.group(1) - - def file_exists(self): - """ returns True or False - """ - if self.html is None: - self.download_html() - if re.search(r"(.*Video not found.*)", self.html) is not None: - return False - else: - return True diff --git a/module/plugins/hoster/XVideosCom.py b/module/plugins/hoster/XVideosCom.py deleted file mode 100644 index b7f3f7b58..000000000 --- a/module/plugins/hoster/XVideosCom.py +++ /dev/null @@ -1,19 +0,0 @@ - -import re -import urllib - -from module.plugins.Hoster import Hoster - -class XVideosCom(Hoster): - __name__ = "XVideos.com" - __version__ = "0.1" - __pattern__ = r"http://www\.xvideos\.com/video([0-9]+)/.*" - __config__ = [] - - def process(self, pyfile): - site = self.load(pyfile.url) - pyfile.name = "%s (%s).flv" %( - re.search(r"

    ([^<]+). - - @author: jeix -""" - -from os.path import join -from os.path import exists -from os import makedirs -import re -import sys -import time -import socket, struct -from select import select -from module.utils import save_join - -from module.plugins.Hoster import Hoster - - -class Xdcc(Hoster): - __name__ = "Xdcc" - __version__ = "0.3" - __pattern__ = r'xdcc://.*?(/#?.*?)?/.*?/#?\d+/?' # xdcc://irc.Abjects.net/#channel/[XDCC]|Shit/#0004/ - __type__ = "hoster" - __config__ = [ - ("nick", "str", "Nickname", "pyload"), - ("ident", "str", "Ident", "pyloadident"), - ("realname", "str", "Realname", "pyloadreal") - ] - __description__ = """A Plugin that allows you to download from an IRC XDCC bot""" - __author_name__ = ("jeix") - __author_mail__ = ("jeix@hasnomail.com") - - def setup(self): - self.debug = 0 #0,1,2 - self.timeout = 30 - self.multiDL = False - - - - def process(self, pyfile): - # change request type - self.req = pyfile.m.core.requestFactory.getRequest(self.__name__, type="XDCC") - - self.pyfile = pyfile - for i in range(0,3): - try: - nmn = self.doDownload(pyfile.url) - self.log.debug("%s: Download of %s finished." % (self.__name__, nmn)) - return - except socket.error, e: - if hasattr(e, "errno"): - errno = e.errno - else: - errno = e.args[0] - - if errno in (10054,): - self.log.debug("XDCC: Server blocked our ip, retry in 5 min") - self.setWait(300) - self.wait() - continue - - self.fail("Failed due to socket errors. Code: %d" % errno) - - self.fail("Server blocked our ip, retry again later manually") - - - def doDownload(self, url): - self.pyfile.setStatus("waiting") # real link - - download_folder = self.config['general']['download_folder'] - location = join(download_folder, self.pyfile.package().folder.decode(sys.getfilesystemencoding())) - if not exists(location): - makedirs(location) - - m = re.search(r'xdcc://(.*?)/#?(.*?)/(.*?)/#?(\d+)/?', url) - server = m.group(1) - chan = m.group(2) - bot = m.group(3) - pack = m.group(4) - nick = self.getConf('nick') - ident = self.getConf('ident') - real = self.getConf('realname') - - temp = server.split(':') - ln = len(temp) - if ln == 2: - host, port = temp - elif ln == 1: - host, port = temp[0], 6667 - else: - self.fail("Invalid hostname for IRC Server (%s)" % server) - - - ####################### - # CONNECT TO IRC AND IDLE FOR REAL LINK - dl_time = time.time() - - sock = socket.socket() - sock.connect((host, int(port))) - if nick == "pyload": - nick = "pyload-%d" % (time.time() % 1000) # last 3 digits - sock.send("NICK %s\r\n" % nick) - sock.send("USER %s %s bla :%s\r\n" % (ident, host, real)) - time.sleep(3) - sock.send("JOIN #%s\r\n" % chan) - sock.send("PRIVMSG %s :xdcc send #%s\r\n" % (bot, pack)) - - # IRC recv loop - readbuffer = "" - done = False - retry = None - m = None - while True: - - # done is set if we got our real link - if done: - break - - if retry: - if time.time() > retry: - retry = None - dl_time = time.time() - sock.send("PRIVMSG %s :xdcc send #%s\r\n" % (bot, pack)) - - else: - if (dl_time + self.timeout) < time.time(): # todo: add in config - sock.send("QUIT :byebye\r\n") - sock.close() - self.fail("XDCC Bot did not answer") - - - fdset = select([sock], [], [], 0) - if sock not in fdset[0]: - continue - - readbuffer += sock.recv(1024) - temp = readbuffer.split("\n") - readbuffer = temp.pop() - - for line in temp: - if self.debug is 2: print "*> " + unicode(line, errors='ignore') - line = line.rstrip() - first = line.split() - - if first[0] == "PING": - sock.send("PONG %s\r\n" % first[1]) - - if first[0] == "ERROR": - self.fail("IRC-Error: %s" % line) - - msg = line.split(None, 3) - if len(msg) != 4: - continue - - msg = { \ - "origin":msg[0][1:], \ - "action":msg[1], \ - "target":msg[2], \ - "text" :msg[3][1:] \ - } - - - if nick == msg["target"][0:len(nick)] and "PRIVMSG" == msg["action"]: - if msg["text"] == "\x01VERSION\x01": - self.log.debug("XDCC: Sending CTCP VERSION.") - sock.send("NOTICE %s :%s\r\n" % (msg['origin'], "pyLoad! IRC Interface")) - elif msg["text"] == "\x01TIME\x01": - self.log.debug("Sending CTCP TIME.") - sock.send("NOTICE %s :%d\r\n" % (msg['origin'], time.time())) - elif msg["text"] == "\x01LAG\x01": - pass # don't know how to answer - - if not (bot == msg["origin"][0:len(bot)] - and nick == msg["target"][0:len(nick)] - and msg["action"] in ("PRIVMSG", "NOTICE")): - continue - - if self.debug is 1: - print "%s: %s" % (msg["origin"], msg["text"]) - - if "You already requested that pack" in msg["text"]: - retry = time.time() + 300 - - if "you must be on a known channel to request a pack" in msg["text"]: - self.fail("Wrong channel") - - m = re.match('\x01DCC SEND (.*?) (\d+) (\d+)(?: (\d+))?\x01', msg["text"]) - if m: - done = True - - # get connection data - ip = socket.inet_ntoa(struct.pack('L', socket.ntohl(int(m.group(2))))) - port = int(m.group(3)) - packname = m.group(1) - - if len(m.groups()) > 3: - self.req.filesize = int(m.group(4)) - - self.pyfile.name = packname - filename = save_join(location, packname) - self.log.info("XDCC: Downloading %s from %s:%d" % (packname, ip, port)) - - self.pyfile.setStatus("downloading") - newname = self.req.download(ip, port, filename, sock, self.pyfile.setProgress) - if newname and newname != filename: - self.log.info("%(name)s saved as %(newname)s" % {"name": self.pyfile.name, "newname": newname}) - filename = newname - - # kill IRC socket - # sock.send("QUIT :byebye\r\n") - sock.close() - - self.lastDownload = filename - return self.lastDownload - diff --git a/module/plugins/hoster/XvidstageCom.py b/module/plugins/hoster/XvidstageCom.py deleted file mode 100644 index 4962c05af..000000000 --- a/module/plugins/hoster/XvidstageCom.py +++ /dev/null @@ -1,114 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: 4Christopher -""" - -import re -import HTMLParser - -from module.plugins.Hoster import Hoster -from module.network.RequestFactory import getURL - - -def setup(self): - self.wantReconnect = False - self.resumeDownload = True - self.multiDL = True - - -def getInfo(urls): - result = [] - - for url in urls: - result.append(parseFileInfo(url, getInfoMode=True)) - yield result - - -def parseFileInfo(url, getInfoMode=False): - html = getURL(url) - info = {"name": url, "size": 0, "status": 3} - try: - info['name'] = re.search(r'(?:Filename|Dateiname):]*?>(.*?)<', html).group(1) - info['size'] = re.search(r'(?:Size|Größe):.*? \((\d+?) bytes\)', html).group(1) - except: ## The file is offline - info['status'] = 1 - else: - info['status'] = 2 - - if getInfoMode: - return info['name'], info['size'], info['status'], url - else: - return info['name'], info['size'], info['status'], html - - -class XvidstageCom(Hoster): - __name__ = 'XvidstageCom' - __version__ = '0.4' - __pattern__ = r'http://(?:www.)?xvidstage.com/(?P[0-9A-Za-z]+)' - __type__ = 'hoster' - __description__ = """A Plugin that allows you to download files from http://xvidstage.com""" - __author_name__ = ('4Christopher') - __author_mail__ = ('4Christopher@gmx.de') - - - def process(self, pyfile): - pyfile.name, pyfile.size, pyfile.status, self.html = parseFileInfo(pyfile.url) - self.logDebug('Name: %s' % pyfile.name) - if pyfile.status == 1: ## offline - self.offline() - self.id = re.search(self.__pattern__, pyfile.url).group('id') - - wait_sec = int(re.search(r'countdown_str">.+?>(\d+?)<', self.html).group(1)) - self.setWait(wait_sec, reconnect=False) - self.logDebug('Waiting %d seconds before submitting the captcha' % wait_sec) - self.wait() - - rand = re.search(r'', self.html).group(1) - self.logDebug('rand: %s, id: %s' % (rand, self.id)) - self.html = self.req.load(pyfile.url, - post={'op': 'download2', 'id': self.id, 'rand': rand, 'code': self.get_captcha()}) - file_url = re.search(r'(?P=url)', self.html).group('url') - try: - hours_file_available = int( - re.search(r'This direct link will be available for your IP next (?P\d+?) hours', - self.html).group('hours')) - self.logDebug( - 'You have %d hours to download this file with your current IP address.' % hours_file_available) - except: - self.logDebug('Failed') - self.logDebug('Download file: %s' % file_url) - self.download(file_url) - check = self.checkDownload({'empty': re.compile(r'^$')}) - - if check == 'empty': - self.logInfo('Downloaded File was empty') - # self.retry() - - def get_captcha(self): - ## 1 … - cap_chars = {} - for pad_left, char in re.findall(r"position:absolute;padding-left:(\d+?)px;.*?;'>(.*?)<", self.html): - cap_chars[int(pad_left)] = char - - h = HTMLParser.HTMLParser() - ## Sorting after padding-left - captcha = '' - for pad_left in sorted(cap_chars): - captcha += h.unescape(cap_chars[pad_left]) - - self.logDebug('The captcha is: %s' % captcha) - return captcha diff --git a/module/plugins/hoster/YibaishiwuCom.py b/module/plugins/hoster/YibaishiwuCom.py deleted file mode 100644 index 901225944..000000000 --- a/module/plugins/hoster/YibaishiwuCom.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.common.json_layer import json_loads - -class YibaishiwuCom(SimpleHoster): - __name__ = "YibaishiwuCom" - __type__ = "hoster" - __pattern__ = r"http://(?:www\.)?(?:u\.)?115.com/file/(?P\w+)" - __version__ = "0.12" - __description__ = """115.com""" - __author_name__ = ("zoidberg") - - FILE_NAME_PATTERN = r"file_name: '(?P[^']+)'" - FILE_SIZE_PATTERN = r"file_size: '(?P[^']+)'" - FILE_OFFLINE_PATTERN = ur'

    哎呀！提取码不存在！不妨搜搜看吧！

    ' - - AJAX_URL_PATTERN = r'(/\?ct=(pickcode|download)[^"\']+)' - - def handleFree(self): - found = re.search(self.AJAX_URL_PATTERN, self.html) - if not found: self.parseError("AJAX URL") - url = found.group(1) - self.logDebug(('FREEUSER' if found.group(2) == 'download' else 'GUEST') + ' URL', url) - - response = json_loads(self.load("http://115.com" + url, decode = False)) - for mirror in (response['urls'] if 'urls' in response else response['data'] if 'data' in response else []): - try: - url = mirror['url'].replace('\\','') - self.logDebug("Trying URL: " + url) - self.download(url) - break - except: - continue - else: self.fail('No working link found') - -getInfo = create_getInfo(YibaishiwuCom) diff --git a/module/plugins/hoster/YoupornCom.py b/module/plugins/hoster/YoupornCom.py deleted file mode 100644 index b17a4ef80..000000000 --- a/module/plugins/hoster/YoupornCom.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from module.plugins.Hoster import Hoster - -class YoupornCom(Hoster): - __name__ = "YoupornCom" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?youporn\.com/watch/.+" - __version__ = "0.2" - __description__ = """Youporn.com Video Download Hoster""" - __author_name__ = ("willnix") - __author_mail__ = ("willnix@pyload.org") - - def setup(self): - self.html = None - - def process(self, pyfile): - self.pyfile = pyfile - - if not self.file_exists(): - self.offline() - - self.pyfile.name = self.get_file_name() - self.download(self.get_file_url()) - - def download_html(self): - url = self.pyfile.url - self.html = self.load(url, post={"user_choice":"Enter"}, cookies=False) - - def get_file_url(self): - """ returns the absolute downloadable filepath - """ - if self.html is None: - self.download_html() - - file_url = re.search(r'(http://download\.youporn\.com/download/\d+\?save=1)">', self.html).group(1) - return file_url - - def get_file_name(self): - if self.html is None: - self.download_html() - - file_name_pattern = r"(.*) - Free Porn Videos - YouPorn" - return re.search(file_name_pattern, self.html).group(1).replace("&", "&").replace("/","") + '.flv' - - def file_exists(self): - """ returns True or False - """ - if self.html is None: - self.download_html() - if re.search(r"(.*invalid video_id.*)", self.html) is not None: - return False - else: - return True diff --git a/module/plugins/hoster/YourfilesTo.py b/module/plugins/hoster/YourfilesTo.py deleted file mode 100644 index b67ccb68d..000000000 --- a/module/plugins/hoster/YourfilesTo.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -import urllib -from module.plugins.Hoster import Hoster - -class YourfilesTo(Hoster): - __name__ = "YourfilesTo" - __type__ = "hoster" - __pattern__ = r"(http://)?(www\.)?yourfiles\.(to|biz)/\?d=[a-zA-Z0-9]+" - __version__ = "0.2" - __description__ = """Youfiles.to Download Hoster""" - __author_name__ = ("jeix", "skydancer") - __author_mail__ = ("jeix@hasnomail.de", "skydancer@hasnomail.de") - - def setup(self): - self.html = None - self.multiDL = True - - def process(self,pyfile): - self.pyfile = pyfile - self.prepare() - self.download(self.get_file_url()) - - def prepare(self): - if not self.file_exists(): - self.offline() - - self.pyfile.name = self.get_file_name() - - wait_time = self.get_waiting_time() - self.setWait(wait_time) - self.log.debug("%s: Waiting %d seconds." % (self.__name__,wait_time)) - self.wait() - - def get_waiting_time(self): - if self.html is None: - self.download_html() - - #var zzipitime = 15; - m = re.search(r'var zzipitime = (\d+);', self.html) - if m: - sec = int(m.group(1)) - else: - sec = 0 - - return sec - - def download_html(self): - url = self.pyfile.url - self.html = self.load(url) - - def get_file_url(self): - """ returns the absolute downloadable filepath - """ - url = re.search(r"var bla = '(.*?)';", self.html) - if url: - url = url.group(1) - url = urllib.unquote(url.replace("http://http:/http://", "http://").replace("dumdidum", "")) - return url - else: - self.fail("absolute filepath could not be found. offline? ") - - def get_file_name(self): - if self.html is None: - self.download_html() - - return re.search("(.*)", self.html).group(1) - - def file_exists(self): - """ returns True or False - """ - if self.html is None: - self.download_html() - - if re.search(r"HTTP Status 404", self.html) is not None: - return False - else: - return True - - - diff --git a/module/plugins/hoster/YoutubeCom.py b/module/plugins/hoster/YoutubeCom.py deleted file mode 100644 index 70db597cf..000000000 --- a/module/plugins/hoster/YoutubeCom.py +++ /dev/null @@ -1,164 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -import subprocess -import os -import os.path -from urllib import unquote - -from module.utils import html_unescape -from module.plugins.Hoster import Hoster - -def which(program): - """Works exactly like the unix command which - - Courtesy of http://stackoverflow.com/a/377028/675646""" - def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - fpath, fname = os.path.split(program) - if fpath: - if is_exe(program): - return program - else: - for path in os.environ["PATH"].split(os.pathsep): - path = path.strip('"') - exe_file = os.path.join(path, program) - if is_exe(exe_file): - return exe_file - - return None - -class YoutubeCom(Hoster): - __name__ = "YoutubeCom" - __type__ = "hoster" - __pattern__ = r"https?://(?:[^/]*?)youtube\.com/watch.*?[?&]v=.*" - __version__ = "0.32" - __config__ = [("quality", "sd;hd;fullhd;240p;360p;480p;720p;1080p;3072p", "Quality Setting", "hd"), - ("fmt", "int", "FMT/ITAG Number (5-102, 0 for auto)", 0), - (".mp4", "bool", "Allow .mp4", True), - (".flv", "bool", "Allow .flv", True), - (".webm", "bool", "Allow .webm", False), - (".3gp", "bool", "Allow .3gp", False), - ("3d", "bool", "Prefer 3D", False)] - __description__ = """Youtube.com Video Download Hoster""" - __author_name__ = ("spoob", "zoidberg") - __author_mail__ = ("spoob@pyload.org", "zoidberg@mujmail.cz") - - # name, width, height, quality ranking, 3D - formats = {5: (".flv", 400, 240, 1, False), - 6: (".flv", 640, 400, 4, False), - 17: (".3gp", 176, 144, 0, False), - 18: (".mp4", 480, 360, 2, False), - 22: (".mp4", 1280, 720, 8, False), - 43: (".webm", 640, 360, 3, False), - 34: (".flv", 640, 360, 4, False), - 35: (".flv", 854, 480, 6, False), - 36: (".3gp", 400, 240, 1, False), - 37: (".mp4", 1920, 1080, 9, False), - 38: (".mp4", 4096, 3072, 10, False), - 44: (".webm", 854, 480, 5, False), - 45: (".webm", 1280, 720, 7, False), - 46: (".webm", 1920, 1080, 9, False), - 82: (".mp4", 640, 360, 3, True), - 83: (".mp4", 400, 240, 1, True), - 84: (".mp4", 1280, 720, 8, True), - 85: (".mp4", 1920, 1080, 9, True), - 100: (".webm", 640, 360, 3, True), - 101: (".webm", 640, 360, 4, True), - 102: (".webm", 1280, 720, 8, True) - } - - def setup(self): - self.resumeDownload = self.multiDL = True - - def process(self, pyfile): - html = self.load(pyfile.url, decode=True) - - if '

    ' in html: - self.offline() - - if "We have been receiving a large volume of requests from your network." in html: - self.tempOffline() - - #get config - use3d = self.getConf("3d") - if use3d: - quality = {"sd":82,"hd":84,"fullhd":85,"240p":83,"360p":82,"480p":82,"720p":84,"1080p":85,"3072p":85} - else: - quality = {"sd":18,"hd":22,"fullhd":37,"240p":5,"360p":18,"480p":35,"720p":22,"1080p":37,"3072p":38} - desired_fmt = self.getConf("fmt") - if desired_fmt and desired_fmt not in self.formats: - self.logWarning("FMT %d unknown - using default." % desired_fmt) - desired_fmt = 0 - if not desired_fmt: - desired_fmt = quality.get(self.getConf("quality"), 18) - - #parse available streams - streams = re.search(r'"url_encoded_fmt_stream_map": "(.*?)",', html).group(1) - streams = [x.split('\u0026') for x in streams.split(',')] - streams = [dict((y.split('=',1)) for y in x) for x in streams] - streams = [(int(x['itag']), "%s&signature=%s" % (unquote(x['url']), x['sig'])) for x in streams] - #self.logDebug("Found links: %s" % streams) - self.logDebug("AVAILABLE STREAMS: %s" % [x[0] for x in streams]) - - #build dictionary of supported itags (3D/2D) - allowed = lambda x: self.getConfig(self.formats[x][0]) - streams = [x for x in streams if x[0] in self.formats and allowed(x[0])] - if not streams: - self.fail("No available stream meets your preferences") - fmt_dict = dict([x for x in streams if self.formats[x[0]][4] == use3d] or streams) - - self.logDebug("DESIRED STREAM: ITAG:%d (%s) %sfound, %sallowed" % - (desired_fmt, - "%s %dx%d Q:%d 3D:%s" % self.formats[desired_fmt], - "" if desired_fmt in fmt_dict else "NOT ", - "" if allowed(desired_fmt) else "NOT ") - ) - - #return fmt nearest to quality index - if desired_fmt in fmt_dict and allowed(desired_fmt): - fmt = desired_fmt - else: - sel = lambda x: self.formats[x][3] #select quality index - comp = lambda x, y: abs(sel(x) - sel(y)) - - self.logDebug("Choosing nearest fmt: %s" % [(x, allowed(x), comp(x, desired_fmt)) for x in fmt_dict.keys()]) - fmt = reduce(lambda x, y: x if comp(x, desired_fmt) <= comp(y, desired_fmt) and - sel(x) > sel(y) else y, fmt_dict.keys()) - - self.logDebug("Chosen fmt: %s" % fmt) - url = fmt_dict[fmt] - self.logDebug("URL: %s" % url) - - #set file name - file_suffix = self.formats[fmt][0] if fmt in self.formats else ".flv" - file_name_pattern = '' - name = re.search(file_name_pattern, html).group(1).replace("/", "") - pyfile.name = html_unescape(name) - - time = re.search(r"t=((\d+)m)?(\d+)s", pyfile.url) - ffmpeg = which("ffmpeg") - if ffmpeg and time: - m, s = time.groups()[1:] - if not m: - m = "0" - - pyfile.name += " (starting at %s:%s)" % (m, s) - pyfile.name += file_suffix - - filename = self.download(url) - - if ffmpeg and time: - inputfile = filename + "_" - os.rename(filename, inputfile) - - subprocess.call([ - ffmpeg, - "-ss", "00:%s:%s" % (m, s), - "-i", inputfile, - "-vcodec", "copy", - "-acodec", "copy", - filename]) - os.remove(inputfile) diff --git a/module/plugins/hoster/ZDF.py b/module/plugins/hoster/ZDF.py deleted file mode 100644 index ea45f4fd8..000000000 --- a/module/plugins/hoster/ZDF.py +++ /dev/null @@ -1,46 +0,0 @@ - -import re -from xml.etree.ElementTree import fromstring - -from module.plugins.Hoster import Hoster - -XML_API = "http://www.zdf.de/ZDFmediathek/xmlservice/web/beitragsDetails?id=%i" - -class ZDF(Hoster): - # Based on zdfm by Roland Beermann - # http://github.com/enkore/zdfm/ - __name__ = "ZDF Mediathek" - __version__ = "0.7" - __pattern__ = r"http://www\.zdf\.de/ZDFmediathek/[^0-9]*([0-9]+)[^0-9]*" - __config__ = [] - - @staticmethod - def video_key(video): - return ( - int(video.findtext("videoBitrate", "0")), - any(f.text == "progressive" for f in video.iter("facet")), - ) - - @staticmethod - def video_valid(video): - return (video.findtext("url").startswith("http") and video.findtext("url").endswith(".mp4")) - - @staticmethod - def get_id(url): - return int(re.search(r"[^0-9]*([0-9]+)[^0-9]*", url).group(1)) - - def process(self, pyfile): - xml = fromstring(self.load(XML_API % self.get_id(pyfile.url))) - - status = xml.findtext("./status/statuscode") - if status != "ok": - self.fail("Error retrieving manifest.") - - video = xml.find("video") - title = video.findtext("information/title") - - pyfile.name = title - - target_url = sorted((v for v in video.iter("formitaet") if self.video_valid(v)), key=self.video_key)[-1].findtext("url") - - self.download(target_url) diff --git a/module/plugins/hoster/ZeveraCom.py b/module/plugins/hoster/ZeveraCom.py deleted file mode 100644 index 8be725d2f..000000000 --- a/module/plugins/hoster/ZeveraCom.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.plugins.Hoster import Hoster -from module.utils import html_unescape -from urllib import quote, unquote -from time import sleep - -class ZeveraCom(Hoster): - __name__ = "ZeveraCom" - __version__ = "0.20" - __type__ = "hoster" - __pattern__ = r"http://zevera.com/.*" - __description__ = """zevera.com hoster plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - def setup(self): - self.resumeDownload = self.multiDL = True - self.chunkLimit = 1 - - def process(self, pyfile): - if not self.account: - self.logError(_("Please enter your zevera.com account or deactivate this plugin")) - self.fail("No zevera.com account provided") - - self.logDebug("zevera.com: Old URL: %s" % pyfile.url) - - if self.account.getAPIData(self.req, cmd = "checklink", olink = pyfile.url) != "Alive": - self.fail("Offline or not downloadable - contact Zevera support") - - header = self.account.getAPIData(self.req, just_header = True, cmd="generatedownloaddirect", olink = pyfile.url) - if not "location" in header: - self.fail("Unable to initialize download - contact Zevera support") - - self.download(header['location'], disposition = True) - - check = self.checkDownload({"error" : 'action="ErrorDownload.aspx'}) - if check == "error": - self.fail("Error response received - contact Zevera support") - - """ - # BitAPI not used - defunct, probably abandoned by Zevera - - api_url = "http://zevera.com/API.ashx" - - def process(self, pyfile): - if not self.account: - self.logError(_("Please enter your zevera.com account or deactivate this plugin")) - self.fail("No zevera.com account provided") - - self.logDebug("zevera.com: Old URL: %s" % pyfile.url) - - last_size = retries = 0 - olink = self.pyfile.url #quote(self.pyfile.url.encode('utf_8')) - - for i in range(100): - self.retData = self.account.loadAPIRequest(self.req, cmd = 'download_request', olink = olink) - self.checkAPIErrors(self.retData) - - if self.retData['FileInfo']['StatusID'] == 100: - break - elif self.retData['FileInfo']['StatusID'] == 99: - self.fail('Failed to initialize download (99)') - else: - if self.retData['FileInfo']['Progress']['BytesReceived'] <= last_size: - if retries >= 6: - self.fail('Failed to initialize download (%d)' % self.retData['FileInfo']['StatusID'] ) - retries += 1 - else: - retries = 0 - - last_size = self.retData['FileInfo']['Progress']['BytesReceived'] - - self.setWait(self.retData['Update_Wait']) - self.wait() - - pyfile.name = self.retData['FileInfo']['RealFileName'] - pyfile.size = self.retData['FileInfo']['FileSizeInBytes'] - - self.retData = self.account.loadAPIRequest(self.req, cmd = 'download_start', FileID = self.retData['FileInfo']['FileID']) - self.checkAPIErrors(self.retData) - - self.download(self.api_url, get = { - 'cmd': "open_stream", - 'login': self.account.loginname, - 'pass': self.account.password, - 'FileID': self.retData['FileInfo']['FileID'], - 'startBytes': 0 - } - ) - - def checkAPIErrors(self, retData): - if not retData: - self.fail('Unknown API response') - - if retData['ErrorCode']: - self.logError(retData['ErrorCode'], retData['ErrorMessage']) - #self.fail('ERROR: ' + retData['ErrorMessage']) - - if self.pyfile.size / 1024000 > retData['AccountInfo']['AvailableTODAYTrafficForUseInMBytes']: - self.logWarning("Not enough data left to download the file") - - def crazyDecode(self, ustring): - # accepts decoded ie. unicode string - API response is double-quoted, double-utf8-encoded - # no idea what the proper order of calling these functions would be :-/ - return html_unescape(unquote(unquote(ustring.replace('@DELIMITER@','#'))).encode('raw_unicode_escape').decode('utf-8')) - """ \ No newline at end of file diff --git a/module/plugins/hoster/ZippyshareCom.py b/module/plugins/hoster/ZippyshareCom.py deleted file mode 100644 index 84974e7ba..000000000 --- a/module/plugins/hoster/ZippyshareCom.py +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re, subprocess, tempfile, os -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, timestamp -from module.plugins.ReCaptcha import ReCaptcha -from module.common.json_layer import json_loads - -class ZippyshareCom(SimpleHoster): - __name__ = "ZippyshareCom" - __type__ = "hoster" - __pattern__ = r"(?Phttp://www\d{0,2}\.zippyshare.com)/v(?:/|iew.jsp.*key=)(?P\d+)" - __version__ = "0.37" - __description__ = """Zippyshare.com Download Hoster""" - __author_name__ = ("spoob", "zoidberg") - __author_mail__ = ("spoob@pyload.org", "zoidberg@mujmail.cz") - __config__ = [("swfdump_path", "string", "Path to swfdump", "")] - - FILE_NAME_PATTERN = r'>Name:\s*]*>(?P[^<]+)
    ' - FILE_SIZE_PATTERN = r'>Size:\s*]*>(?P[0-9.,]+) (?P[kKMG]+)i?B
    ' - FILE_INFO_PATTERN = r'document\.getElementById\(\'dlbutton\'\)\.href = "[^;]*/(?P[^"]+)";' - FILE_OFFLINE_PATTERN = r'>File does not exist on this server

    ' - - DOWNLOAD_URL_PATTERN = r" - - - - - - -
    -
    -
    - -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - © 2008-2013
    - The pyLoad Team
    -
    -
    -
    -

    Powered by

    -
    - Bootstrap
    -
    - -
    -

    pyLoad

    -
    - dsfdsf
    -
    - -
    -

    Community

    -
    - asd
    -
    - -
    -

    Development

    -
    - asd
    -
    -
    -
    -
    - - - diff --git a/module/web/app/scripts/app.js b/module/web/app/scripts/app.js deleted file mode 100644 index 427cb1bc8..000000000 --- a/module/web/app/scripts/app.js +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Global Application Object - * Contains all necessary logic shared across views - */ -define([ - - // Libraries. - 'jquery', - 'underscore', - 'backbone', - 'utils/initHB', - 'utils/animations', - 'utils/lazyRequire', - 'utils/dialogs', - 'marionette', - 'bootstrap', - 'animate' - -], function($, _, Backbone, Handlebars) { - 'use strict'; - - Backbone.Marionette.TemplateCache.prototype.compileTemplate = function(rawTemplate) { - return Handlebars.compile(rawTemplate); - }; - - // TODO: configurable root - var App = new Backbone.Marionette.Application({ - root: '/' - }); - - App.addRegions({ - header: '#header', - notification: '#notification-area', - selection: '#selection-area', - content: '#content', - actionbar: '#actionbar' - }); - - App.navigate = function(url) { - return Backbone.history.navigate(url, true); - }; - - App.apiUrl = function(path) { - var url = window.hostProtocol + window.hostAddress + ':' + window.hostPort + window.pathPrefix + path; - return url; - }; - - // Add Global Helper functions - // Generates options dict that can be used for xhr requests - App.apiRequest = function(method, data, options) { - options || (options = {}); - options.url = App.apiUrl('api/' + method); - options.dataType = 'json'; - - if (data) { - options.type = 'POST'; - options.data = {}; - // Convert arguments to json - _.keys(data).map(function(key) { - options.data[key] = JSON.stringify(data[key]); - }); - } - - return options; - }; - - App.setTitle = function(name) { - var title = window.document.title; - var newTitle; - // page name separator - var index = title.indexOf('-'); - if (index >= 0) - newTitle = name + ' - ' + title.substr(index + 2, title.length); - else - newTitle = name + ' - ' + title; - - window.document.title = newTitle; - }; - - App.openWebSocket = function(path) { - // TODO - return new WebSocket(window.wsAddress.replace('%s', window.hostAddress) + path); - }; - - App.on('initialize:after', function() { -// TODO pushState variable - Backbone.history.start({ - pushState: false, - root: App.root - }); - - // All links should be handled by backbone - $(document).on('click', 'a[data-nav]', function(evt) { - var href = { prop: $(this).prop('href'), attr: $(this).attr('href') }; - var root = location.protocol + '//' + location.host + App.root; - if (href.prop.slice(0, root.length) === root) { - evt.preventDefault(); - Backbone.history.navigate(href.attr, true); - } - }); - }); - - // Returns the app object to be available to other modules through require.js. - return App; -}); \ No newline at end of file diff --git a/module/web/app/scripts/collections/AccountList.js b/module/web/app/scripts/collections/AccountList.js deleted file mode 100644 index bfc2af5a3..000000000 --- a/module/web/app/scripts/collections/AccountList.js +++ /dev/null @@ -1,24 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'models/Account'], function($, Backbone, _, App, Account) { - 'use strict'; - - return Backbone.Collection.extend({ - - model: Account, - - comparator: function(account) { - return account.get('plugin'); - }, - - initialize: function() { - - }, - - fetch: function(options) { - // TODO: refresh options? - options = App.apiRequest('getAccounts/false', null, options); - return Backbone.Collection.prototype.fetch.call(this, options); - } - - }); - -}); \ No newline at end of file diff --git a/module/web/app/scripts/collections/FileList.js b/module/web/app/scripts/collections/FileList.js deleted file mode 100644 index 873f4c0e3..000000000 --- a/module/web/app/scripts/collections/FileList.js +++ /dev/null @@ -1,18 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'models/File'], function($, Backbone, _, File) { - 'use strict'; - - return Backbone.Collection.extend({ - - model: File, - - comparator: function(file) { - return file.get('fileorder'); - }, - - initialize: function() { - - } - - }); - -}); \ No newline at end of file diff --git a/module/web/app/scripts/collections/InteractionList.js b/module/web/app/scripts/collections/InteractionList.js deleted file mode 100644 index 24f8b9248..000000000 --- a/module/web/app/scripts/collections/InteractionList.js +++ /dev/null @@ -1,49 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'models/InteractionTask'], - function($, Backbone, _, App, InteractionTask) { - 'use strict'; - - return Backbone.Collection.extend({ - - model: InteractionTask, - - comparator: function(task) { - return task.get('iid'); - }, - - fetch: function(options) { - options = App.apiRequest('getInteractionTasks/0', null, options); - var self = this; - options.success = function(data) { - self.set(data); - }; - - return $.ajax(options); - }, - - toJSON: function() { - var data = {queries: 0, notifications: 0}; - - this.map(function(task) { - if (task.isNotification()) - data.notifications++; - else - data.queries++; - }); - - return data; - }, - - // a task is waiting for attention (no notification) - hasTaskWaiting: function() { - var tasks = 0; - this.map(function(task) { - if (!task.isNotification()) - tasks++; - }); - - return tasks > 0; - } - - }); - - }); \ No newline at end of file diff --git a/module/web/app/scripts/collections/PackageList.js b/module/web/app/scripts/collections/PackageList.js deleted file mode 100644 index 7bee861a4..000000000 --- a/module/web/app/scripts/collections/PackageList.js +++ /dev/null @@ -1,16 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'models/Package'], function($, Backbone, _, Package) { - 'use strict'; - - return Backbone.Collection.extend({ - - model: Package, - - comparator: function(pack) { - return pack.get('packageorder'); - }, - - initialize: function() { - } - - }); -}); \ No newline at end of file diff --git a/module/web/app/scripts/collections/ProgressList.js b/module/web/app/scripts/collections/ProgressList.js deleted file mode 100644 index 51849d8de..000000000 --- a/module/web/app/scripts/collections/ProgressList.js +++ /dev/null @@ -1,18 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'models/Progress'], function($, Backbone, _, Progress) { - 'use strict'; - - return Backbone.Collection.extend({ - - model: Progress, - - comparator: function(progress) { - return progress.get('eta'); - }, - - initialize: function() { - - } - - }); - -}); \ No newline at end of file diff --git a/module/web/app/scripts/config.js b/module/web/app/scripts/config.js deleted file mode 100644 index 398d97e11..000000000 --- a/module/web/app/scripts/config.js +++ /dev/null @@ -1,73 +0,0 @@ -// Sets the require.js configuration for your application. -'use strict'; -require.config({ - - deps: ['default'], - - paths: { - - jquery: '../components/jquery/jquery', - flot: '../components/flot/jquery.flot', - transit: '../components/jquery.transit/jquery.transit', - animate: '../components/jquery.animate-enhanced/scripts/src/jquery.animate-enhanced', - cookie: '../components/jquery.cookie/jquery.cookie', - omniwindow: 'vendor/jquery.omniwindow', - select2: '../components/select2/select2', - bootstrap: 'vendor/bootstrap-2.3.2', - underscore: '../components/underscore/underscore', - backbone: '../components/backbone/backbone', - marionette: '../components/backbone.marionette/lib/backbone.marionette', -// handlebars: '../components/handlebars.js/dist/handlebars', - handlebars: 'vendor/Handlebars-1.0rc1', - jed: '../components/jed/jed', - - // TODO: Two hbs dependencies could be replaced - i18nprecompile: '../components/require-handlebars-plugin/hbs/i18nprecompile', - json2: '../components/require-handlebars-plugin/hbs/json2', - - // Plugins - text: '../components/requirejs-text/text', - hbs: '../components/require-handlebars-plugin/hbs', - - // Shortcut - tpl: '../templates/default' - }, - - hbs: { - disableI18n: true, - helperPathCallback: // Callback to determine the path to look for helpers - function(name) { - // Some helpers are accumulated into one file - if (name.indexOf('file') === 0) - name = 'fileHelper'; - - return 'helpers/' + name; - }, - templateExtension: 'html' - }, - - // Sets the configuration for your third party scripts that are not AMD compatible - shim: { - underscore: { - exports: '_' - }, - - backbone: { - deps: ['underscore', 'jquery'], - exports: 'Backbone' - }, - - marionette: ['backbone'], -// handlebars: { -// exports: 'Handlebars' -// }, - - flot: ['jquery'], - transit: ['jquery'], - cookie: ['jquery'], - omniwindow: ['jquery'], - select2: ['jquery'], - bootstrap: ['jquery'], - animate: ['jquery'] - } -}); \ No newline at end of file diff --git a/module/web/app/scripts/controller.js b/module/web/app/scripts/controller.js deleted file mode 100644 index 05237914d..000000000 --- a/module/web/app/scripts/controller.js +++ /dev/null @@ -1,67 +0,0 @@ -define([ - 'app', - 'backbone', - - // Views - 'views/headerView', - 'views/notificationView', - 'views/dashboard/dashboardView', - 'views/dashboard/selectionView', - 'views/dashboard/filterView', - 'views/loginView', - 'views/settings/settingsView', - 'views/accounts/accountListView' -], function( - App, Backbone, HeaderView, NotificationView, DashboardView, SelectionView, FilterView, LoginView, SettingsView, AccountListView) { - 'use strict'; - // TODO some views does not need to be loaded instantly - - return { - - header: function() { - if (!App.header.currentView) { - App.header.show(new HeaderView()); - App.header.currentView.init(); - App.notification.attachView(new NotificationView()); - } - }, - - dashboard: function() { - this.header(); - - App.actionbar.show(new FilterView()); - // TODO: not completly visible after reattaching - App.selection.attachView(new SelectionView()); - App.content.show(new DashboardView()); - }, - - login: function() { - App.content.show(new LoginView()); - }, - - logout: function() { - alert('Not implemented'); - }, - - settings: function() { - this.header(); - - var view = new SettingsView(); - App.actionbar.show(new view.actionbar()); - App.content.show(view); - }, - - accounts: function() { - this.header(); - - var view = new AccountListView(); - App.actionbar.show(new view.actionbar()); - App.content.show(view); - }, - - admin: function() { - alert('Not implemented'); - } - }; - -}); diff --git a/module/web/app/scripts/default.js b/module/web/app/scripts/default.js deleted file mode 100644 index a337cee21..000000000 --- a/module/web/app/scripts/default.js +++ /dev/null @@ -1,30 +0,0 @@ -define('default', ['backbone', 'jquery', 'app', 'router', 'models/userSession'], - function(Backbone, $, App, Router, UserSession) { - 'use strict'; - - // Global ajax options - var options = { - statusCode: { - 401: function() { - console.log('Not logged in.'); - App.navigate('login'); - } - }, - xhrFields: {withCredentials: true} - }; - - $.ajaxSetup(options); - - Backbone.ajax = function() { - Backbone.$.ajaxSetup.call(Backbone.$, options); - return Backbone.$.ajax.apply(Backbone.$, arguments); - }; - - $(function() { - App.session = new UserSession(); - App.router = new Router(); - App.start(); - }); - - return App; - }); \ No newline at end of file diff --git a/module/web/app/scripts/helpers/fileHelper.js b/module/web/app/scripts/helpers/fileHelper.js deleted file mode 100644 index 156be58f0..000000000 --- a/module/web/app/scripts/helpers/fileHelper.js +++ /dev/null @@ -1,55 +0,0 @@ -// Helpers to render the file view -define('helpers/fileHelper', ['handlebars', 'utils/apitypes', 'helpers/formatTime'], - function(Handlebars, Api, formatTime) { - 'use strict'; - - function fileClass(file, options) { - if (file.finished) - return 'finished'; - else if (file.failed) - return 'failed'; - else if (file.offline) - return 'offline'; - else if (file.online) - return 'online'; - else if (file.waiting) - return 'waiting'; - else if (file.downloading) - return 'downloading'; - - return ''; - } - - // TODO - function fileIcon(media, options) { - return 'icon-music'; - } - - // TODO rest of the states - function fileStatus(file, options) { - var s; - var msg = file.download.statusmsg; - - if (file.failed) { - s = ' '; - if (file.download.error) - s += file.download.error; - else s += msg; - } else if (file.finished) - s = ' ' + msg; - else if (file.downloading) - s = '
      ' + - formatTime(file.eta) + '
    '; - else if (file.waiting) - s = ' ' + formatTime(file.eta); - else - s = msg; - - return new Handlebars.SafeString(s); - } - - Handlebars.registerHelper('fileClass', fileClass); - Handlebars.registerHelper('fileIcon', fileIcon); - Handlebars.registerHelper('fileStatus', fileStatus); - return fileClass; - }); \ No newline at end of file diff --git a/module/web/app/scripts/helpers/formatSize.js b/module/web/app/scripts/helpers/formatSize.js deleted file mode 100644 index 3b62e74c7..000000000 --- a/module/web/app/scripts/helpers/formatSize.js +++ /dev/null @@ -1,15 +0,0 @@ -// Format bytes in human readable format -define('helpers/formatSize', ['handlebars'], function(Handlebars) { - 'use strict'; - - var sizes = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB']; - function formatSize(bytes, options) { - if (!bytes || bytes === 0) return '0 B'; - var i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)), 10); - // round to two digits - return (bytes / Math.pow(1024, i)).toFixed(2) + ' ' + sizes[i]; - } - - Handlebars.registerHelper('formatSize', formatSize); - return formatSize; -}); \ No newline at end of file diff --git a/module/web/app/scripts/helpers/formatTime.js b/module/web/app/scripts/helpers/formatTime.js deleted file mode 100644 index 757ff73ad..000000000 --- a/module/web/app/scripts/helpers/formatTime.js +++ /dev/null @@ -1,17 +0,0 @@ -// Format bytes in human readable format -define('helpers/formatTime', ['handlebars', 'vendor/remaining'], function(Handlebars, Remaining) { - 'use strict'; - - function formatTime(seconds, options) { - if (seconds === Infinity) - return '∞'; - else if (!seconds || seconds <= 0) - return '-'; - - // TODO: digital or written string - return Remaining.getStringDigital(seconds, window.dates); - } - - Handlebars.registerHelper('formatTime', formatTime); - return formatTime; -}); \ No newline at end of file diff --git a/module/web/app/scripts/helpers/pluginIcon.js b/module/web/app/scripts/helpers/pluginIcon.js deleted file mode 100644 index 6b2fdc67f..000000000 --- a/module/web/app/scripts/helpers/pluginIcon.js +++ /dev/null @@ -1,14 +0,0 @@ -// Resolves name of plugin to icon path -define('helpers/pluginIcon', ['handlebars', 'app'], function(Handlebars, App) { - 'use strict'; - - function pluginIcon(name) { - if (typeof name === 'object' && typeof name.get === 'function') - name = name.get('plugin'); - - return App.apiUrl('icons/' + name); - } - - Handlebars.registerHelper('pluginIcon', pluginIcon); - return pluginIcon; -}); \ No newline at end of file diff --git a/module/web/app/scripts/models/Account.js b/module/web/app/scripts/models/Account.js deleted file mode 100644 index a2e24b056..000000000 --- a/module/web/app/scripts/models/Account.js +++ /dev/null @@ -1,51 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes'], function($, Backbone, _, App, Api) { - 'use strict'; - - return Backbone.Model.extend({ - - // TODO - // generated, not submitted - idAttribute: 'user', - - defaults: { - plugin: null, - loginname: null, - owner: -1, - valid: false, - validuntil: -1, - trafficleft: -1, - maxtraffic: -1, - premium: false, - activated: false, - shared: false, - options: null - }, - - // Model Constructor - initialize: function() { - }, - - // Any time a model attribute is set, this method is called - validate: function(attrs) { - - }, - - save: function(options) { - options = App.apiRequest('updateAccountInfo', {account: this.toJSON()}, options); - return $.ajax(options); - }, - - destroy: function(options) { - options = App.apiRequest('removeAccount', {account: this.toJSON()}, options); - var self = this; - options.success = function() { - self.trigger('destroy', self, self.collection, options); - }; - - // TODO request is not dispatched -// return Backbone.Model.prototype.destroy.call(this, options); - return $.ajax(options); - } - }); - -}); \ No newline at end of file diff --git a/module/web/app/scripts/models/ConfigHolder.js b/module/web/app/scripts/models/ConfigHolder.js deleted file mode 100644 index 40efbc7c0..000000000 --- a/module/web/app/scripts/models/ConfigHolder.js +++ /dev/null @@ -1,68 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', './ConfigItem'], - function($, Backbone, _, App, ConfigItem) { - 'use strict'; - - return Backbone.Model.extend({ - - defaults: { - name: '', - label: '', - description: '', - long_description: null, - // simple list but no collection - items: null, - info: null - }, - - // Model Constructor - initialize: function() { - - }, - - // Loads it from server by name - fetch: function(options) { - options = App.apiRequest('loadConfig/"' + this.get('name') + '"', null, options); - return Backbone.Model.prototype.fetch.call(this, options); - }, - - save: function(options) { - var config = this.toJSON(); - var items = []; - // Convert changed items to json - _.each(config.items, function(item) { - if (item.isChanged()) { - items.push(item.prepareSave()); - } - }); - config.items = items; - // TODO: only set new values on success - - options = App.apiRequest('saveConfig', {config: config}, options); - - return $.ajax(options); - }, - - parse: function(resp) { - // Create item models - resp.items = _.map(resp.items, function(item) { - return new ConfigItem(item); - }); - - return Backbone.Model.prototype.parse.call(this, resp); - }, - - isLoaded: function() { - return this.has('items') || this.has('long_description'); - }, - - // check if any of the items has changes - hasChanges: function() { - var items = this.get('items'); - if (!items) return false; - return _.reduce(items, function(a, b) { - return a || b.isChanged(); - }, false); - } - - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/models/ConfigItem.js b/module/web/app/scripts/models/ConfigItem.js deleted file mode 100644 index 2d325c2a2..000000000 --- a/module/web/app/scripts/models/ConfigItem.js +++ /dev/null @@ -1,40 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes'], - function($, Backbone, _, App, Api) { - 'use strict'; - - return Backbone.Model.extend({ - - defaults: { - name: '', - label: '', - description: '', - input: null, - default_value: null, - value: null, - // additional attributes - inputView: null - }, - - // Model Constructor - initialize: function() { - - }, - - isChanged: function() { - return this.get('inputView') && this.get('inputView').getVal() !== this.get('value'); - }, - - // set new value and return json - prepareSave: function() { - // set the new value - if (this.get('inputView')) - this.set('value', this.get('inputView').getVal()); - - var data = this.toJSON(); - delete data.inputView; - delete data.description; - - return data; - } - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/models/File.js b/module/web/app/scripts/models/File.js deleted file mode 100644 index 3beb7f270..000000000 --- a/module/web/app/scripts/models/File.js +++ /dev/null @@ -1,92 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes'], function($, Backbone, _, App, Api) { - 'use strict'; - - var Finished = [Api.DownloadStatus.Finished, Api.DownloadStatus.Skipped]; - var Failed = [Api.DownloadStatus.Failed, Api.DownloadStatus.Aborted, Api.DownloadStatus.TempOffline, Api.DownloadStatus.Offline]; - // Unfinished - Other - - return Backbone.Model.extend({ - - idAttribute: 'fid', - - defaults: { - fid: -1, - name: null, - package: -1, - owner: -1, - size: -1, - status: -1, - media: -1, - added: -1, - fileorder: -1, - download: null, - - // UI attributes - selected: false, - visible: true, - progress: 0, - eta: 0 - }, - - // Model Constructor - initialize: function() { - - }, - - fetch: function(options) { - options = App.apiRequest( - 'getFileInfo', - {fid: this.get('fid')}, - options); - - return Backbone.Model.prototype.fetch.call(this, options); - }, - - destroy: function(options) { - // also not working when using data - options = App.apiRequest( - 'deleteFiles/[' + this.get('fid') + ']', - null, options); - options.method = 'post'; - - return Backbone.Model.prototype.destroy.call(this, options); - }, - - // Does not send a request to the server - destroyLocal: function(options) { - this.trigger('destroy', this, this.collection, options); - }, - - restart: function(options) { - options = App.apiRequest( - 'restartFile', - {fid: this.get('fid')}, - options); - - return $.ajax(options); - }, - - // Any time a model attribute is set, this method is called - validate: function(attrs) { - - }, - - isDownload: function() { - return this.has('download'); - }, - - isFinished: function() { - return _.indexOf(Finished, this.get('download').status) > -1; - }, - - isUnfinished: function() { - return _.indexOf(Finished, this.get('download').status) === -1 && _.indexOf(Failed, this.get('download').status) === -1; - }, - - isFailed: function() { - return _.indexOf(Failed, this.get('download').status) > -1; - } - - }); - -}); \ No newline at end of file diff --git a/module/web/app/scripts/models/InteractionTask.js b/module/web/app/scripts/models/InteractionTask.js deleted file mode 100644 index 54c739d4b..000000000 --- a/module/web/app/scripts/models/InteractionTask.js +++ /dev/null @@ -1,41 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes'], - function($, Backbone, _, App, Api) { - 'use strict'; - - return Backbone.Model.extend({ - - idAttribute: 'iid', - - defaults: { - iid: -1, - type: null, - input: null, - default_value: null, - title: '', - description: '', - plugin: '', - // additional attributes - result: '' - }, - - // Model Constructor - initialize: function() { - - }, - - save: function(options) { - options = App.apiRequest('setInteractionResult/' + this.get('iid'), - {result: this.get('result')}, options); - - return $.ajax(options); - }, - - isNotification: function() { - return this.get('type') === Api.Interaction.Notification; - }, - - isCaptcha: function() { - return this.get('type') === Api.Interaction.Captcha; - } - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/models/Package.js b/module/web/app/scripts/models/Package.js deleted file mode 100644 index a34ec1c69..000000000 --- a/module/web/app/scripts/models/Package.js +++ /dev/null @@ -1,119 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'collections/FileList', 'require'], - function($, Backbone, _, App, FileList, require) { - 'use strict'; - - return Backbone.Model.extend({ - - idAttribute: 'pid', - - defaults: { - pid: -1, - name: null, - folder: '', - root: -1, - owner: -1, - site: '', - comment: '', - password: '', - added: -1, - tags: null, - status: -1, - shared: false, - packageorder: -1, - stats: null, - fids: null, - pids: null, - files: null, // Collection - packs: null, // Collection - - selected: false // For Checkbox - }, - - // Model Constructor - initialize: function() { - }, - - toJSON: function(options) { - var obj = Backbone.Model.prototype.toJSON.call(this, options); - obj.percent = Math.round(obj.stats.linksdone * 100 / obj.stats.linkstotal); - - return obj; - }, - - // Changes url + method and delegates call to super class - fetch: function(options) { - options = App.apiRequest( - 'getFileTree/' + this.get('pid'), - {full: false}, - options); - - return Backbone.Model.prototype.fetch.call(this, options); - }, - - // Create a pseudo package und use search to populate data - search: function(qry, options) { - options = App.apiRequest( - 'findFiles', - {pattern: qry}, - options); - - return Backbone.Model.prototype.fetch.call(this, options); - }, - - save: function(options) { - // TODO - }, - - destroy: function(options) { - // TODO: Not working when using data?, array seems to break it - options = App.apiRequest( - 'deletePackages/[' + this.get('pid') + ']', - null, options); - options.method = 'post'; - - console.log(options); - - return Backbone.Model.prototype.destroy.call(this, options); - }, - - restart: function(options) { - options = App.apiRequest( - 'restartPackage', - {pid: this.get('pid')}, - options); - - var self = this; - options.success = function() { - self.fetch(); - }; - return $.ajax(options); - }, - - parse: function(resp) { - // Package is loaded from tree collection - if (_.has(resp, 'root')) { - if (!this.has('files')) - resp.root.files = new FileList(_.values(resp.files)); - else - this.get('files').set(_.values(resp.files)); - - // circular dependencies needs to be avoided - var PackageList = require('collections/PackageList'); - - if (!this.has('packs')) - resp.root.packs = new PackageList(_.values(resp.packages)); - else - this.get('packs').set(_.values(resp.packages)); - - return resp.root; - } - return Backbone.model.prototype.parse.call(this, resp); - }, - - // Any time a model attribute is set, this method is called - validate: function(attrs) { - - } - - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/models/Progress.js b/module/web/app/scripts/models/Progress.js deleted file mode 100644 index b0bbb684d..000000000 --- a/module/web/app/scripts/models/Progress.js +++ /dev/null @@ -1,50 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'utils/apitypes'], function($, Backbone, _, Api) { - 'use strict'; - - return Backbone.Model.extend({ - - // generated, not submitted - idAttribute: 'pid', - - defaults: { - pid: -1, - plugin: null, - name: null, - statusmsg: -1, - eta: -1, - done: -1, - total: -1, - download: null - }, - - getPercent: function() { - if (this.get('total') > 0) - return Math.round(this.get('done') * 100 / this.get('total')); - return 0; - }, - - // Model Constructor - initialize: function() { - - }, - - // Any time a model attribute is set, this method is called - validate: function(attrs) { - - }, - - toJSON: function(options) { - var obj = Backbone.Model.prototype.toJSON.call(this, options); - obj.percent = this.getPercent(); - obj.downloading = this.isDownload() && this.get('download').status === Api.DownloadStatus.Downloading; - - return obj; - }, - - isDownload : function() { - return this.has('download'); - } - - }); - -}); \ No newline at end of file diff --git a/module/web/app/scripts/models/ServerStatus.js b/module/web/app/scripts/models/ServerStatus.js deleted file mode 100644 index 59739b41e..000000000 --- a/module/web/app/scripts/models/ServerStatus.js +++ /dev/null @@ -1,47 +0,0 @@ -define(['jquery', 'backbone', 'underscore'], - function($, Backbone, _) { - 'use strict'; - - return Backbone.Model.extend({ - - defaults: { - speed: 0, - linkstotal: 0, - linksqueue: 0, - sizetotal: 0, - sizequeue: 0, - notifications: -1, - paused: false, - download: false, - reconnect: false - }, - - // Model Constructor - initialize: function() { - - }, - - fetch: function(options) { - options || (options = {}); - options.url = 'api/getServerStatus'; - - return Backbone.Model.prototype.fetch.call(this, options); - }, - - toJSON: function(options) { - var obj = Backbone.Model.prototype.toJSON.call(this, options); - - obj.linksdone = obj.linkstotal - obj.linksqueue; - obj.sizedone = obj.sizetotal - obj.sizequeue; - if (obj.speed && obj.speed > 0) - obj.eta = Math.round(obj.sizequeue / obj.speed); - else if (obj.sizequeue > 0) - obj.eta = Infinity; - else - obj.eta = 0; - - return obj; - } - - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/models/TreeCollection.js b/module/web/app/scripts/models/TreeCollection.js deleted file mode 100644 index 2f761e6cc..000000000 --- a/module/web/app/scripts/models/TreeCollection.js +++ /dev/null @@ -1,50 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'models/Package', 'collections/FileList', 'collections/PackageList'], - function($, Backbone, _, App, Package, FileList, PackageList) { - 'use strict'; - - // TreeCollection - // A Model and not a collection, aggregates other collections - return Backbone.Model.extend({ - - defaults: { - root: null, - packages: null, - files: null - }, - - initialize: function() { - - }, - - fetch: function(options) { - options || (options = {}); - var pid = options.pid || -1; - - options = App.apiRequest( - 'getFileTree/' + pid, - {full: false}, - options); - - console.log('Fetching package tree ' + pid); - return Backbone.Model.prototype.fetch.call(this, options); - }, - - // Parse the response and updates the collections - parse: function(resp) { - var ret = {}; - if (!this.has('packages')) - ret.packages = new PackageList(_.values(resp.packages)); - else - this.get('packages').set(_.values(resp.packages)); - - if (!this.has('files')) - ret.files = new FileList(_.values(resp.files)); - else - this.get('files').set(_.values(resp.files)); - - ret.root = new Package(resp.root); - return ret; - } - - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/models/UserSession.js b/module/web/app/scripts/models/UserSession.js deleted file mode 100644 index a7e9aa848..000000000 --- a/module/web/app/scripts/models/UserSession.js +++ /dev/null @@ -1,20 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'utils/apitypes', 'cookie'], - function($, Backbone, _, Api) { - 'use strict'; - - return Backbone.Model.extend({ - - idAttribute: 'username', - - defaults: { - username: null, - permissions: null, - session: null - }, - - // Model Constructor - initialize: function() { - this.set('session', $.cookie('beaker.session.id')); - } - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/router.js b/module/web/app/scripts/router.js deleted file mode 100644 index 68ea5575d..000000000 --- a/module/web/app/scripts/router.js +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Router defines routes that are handled by registered controller - */ -define([ - // Libraries - 'backbone', - 'marionette', - - // Modules - 'controller' -], - function(Backbone, Marionette, Controller) { - 'use strict'; - - return Backbone.Marionette.AppRouter.extend({ - - appRoutes: { - '': 'dashboard', - 'login': 'login', - 'logout': 'logout', - 'settings': 'settings', - 'accounts': 'accounts', - 'admin': 'admin' - }, - - // Our controller to handle the routes - controller: Controller - }); - }); diff --git a/module/web/app/scripts/routers/defaultRouter.js b/module/web/app/scripts/routers/defaultRouter.js deleted file mode 100644 index 4b00d160c..000000000 --- a/module/web/app/scripts/routers/defaultRouter.js +++ /dev/null @@ -1,30 +0,0 @@ -define(['jquery', 'backbone', 'views/headerView'], function($, Backbone, HeaderView) { - 'use strict'; - - var Router = Backbone.Router.extend({ - - initialize: function() { - Backbone.history.start(); - }, - - // All of your Backbone Routes (add more) - routes: { - - // When there is no hash bang on the url, the home method is called - '': 'home' - - }, - - 'home': function() { - // Instantiating mainView and anotherView instances - var headerView = new HeaderView(); - - // Renders the mainView template - headerView.render(); - - } - }); - - // Returns the Router class - return Router; -}); \ No newline at end of file diff --git a/module/web/app/scripts/routers/mobileRouter.js b/module/web/app/scripts/routers/mobileRouter.js deleted file mode 100644 index e24cb7a34..000000000 --- a/module/web/app/scripts/routers/mobileRouter.js +++ /dev/null @@ -1,56 +0,0 @@ -define(['jquery', 'backbone', 'underscore'], function($, Backbone, _) { - 'use strict'; - - return Backbone.Router.extend({ - - initialize: function() { - _.bindAll(this, 'changePage'); - - this.$el = $('#content'); - - // Tells Backbone to start watching for hashchange events - Backbone.history.start(); - - }, - - // All of your Backbone Routes (add more) - routes: { - - // When there is no hash bang on the url, the home method is called - '': 'home' - - }, - - 'home': function() { - - var self = this; - - $('#p1').fastClick(function() { - self.changePage($('

    Page 1


    some content
    sdfdsf
    sdffg

    oiuzz

    ')); - }); - - $('#p2').bind('click', function() { - self.changePage($('

    Page 2


    some content
    sdfdsf

    sdfsdf

    sdffg
    ')); - }); - - }, - - changePage: function(content) { - - var oldpage = this.$el.find('.page'); - content.css({x: '100%'}); - this.$el.append(content); - content.transition({x: 0}, function() { - window.setTimeout(function() { - oldpage.remove(); - }, 400); - }); - -// $("#viewport").transition({x: "100%"}, function(){ -// $("#viewport").html(content); -// $("#viewport").transition({x: 0}); -// }); - } - - }); -}); \ No newline at end of file diff --git a/module/web/app/scripts/utils/animations.js b/module/web/app/scripts/utils/animations.js deleted file mode 100644 index 7f89afef1..000000000 --- a/module/web/app/scripts/utils/animations.js +++ /dev/null @@ -1,129 +0,0 @@ -define(['jquery', 'underscore', 'transit'], function(jQuery, _) { - 'use strict'; - - // Adds an element and computes its height, which is saved as data attribute - // Important function to have slide animations - jQuery.fn.appendWithHeight = function(element, hide) { - var o = jQuery(this[0]); - element = jQuery(element); - - // TODO: additionally it could be placed out of viewport first - // The real height can only be retrieved when element is on DOM and display:true - element.css('visibility', 'hidden'); - o.append(element); - - var height = element.height(); - - // Hide the element - if (hide === true) { - element.hide(); - element.height(0); - } - - element.css('visibility', ''); - element.data('height', height); - - return this; - }; - - // Shortcut to have a animation when element is added - jQuery.fn.appendWithAnimation = function(element, animation) { - var o = jQuery(this[0]); - element = jQuery(element); - - if (animation === true) - element.hide(); - - o.append(element); - - if (animation === true) - element.fadeIn(); - -// element.calculateHeight(); - - return this; - }; - - // calculate the height and write it to data, should be used on invisible elements - jQuery.fn.calculateHeight = function(setHeight) { - var o = jQuery(this[0]); - var height = o.height(); - if (!height) { - var display = o.css('display'); - o.css('visibility', 'hidden'); - o.show(); - height = o.height(); - - o.css('display', display); - o.css('visibility', ''); - } - - if (setHeight) - o.css('height', height); - - o.data('height', height); - return this; - }; - - // TODO: carry arguments, optional height argument - - // reset arguments, sets overflow hidden - jQuery.fn.slideOut = function(reset) { - var o = jQuery(this[0]); - o.animate({height: o.data('height'), opacity: 'show'}, function() { - // reset css attributes; - if (reset) { - this.css('overflow', ''); - this.css('height', ''); - } - }); - return this; - }; - - jQuery.fn.slideIn = function(reset) { - var o = jQuery(this[0]); - if (reset) { - o.css('overflow', 'hidden'); - } - o.animate({height: 0, opacity: 'hide'}); - return this; - }; - - jQuery.fn.initTooltips = function(placement) { - placement || (placement = 'top'); - - var o = jQuery(this[0]); - o.find('[data-toggle="tooltip"]').tooltip( - { - delay: {show: 800, hide: 100}, - placement: placement - }); - - return this; - }; - - jQuery.fn._transit = jQuery.fn.transit; - - // Overriding transit plugin to support hide and show - jQuery.fn.transit = jQuery.fn.transition = function(props, duration, easing, callback) { - var self = this; - var cb = callback; - var newprops = _.extend({}, props); - - if (newprops && (newprops.opacity === 'hide')) { - newprops.opacity = 0; - - callback = function() { - self.css({display: 'none'}); - if (typeof cb === 'function') { - cb.apply(self); - } - }; - } else if (newprops && (newprops.opacity === 'show')) { - newprops.opacity = 1; - this.css({display: 'block'}); - } - - return this._transit(newprops, duration, easing, callback); - }; -}); \ No newline at end of file diff --git a/module/web/app/scripts/utils/apitypes.js b/module/web/app/scripts/utils/apitypes.js deleted file mode 100644 index cbbc9064f..000000000 --- a/module/web/app/scripts/utils/apitypes.js +++ /dev/null @@ -1,16 +0,0 @@ -// Autogenerated, do not edit! -/*jslint -W070: false*/ -define([], function() { - 'use strict'; - return { - DownloadState: {'Failed': 3, 'All': 0, 'Unmanaged': 4, 'Finished': 1, 'Unfinished': 2}, - DownloadStatus: {'Downloading': 10, 'NA': 0, 'Processing': 14, 'Waiting': 9, 'Decrypting': 13, 'Paused': 4, 'Failed': 7, 'Finished': 5, 'Skipped': 6, 'Unknown': 16, 'Aborted': 12, 'Online': 2, 'TempOffline': 11, 'Offline': 1, 'Custom': 15, 'Starting': 8, 'Queued': 3}, - FileStatus: {'Remote': 2, 'Ok': 0, 'Missing': 1}, - InputType: {'Multiple': 10, 'Int': 2, 'NA': 0, 'List': 11, 'Bool': 7, 'File': 3, 'Text': 1, 'Table': 12, 'Folder': 4, 'Password': 6, 'Click': 8, 'Select': 9, 'Textbox': 5}, - Interaction: {'Captcha': 2, 'All': 0, 'Query': 4, 'Notification': 1}, - MediaType: {'All': 0, 'Audio': 2, 'Image': 4, 'Other': 1, 'Video': 8, 'Document': 16, 'Archive': 32}, - PackageStatus: {'Paused': 1, 'Remote': 3, 'Folder': 2, 'Ok': 0}, - Permission: {'All': 0, 'Interaction': 32, 'Modify': 4, 'Add': 1, 'Accounts': 16, 'Plugins': 64, 'Download': 8, 'Delete': 2}, - Role: {'Admin': 0, 'User': 1}, - }; -}); \ No newline at end of file diff --git a/module/web/app/scripts/utils/dialogs.js b/module/web/app/scripts/utils/dialogs.js deleted file mode 100644 index 4933b7ed2..000000000 --- a/module/web/app/scripts/utils/dialogs.js +++ /dev/null @@ -1,16 +0,0 @@ -// Loads all helper and set own handlebars rules -define(['jquery', 'underscore', 'views/abstract/modalView'], function($, _, Modal) { - 'use strict'; - - // Shows the confirm dialog for given context - // on success executes func with context - _.confirm = function(template, func, context) { - template = 'text!tpl/' + template; - _.requireOnce([template], function(html) { - var template = _.compile(html); - var dialog = new Modal(template, _.bind(func, context)); - dialog.show(); - }); - - }; -}); \ No newline at end of file diff --git a/module/web/app/scripts/utils/initHB.js b/module/web/app/scripts/utils/initHB.js deleted file mode 100644 index d7f582521..000000000 --- a/module/web/app/scripts/utils/initHB.js +++ /dev/null @@ -1,11 +0,0 @@ -// Loads all helper and set own handlebars rules -define(['underscore', 'handlebars', - 'helpers/formatSize', 'helpers/fileHelper', 'helpers/formatTime'], - function(_, Handlebars) { - 'use strict'; - // Replace with own lexer rules compiled from handlebars.l - Handlebars.Parser.lexer.rules = [/^(?:[^\x00]*?(?=(<%)))/, /^(?:[^\x00]+)/, /^(?:[^\x00]{2,}?(?=(\{\{|$)))/, /^(?:\{\{>)/, /^(?:<%=)/, /^(?:<%\/)/, /^(?:\{\{\^)/, /^(?:<%\s*else\b)/, /^(?:\{<%%)/, /^(?:\{\{&)/, /^(?:<%![\s\S]*?%>)/, /^(?:<%)/, /^(?:=)/, /^(?:\.(?=[%} ]))/, /^(?:\.\.)/, /^(?:[\/.])/, /^(?:\s+)/, /^(?:%%>)/, /^(?:%>)/, /^(?:"(\\["]|[^"])*")/, /^(?:'(\\[']|[^'])*')/, /^(?:@[a-zA-Z]+)/, /^(?:true(?=[%}\s]))/, /^(?:false(?=[%}\s]))/, /^(?:[0-9]+(?=[%}\s]))/, /^(?:[a-zA-Z0-9_$-]+(?=[=%}\s\/.]))/, /^(?:\[[^\]]*\])/, /^(?:.)/, /^(?:$)/]; - _.compile = Handlebars.compile; - - return Handlebars; - }); \ No newline at end of file diff --git a/module/web/app/scripts/utils/lazyRequire.js b/module/web/app/scripts/utils/lazyRequire.js deleted file mode 100644 index 96c07aa24..000000000 --- a/module/web/app/scripts/utils/lazyRequire.js +++ /dev/null @@ -1,97 +0,0 @@ -// Define the module. -define( - [ - 'require', 'underscore' - ], - function( require, _ ){ - 'use strict'; - - - // Define the states of loading for a given set of modules - // within a require() statement. - var states = { - unloaded: 'UNLOADED', - loading: 'LOADING', - loaded: 'LOADED' - }; - - - // Define the top-level module container. Mostly, we're making - // the top-level container a non-Function so that users won't - // try to invoke this without calling the once() method below. - var lazyRequire = {}; - - - // I will return a new, unique instance of the requrieOnce() - // method. Each instance will only call the require() method - // once internally. - lazyRequire.once = function(){ - - // The modules start in an unloaded state before - // requireOnce() is invoked by the calling code. - var state = states.unloaded; - var args; - - var requireOnce = function(dependencies, loadCallback ){ - - // Use the module state to determine which method to - // invoke (or just to ignore the invocation). - if (state === states.loaded){ - loadCallback.apply(null, args); - - // The modules have not yet been requested - let's - // lazy load them. - } else if (state !== states.loading){ - - // We're about to load the modules asynchronously; - // flag the interim state. - state = states.loading; - - // Load the modules. - require( - dependencies, - function(){ - - args = arguments; - loadCallback.apply( null, args ); - state = states.loaded; - - - } - ); - - // RequireJS is currently loading the modules - // asynchronously, but they have not finished - // loading yet. - } else { - - // Simply ignore this call. - return; - - } - - }; - - // Return the new lazy loader. - return( requireOnce ); - - }; - - - // -------------------------------------------------- // - // -------------------------------------------------- // - - // Set up holder for underscore - var instances = {}; - _.requireOnce = function(dependencies, loadCallback) { - if (!_.has(instances, dependencies)) - instances[dependencies] = lazyRequire.once(); - - return instances[dependencies](dependencies, loadCallback); - }; - - - // Return the module definition. - return( lazyRequire ); - } -); \ No newline at end of file diff --git a/module/web/app/scripts/vendor/Handlebars-1.0rc1.js b/module/web/app/scripts/vendor/Handlebars-1.0rc1.js deleted file mode 100644 index 991242461..000000000 --- a/module/web/app/scripts/vendor/Handlebars-1.0rc1.js +++ /dev/null @@ -1,1927 +0,0 @@ -// lib/handlebars/base.js -(function () { -/*jshint eqnull:true*/ -this.Handlebars = {}; - -(function(Handlebars) { - -Handlebars.VERSION = "1.0.rc.1"; - -Handlebars.helpers = {}; -Handlebars.partials = {}; - -Handlebars.registerHelper = function(name, fn, inverse) { - if(inverse) { fn.not = inverse; } - this.helpers[name] = fn; -}; - -Handlebars.registerPartial = function(name, str) { - this.partials[name] = str; -}; - -Handlebars.registerHelper('helperMissing', function(arg) { - if(arguments.length === 2) { - return undefined; - } else { - throw new Error("Could not find property '" + arg + "'"); - } -}); - -var toString = Object.prototype.toString, functionType = "[object Function]"; - -Handlebars.registerHelper('blockHelperMissing', function(context, options) { - var inverse = options.inverse || function() {}, fn = options.fn; - - - var ret = ""; - var type = toString.call(context); - - if(type === functionType) { context = context.call(this); } - - if(context === true) { - return fn(this); - } else if(context === false || context == null) { - return inverse(this); - } else if(type === "[object Array]") { - if(context.length > 0) { - return Handlebars.helpers.each(context, options); - } else { - return inverse(this); - } - } else { - return fn(context); - } -}); - -Handlebars.K = function() {}; - -Handlebars.createFrame = Object.create || function(object) { - Handlebars.K.prototype = object; - var obj = new Handlebars.K(); - Handlebars.K.prototype = null; - return obj; -}; - -Handlebars.registerHelper('each', function(context, options) { - var fn = options.fn, inverse = options.inverse; - var ret = "", data; - - if (options.data) { - data = Handlebars.createFrame(options.data); - } - - if(context && context.length > 0) { - for(var i=0, j=context.length; i 2) { - expected.push("'" + this.terminals_[p] + "'"); - } - if (this.lexer.showPosition) { - errStr = "Parse error on line " + (yylineno + 1) + ":\n" + this.lexer.showPosition() + "\nExpecting " + expected.join(", ") + ", got '" + (this.terminals_[symbol] || symbol) + "'"; - } else { - errStr = "Parse error on line " + (yylineno + 1) + ": Unexpected " + (symbol == 1?"end of input":"'" + (this.terminals_[symbol] || symbol) + "'"); - } - this.parseError(errStr, {text: this.lexer.match, token: this.terminals_[symbol] || symbol, line: this.lexer.yylineno, loc: yyloc, expected: expected}); - } - } - if (action[0] instanceof Array && action.length > 1) { - throw new Error("Parse Error: multiple actions possible at state: " + state + ", token: " + symbol); - } - switch (action[0]) { - case 1: - stack.push(symbol); - vstack.push(this.lexer.yytext); - lstack.push(this.lexer.yylloc); - stack.push(action[1]); - symbol = null; - if (!preErrorSymbol) { - yyleng = this.lexer.yyleng; - yytext = this.lexer.yytext; - yylineno = this.lexer.yylineno; - yyloc = this.lexer.yylloc; - if (recovering > 0) - recovering--; - } else { - symbol = preErrorSymbol; - preErrorSymbol = null; - } - break; - case 2: - len = this.productions_[action[1]][1]; - yyval.$ = vstack[vstack.length - len]; - yyval._$ = {first_line: lstack[lstack.length - (len || 1)].first_line, last_line: lstack[lstack.length - 1].last_line, first_column: lstack[lstack.length - (len || 1)].first_column, last_column: lstack[lstack.length - 1].last_column}; - if (ranges) { - yyval._$.range = [lstack[lstack.length - (len || 1)].range[0], lstack[lstack.length - 1].range[1]]; - } - r = this.performAction.call(yyval, yytext, yyleng, yylineno, this.yy, action[1], vstack, lstack); - if (typeof r !== "undefined") { - return r; - } - if (len) { - stack = stack.slice(0, -1 * len * 2); - vstack = vstack.slice(0, -1 * len); - lstack = lstack.slice(0, -1 * len); - } - stack.push(this.productions_[action[1]][0]); - vstack.push(yyval.$); - lstack.push(yyval._$); - newState = table[stack[stack.length - 2]][stack[stack.length - 1]]; - stack.push(newState); - break; - case 3: - return true; - } - } - return true; -} -}; -/* Jison generated lexer */ -var lexer = (function(){ -var lexer = ({EOF:1, -parseError:function parseError(str, hash) { - if (this.yy.parser) { - this.yy.parser.parseError(str, hash); - } else { - throw new Error(str); - } - }, -setInput:function (input) { - this._input = input; - this._more = this._less = this.done = false; - this.yylineno = this.yyleng = 0; - this.yytext = this.matched = this.match = ''; - this.conditionStack = ['INITIAL']; - this.yylloc = {first_line:1,first_column:0,last_line:1,last_column:0}; - if (this.options.ranges) this.yylloc.range = [0,0]; - this.offset = 0; - return this; - }, -input:function () { - var ch = this._input[0]; - this.yytext += ch; - this.yyleng++; - this.offset++; - this.match += ch; - this.matched += ch; - var lines = ch.match(/(?:\r\n?|\n).*/g); - if (lines) { - this.yylineno++; - this.yylloc.last_line++; - } else { - this.yylloc.last_column++; - } - if (this.options.ranges) this.yylloc.range[1]++; - - this._input = this._input.slice(1); - return ch; - }, -unput:function (ch) { - var len = ch.length; - var lines = ch.split(/(?:\r\n?|\n)/g); - - this._input = ch + this._input; - this.yytext = this.yytext.substr(0, this.yytext.length-len-1); - //this.yyleng -= len; - this.offset -= len; - var oldLines = this.match.split(/(?:\r\n?|\n)/g); - this.match = this.match.substr(0, this.match.length-1); - this.matched = this.matched.substr(0, this.matched.length-1); - - if (lines.length-1) this.yylineno -= lines.length-1; - var r = this.yylloc.range; - - this.yylloc = {first_line: this.yylloc.first_line, - last_line: this.yylineno+1, - first_column: this.yylloc.first_column, - last_column: lines ? - (lines.length === oldLines.length ? this.yylloc.first_column : 0) + oldLines[oldLines.length - lines.length].length - lines[0].length: - this.yylloc.first_column - len - }; - - if (this.options.ranges) { - this.yylloc.range = [r[0], r[0] + this.yyleng - len]; - } - return this; - }, -more:function () { - this._more = true; - return this; - }, -less:function (n) { - this.unput(this.match.slice(n)); - }, -pastInput:function () { - var past = this.matched.substr(0, this.matched.length - this.match.length); - return (past.length > 20 ? '...':'') + past.substr(-20).replace(/\n/g, ""); - }, -upcomingInput:function () { - var next = this.match; - if (next.length < 20) { - next += this._input.substr(0, 20-next.length); - } - return (next.substr(0,20)+(next.length > 20 ? '...':'')).replace(/\n/g, ""); - }, -showPosition:function () { - var pre = this.pastInput(); - var c = new Array(pre.length + 1).join("-"); - return pre + this.upcomingInput() + "\n" + c+"^"; - }, -next:function () { - if (this.done) { - return this.EOF; - } - if (!this._input) this.done = true; - - var token, - match, - tempMatch, - index, - col, - lines; - if (!this._more) { - this.yytext = ''; - this.match = ''; - } - var rules = this._currentRules(); - for (var i=0;i < rules.length; i++) { - tempMatch = this._input.match(this.rules[rules[i]]); - if (tempMatch && (!match || tempMatch[0].length > match[0].length)) { - match = tempMatch; - index = i; - if (!this.options.flex) break; - } - } - if (match) { - lines = match[0].match(/(?:\r\n?|\n).*/g); - if (lines) this.yylineno += lines.length; - this.yylloc = {first_line: this.yylloc.last_line, - last_line: this.yylineno+1, - first_column: this.yylloc.last_column, - last_column: lines ? lines[lines.length-1].length-lines[lines.length-1].match(/\r?\n?/)[0].length : this.yylloc.last_column + match[0].length}; - this.yytext += match[0]; - this.match += match[0]; - this.matches = match; - this.yyleng = this.yytext.length; - if (this.options.ranges) { - this.yylloc.range = [this.offset, this.offset += this.yyleng]; - } - this._more = false; - this._input = this._input.slice(match[0].length); - this.matched += match[0]; - token = this.performAction.call(this, this.yy, this, rules[index],this.conditionStack[this.conditionStack.length-1]); - if (this.done && this._input) this.done = false; - if (token) return token; - else return; - } - if (this._input === "") { - return this.EOF; - } else { - return this.parseError('Lexical error on line '+(this.yylineno+1)+'. Unrecognized text.\n'+this.showPosition(), - {text: "", token: null, line: this.yylineno}); - } - }, -lex:function lex() { - var r = this.next(); - if (typeof r !== 'undefined') { - return r; - } else { - return this.lex(); - } - }, -begin:function begin(condition) { - this.conditionStack.push(condition); - }, -popState:function popState() { - return this.conditionStack.pop(); - }, -_currentRules:function _currentRules() { - return this.conditions[this.conditionStack[this.conditionStack.length-1]].rules; - }, -topState:function () { - return this.conditionStack[this.conditionStack.length-2]; - }, -pushState:function begin(condition) { - this.begin(condition); - }}); -lexer.options = {}; -lexer.performAction = function anonymous(yy,yy_,$avoiding_name_collisions,YY_START) { - -var YYSTATE=YY_START -switch($avoiding_name_collisions) { -case 0: - if(yy_.yytext.slice(-1) !== "\\") this.begin("mu"); - if(yy_.yytext.slice(-1) === "\\") yy_.yytext = yy_.yytext.substr(0,yy_.yyleng-1), this.begin("emu"); - if(yy_.yytext) return 14; - -break; -case 1: return 14; -break; -case 2: - if(yy_.yytext.slice(-1) !== "\\") this.popState(); - if(yy_.yytext.slice(-1) === "\\") yy_.yytext = yy_.yytext.substr(0,yy_.yyleng-1); - return 14; - -break; -case 3: return 24; -break; -case 4: return 16; -break; -case 5: return 20; -break; -case 6: return 19; -break; -case 7: return 19; -break; -case 8: return 23; -break; -case 9: return 23; -break; -case 10: yy_.yytext = yy_.yytext.substr(3,yy_.yyleng-5); this.popState(); return 15; -break; -case 11: return 22; -break; -case 12: return 35; -break; -case 13: return 34; -break; -case 14: return 34; -break; -case 15: return 37; -break; -case 16: /*ignore whitespace*/ -break; -case 17: this.popState(); return 18; -break; -case 18: this.popState(); return 18; -break; -case 19: yy_.yytext = yy_.yytext.substr(1,yy_.yyleng-2).replace(/\\"/g,'"'); return 29; -break; -case 20: yy_.yytext = yy_.yytext.substr(1,yy_.yyleng-2).replace(/\\"/g,'"'); return 29; -break; -case 21: yy_.yytext = yy_.yytext.substr(1); return 27; -break; -case 22: return 31; -break; -case 23: return 31; -break; -case 24: return 30; -break; -case 25: return 34; -break; -case 26: yy_.yytext = yy_.yytext.substr(1, yy_.yyleng-2); return 34; -break; -case 27: return 'INVALID'; -break; -case 28: return 5; -break; -} -}; -lexer.rules = [/^(?:[^\x00]*?(?=(\{\{)))/,/^(?:[^\x00]+)/,/^(?:[^\x00]{2,}?(?=(\{\{|$)))/,/^(?:\{\{>)/,/^(?:\{\{#)/,/^(?:\{\{\/)/,/^(?:\{\{\^)/,/^(?:\{\{\s*else\b)/,/^(?:\{\{\{)/,/^(?:\{\{&)/,/^(?:\{\{![\s\S]*?\}\})/,/^(?:\{\{)/,/^(?:=)/,/^(?:\.(?=[} ]))/,/^(?:\.\.)/,/^(?:[\/.])/,/^(?:\s+)/,/^(?:\}\}\})/,/^(?:\}\})/,/^(?:"(\\["]|[^"])*")/,/^(?:'(\\[']|[^'])*')/,/^(?:@[a-zA-Z]+)/,/^(?:true(?=[}\s]))/,/^(?:false(?=[}\s]))/,/^(?:[0-9]+(?=[}\s]))/,/^(?:[a-zA-Z0-9_$-]+(?=[=}\s\/.]))/,/^(?:\[[^\]]*\])/,/^(?:.)/,/^(?:$)/]; -lexer.conditions = {"mu":{"rules":[3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28],"inclusive":false},"emu":{"rules":[2],"inclusive":false},"INITIAL":{"rules":[0,1,28],"inclusive":true}}; -return lexer;})() -parser.lexer = lexer; -function Parser () { this.yy = {}; }Parser.prototype = parser;parser.Parser = Parser; -return new Parser; -})(); -if (typeof require !== 'undefined' && typeof exports !== 'undefined') { -exports.parser = handlebars; -exports.Parser = handlebars.Parser; -exports.parse = function () { return handlebars.parse.apply(handlebars, arguments); } -exports.main = function commonjsMain(args) { - if (!args[1]) - throw new Error('Usage: '+args[0]+' FILE'); - var source, cwd; - if (typeof process !== 'undefined') { - source = require('fs').readFileSync(require('path').resolve(args[1]), "utf8"); - } else { - source = require("file").path(require("file").cwd()).join(args[1]).read({charset: "utf-8"}); - } - return exports.parser.parse(source); -} -if (typeof module !== 'undefined' && require.main === module) { - exports.main(typeof process !== 'undefined' ? process.argv.slice(1) : require("system").args); -} -}; -; -// lib/handlebars/compiler/base.js -Handlebars.Parser = handlebars; - -Handlebars.parse = function(string) { - Handlebars.Parser.yy = Handlebars.AST; - return Handlebars.Parser.parse(string); -}; - -Handlebars.print = function(ast) { - return new Handlebars.PrintVisitor().accept(ast); -}; - -Handlebars.logger = { - DEBUG: 0, INFO: 1, WARN: 2, ERROR: 3, level: 3, - - // override in the host environment - log: function(level, str) {} -}; - -Handlebars.log = function(level, str) { Handlebars.logger.log(level, str); }; -; -// lib/handlebars/compiler/ast.js -(function() { - - Handlebars.AST = {}; - - Handlebars.AST.ProgramNode = function(statements, inverse) { - this.type = "program"; - this.statements = statements; - if(inverse) { this.inverse = new Handlebars.AST.ProgramNode(inverse); } - }; - - Handlebars.AST.MustacheNode = function(rawParams, hash, unescaped) { - this.type = "mustache"; - this.escaped = !unescaped; - this.hash = hash; - - var id = this.id = rawParams[0]; - var params = this.params = rawParams.slice(1); - - // a mustache is an eligible helper if: - // * its id is simple (a single part, not `this` or `..`) - var eligibleHelper = this.eligibleHelper = id.isSimple; - - // a mustache is definitely a helper if: - // * it is an eligible helper, and - // * it has at least one parameter or hash segment - this.isHelper = eligibleHelper && (params.length || hash); - - // if a mustache is an eligible helper but not a definite - // helper, it is ambiguous, and will be resolved in a later - // pass or at runtime. - }; - - Handlebars.AST.PartialNode = function(id, context) { - this.type = "partial"; - - // TODO: disallow complex IDs - - this.id = id; - this.context = context; - }; - - var verifyMatch = function(open, close) { - if(open.original !== close.original) { - throw new Handlebars.Exception(open.original + " doesn't match " + close.original); - } - }; - - Handlebars.AST.BlockNode = function(mustache, program, inverse, close) { - verifyMatch(mustache.id, close); - this.type = "block"; - this.mustache = mustache; - this.program = program; - this.inverse = inverse; - - if (this.inverse && !this.program) { - this.isInverse = true; - } - }; - - Handlebars.AST.ContentNode = function(string) { - this.type = "content"; - this.string = string; - }; - - Handlebars.AST.HashNode = function(pairs) { - this.type = "hash"; - this.pairs = pairs; - }; - - Handlebars.AST.IdNode = function(parts) { - this.type = "ID"; - this.original = parts.join("."); - - var dig = [], depth = 0; - - for(var i=0,l=parts.length; i": ">", - '"': """, - "'": "'", - "`": "`" - }; - - var badChars = /[&<>"'`]/g; - var possible = /[&<>"'`]/; - - var escapeChar = function(chr) { - return escape[chr] || "&"; - }; - - Handlebars.Utils = { - escapeExpression: function(string) { - // don't escape SafeStrings, since they're already safe - if (string instanceof Handlebars.SafeString) { - return string.toString(); - } else if (string == null || string === false) { - return ""; - } - - if(!possible.test(string)) { return string; } - return string.replace(badChars, escapeChar); - }, - - isEmpty: function(value) { - if (typeof value === "undefined") { - return true; - } else if (value === null) { - return true; - } else if (value === false) { - return true; - } else if(Object.prototype.toString.call(value) === "[object Array]" && value.length === 0) { - return true; - } else { - return false; - } - } - }; -})();; -// lib/handlebars/compiler/compiler.js - -/*jshint eqnull:true*/ -Handlebars.Compiler = function() {}; -Handlebars.JavaScriptCompiler = function() {}; - -(function(Compiler, JavaScriptCompiler) { - // the foundHelper register will disambiguate helper lookup from finding a - // function in a context. This is necessary for mustache compatibility, which - // requires that context functions in blocks are evaluated by blockHelperMissing, - // and then proceed as if the resulting value was provided to blockHelperMissing. - - Compiler.prototype = { - compiler: Compiler, - - disassemble: function() { - var opcodes = this.opcodes, opcode, out = [], params, param; - - for (var i=0, l=opcodes.length; i 0) { - this.source[1] = this.source[1] + ", " + locals.join(", "); - } - - // Generate minimizer alias mappings - if (!this.isChild) { - var aliases = []; - for (var alias in this.context.aliases) { - this.source[1] = this.source[1] + ', ' + alias + '=' + this.context.aliases[alias]; - } - } - - if (this.source[1]) { - this.source[1] = "var " + this.source[1].substring(2) + ";"; - } - - // Merge children - if (!this.isChild) { - this.source[1] += '\n' + this.context.programs.join('\n') + '\n'; - } - - if (!this.environment.isSimple) { - this.source.push("return buffer;"); - } - - var params = this.isChild ? ["depth0", "data"] : ["Handlebars", "depth0", "helpers", "partials", "data"]; - - for(var i=0, l=this.environment.depths.list.length; i this.stackVars.length) { this.stackVars.push("stack" + this.stackSlot); } - return "stack" + this.stackSlot; - }, - - popStack: function() { - var item = this.compileStack.pop(); - - if (item instanceof Literal) { - return item.value; - } else { - this.stackSlot--; - return item; - } - }, - - topStack: function() { - var item = this.compileStack[this.compileStack.length - 1]; - - if (item instanceof Literal) { - return item.value; - } else { - return item; - } - }, - - quotedString: function(str) { - return '"' + str - .replace(/\\/g, '\\\\') - .replace(/"/g, '\\"') - .replace(/\n/g, '\\n') - .replace(/\r/g, '\\r') + '"'; - }, - - setupHelper: function(paramSize, name) { - var params = []; - this.setupParams(paramSize, params); - var foundHelper = this.nameLookup('helpers', name, 'helper'); - - return { - params: params, - name: foundHelper, - callParams: ["depth0"].concat(params).join(", "), - helperMissingParams: ["depth0", this.quotedString(name)].concat(params).join(", ") - }; - }, - - // the params and contexts arguments are passed in arrays - // to fill in - setupParams: function(paramSize, params) { - var options = [], contexts = [], param, inverse, program; - - options.push("hash:" + this.popStack()); - - inverse = this.popStack(); - program = this.popStack(); - - // Avoid setting fn and inverse if neither are set. This allows - // helpers to do a check for `if (options.fn)` - if (program || inverse) { - if (!program) { - this.context.aliases.self = "this"; - program = "self.noop"; - } - - if (!inverse) { - this.context.aliases.self = "this"; - inverse = "self.noop"; - } - - options.push("inverse:" + inverse); - options.push("fn:" + program); - } - - for(var i=0; i') - .appendTo(document.body) - - this.$backdrop.click( - this.options.backdrop == 'static' ? - $.proxy(this.$element[0].focus, this.$element[0]) - : $.proxy(this.hide, this) - ) - - if (doAnimate) this.$backdrop[0].offsetWidth // force reflow - - this.$backdrop.addClass('in') - - if (!callback) return - - doAnimate ? - this.$backdrop.one($.support.transition.end, callback) : - callback() - - } else if (!this.isShown && this.$backdrop) { - this.$backdrop.removeClass('in') - - $.support.transition && this.$element.hasClass('fade')? - this.$backdrop.one($.support.transition.end, callback) : - callback() - - } else if (callback) { - callback() - } - } - } - - - /* MODAL PLUGIN DEFINITION - * ======================= */ - - var old = $.fn.modal - - $.fn.modal = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('modal') - , options = $.extend({}, $.fn.modal.defaults, $this.data(), typeof option == 'object' && option) - if (!data) $this.data('modal', (data = new Modal(this, options))) - if (typeof option == 'string') data[option]() - else if (options.show) data.show() - }) - } - - $.fn.modal.defaults = { - backdrop: true - , keyboard: true - , show: true - } - - $.fn.modal.Constructor = Modal - - - /* MODAL NO CONFLICT - * ================= */ - - $.fn.modal.noConflict = function () { - $.fn.modal = old - return this - } - - - /* MODAL DATA-API - * ============== */ - - $(document).on('click.modal.data-api', '[data-toggle="modal"]', function (e) { - var $this = $(this) - , href = $this.attr('href') - , $target = $($this.attr('data-target') || (href && href.replace(/.*(?=#[^\s]+$)/, ''))) //strip for ie7 - , option = $target.data('modal') ? 'toggle' : $.extend({ remote:!/#/.test(href) && href }, $target.data(), $this.data()) - - e.preventDefault() - - $target - .modal(option) - .one('hide', function () { - $this.focus() - }) - }) - -}(window.jQuery); - -/* ============================================================ - * bootstrap-dropdown.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#dropdowns - * ============================================================ - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================ */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* DROPDOWN CLASS DEFINITION - * ========================= */ - - var toggle = '[data-toggle=dropdown]' - , Dropdown = function (element) { - var $el = $(element).on('click.dropdown.data-api', this.toggle) - $('html').on('click.dropdown.data-api', function () { - $el.parent().removeClass('open') - }) - } - - Dropdown.prototype = { - - constructor: Dropdown - - , toggle: function (e) { - var $this = $(this) - , $parent - , isActive - - if ($this.is('.disabled, :disabled')) return - - $parent = getParent($this) - - isActive = $parent.hasClass('open') - - clearMenus() - - if (!isActive) { - if ('ontouchstart' in document.documentElement) { - // if mobile we we use a backdrop because click events don't delegate - $('
    ').insertBefore($(this)).on('click', clearMenus) - } - $parent.toggleClass('open') - } - - $this.focus() - - return false - } - - , keydown: function (e) { - var $this - , $items - , $active - , $parent - , isActive - , index - - if (!/(38|40|27)/.test(e.keyCode)) return - - $this = $(this) - - e.preventDefault() - e.stopPropagation() - - if ($this.is('.disabled, :disabled')) return - - $parent = getParent($this) - - isActive = $parent.hasClass('open') - - if (!isActive || (isActive && e.keyCode == 27)) { - if (e.which == 27) $parent.find(toggle).focus() - return $this.click() - } - - $items = $('[role=menu] li:not(.divider):visible a', $parent) - - if (!$items.length) return - - index = $items.index($items.filter(':focus')) - - if (e.keyCode == 38 && index > 0) index-- // up - if (e.keyCode == 40 && index < $items.length - 1) index++ // down - if (!~index) index = 0 - - $items - .eq(index) - .focus() - } - - } - - function clearMenus() { - $('.dropdown-backdrop').remove() - $(toggle).each(function () { - getParent($(this)).removeClass('open') - }) - } - - function getParent($this) { - var selector = $this.attr('data-target') - , $parent - - if (!selector) { - selector = $this.attr('href') - selector = selector && /#/.test(selector) && selector.replace(/.*(?=#[^\s]*$)/, '') //strip for ie7 - } - - $parent = selector && $(selector) - - if (!$parent || !$parent.length) $parent = $this.parent() - - return $parent - } - - - /* DROPDOWN PLUGIN DEFINITION - * ========================== */ - - var old = $.fn.dropdown - - $.fn.dropdown = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('dropdown') - if (!data) $this.data('dropdown', (data = new Dropdown(this))) - if (typeof option == 'string') data[option].call($this) - }) - } - - $.fn.dropdown.Constructor = Dropdown - - - /* DROPDOWN NO CONFLICT - * ==================== */ - - $.fn.dropdown.noConflict = function () { - $.fn.dropdown = old - return this - } - - - /* APPLY TO STANDARD DROPDOWN ELEMENTS - * =================================== */ - - $(document) - .on('click.dropdown.data-api', clearMenus) - .on('click.dropdown.data-api', '.dropdown form', function (e) { e.stopPropagation() }) - .on('click.dropdown.data-api' , toggle, Dropdown.prototype.toggle) - .on('keydown.dropdown.data-api', toggle + ', [role=menu]' , Dropdown.prototype.keydown) - -}(window.jQuery); - -/* ============================================================= - * bootstrap-scrollspy.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#scrollspy - * ============================================================= - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================== */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* SCROLLSPY CLASS DEFINITION - * ========================== */ - - function ScrollSpy(element, options) { - var process = $.proxy(this.process, this) - , $element = $(element).is('body') ? $(window) : $(element) - , href - this.options = $.extend({}, $.fn.scrollspy.defaults, options) - this.$scrollElement = $element.on('scroll.scroll-spy.data-api', process) - this.selector = (this.options.target - || ((href = $(element).attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '')) //strip for ie7 - || '') + ' .nav li > a' - this.$body = $('body') - this.refresh() - this.process() - } - - ScrollSpy.prototype = { - - constructor: ScrollSpy - - , refresh: function () { - var self = this - , $targets - - this.offsets = $([]) - this.targets = $([]) - - $targets = this.$body - .find(this.selector) - .map(function () { - var $el = $(this) - , href = $el.data('target') || $el.attr('href') - , $href = /^#\w/.test(href) && $(href) - return ( $href - && $href.length - && [[ $href.position().top + (!$.isWindow(self.$scrollElement.get(0)) && self.$scrollElement.scrollTop()), href ]] ) || null - }) - .sort(function (a, b) { return a[0] - b[0] }) - .each(function () { - self.offsets.push(this[0]) - self.targets.push(this[1]) - }) - } - - , process: function () { - var scrollTop = this.$scrollElement.scrollTop() + this.options.offset - , scrollHeight = this.$scrollElement[0].scrollHeight || this.$body[0].scrollHeight - , maxScroll = scrollHeight - this.$scrollElement.height() - , offsets = this.offsets - , targets = this.targets - , activeTarget = this.activeTarget - , i - - if (scrollTop >= maxScroll) { - return activeTarget != (i = targets.last()[0]) - && this.activate ( i ) - } - - for (i = offsets.length; i--;) { - activeTarget != targets[i] - && scrollTop >= offsets[i] - && (!offsets[i + 1] || scrollTop <= offsets[i + 1]) - && this.activate( targets[i] ) - } - } - - , activate: function (target) { - var active - , selector - - this.activeTarget = target - - $(this.selector) - .parent('.active') - .removeClass('active') - - selector = this.selector - + '[data-target="' + target + '"],' - + this.selector + '[href="' + target + '"]' - - active = $(selector) - .parent('li') - .addClass('active') - - if (active.parent('.dropdown-menu').length) { - active = active.closest('li.dropdown').addClass('active') - } - - active.trigger('activate') - } - - } - - - /* SCROLLSPY PLUGIN DEFINITION - * =========================== */ - - var old = $.fn.scrollspy - - $.fn.scrollspy = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('scrollspy') - , options = typeof option == 'object' && option - if (!data) $this.data('scrollspy', (data = new ScrollSpy(this, options))) - if (typeof option == 'string') data[option]() - }) - } - - $.fn.scrollspy.Constructor = ScrollSpy - - $.fn.scrollspy.defaults = { - offset: 10 - } - - - /* SCROLLSPY NO CONFLICT - * ===================== */ - - $.fn.scrollspy.noConflict = function () { - $.fn.scrollspy = old - return this - } - - - /* SCROLLSPY DATA-API - * ================== */ - - $(window).on('load', function () { - $('[data-spy="scroll"]').each(function () { - var $spy = $(this) - $spy.scrollspy($spy.data()) - }) - }) - -}(window.jQuery); -/* ======================================================== - * bootstrap-tab.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#tabs - * ======================================================== - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ======================================================== */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* TAB CLASS DEFINITION - * ==================== */ - - var Tab = function (element) { - this.element = $(element) - } - - Tab.prototype = { - - constructor: Tab - - , show: function () { - var $this = this.element - , $ul = $this.closest('ul:not(.dropdown-menu)') - , selector = $this.attr('data-target') - , previous - , $target - , e - - if (!selector) { - selector = $this.attr('href') - selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') //strip for ie7 - } - - if ( $this.parent('li').hasClass('active') ) return - - previous = $ul.find('.active:last a')[0] - - e = $.Event('show', { - relatedTarget: previous - }) - - $this.trigger(e) - - if (e.isDefaultPrevented()) return - - $target = $(selector) - - this.activate($this.parent('li'), $ul) - this.activate($target, $target.parent(), function () { - $this.trigger({ - type: 'shown' - , relatedTarget: previous - }) - }) - } - - , activate: function ( element, container, callback) { - var $active = container.find('> .active') - , transition = callback - && $.support.transition - && $active.hasClass('fade') - - function next() { - $active - .removeClass('active') - .find('> .dropdown-menu > .active') - .removeClass('active') - - element.addClass('active') - - if (transition) { - element[0].offsetWidth // reflow for transition - element.addClass('in') - } else { - element.removeClass('fade') - } - - if ( element.parent('.dropdown-menu') ) { - element.closest('li.dropdown').addClass('active') - } - - callback && callback() - } - - transition ? - $active.one($.support.transition.end, next) : - next() - - $active.removeClass('in') - } - } - - - /* TAB PLUGIN DEFINITION - * ===================== */ - - var old = $.fn.tab - - $.fn.tab = function ( option ) { - return this.each(function () { - var $this = $(this) - , data = $this.data('tab') - if (!data) $this.data('tab', (data = new Tab(this))) - if (typeof option == 'string') data[option]() - }) - } - - $.fn.tab.Constructor = Tab - - - /* TAB NO CONFLICT - * =============== */ - - $.fn.tab.noConflict = function () { - $.fn.tab = old - return this - } - - - /* TAB DATA-API - * ============ */ - - $(document).on('click.tab.data-api', '[data-toggle="tab"], [data-toggle="pill"]', function (e) { - e.preventDefault() - $(this).tab('show') - }) - -}(window.jQuery); -/* =========================================================== - * bootstrap-tooltip.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#tooltips - * Inspired by the original jQuery.tipsy by Jason Frame - * =========================================================== - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ========================================================== */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* TOOLTIP PUBLIC CLASS DEFINITION - * =============================== */ - - var Tooltip = function (element, options) { - this.init('tooltip', element, options) - } - - Tooltip.prototype = { - - constructor: Tooltip - - , init: function (type, element, options) { - var eventIn - , eventOut - , triggers - , trigger - , i - - this.type = type - this.$element = $(element) - this.options = this.getOptions(options) - this.enabled = true - - triggers = this.options.trigger.split(' ') - - for (i = triggers.length; i--;) { - trigger = triggers[i] - if (trigger == 'click') { - this.$element.on('click.' + this.type, this.options.selector, $.proxy(this.toggle, this)) - } else if (trigger != 'manual') { - eventIn = trigger == 'hover' ? 'mouseenter' : 'focus' - eventOut = trigger == 'hover' ? 'mouseleave' : 'blur' - this.$element.on(eventIn + '.' + this.type, this.options.selector, $.proxy(this.enter, this)) - this.$element.on(eventOut + '.' + this.type, this.options.selector, $.proxy(this.leave, this)) - } - } - - this.options.selector ? - (this._options = $.extend({}, this.options, { trigger: 'manual', selector: '' })) : - this.fixTitle() - } - - , getOptions: function (options) { - options = $.extend({}, $.fn[this.type].defaults, this.$element.data(), options) - - if (options.delay && typeof options.delay == 'number') { - options.delay = { - show: options.delay - , hide: options.delay - } - } - - return options - } - - , enter: function (e) { - var defaults = $.fn[this.type].defaults - , options = {} - , self - - this._options && $.each(this._options, function (key, value) { - if (defaults[key] != value) options[key] = value - }, this) - - self = $(e.currentTarget)[this.type](options).data(this.type) - - if (!self.options.delay || !self.options.delay.show) return self.show() - - clearTimeout(this.timeout) - self.hoverState = 'in' - this.timeout = setTimeout(function() { - if (self.hoverState == 'in') self.show() - }, self.options.delay.show) - } - - , leave: function (e) { - var self = $(e.currentTarget)[this.type](this._options).data(this.type) - - if (this.timeout) clearTimeout(this.timeout) - if (!self.options.delay || !self.options.delay.hide) return self.hide() - - self.hoverState = 'out' - this.timeout = setTimeout(function() { - if (self.hoverState == 'out') self.hide() - }, self.options.delay.hide) - } - - , show: function () { - var $tip - , pos - , actualWidth - , actualHeight - , placement - , tp - , e = $.Event('show') - - if (this.hasContent() && this.enabled) { - this.$element.trigger(e) - if (e.isDefaultPrevented()) return - $tip = this.tip() - this.setContent() - - if (this.options.animation) { - $tip.addClass('fade') - } - - placement = typeof this.options.placement == 'function' ? - this.options.placement.call(this, $tip[0], this.$element[0]) : - this.options.placement - - $tip - .detach() - .css({ top: 0, left: 0, display: 'block' }) - - this.options.container ? $tip.appendTo(this.options.container) : $tip.insertAfter(this.$element) - - pos = this.getPosition() - - actualWidth = $tip[0].offsetWidth - actualHeight = $tip[0].offsetHeight - - switch (placement) { - case 'bottom': - tp = {top: pos.top + pos.height, left: pos.left + pos.width / 2 - actualWidth / 2} - break - case 'top': - tp = {top: pos.top - actualHeight, left: pos.left + pos.width / 2 - actualWidth / 2} - break - case 'left': - tp = {top: pos.top + pos.height / 2 - actualHeight / 2, left: pos.left - actualWidth} - break - case 'right': - tp = {top: pos.top + pos.height / 2 - actualHeight / 2, left: pos.left + pos.width} - break - } - - this.applyPlacement(tp, placement) - this.$element.trigger('shown') - } - } - - , applyPlacement: function(offset, placement){ - var $tip = this.tip() - , width = $tip[0].offsetWidth - , height = $tip[0].offsetHeight - , actualWidth - , actualHeight - , delta - , replace - - $tip - .offset(offset) - .addClass(placement) - .addClass('in') - - actualWidth = $tip[0].offsetWidth - actualHeight = $tip[0].offsetHeight - - if (placement == 'top' && actualHeight != height) { - offset.top = offset.top + height - actualHeight - replace = true - } - - if (placement == 'bottom' || placement == 'top') { - delta = 0 - - if (offset.left < 0){ - delta = offset.left * -2 - offset.left = 0 - $tip.offset(offset) - actualWidth = $tip[0].offsetWidth - actualHeight = $tip[0].offsetHeight - } - - this.replaceArrow(delta - width + actualWidth, actualWidth, 'left') - } else { - this.replaceArrow(actualHeight - height, actualHeight, 'top') - } - - if (replace) $tip.offset(offset) - } - - , replaceArrow: function(delta, dimension, position){ - this - .arrow() - .css(position, delta ? (50 * (1 - delta / dimension) + "%") : '') - } - - , setContent: function () { - var $tip = this.tip() - , title = this.getTitle() - - $tip.find('.tooltip-inner')[this.options.html ? 'html' : 'text'](title) - $tip.removeClass('fade in top bottom left right') - } - - , hide: function () { - var that = this - , $tip = this.tip() - , e = $.Event('hide') - - this.$element.trigger(e) - if (e.isDefaultPrevented()) return - - $tip.removeClass('in') - - function removeWithAnimation() { - var timeout = setTimeout(function () { - $tip.off($.support.transition.end).detach() - }, 500) - - $tip.one($.support.transition.end, function () { - clearTimeout(timeout) - $tip.detach() - }) - } - - $.support.transition && this.$tip.hasClass('fade') ? - removeWithAnimation() : - $tip.detach() - - this.$element.trigger('hidden') - - return this - } - - , fixTitle: function () { - var $e = this.$element - if ($e.attr('title') || typeof($e.attr('data-original-title')) != 'string') { - $e.attr('data-original-title', $e.attr('title') || '').attr('title', '') - } - } - - , hasContent: function () { - return this.getTitle() - } - - , getPosition: function () { - var el = this.$element[0] - return $.extend({}, (typeof el.getBoundingClientRect == 'function') ? el.getBoundingClientRect() : { - width: el.offsetWidth - , height: el.offsetHeight - }, this.$element.offset()) - } - - , getTitle: function () { - var title - , $e = this.$element - , o = this.options - - title = $e.attr('data-original-title') - || (typeof o.title == 'function' ? o.title.call($e[0]) : o.title) - - return title - } - - , tip: function () { - return this.$tip = this.$tip || $(this.options.template) - } - - , arrow: function(){ - return this.$arrow = this.$arrow || this.tip().find(".tooltip-arrow") - } - - , validate: function () { - if (!this.$element[0].parentNode) { - this.hide() - this.$element = null - this.options = null - } - } - - , enable: function () { - this.enabled = true - } - - , disable: function () { - this.enabled = false - } - - , toggleEnabled: function () { - this.enabled = !this.enabled - } - - , toggle: function (e) { - var self = e ? $(e.currentTarget)[this.type](this._options).data(this.type) : this - self.tip().hasClass('in') ? self.hide() : self.show() - } - - , destroy: function () { - this.hide().$element.off('.' + this.type).removeData(this.type) - } - - } - - - /* TOOLTIP PLUGIN DEFINITION - * ========================= */ - - var old = $.fn.tooltip - - $.fn.tooltip = function ( option ) { - return this.each(function () { - var $this = $(this) - , data = $this.data('tooltip') - , options = typeof option == 'object' && option - if (!data) $this.data('tooltip', (data = new Tooltip(this, options))) - if (typeof option == 'string') data[option]() - }) - } - - $.fn.tooltip.Constructor = Tooltip - - $.fn.tooltip.defaults = { - animation: true - , placement: 'top' - , selector: false - , template: '
    ' - , trigger: 'hover focus' - , title: '' - , delay: 0 - , html: false - , container: false - } - - - /* TOOLTIP NO CONFLICT - * =================== */ - - $.fn.tooltip.noConflict = function () { - $.fn.tooltip = old - return this - } - -}(window.jQuery); - -/* =========================================================== - * bootstrap-popover.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#popovers - * =========================================================== - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * =========================================================== */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* POPOVER PUBLIC CLASS DEFINITION - * =============================== */ - - var Popover = function (element, options) { - this.init('popover', element, options) - } - - - /* NOTE: POPOVER EXTENDS BOOTSTRAP-TOOLTIP.js - ========================================== */ - - Popover.prototype = $.extend({}, $.fn.tooltip.Constructor.prototype, { - - constructor: Popover - - , setContent: function () { - var $tip = this.tip() - , title = this.getTitle() - , content = this.getContent() - - $tip.find('.popover-title')[this.options.html ? 'html' : 'text'](title) - $tip.find('.popover-content')[this.options.html ? 'html' : 'text'](content) - - $tip.removeClass('fade top bottom left right in') - } - - , hasContent: function () { - return this.getTitle() || this.getContent() - } - - , getContent: function () { - var content - , $e = this.$element - , o = this.options - - content = (typeof o.content == 'function' ? o.content.call($e[0]) : o.content) - || $e.attr('data-content') - - return content - } - - , tip: function () { - if (!this.$tip) { - this.$tip = $(this.options.template) - } - return this.$tip - } - - , destroy: function () { - this.hide().$element.off('.' + this.type).removeData(this.type) - } - - }) - - - /* POPOVER PLUGIN DEFINITION - * ======================= */ - - var old = $.fn.popover - - $.fn.popover = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('popover') - , options = typeof option == 'object' && option - if (!data) $this.data('popover', (data = new Popover(this, options))) - if (typeof option == 'string') data[option]() - }) - } - - $.fn.popover.Constructor = Popover - - $.fn.popover.defaults = $.extend({} , $.fn.tooltip.defaults, { - placement: 'right' - , trigger: 'click' - , content: '' - , template: '

    ' - }) - - - /* POPOVER NO CONFLICT - * =================== */ - - $.fn.popover.noConflict = function () { - $.fn.popover = old - return this - } - -}(window.jQuery); - -/* ========================================================== - * bootstrap-affix.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#affix - * ========================================================== - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ========================================================== */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* AFFIX CLASS DEFINITION - * ====================== */ - - var Affix = function (element, options) { - this.options = $.extend({}, $.fn.affix.defaults, options) - this.$window = $(window) - .on('scroll.affix.data-api', $.proxy(this.checkPosition, this)) - .on('click.affix.data-api', $.proxy(function () { setTimeout($.proxy(this.checkPosition, this), 1) }, this)) - this.$element = $(element) - this.checkPosition() - } - - Affix.prototype.checkPosition = function () { - if (!this.$element.is(':visible')) return - - var scrollHeight = $(document).height() - , scrollTop = this.$window.scrollTop() - , position = this.$element.offset() - , offset = this.options.offset - , offsetBottom = offset.bottom - , offsetTop = offset.top - , reset = 'affix affix-top affix-bottom' - , affix - - if (typeof offset != 'object') offsetBottom = offsetTop = offset - if (typeof offsetTop == 'function') offsetTop = offset.top() - if (typeof offsetBottom == 'function') offsetBottom = offset.bottom() - - affix = this.unpin != null && (scrollTop + this.unpin <= position.top) ? - false : offsetBottom != null && (position.top + this.$element.height() >= scrollHeight - offsetBottom) ? - 'bottom' : offsetTop != null && scrollTop <= offsetTop ? - 'top' : false - - if (this.affixed === affix) return - - this.affixed = affix - this.unpin = affix == 'bottom' ? position.top - scrollTop : null - - this.$element.removeClass(reset).addClass('affix' + (affix ? '-' + affix : '')) - } - - - /* AFFIX PLUGIN DEFINITION - * ======================= */ - - var old = $.fn.affix - - $.fn.affix = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('affix') - , options = typeof option == 'object' && option - if (!data) $this.data('affix', (data = new Affix(this, options))) - if (typeof option == 'string') data[option]() - }) - } - - $.fn.affix.Constructor = Affix - - $.fn.affix.defaults = { - offset: 0 - } - - - /* AFFIX NO CONFLICT - * ================= */ - - $.fn.affix.noConflict = function () { - $.fn.affix = old - return this - } - - - /* AFFIX DATA-API - * ============== */ - - $(window).on('load', function () { - $('[data-spy="affix"]').each(function () { - var $spy = $(this) - , data = $spy.data() - - data.offset = data.offset || {} - - data.offsetBottom && (data.offset.bottom = data.offsetBottom) - data.offsetTop && (data.offset.top = data.offsetTop) - - $spy.affix(data) - }) - }) - - -}(window.jQuery); -/* ========================================================== - * bootstrap-alert.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#alerts - * ========================================================== - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ========================================================== */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* ALERT CLASS DEFINITION - * ====================== */ - - var dismiss = '[data-dismiss="alert"]' - , Alert = function (el) { - $(el).on('click', dismiss, this.close) - } - - Alert.prototype.close = function (e) { - var $this = $(this) - , selector = $this.attr('data-target') - , $parent - - if (!selector) { - selector = $this.attr('href') - selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') //strip for ie7 - } - - $parent = $(selector) - - e && e.preventDefault() - - $parent.length || ($parent = $this.hasClass('alert') ? $this : $this.parent()) - - $parent.trigger(e = $.Event('close')) - - if (e.isDefaultPrevented()) return - - $parent.removeClass('in') - - function removeElement() { - $parent - .trigger('closed') - .remove() - } - - $.support.transition && $parent.hasClass('fade') ? - $parent.on($.support.transition.end, removeElement) : - removeElement() - } - - - /* ALERT PLUGIN DEFINITION - * ======================= */ - - var old = $.fn.alert - - $.fn.alert = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('alert') - if (!data) $this.data('alert', (data = new Alert(this))) - if (typeof option == 'string') data[option].call($this) - }) - } - - $.fn.alert.Constructor = Alert - - - /* ALERT NO CONFLICT - * ================= */ - - $.fn.alert.noConflict = function () { - $.fn.alert = old - return this - } - - - /* ALERT DATA-API - * ============== */ - - $(document).on('click.alert.data-api', dismiss, Alert.prototype.close) - -}(window.jQuery); -/* ============================================================ - * bootstrap-button.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#buttons - * ============================================================ - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================ */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* BUTTON PUBLIC CLASS DEFINITION - * ============================== */ - - var Button = function (element, options) { - this.$element = $(element) - this.options = $.extend({}, $.fn.button.defaults, options) - } - - Button.prototype.setState = function (state) { - var d = 'disabled' - , $el = this.$element - , data = $el.data() - , val = $el.is('input') ? 'val' : 'html' - - state = state + 'Text' - data.resetText || $el.data('resetText', $el[val]()) - - $el[val](data[state] || this.options[state]) - - // push to event loop to allow forms to submit - setTimeout(function () { - state == 'loadingText' ? - $el.addClass(d).attr(d, d) : - $el.removeClass(d).removeAttr(d) - }, 0) - } - - Button.prototype.toggle = function () { - var $parent = this.$element.closest('[data-toggle="buttons-radio"]') - - $parent && $parent - .find('.active') - .removeClass('active') - - this.$element.toggleClass('active') - } - - - /* BUTTON PLUGIN DEFINITION - * ======================== */ - - var old = $.fn.button - - $.fn.button = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('button') - , options = typeof option == 'object' && option - if (!data) $this.data('button', (data = new Button(this, options))) - if (option == 'toggle') data.toggle() - else if (option) data.setState(option) - }) - } - - $.fn.button.defaults = { - loadingText: 'loading...' - } - - $.fn.button.Constructor = Button - - - /* BUTTON NO CONFLICT - * ================== */ - - $.fn.button.noConflict = function () { - $.fn.button = old - return this - } - - - /* BUTTON DATA-API - * =============== */ - - $(document).on('click.button.data-api', '[data-toggle^=button]', function (e) { - var $btn = $(e.target) - if (!$btn.hasClass('btn')) $btn = $btn.closest('.btn') - $btn.button('toggle') - }) - -}(window.jQuery); -/* ============================================================= - * bootstrap-collapse.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#collapse - * ============================================================= - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================ */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* COLLAPSE PUBLIC CLASS DEFINITION - * ================================ */ - - var Collapse = function (element, options) { - this.$element = $(element) - this.options = $.extend({}, $.fn.collapse.defaults, options) - - if (this.options.parent) { - this.$parent = $(this.options.parent) - } - - this.options.toggle && this.toggle() - } - - Collapse.prototype = { - - constructor: Collapse - - , dimension: function () { - var hasWidth = this.$element.hasClass('width') - return hasWidth ? 'width' : 'height' - } - - , show: function () { - var dimension - , scroll - , actives - , hasData - - if (this.transitioning || this.$element.hasClass('in')) return - - dimension = this.dimension() - scroll = $.camelCase(['scroll', dimension].join('-')) - actives = this.$parent && this.$parent.find('> .accordion-group > .in') - - if (actives && actives.length) { - hasData = actives.data('collapse') - if (hasData && hasData.transitioning) return - actives.collapse('hide') - hasData || actives.data('collapse', null) - } - - this.$element[dimension](0) - this.transition('addClass', $.Event('show'), 'shown') - $.support.transition && this.$element[dimension](this.$element[0][scroll]) - } - - , hide: function () { - var dimension - if (this.transitioning || !this.$element.hasClass('in')) return - dimension = this.dimension() - this.reset(this.$element[dimension]()) - this.transition('removeClass', $.Event('hide'), 'hidden') - this.$element[dimension](0) - } - - , reset: function (size) { - var dimension = this.dimension() - - this.$element - .removeClass('collapse') - [dimension](size || 'auto') - [0].offsetWidth - - this.$element[size !== null ? 'addClass' : 'removeClass']('collapse') - - return this - } - - , transition: function (method, startEvent, completeEvent) { - var that = this - , complete = function () { - if (startEvent.type == 'show') that.reset() - that.transitioning = 0 - that.$element.trigger(completeEvent) - } - - this.$element.trigger(startEvent) - - if (startEvent.isDefaultPrevented()) return - - this.transitioning = 1 - - this.$element[method]('in') - - $.support.transition && this.$element.hasClass('collapse') ? - this.$element.one($.support.transition.end, complete) : - complete() - } - - , toggle: function () { - this[this.$element.hasClass('in') ? 'hide' : 'show']() - } - - } - - - /* COLLAPSE PLUGIN DEFINITION - * ========================== */ - - var old = $.fn.collapse - - $.fn.collapse = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('collapse') - , options = $.extend({}, $.fn.collapse.defaults, $this.data(), typeof option == 'object' && option) - if (!data) $this.data('collapse', (data = new Collapse(this, options))) - if (typeof option == 'string') data[option]() - }) - } - - $.fn.collapse.defaults = { - toggle: true - } - - $.fn.collapse.Constructor = Collapse - - - /* COLLAPSE NO CONFLICT - * ==================== */ - - $.fn.collapse.noConflict = function () { - $.fn.collapse = old - return this - } - - - /* COLLAPSE DATA-API - * ================= */ - - $(document).on('click.collapse.data-api', '[data-toggle=collapse]', function (e) { - var $this = $(this), href - , target = $this.attr('data-target') - || e.preventDefault() - || (href = $this.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '') //strip for ie7 - , option = $(target).data('collapse') ? 'toggle' : $this.data() - $this[$(target).hasClass('in') ? 'addClass' : 'removeClass']('collapsed') - $(target).collapse(option) - }) - -}(window.jQuery); -/* ========================================================== - * bootstrap-carousel.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#carousel - * ========================================================== - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ========================================================== */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* CAROUSEL CLASS DEFINITION - * ========================= */ - - var Carousel = function (element, options) { - this.$element = $(element) - this.$indicators = this.$element.find('.carousel-indicators') - this.options = options - this.options.pause == 'hover' && this.$element - .on('mouseenter', $.proxy(this.pause, this)) - .on('mouseleave', $.proxy(this.cycle, this)) - } - - Carousel.prototype = { - - cycle: function (e) { - if (!e) this.paused = false - if (this.interval) clearInterval(this.interval); - this.options.interval - && !this.paused - && (this.interval = setInterval($.proxy(this.next, this), this.options.interval)) - return this - } - - , getActiveIndex: function () { - this.$active = this.$element.find('.item.active') - this.$items = this.$active.parent().children() - return this.$items.index(this.$active) - } - - , to: function (pos) { - var activeIndex = this.getActiveIndex() - , that = this - - if (pos > (this.$items.length - 1) || pos < 0) return - - if (this.sliding) { - return this.$element.one('slid', function () { - that.to(pos) - }) - } - - if (activeIndex == pos) { - return this.pause().cycle() - } - - return this.slide(pos > activeIndex ? 'next' : 'prev', $(this.$items[pos])) - } - - , pause: function (e) { - if (!e) this.paused = true - if (this.$element.find('.next, .prev').length && $.support.transition.end) { - this.$element.trigger($.support.transition.end) - this.cycle(true) - } - clearInterval(this.interval) - this.interval = null - return this - } - - , next: function () { - if (this.sliding) return - return this.slide('next') - } - - , prev: function () { - if (this.sliding) return - return this.slide('prev') - } - - , slide: function (type, next) { - var $active = this.$element.find('.item.active') - , $next = next || $active[type]() - , isCycling = this.interval - , direction = type == 'next' ? 'left' : 'right' - , fallback = type == 'next' ? 'first' : 'last' - , that = this - , e - - this.sliding = true - - isCycling && this.pause() - - $next = $next.length ? $next : this.$element.find('.item')[fallback]() - - e = $.Event('slide', { - relatedTarget: $next[0] - , direction: direction - }) - - if ($next.hasClass('active')) return - - if (this.$indicators.length) { - this.$indicators.find('.active').removeClass('active') - this.$element.one('slid', function () { - var $nextIndicator = $(that.$indicators.children()[that.getActiveIndex()]) - $nextIndicator && $nextIndicator.addClass('active') - }) - } - - if ($.support.transition && this.$element.hasClass('slide')) { - this.$element.trigger(e) - if (e.isDefaultPrevented()) return - $next.addClass(type) - $next[0].offsetWidth // force reflow - $active.addClass(direction) - $next.addClass(direction) - this.$element.one($.support.transition.end, function () { - $next.removeClass([type, direction].join(' ')).addClass('active') - $active.removeClass(['active', direction].join(' ')) - that.sliding = false - setTimeout(function () { that.$element.trigger('slid') }, 0) - }) - } else { - this.$element.trigger(e) - if (e.isDefaultPrevented()) return - $active.removeClass('active') - $next.addClass('active') - this.sliding = false - this.$element.trigger('slid') - } - - isCycling && this.cycle() - - return this - } - - } - - - /* CAROUSEL PLUGIN DEFINITION - * ========================== */ - - var old = $.fn.carousel - - $.fn.carousel = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('carousel') - , options = $.extend({}, $.fn.carousel.defaults, typeof option == 'object' && option) - , action = typeof option == 'string' ? option : options.slide - if (!data) $this.data('carousel', (data = new Carousel(this, options))) - if (typeof option == 'number') data.to(option) - else if (action) data[action]() - else if (options.interval) data.pause().cycle() - }) - } - - $.fn.carousel.defaults = { - interval: 5000 - , pause: 'hover' - } - - $.fn.carousel.Constructor = Carousel - - - /* CAROUSEL NO CONFLICT - * ==================== */ - - $.fn.carousel.noConflict = function () { - $.fn.carousel = old - return this - } - - /* CAROUSEL DATA-API - * ================= */ - - $(document).on('click.carousel.data-api', '[data-slide], [data-slide-to]', function (e) { - var $this = $(this), href - , $target = $($this.attr('data-target') || (href = $this.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '')) //strip for ie7 - , options = $.extend({}, $target.data(), $this.data()) - , slideIndex - - $target.carousel(options) - - if (slideIndex = $this.attr('data-slide-to')) { - $target.data('carousel').pause().to(slideIndex).cycle() - } - - e.preventDefault() - }) - -}(window.jQuery); -/* ============================================================= - * bootstrap-typeahead.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#typeahead - * ============================================================= - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================ */ - - -!function($){ - - "use strict"; // jshint ;_; - - - /* TYPEAHEAD PUBLIC CLASS DEFINITION - * ================================= */ - - var Typeahead = function (element, options) { - this.$element = $(element) - this.options = $.extend({}, $.fn.typeahead.defaults, options) - this.matcher = this.options.matcher || this.matcher - this.sorter = this.options.sorter || this.sorter - this.highlighter = this.options.highlighter || this.highlighter - this.updater = this.options.updater || this.updater - this.source = this.options.source - this.$menu = $(this.options.menu) - this.shown = false - this.listen() - } - - Typeahead.prototype = { - - constructor: Typeahead - - , select: function () { - var val = this.$menu.find('.active').attr('data-value') - this.$element - .val(this.updater(val)) - .change() - return this.hide() - } - - , updater: function (item) { - return item - } - - , show: function () { - var pos = $.extend({}, this.$element.position(), { - height: this.$element[0].offsetHeight - }) - - this.$menu - .insertAfter(this.$element) - .css({ - top: pos.top + pos.height - , left: pos.left - }) - .show() - - this.shown = true - return this - } - - , hide: function () { - this.$menu.hide() - this.shown = false - return this - } - - , lookup: function (event) { - var items - - this.query = this.$element.val() - - if (!this.query || this.query.length < this.options.minLength) { - return this.shown ? this.hide() : this - } - - items = $.isFunction(this.source) ? this.source(this.query, $.proxy(this.process, this)) : this.source - - return items ? this.process(items) : this - } - - , process: function (items) { - var that = this - - items = $.grep(items, function (item) { - return that.matcher(item) - }) - - items = this.sorter(items) - - if (!items.length) { - return this.shown ? this.hide() : this - } - - return this.render(items.slice(0, this.options.items)).show() - } - - , matcher: function (item) { - return ~item.toLowerCase().indexOf(this.query.toLowerCase()) - } - - , sorter: function (items) { - var beginswith = [] - , caseSensitive = [] - , caseInsensitive = [] - , item - - while (item = items.shift()) { - if (!item.toLowerCase().indexOf(this.query.toLowerCase())) beginswith.push(item) - else if (~item.indexOf(this.query)) caseSensitive.push(item) - else caseInsensitive.push(item) - } - - return beginswith.concat(caseSensitive, caseInsensitive) - } - - , highlighter: function (item) { - var query = this.query.replace(/[\-\[\]{}()*+?.,\\\^$|#\s]/g, '\\$&') - return item.replace(new RegExp('(' + query + ')', 'ig'), function ($1, match) { - return '' + match + '' - }) - } - - , render: function (items) { - var that = this - - items = $(items).map(function (i, item) { - i = $(that.options.item).attr('data-value', item) - i.find('a').html(that.highlighter(item)) - return i[0] - }) - - items.first().addClass('active') - this.$menu.html(items) - return this - } - - , next: function (event) { - var active = this.$menu.find('.active').removeClass('active') - , next = active.next() - - if (!next.length) { - next = $(this.$menu.find('li')[0]) - } - - next.addClass('active') - } - - , prev: function (event) { - var active = this.$menu.find('.active').removeClass('active') - , prev = active.prev() - - if (!prev.length) { - prev = this.$menu.find('li').last() - } - - prev.addClass('active') - } - - , listen: function () { - this.$element - .on('focus', $.proxy(this.focus, this)) - .on('blur', $.proxy(this.blur, this)) - .on('keypress', $.proxy(this.keypress, this)) - .on('keyup', $.proxy(this.keyup, this)) - - if (this.eventSupported('keydown')) { - this.$element.on('keydown', $.proxy(this.keydown, this)) - } - - this.$menu - .on('click', $.proxy(this.click, this)) - .on('mouseenter', 'li', $.proxy(this.mouseenter, this)) - .on('mouseleave', 'li', $.proxy(this.mouseleave, this)) - } - - , eventSupported: function(eventName) { - var isSupported = eventName in this.$element - if (!isSupported) { - this.$element.setAttribute(eventName, 'return;') - isSupported = typeof this.$element[eventName] === 'function' - } - return isSupported - } - - , move: function (e) { - if (!this.shown) return - - switch(e.keyCode) { - case 9: // tab - case 13: // enter - case 27: // escape - e.preventDefault() - break - - case 38: // up arrow - e.preventDefault() - this.prev() - break - - case 40: // down arrow - e.preventDefault() - this.next() - break - } - - e.stopPropagation() - } - - , keydown: function (e) { - this.suppressKeyPressRepeat = ~$.inArray(e.keyCode, [40,38,9,13,27]) - this.move(e) - } - - , keypress: function (e) { - if (this.suppressKeyPressRepeat) return - this.move(e) - } - - , keyup: function (e) { - switch(e.keyCode) { - case 40: // down arrow - case 38: // up arrow - case 16: // shift - case 17: // ctrl - case 18: // alt - break - - case 9: // tab - case 13: // enter - if (!this.shown) return - this.select() - break - - case 27: // escape - if (!this.shown) return - this.hide() - break - - default: - this.lookup() - } - - e.stopPropagation() - e.preventDefault() - } - - , focus: function (e) { - this.focused = true - } - - , blur: function (e) { - this.focused = false - if (!this.mousedover && this.shown) this.hide() - } - - , click: function (e) { - e.stopPropagation() - e.preventDefault() - this.select() - this.$element.focus() - } - - , mouseenter: function (e) { - this.mousedover = true - this.$menu.find('.active').removeClass('active') - $(e.currentTarget).addClass('active') - } - - , mouseleave: function (e) { - this.mousedover = false - if (!this.focused && this.shown) this.hide() - } - - } - - - /* TYPEAHEAD PLUGIN DEFINITION - * =========================== */ - - var old = $.fn.typeahead - - $.fn.typeahead = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('typeahead') - , options = typeof option == 'object' && option - if (!data) $this.data('typeahead', (data = new Typeahead(this, options))) - if (typeof option == 'string') data[option]() - }) - } - - $.fn.typeahead.defaults = { - source: [] - , items: 8 - , menu: '
      ' - , item: '
      • ' - , minLength: 1 - } - - $.fn.typeahead.Constructor = Typeahead - - - /* TYPEAHEAD NO CONFLICT - * =================== */ - - $.fn.typeahead.noConflict = function () { - $.fn.typeahead = old - return this - } - - - /* TYPEAHEAD DATA-API - * ================== */ - - $(document).on('focus.typeahead.data-api', '[data-provide="typeahead"]', function (e) { - var $this = $(this) - if ($this.data('typeahead')) return - $this.typeahead($this.data()) - }) - -}(window.jQuery); diff --git a/module/web/app/scripts/vendor/jquery.omniwindow.js b/module/web/app/scripts/vendor/jquery.omniwindow.js deleted file mode 100644 index e1f0b8f77..000000000 --- a/module/web/app/scripts/vendor/jquery.omniwindow.js +++ /dev/null @@ -1,141 +0,0 @@ -// jQuery OmniWindow plugin -// @version: 0.7.0 -// @author: Rudenka Alexander (mur.mailbox@gmail.com) -// @license: MIT - -;(function($) { - "use strict"; - $.fn.extend({ - omniWindow: function(options) { - - options = $.extend(true, { - animationsPriority: { - show: ['overlay', 'modal'], - hide: ['modal', 'overlay'] - }, - overlay: { - selector: '.ow-overlay', - hideClass: 'ow-closed', - animations: { - show: function(subjects, internalCallback) { return internalCallback(subjects); }, - hide: function(subjects, internalCallback) { return internalCallback(subjects); }, - internal: { - show: function(subjects){ subjects.overlay.removeClass(options.overlay.hideClass); }, - hide: function(subjects){ subjects.overlay.addClass(options.overlay.hideClass); } - } - } - }, - modal: { - hideClass: 'ow-closed', - animations: { - show: function(subjects, internalCallback) { return internalCallback(subjects); }, - hide: function(subjects, internalCallback) { return internalCallback(subjects); }, - internal: { - show: function(subjects){ subjects.modal.removeClass(options.modal.hideClass); }, - hide: function(subjects){ subjects.modal.addClass(options.modal.hideClass); } - } - }, - internal: { - stateAttribute: 'ow-active' - } - }, - eventsNames: { - show: 'show.ow', - hide: 'hide.ow', - internal: { - overlayClick: 'click.ow', - keyboardKeyUp: 'keyup.ow' - } - }, - callbacks: { // Callbacks execution chain - beforeShow: function(subjects, internalCallback) { return internalCallback(subjects); }, // 1 (stop if retruns false) - positioning: function(subjects, internalCallback) { return internalCallback(subjects); }, // 2 - afterShow: function(subjects, internalCallback) { return internalCallback(subjects); }, // 3 - beforeHide: function(subjects, internalCallback) { return internalCallback(subjects); }, // 4 (stop if retruns false) - afterHide: function(subjects, internalCallback) { return internalCallback(subjects); }, // 5 - internal: { - beforeShow: function(subjects) { - if (subjects.modal.data(options.modal.internal.stateAttribute)) { - return false; - } else { - subjects.modal.data(options.modal.internal.stateAttribute, true); - return true; - } - }, - afterShow: function(subjects) { - $(document).on(options.eventsNames.internal.keyboardKeyUp, function(e) { - if (e.keyCode === 27) { // if the key pressed is the ESC key - subjects.modal.trigger(options.eventsNames.hide); - } - }); - - subjects.overlay.on(options.eventsNames.internal.overlayClick, function(){ - subjects.modal.trigger(options.eventsNames.hide); - }); - }, - positioning: function(subjects) { - subjects.modal.css('margin-left', Math.round(subjects.modal.outerWidth() / -2)); - }, - beforeHide: function(subjects) { - if (subjects.modal.data(options.modal.internal.stateAttribute)) { - subjects.modal.data(options.modal.internal.stateAttribute, false); - return true; - } else { - return false; - } - }, - afterHide: function(subjects) { - subjects.overlay.off(options.eventsNames.internal.overlayClick); - $(document).off(options.eventsNames.internal.keyboardKeyUp); - - subjects.overlay.css('display', ''); // clear inline styles after jQ animations - subjects.modal.css('display', ''); - } - } - } - }, options); - - var animate = function(process, subjects, callbackName) { - var first = options.animationsPriority[process][0], - second = options.animationsPriority[process][1]; - - options[first].animations[process](subjects, function(subjs) { // call USER's FIRST animation (depends on priority) - options[first].animations.internal[process](subjs); // call internal FIRST animation - - options[second].animations[process](subjects, function(subjs) { // call USER's SECOND animation - options[second].animations.internal[process](subjs); // call internal SECOND animation - - // then we need to call USER's - // afterShow of afterHide callback - options.callbacks[callbackName](subjects, options.callbacks.internal[callbackName]); - }); - }); - }; - - var showModal = function(subjects) { - if (!options.callbacks.beforeShow(subjects, options.callbacks.internal.beforeShow)) { return; } // cancel showing if beforeShow callback return false - - options.callbacks.positioning(subjects, options.callbacks.internal.positioning); - - animate('show', subjects, 'afterShow'); - }; - - var hideModal = function(subjects) { - if (!options.callbacks.beforeHide(subjects, options.callbacks.internal.beforeHide)) { return; } // cancel hiding if beforeHide callback return false - - animate('hide', subjects, 'afterHide'); - }; - - - var $overlay = $(options.overlay.selector); - - return this.each(function() { - var $modal = $(this); - var subjects = {modal: $modal, overlay: $overlay}; - - $modal.bind(options.eventsNames.show, function(){ showModal(subjects); }) - .bind(options.eventsNames.hide, function(){ hideModal(subjects); }); - }); - } - }); -})(jQuery); \ No newline at end of file diff --git a/module/web/app/scripts/vendor/remaining.js b/module/web/app/scripts/vendor/remaining.js deleted file mode 100644 index d66a2931a..000000000 --- a/module/web/app/scripts/vendor/remaining.js +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Javascript Countdown - * Copyright (c) 2009 Markus Hedlund - * Version 1.1 - * Licensed under MIT license - * http://www.opensource.org/licenses/mit-license.php - * http://labs.mimmin.com/countdown - */ -define([], function() { - var remaining = { - /** - * Get the difference of the passed date, and now. The different formats of the taget parameter are: - * January 12, 2009 15:14:00 (Month dd, yyyy hh:mm:ss) - * January 12, 2009 (Month dd, yyyy) - * 09,00,12,15,14,00 (yy,mm,dd,hh,mm,ss) Months range from 0-11, not 1-12. - * 09,00,12 (yy,mm,dd) Months range from 0-11, not 1-12. - * 500 (milliseconds) - * 2009-01-12 15:14:00 (yyyy-mm-dd hh-mm-ss) - * 2009-01-12 15:14 (yyyy-mm-dd hh-mm) - * @param target Target date. Can be either a date object or a string (formated like '24 December, 2010 15:00:00') - * @return Difference in seconds - */ - getSeconds: function(target) { - var today = new Date(); - - if (typeof(target) == 'object') { - var targetDate = target; - } else { - var matches = target.match(/(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2})(:(\d{2}))?/); // YYYY-MM-DD HH-MM-SS - if (matches != null) { - matches[7] = typeof(matches[7]) == 'undefined' ? '00' : matches[7]; - var targetDate = new Date(matches[1], matches[2] - 1, matches[3], matches[4], matches[5], matches[7]); - } else { - var targetDate = new Date(target); - } - } - - return Math.floor((targetDate.getTime() - today.getTime()) / 1000); - }, - - /** - * @param seconds Difference in seconds - * @param i18n A language object (see code) - * @param onlyLargestUnit Return only the largest unit (see documentation) - * @param hideEmpty Hide empty units (see documentation) - * @return String formated something like '1 week, 1 hours, 1 second' - */ - getString: function(seconds, i18n, onlyLargestUnit, hideEmpty) { - if (seconds < 1) { - return ''; - } - - if (typeof(hideEmpty) == 'undefined' || hideEmpty == null) { - hideEmpty = true; - } - if (typeof(onlyLargestUnit) == 'undefined' || onlyLargestUnit == null) { - onlyLargestUnit = false; - } - if (typeof(i18n) == 'undefined' || i18n == null) { - i18n = { - weeks: ['week', 'weeks'], - days: ['day', 'days'], - hours: ['hour', 'hours'], - minutes: ['minute', 'minutes'], - seconds: ['second', 'seconds'] - }; - } - - var units = { - weeks: 7 * 24 * 60 * 60, - days: 24 * 60 * 60, - hours: 60 * 60, - minutes: 60, - seconds: 1 - }; - - var returnArray = []; - var value; - for (unit in units) { - value = units[unit]; - if (seconds / value >= 1 || unit == 'seconds' || !hideEmpty) { - secondsConverted = Math.floor(seconds / value); - var i18nUnit = i18n[unit][secondsConverted == 1 ? 0 : 1]; - returnArray.push(secondsConverted + ' ' + i18nUnit); - seconds -= secondsConverted * value; - - if (onlyLargestUnit) { - break; - } - } - } - ; - - return returnArray.join(', '); - }, - - /** - * @param seconds Difference in seconds - * @return String formated something like '169:00:01' - */ - getStringDigital: function(seconds) { - if (seconds < 1) { - return ''; - } - - remainingTime = remaining.getArray(seconds); - - for (index in remainingTime) { - remainingTime[index] = remaining.padNumber(remainingTime[index]); - } - ; - - return remainingTime.join(':'); - }, - - /** - * @param seconds Difference in seconds - * @return Array with hours, minutes and seconds - */ - getArray: function(seconds) { - if (seconds < 1) { - return []; - } - - var units = [60 * 60, 60, 1]; - - var returnArray = []; - var value; - for (index in units) { - value = units[index]; - secondsConverted = Math.floor(seconds / value); - returnArray.push(secondsConverted); - seconds -= secondsConverted * value; - } - ; - - return returnArray; - }, - - /** - * @param number An integer - * @return Integer padded with a 0 if necessary - */ - padNumber: function(number) { - return (number >= 0 && number < 10) ? '0' + number : number; - } - }; - return remaining; -}); \ No newline at end of file diff --git a/module/web/app/scripts/views/abstract/itemView.js b/module/web/app/scripts/views/abstract/itemView.js deleted file mode 100644 index c37118a4c..000000000 --- a/module/web/app/scripts/views/abstract/itemView.js +++ /dev/null @@ -1,47 +0,0 @@ -define(['jquery', 'backbone', 'underscore'], function($, Backbone, _) { - 'use strict'; - - // A view that is meant for temporary displaying - // All events must be unbound in onDestroy - return Backbone.View.extend({ - - tagName: 'li', - destroy: function() { - this.undelegateEvents(); - this.unbind(); - if (this.onDestroy) { - this.onDestroy(); - } - this.$el.removeData().unbind(); - this.remove(); - }, - - hide: function() { - this.$el.slideUp(); - }, - - show: function() { - this.$el.slideDown(); - }, - - unrender: function() { - var self = this; - this.$el.slideUp(function() { - self.destroy(); - }); - }, - - deleteItem: function(e) { - if (e) - e.stopPropagation(); - this.model.destroy(); - }, - - restart: function(e) { - if(e) - e.stopPropagation(); - this.model.restart(); - } - - }); -}); \ No newline at end of file diff --git a/module/web/app/scripts/views/abstract/modalView.js b/module/web/app/scripts/views/abstract/modalView.js deleted file mode 100644 index 9d1d72869..000000000 --- a/module/web/app/scripts/views/abstract/modalView.js +++ /dev/null @@ -1,125 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'omniwindow'], function($, Backbone, _) { - 'use strict'; - - return Backbone.View.extend({ - - events: { - 'click .btn-confirm': 'confirm', - 'click .btn-close': 'hide', - 'click .close': 'hide' - }, - - template: null, - dialog: null, - - onHideDestroy: false, - confirmCallback: null, - - initialize: function(template, confirm) { - this.confirmCallback = confirm; - var self = this; - if (this.template === null) { - if (template) { - this.template = template; - // When template was provided this is a temporary dialog - this.onHideDestroy = true; - } - else - require(['text!tpl/default/modal.html'], function(template) { - self.template = template; - }); - } - - }, - - // TODO: whole modal stuff is not very elegant - render: function() { - this.$el.html(this.template(this.renderContent())); - this.onRender(); - - if (this.dialog === null) { - this.$el.addClass('modal hide'); - this.$el.css({opacity: 0, scale: 0.7}); - - var self = this; - $('body').append(this.el); - this.dialog = this.$el.omniWindow({ - overlay: { - selector: '#modal-overlay', - hideClass: 'hide', - animations: { - hide: function(subjects, internalCallback) { - subjects.overlay.transition({opacity: 'hide', delay: 100}, 300, function() { - internalCallback(subjects); - self.onHide(); - if (self.onHideDestroy) - self.destroy(); - }); - }, - show: function(subjects, internalCallback) { - subjects.overlay.fadeIn(300); - internalCallback(subjects); - }}}, - modal: { - hideClass: 'hide', - animations: { - hide: function(subjects, internalCallback) { - subjects.modal.transition({opacity: 'hide', scale: 0.7}, 300); - internalCallback(subjects); - }, - - show: function(subjects, internalCallback) { - subjects.modal.transition({opacity: 'show', scale: 1, delay: 100}, 300, function() { - internalCallback(subjects); - }); - }} - }}); - } - - return this; - }, - - onRender: function() { - - }, - - renderContent: function() { - return {}; - }, - - show: function() { - if (this.dialog === null) - this.render(); - - this.dialog.trigger('show'); - - this.onShow(); - }, - - onShow: function() { - - }, - - hide: function() { - this.dialog.trigger('hide'); - }, - - onHide: function() { - - }, - - confirm: function() { - if (this.confirmCallback) - this.confirmCallback.apply(); - - this.hide(); - }, - - destroy: function() { - this.$el.remove(); - this.dialog = null; - this.remove(); - } - - }); -}); \ No newline at end of file diff --git a/module/web/app/scripts/views/accounts/accountListView.js b/module/web/app/scripts/views/accounts/accountListView.js deleted file mode 100644 index 4eb5bfe7d..000000000 --- a/module/web/app/scripts/views/accounts/accountListView.js +++ /dev/null @@ -1,52 +0,0 @@ -define(['jquery', 'underscore', 'backbone', 'app', 'collections/AccountList', './accountView', - 'hbs!tpl/accounts/layout', 'hbs!tpl/accounts/actionbar'], - function($, _, Backbone, App, AccountList, accountView, template, templateBar) { - 'use strict'; - - // Renders settings over view page - return Backbone.Marionette.CollectionView.extend({ - - itemView: accountView, - template: template, - - collection: null, - modal: null, - - initialize: function() { - this.actionbar = Backbone.Marionette.ItemView.extend({ - template: templateBar, - events: { - 'click .btn': 'addAccount' - }, - addAccount: _.bind(this.addAccount, this) - }); - - this.collection = new AccountList(); - this.update(); - - this.listenTo(App.vent, 'accounts:updated', this.update); - }, - - update: function() { - this.collection.fetch(); - }, - - onBeforeRender: function() { - this.$el.html(template()); - }, - - appendHtml: function(collectionView, itemView, index) { - this.$('.account-list').append(itemView.el); - }, - - addAccount: function() { - var self = this; - _.requireOnce(['views/accounts/accountModal'], function(Modal) { - if (self.modal === null) - self.modal = new Modal(); - - self.modal.show(); - }); - } - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/views/accounts/accountModal.js b/module/web/app/scripts/views/accounts/accountModal.js deleted file mode 100644 index 6c2b226df..000000000 --- a/module/web/app/scripts/views/accounts/accountModal.js +++ /dev/null @@ -1,72 +0,0 @@ -define(['jquery', 'underscore', 'app', 'views/abstract/modalView', 'hbs!tpl/dialogs/addAccount', 'helpers/pluginIcon', 'select2'], - function($, _, App, modalView, template, pluginIcon) { - 'use strict'; - return modalView.extend({ - - events: { - 'submit form': 'add', - 'click .btn-add': 'add' - }, - template: template, - plugins: null, - select: null, - - initialize: function() { - // Inherit parent events - this.events = _.extend({}, modalView.prototype.events, this.events); - var self = this; - $.ajax(App.apiRequest('getAccountTypes', null, {success: function(data) { - self.plugins = _.sortBy(data, function(item) { - return item; - }); - self.render(); - }})); - }, - - onRender: function() { - // TODO: could be a separate input type if needed on multiple pages - if (this.plugins) - this.select = this.$('#pluginSelect').select2({ - escapeMarkup: function(m) { - return m; - }, - formatResult: this.format, - formatSelection: this.format, - data: {results: this.plugins, text: function(item) { - return item; - }}, - id: function(item) { - return item; - } - }); - }, - - onShow: function() { - }, - - onHide: function() { - }, - - format: function(data) { - return ' ' + data; - }, - - add: function(e) { - e.stopPropagation(); - if (this.select) { - var plugin = this.select.val(), - login = this.$('#login').val(), - password = this.$('#password').val(), - self = this; - - $.ajax(App.apiRequest('updateAccount', { - plugin: plugin, login: login, password: password - }, { success: function() { - App.vent.trigger('accounts:updated'); - self.hide(); - }})); - } - return false; - } - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/views/accounts/accountView.js b/module/web/app/scripts/views/accounts/accountView.js deleted file mode 100644 index 89f69d7e7..000000000 --- a/module/web/app/scripts/views/accounts/accountView.js +++ /dev/null @@ -1,18 +0,0 @@ -define(['jquery', 'underscore', 'backbone', 'app', 'hbs!tpl/accounts/account'], - function($, _, Backbone, App, template) { - 'use strict'; - - return Backbone.Marionette.ItemView.extend({ - - tagName: 'tr', - template: template, - - events: { - 'click .btn-danger': 'deleteAccount' - }, - - deleteAccount: function() { - this.model.destroy(); - } - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/views/dashboard/dashboardView.js b/module/web/app/scripts/views/dashboard/dashboardView.js deleted file mode 100644 index e7adba475..000000000 --- a/module/web/app/scripts/views/dashboard/dashboardView.js +++ /dev/null @@ -1,168 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'models/TreeCollection', - './packageView', './fileView', 'hbs!tpl/dashboard/layout', 'select2'], - function($, Backbone, _, App, TreeCollection, PackageView, FileView, template) { - 'use strict'; - // Renders whole dashboard - return Backbone.Marionette.ItemView.extend({ - - // TODO: refactor - active: $('.breadcrumb .active'), - - template: template, - - events: { - }, - - ui: { - 'packages': '.package-list', - 'files': '.file-list' - }, - - // Package tree - tree: null, - // Current open files - files: null, - // True when loading animation is running - isLoading: false, - - initialize: function() { - App.dashboard = this; - this.tree = new TreeCollection(); - - var self = this; - // When package is added we reload the data - App.vent.on('package:added', function() { - console.log('Package tree caught, package:added event'); - self.tree.fetch(); - }); - - App.vent.on('file:updated', _.bind(this.fileUpdated, this)); - - // TODO: merge? - this.init(); - // TODO: file:added - // TODO: package:deleted - // TODO: package:updated - }, - - init: function() { - var self = this; - // TODO: put in separated function - // TODO: order of elements? - // Init the tree and callback for package added - this.tree.fetch({success: function() { - self.update(); - self.tree.get('packages').on('add', function(pack) { - console.log('Package ' + pack.get('pid') + ' added to tree'); - self.appendPackage(pack, 0, true); - self.openPackage(pack); - }); - }}); - - this.$('.input').select2({tags: ['a', 'b', 'sdf']}); - }, - - update: function() { - console.log('Update package list'); - - // TODO: Both null - var packs = this.tree.get('packages'); - this.files = this.tree.get('files'); - - if (packs) - packs.each(_.bind(this.appendPackage, this)); - - if (!this.files || this.files.length === 0) { - // no files are displayed - this.files = null; - // Open the first package - if (packs && packs.length >= 1) - this.openPackage(packs.at(0)); - } - else - this.files.each(_.bind(this.appendFile, this)); - - return this; - }, - - // TODO sorting ?! - // Append a package to the list, index, animate it - appendPackage: function(pack, i, animation) { - var el = new PackageView({model: pack}).render().el; - $(this.ui.packages).appendWithAnimation(el, animation); - }, - - appendFile: function(file, i, animation) { - var el = new FileView({model: file}).render().el; - $(this.ui.files).appendWithAnimation(el, animation); - }, - - // Show content of the packages on main view - openPackage: function(pack) { - var self = this; - - // load animation only when something is shown and its different from current package - if (this.files && this.files !== pack.get('files')) - self.loading(); - - pack.fetch({silent: true, success: function() { - console.log('Package ' + pack.get('pid') + ' loaded'); - self.active.text(pack.get('name')); - self.contentReady(pack.get('files')); - }, failure: function() { - self.failure(); - }}); - - }, - - contentReady: function(files) { - var old_files = this.files; - this.files = files; - App.vent.trigger('dashboard:contentReady'); - - // show the files when no loading animation is running and not already open - if (!this.isLoading && old_files !== files) - this.show(); - }, - - // Do load animation, remove the old stuff - loading: function() { - this.isLoading = true; - this.files = null; - var self = this; - $(this.ui.files).fadeOut({complete: function() { - // All file views should vanish - App.vent.trigger('dashboard:destroyContent'); - - // Loading was faster than animation - if (self.files) - self.show(); - - self.isLoading = false; - }}); - }, - - failure: function() { - // TODO - }, - - show: function() { - // fileUL has to be resetted before - this.files.each(_.bind(this.appendFile, this)); - //TODO: show placeholder when nothing is displayed (filtered content empty) - $(this.ui.files).fadeIn(); - App.vent.trigger('dashboard:updated'); - }, - - // Refresh the file if it is currently shown - fileUpdated: function(data) { - // this works with ids and object - var file = this.files.get(data); - if (file) - if (_.isObject(data)) // update directly - file.set(data); - else // fetch from server - file.fetch(); - } - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/views/dashboard/fileView.js b/module/web/app/scripts/views/dashboard/fileView.js deleted file mode 100644 index 4e5884ed8..000000000 --- a/module/web/app/scripts/views/dashboard/fileView.js +++ /dev/null @@ -1,102 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes', 'views/abstract/itemView', 'helpers/formatTime', 'hbs!tpl/dashboard/file'], - function($, Backbone, _, App, Api, ItemView, formatTime, template) { - 'use strict'; - - // Renders single file item - return ItemView.extend({ - - tagName: 'li', - className: 'file-view row-fluid', - template: template, - events: { - 'click .checkbox': 'select', - 'click .btn-delete': 'deleteItem', - 'click .btn-restart': 'restart' - }, - - initialize: function() { - this.listenTo(this.model, 'change', this.render); - // This will be triggered manually and changed before with silent=true - this.listenTo(this.model, 'change:visible', this.visibility_changed); - this.listenTo(this.model, 'change:progress', this.progress_changed); - this.listenTo(this.model, 'remove', this.unrender); - this.listenTo(App.vent, 'dashboard:destroyContent', this.destroy); - }, - - onDestroy: function() { - }, - - render: function() { - var data = this.model.toJSON(); - if (data.download) { - var status = data.download.status; - if (status === Api.DownloadStatus.Offline || status === Api.DownloadStatus.TempOffline) - data.offline = true; - else if (status === Api.DownloadStatus.Online) - data.online = true; - else if (status === Api.DownloadStatus.Waiting) - data.waiting = true; - else if (status === Api.DownloadStatus.Downloading) - data.downloading = true; - else if (this.model.isFailed()) - data.failed = true; - else if (this.model.isFinished()) - data.finished = true; - } - - this.$el.html(this.template(data)); - if (this.model.get('selected')) - this.$el.addClass('ui-selected'); - else - this.$el.removeClass('ui-selected'); - - if (this.model.get('visible')) - this.$el.show(); - else - this.$el.hide(); - - return this; - }, - - select: function(e) { - e.preventDefault(); - var checked = this.$el.hasClass('ui-selected'); - // toggle class immediately, so no re-render needed - this.model.set('selected', !checked, {silent: true}); - this.$el.toggleClass('ui-selected'); - App.vent.trigger('file:selection'); - }, - - visibility_changed: function(visible) { - // TODO: improve animation, height is not available when element was not visible - if (visible) - this.$el.slideOut(true); - else { - this.$el.calculateHeight(true); - this.$el.slideIn(true); - } - }, - - progress_changed: function() { - if (!this.model.isDownload()) - return; - - if (this.model.get('download').status === Api.DownloadStatus.Downloading) { - var bar = this.$('.progress .bar'); - if (!bar) { // ensure that the dl bar is rendered - this.render(); - bar = this.$('.progress .bar'); - } - - bar.width(this.model.get('progress') + '%'); - bar.html('  ' + formatTime(this.model.get('eta'))); - } else if (this.model.get('download').status === Api.DownloadStatus.Waiting) { - this.$('.second').html( - ' ' + formatTime(this.model.get('eta'))); - - } else // Every else state can be renderred normally - this.render(); - - } - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/views/dashboard/filterView.js b/module/web/app/scripts/views/dashboard/filterView.js deleted file mode 100644 index 64bc56724..000000000 --- a/module/web/app/scripts/views/dashboard/filterView.js +++ /dev/null @@ -1,133 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes', 'models/Package', 'hbs!tpl/dashboard/actionbar'], - /*jslint -W040: false*/ - function($, Backbone, _, App, Api, Package, template) { - 'use strict'; - - // Modified version of type ahead show, nearly the same without absolute positioning - function show() { - this.$menu - .insertAfter(this.$element) - .show(); - - this.shown = true; - return this; - } - - // Renders the actionbar for the dashboard, handles everything related to filtering displayed files - return Backbone.Marionette.ItemView.extend({ - - events: { - 'click .filter-type': 'filter_type', - 'click .filter-state': 'switch_filter', - 'submit .form-search': 'search' - }, - - ui: { - 'search': '.search-query', - 'stateMenu': '.dropdown-toggle .state' - }, - - template: template, - state: null, - - initialize: function() { - this.state = Api.DownloadState.All; - - // Apply the filter before the content is shown - App.vent.on('dashboard:contentReady', _.bind(this.apply_filter, this)); - }, - - onRender: function() { - // use our modified method - $.fn.typeahead.Constructor.prototype.show = show; - this.ui.search.typeahead({ - minLength: 2, - source: this.getSuggestions - }); - - }, - - // TODO: app level api request - search: function(e) { - e.stopPropagation(); - var query = this.ui.search.val(); - this.ui.search.val(''); - - var pack = new Package(); - // Overwrite fetch method to use a search - // TODO: quite hackish, could be improved to filter packages - // or show performed search - pack.fetch = function(options) { - pack.search(query, options); - }; - - App.dashboard.openPackage(pack); - }, - - getSuggestions: function(query, callback) { - $.ajax(App.apiRequest('searchSuggestions', {pattern: query}, { - method: 'POST', - success: function(data) { - callback(data); - } - })); - }, - - switch_filter: function(e) { - e.stopPropagation(); - var element = $(e.target); - var state = parseInt(element.data('state'), 10); - var menu = this.ui.stateMenu.parent().parent(); - menu.removeClass('open'); - - if (state === Api.DownloadState.Finished) { - menu.removeClass().addClass('dropdown finished'); - } else if (state === Api.DownloadState.Unfinished) { - menu.removeClass().addClass('dropdown active'); - } else if (state === Api.DownloadState.Failed) { - menu.removeClass().addClass('dropdown failed'); - } else { - menu.removeClass().addClass('dropdown'); - } - - this.state = state; - this.ui.stateMenu.text(element.text()); - this.apply_filter(); - }, - - // Applies the filtering to current open files - apply_filter: function() { - if (!App.dashboard.files) - return; - - var self = this; - App.dashboard.files.map(function(file) { - var visible = file.get('visible'); - if (visible !== self.is_visible(file)) { - file.set('visible', !visible, {silent: true}); - file.trigger('change:visible', !visible); - } - }); - - App.vent.trigger('dashboard:filtered'); - }, - - // determine if a file should be visible - // TODO: non download files - is_visible: function(file) { - if (this.state === Api.DownloadState.Finished) - return file.isFinished(); - else if (this.state === Api.DownloadState.Unfinished) - return file.isUnfinished(); - else if (this.state === Api.DownloadState.Failed) - return file.isFailed(); - - return true; - }, - - filter_type: function(e) { - - } - - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/views/dashboard/packageView.js b/module/web/app/scripts/views/dashboard/packageView.js deleted file mode 100644 index 2738fcbea..000000000 --- a/module/web/app/scripts/views/dashboard/packageView.js +++ /dev/null @@ -1,75 +0,0 @@ -define(['jquery', 'app', 'views/abstract/itemView', 'underscore', 'hbs!tpl/dashboard/package'], - function($, App, itemView, _, template) { - 'use strict'; - - // Renders a single package item - return itemView.extend({ - - tagName: 'li', - className: 'package-view', - template: template, - events: { - 'click .package-name, .btn-open': 'open', - 'click .icon-refresh': 'restart', - 'click .select': 'select', - 'click .btn-delete': 'deleteItem' - }, - - // Ul for child packages (unused) - ul: null, - // Currently unused - expanded: false, - - initialize: function() { - this.listenTo(this.model, 'filter:added', this.hide); - this.listenTo(this.model, 'filter:removed', this.show); - this.listenTo(this.model, 'change', this.render); - this.listenTo(this.model, 'remove', this.unrender); - - // Clear drop down menu - var self = this; - this.$el.on('mouseleave', function() { - self.$('.dropdown-menu').parent().removeClass('open'); - }); - }, - - onDestroy: function() { - }, - - // Render everything, optional only the fileViews - render: function() { - this.$el.html(this.template(this.model.toJSON())); - this.$el.initTooltips(); - - return this; - }, - - unrender: function() { - itemView.prototype.unrender.apply(this); - - // TODO: display other package - App.vent.trigger('dashboard:loading', null); - }, - - - // TODO - // Toggle expanding of packages - expand: function(e) { - e.preventDefault(); - }, - - open: function(e) { - e.preventDefault(); - App.dashboard.openPackage(this.model); - }, - - select: function(e) { - e.preventDefault(); - var checked = this.$('.select').hasClass('icon-check'); - // toggle class immediately, so no re-render needed - this.model.set('selected', !checked, {silent: true}); - this.$('.select').toggleClass('icon-check').toggleClass('icon-check-empty'); - App.vent.trigger('package:selection'); - } - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/views/dashboard/selectionView.js b/module/web/app/scripts/views/dashboard/selectionView.js deleted file mode 100644 index 8685fd849..000000000 --- a/module/web/app/scripts/views/dashboard/selectionView.js +++ /dev/null @@ -1,155 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'hbs!tpl/dashboard/select'], - function($, Backbone, _, App, template) { - 'use strict'; - - // Renders context actions for selection packages and files - return Backbone.Marionette.ItemView.extend({ - - el: '#selection-area', - template: template, - - events: { - 'click .icon-check': 'deselect', - 'click .icon-pause': 'pause', - 'click .icon-trash': 'trash', - 'click .icon-refresh': 'restart' - }, - - // Element of the action bar - actionBar: null, - // number of currently selected elements - current: 0, - - initialize: function() { - this.$el.calculateHeight().height(0); - var render = _.bind(this.render, this); - - App.vent.on('dashboard:updated', render); - App.vent.on('dashboard:filtered', render); - App.vent.on('package:selection', render); - App.vent.on('file:selection', render); - - this.actionBar = $('.actionbar .btn-check'); - this.actionBar.parent().click(_.bind(this.select_toggle, this)); - - // API events, maybe better to rely on internal ones? - App.vent.on('package:deleted', render); - App.vent.on('file:deleted', render); - }, - - get_files: function(all) { - var files = []; - if (App.dashboard.files) - if (all) - files = App.dashboard.files.where({visible: true}); - else - files = App.dashboard.files.where({selected: true, visible: true}); - - return files; - }, - - get_packs: function() { - if (!App.dashboard.tree.get('packages')) - return []; // TODO - - return App.dashboard.tree.get('packages').where({selected: true}); - }, - - render: function() { - var files = this.get_files().length; - var packs = this.get_packs().length; - - if (files + packs > 0) { - this.$el.html(this.template({files: files, packs: packs})); - this.$el.initTooltips('bottom'); - } - - if (files + packs > 0 && this.current === 0) - this.$el.slideOut(); - else if (files + packs === 0 && this.current > 0) - this.$el.slideIn(); - - // TODO: accessing ui directly, should be events - if (files > 0) { - this.actionBar.addClass('icon-check').removeClass('icon-check-empty'); - App.dashboard.ui.packages.addClass('ui-files-selected'); - } - else { - this.actionBar.addClass('icon-check-empty').removeClass('icon-check'); - App.dashboard.ui.packages.removeClass('ui-files-selected'); - } - - this.current = files + packs; - }, - - // Deselects all items - deselect: function() { - this.get_files().map(function(file) { - file.set('selected', false); - }); - - this.get_packs().map(function(pack) { - pack.set('selected', false); - }); - - this.render(); - }, - - pause: function() { - alert('Not implemented yet'); - this.deselect(); - }, - - trash: function() { - _.confirm('default/confirmDialog.html', function() { - - var pids = []; - // TODO: delete many at once - this.get_packs().map(function(pack) { - pids.push(pack.get('pid')); - pack.destroy(); - }); - - // get only the fids of non deleted packages - var fids = _.filter(this.get_files(),function(file) { - return !_.contains(pids, file.get('package')); - }).map(function(file) { - file.destroyLocal(); - return file.get('fid'); - }); - - if (fids.length > 0) - $.ajax(App.apiRequest('deleteFiles', {fids: fids})); - - this.deselect(); - }, this); - }, - - restart: function() { - this.get_files().map(function(file) { - file.restart(); - }); - this.get_packs().map(function(pack) { - pack.restart(); - }); - - this.deselect(); - }, - - // Select or deselect all visible files - select_toggle: function() { - var files = this.get_files(); - if (files.length === 0) { - this.get_files(true).map(function(file) { - file.set('selected', true); - }); - - } else - files.map(function(file) { - file.set('selected', false); - }); - - this.render(); - } - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/views/headerView.js b/module/web/app/scripts/views/headerView.js deleted file mode 100644 index 512c7259b..000000000 --- a/module/web/app/scripts/views/headerView.js +++ /dev/null @@ -1,240 +0,0 @@ -define(['jquery', 'underscore', 'backbone', 'app', 'models/ServerStatus', 'collections/ProgressList', - 'views/progressView', 'views/notificationView', 'helpers/formatSize', 'hbs!tpl/header/layout', - 'hbs!tpl/header/status', 'hbs!tpl/header/progressbar' , 'flot'], - function($, _, Backbone, App, ServerStatus, ProgressList, ProgressView, NotificationView, formatSize, - template, templateStatus, templateHeader) { - 'use strict'; - // Renders the header with all information - return Backbone.Marionette.ItemView.extend({ - - events: { - 'click .icon-list': 'toggle_taskList', - 'click .popover .close': 'toggle_taskList', - 'click .btn-grabber': 'open_grabber' - }, - - ui: { - progress: '.progress-list', - speedgraph: '#speedgraph' - }, - - // todo: maybe combine these - template: template, - templateStatus: templateStatus, - templateHeader: templateHeader, - - // view - grabber: null, - speedgraph: null, - - // models and data - ws: null, - status: null, - progressList: null, - speeds: null, - - // sub view - notificationView: null, - - // save if last progress was empty - wasEmpty: false, - - initialize: function() { - var self = this; - this.notificationView = new NotificationView(); - - this.status = new ServerStatus(); - this.listenTo(this.status, 'change', this.update); - - this.progressList = new ProgressList(); - this.listenTo(this.progressList, 'add', function(model) { - self.ui.progress.appendWithAnimation(new ProgressView({model: model}).render().el); - }); - - // TODO: button to start stop refresh - var ws = App.openWebSocket('/async'); - ws.onopen = function() { - ws.send(JSON.stringify('start')); - }; - // TODO compare with polling - ws.onmessage = _.bind(this.onData, this); - ws.onerror = function(error) { - console.log(error); - alert('WebSocket error' + error); - }; - - this.ws = ws; - }, - - initGraph: function() { - var totalPoints = 120; - var data = []; - - // init with empty data - while (data.length < totalPoints) - data.push([data.length, 0]); - - this.speeds = data; - this.speedgraph = $.plot(this.ui.speedgraph, [this.speeds], { - series: { - lines: { show: true, lineWidth: 2 }, - shadowSize: 0, - color: '#fee247' - }, - xaxis: { ticks: [] }, - yaxis: { ticks: [], min: 1, autoscaleMargin: 0.1, tickFormatter: function(data) { - return formatSize(data * 1024); - }, position: 'right' }, - grid: { - show: true, -// borderColor: "#757575", - borderColor: 'white', - borderWidth: 1, - labelMargin: 0, - axisMargin: 0, - minBorderMargin: 0 - } - }); - - }, - - // Must be called after view was attached - init: function() { - this.initGraph(); - this.update(); - }, - - update: function() { - // TODO: what should be displayed in the header - // queue/processing size? - - var status = this.status.toJSON(); - status.maxspeed = _.max(this.speeds, function(speed) { - return speed[1]; - })[1] * 1024; - this.$('.status-block').html( - this.templateStatus(status) - ); - - var data = {tasks: 0, downloads: 0, speed: 0, single: false}; - this.progressList.each(function(progress) { - if (progress.isDownload()) { - data.downloads += 1; - data.speed += progress.get('download').speed; - } else - data.tasks++; - }); - - // Show progress of one task - if (data.tasks + data.downloads === 1) { - var progress = this.progressList.at(0); - data.single = true; - data.eta = progress.get('eta'); - data.percent = progress.getPercent(); - data.name = progress.get('name'); - data.statusmsg = progress.get('statusmsg'); - } - // TODO: better progressbar rendering - - data.etaqueue = status.eta; - data.linksqueue = status.linksqueue; - data.sizequeue = status.sizequeue; - - this.$('#progress-info').html( - this.templateHeader(data) - ); - return this; - }, - - toggle_taskList: function() { - this.$('.popover').animate({opacity: 'toggle'}); - }, - - open_grabber: function() { - var self = this; - _.requireOnce(['views/linkGrabberModal'], function(ModalView) { - if (self.grabber === null) - self.grabber = new ModalView(); - - self.grabber.show(); - }); - }, - - onData: function(evt) { - var data = JSON.parse(evt.data); - if (data === null) return; - - if (data['@class'] === 'ServerStatus') { - // TODO: load interaction when none available - this.status.set(data); - - // There tasks at the server, but not in queue: so fetch them - // or there are tasks in our queue but not on the server - if (this.status.get('notifications') && !this.notificationView.tasks.hasTaskWaiting() || - !this.status.get('notifications') && this.notificationView.tasks.hasTaskWaiting()) - this.notificationView.tasks.fetch(); - - this.speeds = this.speeds.slice(1); - this.speeds.push([this.speeds[this.speeds.length - 1][0] + 1, Math.floor(data.speed / 1024)]); - - // TODO: if everything is 0 re-render is not needed - this.speedgraph.setData([this.speeds]); - // adjust the axis - this.speedgraph.setupGrid(); - this.speedgraph.draw(); - - } - else if (_.isArray(data)) - this.onProgressUpdate(data); - else if (data['@class'] === 'EventInfo') - this.onEvent(data.eventname, data.event_args); - else - console.log('Unknown Async input', data); - - }, - - onProgressUpdate: function(progress) { - // generate a unique id - _.each(progress, function(prog) { - if (prog.download) - prog.pid = prog.download.fid; - else - prog.pid = prog.plugin + prog.name; - }); - - this.progressList.set(progress); - // update currently open files with progress - this.progressList.each(function(prog) { - if (prog.isDownload() && App.dashboard.files) { - var file = App.dashboard.files.get(prog.get('download').fid); - if (file) { - file.set({ - progress: prog.getPercent(), - eta: prog.get('eta') - }, {silent: true}); - - file.trigger('change:progress'); - } - } - }); - - if (progress.length === 0) { - // only render one time when last was not empty already - if (!this.wasEmpty) { - this.update(); - this.wasEmpty = true; - } - } else { - this.wasEmpty = false; - this.update(); - } - }, - - onEvent: function(event, args) { - args.unshift(event); - console.log('Core send event', args); - App.vent.trigger.apply(App.vent, args); - } - - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/views/input/inputLoader.js b/module/web/app/scripts/views/input/inputLoader.js deleted file mode 100644 index 11665abb4..000000000 --- a/module/web/app/scripts/views/input/inputLoader.js +++ /dev/null @@ -1,8 +0,0 @@ -define(['./textInput'], function(textInput) { - 'use strict'; - - // selects appropriate input element - return function(input, value, default_value, description) { - return textInput; - }; -}); \ No newline at end of file diff --git a/module/web/app/scripts/views/input/inputView.js b/module/web/app/scripts/views/input/inputView.js deleted file mode 100644 index 1fbe5042d..000000000 --- a/module/web/app/scripts/views/input/inputView.js +++ /dev/null @@ -1,86 +0,0 @@ -define(['jquery', 'backbone', 'underscore'], function($, Backbone, _) { - 'use strict'; - - // Renders input elements - return Backbone.View.extend({ - - tagName: 'input', - - input: null, - value: null, - default_value: null, - description: null, - - // enables tooltips - tooltip: true, - - initialize: function(options) { - this.input = options.input; - this.value = options.value; - this.default_value = options.default_value; - this.description = options.description; - }, - - render: function() { - this.renderInput(); - // data for tooltips - if (this.description && this.tooltip) { - this.$el.data('content', this.description); - // TODO: render default value in popup? -// this.$el.data('title', "TODO: title"); - this.$el.popover({ - placement: 'right', - trigger: 'hover' -// delay: { show: 500, hide: 100 } - }); - } - - return this; - }, - - renderInput: function() { - // Overwrite this - }, - - showTooltip: function() { - if (this.description && this.tooltip) - this.$el.popover('show'); - }, - - hideTooltip: function() { - if (this.description && this.tooltip) - this.$el.popover('hide'); - }, - - destroy: function() { - this.undelegateEvents(); - this.unbind(); - if (this.onDestroy) { - this.onDestroy(); - } - this.$el.removeData().unbind(); - this.remove(); - }, - - // focus the input element - focus: function() { - this.$el.focus(); - }, - - // Clear the input - clear: function() { - - }, - - // retrieve value of the input - getVal: function() { - return this.value; - }, - - // the child class must call this when the value changed - setVal: function(value) { - this.value = value; - this.trigger('change', value); - } - }); -}); \ No newline at end of file diff --git a/module/web/app/scripts/views/input/textInput.js b/module/web/app/scripts/views/input/textInput.js deleted file mode 100644 index 0eebbf91e..000000000 --- a/module/web/app/scripts/views/input/textInput.js +++ /dev/null @@ -1,36 +0,0 @@ -define(['jquery', 'backbone', 'underscore', './inputView'], function($, Backbone, _, inputView) { - 'use strict'; - - return inputView.extend({ - - // TODO - tagName: 'input', - events: { - 'keyup': 'onChange', - 'focus': 'showTooltip', - 'focusout': 'hideTooltip' - }, - - renderInput: function() { - this.$el.attr('type', 'text'); - this.$el.attr('name', 'textInput'); - - if (this.default_value) - this.$el.attr('placeholder', this.default_value); - - if (this.value) - this.$el.val(this.value); - - return this; - }, - - clear: function() { - this.$el.val(''); - }, - - onChange: function(e) { - this.setVal(this.$el.val()); - } - - }); -}); \ No newline at end of file diff --git a/module/web/app/scripts/views/linkGrabberModal.js b/module/web/app/scripts/views/linkGrabberModal.js deleted file mode 100644 index e6f59c134..000000000 --- a/module/web/app/scripts/views/linkGrabberModal.js +++ /dev/null @@ -1,49 +0,0 @@ -define(['jquery', 'underscore', 'app', 'views/abstract/modalView', 'hbs!tpl/dialogs/linkgrabber'], - function($, _, App, modalView, template) { - 'use strict'; - // Modal dialog for package adding - triggers package:added when package was added - return modalView.extend({ - - events: { - 'click .btn-success': 'addPackage', - 'keypress #inputPackageName': 'addOnEnter' - }, - - template: template, - - initialize: function() { - // Inherit parent events - this.events = _.extend({}, modalView.prototype.events, this.events); - }, - - addOnEnter: function(e) { - if (e.keyCode !== 13) return; - this.addPackage(e); - }, - - addPackage: function(e) { - var self = this; - var options = App.apiRequest('addPackage', - { - name: $('#inputPackageName').val(), - // TODO: better parsing / tokenization - links: $('#inputLinks').val().split('\n') - }, - { - success: function() { - App.vent.trigger('package:added'); - self.hide(); - } - }); - - $.ajax(options); - $('#inputPackageName').val(''); - $('#inputLinks').val(''); - }, - - onShow: function() { - this.$('#inputPackageName').focus(); - } - - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/views/loginView.js b/module/web/app/scripts/views/loginView.js deleted file mode 100644 index 891b3ec99..000000000 --- a/module/web/app/scripts/views/loginView.js +++ /dev/null @@ -1,37 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'hbs!tpl/login'], - function($, Backbone, _, App, template) { - 'use strict'; - - // Renders context actions for selection packages and files - return Backbone.Marionette.ItemView.extend({ - template: template, - - events: { - 'submit form': 'login' - }, - - ui: { - 'form': 'form' - }, - - login: function(e) { - e.stopPropagation(); - - var options = App.apiRequest('login', null, { - data: this.ui.form.serialize(), - type : 'post', - success: function(data) { - // TODO: go to last page, better error - if (data) - App.navigate(''); - else - alert('Wrong login'); - } - }); - - $.ajax(options); - return false; - } - - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/views/notificationView.js b/module/web/app/scripts/views/notificationView.js deleted file mode 100644 index abfcd8079..000000000 --- a/module/web/app/scripts/views/notificationView.js +++ /dev/null @@ -1,83 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'collections/InteractionList', 'hbs!tpl/notification'], - function($, Backbone, _, App, InteractionList, queryModal, template) { - 'use strict'; - - // Renders context actions for selection packages and files - return Backbone.View.extend({ - - // Only view for this area so it's hardcoded - el: '#notification-area', - template: template, - - events: { - 'click .btn-query': 'openQuery', - 'click .btn-notification': 'openNotifications' - }, - - tasks: null, - // area is slided out - visible: false, - // the dialog - modal: null, - - initialize: function() { - this.tasks = new InteractionList(); - - this.$el.calculateHeight().height(0); - - App.vent.on('interaction:added', _.bind(this.onAdd, this)); - App.vent.on('interaction:deleted', _.bind(this.onDelete, this)); - - var render = _.bind(this.render, this); - this.listenTo(this.tasks, 'add', render); - this.listenTo(this.tasks, 'remove', render); - - }, - - onAdd: function(task) { - this.tasks.add(task); - }, - - onDelete: function(task) { - this.tasks.remove(task); - }, - - render: function() { - - // only render when it will be visible - if (this.tasks.length > 0) - this.$el.html(this.template(this.tasks.toJSON())); - - if (this.tasks.length > 0 && !this.visible) { - this.$el.slideOut(); - this.visible = true; - } - else if (this.tasks.length === 0 && this.visible) { - this.$el.slideIn(); - this.visible = false; - } - - return this; - }, - - openQuery: function() { - var self = this; - - _.requireOnce(['views/queryModal'], function(ModalView) { - if (self.modal === null) { - self.modal = new ModalView(); - self.modal.parent = self; - } - - self.modal.model = self.tasks.at(0); - self.modal.render(); - self.modal.show(); - }); - - }, - - openNotifications: function() { - - } - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/views/progressView.js b/module/web/app/scripts/views/progressView.js deleted file mode 100644 index 3a4bb2825..000000000 --- a/module/web/app/scripts/views/progressView.js +++ /dev/null @@ -1,33 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes', 'views/abstract/itemView', - 'hbs!tpl/header/progress', 'helpers/pluginIcon'], - function($, Backbone, _, App, Api, ItemView, template, pluginIcon) { - 'use strict'; - - // Renders single file item - return ItemView.extend({ - - idAttribute: 'pid', - tagName: 'li', - template: template, - events: { - }, - - initialize: function() { - this.listenTo(this.model, 'change', this.render); - this.listenTo(this.model, 'remove', this.unrender); - }, - - onDestroy: function() { - }, - - render: function() { - // TODO: icon - // TODO: other states - // TODO: non download progress - // TODO: better progressbar rendering - this.$el.css('background-image', 'url('+ pluginIcon('todo') +')'); - this.$el.html(this.template(this.model.toJSON())); - return this; - } - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/views/queryModal.js b/module/web/app/scripts/views/queryModal.js deleted file mode 100644 index 7c6439b49..000000000 --- a/module/web/app/scripts/views/queryModal.js +++ /dev/null @@ -1,69 +0,0 @@ -define(['jquery', 'underscore', 'app', 'views/abstract/modalView', './input/inputLoader', 'text!tpl/default/queryDialog.html'], - function($, _, App, modalView, load_input, template) { - 'use strict'; - return modalView.extend({ - - // TODO: submit on enter reloads the page sometimes - events: { - 'click .btn-success': 'submit', - 'submit form': 'submit' - }, - template: _.compile(template), - - // the notificationView - parent: null, - - model: null, - input: null, - - initialize: function() { - // Inherit parent events - this.events = _.extend({}, modalView.prototype.events, this.events); - }, - - renderContent: function() { - var data = { - title: this.model.get('title'), - plugin: this.model.get('plugin'), - description: this.model.get('description') - }; - - var input = this.model.get('input').data; - if (this.model.isCaptcha()) { - data.captcha = input[0]; - data.type = input[1]; - } - return data; - }, - - onRender: function() { - // instantiate the input - var input = this.model.get('input'); - var InputView = load_input(input); - this.input = new InputView(input); - // only renders after wards - this.$('#inputField').append(this.input.render().el); - }, - - submit: function(e) { - e.stopPropagation(); - // TODO: load next task - - this.model.set('result', this.input.getVal()); - var self = this; - this.model.save({success: function() { - self.hide(); - }}); - - this.input.clear(); - }, - - onShow: function() { - this.input.focus(); - }, - - onHide: function() { - this.input.destroy(); - } - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/views/settings/configSectionView.js b/module/web/app/scripts/views/settings/configSectionView.js deleted file mode 100644 index e05701b2a..000000000 --- a/module/web/app/scripts/views/settings/configSectionView.js +++ /dev/null @@ -1,99 +0,0 @@ -define(['jquery', 'underscore', 'backbone', 'app', '../abstract/itemView', '../input/inputLoader', - 'hbs!tpl/settings/config', 'hbs!tpl/settings/configItem'], - function($, _, Backbone, App, itemView, load_input, template, templateItem) { - 'use strict'; - - // Renders settings over view page - return itemView.extend({ - - tagName: 'div', - - template: template, - templateItem: templateItem, - - // Will only render one time with further attribute updates - rendered: false, - - events: { - 'click .btn-primary': 'submit', - 'click .btn-reset': 'reset' - }, - - initialize: function() { - this.listenTo(this.model, 'destroy', this.destroy); - }, - - render: function() { - if (!this.rendered) { - this.$el.html(this.template(this.model.toJSON())); - - // initialize the popover - this.$('.page-header a').popover({ - placement: 'left' -// trigger: 'hover' - }); - - var container = this.$('.control-content'); - var self = this; - _.each(this.model.get('items'), function(item) { - var json = item.toJSON(); - var el = $('
      ').html(self.templateItem(json)); - var InputView = load_input(item.get('input')); - var input = new InputView(json).render(); - item.set('inputView', input); - - self.listenTo(input, 'change', _.bind(self.render, self)); - el.find('.controls').append(input.el); - container.append(el); - }); - this.rendered = true; - } - // Enable button if something is changed - if (this.model.hasChanges()) - this.$('.btn-primary').removeClass('disabled'); - else - this.$('.btn-primary').addClass('disabled'); - - // Mark all inputs that are modified - _.each(this.model.get('items'), function(item) { - var input = item.get('inputView'); - var el = input.$el.parent().parent(); - if (item.isChanged()) - el.addClass('info'); - else - el.removeClass('info'); - }); - - return this; - }, - - onDestroy: function() { - // TODO: correct cleanup after building up so many views and models - }, - - submit: function(e) { - e.stopPropagation(); - // TODO: success / failure popups - var self = this; - this.model.save({success: function() { - self.render(); - App.settingsView.refresh(); - }}); - - }, - - reset: function(e) { - e.stopPropagation(); - // restore the original value - _.each(this.model.get('items'), function(item) { - if (item.has('inputView')) { - var input = item.get('inputView'); - input.setVal(item.get('value')); - input.render(); - } - }); - this.render(); - } - - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/views/settings/pluginChooserModal.js b/module/web/app/scripts/views/settings/pluginChooserModal.js deleted file mode 100644 index 91e9f11b3..000000000 --- a/module/web/app/scripts/views/settings/pluginChooserModal.js +++ /dev/null @@ -1,69 +0,0 @@ -define(['jquery', 'underscore', 'app', 'views/abstract/modalView', 'hbs!tpl/dialogs/addPluginConfig', - 'helpers/pluginIcon', 'select2'], - function($, _, App, modalView, template, pluginIcon) { - 'use strict'; - return modalView.extend({ - - events: { - 'click .btn-add': 'add' - }, - template: template, - plugins: null, - select: null, - - initialize: function() { - // Inherit parent events - this.events = _.extend({}, modalView.prototype.events, this.events); - var self = this; - $.ajax(App.apiRequest('getAvailablePlugins', null, {success: function(data) { - self.plugins = _.sortBy(data, function(item) { - return item.name; - }); - self.render(); - }})); - }, - - onRender: function() { - // TODO: could be a seperate input type if needed on multiple pages - if (this.plugins) - this.select = this.$('#pluginSelect').select2({ - escapeMarkup: function(m) { - return m; - }, - formatResult: this.format, - formatSelection: this.formatSelection, - data: {results: this.plugins, text: function(item) { - return item.label; - }}, - id: function(item) { - return item.name; - } - }); - }, - - onShow: function() { - }, - - onHide: function() { - }, - - format: function(data) { - var s = '
      ' + data.label; - s += '
      ' + data.description + '
      '; - return s; - }, - - formatSelection: function(data) { - return ' ' + data.label; - }, - - add: function(e) { - e.stopPropagation(); - if (this.select) { - var plugin = this.select.val(); - App.vent.trigger('config:open', plugin); - this.hide(); - } - } - }); - }); \ No newline at end of file diff --git a/module/web/app/scripts/views/settings/settingsView.js b/module/web/app/scripts/views/settings/settingsView.js deleted file mode 100644 index cad5ab075..000000000 --- a/module/web/app/scripts/views/settings/settingsView.js +++ /dev/null @@ -1,184 +0,0 @@ -define(['jquery', 'underscore', 'backbone', 'app', 'models/ConfigHolder', './configSectionView', - 'hbs!tpl/settings/layout', 'hbs!tpl/settings/menu', 'hbs!tpl/settings/actionbar'], - function($, _, Backbone, App, ConfigHolder, ConfigSectionView, template, templateMenu, templateBar) { - 'use strict'; - - // Renders settings over view page - return Backbone.Marionette.ItemView.extend({ - - template: template, - templateMenu: templateMenu, - - events: { - 'click .settings-menu li > a': 'change_section', - 'click .btn-add': 'choosePlugin', // TODO not in scope - 'click .icon-remove': 'deleteConfig' - }, - - ui: { - 'menu': '.settings-menu', - 'content': '.setting-box > form' - }, - - selected: null, - modal: null, - - coreConfig: null, // It seems collections are not needed - pluginConfig: null, - - // currently open configHolder - config: null, - lastConfig: null, - isLoading: false, - - initialize: function() { - this.actionbar = Backbone.Marionette.ItemView.extend({ - template: templateBar, - events: { - 'click .btn': 'choosePlugin' - }, - choosePlugin: _.bind(this.choosePlugin, this) - - }); - this.listenTo(App.vent, 'config:open', this.openConfig); - - this.refresh(); - }, - - refresh: function() { - var self = this; - $.ajax(App.apiRequest('getCoreConfig', null, {success: function(data) { - self.coreConfig = data; - self.renderMenu(); - }})); - $.ajax(App.apiRequest('getPluginConfig', null, {success: function(data) { - self.pluginConfig = data; - self.renderMenu(); - }})); - }, - - onRender: function() { - // set a height with css so animations will work - this.ui.content.height(this.ui.content.height()); - }, - - renderMenu: function() { - var plugins = [], - addons = []; - - // separate addons and default plugins - // addons have an activated state - _.each(this.pluginConfig, function(item) { - if (item.activated === null) - plugins.push(item); - else - addons.push(item); - }); - - this.ui.menu.html(this.templateMenu({ - core: this.coreConfig, - plugin: plugins, - addon: addons - })); - - // mark the selected element - this.$('li[data-name="' + this.selected + '"]').addClass('active'); - }, - - openConfig: function(name) { - // Do nothing when this config is already open - if (this.config && this.config.get('name') === name) - return; - - this.lastConfig = this.config; - this.config = new ConfigHolder({name: name}); - this.loading(); - - var self = this; - this.config.fetch({success: function() { - if (!self.isLoading) - self.show(); - - }, failure: _.bind(this.failure, this)}); - - }, - - loading: function() { - this.isLoading = true; - var self = this; - this.ui.content.fadeOut({complete: function() { - if (self.config.isLoaded()) - self.show(); - - self.isLoading = false; - }}); - - }, - - show: function() { - // TODO animations are bit sloppy - this.ui.content.css('display', 'block'); - var oldHeight = this.ui.content.height(); - - // this will destroy the old view - if (this.lastConfig) - this.lastConfig.trigger('destroy'); - else - this.ui.content.empty(); - - // reset the height - this.ui.content.css('height', ''); - // append the new element - this.ui.content.append(new ConfigSectionView({model: this.config}).render().el); - // get the new height - var height = this.ui.content.height(); - // set the old height again - this.ui.content.height(oldHeight); - this.ui.content.animate({ - opacity: 'show', - height: height - }); - }, - - failure: function() { - // TODO - this.config = null; - }, - - change_section: function(e) { - // TODO check for changes - // TODO move this into render? - - var el = $(e.target).closest('li'); - - this.selected = el.data('name'); - this.openConfig(this.selected); - - this.ui.menu.find('li.active').removeClass('active'); - el.addClass('active'); - e.preventDefault(); - }, - - choosePlugin: function(e) { - var self = this; - _.requireOnce(['views/settings/pluginChooserModal'], function(Modal) { - if (self.modal === null) - self.modal = new Modal(); - - self.modal.show(); - }); - }, - - deleteConfig: function(e) { - e.stopPropagation(); - var el = $(e.target).parent().parent(); - var name = el.data('name'); - var self = this; - $.ajax(App.apiRequest('deleteConfig', {plugin: name}, { success: function() { - self.refresh(); - }})); - - } - - }); - }); \ No newline at end of file diff --git a/module/web/app/styles/default/accounts.less b/module/web/app/styles/default/accounts.less deleted file mode 100644 index 9b45b64b3..000000000 --- a/module/web/app/styles/default/accounts.less +++ /dev/null @@ -1,6 +0,0 @@ -@import "common"; - -.logo-select { - width: 20px; - height: 20px; -} \ No newline at end of file diff --git a/module/web/app/styles/default/admin.less b/module/web/app/styles/default/admin.less deleted file mode 100644 index 92524c153..000000000 --- a/module/web/app/styles/default/admin.less +++ /dev/null @@ -1,17 +0,0 @@ -@import "common"; - -/* - Admin -*/ - -#btn_newuser { - float: right; -} - -#user_permissions { - float: right; -} - -.userperm { - width: 115px; -} \ No newline at end of file diff --git a/module/web/app/styles/default/base.less b/module/web/app/styles/default/base.less deleted file mode 100644 index bd318127e..000000000 --- a/module/web/app/styles/default/base.less +++ /dev/null @@ -1,163 +0,0 @@ -@import "common"; - -/* - General -*/ - -* { - margin: 0; -} - -html, body { - height: 100%; -} - -body { - margin: 0; - padding: 0; - font-family: 'Abel', sans-serif; - font-size: 16px; - background: url("../../images/default/bgpattern.png") repeat scroll 0 0 transparent; -} - -h1, h2, h3 { - margin: 0; - padding: 0; - font-weight: normal; -} - -a { - text-decoration: none; - color: @blue; -} - -a:hover { - text-decoration: none; - color: @emph; -} - -#wrap { - min-height: 100%; -} - -#content { - padding-bottom: @footer-height; -} - -#content-container:before { - display: block; - content: " "; - height: @header-height; -} - -/* - Additional Responsive Class for larger desktop -*/ - -.visible-large-screen { - display: inherit !important; -} - -@media (max-width: @large-screen) { - .visible-large-screen { - display: none !important; - } -} - -.btn-blue { - background-color: hsl(206, 49%, 35%) !important; - background-repeat: repeat-x; - background-image: -khtml-gradient(linear, left top, left bottom, from(#5493c4), to(#2d5f84)); - background-image: -moz-linear-gradient(top, #5493c4, #2d5f84); - background-image: -ms-linear-gradient(top, #5493c4, #2d5f84); - background-image: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #5493c4), color-stop(100%, #2d5f84)); - background-image: -webkit-linear-gradient(top, #5493c4, #2d5f84); - background-image: -o-linear-gradient(top, #5493c4, #2d5f84); - background-image: linear-gradient(#5493c4, #2d5f84); - border-color: #2d5f84 #2d5f84 hsl(206, 49%, 30%); - color: #fff !important; - text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.33); - -webkit-font-smoothing: antialiased; - - .caret { - border-bottom-color: #FFFFFF; - border-top-color: #FFFFFF; - } - -} - -.btn-yellow { - background-color: hsl(46, 100%, 57%) !important; - background-repeat: repeat-x; - background-image: -khtml-gradient(linear, left top, left bottom, from(#ffe389), to(#fecb23)); - background-image: -moz-linear-gradient(top, #ffe389, #fecb23); - background-image: -ms-linear-gradient(top, #ffe389, #fecb23); - background-image: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #ffe389), color-stop(100%, #fecb23)); - background-image: -webkit-linear-gradient(top, #ffe389, #fecb23); - background-image: -o-linear-gradient(top, #ffe389, #fecb23); - background-image: linear-gradient(#ffe389, #fecb23); - border-color: #fecb23 #fecb23 hsl(46, 100%, 52%); - color: #333 !important; - text-shadow: 0 1px 1px rgba(255, 255, 255, 0.33); - -webkit-font-smoothing: antialiased; - - .caret { - border-bottom-color: #FFFFFF; - border-top-color: #FFFFFF; - } -} - -.icon-8x { - font-size: 8em; -} - -// Will not work well in buttons/navs, would require more rules -.icon-larger { - vertical-align: -10%; - font-size: 1.5em; -} - -/* - Modal Overlay -*/ -#modal-overlay { - content: " "; - height: 100%; - width: 100%; - position: absolute; - left: 0; - top: 0; - background: -moz-radial-gradient(center, ellipse cover, rgba(236, 208, 66, 0) 0%, rgba(40, 119, 171, 0.9) 100%); - background: -webkit-gradient(radial, center center, 0px, center center, 100%, color-stop(0%, rgba(236, 208, 66, 0)), color-stop(100%, rgba(40, 119, 171, 0.9))); - background: -webkit-radial-gradient(center, ellipse cover, rgba(236, 208, 66, 0) 0%, rgba(40, 119, 171, 0.9) 100%); - background: -o-radial-gradient(center, ellipse cover, rgba(236, 208, 66, 0) 0%, rgba(40, 119, 171, 0.9) 100%); - background: -ms-radial-gradient(center, ellipse cover, rgba(236, 208, 66, 0) 0%, rgba(40, 119, 171, 0.9) 100%); - background: radial-gradient(center, ellipse cover, rgba(236, 208, 66, 0) 0%, rgba(40, 119, 171, 0.9) 100%); - z-index: 100; - opacity: 0; -} - -div.modal-header { - border-top-left-radius: 4px; - border-top-right-radius: 4px; - background-color: @blueDark; - color: #FFFFFF; - - .close { - color: @light; - } - -} - -.select2-container { - min-width: 220px; // same as other input fields -} - -div.modal-body { - max-height: 100%; -} - -div.modal { - width: 600px; - -} \ No newline at end of file diff --git a/module/web/app/styles/default/common.less b/module/web/app/styles/default/common.less deleted file mode 100644 index 338ed38e7..000000000 --- a/module/web/app/styles/default/common.less +++ /dev/null @@ -1,90 +0,0 @@ -/* - Definitions -*/ - - -/* - Threshold for slightly larger screen -*/ -@large-screen: 1150px; - -@header-height: 70px; -@actionbar-height: 40px; -@footer-height: 66px; - -@light: #ffffff; -@dark: #333333; - -@grey: #757575; -@greyLight: #E5E5E5; -@greyLighter: #F5F5F5; -@greyDark: #444444; -@greyDarker: #111113; - -@yellow: #ffd856; -@yellowLighter: lighten(spin(@yellow, 10), 20%); -@yellowLightest: lighten(spin(@yellow, 15), 30%); -@yellowDark: darken(@yellow, 10%); - -@blue: #3571a2; -@blueLight: lighten(spin(@blue, 5), 10%); -@blueLighter: lighten(spin(@blue, 10), 20%); -@blueLightest: lighten(spin(@blue, 20), 40%); -@blueDark: darken(spin(@blue, -5), 10%); -@blueDarker: darken(spin(@blue, -10), 20%); - -@green: #468847; -@greenLight: lighten(spin(@green, 5), 10%); -@greenDark: darken(spin(@green, -5), 10%); - -@red: #b94a48; -@redLight: lighten(spin(@red, 5), 10%); -@redDark: darken(spin(@red, -5), 10%); - -@emph: #FF7637; - - -/* - Mixins -*/ - -.gradient(@origin: left, @start: #ffffff, @stop: #000000) { - background-color: @start; - background-image: -webkit-linear-gradient(@origin, @start, @stop); - background-image: -moz-linear-gradient(@origin, @start, @stop); - background-image: -o-linear-gradient(@origin, @start, @stop); - background-image: -ms-linear-gradient(@origin, @start, @stop); - background-image: linear-gradient(@origin, @start, @stop); -} - -.transition(@prop: all, @time: 0.25s, @ease: linear) { - -webkit-transition: @prop @time @ease; - -moz-transition: @prop @time @ease; - -o-transition: @prop @time @ease; - -ms-transition: @prop @time @ease; - transition: @prop @time @ease; -} - -.stripes(@color, @color2: rgba(255, 255, 255, 0.15)) { - background-color: @color; - background-image: -webkit-linear-gradient(45deg, @color2 25%, transparent 25%, transparent 50%, @color2 50%, @color2 75%, transparent 75%, transparent); - background-image: -moz-linear-gradient(45deg, @color2 25%, transparent 25%, transparent 50%, @color2 50%, @color2 75%, transparent 75%, transparent); - background-image: -o-linear-gradient(45deg, @color2 25%, transparent 25%, transparent 50%, @color2 50%, @color2 75%, transparent 75%, transparent); - background-image: linear-gradient(45deg, @color2 25%, transparent 25%, transparent 50%, @color2 50%, @color2 75%, transparent 75%, transparent); - -webkit-background-size: 40px 40px; - -moz-background-size: 40px 40px; - -o-background-size: 40px 40px; - background-size: 40px 40px; -} -.stripes-animated { - -webkit-animation: progress-bar-stripes 2s linear infinite; - -moz-animation: progress-bar-stripes 2s linear infinite; - -ms-animation: progress-bar-stripes 2s linear infinite; - -o-animation: progress-bar-stripes 2s linear infinite; - animation: progress-bar-stripes 2s linear infinite; -} - - -.default-shadow { - box-shadow: 0 0 5px @dark; -} \ No newline at end of file diff --git a/module/web/app/styles/default/dashboard.less b/module/web/app/styles/default/dashboard.less deleted file mode 100644 index ab61ba29b..000000000 --- a/module/web/app/styles/default/dashboard.less +++ /dev/null @@ -1,331 +0,0 @@ -@import "common"; - -/* - Dashboard -*/ - -#dashboard ul { - margin: 0; - list-style: none; -} - -.sidebar-header { - font-size: 25px; - line-height: 25px; - margin: 4px 0; - border-bottom: 1px dashed @grey; -} - -/* - Packages -*/ - -.package-list { - list-style: none; - margin-left: 0; -} - -@frame-top: 20px; -@frame-bottom: 18px; - -.package-frame { - position: absolute; - top: -@frame-top; - left: -@frame-top / 2; - right: -@frame-top / 2; - bottom: -@frame-bottom + 2px; // + size of visible bar - z-index: -1; // lies under package - border: 1px solid @grey; - border-radius: 5px; - box-shadow: 3px 3px 6px rgba(0, 0, 0, 0.75); -} - -.package-view { - padding-bottom: 4px; - margin: 8px 0; - position: relative; - overflow: hidden; - - -webkit-hyphens: auto; - -moz-hyphens: auto; - hyphens: auto; - - i { - cursor: pointer; - } - - & > i { - vertical-align: middle; - } - - .progress { - position: absolute; - height: @frame-bottom; - line-height: @frame-bottom; - font-size: 12px; - text-align: center; - border-radius: 0; - border-bottom-left-radius: 5px; - border-bottom-right-radius: 5px; - bottom: 0; - left: 0; - right: 0; - margin-bottom: 0; - background-image: none; - color: @light; - background-color: @yellow; - } - - .bar-info { - background-image: none; - background-color: @blue; - } - - &:hover { - overflow: visible; - z-index: 10; - - .package-frame { - background-color: @light; - } - } - - &.ui-selected:hover { - color: @light; - - .package-frame { - background-color: @dark; - } - - } -} - -.package-name { - cursor: pointer; -} - -.package-indicator { - position: absolute; - top: 0; - right: 0; - float: right; - color: @blue; - text-shadow: @yellowDark 1px 1px; - height: @frame-top; - line-height: @frame-top; - - & > i:hover { - color: @green; - } - - .dropdown-menu { - text-shadow: none; - } - - .tooltip { - text-shadow: none; - width: 100%; - } - - .btn-move { - color: @green; - display: none; - } - -} - -.ui-files-selected .btn-move { - display: inline; -} - -// Tag area with different effect on hover -.tag-area { - position: absolute; - top: -2px; - left: 0; - - .badge { - font-size: 11px; - line-height: 11px; - } - - .badge i { - cursor: pointer; - &:hover:before { - content: "\f024"; // show Remove icon - } - } - - .badge-ghost { - visibility: hidden; - cursor: pointer; - opacity: 0.5; - } - - &:hover .badge-ghost { - visibility: visible; - } - -} - -/* - File View -*/ - -.file-list { - list-style: none; - margin: 0; -} - -@file-height: 22px; - -.file-view { - position: relative; - padding: 0 4px; - border-top: 1px solid #dddddd; - line-height: @file-height; - - &:first-child { - border-top: none; - } - - &:hover, &.ui-selected:hover { - border-radius: 5px; - .gradient(top, @blue, @blueLight); - color: @light; - } - - &.ui-selected { - .gradient(top, @yellow, @yellowDark); - color: @dark; - border-color: @greenDark; - - .file-row.downloading .bar { - .gradient(top, @green, @greenLight); - } - - } - - img { // plugin logo - margin-top: -2px; - padding: 0 2px; - height: @file-height; - width: @file-height; - } - - .icon-chevron-down:hover { - cursor: pointer; - color: @yellow; - } - -} - -.file-row { - min-height: 0 !important; -// padding-left: 5px; - padding-top: 4px; - padding-bottom: 4px; - - // TODO: better styling for filestatus - &.second { -// border-radius: 4px; -// background: @light; - font-size: small; - font-weight: bold; -// box-shadow: 3px 3px 6px rgba(0, 0, 0, 0.75); -// .default-shadow; - } - - &.third { - margin-left: 0; - position: relative; - font-size: small; - } - - .dropdown-menu { - font-size: medium; - } -} - -/* - TODO: more colorful states - better fileView design -*/ - -.file-row.finished { -// .gradient(top, @green, @greenLight); -// color: @light; - color: @green; -} - -.file-row.failed { -// .gradient(top, @red, @redLight); -// color: @light; - color: @red; -} - -.file-row.downloading { - - .progress { - height: @file-height; - background: @light; - margin: 0; - } - - .bar { - text-align: left; - .gradient(top, @yellow, @yellowDark); - color: @dark; - } - -} - -/* -FANCY CHECKBOXES -*/ -.file-view .checkbox { - width: 20px; - height: 21px; - background: url(../../images/default/checks_sheet.png) left top no-repeat; - cursor: pointer; -} - -.file-view.ui-selected .checkbox { - background: url(../../images/default/checks_sheet.png) -21px top no-repeat; -} - -/* - Actionbar -*/ - -.form-search { - position: relative; - - .dropdown-menu { - min-width: 100%; - position: absolute; - right: 0; - left: auto; - } - -} - -li.finished > a, li.finished:hover > a { - background-color: @green; - color: @light; - - .caret, .caret:hover { - border-bottom-color: @light !important; - border-top-color: @light !important; - } -} - -li.failed > a, li.failed:hover > a { - background-color: @red; - color: @light; - - .caret, .caret:hover { - border-bottom-color: @light !important; - border-top-color: @light !important; - } -} \ No newline at end of file diff --git a/module/web/app/styles/default/main.less b/module/web/app/styles/default/main.less deleted file mode 100644 index 15d9943f8..000000000 --- a/module/web/app/styles/default/main.less +++ /dev/null @@ -1,11 +0,0 @@ -@import "bootstrap/less/bootstrap"; -@import "bootstrap/less/responsive"; -@import "font-awesome/build/assets/font-awesome/less/font-awesome"; - -@FontAwesomePath: "../../fonts"; - -@import "style"; -@import "dashboard"; -@import "settings"; -@import "accounts"; -@import "admin"; diff --git a/module/web/app/styles/default/settings.less b/module/web/app/styles/default/settings.less deleted file mode 100644 index 34bfcb92a..000000000 --- a/module/web/app/styles/default/settings.less +++ /dev/null @@ -1,121 +0,0 @@ -@import "common"; - -/* - Settings -*/ -.settings-menu { - background-color: #FFF; - box-shadow: 0 0 5px #000; // border: 10px solid #EEE; - - .nav-header { - background: @blueDark; - color: @light; - } - - li > a, .nav-header { - margin-left: -16px; - margin-right: -16px; - text-shadow: none; - } - - i { - margin-top: 0; - } - - .plugin, .addon { - a { - padding-left: 28px; - background-position: 4px 2px; - background-repeat: no-repeat; - background-size: 20px 20px; - } - - .icon-remove { - display: none; - } - - &:hover { - i { - display: block; - } - } - - } - - .addon { - div { - font-size: small; - } - .addon-on { - color: @green; - } - - .addon-off { - color: @red; - } - - } - - border-top-left-radius: 0; - border-top-right-radius: 0; - - .nav > li > a:hover { - color: @blueDark; - } -} - -.setting-box { - border: 10px solid @blueDark; - box-shadow: 0 0 5px @dark; // .gradient(bottom, @yellowLightest, @light); - overflow: hidden; - - .page-header { - margin: 0; - - .btn { - float: right; - margin-top: 5px; - } - - .popover { - font-size: medium; - } - - } - - // Bit wider control labels - .control-label { - width: 180px; - } - .controls { - margin-left: 200px; - } - .form-actions { - padding-left: 200px; - } - -} - -/* - Plugin select -*/ - -.plugin-select { - background-position: left 2px; - background-repeat: no-repeat; - background-size: 20px 20px; - padding-left: 24px; - - font-weight: bold; - span { - line-height: 14px; - font-size: small; - font-weight: normal; - } - -} - -.logo-select { - width: 20px; - height: 20px; -} \ No newline at end of file diff --git a/module/web/app/styles/default/style.less b/module/web/app/styles/default/style.less deleted file mode 100644 index 6fb5a4857..000000000 --- a/module/web/app/styles/default/style.less +++ /dev/null @@ -1,366 +0,0 @@ -@import "default/base"; - - -/* - Header -*/ - -header { // background-color: @greyDark; - .gradient(to bottom, #222222, #111111); - height: @header-height; - position: fixed; - top: 0; - vertical-align: top; - width: 100%; - z-index: 10; - color: #ffffff; - - a { - color: #ffffff; - } - .container-fluid, .row-fluid { - height: @header-height; - } - -} - -@header-inner-height: @header-height - 16px; - -// centered header element -.centered { - height: @header-inner-height; - margin: 8px 0; -} - -header:before { - position: absolute; - content: ' '; - top: 0; - right: 0; - bottom: 0; - left: 0; - background-color: transparent; - z-index: -1; - .default-shadow; -} - -header span.title { - color: white; - float: left; - font-family: SansationRegular, sans-serif; - font-size: 40px; - line-height: @header-height; - cursor: default; -} - -header .logo { - float: left; - margin-right: 10px; - margin-top: 10px; - width: 105px; - height: 107px; - background: url("../../images/default/logo.png") no-repeat; -} - -.header-block { - .centered; - float: left; - line-height: @header-inner-height / 3; // 3 rows - font-size: small; -} - -.status-block { - min-width: 15%; -} - -.header-btn { - float: right; - position: relative; - .centered; - - .lower { - position: absolute; - bottom: 0; - left: 0; - right: 0; - margin-left: 0; - - button { - width: 100% / 3; // 3 buttons - } - - } -} - -#progress-area { - .centered; - position: relative; - margin-top: 8px; - line-height: 16px; - - #progress-info { - padding-left: 2px; - } - - .sub { - font-size: small; - padding: 0 2px; - } - - .popover { // display: block; - max-width: none; - width: 120%; - left: -60%; // Half of width - margin-left: 50%; - top: 100%; - } - - .popover-title, .popover-content { - color: @greyDark; - } - - .icon-list { - cursor: pointer; - margin-right: 2px; // same as globalprogress margin - - &:hover { - color: @yellow; - } - } - .close { - line-height: 14px; - } -} - -.progress-list { - list-style: none; - margin: 0; - font-size: small; - - li { - background-repeat: no-repeat; - background-size: 32px 32px; - background-position: 0px 8px; - padding-left: 40px; - - &:not(:last-child) { - margin-bottom: 5px; - padding-bottom: 5px; - border-bottom: 1px dashed @greyLight; - } - - .progress { - height: 8px; - margin-bottom: 0; - - .bar { - .gradient(bottom, @blue, @blueLight); - } - } - } -} - -#globalprogress { - background-color: @greyDark; - background-image: none; - height: 8px; - margin: 4px 0; - border-radius: 8px; - border: 2px solid @grey; - - .bar { - color: @dark; - background-image: none; - background-color: @yellow; - - &.running { - width: 100%; - .stripes(@yellowLighter, @yellowDark); - } - } -} - -.speedgraph-container { - // Allows speedgraph to take up remaining space - display: block; - overflow: hidden; - padding: 0 8px; - - #speedgraph { - float: right; - width: 100%; - .centered; -// height: @header-height - 16px; -// margin: 8px 0; - font-family: sans-serif; - } -} - -.header-area { - display: none; // hidden by default - position: absolute; - bottom: -28px; - line-height: 18px; - top: @header-height; - padding: 4px 10px 6px 10px; - text-align: center; - border-radius: 0 0 6px 6px; - color: @light; - background-color: @greyDark; - .default-shadow; -} - -#notification-area { - .header-area; - left: 140px; - - .badge { - vertical-align: top; - } - - .btn-query, .btn-notification { - cursor: pointer; - } -} - -#selection-area { - .header-area; - left: 50%; - min-width: 15%; - - i { - cursor: pointer; - - &:hover { - color: @yellow; - } - } - -} - -/* - Actionbar -*/ - -.nav > li > a:hover { - color: @blue; -} - -.actionbar { - padding-bottom: 3px; - margin-bottom: 0; - border-bottom: 1px dashed @grey; - - height: @actionbar-height; - - padding-top: 2px; - margin-bottom: 5px; - -} - -.actionbar > li > a { - margin-top: 4px; -} - -.actionbar .breadcrumb { - margin: 0; - padding-top: 10px; - padding-bottom: 0; - - .active { - color: @grey; - } - -} - -.actionbar form { - margin-top: 6px; - margin-bottom: 0; -} - -.actionbar input, .actionbar button { - padding-top: 2px; - padding-bottom: 2px; -} - -.actionbar .dropdown-menu i { - margin-top: 4px; - padding-right: 5px; -} - -/* - Login -*/ -.login { - vertical-align: middle; - border: 2px solid @dark; - padding: 15px 50px; - font-size: 17px; - border-radius: 15px; - -moz-border-radius: 15px; - -webkit-border-radius: 15px; -} - - -/* - Footer -*/ -footer { // Same gradient as navbar - .gradient(top, #222222, #111111); - color: @grey; - min-height: @footer-height; - margin-top: -@footer-height; - position: relative; - width: 100%; - line-height: 16px; - z-index: 10; -} - -footer:before { - position: absolute; - content: ' '; - top: 0; - right: 0; - bottom: 0; - left: 0; - background-color: transparent; - box-shadow: 0 0 5px black; - z-index: -1; -} - -footer hr { - margin: 4px 0; - border-top-color: @greyDarker; - border-bottom-color: @greyDark; -} - -footer .span2 { - font-size: 12px; - padding-top: 12px; - padding-bottom: 12px; -} - -// This is the copyright span -footer .offset1 { - padding-top: 8px; - padding-bottom: 0; -} - -footer .copyright { - background: url(../../images/default/logo_grey.png) no-repeat; - background-size: 40px 40px; - background-position: 12px center; - height: 40px; - padding-left: 40px; - padding-top: 10px; - text-align: center; -} - -footer h2 { - color: @light; - font-family: SansationLight, sans-serif; - font-size: 16px; - font-weight: normal; - line-height: 16px; - margin: 0; -} \ No newline at end of file diff --git a/module/web/app/styles/font.css b/module/web/app/styles/font.css deleted file mode 100644 index ee117d43b..000000000 --- a/module/web/app/styles/font.css +++ /dev/null @@ -1,37 +0,0 @@ -/** - * @file - * Font styling - */ - -@font-face { - font-family: 'SansationRegular'; - src: url('../fonts/Sansation_Regular-webfont.eot'); - src: url('../fonts/Sansation_Regular-webfont.eot?#iefix') format('embedded-opentype'), - url('../fonts/Sansation_Regular-webfont.woff') format('woff'), - url('../fonts/Sansation_Regular-webfont.ttf') format('truetype'), - url('../fonts/Sansation_Regular-webfont.svg#SansationRegular') format('svg'); - font-weight: normal; - font-style: normal; -} - -@font-face { - font-family: 'SansationLight'; - src: url('../fonts/Sansation_Light-webfont.eot'); - src: url('../fonts/Sansation_Light-webfont.eot?#iefix') format('embedded-opentype'), - url('../fonts/Sansation_Light-webfont.woff') format('woff'), - url('../fonts/Sansation_Light-webfont.ttf') format('truetype'), - url('../fonts/Sansation_Light-webfont.svg#SansationLight') format('svg'); - font-weight: normal; - font-style: normal; -} - -@font-face { - font-family: 'SansationBold'; - src: url('../fonts/Sansation_Bold-webfont.eot'); - src: url('../fonts/Sansation_Bold-webfont.eot?#iefix') format('embedded-opentype'), - url('../fonts/Sansation_Bold-webfont.woff') format('woff'), - url('../fonts/Sansation_Bold-webfont.ttf') format('truetype'), - url('../fonts/Sansation_Bold-webfont.svg#SansationBold') format('svg'); - font-weight: normal; - font-style: normal; -} \ No newline at end of file diff --git a/module/web/app/templates/default/accounts/account.html b/module/web/app/templates/default/accounts/account.html deleted file mode 100644 index c2ded16f6..000000000 --- a/module/web/app/templates/default/accounts/account.html +++ /dev/null @@ -1,10 +0,0 @@ -<% plugin %> -<% loginname %> -<% valid %> -<% premium %> -<% trafficleft %> -<% shared %> -<% activated %> - - - \ No newline at end of file diff --git a/module/web/app/templates/default/accounts/actionbar.html b/module/web/app/templates/default/accounts/actionbar.html deleted file mode 100644 index f4652ec42..000000000 --- a/module/web/app/templates/default/accounts/actionbar.html +++ /dev/null @@ -1,5 +0,0 @@ -
      -
      - - - \ No newline at end of file diff --git a/module/web/app/templates/default/accounts/layout.html b/module/web/app/templates/default/accounts/layout.html deleted file mode 100644 index e6627500d..000000000 --- a/module/web/app/templates/default/accounts/layout.html +++ /dev/null @@ -1,19 +0,0 @@ - -
      - - - - - - - - - - - - - - - -
      PluginNameValidPremiumTrafficSharedActivatedDelete
      -
      \ No newline at end of file diff --git a/module/web/app/templates/default/admin.html b/module/web/app/templates/default/admin.html deleted file mode 100644 index 2eb90d7e0..000000000 --- a/module/web/app/templates/default/admin.html +++ /dev/null @@ -1,223 +0,0 @@ -{% extends 'default/base.html' %} - -{% block title %}{{ _("Admin") }} - {{ super() }} {% endblock %} -{% block subtitle %}{{ _("Admin") }} -{% endblock %} - -{% block css %} - - -{% endblock %} - -{% block require %} -{% endblock %} - -{% block content %} -
      -
      -
      -
      -

      Admin Bereich - Userverwaltung, Systeminfos - -

      - - - -
      - -
      - User - - -
      - - - - - -
      - -
      -
      -

      Support

      - - - - - - - - - - - - - - - - - -
      - - - Wiki | - Forum | - Chat -
      - - - Documentation -
      - - - Development -
      - - - Issue Tracker -
      -
      - System - -
      -
      -
      - - - {##} - - -{% endblock %} \ No newline at end of file diff --git a/module/web/app/templates/default/dashboard/actionbar.html b/module/web/app/templates/default/dashboard/actionbar.html deleted file mode 100644 index 815d4593c..000000000 --- a/module/web/app/templates/default/dashboard/actionbar.html +++ /dev/null @@ -1,54 +0,0 @@ -
      -
      - \ No newline at end of file diff --git a/module/web/app/templates/default/dashboard/file.html b/module/web/app/templates/default/dashboard/file.html deleted file mode 100644 index 6256254da..000000000 --- a/module/web/app/templates/default/dashboard/file.html +++ /dev/null @@ -1,34 +0,0 @@ -
      -   - - <% name %> - -
      -
      - <% fileStatus this %> -
      - -
      -   - <% formatSize size %> - - - <% download.plugin %>  - - - -
      \ No newline at end of file diff --git a/module/web/app/templates/default/dashboard/layout.html b/module/web/app/templates/default/dashboard/layout.html deleted file mode 100644 index 945d11762..000000000 --- a/module/web/app/templates/default/dashboard/layout.html +++ /dev/null @@ -1,35 +0,0 @@ -
      -
      - Local -
      - -
      -
      -
      -
        - -
      -
      - Shared -
      -
        -
      • Content from
        • -
      • Other user
        • -
      • which they shared
        • -
      -
      - Remote -
      -
        -
      • Content from
        • -
      • remote sites
        • -
      • mega
        • -
      • dropbox
        • -
      • other pyloads
        • -
      -
      -
      -
        - -
      -
      \ No newline at end of file diff --git a/module/web/app/templates/default/dashboard/package.html b/module/web/app/templates/default/dashboard/package.html deleted file mode 100644 index c0690a9bf..000000000 --- a/module/web/app/templates/default/dashboard/package.html +++ /dev/null @@ -1,50 +0,0 @@ -<%= if selected %> - - <% else %> - - <%/if%> - - <% name %> - - -
      -
      - video - Add Tag -
      -
      - - - - <%= if shared %> - - <% else %> - - <%/if%> - - - -
      -
      - - <% stats.linksdone %> / <% stats.linkstotal %> - -
      - - <% formatSize stats.sizedone %> / <% formatSize stats.sizetotal %> - -
      -
      \ No newline at end of file diff --git a/module/web/app/templates/default/dashboard/select.html b/module/web/app/templates/default/dashboard/select.html deleted file mode 100644 index 73ea391cd..000000000 --- a/module/web/app/templates/default/dashboard/select.html +++ /dev/null @@ -1,11 +0,0 @@ -  -<%= if packs %><% packs %> package(s)<%/if %> -<%= if files %> -<%= if packs %>, <%/if %> -<% files %> file(s) -<%/if %> -selected - |  -  -  - \ No newline at end of file diff --git a/module/web/app/templates/default/dialogs/addAccount.html b/module/web/app/templates/default/dialogs/addAccount.html deleted file mode 100755 index bdc8a609a..000000000 --- a/module/web/app/templates/default/dialogs/addAccount.html +++ /dev/null @@ -1,42 +0,0 @@ -
      - -

      Add an account

      -
      -
      -
      - - Please enter your account data - -
      - - -
      - -
      -
      -
      - - -
      - -
      -
      -
      - - -
      - -
      -
      -
      -
      -
      - Add - Close -
      \ No newline at end of file diff --git a/module/web/app/templates/default/dialogs/addPluginConfig.html b/module/web/app/templates/default/dialogs/addPluginConfig.html deleted file mode 100755 index e7a42a208..000000000 --- a/module/web/app/templates/default/dialogs/addPluginConfig.html +++ /dev/null @@ -1,26 +0,0 @@ -
      - -

      - Choose a plugin -

      -
      -
      -
      - - Please choose a plugin, which you want to configure - -
      - - -
      - -
      -
      -
      -
      -
      - Add - Close -
      \ No newline at end of file diff --git a/module/web/app/templates/default/dialogs/confirmDelete.html b/module/web/app/templates/default/dialogs/confirmDelete.html deleted file mode 100644 index 65ae1cb21..000000000 --- a/module/web/app/templates/default/dialogs/confirmDelete.html +++ /dev/null @@ -1,11 +0,0 @@ -
      - -

      Please confirm

      -
      -
      - Do you want to delete the selected items? -
      -
      - Delete - Cancel -
      \ No newline at end of file diff --git a/module/web/app/templates/default/dialogs/interactionTask.html b/module/web/app/templates/default/dialogs/interactionTask.html deleted file mode 100755 index ae325e83d..000000000 --- a/module/web/app/templates/default/dialogs/interactionTask.html +++ /dev/null @@ -1,37 +0,0 @@ -
      - -

      - <% title %> - - <% plugin %> - -

      -
      -
      -
      - <% description %> - <%= if captcha %> -
      - - -
      - -
      -
      -
      - - -
      -
      -
      - <% else %> - <% content %> - <%/if%> -
      -
      -
      - Submit - Close -
      \ No newline at end of file diff --git a/module/web/app/templates/default/dialogs/linkgrabber.html b/module/web/app/templates/default/dialogs/linkgrabber.html deleted file mode 100755 index 08418cf03..000000000 --- a/module/web/app/templates/default/dialogs/linkgrabber.html +++ /dev/null @@ -1,49 +0,0 @@ -
      - -

      - AddPackage - paste&add links to pyLoad -

      -
      - -
      - -
      -
      - - -
      - -
      -
      -
      - - -
      - -
      -
      -
      - - -
      - -
      -
      -
      - - -
      - - -
      -
      -
      -
      - -
      - Add - Close -
      \ No newline at end of file diff --git a/module/web/app/templates/default/dialogs/modal.html b/module/web/app/templates/default/dialogs/modal.html deleted file mode 100755 index 1e44cc99c..000000000 --- a/module/web/app/templates/default/dialogs/modal.html +++ /dev/null @@ -1,10 +0,0 @@ -
      - -

      Dialog

      -
      -
      -
      -
      - Close - Save -
      \ No newline at end of file diff --git a/module/web/app/templates/default/header/layout.html b/module/web/app/templates/default/header/layout.html deleted file mode 100644 index 890a5b018..000000000 --- a/module/web/app/templates/default/header/layout.html +++ /dev/null @@ -1,62 +0,0 @@ -
      - - pyLoad -
      -
      -
      - - - -
      -
      -
      -

      - Running... - -

      -
      -
        -
        -
        -
        -
        -
        -
        -
        - Max. Speed:
        - Running:
        - Reconnect:
        -
        - -
        - -
        -
        - User - - -
        -
        - - - -
        -
        - -
        -         -
        \ No newline at end of file diff --git a/module/web/app/templates/default/header/progress.html b/module/web/app/templates/default/header/progress.html deleted file mode 100644 index 65ae9a880..000000000 --- a/module/web/app/templates/default/header/progress.html +++ /dev/null @@ -1,14 +0,0 @@ -<% name %> -<% plugin %> - -
        -
        -
        -<%= if downloading %> -<% formatSize done %> of <% formatSize total %> (<% formatSize download.speed %>/s) -<% else %> -<% statusmsg %> -<%/if%> - - <% formatTime eta %> - \ No newline at end of file diff --git a/module/web/app/templates/default/header/progressbar.html b/module/web/app/templates/default/header/progressbar.html deleted file mode 100644 index 41645c92f..000000000 --- a/module/web/app/templates/default/header/progressbar.html +++ /dev/null @@ -1,27 +0,0 @@ - <%= if single %> - <% name %> (<% statusmsg %>) - <% else %> - <%= if downloads %> - <% downloads %> downloads running <%= if speed %>(<% formatSize speed %>/s)<%/if%> - <% else %> - No running tasks - <%/if%> - <%/if%> - - -
        - <%= if single %> -
        - <% else %> -
        - <%/if%> -
        -
        -
        - <%= if linksqueue %> - <% linksqueue %> downloads left (<% formatSize sizequeue %>) - <%/if%> - - <% formatTime etaqueue %> - -
        \ No newline at end of file diff --git a/module/web/app/templates/default/header/status.html b/module/web/app/templates/default/header/status.html deleted file mode 100644 index 3a22bb75b..000000000 --- a/module/web/app/templates/default/header/status.html +++ /dev/null @@ -1,3 +0,0 @@ -<% formatSize maxspeed %>/s
        -<% paused %>
        -<%= if reconnect %>true<% else %>false<%/if%> \ No newline at end of file diff --git a/module/web/app/templates/default/login.html b/module/web/app/templates/default/login.html deleted file mode 100644 index 9e8d9eeb6..000000000 --- a/module/web/app/templates/default/login.html +++ /dev/null @@ -1,28 +0,0 @@ -
        - -
        - diff --git a/module/web/app/templates/default/notification.html b/module/web/app/templates/default/notification.html deleted file mode 100644 index 0f9b2c9d2..000000000 --- a/module/web/app/templates/default/notification.html +++ /dev/null @@ -1,11 +0,0 @@ -<%= if queries %> - -Queries <% queries %> - -<%/if%> -<%= if notifications %> - -Notifications <% notifications %> - -<%/if%> - \ No newline at end of file diff --git a/module/web/app/templates/default/settings/actionbar.html b/module/web/app/templates/default/settings/actionbar.html deleted file mode 100644 index 25b10d463..000000000 --- a/module/web/app/templates/default/settings/actionbar.html +++ /dev/null @@ -1,5 +0,0 @@ -
        -
        - - - \ No newline at end of file diff --git a/module/web/app/templates/default/settings/config.html b/module/web/app/templates/default/settings/config.html deleted file mode 100644 index a9ca6214c..000000000 --- a/module/web/app/templates/default/settings/config.html +++ /dev/null @@ -1,17 +0,0 @@ - -
        -

        <% label %> - <% description %> - <%= if long_description %> - - <%/if%> -

        -
        -         -
        -
        -
        - - -
        \ No newline at end of file diff --git a/module/web/app/templates/default/settings/configItem.html b/module/web/app/templates/default/settings/configItem.html deleted file mode 100644 index 3ddf16c84..000000000 --- a/module/web/app/templates/default/settings/configItem.html +++ /dev/null @@ -1,7 +0,0 @@ -
        - - -
        - -
        -
        \ No newline at end of file diff --git a/module/web/app/templates/default/settings/layout.html b/module/web/app/templates/default/settings/layout.html deleted file mode 100644 index 39f1a2ec9..000000000 --- a/module/web/app/templates/default/settings/layout.html +++ /dev/null @@ -1,11 +0,0 @@ -
        -
          -
        -
        -
        -
        -
        -

        Please choose a config section

        -
        -
        -
        \ No newline at end of file diff --git a/module/web/app/templates/default/settings/menu.html b/module/web/app/templates/default/settings/menu.html deleted file mode 100644 index ef814414a..000000000 --- a/module/web/app/templates/default/settings/menu.html +++ /dev/null @@ -1,40 +0,0 @@ -<%=if core%> -
      • General
        • -<%= each core%> -
      • <% label %>
        • -<%/each%> -<%/if%> -
      • -
      • Addons
        • -<%= each addon %> -
      • - - <% label %> - - <%= if activated %> -
        - active - <%else%> -
        - inactive - <%/if%> - <%= if user_context %> - - - <%else%> - - <%/if%> -
        -         -
        • -<%/each%> -
      • -
      • Plugin Configs
        • -<%= each plugin %> -
      • - - <% label %> - - -
        • -<%/each%> \ No newline at end of file diff --git a/module/web/app/templates/default/setup.html b/module/web/app/templates/default/setup.html deleted file mode 100644 index e5c9f4b8c..000000000 --- a/module/web/app/templates/default/setup.html +++ /dev/null @@ -1,16 +0,0 @@ -{% extends 'default/base.html' %} -{% block title %} - {{_("Setup")}} - {{ super()}} -{% endblock %} - -{% block content %} -
        -

        You did it!

        -

        pyLoad is running and ready for configuration.

        -

        - - Go on - -

        -
        -{% endblock %} \ No newline at end of file diff --git a/module/web/app/unavailable.html b/module/web/app/unavailable.html deleted file mode 100644 index 6706a693c..000000000 --- a/module/web/app/unavailable.html +++ /dev/null @@ -1,18 +0,0 @@ - - - - WebUI not available - - - -

        WebUI not available

        -You are running a pyLoad version without prebuilt webUI. You can download a build from our website or deactivate the dev mode. -If desired you can build it yourself by running: -
          -
        • npm install
          • -
        • bower install
          • -
        • grunt build
          • -
        - - - \ No newline at end of file diff --git a/module/web/bower.json b/module/web/bower.json deleted file mode 100644 index 6d8adb8a7..000000000 --- a/module/web/bower.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "name": "pyload", - "version": "0.1.0", - "dependencies": { - "requirejs": "~2.1.6", - "requirejs-text": "*", - "require-handlebars-plugin": "*", - "jquery": "~1.9.1", - "jquery.transit": "~0.9.9", - "jquery.cookie": "~1.3.1", - "jquery.animate-enhanced": "*", - "flot": "~0.8.1", - "underscore": "~1.4.4", - "backbone": "~1.0.0", - "backbone.marionette": "~1.0.3", - "handlebars.js": "~1.0.0", - "jed": "~0.5.4", - "select2": "~3.4.0", - "bootstrap": "~2.3.2", - "font-awesome": "~3.1.1" - }, - "devDependencies": {} -} diff --git a/module/web/cnl_app.py b/module/web/cnl_app.py deleted file mode 100644 index b6a98a0a8..000000000 --- a/module/web/cnl_app.py +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -from os.path import join -import re -from urllib import unquote -from base64 import standard_b64decode -from binascii import unhexlify - -from module.utils.fs import save_filename - -from bottle import route, request, HTTPError -from webinterface import PYLOAD, DL_ROOT, JS - -try: - from Crypto.Cipher import AES -except: - pass - - -def local_check(function): - def _view(*args, **kwargs): - if request.environ.get('REMOTE_ADDR', "0") in ('127.0.0.1', 'localhost') \ - or request.environ.get('HTTP_HOST','0') in ('127.0.0.1:9666', 'localhost:9666'): - return function(*args, **kwargs) - else: - return HTTPError(403, "Forbidden") - - return _view - - -@route("/flash") -@route("/flash/:id") -@route("/flash", method="POST") -@local_check -def flash(id="0"): - return "JDownloader\r\n" - -@route("/flash/add", method="POST") -@local_check -def add(request): - package = request.POST.get('referer', None) - urls = filter(lambda x: x != "", request.POST['urls'].split("\n")) - - if package: - PYLOAD.addPackage(package, urls, 0) - else: - PYLOAD.generateAndAddPackages(urls, 0) - - return "" - -@route("/flash/addcrypted", method="POST") -@local_check -def addcrypted(): - - package = request.forms.get('referer', 'ClickAndLoad Package') - dlc = request.forms['crypted'].replace(" ", "+") - - dlc_path = join(DL_ROOT, save_filename(package) + ".dlc") - dlc_file = open(dlc_path, "wb") - dlc_file.write(dlc) - dlc_file.close() - - try: - PYLOAD.addPackage(package, [dlc_path], 0) - except: - return HTTPError() - else: - return "success\r\n" - -@route("/flash/addcrypted2", method="POST") -@local_check -def addcrypted2(): - - package = request.forms.get("source", None) - crypted = request.forms["crypted"] - jk = request.forms["jk"] - - crypted = standard_b64decode(unquote(crypted.replace(" ", "+"))) - if JS: - jk = "%s f()" % jk - jk = JS.eval(jk) - - else: - try: - jk = re.findall(r"return ('|\")(.+)('|\")", jk)[0][1] - except: - ## Test for some known js functions to decode - if jk.find("dec") > -1 and jk.find("org") > -1: - org = re.findall(r"var org = ('|\")([^\"']+)", jk)[0][1] - jk = list(org) - jk.reverse() - jk = "".join(jk) - else: - print "Could not decrypt key, please install py-spidermonkey or ossp-js" - - try: - Key = unhexlify(jk) - except: - print "Could not decrypt key, please install py-spidermonkey or ossp-js" - return "failed" - - IV = Key - - obj = AES.new(Key, AES.MODE_CBC, IV) - result = obj.decrypt(crypted).replace("\x00", "").replace("\r","").split("\n") - - result = filter(lambda x: x != "", result) - - try: - if package: - PYLOAD.addPackage(package, result, 0) - else: - PYLOAD.generateAndAddPackages(result, 0) - except: - return "failed can't add" - else: - return "success\r\n" - -@route("/flashgot_pyload") -@route("/flashgot_pyload", method="POST") -@route("/flashgot") -@route("/flashgot", method="POST") -@local_check -def flashgot(): - if request.environ['HTTP_REFERER'] != "http://localhost:9666/flashgot" and request.environ['HTTP_REFERER'] != "http://127.0.0.1:9666/flashgot": - return HTTPError() - - autostart = int(request.forms.get('autostart', 0)) - package = request.forms.get('package', None) - urls = filter(lambda x: x != "", request.forms['urls'].split("\n")) - folder = request.forms.get('dir', None) - - if package: - PYLOAD.addPackage(package, urls, autostart) - else: - PYLOAD.generateAndAddPackages(urls, autostart) - - return "" - -@route("/crossdomain.xml") -@local_check -def crossdomain(): - rep = "\n" - rep += "\n" - rep += "\n" - rep += "\n" - rep += "" - return rep - - -@route("/flash/checkSupportForUrl") -@local_check -def checksupport(): - - url = request.GET.get("url") - res = PYLOAD.checkURLs([url]) - supported = (not res[0][1] is None) - - return str(supported).lower() - -@route("/jdcheck.js") -@local_check -def jdcheck(): - rep = "jdownloader=true;\n" - rep += "var version='9.581;'" - return rep diff --git a/module/web/middlewares.py b/module/web/middlewares.py deleted file mode 100644 index ae0911cc3..000000000 --- a/module/web/middlewares.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# gzip is optional on some platform -try: - import gzip -except ImportError: - gzip = None - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO - -class StripPathMiddleware(object): - def __init__(self, app): - self.app = app - - def __call__(self, e, h): - e['PATH_INFO'] = e['PATH_INFO'].rstrip('/') - return self.app(e, h) - - -class PrefixMiddleware(object): - def __init__(self, app, prefix="/pyload"): - self.app = app - self.prefix = prefix - - def __call__(self, e, h): - path = e["PATH_INFO"] - if path.startswith(self.prefix): - e['PATH_INFO'] = path.replace(self.prefix, "", 1) - return self.app(e, h) - -# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org) -# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php - -# WSGI middleware -# Gzip-encodes the response. - -class GZipMiddleWare(object): - - def __init__(self, application, compress_level=6): - self.application = application - self.compress_level = int(compress_level) - - def __call__(self, environ, start_response): - if 'gzip' not in environ.get('HTTP_ACCEPT_ENCODING', ''): - # nothing for us to do, so this middleware will - # be a no-op: - return self.application(environ, start_response) - response = GzipResponse(start_response, self.compress_level) - app_iter = self.application(environ, - response.gzip_start_response) - if app_iter is not None: - response.finish_response(app_iter) - - return response.write() - -def header_value(headers, key): - for header, value in headers: - if key.lower() == header.lower(): - return value - -def update_header(headers, key, value): - remove_header(headers, key) - headers.append((key, value)) - -def remove_header(headers, key): - for header, value in headers: - if key.lower() == header.lower(): - headers.remove((header, value)) - break - -class GzipResponse(object): - - def __init__(self, start_response, compress_level): - self.start_response = start_response - self.compress_level = compress_level - self.buffer = StringIO() - self.compressible = False - self.content_length = None - self.headers = () - - def gzip_start_response(self, status, headers, exc_info=None): - self.headers = headers - ct = header_value(headers,'content-type') - ce = header_value(headers,'content-encoding') - cl = header_value(headers, 'content-length') - - # don't compress on unknown size, it may be too huge - cl = int(cl) if cl else 0 - - if ce: - self.compressible = False - elif gzip is not None and ct and (ct.startswith('text/') or ct.startswith('application/')) \ - and 'zip' not in ct and 200 < cl < 1024*1024: - self.compressible = True - headers.append(('content-encoding', 'gzip')) - headers.append(('vary', 'Accept-Encoding')) - - remove_header(headers, 'content-length') - self.headers = headers - self.status = status - return self.buffer.write - - def write(self): - out = self.buffer - out.seek(0) - s = out.getvalue() - out.close() - return [s] - - def finish_response(self, app_iter): - if self.compressible: - output = gzip.GzipFile(mode='wb', compresslevel=self.compress_level, - fileobj=self.buffer) - else: - output = self.buffer - try: - for s in app_iter: - output.write(s) - if self.compressible: - output.close() - finally: - if hasattr(app_iter, 'close'): - try: - app_iter.close() - except : - pass - - content_length = self.buffer.tell() - update_header(self.headers, "Content-Length" , str(content_length)) - self.start_response(self.status, self.headers) \ No newline at end of file diff --git a/module/web/package.json b/module/web/package.json deleted file mode 100644 index aaf1c9a8d..000000000 --- a/module/web/package.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "pyload", - "version": "0.1.0", - "dependencies": {}, - "devDependencies": { - "grunt": "~0.4.1", - "grunt-contrib-copy": "~0.4.1", - "grunt-contrib-concat": "~0.1.3", - "grunt-contrib-uglify": "~0.2.0", - "grunt-contrib-jshint": "~0.4.1", - "grunt-contrib-less": "~0.5.2", - "grunt-contrib-cssmin": "~0.6.0", - "grunt-contrib-connect": "~0.2.0", - "grunt-contrib-clean": "~0.4.0", - "grunt-contrib-htmlmin": "~0.1.3", - "grunt-bower-requirejs": "~0.4.3", - "grunt-contrib-requirejs": "~0.4.0", - "grunt-contrib-imagemin": "~0.1.3", - "grunt-contrib-watch": "~0.4.0", - "grunt-rev": "~0.1.0", - "grunt-usemin": "~0.1.10", - "grunt-mocha": "~0.3.0", - "grunt-open": "~0.2.0", - "grunt-svgmin": "~0.1.0", - "grunt-concurrent": "~0.1.0", - "matchdep": "~0.1.1", - "connect-livereload": "~0.1.1" - }, - "engines": { - "node": ">=0.8.0" - } -} diff --git a/module/web/pyload_app.py b/module/web/pyload_app.py deleted file mode 100644 index 724fddec2..000000000 --- a/module/web/pyload_app.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" -import time -from os.path import join, exists - -from bottle import route, static_file, response, redirect - -from webinterface import PROJECT_DIR, SETUP, DEVELOP - -from utils import login_required - -########## -# Helper -########## - -app_path = "app" -UNAVAILALBE = False - -# webUI build is available -if exists(join(PROJECT_DIR, "dist", "index.html")) and not DEVELOP: - app_path = "dist" -elif not exists(join(PROJECT_DIR, "app", "components")) or not exists(join(PROJECT_DIR, ".tmp")): - UNAVAILALBE = True - - -@route('/icons/') -def serve_icon(path): - # TODO - return redirect('/images/icon.png') - # return static_file(path, root=join("tmp", "icons")) - -@route("/download/:fid") -@login_required('Download') -def download(fid, api): - path, name = api.getFilePath(fid) - return static_file(name, path, download=True) - - -@route('/') -def index(): - if UNAVAILALBE: - return server_static("unavailable.html") - - if SETUP: - # TODO show different page - pass - - # TODO: render it as simple template with configuration - return server_static('index.html') - -# Very last route that is registered, could match all uris -@route('/') -def server_static(path): - response.headers['Expires'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", - time.gmtime(time.time() + 60 * 60 * 24 * 7)) - response.headers['Cache-control'] = "public" - resp = static_file(path, root=join(PROJECT_DIR, app_path)) - # Also serve from .tmp folder in dev mode - if resp.status_code == 404 and app_path == "app": - return static_file(path, root=join(PROJECT_DIR, '.tmp')) - - return resp \ No newline at end of file diff --git a/module/web/servers.py b/module/web/servers.py deleted file mode 100644 index a3c51e36b..000000000 --- a/module/web/servers.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from bottle import ServerAdapter as BaseAdapter - -class ServerAdapter(BaseAdapter): - SSL = False - NAME = "" - - def __init__(self, host, port, key, cert, connections, debug, **kwargs): - BaseAdapter.__init__(self, host, port, **kwargs) - self.key = key - self.cert = cert - self.connection = connections - self.debug = debug - - @classmethod - def find(cls): - """ Check if server is available by trying to import it - - :raises Exception: importing C dependant library could also fail with other reasons - :return: True on success - """ - try: - __import__(cls.NAME) - return True - except ImportError: - return False - - def run(self, handler): - raise NotImplementedError - - -class CherryPyWSGI(ServerAdapter): - SSL = True - NAME = "threaded" - - @classmethod - def find(cls): - return True - - def run(self, handler): - from wsgiserver import CherryPyWSGIServer - - if self.cert and self.key: - CherryPyWSGIServer.ssl_certificate = self.cert - CherryPyWSGIServer.ssl_private_key = self.key - - server = CherryPyWSGIServer((self.host, self.port), handler, numthreads=self.connection) - server.start() - - -class FapwsServer(ServerAdapter): - """ Does not work very good currently """ - - NAME = "fapws" - - def run(self, handler): # pragma: no cover - import fapws._evwsgi as evwsgi - from fapws import base, config - - port = self.port - if float(config.SERVER_IDENT[-2:]) > 0.4: - # fapws3 silently changed its API in 0.5 - port = str(port) - evwsgi.start(self.host, port) - evwsgi.set_base_module(base) - - def app(environ, start_response): - environ['wsgi.multiprocess'] = False - return handler(environ, start_response) - - evwsgi.wsgi_cb(('', app)) - evwsgi.run() - - -# TODO: ssl -class MeinheldServer(ServerAdapter): - SSL = True - NAME = "meinheld" - - def run(self, handler): - from meinheld import server - - if self.quiet: - server.set_access_logger(None) - server.set_error_logger(None) - - server.listen((self.host, self.port)) - server.run(handler) - -# todo:ssl -class TornadoServer(ServerAdapter): - """ The super hyped asynchronous server by facebook. Untested. """ - - SSL = True - NAME = "tornado" - - def run(self, handler): # pragma: no cover - import tornado.wsgi, tornado.httpserver, tornado.ioloop - - container = tornado.wsgi.WSGIContainer(handler) - server = tornado.httpserver.HTTPServer(container) - server.listen(port=self.port) - tornado.ioloop.IOLoop.instance().start() - - -class BjoernServer(ServerAdapter): - """ Fast server written in C: https://github.com/jonashaag/bjoern """ - - NAME = "bjoern" - - def run(self, handler): - from bjoern import run - - run(handler, self.host, self.port) - - -# todo: ssl -class EventletServer(ServerAdapter): - - SSL = True - NAME = "eventlet" - - def run(self, handler): - from eventlet import wsgi, listen - - try: - wsgi.server(listen((self.host, self.port)), handler, - log_output=(not self.quiet)) - except TypeError: - # Needed to ignore the log - class NoopLog: - def write(self, *args): - pass - - # Fallback, if we have old version of eventlet - wsgi.server(listen((self.host, self.port)), handler, log=NoopLog()) - - -class FlupFCGIServer(ServerAdapter): - - SSL = False - NAME = "flup" - - def run(self, handler): # pragma: no cover - import flup.server.fcgi - from flup.server.threadedserver import ThreadedServer - - def noop(*args, **kwargs): - pass - - # Monkey patch signal handler, it does not work from threads - ThreadedServer._installSignalHandlers = noop - - self.options.setdefault('bindAddress', (self.host, self.port)) - flup.server.fcgi.WSGIServer(handler, **self.options).run() - -# Order is important and gives every server precedence over others! -all_server = [BjoernServer, TornadoServer, EventletServer, CherryPyWSGI] -# Some are deactivated because they have some flaws -##all_server = [FapwsServer, MeinheldServer, BjoernServer, TornadoServer, EventletServer, CherryPyWSGI] \ No newline at end of file diff --git a/module/web/setup_app.py b/module/web/setup_app.py deleted file mode 100644 index cd44ad08e..000000000 --- a/module/web/setup_app.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from bottle import route, request, response, HTTPError, redirect - -from webinterface import PROJECT_DIR, SETUP - -def setup_required(func): - def _view(*args, **kwargs): - # setup needs to be running - if SETUP is None: - redirect("/nopermission") - - return func(*args, **kwargs) - return _view - - -@route("/setup") -@setup_required -def setup(): - pass # TODO diff --git a/module/web/utils.py b/module/web/utils.py deleted file mode 100644 index b5a933b26..000000000 --- a/module/web/utils.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from bottle import request, HTTPError, redirect - -from webinterface import PYLOAD, SETUP - - -def set_session(request, user): - s = request.environ.get('beaker.session') - s["uid"] = user.uid - s.save() - return s - -def get_user_api(s): - if s: - uid = s.get("uid", None) - if (uid is not None) and (PYLOAD is not None): - return PYLOAD.withUserContext(uid) - return None - -def is_mobile(): - if request.get_cookie("mobile"): - if request.get_cookie("mobile") == "True": - return True - else: - return False - mobile_ua = request.headers.get('User-Agent', '').lower() - if mobile_ua.find('opera mini') > 0: - return True - if mobile_ua.find('windows') > 0: - return False - if request.headers.get('Accept', '').lower().find('application/vnd.wap.xhtml+xml') > 0: - return True - if re.search('(up.browser|up.link|mmp|symbian|smartphone|midp|wap|phone|android)', mobile_ua) is not None: - return True - mobile_ua = mobile_ua[:4] - mobile_agents = ['w3c ','acs-','alav','alca','amoi','audi','avan','benq','bird','blac','blaz','brew','cell','cldc','cmd-', - 'dang','doco','eric','hipt','inno','ipaq','java','jigs','kddi','keji','leno','lg-c','lg-d','lg-g','lge-', - 'maui','maxo','midp','mits','mmef','mobi','mot-','moto','mwbp','nec-','newt','noki','palm','pana','pant', - 'phil','play','port','prox','qwap','sage','sams','sany','sch-','sec-','send','seri','sgh-','shar','sie-', - 'siem','smal','smar','sony','sph-','symb','t-mo','teli','tim-','tosh','tsm-','upg1','upsi','vk-v','voda', - 'wap-','wapa','wapi','wapp','wapr','webc','winw','winw','xda ','xda-'] - if mobile_ua in mobile_agents: - return True - return False - - -def login_required(perm=None): - def _dec(func): - def _view(*args, **kwargs): - - # In case of setup, no login methods can be accessed - if SETUP is not None: - redirect("/setup") - - s = request.environ.get('beaker.session') - api = get_user_api(s) - if api is not None: - if perm: - if api.user.hasPermission(perm): - if request.headers.get('X-Requested-With') == 'XMLHttpRequest': - return HTTPError(403, "Forbidden") - else: - return redirect("/nopermission") - - kwargs["api"] = api - return func(*args, **kwargs) - else: - if request.headers.get('X-Requested-With') == 'XMLHttpRequest': - return HTTPError(403, "Forbidden") - else: - return redirect("/login") - - return _view - - return _dec diff --git a/module/web/webinterface.py b/module/web/webinterface.py deleted file mode 100644 index 30c7d141f..000000000 --- a/module/web/webinterface.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" - -import sys - -from os.path import join, abspath, dirname - -PROJECT_DIR = abspath(dirname(__file__)) -PYLOAD_DIR = abspath(join(PROJECT_DIR, "..", "..")) - -sys.path.append(PYLOAD_DIR) - -from module import InitHomeDir - -import bottle -from bottle import run, app - -from middlewares import StripPathMiddleware, GZipMiddleWare, PrefixMiddleware - -SETUP = None -PYLOAD = None - -from module.web import ServerThread - -if not ServerThread.core: - if ServerThread.setup: - SETUP = ServerThread.setup - config = SETUP.config - else: - raise Exception("Could not access pyLoad Core") -else: - PYLOAD = ServerThread.core.api - config = ServerThread.core.config - -from module.utils.JsEngine import JsEngine - -JS = JsEngine() - -TEMPLATE = config.get('webinterface', 'template') -DL_ROOT = config.get('general', 'download_folder') -PREFIX = config.get('webinterface', 'prefix') -DEVELOP = config.get('webinterface', 'develop') - -if PREFIX: - PREFIX = PREFIX.rstrip("/") - if PREFIX and not PREFIX.startswith("/"): - PREFIX = "/" + PREFIX - -DEBUG = config.get("general", "debug_mode") or "-d" in sys.argv or "--debug" in sys.argv -bottle.debug(DEBUG) - - -# Middlewares -from beaker.middleware import SessionMiddleware - -session_opts = { - 'session.type': 'file', - 'session.cookie_expires': False, - 'session.data_dir': './tmp', - 'session.auto': False -} - -session = SessionMiddleware(app(), session_opts) -web = StripPathMiddleware(session) -web = GZipMiddleWare(web) - -if PREFIX: - web = PrefixMiddleware(web, prefix=PREFIX) - -import api_app -import cnl_app -import setup_app -# Last routes to register, -import pyload_app - -# Server Adapter -def run_server(host, port, server): - run(app=web, host=host, port=port, quiet=True, server=server) - - -if __name__ == "__main__": - run(app=web, port=8001) diff --git a/pavement.py b/pavement.py index 4a7b5c0b0..3682281b3 100644 --- a/pavement.py +++ b/pavement.py @@ -26,9 +26,7 @@ path.fnmatch = new_fnmatch import sys import re -from urllib import urlretrieve from subprocess import call, Popen -from zipfile import ZipFile PROJECT_DIR = path(__file__).dirname() sys.path.append(PROJECT_DIR) @@ -59,7 +57,7 @@ setup( include_package_data=True, exclude_package_data={'pyload': ['docs*', 'scripts*', 'tests*']}, #exluced from build but not from sdist # 'bottle >= 0.10.0' not in list, because its small and contain little modifications - install_requires=['pycurl', 'jinja2 >= 2.6', 'Beaker >= 1.6'] + extradeps, + install_requires=['pycurl', 'Beaker >= 1.6'] + extradeps, extras_require={ 'SSL': ["pyOpenSSL"], 'DLC': ['pycrypto'], @@ -70,12 +68,12 @@ setup( #setup_requires=["setuptools_hg"], test_suite='nose.collector', tests_require=['nose', 'websocket-client >= 0.8.0', 'requests >= 1.2.2'], -# scripts=['pyload', 'pyload-cli'], - entry_points={ - 'console_scripts': [ - 'pyload = pyload:main', - 'pyload-cli = pyload_cli:main' #TODO fix - ]}, + scripts=['pyload.py', 'pyload-cli.py'], + # entry_points={ + # 'console_scripts': [ + # 'pyload = pyload:main', + # 'pyload-cli = pyload_cli:main' #TODO fix + # ]}, zip_safe=False, classifiers=[ "Development Status :: 5 - Production/Stable", @@ -94,11 +92,6 @@ options( builddir="_build", sourcedir="" ), - get_source=Bunch( - src="https://bitbucket.org/spoob/pyload/get/tip.zip", - rev=None, - clean=False - ), apitypes=Bunch( path="thrift", ), @@ -116,7 +109,6 @@ options( xargs = ["--from-code=utf-8", "--copyright-holder=pyLoad Team", "--package-name=pyLoad", "--package-version=%s" % options.version, "--msgid-bugs-address='bugs@pyload.org'"] - @task @needs('cog') def html(): @@ -127,50 +119,7 @@ def html(): @task -@cmdopts([ - ('src=', 's', 'Url to source'), - ('rev=', 'r', "HG revision"), - ("clean", 'c', 'Delete old source folder') -]) -def get_source(options): - """ Downloads pyload source from bitbucket tip or given rev""" - if options.rev: options.url = "https://bitbucket.org/spoob/pyload/get/%s.zip" % options.rev - - pyload = path("pyload") - - if len(pyload.listdir()) and not options.clean: - return - elif pyload.exists(): - pyload.rmtree() - - urlretrieve(options.src, "pyload_src.zip") - zip = ZipFile("pyload_src.zip") - zip.extractall() - path("pyload_src.zip").remove() - - folder = [x for x in path(".").dirs() if x.name.startswith("spoob-pyload-")][0] - folder.move(pyload) - - change_mode(pyload, 0644) - change_mode(pyload, 0755, folder=True) - - for file in pyload.files(): - if file.name.endswith(".py"): - file.chmod(0755) - - (pyload / ".hgtags").remove() - (pyload / ".hgignore").remove() - #(pyload / "docs").rmtree() - - f = open(pyload / "__init__.py", "wb") - f.close() - - #options.setup.packages = find_packages() - #options.setup.package_data = find_package_data() - - -@task -@needs('clean', 'generate_setup', 'get_source', 'setuptools.command.sdist') +@needs('clean', 'generate_setup', 'webapp', 'setuptools.command.sdist') def sdist(): """ Build source code package with distutils """ diff --git a/pyload-cli.py b/pyload-cli.py index 684958f1e..b91e7d215 100755 --- a/pyload-cli.py +++ b/pyload-cli.py @@ -1,590 +1,20 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- + +############################################################################### +# Copyright(c) 2008-2013 pyLoad Team +# http://www.pyload.org # -#Copyright (C) 2012 RaNaN -# -#This program is free software; you can redistribute it and/or modify -#it under the terms of the GNU General Public License as published by -#the Free Software Foundation; either version 3 of the License, -#or (at your option) any later version. -# -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#See the GNU General Public License for more details. -# -#You should have received a copy of the GNU General Public License -# along with this program; if not, see . +# This file is part of pyLoad. +# pyLoad is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. # -### -from __future__ import with_statement -from getopt import GetoptError, getopt - -import module.common.pylgettext as gettext -import os -from os import _exit -from os.path import join, exists, abspath, basename -import sys -from sys import exit -from threading import Thread, Lock -from time import sleep -from traceback import print_exc - -import ConfigParser - -from codecs import getwriter - -if os.name == "nt": - enc = "cp850" -else: - enc = "utf8" - -sys.stdout = getwriter(enc)(sys.stdout, errors="replace") - -from module import InitHomeDir -from module.cli.printer import * -from module.cli import AddPackage, ManageFiles - -from module.Api import Destination -from module.utils import formatSize, decode -from module.remote.thriftbackend.ThriftClient import ThriftClient, NoConnection, NoSSL, WrongLogin, ConnectionClosed -from module.lib.Getch import Getch -from module.lib.rename_process import renameProcess - -class Cli: - def __init__(self, client, command): - self.client = client - self.command = command - - if not self.command: - renameProcess('pyLoadCli') - self.getch = Getch() - self.input = "" - self.inputline = 0 - self.lastLowestLine = 0 - self.menuline = 0 - - self.lock = Lock() - - #processor funcions, these will be changed dynamically depending on control flow - self.headerHandler = self #the download status - self.bodyHandler = self #the menu section - self.inputHandler = self - - os.system("clear") - println(1, blue("py") + yellow("Load") + white(_(" Command Line Interface"))) - println(2, "") - - self.thread = RefreshThread(self) - self.thread.start() - - self.start() - else: - self.processCommand() - - def reset(self): - """ reset to initial main menu """ - self.input = "" - self.headerHandler = self.bodyHandler = self.inputHandler = self - - def start(self): - """ main loop. handle input """ - while True: - #inp = raw_input() - inp = self.getch.impl() - if ord(inp) == 3: - os.system("clear") - sys.exit() # ctrl + c - elif ord(inp) == 13: #enter - try: - self.lock.acquire() - self.inputHandler.onEnter(self.input) - - except Exception, e: - println(2, red(e)) - finally: - self.lock.release() - - elif ord(inp) == 127: - self.input = self.input[:-1] #backspace - try: - self.lock.acquire() - self.inputHandler.onBackSpace() - finally: - self.lock.release() - - elif ord(inp) == 27: #ugly symbol - pass - else: - self.input += inp - try: - self.lock.acquire() - self.inputHandler.onChar(inp) - finally: - self.lock.release() - - self.inputline = self.bodyHandler.renderBody(self.menuline) - self.renderFooter(self.inputline) - - - def refresh(self): - """refresh screen""" - - println(1, blue("py") + yellow("Load") + white(_(" Command Line Interface"))) - println(2, "") - - self.lock.acquire() - - self.menuline = self.headerHandler.renderHeader(3) + 1 - println(self.menuline - 1, "") - self.inputline = self.bodyHandler.renderBody(self.menuline) - self.renderFooter(self.inputline) - - self.lock.release() - - - def setInput(self, string=""): - self.input = string - - def setHandler(self, klass): - #create new handler with reference to cli - self.bodyHandler = self.inputHandler = klass(self) - self.input = "" - - def renderHeader(self, line): - """ prints download status """ - #print updated information - # print "\033[J" #clear screen - # self.println(1, blue("py") + yellow("Load") + white(_(" Command Line Interface"))) - # self.println(2, "") - # self.println(3, white(_("%s Downloads:") % (len(data)))) - - data = self.client.statusDownloads() - speed = 0 - - println(line, white(_("%s Downloads:") % (len(data)))) - line += 1 - - for download in data: - if download.status == 12: # downloading - percent = download.percent - z = percent / 4 - speed += download.speed - println(line, cyan(download.name)) - line += 1 - println(line, - blue("[") + yellow(z * "#" + (25 - z) * " ") + blue("] ") + green(str(percent) + "%") + _( - " Speed: ") + green(formatSize(download.speed) + "/s") + _(" Size: ") + green( - download.format_size) + _(" Finished in: ") + green(download.format_eta) + _( - " ID: ") + green(download.fid)) - line += 1 - if download.status == 5: - println(line, cyan(download.name)) - line += 1 - println(line, _("waiting: ") + green(download.format_wait)) - line += 1 - - println(line, "") - line += 1 - status = self.client.statusServer() - if status.pause: - paused = _("Status:") + " " + red(_("paused")) - else: - paused = _("Status:") + " " + red(_("running")) - - println(line,"%s %s: %s %s: %s %s: %s" % ( - paused, _("total Speed"), red(formatSize(speed) + "/s"), _("Files in queue"), red( - status.queue), _("Total"), red(status.total))) - - return line + 1 - - def renderBody(self, line): - """ prints initial menu """ - println(line, white(_("Menu:"))) - println(line + 1, "") - println(line + 2, mag("1.") + _(" Add Links")) - println(line + 3, mag("2.") + _(" Manage Queue")) - println(line + 4, mag("3.") + _(" Manage Collector")) - println(line + 5, mag("4.") + _(" (Un)Pause Server")) - println(line + 6, mag("5.") + _(" Kill Server")) - println(line + 7, mag("6.") + _(" Quit")) - - return line + 8 - - def renderFooter(self, line): - """ prints out the input line with input """ - println(line, "") - line += 1 - - println(line, white(" Input: ") + decode(self.input)) - - #clear old output - if line < self.lastLowestLine: - for i in range(line + 1, self.lastLowestLine + 1): - println(i, "") - - self.lastLowestLine = line - - #set cursor to position - print "\033[" + str(self.inputline) + ";0H" - - def onChar(self, char): - """ default no special handling for single chars """ - if char == "1": - self.setHandler(AddPackage) - elif char == "2": - self.setHandler(ManageFiles) - elif char == "3": - self.setHandler(ManageFiles) - self.bodyHandler.target = Destination.Collector - elif char == "4": - self.client.togglePause() - self.setInput() - elif char == "5": - self.client.kill() - self.client.close() - sys.exit() - elif char == "6": - os.system('clear') - sys.exit() - - def onEnter(self, inp): - pass - - def onBackSpace(self): - pass - - def processCommand(self): - command = self.command[0] - args = [] - if len(self.command) > 1: - args = self.command[1:] - - if command == "status": - files = self.client.statusDownloads() - - if not files: - print "No downloads running." - - for download in files: - if download.status == 12: # downloading - print print_status(download) - print "\tDownloading: %s @ %s/s\t %s (%s%%)" % ( - download.format_eta, formatSize(download.speed), formatSize(download.size - download.bleft), - download.percent) - elif download.status == 5: - print print_status(download) - print "\tWaiting: %s" % download.format_wait - else: - print print_status(download) - - elif command == "queue": - print_packages(self.client.getQueueData()) - - elif command == "collector": - print_packages(self.client.getCollectorData()) - - elif command == "add": - if len(args) < 2: - print _("Please use this syntax: add ...") - return - - self.client.addPackage(args[0], args[1:], Destination.Queue, "") - - elif command == "add_coll": - if len(args) < 2: - print _("Please use this syntax: add ...") - return - - self.client.addPackage(args[0], args[1:], Destination.Collector, "") - - elif command == "del_file": - self.client.deleteFiles([int(x) for x in args]) - print "Files deleted." - - elif command == "del_package": - self.client.deletePackages([int(x) for x in args]) - print "Packages deleted." - - elif command == "move": - for pid in args: - pack = self.client.getPackageInfo(int(pid)) - self.client.movePackage((pack.dest + 1) % 2, pack.pid) - - elif command == "check": - print _("Checking %d links:") % len(args) - print - rid = self.client.checkOnlineStatus(args).rid - self.printOnlineCheck(self.client, rid) - - - elif command == "check_container": - path = args[0] - if not exists(join(owd, path)): - print _("File does not exists.") - return - - f = open(join(owd, path), "rb") - content = f.read() - f.close() - - rid = self.client.checkOnlineStatusContainer([], basename(f.name), content).rid - self.printOnlineCheck(self.client, rid) - - - elif command == "pause": - self.client.pauseServer() - - elif command == "unpause": - self.client.unpauseServer() - - elif command == "toggle": - self.client.togglePause() - - elif command == "kill": - self.client.kill() - elif command == "restart_file": - for x in args: - self.client.restartFile(int(x)) - print "Files restarted." - elif command == "restart_package": - for pid in args: - self.client.restartPackage(int(pid)) - print "Packages restarted." - - else: - print_commands() - - def printOnlineCheck(self, client, rid): - while True: - sleep(1) - result = client.pollResults(rid) - for url, status in result.data.iteritems(): - if status.status == 2: check = "Online" - elif status.status == 1: check = "Offline" - else: check = "Unknown" - - print "%-45s %-12s\t %-15s\t %s" % (status.name, formatSize(status.size), status.plugin, check) - - if result.rid == -1: break - - -class RefreshThread(Thread): - def __init__(self, cli): - Thread.__init__(self) - self.setDaemon(True) - self.cli = cli - - def run(self): - while True: - sleep(1) - try: - self.cli.refresh() - except ConnectionClosed: - os.system("clear") - print _("pyLoad was terminated") - _exit(0) - except Exception, e: - println(2, red(str(e))) - self.cli.reset() - print_exc() - - -def print_help(config): - print - print "pyLoadCli Copyright (c) 2008-2012 the pyLoad Team" - print - print "Usage: [python] pyLoadCli.py [options] [command]" - print - print "" - print "See pyLoadCli.py -c for a complete listing." - print - print "" - print " -i, --interactive", " Start in interactive mode" - print - print " -u, --username=", " " * 2, "Specify user name" - print " --pw=", " " * 2, "Password" - print " -a, --address=", " " * 3, "Use address (current=%s)" % config["addr"] - print " -p, --port", " " * 7, "Use port (current=%s)" % config["port"] - print - print " -l, --language", " " * 3, "Set user interface language (current=%s)" % config["language"] - print " -h, --help", " " * 7, "Display this help text" - print " -c, --commands", " " * 3, "List all available commands" - print - - -def print_packages(data): - for pack in data: - print "Package %s (#%s):" % (pack.name, pack.pid) - for download in pack.links: - print "\t" + print_file(download) - print - - -def print_file(download): - return "#%(id)-6d %(name)-30s %(statusmsg)-10s %(plugin)-8s" % { - "id": download.fid, - "name": download.name, - "statusmsg": download.statusmsg, - "plugin": download.plugin - } - - -def print_status(download): - return "#%(id)-6s %(name)-40s Status: %(statusmsg)-10s Size: %(size)s" % { - "id": download.fid, - "name": download.name, - "statusmsg": download.statusmsg, - "size": download.format_size - } - - -def print_commands(): - commands = [("status", _("Prints server status")), - ("queue", _("Prints downloads in queue")), - ("collector", _("Prints downloads in collector")), - ("add ...", _("Adds package to queue")), - ("add_coll ...", _("Adds package to collector")), - ("del_file ...", _("Delete Files from Queue/Collector")), - ("del_package ...", _("Delete Packages from Queue/Collector")), - ("move ...", _("Move Packages from Queue to Collector or vice versa")), - ("restart_file ...", _("Restart files")), - ("restart_package ...", _("Restart packages")), - ("check ...", _("Check online status, works with local container")), - ("check_container path", _("Checks online status of a container file")), - ("pause", _("Pause the server")), - ("unpause", _("continue downloads")), - ("toggle", _("Toggle pause/unpause")), - ("kill", _("kill server")), ] - - print _("List of commands:") - print - for c in commands: - print "%-35s %s" % c - - -def writeConfig(opts): - try: - with open(join(homedir, ".pyloadcli"), "w") as cfgfile: - cfgfile.write("[cli]") - for opt in opts: - cfgfile.write("%s=%s\n" % (opt, opts[opt])) - except: - print _("Couldn't write user config file") - - -def main(): - config = {"addr": "127.0.0.1", "port": "7227", "language": "en"} - try: - config["language"] = os.environ["LANG"][0:2] - except: - pass - - if (not exists(join(pypath, "locale", config["language"]))) or config["language"] == "": - config["language"] = "en" - - configFile = ConfigParser.ConfigParser() - configFile.read(join(homedir, ".pyloadcli")) - - if configFile.has_section("cli"): - for opt in configFile.items("cli"): - config[opt[0]] = opt[1] - - gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) - translation = gettext.translation("pyLoadCli", join(pypath, "locale"), - languages=[config["language"],"en"],fallback=True) - translation.install(unicode=True) - - interactive = False - command = None - username = "" - password = "" - - shortOptions = 'iu:p:a:hcl:' - longOptions = ['interactive', "username=", "pw=", "address=", "port=", "help", "commands", "language="] - - try: - opts, extraparams = getopt(sys.argv[1:], shortOptions, longOptions) - for option, params in opts: - if option in ("-i", "--interactive"): - interactive = True - elif option in ("-u", "--username"): - username = params - elif option in ("-a", "--address"): - config["addr"] = params - elif option in ("-p", "--port"): - config["port"] = params - elif option in ("-l", "--language"): - config["language"] = params - gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) - translation = gettext.translation("pyLoadCli", join(pypath, "locale"), - languages=[config["language"],"en"],fallback=True) - translation.install(unicode=True) - elif option in ("-h", "--help"): - print_help(config) - exit() - elif option in ("--pw"): - password = params - elif option in ("-c", "--comands"): - print_commands() - exit() - - except GetoptError: - print 'Unknown Argument(s) "%s"' % " ".join(sys.argv[1:]) - print_help(config) - exit() - - if len(extraparams) >= 1: - command = extraparams - - client = False - - if interactive: - try: - client = ThriftClient(config["addr"], int(config["port"]), username, password) - except WrongLogin: - pass - except NoSSL: - print _("You need py-openssl to connect to this pyLoad Core.") - exit() - except NoConnection: - config["addr"] = False - config["port"] = False - - if not client: - if not config["addr"]: config["addr"] = raw_input(_("Address: ")) - if not config["port"]: config["port"] = raw_input(_("Port: ")) - if not username: username = raw_input(_("Username: ")) - if not password: - from getpass import getpass - - password = getpass(_("Password: ")) - - try: - client = ThriftClient(config["addr"], int(config["port"]), username, password) - except WrongLogin: - print _("Login data is wrong.") - except NoConnection: - print _("Could not establish connection to %(addr)s:%(port)s." % {"addr": config["addr"], - "port": config["port"]}) - - else: - try: - client = ThriftClient(config["addr"], int(config["port"]), username, password) - except WrongLogin: - print _("Login data is wrong.") - except NoConnection: - print _("Could not establish connection to %(addr)s:%(port)s." % {"addr": config["addr"], - "port": config["port"]}) - except NoSSL: - print _("You need py-openssl to connect to this pyLoad core.") - - if interactive and command: print _("Interactive mode ignored since you passed some commands.") - - if client: - writeConfig(config) - cli = Cli(client, command) +# Subjected to the terms and conditions in LICENSE +############################################################################### +from pyload.cli.Cli import main if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/pyload.py b/pyload.py index b1b76d6af..19d51ed38 100755 --- a/pyload.py +++ b/pyload.py @@ -1,607 +1,29 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -""" - Copyright(c) 2008-2013 pyLoad Team - http://www.pyload.org - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. +############################################################################### +# Copyright(c) 2008-2013 pyLoad Team +# http://www.pyload.org +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: spoob +# @author: sebnapi +# @author: RaNaN +# @author: mkaay +# @version: v0.5.0 +############################################################################### - Subjected to the terms and conditions in LICENSE - - @author: spoob - @author: sebnapi - @author: RaNaN - @author: mkaay - @version: v0.4.9 -""" -CURRENT_VERSION = '0.4.9.9-dev' - -import __builtin__ - -from getopt import getopt, GetoptError -import logging -import logging.handlers import os -from os import _exit, execl, getcwd, remove, walk, chdir, close -import signal import sys -from sys import argv, executable, exit -from time import time, sleep -from traceback import print_exc - -import locale -locale.locale_alias = locale.windows_locale = {} #save ~100kb ram, no known sideeffects for now - -import subprocess -subprocess.__doc__ = None # the module with the largest doc we are using - -from module import InitHomeDir -from module.AccountManager import AccountManager -from module.config.ConfigParser import ConfigParser -from module.config.ConfigManager import ConfigManager -from module.PluginManager import PluginManager -from module.interaction.EventManager import EventManager -from module.network.RequestFactory import RequestFactory -from module.web.ServerThread import WebServer -from module.Scheduler import Scheduler -from module.remote.RemoteManager import RemoteManager -from module.utils.JsEngine import JsEngine - -import module.utils.pylgettext as gettext -from module.utils import formatSize, get_console_encoding -from module.utils.fs import free_space, exists, makedirs, join, chmod - -from codecs import getwriter - -# test runner overwrites sys.stdout -if hasattr(sys.stdout, "encoding"): enc = get_console_encoding(sys.stdout.encoding) -else: enc = "utf8" - -sys._stdout = sys.stdout -sys.stdout = getwriter(enc)(sys.stdout, errors="replace") - -# TODO List -# - configurable auth system ldap/mysql -# - cron job like sheduler -# - plugin stack / multi decrypter -# - media plugin type -# - general progress info -# - content attribute for files / sync status -# - sync with disk content / file manager / nested packages -# - sync between pyload cores -# - new attributes (date|sync status) -# - embedded packages -# - would require new/modified link collector concept -# - pausable links/packages -# - toggable accounts -# - interaction manager -# - improve external scripts -# - make pyload undestructable to fail plugins -> see ConfigParser first - -class Core(object): - """pyLoad Core, one tool to rule them all... (the filehosters) :D""" - - def __init__(self): - self.doDebug = False - self.running = False - self.daemon = False - self.remote = True - self.pdb = None - self.arg_links = [] - self.pidfile = "pyload.pid" - self.deleteLinks = False # will delete links on startup - - if len(argv) > 1: - try: - options, args = getopt(argv[1:], 'vchdusqp:', - ["version", "clear", "clean", "help", "debug", "user", - "setup", "configdir=", "changedir", "daemon", - "quit", "status", "no-remote","pidfile="]) - - for option, argument in options: - if option in ("-v", "--version"): - print "pyLoad", CURRENT_VERSION - exit() - elif option in ("-p", "--pidfile"): - self.pidfile = argument - elif option == "--daemon": - self.daemon = True - elif option in ("-c", "--clear"): - self.deleteLinks = True - elif option in ("-h", "--help"): - self.print_help() - exit() - elif option in ("-d", "--debug"): - self.doDebug = True - elif option in ("-u", "--user"): - from module.Setup import Setup - - self.config = ConfigParser() - s = Setup(pypath, self.config) - s.set_user() - exit() - elif option in ("-s", "--setup"): - from module.Setup import Setup - - self.config = ConfigParser() - s = Setup(pypath, self.config) - s.start() - exit() - elif option == "--changedir": - from module.Setup import Setup - - self.config = ConfigParser() - s = Setup(pypath, self.config) - s.conf_path(True) - exit() - elif option in ("-q", "--quit"): - self.quitInstance() - exit() - elif option == "--status": - pid = self.isAlreadyRunning() - if self.isAlreadyRunning(): - print pid - exit(0) - else: - print "false" - exit(1) - elif option == "--clean": - self.cleanTree() - exit() - elif option == "--no-remote": - self.remote = False - - except GetoptError: - print 'Unknown Argument(s) "%s"' % " ".join(argv[1:]) - self.print_help() - exit() - - def print_help(self): - print "" - print "pyLoad v%s 2008-2013 the pyLoad Team" % CURRENT_VERSION - print "" - if sys.argv[0].endswith(".py"): - print "Usage: python pyload.py [options]" - else: - print "Usage: pyload [options]" - print "" - print "" - print " -v, --version", " " * 10, "Print version to terminal" - print " -c, --clear", " " * 12, "Delete all saved packages/links" - #print " -a, --add=", " " * 2, "Add the specified links" - print " -u, --user", " " * 13, "Manages users" - print " -d, --debug", " " * 12, "Enable debug mode" - print " -s, --setup", " " * 12, "Run setup assistant" - print " --configdir=        ", " " * 6, "Run with as configuration directory" - print " -p, --pidfile=", " " * 3, "Set pidfile to " - print " --changedir", " " * 12, "Change configuration directory permanently" - print " --daemon", " " * 15, "Daemonize after startup" - print " --no-remote", " " * 12, "Disable remote access" - print " --status", " " * 15, "Display pid if running or False" - print " --clean", " " * 16, "Remove .pyc/.pyo files" - print " -q, --quit", " " * 13, "Quit a running pyLoad instance" - print " -h, --help", " " * 13, "Display this help screen" - print "" - - - def quit(self, a, b): - self.shutdown() - self.log.info(_("Received Quit signal")) - _exit(1) - - def writePidFile(self): - self.deletePidFile() - pid = os.getpid() - f = open(self.pidfile, "wb") - f.write(str(pid)) - f.close() - chmod(self.pidfile, 0660) - - def deletePidFile(self): - if self.checkPidFile(): - self.log.debug("Deleting old pidfile %s" % self.pidfile) - os.remove(self.pidfile) - - def checkPidFile(self): - """ return pid as int or 0""" - if os.path.isfile(self.pidfile): - f = open(self.pidfile, "rb") - pid = f.read().strip() - f.close() - if pid: - pid = int(pid) - return pid - - return 0 - - def isAlreadyRunning(self): - pid = self.checkPidFile() - if not pid or os.name == "nt": return False - try: - os.kill(pid, 0) # 0 - default signal (does nothing) - except: - return 0 - - return pid - - def quitInstance(self): - if os.name == "nt": - print "Not supported on windows." - return - - pid = self.isAlreadyRunning() - if not pid: - print "No pyLoad running." - return - - try: - os.kill(pid, 3) #SIGUIT - - t = time() - print "waiting for pyLoad to quit" - - while exists(self.pidfile) and t + 10 > time(): - sleep(0.25) - - if not exists(self.pidfile): - print "pyLoad successfully stopped" - else: - os.kill(pid, 9) #SIGKILL - print "pyLoad did not respond" - print "Kill signal was send to process with id %s" % pid - - except: - print "Error quitting pyLoad" - - - def cleanTree(self): - for path, dirs, files in walk(self.path("")): - for f in files: - if not f.endswith(".pyo") and not f.endswith(".pyc"): - continue - - if "_25" in f or "_26" in f or "_27" in f: - continue - - print join(path, f) - remove(join(path, f)) - - def start(self, rpc=True, web=True, tests=False): - """ starts the fun :D """ - - self.version = CURRENT_VERSION - - # TODO: Re-enable when its working again - # TODO: Don't forget it - if False and not exists("pyload.conf") and not tests: - from module.Setup import Setup - - print "This is your first start, running configuration assistant now." - self.config = ConfigParser() - s = Setup(pypath, self.config) - res = False - try: - res = s.start() - except SystemExit: - pass - except KeyboardInterrupt: - print "\nSetup interrupted" - except: - res = False - print_exc() - print "Setup failed" - if not res: - remove("pyload.conf") - - exit() - - try: signal.signal(signal.SIGQUIT, self.quit) - except: pass - - self.config = ConfigParser() - - gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) - translation = gettext.translation("pyLoad", self.path("locale"), - languages=[self.config['general']['language'],"en"],fallback=True) - translation.install(True) - - # load again so translations are propagated - self.config.loadDefault() - - self.debug = self.doDebug or self.config['general']['debug_mode'] - self.remote &= self.config['remote']['activated'] - - pid = self.isAlreadyRunning() - # don't exit when in test runner - if pid and not tests: - print _("pyLoad already running with pid %s") % pid - exit() - - if os.name != "nt" and self.config["general"]["renice"]: - os.system("renice %d %d" % (self.config["general"]["renice"], os.getpid())) - - if self.config["permission"]["change_group"]: - if os.name != "nt": - try: - from grp import getgrnam - - group = getgrnam(self.config["permission"]["group"]) - os.setgid(group[2]) - except Exception, e: - print _("Failed changing group: %s") % e - - if self.config["permission"]["change_user"]: - if os.name != "nt": - try: - from pwd import getpwnam - - user = getpwnam(self.config["permission"]["user"]) - os.setuid(user[2]) - except Exception, e: - print _("Failed changing user: %s") % e - - if self.debug: - self.init_logger(logging.DEBUG) # logging level - else: - self.init_logger(logging.INFO) # logging level - - self.do_kill = False - self.do_restart = False - self.shuttedDown = False - - self.log.info(_("Starting") + " pyLoad %s" % CURRENT_VERSION) - self.log.info(_("Using home directory: %s") % getcwd()) - - if not tests: - self.writePidFile() +from os import _exit - self.captcha = True # checks seems to fail, although tesseract is available - - self.eventManager = self.evm = EventManager(self) - self.setupDB() - - # Upgrade to configManager - self.config = ConfigManager(self, self.config) - - if self.deleteLinks: - self.log.info(_("All links removed")) - self.db.purgeLinks() - - self.requestFactory = RequestFactory(self) - __builtin__.pyreq = self.requestFactory - - # deferred import, could improve start-up time - from module.Api import Api - from module.AddonManager import AddonManager - from module.interaction.InteractionManager import InteractionManager - from module.threads.ThreadManager import ThreadManager - - Api.initComponents() - self.api = Api(self) - - self.scheduler = Scheduler(self) - - #hell yeah, so many important managers :D - self.pluginManager = PluginManager(self) - self.interactionManager = self.im = InteractionManager(self) - self.accountManager = AccountManager(self) - self.threadManager = ThreadManager(self) - self.addonManager = AddonManager(self) - self.remoteManager = RemoteManager(self) - - self.js = JsEngine() - - # enough initialization for test cases - if tests: return - - self.log.info(_("Download time: %s") % self.api.isTimeDownload()) - - if rpc: - self.remoteManager.startBackends() - - if web: - self.init_webserver() - - dl_folder = self.config["general"]["download_folder"] - - if not exists(dl_folder): - makedirs(dl_folder) - - spaceLeft = free_space(dl_folder) - - self.log.info(_("Free space: %s") % formatSize(spaceLeft)) - - self.config.save() #save so config files gets filled - - link_file = join(pypath, "links.txt") - - if exists(link_file): - f = open(link_file, "rb") - if f.read().strip(): - self.api.addPackage("links.txt", [link_file], 1) - f.close() - - link_file = "links.txt" - if exists(link_file): - f = open(link_file, "rb") - if f.read().strip(): - self.api.addPackage("links.txt", [link_file], 1) - f.close() - - #self.scheduler.addJob(0, self.accountManager.getAccountInfos) - self.log.info(_("Activating Accounts...")) - self.accountManager.refreshAllAccounts() - - #restart failed - if self.config["download"]["restart_failed"]: - self.log.info(_("Restarting failed downloads...")) - self.api.restartFailed() - - self.threadManager.pause = False - self.running = True - - self.addonManager.activateAddons() - - self.log.info(_("pyLoad is up and running")) - self.eventManager.dispatchEvent("core:ready") - - #test api -# from module.common.APIExerciser import startApiExerciser -# startApiExerciser(self, 3) - - #some memory stats -# from guppy import hpy -# hp=hpy() -# print hp.heap() -# import objgraph -# objgraph.show_most_common_types(limit=30) -# import memdebug -# memdebug.start(8002) -# from meliae import scanner -# scanner.dump_all_objects(self.path('objs.json')) - - locals().clear() - - while True: - sleep(1.5) - if self.do_restart: - self.log.info(_("restarting pyLoad")) - self.restart() - if self.do_kill: - self.shutdown() - self.log.info(_("pyLoad quits")) - self.removeLogger() - _exit(0) - # TODO check exits codes, clean exit is still blocked - - self.threadManager.work() - self.interactionManager.work() - self.scheduler.work() - - def setupDB(self): - from module.database import DatabaseBackend - from module.FileManager import FileManager - - self.db = DatabaseBackend(self) # the backend - self.db.setup() - - self.files = FileManager(self) - self.db.manager = self.files #ugly? - - def init_webserver(self): - if self.config['webinterface']['activated']: - self.webserver = WebServer(self) - self.webserver.start() - - def init_logger(self, level): - console = logging.StreamHandler(sys.stdout) - - # try to get a time formatting depending on system locale - tfrm = None - try: # change current locale to default if it is not set - current_locale = locale.getlocale() - if current_locale == (None, None): - current_locale = locale.setlocale(locale.LC_ALL, '') - - # We use timeformat provided by locale when available - if current_locale != (None, None): - tfrm = locale.nl_langinfo(locale.D_FMT) + " " + locale.nl_langinfo(locale.T_FMT) - except: # something did go wrong, locale is heavily platform dependant - pass - - # default formatting when no one was obtained - if not tfrm: - tfrm = "%d.%m.%Y %H:%M:%S" - - frm = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s", tfrm) - console.setFormatter(frm) - self.log = logging.getLogger("log") # setable in config - - if not exists(self.config['log']['log_folder']): - makedirs(self.config['log']['log_folder'], 0700) - - if self.config['log']['file_log']: - if self.config['log']['log_rotate']: - file_handler = logging.handlers.RotatingFileHandler(join(self.config['log']['log_folder'], 'log.txt'), - maxBytes=self.config['log']['log_size'] * 1024, - backupCount=int(self.config['log']['log_count']), - encoding="utf8") - else: - file_handler = logging.FileHandler(join(self.config['log']['log_folder'], 'log.txt'), encoding="utf8") - - file_handler.setFormatter(frm) - self.log.addHandler(file_handler) - - self.log.addHandler(console) #if console logging - self.log.setLevel(level) - - def removeLogger(self): - for h in list(self.log.handlers): - self.log.removeHandler(h) - h.close() - - - def restart(self): - self.shutdown() - chdir(owd) - # close some open fds - for i in range(3,50): - try: - close(i) - except : - pass - - execl(executable, executable, *sys.argv) - _exit(0) - - def shutdown(self): - self.log.info(_("shutting down...")) - self.eventManager.dispatchEvent("coreShutdown") - try: - if self.config['webinterface']['activated'] and hasattr(self, "webserver"): - pass # TODO: quit webserver? -# self.webserver.quit() - - for thread in self.threadManager.threads: - thread.put("quit") - - self.api.stopAllDownloads() - self.addonManager.deactivateAddons() - - except: - self.print_exc() - self.log.info(_("error while shutting down")) - - finally: - self.files.syncSave() - self.db.shutdown() - self.shuttedDown = True - - self.deletePidFile() - - def shell(self): - """ stop and open an ipython shell inplace""" - if self.debug: - from IPython import embed - sys.stdout = sys._stdout - embed() - - def breakpoint(self): - if self.debug: - from IPython.core.debugger import Pdb - sys.stdout = sys._stdout - if not self.pdb: self.pdb = Pdb() - self.pdb.set_trace() - - def print_exc(self): - if self.debug: - print_exc() - - def path(self, *args): - return join(pypath, *args) +from pyload.Core import Core def deamon(): try: @@ -612,7 +34,7 @@ def deamon(): print >> sys.stderr, "fork #1 failed: %d (%s)" % (e.errno, e.strerror) sys.exit(1) - # decouple from parent environment + # decouple from parent environment os.setsid() os.umask(0) @@ -658,7 +80,6 @@ def main(): pyload_core.removeLogger() _exit(1) -# And so it begins... +# And so it begins... if __name__ == "__main__": - main() - + main() \ No newline at end of file diff --git a/pyload/AccountManager.py b/pyload/AccountManager.py new file mode 100644 index 000000000..a476c75c1 --- /dev/null +++ b/pyload/AccountManager.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +############################################################################### +# Copyright(c) 2008-2012 pyLoad Team +# http://www.pyload.org +# +# This file is part of pyLoad. +# pyLoad is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN, mkaay +############################################################################### + +from threading import Lock +from random import choice + +from pyload.utils import lock, json + +class AccountManager: + """manages all accounts""" + + def __init__(self, core): + """Constructor""" + + self.core = core + self.lock = Lock() + + self.loadAccounts() + + def loadAccounts(self): + """loads all accounts available""" + + self.accounts = {} + + for plugin, loginname, activated, password, options in self.core.db.loadAccounts(): + # put into options as used in other context + options = json.loads(options) if options else {} + options["activated"] = activated + + self.createAccount(plugin, loginname, password, options) + + + def iterAccounts(self): + """ yields login, account for all accounts""" + for name, data in self.accounts.iteritems(): + for login, account in data.iteritems(): + yield login, account + + def saveAccounts(self): + """save all account information""" + # TODO: multi user + # TODO: activated + + data = [] + for name, plugin in self.accounts.iteritems(): + data.extend( + [(name, acc.loginname, 1 if acc.activated else 0, acc.password, json.dumps(acc.options)) for acc in + plugin.itervalues()]) + self.core.db.saveAccounts(data) + + def createAccount(self, plugin, loginname, password, options): + klass = self.core.pluginManager.loadClass("accounts", plugin) + if not klass: + self.core.log.warning(_("Unknown account plugin %s") % plugin) + return + + if plugin not in self.accounts: + self.accounts[plugin] = {} + + self.core.log.debug("Create account %s:%s" % (plugin, loginname)) + + self.accounts[plugin][loginname] = klass(self, loginname, password, options) + + + def getAccount(self, plugin, user): + return self.accounts[plugin].get(user, None) + + @lock + def updateAccount(self, plugin, user, password=None, options={}): + """add or update account""" + if plugin in self.accounts and user in self.accounts[plugin]: + acc = self.accounts[plugin][user] + updated = acc.update(password, options) + + self.saveAccounts() + if updated: acc.scheduleRefresh(force=True) + else: + self.createAccount(plugin, user, password, options) + self.saveAccounts() + + self.sendChange(plugin, user) + + @lock + def removeAccount(self, plugin, user): + """remove account""" + if plugin in self.accounts and user in self.accounts[plugin]: + del self.accounts[plugin][user] + self.core.db.removeAccount(plugin, user) + self.core.eventManager.dispatchEvent("account:deleted", plugin, user) + else: + self.core.log.debug("Remove non existent account %s %s" % (plugin, user)) + + + @lock + def getAccountForPlugin(self, plugin): + if plugin in self.accounts: + accs = [x for x in self.accounts[plugin].values() if x.isUsable()] + if accs: return choice(accs) + + return None + + @lock + def getAllAccounts(self, refresh=False): + """ Return account info, refresh afterwards if needed + + :param refresh: + :return: + """ + if refresh: + self.core.scheduler.addJob(0, self.core.accountManager.getAllAccounts) + + # load unavailable account info + for p_dict in self.accounts.itervalues(): + for acc in p_dict.itervalues(): + acc.getAccountInfo() + + return self.accounts + + def refreshAllAccounts(self): + """ Force a refresh of every account """ + for p in self.accounts.itervalues(): + for acc in p.itervalues(): + acc.getAccountInfo(True) + + def sendChange(self, plugin, name): + self.core.eventManager.dispatchEvent("account:updated", plugin, name) \ No newline at end of file diff --git a/pyload/AddonManager.py b/pyload/AddonManager.py new file mode 100644 index 000000000..5637b1a8e --- /dev/null +++ b/pyload/AddonManager.py @@ -0,0 +1,248 @@ +# -*- coding: utf-8 -*- + +############################################################################### +# Copyright(c) 2008-2013 pyLoad Team +# http://www.pyload.org +# +# This file is part of pyLoad. +# pyLoad is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### + +import __builtin__ + +from thread import start_new_thread +from threading import RLock + +from types import MethodType + +from pyload.threads.AddonThread import AddonThread +from pyload.PluginManager import literal_eval +from utils import lock, to_string + +class AddonManager: + """ Manages addons, loading, unloading. """ + + def __init__(self, core): + self.core = core + self.config = self.core.config + + __builtin__.addonManager = self #needed to let addons register themselves + + self.log = self.core.log + # TODO: multiuser, addons can store the user itself, probably not needed here + self.plugins = {} + self.methods = {} # dict of names and list of methods usable by rpc + self.events = {} # Contains event that will be registered + + self.lock = RLock() + self.createIndex() + + # manage addons on config change + self.addEvent("config:changed", self.manageAddons) + + @lock + def callInHooks(self, event, eventName, *args): + """ Calls a method in all addons and catch / log errors""" + for plugin in self.plugins.itervalues(): + self.call(plugin, event, *args) + self.dispatchEvent(eventName, *args) + + def call(self, addon, f, *args): + try: + func = getattr(addon, f) + return func(*args) + except Exception, e: + addon.logError(_("Error when executing %s" % f), e) + self.core.print_exc() + + def addRPC(self, plugin, func, doc): + doc = doc.strip() if doc else "" + + if plugin in self.methods: + self.methods[plugin][func] = doc + else: + self.methods[plugin] = {func: doc} + + def callRPC(self, plugin, func, args): + if not args: args = [] + else: + args = literal_eval(args) + + plugin = self.plugins[plugin] + f = getattr(plugin, func) + return f(*args) + + @lock + def createIndex(self): + active = [] + deactive = [] + + for pluginname in self.core.pluginManager.getPlugins("addons"): + try: + # check first for builtin plugin + attrs = self.core.pluginManager.loadAttributes("addons", pluginname) + internal = attrs.get("internal", False) + + if internal or self.core.config.get(pluginname, "activated"): + pluginClass = self.core.pluginManager.loadClass("addons", pluginname) + + if not pluginClass: continue + + plugin = pluginClass(self.core, self) + self.plugins[pluginClass.__name__] = plugin + + # hide internals from printing + if not internal and plugin.isActivated(): + active.append(pluginClass.__name__) + else: + self.log.debug("Loaded internal plugin: %s" % pluginClass.__name__) + else: + deactive.append(pluginname) + + + except: + self.log.warning(_("Failed activating %(name)s") % {"name": pluginname}) + self.core.print_exc() + + self.log.info(_("Activated addons: %s") % ", ".join(sorted(active))) + self.log.info(_("Deactivate addons: %s") % ", ".join(sorted(deactive))) + + def manageAddons(self, plugin, name, value): + # TODO: user + + # check if section was a plugin + if plugin not in self.core.pluginManager.getPlugins("addons"): + return + + if name == "activated" and value: + self.activateAddon(plugin) + elif name == "activated" and not value: + self.deactivateAddon(plugin) + + @lock + def activateAddon(self, plugin): + #check if already loaded + if plugin in self.plugins: + return + + pluginClass = self.core.pluginManager.loadClass("addons", plugin) + + if not pluginClass: return + + self.log.debug("Plugin loaded: %s" % plugin) + + plugin = pluginClass(self.core, self) + self.plugins[pluginClass.__name__] = plugin + + # active the addon in new thread + start_new_thread(plugin.activate, tuple()) + self.registerEvents() # TODO: BUG: events will be destroyed and not re-registered + + @lock + def deactivateAddon(self, plugin): + if plugin not in self.plugins: + return + else: + addon = self.plugins[plugin] + + if addon.__internal__: return + + self.call(addon, "deactivate") + self.log.debug("Plugin deactivated: %s" % plugin) + + #remove periodic call + self.log.debug("Removed callback %s" % self.core.scheduler.removeJob(addon.cb)) + del self.plugins[addon.__name__] + + #remove event listener + for f in dir(addon): + if f.startswith("__") or type(getattr(addon, f)) != MethodType: + continue + self.core.eventManager.removeFromEvents(getattr(addon, f)) + + def activateAddons(self): + self.log.info(_("Activating Plugins...")) + for plugin in self.plugins.itervalues(): + if plugin.isActivated(): + self.call(plugin, "activate") + + self.registerEvents() + + def deactivateAddons(self): + """ Called when core is shutting down """ + self.log.info(_("Deactivating Plugins...")) + for plugin in self.plugins.itervalues(): + self.call(plugin, "deactivate") + + def downloadPreparing(self, pyfile): + self.callInHooks("downloadPreparing", "download:preparing", pyfile) + + def downloadFinished(self, pyfile): + self.callInHooks("downloadFinished", "download:finished", pyfile) + + def downloadFailed(self, pyfile): + self.callInHooks("downloadFailed", "download:failed", pyfile) + + def packageFinished(self, package): + self.callInHooks("packageFinished", "package:finished", package) + + def beforeReconnecting(self, ip): + self.callInHooks("beforeReconnecting", "reconnecting:before", ip) + + def afterReconnecting(self, ip): + self.callInHooks("afterReconnecting", "reconnecting:after", ip) + + @lock + def startThread(self, function, *args, **kwargs): + AddonThread(self.core.threadManager, function, args, kwargs) + + def activePlugins(self): + """ returns all active plugins """ + return [x for x in self.plugins.itervalues() if x.isActivated()] + + def getAllInfo(self): + """returns info stored by addon plugins""" + info = {} + for name, plugin in self.plugins.iteritems(): + if plugin.info: + #copy and convert so str + info[name] = dict( + [(x, to_string(y)) for x, y in plugin.info.iteritems()]) + return info + + def getInfo(self, plugin): + info = {} + if plugin in self.plugins and self.plugins[plugin].info: + info = dict([(x, to_string(y)) + for x, y in self.plugins[plugin].info.iteritems()]) + + return info + + def addEventListener(self, plugin, func, event): + """ add the event to the list """ + if plugin not in self.events: + self.events[plugin] = [] + self.events[plugin].append((func, event)) + + def registerEvents(self): + """ actually register all saved events """ + for name, plugin in self.plugins.iteritems(): + if name in self.events: + for func, event in self.events[name]: + self.addEvent(event, getattr(plugin, func)) + # clean up + del self.events[name] + + def addEvent(self, *args): + self.core.eventManager.addEvent(*args) + + def dispatchEvent(self, *args): + self.core.eventManager.dispatchEvent(*args) + diff --git a/pyload/Api.py b/pyload/Api.py new file mode 100644 index 000000000..32a077c08 --- /dev/null +++ b/pyload/Api.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +############################################################################### +# Copyright(c) 2008-2013 pyLoad Team +# http://www.pyload.org +# +# This file is part of pyLoad. +# pyLoad is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### + +import re +from types import MethodType + +from remote.apitypes import * + +# contains function names mapped to their permissions +# unlisted functions are for admins only +perm_map = {} + +# decorator only called on init, never initialized, so has no effect on runtime +def RequirePerm(bits): + class _Dec(object): + def __new__(cls, func, *args, **kwargs): + perm_map[func.__name__] = bits + return func + + return _Dec + +urlmatcher = re.compile(r"((https?|ftps?|xdcc|sftp):((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-=\\\.&]*)", re.IGNORECASE) + +stateMap = { + DownloadState.All: frozenset(getattr(DownloadStatus, x) for x in dir(DownloadStatus) if not x.startswith("_")), + DownloadState.Finished: frozenset((DownloadStatus.Finished, DownloadStatus.Skipped)), + DownloadState.Unfinished: None, # set below + DownloadState.Failed: frozenset((DownloadStatus.Failed, DownloadStatus.TempOffline, DownloadStatus.Aborted)), + DownloadState.Unmanaged: None, #TODO +} + +stateMap[DownloadState.Unfinished] = frozenset(stateMap[DownloadState.All].difference(stateMap[DownloadState.Finished])) + +def state_string(state): + return ",".join(str(x) for x in stateMap[state]) + +from datatypes.User import User + +class Api(Iface): + """ + **pyLoads API** + + This is accessible either internal via core.api, websocket backend or json api. + + see Thrift specification file remote/thriftbackend/pyload.thrift\ + for information about data structures and what methods are usable with rpc. + + Most methods requires specific permissions, please look at the source code if you need to know.\ + These can be configured via web interface. + Admin user have all permissions, and are the only ones who can access the methods with no specific permission. + """ + + EXTERNAL = Iface # let the json api know which methods are external + EXTEND = False # only extendable when set too true + + def __init__(self, core): + self.core = core + self.user_apis = {} + + @property + def user(self): + return None #TODO return default user? + + @property + def primaryUID(self): + return self.user.primary if self.user else None + + @classmethod + def initComponents(cls): + # Allow extending the api + # This prevents unintentionally registering of the components, + # but will only work once when they are imported + cls.EXTEND = True + # Import all Api modules, they register themselves. + import pyload.api + # they will vanish from the namespace afterwards + + + @classmethod + def extend(cls, api): + """Takes all params from api and extends cls with it. + api class can be removed afterwards + + :param api: Class with methods to extend + """ + if cls.EXTEND: + for name, func in api.__dict__.iteritems(): + if name.startswith("_"): continue + setattr(cls, name, MethodType(func, None, cls)) + + return cls.EXTEND + + def withUserContext(self, uid): + """ Returns a proxy version of the api, to call method in user context + + :param uid: user or userData instance or uid + :return: :class:`UserApi` + """ + if isinstance(uid, User): + uid = uid.uid + + if uid not in self.user_apis: + user = self.core.db.getUserData(uid=uid) + if not user: #TODO: anonymous user? + return None + + self.user_apis[uid] = UserApi(self.core, User.fromUserData(self, user)) + + return self.user_apis[uid] + + + ############################# + # Auth+User Information + ############################# + + # TODO + + @RequirePerm(Permission.All) + def login(self, username, password, remoteip=None): + """Login into pyLoad, this **must** be called when using rpc before any methods can be used. + + :param username: + :param password: + :param remoteip: Omit this argument, its only used internal + :return: bool indicating login was successful + """ + return True if self.checkAuth(username, password, remoteip) else False + + def checkAuth(self, username, password, remoteip=None): + """Check authentication and returns details + + :param username: + :param password: + :param remoteip: + :return: dict with info, empty when login is incorrect + """ + self.core.log.info(_("User '%s' tries to log in") % username) + + return self.core.db.checkAuth(username, password) + + def isAuthorized(self, func, user): + """checks if the user is authorized for specific method + + :param func: function name + :param user: `User` + :return: boolean + """ + if user.isAdmin(): + return True + elif func in perm_map and user.hasPermission(perm_map[func]): + return True + else: + return False + + # TODO + @RequirePerm(Permission.All) + def getUserData(self, username, password): + """similar to `checkAuth` but returns UserData thrift type """ + user = self.checkAuth(username, password) + if not user: + raise UserDoesNotExists(username) + + return user.toUserData() + + def getAllUserData(self): + """returns all known user and info""" + return self.core.db.getAllUserData() + + def changePassword(self, username, oldpw, newpw): + """ changes password for specific user """ + return self.core.db.changePassword(username, oldpw, newpw) + + def setUserPermission(self, user, permission, role): + self.core.db.setPermission(user, permission) + self.core.db.setRole(user, role) + + +class UserApi(Api): + """ Proxy object for api that provides all methods in user context """ + + def __init__(self, core, user): + # No need to init super class + self.core = core + self._user = user + + def withUserContext(self, uid): + raise Exception("Not allowed") + + @property + def user(self): + return self._user \ No newline at end of file diff --git a/pyload/FileManager.py b/pyload/FileManager.py new file mode 100644 index 000000000..b1d3891e9 --- /dev/null +++ b/pyload/FileManager.py @@ -0,0 +1,582 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +############################################################################### +# Copyright(c) 2008-2012 pyLoad Team +# http://www.pyload.org +# +# This file is part of pyLoad. +# pyLoad is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### + +from time import time +from ReadWriteLock import ReadWriteLock + +from pyload.utils import lock, read_lock + +from Api import PackageStatus, DownloadStatus as DS, TreeCollection, PackageDoesNotExists +from datatypes.PyFile import PyFile +from datatypes.PyPackage import PyPackage, RootPackage + +# invalidates the cache +def invalidate(func): + def new(*args): + args[0].downloadstats = {} + args[0].queuestats = {} + args[0].jobCache = {} + return func(*args) + + return new + +# TODO: needs to be replaced later +OWNER = 0 + +class FileManager: + """Handles all request made to obtain information, + modify status or other request for links or packages""" + + ROOT_PACKAGE = -1 + + def __init__(self, core): + """Constructor""" + self.core = core + self.evm = core.eventManager + + # translations + self.statusMsg = [_("none"), _("offline"), _("online"), _("queued"), _("paused"), + _("finished"), _("skipped"), _("failed"), _("starting"), + _("waiting"), _("downloading"), _("temp. offline"), _("aborted"), + _("decrypting"), _("processing"), _("custom"), _("unknown")] + + self.files = {} # holds instances for files + self.packages = {} # same for packages + + self.jobCache = {} + + # locking the caches, db is already locked implicit + self.lock = ReadWriteLock() + #self.lock._Verbose__verbose = True + + self.downloadstats = {} # cached dl stats + self.queuestats = {} # cached queue stats + + self.db = self.core.db + + def save(self): + """saves all data to backend""" + self.db.commit() + + @read_lock + def syncSave(self): + """saves all data to backend and waits until all data are written""" + for pyfile in self.files.values(): + pyfile.sync() + + for pypack in self.packages.values(): + pypack.sync() + + self.db.syncSave() + + def cachedFiles(self): + return self.files.values() + + def cachedPackages(self): + return self.packages.values() + + def getCollector(self): + pass + + @invalidate + def addLinks(self, data, package): + """Add links, data = (plugin, url) tuple. Internal method should use API.""" + self.db.addLinks(data, package, OWNER) + self.evm.dispatchEvent("package:updated", package) + + + @invalidate + def addPackage(self, name, folder, root, password, site, comment, paused): + """Adds a package to database""" + pid = self.db.addPackage(name, folder, root, password, site, comment, + PackageStatus.Paused if paused else PackageStatus.Ok, OWNER) + p = self.db.getPackageInfo(pid) + + self.evm.dispatchEvent("package:inserted", pid, p.root, p.packageorder) + return pid + + + @lock + def getPackage(self, pid): + """return package instance""" + if pid == self.ROOT_PACKAGE: + return RootPackage(self, OWNER) + elif pid in self.packages: + pack = self.packages[pid] + pack.timestamp = time() + return pack + else: + info = self.db.getPackageInfo(pid, False) + if not info: return None + + pack = PyPackage.fromInfoData(self, info) + self.packages[pid] = pack + + return pack + + @read_lock + def getPackageInfo(self, pid): + """returns dict with package information""" + if pid == self.ROOT_PACKAGE: + pack = RootPackage(self, OWNER).toInfoData() + elif pid in self.packages: + pack = self.packages[pid].toInfoData() + pack.stats = self.db.getStatsForPackage(pid) + else: + pack = self.db.getPackageInfo(pid) + + if not pack: return None + + # todo: what does this todo mean?! + #todo: fill child packs and files + packs = self.db.getAllPackages(root=pid) + if pid in packs: del packs[pid] + pack.pids = packs.keys() + + files = self.db.getAllFiles(package=pid) + pack.fids = files.keys() + + return pack + + @lock + def getFile(self, fid): + """returns pyfile instance""" + if fid in self.files: + return self.files[fid] + else: + info = self.db.getFileInfo(fid) + if not info: return None + + f = PyFile.fromInfoData(self, info) + self.files[fid] = f + return f + + @read_lock + def getFileInfo(self, fid): + """returns dict with file information""" + if fid in self.files: + return self.files[fid].toInfoData() + + return self.db.getFileInfo(fid) + + @read_lock + def getTree(self, pid, full, state, search=None): + """ return a TreeCollection and fill the info data of containing packages. + optional filter only unfnished files + """ + view = TreeCollection(pid) + + # for depth=1, we don't need to retrieve all files/packages + root = pid if not full else None + + packs = self.db.getAllPackages(root) + files = self.db.getAllFiles(package=root, state=state, search=search) + + # updating from cache + for fid, f in self.files.iteritems(): + if fid in files: + files[fid] = f.toInfoData() + + # foreign pid, don't overwrite local pid ! + for fpid, p in self.packages.iteritems(): + if fpid in packs: + # copy the stats data + stats = packs[fpid].stats + packs[fpid] = p.toInfoData() + packs[fpid].stats = stats + + # root package is not in database, create an instance + if pid == self.ROOT_PACKAGE: + view.root = RootPackage(self, OWNER).toInfoData() + packs[self.ROOT_PACKAGE] = view.root + elif pid in packs: + view.root = packs[pid] + else: # package does not exists + return view + + # linear traversal over all data + for fpid, p in packs.iteritems(): + if p.fids is None: p.fids = [] + if p.pids is None: p.pids = [] + + root = packs.get(p.root, None) + if root: + if root.pids is None: root.pids = [] + root.pids.append(fpid) + + for fid, f in files.iteritems(): + p = packs.get(f.package, None) + if p: p.fids.append(fid) + + + # cutting of tree is not good in runtime, only saves bandwidth + # need to remove some entries + if full and pid > -1: + keep = [] + queue = [pid] + while queue: + fpid = queue.pop() + keep.append(fpid) + queue.extend(packs[fpid].pids) + + # now remove unneeded data + for fpid in packs.keys(): + if fpid not in keep: + del packs[fpid] + + for fid, f in files.items(): + if f.package not in keep: + del files[fid] + + #remove root + del packs[pid] + view.files = files + view.packages = packs + + return view + + + @lock + def getJob(self, occ): + """get suitable job""" + + #TODO only accessed by one thread, should not need a lock + #TODO needs to be approved for new database + #TODO clean mess + #TODO improve selection of valid jobs + + if occ in self.jobCache: + if self.jobCache[occ]: + id = self.jobCache[occ].pop() + if id == "empty": + pyfile = None + self.jobCache[occ].append("empty") + else: + pyfile = self.getFile(id) + else: + jobs = self.db.getJob(occ) + jobs.reverse() + if not jobs: + self.jobCache[occ].append("empty") + pyfile = None + else: + self.jobCache[occ].extend(jobs) + pyfile = self.getFile(self.jobCache[occ].pop()) + + else: + self.jobCache = {} #better not caching to much + jobs = self.db.getJob(occ) + jobs.reverse() + self.jobCache[occ] = jobs + + if not jobs: + self.jobCache[occ].append("empty") + pyfile = None + else: + pyfile = self.getFile(self.jobCache[occ].pop()) + + + return pyfile + + def getDownloadStats(self, user=None): + """ return number of downloads """ + if user not in self.downloadstats: + self.downloadstats[user] = self.db.downloadstats(user) + + return self.downloadstats[user] + + def getQueueStats(self, user=None, force=False): + """number of files that have to be processed, failed files will not be included""" + if user not in self.queuestats or force: + self.queuestats[user] = self.db.queuestats(user) + + return self.queuestats[user] + + def scanDownloadFolder(self): + pass + + @lock + @invalidate + def deletePackage(self, pid): + """delete package and all contained links""" + + p = self.getPackage(pid) + if not p: return + + oldorder = p.packageorder + root = p.root + + for pyfile in self.cachedFiles(): + if pyfile.packageid == pid: + pyfile.abortDownload() + + # TODO: delete child packages + # TODO: delete folder + + self.db.deletePackage(pid) + self.releasePackage(pid) + + for pack in self.cachedPackages(): + if pack.root == root and pack.packageorder > oldorder: + pack.packageorder -= 1 + + self.evm.dispatchEvent("package:deleted", pid) + + @lock + @invalidate + def deleteFile(self, fid): + """deletes links""" + + f = self.getFile(fid) + if not f: return + + pid = f.packageid + order = f.fileorder + + if fid in self.core.threadManager.processingIds(): + f.abortDownload() + + # TODO: delete real file + + self.db.deleteFile(fid, f.fileorder, f.packageid) + self.releaseFile(fid) + + for pyfile in self.files.itervalues(): + if pyfile.packageid == pid and pyfile.fileorder > order: + pyfile.fileorder -= 1 + + self.evm.dispatchEvent("file:deleted", fid, pid) + + @lock + def releaseFile(self, fid): + """removes pyfile from cache""" + if fid in self.files: + del self.files[fid] + + @lock + def releasePackage(self, pid): + """removes package from cache""" + if pid in self.packages: + del self.packages[pid] + + def updateFile(self, pyfile): + """updates file""" + self.db.updateFile(pyfile) + + # This event is thrown with pyfile or only fid + self.evm.dispatchEvent("file:updated", pyfile) + + def updatePackage(self, pypack): + """updates a package""" + self.db.updatePackage(pypack) + self.evm.dispatchEvent("package:updated", pypack.pid) + + @invalidate + def updateFileInfo(self, data, pid): + """ updates file info (name, size, status,[ hash,] url)""" + self.db.updateLinkInfo(data) + self.evm.dispatchEvent("package:updated", pid) + + def checkAllLinksFinished(self): + """checks if all files are finished and dispatch event""" + + # TODO: user context? + if not self.db.queuestats()[0]: + self.core.addonManager.dispatchEvent("download:allFinished") + self.core.log.debug("All downloads finished") + return True + + return False + + def checkAllLinksProcessed(self, fid=-1): + """checks if all files was processed and pyload would idle now, needs fid which will be ignored when counting""" + + # reset count so statistic will update (this is called when dl was processed) + self.resetCount() + + # TODO: user context? + if not self.db.processcount(fid): + self.core.addonManager.dispatchEvent("download:allProcessed") + self.core.log.debug("All downloads processed") + return True + + return False + + def checkPackageFinished(self, pyfile): + """ checks if package is finished and calls addonmanager """ + + ids = self.db.getUnfinished(pyfile.packageid) + if not ids or (pyfile.id in ids and len(ids) == 1): + if not pyfile.package().setFinished: + self.core.log.info(_("Package finished: %s") % pyfile.package().name) + self.core.addonManager.packageFinished(pyfile.package()) + pyfile.package().setFinished = True + + def resetCount(self): + self.queuecount = -1 + + @read_lock + @invalidate + def restartPackage(self, pid): + """restart package""" + for pyfile in self.cachedFiles(): + if pyfile.packageid == pid: + self.restartFile(pyfile.id) + + self.db.restartPackage(pid) + + if pid in self.packages: + self.packages[pid].setFinished = False + + self.evm.dispatchEvent("package:updated", pid) + + @read_lock + @invalidate + def restartFile(self, fid): + """ restart file""" + if fid in self.files: + f = self.files[fid] + f.status = DS.Queued + f.name = f.url + f.error = "" + f.abortDownload() + + self.db.restartFile(fid) + self.evm.dispatchEvent("file:updated", fid) + + + @lock + @invalidate + def orderPackage(self, pid, position): + + p = self.getPackageInfo(pid) + self.db.orderPackage(pid, p.root, p.packageorder, position) + + for pack in self.packages.itervalues(): + if pack.root != p.root or pack.packageorder < 0: continue + if pack.pid == pid: + pack.packageorder = position + if p.packageorder > position: + if position <= pack.packageorder < p.packageorder: + pack.packageorder += 1 + elif p.order < position: + if position >= pack.packageorder > p.packageorder: + pack.packageorder -= 1 + + self.db.commit() + + self.evm.dispatchEvent("package:reordered", pid, position, p.root) + + @lock + @invalidate + def orderFiles(self, fids, pid, position): + + files = [self.getFileInfo(fid) for fid in fids] + orders = [f.fileorder for f in files] + if min(orders) + len(files) != max(orders) + 1: + raise Exception("Tried to reorder non continous block of files") + + # minimum fileorder + f = reduce(lambda x,y: x if x.fileorder < y.fileorder else y, files) + order = f.fileorder + + self.db.orderFiles(pid, fids, order, position) + diff = len(fids) + + if f.fileorder > position: + for pyfile in self.files.itervalues(): + if pyfile.packageid != f.package or pyfile.fileorder < 0: continue + if position <= pyfile.fileorder < f.fileorder: + pyfile.fileorder += diff + + for i, fid in enumerate(fids): + if fid in self.files: + self.files[fid].fileorder = position + i + + elif f.fileorder < position: + for pyfile in self.files.itervalues(): + if pyfile.packageid != f.package or pyfile.fileorder < 0: continue + if position >= pyfile.fileorder >= f.fileorder+diff: + pyfile.fileorder -= diff + + for i, fid in enumerate(fids): + if fid in self.files: + self.files[fid].fileorder = position -diff + i + 1 + + self.db.commit() + + self.evm.dispatchEvent("file:reordered", pid) + + @read_lock + @invalidate + def movePackage(self, pid, root): + """ move pid - root """ + + p = self.getPackageInfo(pid) + dest = self.getPackageInfo(root) + if not p: raise PackageDoesNotExists(pid) + if not dest: raise PackageDoesNotExists(root) + + # cantor won't be happy if we put the package in itself + if pid == root or p.root == root: return False + + # TODO move real folders + + # we assume pack is not in use anyway, so we can release it + self.releasePackage(pid) + self.db.movePackage(p.root, p.packageorder, pid, root) + + return True + + @read_lock + @invalidate + def moveFiles(self, fids, pid): + """ move all fids to pid """ + + f = self.getFileInfo(fids[0]) + if not f or f.package == pid: + return False + if not self.getPackageInfo(pid): + raise PackageDoesNotExists(pid) + + # TODO move real files + + self.db.moveFiles(f.package, fids, pid) + + return True + + + @invalidate + def reCheckPackage(self, pid): + """ recheck links in package """ + data = self.db.getPackageData(pid) + + urls = [] + + for pyfile in data.itervalues(): + if pyfile.status not in (DS.NA, DS.Finished, DS.Skipped): + urls.append((pyfile.url, pyfile.pluginname)) + + self.core.threadManager.createInfoThread(urls, pid) + + + @invalidate + def restartFailed(self): + """ restart all failed links """ + # failed should not be in cache anymore, so working on db is sufficient + self.db.restartFailed() diff --git a/pyload/InitHomeDir.py b/pyload/InitHomeDir.py new file mode 100644 index 000000000..8fca42196 --- /dev/null +++ b/pyload/InitHomeDir.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN + + This modules inits working directories and global variables, pydir and homedir +""" + +from os import makedirs, path, chdir +from os.path import join +import sys +from sys import argv, platform + +import __builtin__ + +__builtin__.owd = path.abspath("") #original working directory +__builtin__.pypath = path.abspath(path.join(__file__, "..", "..")) + +sys.path.append(join(pypath, "pyload", "lib")) + +homedir = "" + +if platform == 'nt': + homedir = path.expanduser("~") + if homedir == "~": + import ctypes + + CSIDL_APPDATA = 26 + _SHGetFolderPath = ctypes.windll.shell32.SHGetFolderPathW + _SHGetFolderPath.argtypes = [ctypes.wintypes.HWND, + ctypes.c_int, + ctypes.wintypes.HANDLE, + ctypes.wintypes.DWORD, ctypes.wintypes.LPCWSTR] + + path_buf = ctypes.wintypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH) + result = _SHGetFolderPath(0, CSIDL_APPDATA, 0, 0, path_buf) + homedir = path_buf.value +else: + homedir = path.expanduser("~") + +__builtin__.homedir = homedir + +configdir = None +args = " ".join(argv) +# dirty method to set configdir from commandline arguments +if "--configdir=" in args: + for arg in argv: + if arg.startswith("--configdir="): + configdir = arg.replace('--configdir=', '').strip() + +elif "nosetests" in args: + print "Running in test mode" + configdir = join(pypath, "tests", "config") + +elif path.exists(path.join(pypath, "pyload", "config", "configdir")): + f = open(path.join(pypath, "pyload", "config", "configdir"), "rb") + c = f.read().strip() + f.close() + configdir = path.join(pypath, c) + +# default config dir +if not configdir: + if platform in ("posix", "linux2", "darwin"): + configdir = path.join(homedir, ".pyload") + else: + configdir = path.join(homedir, "pyload") + +if not path.exists(configdir): + makedirs(configdir, 0700) + +__builtin__.configdir = configdir +chdir(configdir) + +#print "Using %s as working directory." % configdir diff --git a/pyload/PluginManager.py b/pyload/PluginManager.py new file mode 100644 index 000000000..159e7d9de --- /dev/null +++ b/pyload/PluginManager.py @@ -0,0 +1,426 @@ +# -*- coding: utf-8 -*- + +############################################################################### +# Copyright(c) 2008-2013 pyLoad Team +# http://www.pyload.org +# +# This file is part of pyLoad. +# pyLoad is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN, mkaay +############################################################################### + +import re +import sys + +from os import listdir, makedirs +from os.path import isfile, join, exists, abspath, basename +from sys import version_info +from time import time + +from pyload.lib.SafeEval import const_eval as literal_eval +from pyload.plugins.Base import Base + +from new_collections import namedtuple + +#TODO: ignores not updatable + +# ignore these plugin configs, mainly because plugins were wiped out +IGNORE = ( + "FreakshareNet", "SpeedManager", "ArchiveTo", "ShareCx", ('addons', 'UnRar'), + 'EasyShareCom', 'FlyshareCz' + ) + +PluginTuple = namedtuple("PluginTuple", "version re deps category user path") + +class PluginManager: + ROOT = "pyload.plugins." + LOCALROOT = "localplugins." + TYPES = ("crypter", "hoster", "accounts", "addons", "internal") + + BUILTIN = re.compile(r'__(?P[a-z0-9_]+)__\s*=\s?(True|False|None|[0-9x.]+)', re.I) + SINGLE = re.compile(r'__(?P[a-z0-9_]+)__\s*=\s*(?:r|u|_)?((?:(?[a-z0-9_]+)__\s*=\s*((?:\{|\[|"{3}).*?(?:"""|\}|\]))', re.DOTALL | re.M | re.I) + + NO_MATCH = re.compile(r'^no_match$') + + def __init__(self, core): + self.core = core + + #self.config = self.core.config + self.log = core.log + + self.plugins = {} + self.modules = {} # cached modules + self.history = [] # match history to speedup parsing (type, name) + self.user_context = {} # plugins working with user context + self.createIndex() + + #register for import addon + sys.meta_path.append(self) + + def logDebug(self, type, plugin, msg): + self.log.debug("Plugin %s | %s: %s" % (type, plugin, msg)) + + def createIndex(self): + """create information for all plugins available""" + # add to path, so we can import from userplugins + sys.path.append(abspath("")) + + if not exists("userplugins"): + makedirs("userplugins") + if not exists(join("userplugins", "__init__.py")): + f = open(join("userplugins", "__init__.py"), "wb") + f.close() + + a = time() + for type in self.TYPES: + self.plugins[type] = self.parse(type) + + self.log.debug("Created index of plugins in %.2f ms", (time() - a) * 1000) + + def parse(self, folder, home=None): + """ Analyze and parses all plugins in folder """ + plugins = {} + if home: + pfolder = join("userplugins", folder) + if not exists(pfolder): + makedirs(pfolder) + if not exists(join(pfolder, "__init__.py")): + f = open(join(pfolder, "__init__.py"), "wb") + f.close() + + else: + pfolder = join(pypath, "pyload", "plugins", folder) + + for f in listdir(pfolder): + if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( + "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): + if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): + continue + elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): + continue + elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): + continue + + # replace suffix and version tag + name = f[:-3] + if name[-1] == ".": name = name[:-4] + + plugin = self.parsePlugin(join(pfolder, f), folder, name, home) + if plugin: + plugins[name] = plugin + + if not home: + temp = self.parse(folder, plugins) + plugins.update(temp) + + return plugins + + def parseAttributes(self, filename, name, folder=""): + """ Parse attribute dict from plugin""" + data = open(filename, "rb") + content = data.read() + data.close() + + attrs = {} + for m in self.BUILTIN.findall(content) + self.SINGLE.findall(content) + self.MULTI.findall(content): + #replace gettext function and eval result + try: + attrs[m[0]] = literal_eval(m[-1].replace("_(", "(")) + except: + self.logDebug(folder, name, "Error when parsing: %s" % m[-1]) + self.core.print_exc() + + if not hasattr(Base, "__%s__" % m[0]): + if m[0] != "type": #TODO remove type from all plugins, its not needed + self.logDebug(folder, name, "Unknown attribute '%s'" % m[0]) + + return attrs + + def parsePlugin(self, filename, folder, name, home=None): + """ Parses a plugin from disk, folder means plugin type in this context. Also sets config. + + :arg home: dict with plugins, of which the found one will be matched against (according version) + :returns PluginTuple""" + + attrs = self.parseAttributes(filename, name, folder) + if not attrs: return + + version = 0 + + if "version" in attrs: + try: + version = float(attrs["version"]) + except ValueError: + self.logDebug(folder, name, "Invalid version %s" % attrs["version"]) + version = 9 #TODO remove when plugins are fixed, causing update loops + else: + self.logDebug(folder, name, "No version attribute") + + # home contains plugins from pyload root + if home and name in home: + if home[name].version >= version: + return + + if name in IGNORE or (folder, name) in IGNORE: + return + + if "pattern" in attrs and attrs["pattern"]: + try: + plugin_re = re.compile(attrs["pattern"], re.I) + except: + self.logDebug(folder, name, "Invalid regexp pattern '%s'" % attrs["pattern"]) + plugin_re = self.NO_MATCH + else: plugin_re = self.NO_MATCH + + deps = attrs.get("dependencies", None) + category = attrs.get("category", None) if folder == "addons" else None + + # create plugin tuple + plugin = PluginTuple(version, plugin_re, deps, category, bool(home), filename) + + # internals have no config + if folder == "internal": + return plugin + + if folder == "addons" and "config" not in attrs and not attrs.get("internal", False): + attrs["config"] = (["activated", "bool", "Activated", False],) + + if "config" in attrs and attrs["config"]: + config = attrs["config"] + desc = attrs.get("description", "") + long_desc = attrs.get("long_description", "") + + if type(config[0]) == tuple: + config = [list(x) for x in config] + else: + config = [list(config)] + + if folder == "addons" and not attrs.get("internal", False): + for item in config: + if item[0] == "activated": break + else: # activated flag missing + config.insert(0, ("activated", "bool", "Activated", False)) + + # Everything that is no addon and user_context=True, is added to dict + if folder != "addons" or attrs.get("user_context", False): + self.user_context[name] = True + + try: + self.core.config.addConfigSection(name, name, desc, long_desc, config) + except: + self.logDebug(folder, name, "Invalid config %s" % config) + + return plugin + + + def parseUrls(self, urls): + """parse plugins for given list of urls, separate to crypter and hoster""" + + res = {"hoster": [], "crypter": []} # tupels of (url, plugin) + + for url in urls: + if type(url) not in (str, unicode, buffer): + self.log.debug("Parsing invalid type %s" % type(url)) + continue + + found = False + + for ptype, name in self.history: + if self.plugins[ptype][name].re.match(url): + res[ptype].append((url, name)) + found = (ptype, name) + break # need to exit this loop first + + if found: # found match + if self.history[0] != found: #update history + self.history.remove(found) + self.history.insert(0, found) + continue + + for ptype in ("crypter", "hoster"): + for name, plugin in self.plugins[ptype].iteritems(): + if plugin.re.match(url): + res[ptype].append((url, name)) + self.history.insert(0, (ptype, name)) + del self.history[10:] # cut down to size of 10 + found = True + break + + if not found: + res["hoster"].append((url, "BasePlugin")) + + return res["hoster"], res["crypter"] + + def getPlugins(self, type): + return self.plugins.get(type, None) + + def findPlugin(self, name, pluginlist=("hoster", "crypter")): + for ptype in pluginlist: + if name in self.plugins[ptype]: + return ptype, self.plugins[ptype][name] + return None, None + + def getPluginModule(self, name): + """ Decprecated: return plugin module from hoster|crypter""" + self.log.debug("Deprecated method: .getPluginModule()") + type, plugin = self.findPlugin(name) + return self.loadModule(type, name) + + def getPluginClass(self, name): + """ return plugin class from hoster|crypter, always the not overwritten one """ + type, plugin = self.findPlugin(name) + return self.loadClass(type, name) + + # MultiHoster will overwrite this + getPlugin = getPluginClass + + def loadAttributes(self, type, name): + plugin = self.plugins[type][name] + return self.parseAttributes(plugin.path, name, type) + + def loadModule(self, type, name): + """ Returns loaded module for plugin + + :param type: plugin type, subfolder of module.plugins + :param name: + """ + plugins = self.plugins[type] + if name in plugins: + if (type, name) in self.modules: return self.modules[(type, name)] + try: + # convert path to python recognizable import + path = basename(plugins[name].path).replace(".pyc", "").replace(".py", "") + module = __import__(self.ROOT + "%s.%s" % (type, path), globals(), locals(), path) + self.modules[(type, name)] = module # cache import, maybe unneeded + return module + except Exception, e: + self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) + self.core.print_exc() + + def loadClass(self, type, name): + """Returns the class of a plugin with the same name""" + module = self.loadModule(type, name) + if module: return getattr(module, name) + + def find_module(self, fullname, path=None): + #redirecting imports if necesarry + if fullname.startswith(self.ROOT) or fullname.startswith(self.LOCALROOT): #separate pyload plugins + if fullname.startswith(self.LOCALROOT): user = 1 + else: user = 0 #used as bool and int + + split = fullname.split(".") + if len(split) != 4 - user: return + type, name = split[2 - user:4 - user] + + if type in self.plugins and name in self.plugins[type]: + #userplugin is a newer version + if not user and self.plugins[type][name].user: + return self + #imported from userdir, but pyloads is newer + if user and not self.plugins[type][name].user: + return self + + # TODO: Remove when all plugin imports are adapted + if "module" in fullname: + return self + + + def load_module(self, name, replace=True): + if name not in sys.modules: #could be already in modules + + # TODO: only temporary + if name.endswith("module"): + # name = "pyload." + name = name.replace(".module", "") + self.log.debug("Old import reference detected, use %s" % name) + replace = False + return __import__("pyload") + if name.startswith("module"): + name = name.replace("module", "pyload") + self.log.debug("Old import reference detected, use %s" % name) + replace = False + + + if replace: + if self.ROOT in name: + newname = name.replace(self.ROOT, self.LOCALROOT) + else: + newname = name.replace(self.LOCALROOT, self.ROOT) + else: newname = name + + base, plugin = newname.rsplit(".", 1) + + self.log.debug("Redirected import %s -> %s" % (name, newname)) + + module = __import__(newname, globals(), locals(), [plugin]) + #inject under new an old name + sys.modules[name] = module + sys.modules[newname] = module + + return sys.modules[name] + + def reloadPlugins(self, type_plugins): + """ reloads and reindexes plugins """ + if not type_plugins: return False + + self.log.debug("Request reload of plugins: %s" % type_plugins) + + as_dict = {} + for t, n in type_plugins: + if t in as_dict: + as_dict[t].append(n) + else: + as_dict[t] = [n] + + # we do not reload addons or internals, would cause to much side effects + if "addons" in as_dict or "internal" in as_dict: + return False + + for type in as_dict.iterkeys(): + for plugin in as_dict[type]: + if plugin in self.plugins[type]: + if (type, plugin) in self.modules: + self.log.debug("Reloading %s" % plugin) + reload(self.modules[(type, plugin)]) + + # index re-creation + for type in ("crypter", "container", "hoster", "captcha", "accounts"): + self.plugins[type] = self.parse(type) + + if "accounts" in as_dict: #accounts needs to be reloaded + self.core.accountManager.initPlugins() + self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) + + return True + + def isUserPlugin(self, plugin): + """ A plugin suitable for multiple user """ + return plugin in self.user_context + + def isPluginType(self, plugin, type): + return plugin in self.plugins[type] + + def getCategory(self, plugin): + if plugin in self.plugins["addons"]: + return self.plugins["addons"][plugin].category or "addon" + + def loadIcon(self, name): + """ load icon for single plugin, base64 encoded""" + pass + + def checkDependencies(self, type, name): + """ Check deps for given plugin + + :return: List of unfullfilled dependencies + """ + pass + diff --git a/pyload/Scheduler.py b/pyload/Scheduler.py new file mode 100644 index 000000000..0bc396b69 --- /dev/null +++ b/pyload/Scheduler.py @@ -0,0 +1,141 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" + +from time import time +from heapq import heappop, heappush +from thread import start_new_thread +from threading import Lock + +class AlreadyCalled(Exception): + pass + + +class Deferred(): + def __init__(self): + self.call = [] + self.result = () + + def addCallback(self, f, *cargs, **ckwargs): + self.call.append((f, cargs, ckwargs)) + + def callback(self, *args, **kwargs): + if self.result: + raise AlreadyCalled + self.result = (args, kwargs) + for f, cargs, ckwargs in self.call: + args += tuple(cargs) + kwargs.update(ckwargs) + f(*args ** kwargs) + + +class Scheduler(): + def __init__(self, core): + self.core = core + + self.queue = PriorityQueue() + + def addJob(self, t, call, args=[], kwargs={}, threaded=True): + d = Deferred() + t += time() + j = Job(t, call, args, kwargs, d, threaded) + self.queue.put((t, j)) + return d + + + def removeJob(self, d): + """ + :param d: defered object + :return: if job was deleted + """ + index = -1 + + for i, j in enumerate(self.queue): + if j[1].deferred == d: + index = i + + if index >= 0: + del self.queue[index] + return True + + return False + + def work(self): + while True: + t, j = self.queue.get() + if not j: + break + else: + if t <= time(): + j.start() + else: + self.queue.put((t, j)) + break + + +class Job(): + def __init__(self, time, call, args=[], kwargs={}, deferred=None, threaded=True): + self.time = float(time) + self.call = call + self.args = args + self.kwargs = kwargs + self.deferred = deferred + self.threaded = threaded + + def run(self): + ret = self.call(*self.args, **self.kwargs) + if self.deferred is None: + return + else: + self.deferred.callback(ret) + + def start(self): + if self.threaded: + start_new_thread(self.run, ()) + else: + self.run() + + +class PriorityQueue(): + """ a non blocking priority queue """ + + def __init__(self): + self.queue = [] + self.lock = Lock() + + def __iter__(self): + return iter(self.queue) + + def __delitem__(self, key): + del self.queue[key] + + def put(self, element): + self.lock.acquire() + heappush(self.queue, element) + self.lock.release() + + def get(self): + """ return element or None """ + self.lock.acquire() + try: + el = heappop(self.queue) + return el + except IndexError: + return None, None + finally: + self.lock.release() \ No newline at end of file diff --git a/pyload/Setup.py b/pyload/Setup.py new file mode 100644 index 000000000..2ecd6ebe6 --- /dev/null +++ b/pyload/Setup.py @@ -0,0 +1,418 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +############################################################################### +# Copyright(c) 2008-2012 pyLoad Team +# http://www.pyload.org +# +# This file is part of pyLoad. +# pyLoad is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### + +import pyload.utils.pylgettext as gettext +import os +import sys +import socket +import webbrowser + +from getpass import getpass +from time import time +from sys import exit + +from pyload.utils.fs import abspath, dirname, exists, join, makedirs +from pyload.utils import get_console_encoding +from pyload.web.ServerThread import WebServer + + +class Setup(): + """ + pyLoads initial setup configuration assistant + """ + + def __init__(self, path, config): + self.path = path + self.config = config + self.stdin_encoding = get_console_encoding(sys.stdin.encoding) + self.lang = None + # We will create a timestamp so that the setup will be completed in a specific interval + self.timestamp = time() + + # TODO: probably unneeded + self.yes = "yes" + self.no = "no" + + + def start(self): + web = WebServer(pysetup=self) + web.start() + + error = web.check_error() + if error: #todo errno 44 port already in use + print error + + url = "http://%s:%d/" % (socket.gethostbyname(socket.gethostname()), web.port) + + print "Setup is started" + + opened = webbrowser.open_new_tab(url) + if not opened: + print "Please point your browser to %s" % url + + + self.ask_lang() + + print "" + print _("Would you like to configure pyLoad via Webinterface?") + print _("You need a Browser and a connection to this PC for it.") + print _("Url would be: http://hostname:8000/") + viaweb = self.ask(_("Start initial webinterface for configuration?"), self.yes, bool=True) + if viaweb: + self.start_web() + else: + self.start_cli() + + + + def start_cli(self): + + + print _("Welcome to the pyLoad Configuration Assistent.") + print _("It will check your system and make a basic setup in order to run pyLoad.") + print "" + print _("The value in brackets [] always is the default value,") + print _("in case you don't want to change it or you are unsure what to choose, just hit enter.") + print _( + "Don't forget: You can always rerun this assistent with --setup or -s parameter, when you start pyLoadCore.") + print _("If you have any problems with this assistent hit STRG-C,") + print _("to abort and don't let him start with pyLoadCore automatically anymore.") + print "" + print _("When you are ready for system check, hit enter.") + raw_input() + + #self.get_page_next() + + + if len(avail) < 5: + print _("Features missing: ") + print + + if not self.check_module("Crypto"): + print _("no py-crypto available") + print _("You need this if you want to decrypt container files.") + print "" + + if not ssl: + print _("no SSL available") + print _("This is needed if you want to establish a secure connection to core or webinterface.") + print _("If you only want to access locally to pyLoad ssl is not useful.") + print "" + + if not captcha: + print _("no Captcha Recognition available") + print _("Only needed for some hosters and as freeuser.") + print "" + + if not js: + print _("no JavaScript engine found") + print _("You will need this for some Click'N'Load links. Install Spidermonkey, ossp-js, pyv8 or rhino") + + print _("You can abort the setup now and fix some dependencies if you want.") + + con = self.ask(_("Continue with setup?"), self.yes, bool=True) + + if not con: + return False + + print "" + print _("Do you want to change the config path? Current is %s") % abspath("") + print _( + "If you use pyLoad on a server or the home partition lives on an internal flash it may be a good idea to change it.") + path = self.ask(_("Change config path?"), self.no, bool=True) + if path: + self.conf_path() + #calls exit when changed + + print "" + print _("Do you want to configure login data and basic settings?") + print _("This is recommend for first run.") + con = self.ask(_("Make basic setup?"), self.yes, bool=True) + + if con: + self.conf_basic() + + if ssl: + print "" + print _("Do you want to configure ssl?") + ssl = self.ask(_("Configure ssl?"), self.no, bool=True) + if ssl: + self.conf_ssl() + + if web: + print "" + print _("Do you want to configure webinterface?") + web = self.ask(_("Configure webinterface?"), self.yes, bool=True) + if web: + self.conf_web() + + print "" + print _("Setup finished successfully.") + print _("Hit enter to exit and restart pyLoad") + raw_input() + return True + + + def start_web(self): + print "" + print _("Webinterface running for setup.") + # TODO start browser? + try: + from pyload.web import ServerThread + ServerThread.setup = self + from pyload.web import webinterface + webinterface.run_simple() + self.web = True + return True + except Exception, e: + print "Webinterface failed with this error: ", e + print "Falling back to commandline setup." + self.start_cli() + + + def conf_basic(self): + print "" + print _("## Basic Setup ##") + + print "" + print _("The following logindata is valid for CLI, GUI and webinterface.") + + from pyload.database import DatabaseBackend + + db = DatabaseBackend(None) + db.setup() + username = self.ask(_("Username"), "User") + password = self.ask("", "", password=True) + db.addUser(username, password) + db.shutdown() + + print "" + print _("External clients (GUI, CLI or other) need remote access to work over the network.") + print _("However, if you only want to use the webinterface you may disable it to save ram.") + self.config["remote"]["activated"] = self.ask(_("Enable remote access"), self.yes, bool=True) + + print "" + langs = self.config.getMetaData("general", "language") + self.config["general"]["language"] = self.ask(_("Language"), "en", langs.type.split(";")) + + self.config["general"]["download_folder"] = self.ask(_("Downloadfolder"), "Downloads") + self.config["download"]["max_downloads"] = self.ask(_("Max parallel downloads"), "3") + #print _("You should disable checksum proofing, if you have low hardware requirements.") + #self.config["general"]["checksum"] = self.ask(_("Proof checksum?"), "y", bool=True) + + reconnect = self.ask(_("Use Reconnect?"), self.no, bool=True) + self.config["reconnect"]["activated"] = reconnect + if reconnect: + self.config["reconnect"]["method"] = self.ask(_("Reconnect script location"), "./reconnect.sh") + + + def conf_web(self): + print "" + print _("## Webinterface Setup ##") + + print "" + self.config["webinterface"]["activated"] = self.ask(_("Activate webinterface?"), self.yes, bool=True) + print "" + print _("Listen address, if you use 127.0.0.1 or localhost, the webinterface will only accessible locally.") + self.config["webinterface"]["host"] = self.ask(_("Address"), "0.0.0.0") + self.config["webinterface"]["port"] = self.ask(_("Port"), "8000") + print "" + print _("pyLoad offers several server backends, now following a short explanation.") + print "threaded:", _("Default server, this server offers SSL and is a good alternative to builtin.") + print "fastcgi:", _( + "Can be used by apache, lighttpd, requires you to configure them, which is not too easy job.") + print "lightweight:", _("Very fast alternative written in C, requires libev and linux knowledge.") + print "\t", _("Get it from here: https://github.com/jonashaag/bjoern, compile it") + print "\t", _("and copy bjoern.so to pyload/lib") + + print + print _( + "Attention: In some rare cases the builtin server is not working, if you notice problems with the webinterface") + print _("come back here and change the builtin server to the threaded one here.") + + self.config["webinterface"]["server"] = self.ask(_("Server"), "threaded", + ["builtin", "threaded", "fastcgi", "lightweight"]) + + def conf_ssl(self): + print "" + print _("## SSL Setup ##") + print "" + print _("Execute these commands from pyLoad config folder to make ssl certificates:") + print "" + print "openssl genrsa -out ssl.key 1024" + print "openssl req -new -key ssl.key -out ssl.csr" + print "openssl req -days 36500 -x509 -key ssl.key -in ssl.csr > ssl.crt " + print "" + print _("If you're done and everything went fine, you can activate ssl now.") + self.config["ssl"]["activated"] = self.ask(_("Activate SSL?"), self.yes, bool=True) + + def set_user(self): + gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) + translation = gettext.translation("setup", join(self.path, "locale"), + languages=[self.config["general"]["language"], "en"], fallback=True) + translation.install(True) + + from pyload.database import DatabaseBackend + + db = DatabaseBackend(None) + db.setup() + + noaction = True + try: + while True: + print _("Select action") + print _("1 - Create/Edit user") + print _("2 - List users") + print _("3 - Remove user") + print _("4 - Quit") + action = raw_input("[1]/2/3/4: ") + if not action in ("1", "2", "3", "4"): + continue + elif action == "1": + print "" + username = self.ask(_("Username"), "User") + password = self.ask("", "", password=True) + db.addUser(username, password) + noaction = False + elif action == "2": + print "" + print _("Users") + print "-----" + users = db.getAllUserData() + noaction = False + for user in users.itervalues(): + print user.name + print "-----" + print "" + elif action == "3": + print "" + username = self.ask(_("Username"), "") + if username: + db.removeUser(username) + noaction = False + elif action == "4": + db.syncSave() + break + finally: + if not noaction: + db.shutdown() + + def conf_path(self, trans=False): + if trans: + gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) + translation = gettext.translation("setup", join(self.path, "locale"), + languages=[self.config["general"]["language"], "en"], fallback=True) + translation.install(True) + + print _("Setting new configpath, current configuration will not be transferred!") + path = self.ask(_("Configpath"), abspath("")) + try: + path = join(pypath, path) + if not exists(path): + makedirs(path) + f = open(join(pypath, "pyload", "config", "configdir"), "wb") + f.write(path) + f.close() + print _("Configpath changed, setup will now close, please restart to go on.") + print _("Press Enter to exit.") + raw_input() + exit() + except Exception, e: + print _("Setting config path failed: %s") % str(e) + + + def ask_lang(self): + langs = self.config.getMetaData("general", "language").type.split(";") + self.lang = self.ask(u"Choose your Language / Wähle deine Sprache", "en", langs) + gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) + translation = gettext.translation("setup", join(self.path, "locale"), languages=[self.lang, "en"], fallback=True) + translation.install(True) + + #l10n Input shorthand for yes + self.yes = _("y") + #l10n Input shorthand for no + self.no = _("n") + + def ask(self, qst, default, answers=[], bool=False, password=False): + """ Generate dialog on command line """ + + if answers: + info = "(" + for i, answer in enumerate(answers): + info += (", " if i != 0 else "") + str((answer == default and "[%s]" % answer) or answer) + + info += ")" + elif bool: + if default == self.yes: + info = "([%s]/%s)" % (self.yes, self.no) + else: + info = "(%s/[%s])" % (self.yes, self.no) + else: + info = "[%s]" % default + + if password: + p1 = True + p2 = False + while p1 != p2: + # getpass(_("Password: ")) will crash on systems with broken locales (Win, NAS) + sys.stdout.write(_("Password: ")) + p1 = getpass("") + + if len(p1) < 4: + print _("Password too short. Use at least 4 symbols.") + continue + + sys.stdout.write(_("Password (again): ")) + p2 = getpass("") + + if p1 == p2: + return p1 + else: + print _("Passwords did not match.") + + while True: + input = raw_input(qst + " %s: " % info) + input = input.decode(self.stdin_encoding) + + if input.strip() == "": + input = default + + if bool: + #l10n yes, true,t are inputs for booleans with value true + if input.lower().strip() in [self.yes, _("yes"), _("true"), _("t"), "yes"]: + return True + #l10n no, false,f are inputs for booleans with value false + elif input.lower().strip() in [self.no, _("no"), _("false"), _("f"), "no"]: + return False + else: + print _("Invalid Input") + continue + + if not answers: + return input + + else: + if input in answers: + return input + else: + print _("Invalid Input") + + +if __name__ == "__main__": + test = Setup(join(abspath(dirname(__file__)), ".."), None) + test.start() diff --git a/pyload/__init__.py b/pyload/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyload/api/AccountApi.py b/pyload/api/AccountApi.py new file mode 100644 index 000000000..999484974 --- /dev/null +++ b/pyload/api/AccountApi.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pyload.Api import Api, RequirePerm, Permission + +from ApiComponent import ApiComponent + + +class AccountApi(ApiComponent): + """ All methods to control accounts """ + + @RequirePerm(Permission.Accounts) + def getAccounts(self, refresh): + """Get information about all entered accounts. + + :param refresh: reload account info + :return: list of `AccountInfo` + """ + accs = self.core.accountManager.getAllAccounts(refresh) + accounts = [] + for plugin in accs.itervalues(): + accounts.extend([acc.toInfoData() for acc in plugin.values()]) + + return accounts + + @RequirePerm(Permission.All) + def getAccountTypes(self): + """All available account types. + + :return: string list + """ + return self.core.pluginManager.getPlugins("accounts").keys() + + @RequirePerm(Permission.Accounts) + def updateAccount(self, plugin, login, password): + """Changes pw/options for specific account.""" + # TODO: options + self.core.accountManager.updateAccount(plugin, login, password, {}) + + def updateAccountInfo(self, account): + """ Update account from :class:`AccountInfo` """ + #TODO + + @RequirePerm(Permission.Accounts) + def removeAccount(self, account): + """Remove account from pyload. + + :param account: :class:`ÀccountInfo` instance + """ + self.core.accountManager.removeAccount(account.plugin, account.loginname) + + +if Api.extend(AccountApi): + del AccountApi \ No newline at end of file diff --git a/pyload/api/AddonApi.py b/pyload/api/AddonApi.py new file mode 100644 index 000000000..4ae686d2d --- /dev/null +++ b/pyload/api/AddonApi.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pyload.Api import Api, RequirePerm, Permission + +from ApiComponent import ApiComponent + +class AddonApi(ApiComponent): + """ Methods to interact with addons """ + + def getAllInfo(self): + """Returns all information stored by addon plugins. Values are always strings + + :return: {"plugin": {"name": value } } + """ + return self.core.addonManager.getAllInfo() + + def getInfoByPlugin(self, plugin): + """Returns information stored by a specific plugin. + + :param plugin: pluginname + :return: dict of attr names mapped to value {"name": value} + """ + return self.core.addonManager.getInfo(plugin) + +if Api.extend(AddonApi): + del AddonApi \ No newline at end of file diff --git a/pyload/api/ApiComponent.py b/pyload/api/ApiComponent.py new file mode 100644 index 000000000..bb333c259 --- /dev/null +++ b/pyload/api/ApiComponent.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pyload.remote.apitypes import Iface + +# Workaround to let code-completion think, this is subclass of Iface +Iface = object +class ApiComponent(Iface): + + __slots__ = [] + + def __init__(self, core, user): + # Only for auto completion, this class can not be instantiated + from pyload import Core + from pyload.datatypes.User import User + assert isinstance(core, Core) + assert issubclass(ApiComponent, Iface) + self.core = core + assert isinstance(user, User) + self.user = user + self.primaryUID = 0 + # No instantiating! + raise Exception() \ No newline at end of file diff --git a/pyload/api/CollectorApi.py b/pyload/api/CollectorApi.py new file mode 100644 index 000000000..49340285e --- /dev/null +++ b/pyload/api/CollectorApi.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pyload.Api import Api, RequirePerm, Permission + +from ApiComponent import ApiComponent + +class CollectorApi(ApiComponent): + """ Link collector """ + + @RequirePerm(Permission.All) + def getCollector(self): + pass + + @RequirePerm(Permission.Add) + def addToCollector(self, links): + pass + + @RequirePerm(Permission.Add) + def addFromCollector(self, name, new_name): + pass + + @RequirePerm(Permission.Delete) + def deleteCollPack(self, name): + pass + + @RequirePerm(Permission.Add) + def renameCollPack(self, name, new_name): + pass + + @RequirePerm(Permission.Delete) + def deleteCollLink(self, url): + pass + + +if Api.extend(CollectorApi): + del CollectorApi \ No newline at end of file diff --git a/pyload/api/ConfigApi.py b/pyload/api/ConfigApi.py new file mode 100644 index 000000000..82cfdd418 --- /dev/null +++ b/pyload/api/ConfigApi.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pyload.Api import Api, RequirePerm, Permission, ConfigHolder, ConfigItem, ConfigInfo +from pyload.utils import to_string + +from ApiComponent import ApiComponent + +# helper function to create a ConfigHolder +def toConfigHolder(section, config, values): + holder = ConfigHolder(section, config.name, config.description, config.long_desc) + holder.items = [ConfigItem(option, x.name, x.description, x.type, to_string(x.default), + to_string(values.get(option, x.default))) for option, x in + config.config.iteritems()] + return holder + + +class ConfigApi(ApiComponent): + """ Everything related to configuration """ + + def getConfigValue(self, section, option): + """Retrieve config value. + + :param section: name of category, or plugin + :param option: config option + :rtype: str + :return: config value as string + """ + value = self.core.config.get(section, option, self.primaryUID) + return to_string(value) + + def setConfigValue(self, section, option, value): + """Set new config value. + + :param section: + :param option: + :param value: new config value + """ + if option in ("limit_speed", "max_speed"): #not so nice to update the limit + self.core.requestFactory.updateBucket() + + self.core.config.set(section, option, value, self.primaryUID) + + def getConfig(self): + """Retrieves complete config of core. + + :rtype: dict of section -> ConfigHolder + """ + data = {} + for section, config, values in self.core.config.iterCoreSections(): + data[section] = toConfigHolder(section, config, values) + return data + + def getCoreConfig(self): + """ Retrieves core config sections + + :rtype: list of PluginInfo + """ + return [ConfigInfo(section, config.name, config.description, False, False) + for section, config, values in self.core.config.iterCoreSections()] + + @RequirePerm(Permission.Plugins) + def getPluginConfig(self): + """All plugins and addons the current user has configured + + :rtype: list of PluginInfo + """ + # TODO: include addons that are activated by default + # TODO: multi user + # TODO: better plugin / addon activated config + data = [] + active = [x.getName() for x in self.core.addonManager.activePlugins()] + for name, config, values in self.core.config.iterSections(self.primaryUID): + # skip unmodified and inactive addons + if not values and name not in active: continue + + item = ConfigInfo(name, config.name, config.description, + self.core.pluginManager.getCategory(name), + self.core.pluginManager.isUserPlugin(name), + values.get("activated", None if "activated" not in config.config else config.config[ + "activated"].default)) + data.append(item) + + return data + + @RequirePerm(Permission.Plugins) + def getAvailablePlugins(self): + """List of all available plugins, that are configurable + + :rtype: list of PluginInfo + """ + # TODO: filter user_context / addons when not allowed + plugins = [ConfigInfo(name, config.name, config.description, + self.core.pluginManager.getCategory(name), + self.core.pluginManager.isUserPlugin(name)) + for name, config, values in self.core.config.iterSections(self.primaryUID)] + + return plugins + + @RequirePerm(Permission.Plugins) + def loadConfig(self, name): + """Get complete config options for desired section + + :param name: Name of plugin or config section + :rtype: ConfigHolder + """ + # requires at least plugin permissions, but only admin can load core config + config, values = self.core.config.getSection(name, self.primaryUID) + return toConfigHolder(name, config, values) + + + @RequirePerm(Permission.Plugins) + def saveConfig(self, config): + """Used to save a configuration, core config can only be saved by admins + + :param config: :class:`ConfigHolder` + """ + for item in config.items: + self.core.config.set(config.name, item.name, item.value, sync=False, user=self.primaryUID) + # save the changes + self.core.config.saveValues(self.primaryUID, config.name) + + @RequirePerm(Permission.Plugins) + def deleteConfig(self, plugin): + """Deletes modified config + + :param plugin: plugin name + """ + #TODO: delete should deactivate addons? + self.core.config.delete(plugin, self.primaryUID) + + +if Api.extend(ConfigApi): + del ConfigApi \ No newline at end of file diff --git a/pyload/api/CoreApi.py b/pyload/api/CoreApi.py new file mode 100644 index 000000000..ebb194134 --- /dev/null +++ b/pyload/api/CoreApi.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pyload.Api import Api, RequirePerm, Permission, ServerStatus, Interaction +from pyload.utils.fs import join, free_space +from pyload.utils import compare_time + +from ApiComponent import ApiComponent + +class CoreApi(ApiComponent): + """ This module provides methods for general interaction with the core, like status or progress retrieval """ + + @RequirePerm(Permission.All) + def getServerVersion(self): + """pyLoad Core version """ + return self.core.version + + @RequirePerm(Permission.All) + def getWSAddress(self): + """Gets and address for the websocket based on configuration""" + # TODO SSL (wss) + return "ws://%%s:%d" % self.core.config['remote']['port'] + + @RequirePerm(Permission.All) + def getServerStatus(self): + """Some general information about the current status of pyLoad. + + :return: `ServerStatus` + """ + queue = self.core.files.getQueueStats(self.primaryUID) + total = self.core.files.getDownloadStats(self.primaryUID) + + serverStatus = ServerStatus(0, + total[0], queue[0], + total[1], queue[1], + self.isInteractionWaiting(Interaction.All), + not self.core.threadManager.pause and self.isTimeDownload(), + self.core.threadManager.pause, + self.core.config['reconnect']['activated'] and self.isTimeReconnect()) + + + for pyfile in self.core.threadManager.getActiveDownloads(self.primaryUID): + serverStatus.speed += pyfile.getSpeed() #bytes/s + + return serverStatus + + @RequirePerm(Permission.All) + def getProgressInfo(self): + """ Status of all currently running tasks + + :rtype: list of :class:`ProgressInfo` + """ + return self.core.threadManager.getProgressList(self.primaryUID) + + def pauseServer(self): + """Pause server: It won't start any new downloads, but nothing gets aborted.""" + self.core.threadManager.pause = True + + def unpauseServer(self): + """Unpause server: New Downloads will be started.""" + self.core.threadManager.pause = False + + def togglePause(self): + """Toggle pause state. + + :return: new pause state + """ + self.core.threadManager.pause ^= True + return self.core.threadManager.pause + + def toggleReconnect(self): + """Toggle reconnect activation. + + :return: new reconnect state + """ + self.core.config["reconnect"]["activated"] ^= True + return self.core.config["reconnect"]["activated"] + + def freeSpace(self): + """Available free space at download directory in bytes""" + return free_space(self.core.config["general"]["download_folder"]) + + + def quit(self): + """Clean way to quit pyLoad""" + self.core.do_kill = True + + def restart(self): + """Restart pyload core""" + self.core.do_restart = True + + def getLog(self, offset=0): + """Returns most recent log entries. + + :param offset: line offset + :return: List of log entries + """ + filename = join(self.core.config['log']['log_folder'], 'log.txt') + try: + fh = open(filename, "r") + lines = fh.readlines() + fh.close() + if offset >= len(lines): + return [] + return lines[offset:] + except: + return ['No log available'] + + @RequirePerm(Permission.All) + def isTimeDownload(self): + """Checks if pyload will start new downloads according to time in config. + + :return: bool + """ + start = self.core.config['downloadTime']['start'].split(":") + end = self.core.config['downloadTime']['end'].split(":") + return compare_time(start, end) + + @RequirePerm(Permission.All) + def isTimeReconnect(self): + """Checks if pyload will try to make a reconnect + + :return: bool + """ + start = self.core.config['reconnect']['startTime'].split(":") + end = self.core.config['reconnect']['endTime'].split(":") + return compare_time(start, end) and self.core.config["reconnect"]["activated"] + + +if Api.extend(CoreApi): + del CoreApi \ No newline at end of file diff --git a/pyload/api/DownloadApi.py b/pyload/api/DownloadApi.py new file mode 100644 index 000000000..0a01007b5 --- /dev/null +++ b/pyload/api/DownloadApi.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from os.path import isabs + +from pyload.Api import Api, RequirePerm, Permission +from pyload.utils.fs import join + +from ApiComponent import ApiComponent + +class DownloadApi(ApiComponent): + """ Component to create, add, delete or modify downloads.""" + + @RequirePerm(Permission.Add) + def generateAndAddPackages(self, links, paused=False): + """Generates and add packages + + :param links: list of urls + :param paused: paused package + :return: list of package ids + """ + return [self.addPackageP(name, urls, "", paused) for name, urls + in self.generatePackages(links).iteritems()] + + @RequirePerm(Permission.Add) + def createPackage(self, name, folder, root, password="", site="", comment="", paused=False): + """Create a new package. + + :param name: display name of the package + :param folder: folder name or relative path, abs path are not allowed + :param root: package id of root package, -1 for top level package + :param password: single pw or list of passwords separated with new line + :param site: arbitrary url to site for more information + :param comment: arbitrary comment + :param paused: No downloads will be started when True + :return: pid of newly created package + """ + + if isabs(folder): + folder = folder.replace("/", "_") + + folder = folder.replace("http://", "").replace(":", "").replace("\\", "_").replace("..", "") + + self.core.log.info(_("Added package %(name)s as folder %(folder)s") % {"name": name, "folder": folder}) + pid = self.core.files.addPackage(name, folder, root, password, site, comment, paused) + + return pid + + + @RequirePerm(Permission.Add) + def addPackage(self, name, links, password=""): + """Convenient method to add a package to the top-level and for adding links. + + :return: package id + """ + return self.addPackageChild(name, links, password, -1, False) + + @RequirePerm(Permission.Add) + def addPackageP(self, name, links, password, paused): + """ Same as above with additional paused attribute. """ + return self.addPackageChild(name, links, password, -1, paused) + + @RequirePerm(Permission.Add) + def addPackageChild(self, name, links, password, root, paused): + """Adds a package, with links to desired package. + + :param root: parents package id + :return: package id of the new package + """ + if self.core.config['general']['folder_per_package']: + folder = name + else: + folder = "" + + pid = self.createPackage(name, folder, root, password) + self.addLinks(pid, links) + + return pid + + @RequirePerm(Permission.Add) + def addLinks(self, pid, links): + """Adds links to specific package. Initiates online status fetching. + + :param pid: package id + :param links: list of urls + """ + hoster, crypter = self.core.pluginManager.parseUrls(links) + + if hoster: + self.core.files.addLinks(hoster, pid) + self.core.threadManager.createInfoThread(hoster, pid) + + self.core.threadManager.createDecryptThread(crypter, pid) + + self.core.log.info((_("Added %d links to package") + " #%d" % pid) % len(hoster)) + self.core.files.save() + + @RequirePerm(Permission.Add) + def uploadContainer(self, filename, data): + """Uploads and adds a container file to pyLoad. + + :param filename: filename, extension is important so it can correctly decrypted + :param data: file content + """ + th = open(join(self.core.config["general"]["download_folder"], "tmp_" + filename), "wb") + th.write(str(data)) + th.close() + + return self.addPackage(th.name, [th.name]) + + @RequirePerm(Permission.Delete) + def deleteFiles(self, fids): + """Deletes several file entries from pyload. + + :param fids: list of file ids + """ + for fid in fids: + self.core.files.deleteFile(fid) + + self.core.files.save() + + @RequirePerm(Permission.Delete) + def deletePackages(self, pids): + """Deletes packages and containing links. + + :param pids: list of package ids + """ + for pid in pids: + self.core.files.deletePackage(pid) + + self.core.files.save() + + + @RequirePerm(Permission.Modify) + def restartPackage(self, pid): + """Restarts a package, resets every containing files. + + :param pid: package id + """ + self.core.files.restartPackage(pid) + + @RequirePerm(Permission.Modify) + def restartFile(self, fid): + """Resets file status, so it will be downloaded again. + + :param fid: file id + """ + self.core.files.restartFile(fid) + + @RequirePerm(Permission.Modify) + def recheckPackage(self, pid): + """Check online status of all files in a package, also a default action when package is added. """ + self.core.files.reCheckPackage(pid) + + @RequirePerm(Permission.Modify) + def restartFailed(self): + """Restarts all failed failes.""" + self.core.files.restartFailed() + + @RequirePerm(Permission.Modify) + def stopAllDownloads(self): + """Aborts all running downloads.""" + + pyfiles = self.core.files.cachedFiles() + for pyfile in pyfiles: + pyfile.abortDownload() + + @RequirePerm(Permission.Modify) + def stopDownloads(self, fids): + """Aborts specific downloads. + + :param fids: list of file ids + :return: + """ + pyfiles = self.core.files.cachedFiles() + for pyfile in pyfiles: + if pyfile.id in fids: + pyfile.abortDownload() + + +if Api.extend(DownloadApi): + del DownloadApi \ No newline at end of file diff --git a/pyload/api/DownloadPreparingApi.py b/pyload/api/DownloadPreparingApi.py new file mode 100644 index 000000000..0a47fe5ab --- /dev/null +++ b/pyload/api/DownloadPreparingApi.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from itertools import chain + +from pyload.Api import Api, RequirePerm, Permission, OnlineCheck, LinkStatus, urlmatcher +from pyload.utils.fs import join +from pyload.utils.packagetools import parseNames +from pyload.network.RequestFactory import getURL + +from ApiComponent import ApiComponent + +class DownloadPreparingApi(ApiComponent): + """ All kind of methods to parse links or retrieve online status """ + + @RequirePerm(Permission.Add) + def parseURLs(self, html=None, url=None): + """Parses html content or any arbitrary text for links and returns result of `checkURLs` + + :param html: html source + :return: + """ + urls = [] + + if html: + urls += [x[0] for x in urlmatcher.findall(html)] + + if url: + page = getURL(url) + urls += [x[0] for x in urlmatcher.findall(page)] + + # remove duplicates + return self.checkURLs(set(urls)) + + + @RequirePerm(Permission.Add) + def checkURLs(self, urls): + """ Gets urls and returns pluginname mapped to list of matching urls. + + :param urls: + :return: {plugin: urls} + """ + data, crypter = self.core.pluginManager.parseUrls(urls) + plugins = {} + + for url, plugin in chain(data, crypter): + if plugin in plugins: + plugins[plugin].append(url) + else: + plugins[plugin] = [url] + + return plugins + + @RequirePerm(Permission.Add) + def checkOnlineStatus(self, urls): + """ initiates online status check, will also decrypt files. + + :param urls: + :return: initial set of data as :class:`OnlineCheck` instance containing the result id + """ + data, crypter = self.core.pluginManager.parseUrls(urls) + + # initial result does not contain the crypter links + tmp = [(url, (url, LinkStatus(url, pluginname, "unknown", 3, 0))) for url, pluginname in data] + data = parseNames(tmp) + result = {} + + for k, v in data.iteritems(): + for url, status in v: + status.packagename = k + result[url] = status + + data.update(crypter) # hoster and crypter will be processed + rid = self.core.threadManager.createResultThread(data, False) + + return OnlineCheck(rid, result) + + @RequirePerm(Permission.Add) + def checkOnlineStatusContainer(self, urls, container, data): + """ checks online status of urls and a submitted container file + + :param urls: list of urls + :param container: container file name + :param data: file content + :return: :class:`OnlineCheck` + """ + th = open(join(self.core.config["general"]["download_folder"], "tmp_" + container), "wb") + th.write(str(data)) + th.close() + urls.append(th.name) + return self.checkOnlineStatus(urls) + + @RequirePerm(Permission.Add) + def pollResults(self, rid): + """ Polls the result available for ResultID + + :param rid: `ResultID` + :return: `OnlineCheck`, if rid is -1 then there is no more data available + """ + result = self.core.threadManager.getInfoResult(rid) + + if "ALL_INFO_FETCHED" in result: + del result["ALL_INFO_FETCHED"] + return OnlineCheck(-1, result) + else: + return OnlineCheck(rid, result) + + + @RequirePerm(Permission.Add) + def generatePackages(self, links): + """ Parses links, generates packages names from urls + + :param links: list of urls + :return: package names mapped to urls + """ + result = parseNames((x, x) for x in links) + return result + + +if Api.extend(DownloadPreparingApi): + del DownloadPreparingApi \ No newline at end of file diff --git a/pyload/api/FileApi.py b/pyload/api/FileApi.py new file mode 100644 index 000000000..2ca409165 --- /dev/null +++ b/pyload/api/FileApi.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pyload.Api import Api, RequirePerm, Permission, DownloadState, PackageDoesNotExists, FileDoesNotExists +from pyload.utils import uniqify + +from ApiComponent import ApiComponent + +# TODO: user context +class FileApi(ApiComponent): + """Everything related to available packages or files. Deleting, Modifying and so on.""" + + @RequirePerm(Permission.All) + def getAllFiles(self): + """ same as `getFileTree` for toplevel root and full tree""" + return self.getFileTree(-1, True) + + @RequirePerm(Permission.All) + def getFilteredFiles(self, state): + """ same as `getFilteredFileTree` for toplevel root and full tree""" + return self.getFilteredFileTree(-1, state, True) + + @RequirePerm(Permission.All) + def getFileTree(self, pid, full): + """ Retrieve data for specific package. full=True will retrieve all data available + and can result in greater delays. + + :param pid: package id + :param full: go down the complete tree or only the first layer + :return: :class:`TreeCollection` + """ + return self.core.files.getTree(pid, full, DownloadState.All) + + @RequirePerm(Permission.All) + def getFilteredFileTree(self, pid, full, state): + """ Same as `getFileTree` but only contains files with specific download state. + + :param pid: package id + :param full: go down the complete tree or only the first layer + :param state: :class:`DownloadState`, the attributes used for filtering + :return: :class:`TreeCollection` + """ + return self.core.files.getTree(pid, full, state) + + @RequirePerm(Permission.All) + def getPackageContent(self, pid): + """ Only retrieve content of a specific package. see `getFileTree`""" + return self.getFileTree(pid, False) + + @RequirePerm(Permission.All) + def getPackageInfo(self, pid): + """Returns information about package, without detailed information about containing files + + :param pid: package id + :raises PackageDoesNotExists: + :return: :class:`PackageInfo` + """ + info = self.core.files.getPackageInfo(pid) + if not info: + raise PackageDoesNotExists(pid) + return info + + @RequirePerm(Permission.All) + def getFileInfo(self, fid): + """ Info for specific file + + :param fid: file id + :raises FileDoesNotExists: + :return: :class:`FileInfo` + + """ + info = self.core.files.getFileInfo(fid) + if not info: + raise FileDoesNotExists(fid) + return info + + @RequirePerm(Permission.Download) + def getFilePath(self, fid): + """ Internal method to get the filepath""" + info = self.getFileInfo(fid) + pack = self.core.files.getPackage(info.package) + return pack.getPath(), info.name + + + @RequirePerm(Permission.All) + def findFiles(self, pattern): + return self.core.files.getTree(-1, True, DownloadState.All, pattern) + + @RequirePerm(Permission.All) + def searchSuggestions(self, pattern): + names = self.core.db.getMatchingFilenames(pattern, self.primaryUID) + # TODO: stemming and reducing the names to provide better suggestions + return uniqify(names) + + @RequirePerm(Permission.All) + def findPackages(self, tags): + pass + + @RequirePerm(Permission.Modify) + def updatePackage(self, pack): + """Allows to modify several package attributes. + + :param pid: package id + :param data: :class:`PackageInfo` + """ + pid = pack.pid + p = self.core.files.getPackage(pid) + if not p: raise PackageDoesNotExists(pid) + + #TODO: fix + for key, value in data.iteritems(): + if key == "id": continue + setattr(p, key, value) + + p.sync() + self.core.files.save() + + @RequirePerm(Permission.Modify) + def setPackageFolder(self, pid, path): + pass + + @RequirePerm(Permission.Modify) + def movePackage(self, pid, root): + """ Set a new root for specific package. This will also moves the files on disk\ + and will only work when no file is currently downloading. + + :param pid: package id + :param root: package id of new root + :raises PackageDoesNotExists: When pid or root is missing + :return: False if package can't be moved + """ + return self.core.files.movePackage(pid, root) + + @RequirePerm(Permission.Modify) + def moveFiles(self, fids, pid): + """Move multiple files to another package. This will move the files on disk and\ + only work when files are not downloading. All files needs to be continuous ordered + in the current package. + + :param fids: list of file ids + :param pid: destination package + :return: False if files can't be moved + """ + return self.core.files.moveFiles(fids, pid) + + @RequirePerm(Permission.Modify) + def orderPackage(self, pid, position): + """Set new position for a package. + + :param pid: package id + :param position: new position, 0 for very beginning + """ + self.core.files.orderPackage(pid, position) + + @RequirePerm(Permission.Modify) + def orderFiles(self, fids, pid, position): + """ Set a new position for a bunch of files within a package. + All files have to be in the same package and must be **continuous**\ + in the package. That means no gaps between them. + + :param fids: list of file ids + :param pid: package id of parent package + :param position: new position: 0 for very beginning + """ + self.core.files.orderFiles(fids, pid, position) + + +if Api.extend(FileApi): + del FileApi \ No newline at end of file diff --git a/pyload/api/UserInteractionApi.py b/pyload/api/UserInteractionApi.py new file mode 100644 index 000000000..f5a9e9290 --- /dev/null +++ b/pyload/api/UserInteractionApi.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pyload.Api import Api, RequirePerm, Permission, Interaction + +from ApiComponent import ApiComponent + +class UserInteractionApi(ApiComponent): + """ Everything needed for user interaction """ + + @RequirePerm(Permission.Interaction) + def isInteractionWaiting(self, mode): + """ Check if task is waiting. + + :param mode: binary or'ed output type + :return: boolean + """ + return self.core.interactionManager.isTaskWaiting(self.primaryUID, mode) + + @RequirePerm(Permission.Interaction) + def getInteractionTasks(self, mode): + """Retrieve task for specific mode. + + :param mode: binary or'ed interaction types which should be retrieved + :rtype list of :class:`InteractionTask` + """ + tasks = self.core.interactionManager.getTasks(self.primaryUID, mode) + # retrieved tasks count as seen + for t in tasks: + t.seen = True + if t.type == Interaction.Notification: + t.setWaiting(self.core.interactionManager.CLIENT_THRESHOLD) + + return tasks + + @RequirePerm(Permission.Interaction) + def setInteractionResult(self, iid, result): + """Set Result for a interaction task. It will be immediately removed from task queue afterwards + + :param iid: interaction id + :param result: result as json string + """ + task = self.core.interactionManager.getTaskByID(iid) + if task and self.primaryUID == task.owner: + task.setResult(result) + + @RequirePerm(Permission.Interaction) + def getAddonHandler(self): + pass + + @RequirePerm(Permission.Interaction) + def callAddonHandler(self, plugin, func, pid_or_fid): + pass + + @RequirePerm(Permission.Download) + def generateDownloadLink(self, fid, timeout): + pass + + +if Api.extend(UserInteractionApi): + del UserInteractionApi \ No newline at end of file diff --git a/pyload/api/__init__.py b/pyload/api/__init__.py new file mode 100644 index 000000000..1348fd26f --- /dev/null +++ b/pyload/api/__init__.py @@ -0,0 +1,8 @@ +__all__ = ["CoreApi", "ConfigApi", "DownloadApi", "DownloadPreparingApi", "FileApi", + "CollectorApi", "UserInteractionApi", "AccountApi", "AddonApi"] + +# Import all components +# from .import * +# Above does not work in py 2.5 +for name in __all__: + __import__(__name__ + "." + name) \ No newline at end of file diff --git a/pyload/cli/AddPackage.py b/pyload/cli/AddPackage.py new file mode 100644 index 000000000..a73401586 --- /dev/null +++ b/pyload/cli/AddPackage.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +#Copyright (C) 2011 RaNaN +# +#This program is free software; you can redistribute it and/or modify +#it under the terms of the GNU General Public License as published by +#the Free Software Foundation; either version 3 of the License, +#or (at your option) any later version. +# +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +#See the GNU General Public License for more details. +# +#You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +### + +from Handler import Handler +from printer import * + +class AddPackage(Handler): + """ let the user add packages """ + + def init(self): + self.name = "" + self.urls = [] + + def onEnter(self, inp): + if inp == "0": + self.cli.reset() + + if not self.name: + self.name = inp + self.setInput() + elif inp == "END": + #add package + self.client.addPackage(self.name, self.urls, 1) + self.cli.reset() + else: + if inp.strip(): + self.urls.append(inp) + self.setInput() + + def renderBody(self, line): + println(line, white(_("Add Package:"))) + println(line + 1, "") + line += 2 + + if not self.name: + println(line, _("Enter a name for the new package")) + println(line + 1, "") + line += 2 + else: + println(line, _("Package: %s") % self.name) + println(line + 1, _("Parse the links you want to add.")) + println(line + 2, _("Type %s when done.") % mag("END")) + println(line + 3, _("Links added: ") + mag(len(self.urls))) + line += 4 + + println(line, "") + println(line + 1, mag("0.") + _(" back to main menu")) + + return line + 2 \ No newline at end of file diff --git a/pyload/cli/Handler.py b/pyload/cli/Handler.py new file mode 100644 index 000000000..476d09386 --- /dev/null +++ b/pyload/cli/Handler.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +#Copyright (C) 2011 RaNaN +# +#This program is free software; you can redistribute it and/or modify +#it under the terms of the GNU General Public License as published by +#the Free Software Foundation; either version 3 of the License, +#or (at your option) any later version. +# +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +#See the GNU General Public License for more details. +# +#You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +### +class Handler: + def __init__(self, cli): + self.cli = cli + self.init() + + client = property(lambda self: self.cli.client) + input = property(lambda self: self.cli.input) + + def init(self): + pass + + def onChar(self, char): + pass + + def onBackSpace(self): + pass + + def onEnter(self, inp): + pass + + def setInput(self, inp=""): + self.cli.setInput(inp) + + def backspace(self): + self.cli.setInput(self.input[:-1]) + + def renderBody(self, line): + """ gets the line where to render output and should return the line number below its content """ + return line + 1 \ No newline at end of file diff --git a/pyload/cli/ManageFiles.py b/pyload/cli/ManageFiles.py new file mode 100644 index 000000000..2304af355 --- /dev/null +++ b/pyload/cli/ManageFiles.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +#Copyright (C) 2011 RaNaN +# +#This program is free software; you can redistribute it and/or modify +#it under the terms of the GNU General Public License as published by +#the Free Software Foundation; either version 3 of the License, +#or (at your option) any later version. +# +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +#See the GNU General Public License for more details. +# +#You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +### + +from itertools import islice +from time import time + +from Handler import Handler +from printer import * + +from pyload.Api import Destination, PackageData + +class ManageFiles(Handler): + """ possibility to manage queue/collector """ + + def init(self): + self.target = Destination.Queue + self.pos = 0 #position in queue + self.package = -1 #chosen package + self.mode = "" # move/delete/restart + + self.cache = None + self.links = None + self.time = 0 + + def onChar(self, char): + if char in ("m", "d", "r"): + self.mode = char + self.setInput() + elif char == "p": + self.pos = max(0, self.pos - 5) + self.backspace() + elif char == "n": + self.pos += 5 + self.backspace() + + def onBackSpace(self): + if not self.input and self.mode: + self.mode = "" + if not self.input and self.package > -1: + self.package = -1 + + def onEnter(self, input): + if input == "0": + self.cli.reset() + elif self.package < 0 and self.mode: + #mode select + packs = self.parseInput(input) + if self.mode == "m": + [self.client.movePackage((self.target + 1) % 2, x) for x in packs] + elif self.mode == "d": + self.client.deletePackages(packs) + elif self.mode == "r": + [self.client.restartPackage(x) for x in packs] + + elif self.mode: + #edit links + links = self.parseInput(input, False) + + if self.mode == "d": + self.client.deleteFiles(links) + elif self.mode == "r": + map(self.client.restartFile, links) + + else: + #look into package + try: + self.package = int(input) + except: + pass + + self.cache = None + self.links = None + self.pos = 0 + self.mode = "" + self.setInput() + + + def renderBody(self, line): + if self.package < 0: + println(line, white(_("Manage Packages:"))) + else: + println(line, white((_("Manage Links:")))) + line += 1 + + if self.mode: + if self.mode == "m": + println(line, _("What do you want to move?")) + elif self.mode == "d": + println(line, _("What do you want to delete?")) + elif self.mode == "r": + println(line, _("What do you want to restart?")) + + println(line + 1, "Enter a single number, comma separated numbers or ranges. e.g.: 1,2,3 or 1-3.") + line += 2 + else: + println(line, _("Choose what you want to do, or enter package number.")) + println(line + 1, ("%s - %%s, %s - %%s, %s - %%s" % (mag("d"), mag("m"), mag("r"))) % ( + _("delete"), _("move"), _("restart"))) + line += 2 + + if self.package < 0: + #print package info + pack = self.getPackages() + i = 0 + for value in islice(pack, self.pos, self.pos + 5): + try: + println(line, mag(str(value.pid)) + ": " + value.name) + line += 1 + i += 1 + except Exception, e: + pass + for x in range(5 - i): + println(line, "") + line += 1 + else: + #print links info + pack = self.getLinks() + i = 0 + for value in islice(pack.links, self.pos, self.pos + 5): + try: + println(line, mag(value.fid) + ": %s | %s | %s" % ( + value.name, value.statusmsg, value.plugin)) + line += 1 + i += 1 + except Exception, e: + pass + for x in range(5 - i): + println(line, "") + line += 1 + + println(line, mag("p") + _(" - previous") + " | " + mag("n") + _(" - next")) + println(line + 1, mag("0.") + _(" back to main menu")) + + return line + 2 + + + def getPackages(self): + if self.cache and self.time + 2 < time(): + return self.cache + + if self.target == Destination.Queue: + data = self.client.getQueue() + else: + data = self.client.getCollector() + + + self.cache = data + self.time = time() + + return data + + def getLinks(self): + if self.links and self.time + 1 < time(): + return self.links + + try: + data = self.client.getPackageData(self.package) + except: + data = PackageData(links=[]) + + self.links = data + self.time = time() + + return data + + def parseInput(self, inp, package=True): + inp = inp.strip() + if "-" in inp: + l, n, h = inp.partition("-") + l = int(l) + h = int(h) + r = range(l, h + 1) + + ret = [] + if package: + for p in self.cache: + if p.pid in r: + ret.append(p.pid) + else: + for l in self.links.links: + if l.lid in r: + ret.append(l.lid) + + return ret + + else: + return [int(x) for x in inp.split(",")] diff --git a/pyload/cli/__init__.py b/pyload/cli/__init__.py new file mode 100644 index 000000000..fa8a09291 --- /dev/null +++ b/pyload/cli/__init__.py @@ -0,0 +1,2 @@ +from AddPackage import AddPackage +from ManageFiles import ManageFiles \ No newline at end of file diff --git a/pyload/cli/printer.py b/pyload/cli/printer.py new file mode 100644 index 000000000..c62c1800e --- /dev/null +++ b/pyload/cli/printer.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +def blue(string): + return "\033[1;34m" + unicode(string) + "\033[0m" + +def green(string): + return "\033[1;32m" + unicode(string) + "\033[0m" + +def yellow(string): + return "\033[1;33m" + unicode(string) + "\033[0m" + +def red(string): + return "\033[1;31m" + unicode(string) + "\033[0m" + +def cyan(string): + return "\033[1;36m" + unicode(string) + "\033[0m" + +def mag(string): + return "\033[1;35m" + unicode(string) + "\033[0m" + +def white(string): + return "\033[1;37m" + unicode(string) + "\033[0m" + +def println(line, content): + print "\033[" + str(line) + ";0H\033[2K" + content \ No newline at end of file diff --git a/pyload/config/ConfigManager.py b/pyload/config/ConfigManager.py new file mode 100644 index 000000000..f9dc3795b --- /dev/null +++ b/pyload/config/ConfigManager.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from new_collections import OrderedDict + +from pyload.Api import InvalidConfigSection +from pyload.utils import from_string, json + +from ConfigParser import ConfigParser + + +def convertKeyError(func): + """ converts KeyError into InvalidConfigSection """ + + def conv(*args, **kwargs): + try: + return func(*args, **kwargs) + except KeyError: + raise InvalidConfigSection(args[1]) + + return conv + + +class ConfigManager(ConfigParser): + """ Manages the core config and configs for addons and single user. + Has similar interface to ConfigParser. """ + + def __init__(self, core, parser): + # No __init__ call to super class is needed! + + self.core = core + self.db = core.db + # The config parser, holding the core config + self.parser = parser + + # similar to parser, separated meta data and values + self.config = OrderedDict() + + # Value cache for multiple user configs + # Values are populated from db on first access + # Entries are saved as (user, section) keys + self.values = {} + # TODO: similar to a cache, could be deleted periodically + + def save(self): + self.parser.save() + + @convertKeyError + def get(self, section, option, user=None): + """get config value, core config only available for admins. + if user is not valid default value will be returned""" + + # Core config loaded from parser, when no user is given or he is admin + if section in self.parser and user is None: + return self.parser.get(section, option) + else: + # We need the id and not the instance + # Will be None for admin user and so the same as internal access + try: + # Check if this config exists + # Configs without meta data can not be loaded! + data = self.config[section].config[option] + return self.loadValues(user, section)[option] + except KeyError: + pass # Returns default value later + + return self.config[section].config[option].default + + def loadValues(self, user, section): + if (user, section) not in self.values: + conf = self.db.loadConfig(section, user) + try: + self.values[user, section] = json.loads(conf) if conf else {} + except ValueError: # Something did go wrong when parsing + self.values[user, section] = {} + self.core.print_exc() + + return self.values[user, section] + + @convertKeyError + def set(self, section, option, value, sync=True, user=None): + """ set config value """ + + changed = False + if section in self.parser and user is None: + changed = self.parser.set(section, option, value, sync) + else: + data = self.config[section].config[option] + value = from_string(value, data.type) + old_value = self.get(section, option) + + # Values will always be saved to db, sync is ignored + if value != old_value: + changed = True + self.values[user, section][option] = value + if sync: self.saveValues(user, section) + + if changed: self.core.evm.dispatchEvent("config:changed", section, option, value) + return changed + + def saveValues(self, user, section): + if section in self.parser and user is None: + self.save() + elif (user, section) in self.values: + self.db.saveConfig(section, json.dumps(self.values[user, section]), user) + + def delete(self, section, user=None): + """ Deletes values saved in db and cached values for given user, NOT meta data + Does not trigger an error when nothing was deleted. """ + if (user, section) in self.values: + del self.values[user, section] + + self.db.deleteConfig(section, user) + self.core.evm.dispatchEvent("config:deleted", section, user) + + def iterCoreSections(self): + return self.parser.iterSections() + + def iterSections(self, user=None): + """ Yields: section, metadata, values """ + values = self.db.loadConfigsForUser(user) + + # Every section needs to be json decoded + for section, data in values.items(): + try: + values[section] = json.loads(data) if data else {} + except ValueError: + values[section] = {} + self.core.print_exc() + + for name, config in self.config.iteritems(): + yield name, config, values[name] if name in values else {} + + def getSection(self, section, user=None): + if section in self.parser and user is None: + return self.parser.getSection(section) + + values = self.loadValues(user, section) + return self.config.get(section), values diff --git a/pyload/config/ConfigParser.py b/pyload/config/ConfigParser.py new file mode 100644 index 000000000..fd22e93b9 --- /dev/null +++ b/pyload/config/ConfigParser.py @@ -0,0 +1,207 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement +from time import sleep +from os.path import exists +from gettext import gettext +from new_collections import namedtuple, OrderedDict + +from pyload.utils import from_string +from pyload.utils.fs import chmod + +from default import make_config + +CONF_VERSION = 2 +SectionTuple = namedtuple("SectionTuple", "name description long_desc config") +ConfigData = namedtuple("ConfigData", "name type description default") + +class ConfigParser: + """ + Holds and manages the configuration + meta data for config read from file. + """ + + CONFIG = "pyload.conf" + + def __init__(self, config=None): + + if config: self.CONFIG = config + + # Meta data information + self.config = OrderedDict() + # The actual config values + self.values = {} + + self.checkVersion() + + self.loadDefault() + self.parseValues(self.CONFIG) + + def loadDefault(self): + make_config(self) + + def checkVersion(self): + """Determines if config needs to be deleted""" + e = None + # workaround conflict, with GUI (which also accesses the config) so try read in 3 times + for i in range(0, 3): + try: + if exists(self.CONFIG): + f = open(self.CONFIG, "rb") + v = f.readline() + f.close() + v = v[v.find(":") + 1:].strip() + + if not v or int(v) < CONF_VERSION: + f = open(self.CONFIG, "wb") + f.write("version: " + str(CONF_VERSION)) + f.close() + print "Old version of %s deleted" % self.CONFIG + else: + f = open(self.CONFIG, "wb") + f.write("version:" + str(CONF_VERSION)) + f.close() + + except Exception, ex: + e = ex + sleep(0.3) + if e: raise e + + + def parseValues(self, filename): + """read config values from file""" + f = open(filename, "rb") + config = f.readlines()[1:] + + # save the current section + section = "" + + for line in config: + line = line.strip() + + # comment line, different variants + if not line or line.startswith("#") or line.startswith("//") or line.startswith(";"): continue + + if line.startswith("["): + section = line.replace("[", "").replace("]", "") + + if section not in self.config: + print "Unrecognized section", section + section = "" + + else: + name, non, value = line.rpartition("=") + name = name.strip() + value = value.strip() + + if not section: + print "Value without section", name + continue + + if name in self.config[section].config: + self.set(section, name, value, sync=False) + else: + print "Unrecognized option", section, name + + + def save(self): + """saves config to filename""" + + configs = [] + f = open(self.CONFIG, "wb") + configs.append(f) + chmod(self.CONFIG, 0600) + f.write("version: %i\n\n" % CONF_VERSION) + + for section, data in self.config.iteritems(): + f.write("[%s]\n" % section) + + for option, data in data.config.iteritems(): + value = self.get(section, option) + if type(value) == unicode: value = value.encode("utf8") + else: value = str(value) + + f.write('%s = %s\n' % (option, value)) + + f.write("\n") + + f.close() + + def __getitem__(self, section): + """provides dictionary like access: c['section']['option']""" + return Section(self, section) + + def __contains__(self, section): + """ checks if parser contains section """ + return section in self.config + + def get(self, section, option): + """get value or default""" + try: + return self.values[section][option] + except KeyError: + return self.config[section].config[option].default + + def set(self, section, option, value, sync=True): + """set value""" + + data = self.config[section].config[option] + value = from_string(value, data.type) + old_value = self.get(section, option) + + # only save when different values + if value != old_value: + if section not in self.values: self.values[section] = {} + self.values[section][option] = value + if sync: + self.save() + return True + + return False + + def getMetaData(self, section, option): + """ get all config data for an option """ + return self.config[section].config[option] + + def iterSections(self): + """ Yields section, config info, values, for all sections """ + + for name, config in self.config.iteritems(): + yield name, config, self.values[name] if name in self.values else {} + + def getSection(self, section): + """ Retrieves single config as tuple (section, values) """ + return self.config[section], self.values[section] if section in self.values else {} + + def addConfigSection(self, section, name, desc, long_desc, config): + """Adds a section to the config. `config` is a list of config tuples as used in plugin api defined as: + The order of the config elements is preserved with OrderedDict + """ + d = OrderedDict() + + for entry in config: + if len(entry) == 5: + conf_name, type, conf_desc, conf_verbose, default = entry + else: # config options without description + conf_name, type, conf_desc, default = entry + conf_verbose = "" + + d[conf_name] = ConfigData(gettext(conf_desc), type, gettext(conf_verbose), from_string(default, type)) + + data = SectionTuple(gettext(name), gettext(desc), gettext(long_desc), d) + self.config[section] = data + +class Section: + """provides dictionary like access for configparser""" + + def __init__(self, parser, section): + """Constructor""" + self.parser = parser + self.section = section + + def __getitem__(self, item): + """getitem""" + return self.parser.get(self.section, item) + + def __setitem__(self, item, value): + """setitem""" + self.parser.set(self.section, item, value) diff --git a/pyload/config/__init__.py b/pyload/config/__init__.py new file mode 100644 index 000000000..4b31e848b --- /dev/null +++ b/pyload/config/__init__.py @@ -0,0 +1 @@ +__author__ = 'christian' diff --git a/pyload/config/default.py b/pyload/config/default.py new file mode 100644 index 000000000..8a2044281 --- /dev/null +++ b/pyload/config/default.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- + +""" +Configuration layout for default base config +""" + +#TODO: write tooltips and descriptions +#TODO: use apis config related classes + +def make_config(config): + # Check if gettext is installed + _ = lambda x: x + + config.addConfigSection("remote", _("Remote"), _("Description"), _("Long description"), + [ + ("activated", "bool", _("Activated"), _("Tooltip"), True), + ("port", "int", _("Port"), _("Tooltip"), 7227), + ("listenaddr", "ip", _("Adress"), _("Tooltip"), "0.0.0.0"), + ]) + + config.addConfigSection("log", _("Log"), _("Description"), _("Long description"), + [ + ("log_size", "int", _("Size in kb"), _("Tooltip"), 100), + ("log_folder", "folder", _("Folder"), _("Tooltip"), "Logs"), + ("file_log", "bool", _("File Log"), _("Tooltip"), True), + ("log_count", "int", _("Count"), _("Tooltip"), 5), + ("log_rotate", "bool", _("Log Rotate"), _("Tooltip"), True), + ]) + + config.addConfigSection("permission", _("Permissions"), _("Description"), _("Long description"), + [ + ("group", "str", _("Groupname"), _("Tooltip"), "users"), + ("change_dl", "bool", _("Change Group and User of Downloads"), _("Tooltip"), False), + ("change_file", "bool", _("Change file mode of downloads"), _("Tooltip"), False), + ("user", "str", _("Username"), _("Tooltip"), "user"), + ("file", "str", _("Filemode for Downloads"), _("Tooltip"), "0644"), + ("change_group", "bool", _("Change group of running process"), _("Tooltip"), False), + ("folder", "str", _("Folder Permission mode"), _("Tooltip"), "0755"), + ("change_user", "bool", _("Change user of running process"), _("Tooltip"), False), + ]) + + config.addConfigSection("general", _("General"), _("Description"), _("Long description"), + [ + ("language", "en;de;fr;it;es;nl;sv;ru;pl;cs;sr;pt_BR", _("Language"), _("Tooltip"), "en"), + ("download_folder", "folder", _("Download Folder"), _("Tooltip"), "Downloads"), + ("checksum", "bool", _("Use Checksum"), _("Tooltip"), False), + ("folder_per_package", "bool", _("Create folder for each package"), _("Tooltip"), True), + ("debug_mode", "bool", _("Debug Mode"), _("Tooltip"), False), + ("min_free_space", "int", _("Min Free Space (MB)"), _("Tooltip"), 200), + ("renice", "int", _("CPU Priority"), _("Tooltip"), 0), + ]) + + config.addConfigSection("ssl", _("SSL"), _("Description"), _("Long description"), + [ + ("cert", "file", _("SSL Certificate"), _("Tooltip"), "ssl.crt"), + ("activated", "bool", _("Activated"), _("Tooltip"), False), + ("key", "file", _("SSL Key"), _("Tooltip"), "ssl.key"), + ]) + + config.addConfigSection("webinterface", _("Webinterface"), _("Description"), _("Long description"), + [ + ("template", "str", _("Template"), _("Tooltip"), "default"), + ("activated", "bool", _("Activated"), _("Tooltip"), True), + ("prefix", "str", _("Path Prefix"), _("Tooltip"), ""), + ("server", "auto;threaded;fallback;fastcgi", _("Server"), _("Tooltip"), "auto"), + ("force_server", "str", _("Favor specific server"), _("Tooltip"), ""), + ("host", "ip", _("IP"), _("Tooltip"), "0.0.0.0"), + ("https", "bool", _("Use HTTPS"), _("Tooltip"), False), + ("port", "int", _("Port"), _("Tooltip"), 8001), + ("develop", "str", _("Development mode"), _(""), False), + ]) + + config.addConfigSection("proxy", _("Proxy"), _("Description"), _("Long description"), + [ + ("username", "str", _("Username"), _("Tooltip"), ""), + ("proxy", "bool", _("Use Proxy"), _("Tooltip"), False), + ("address", "str", _("Address"), _("Tooltip"), "localhost"), + ("password", "password", _("Password"), _("Tooltip"), ""), + ("type", "http;socks4;socks5", _("Protocol"), _("Tooltip"), "http"), + ("port", "int", _("Port"), _("Tooltip"), 7070), + ]) + + config.addConfigSection("reconnect", _("Reconnect"), _("Description"), _("Long description"), + [ + ("endTime", "time", _("End"), _("Tooltip"), "0:00"), + ("activated", "bool", _("Use Reconnect"), _("Tooltip"), False), + ("method", "str", _("Method"), _("Tooltip"), "./reconnect.sh"), + ("startTime", "time", _("Start"), _("Tooltip"), "0:00"), + ]) + + config.addConfigSection("download", _("Download"), _("Description"), _("Long description"), + [ + ("max_downloads", "int", _("Max Parallel Downloads"), _("Tooltip"), 3), + ("limit_speed", "bool", _("Limit Download Speed"), _("Tooltip"), False), + ("interface", "str", _("Download interface to bind (ip or Name)"), _("Tooltip"), ""), + ("skip_existing", "bool", _("Skip already existing files"), _("Tooltip"), False), + ("max_speed", "int", _("Max Download Speed in kb/s"), _("Tooltip"), -1), + ("ipv6", "bool", _("Allow IPv6"), _("Tooltip"), False), + ("chunks", "int", _("Max connections for one download"), _("Tooltip"), 3), + ("restart_failed", "bool", _("Restart failed downloads on startup"), _("Tooltip"), False), + ]) + + config.addConfigSection("downloadTime", _("Download Time"), _("Description"), _("Long description"), + [ + ("start", "time", _("Start"), _("Tooltip"), "0:00"), + ("end", "time", _("End"), _("Tooltip"), "0:00"), + ]) diff --git a/pyload/database/AccountDatabase.py b/pyload/database/AccountDatabase.py new file mode 100644 index 000000000..eaa1a3203 --- /dev/null +++ b/pyload/database/AccountDatabase.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +from pyload.database import queue, async +from pyload.database import DatabaseBackend + + +class AccountMethods: + @queue + def loadAccounts(db): + db.c.execute('SELECT plugin, loginname, activated, password, options FROM accounts;') + return db.c.fetchall() + + @async + def saveAccounts(db, data): + # TODO: owner, shared + + db.c.executemany( + 'INSERT INTO accounts(plugin, loginname, activated, password, options) VALUES(?,?,?,?,?)', data) + + @async + def removeAccount(db, plugin, loginname): + db.c.execute('DELETE FROM accounts WHERE plugin=? AND loginname=?', (plugin, loginname)) + + +DatabaseBackend.registerSub(AccountMethods) \ No newline at end of file diff --git a/pyload/database/ConfigDatabase.py b/pyload/database/ConfigDatabase.py new file mode 100644 index 000000000..0c0dd72dd --- /dev/null +++ b/pyload/database/ConfigDatabase.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pyload.database import DatabaseMethods, queue, async + +class ConfigMethods(DatabaseMethods): + + @async + def saveConfig(self, plugin, config, user=None): + if user is None: user = -1 + self.c.execute('INSERT INTO settings(plugin, config, user) VALUES(?,?,?)', (plugin, config, user)) + + + @queue + def loadConfig(self, plugin, user=None): + if user is None: user = -1 + self.c.execute('SELECT config FROM settings WHERE plugin=? AND user=?', (plugin, user)) + + r = self.c.fetchone() + return r[0] if r else "" + + @async + def deleteConfig(self, plugin, user=None): + if user is None: + self.c.execute('DELETE FROM settings WHERE plugin=?', (plugin, )) + else: + self.c.execute('DELETE FROM settings WHERE plugin=? AND user=?', (plugin, user)) + + @queue + def loadAllConfigs(self): + self.c.execute('SELECT user, plugin, config FROM settings') + configs = {} + for r in self.c: + if r[0] in configs: + configs[r[0]][r[1]] = r[2] + else: + configs[r[0]] = {r[1]: r[2]} + + return configs + + @queue + def loadConfigsForUser(self, user=None): + if user is None: user = -1 + self.c.execute('SELECT plugin, config FROM settings WHERE user=?', (user,)) + configs = {} + for r in self.c: + configs[r[0]] = r[1] + + return configs + + @async + def clearAllConfigs(self): + self.c.execute('DELETE FROM settings') + + +ConfigMethods.register() \ No newline at end of file diff --git a/pyload/database/DatabaseBackend.py b/pyload/database/DatabaseBackend.py new file mode 100644 index 000000000..99a406d9c --- /dev/null +++ b/pyload/database/DatabaseBackend.py @@ -0,0 +1,492 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +############################################################################### +# Copyright(c) 2008-2012 pyLoad Team +# http://www.pyload.org +# +# This file is part of pyLoad. +# pyLoad is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN, mkaay +############################################################################### + +from threading import Thread, Event +from shutil import move + +from Queue import Queue +from traceback import print_exc + +from pyload.utils.fs import chmod, exists, remove + +try: + from pysqlite2 import dbapi2 as sqlite3 +except: + import sqlite3 + +DB = None +DB_VERSION = 6 + +def set_DB(db): + global DB + DB = db + + +def queue(f): + @staticmethod + def x(*args, **kwargs): + if DB: + return DB.queue(f, *args, **kwargs) + + return x + + +def async(f): + @staticmethod + def x(*args, **kwargs): + if DB: + return DB.async(f, *args, **kwargs) + + return x + + +def inner(f): + @staticmethod + def x(*args, **kwargs): + if DB: + return f(DB, *args, **kwargs) + + return x + + +class DatabaseMethods: + # stubs for autocompletion + core = None + manager = None + conn = None + c = None + + @classmethod + def register(cls): + DatabaseBackend.registerSub(cls) + + +class DatabaseJob(): + def __init__(self, f, *args, **kwargs): + self.done = Event() + + self.f = f + self.args = args + self.kwargs = kwargs + + self.result = None + self.exception = False + + # import inspect + # self.frame = inspect.currentframe() + + def __repr__(self): + from os.path import basename + + frame = self.frame.f_back + output = "" + for i in range(5): + output += "\t%s:%s, %s\n" % (basename(frame.f_code.co_filename), frame.f_lineno, frame.f_code.co_name) + frame = frame.f_back + del frame + del self.frame + + return "DataBase Job %s:%s\n%sResult: %s" % (self.f.__name__, self.args[1:], output, self.result) + + def processJob(self): + try: + self.result = self.f(*self.args, **self.kwargs) + except Exception, e: + print_exc() + try: + print "Database Error @", self.f.__name__, self.args[1:], self.kwargs, e + except: + pass + + self.exception = e + finally: + self.done.set() + + def wait(self): + self.done.wait() + + +class DatabaseBackend(Thread): + subs = [] + + DB_FILE = "pyload.db" + VERSION_FILE = "db.version" + + def __init__(self, core): + Thread.__init__(self) + self.setDaemon(True) + self.core = core + self.manager = None # set later + self.running = Event() + + self.jobs = Queue() + + set_DB(self) + + def setup(self): + """ *MUST* be called before db can be used !""" + self.start() + self.running.wait() + + def init(self): + """main loop, which executes commands""" + + version = self._checkVersion() + + self.conn = sqlite3.connect(self.DB_FILE) + chmod(self.DB_FILE, 0600) + + self.c = self.conn.cursor() + + if version is not None and version < DB_VERSION: + success = self._convertDB(version) + + # delete database + if not success: + self.c.close() + self.conn.close() + + try: + self.manager.core.log.warning(_("Database was deleted due to incompatible version.")) + except: + print "Database was deleted due to incompatible version." + + remove(self.VERSION_FILE) + move(self.DB_FILE, self.DB_FILE + ".backup") + f = open(self.VERSION_FILE, "wb") + f.write(str(DB_VERSION)) + f.close() + + self.conn = sqlite3.connect(self.DB_FILE) + chmod(self.DB_FILE, 0600) + self.c = self.conn.cursor() + + self._createTables() + self.conn.commit() + + + def run(self): + try: + self.init() + finally: + self.running.set() + + while True: + j = self.jobs.get() + if j == "quit": + self.c.close() + self.conn.commit() + self.conn.close() + self.closing.set() + break + j.processJob() + + + def shutdown(self): + self.running.clear() + self.closing = Event() + self.jobs.put("quit") + self.closing.wait(1) + + def _checkVersion(self): + """ get db version""" + if not exists(self.VERSION_FILE): + f = open(self.VERSION_FILE, "wb") + f.write(str(DB_VERSION)) + f.close() + return + + f = open(self.VERSION_FILE, "rb") + v = int(f.read().strip()) + f.close() + + return v + + def _convertDB(self, v): + try: + return getattr(self, "_convertV%i" % v)() + except: + return False + + #--convert scripts start + + def _convertV6(self): + return False + + #--convert scripts end + + def _createTables(self): + """create tables for database""" + + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "packages" (' + '"pid" INTEGER PRIMARY KEY AUTOINCREMENT, ' + '"name" TEXT NOT NULL, ' + '"folder" TEXT DEFAULT "" NOT NULL, ' + '"site" TEXT DEFAULT "" NOT NULL, ' + '"comment" TEXT DEFAULT "" NOT NULL, ' + '"password" TEXT DEFAULT "" NOT NULL, ' + '"added" INTEGER DEFAULT 0 NOT NULL,' # set by trigger + '"status" INTEGER DEFAULT 0 NOT NULL,' + '"tags" TEXT DEFAULT "" NOT NULL,' + '"shared" INTEGER DEFAULT 0 NOT NULL,' + '"packageorder" INTEGER DEFAULT -1 NOT NULL,' #incremented by trigger + '"root" INTEGER DEFAULT -1 NOT NULL, ' + '"owner" INTEGER NOT NULL, ' + 'FOREIGN KEY(owner) REFERENCES users(uid), ' + 'CHECK (root != pid)' + ')' + ) + + self.c.execute( + 'CREATE TRIGGER IF NOT EXISTS "insert_package" AFTER INSERT ON "packages"' + 'BEGIN ' + 'UPDATE packages SET added = strftime("%s", "now"), ' + 'packageorder = (SELECT max(p.packageorder) + 1 FROM packages p WHERE p.root=new.root) ' + 'WHERE rowid = new.rowid;' + 'END' + ) + + self.c.execute( + 'CREATE TRIGGER IF NOT EXISTS "delete_package" AFTER DELETE ON "packages"' + 'BEGIN ' + 'DELETE FROM files WHERE package = old.pid;' + 'UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > old.packageorder AND root=old.pid;' + 'END' + ) + self.c.execute('CREATE INDEX IF NOT EXISTS "package_index" ON packages(root, owner)') + self.c.execute('CREATE INDEX IF NOT EXISTS "package_owner" ON packages(owner)') + + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "files" (' + '"fid" INTEGER PRIMARY KEY AUTOINCREMENT, ' + '"name" TEXT NOT NULL, ' + '"size" INTEGER DEFAULT 0 NOT NULL, ' + '"status" INTEGER DEFAULT 0 NOT NULL, ' + '"media" INTEGER DEFAULT 1 NOT NULL,' + '"added" INTEGER DEFAULT 0 NOT NULL,' + '"fileorder" INTEGER DEFAULT -1 NOT NULL, ' + '"url" TEXT DEFAULT "" NOT NULL, ' + '"plugin" TEXT DEFAULT "" NOT NULL, ' + '"hash" TEXT DEFAULT "" NOT NULL, ' + '"dlstatus" INTEGER DEFAULT 0 NOT NULL, ' + '"error" TEXT DEFAULT "" NOT NULL, ' + '"package" INTEGER NOT NULL, ' + '"owner" INTEGER NOT NULL, ' + 'FOREIGN KEY(owner) REFERENCES users(uid), ' + 'FOREIGN KEY(package) REFERENCES packages(id)' + ')' + ) + self.c.execute('CREATE INDEX IF NOT EXISTS "file_index" ON files(package, owner)') + self.c.execute('CREATE INDEX IF NOT EXISTS "file_owner" ON files(owner)') + + self.c.execute( + 'CREATE TRIGGER IF NOT EXISTS "insert_file" AFTER INSERT ON "files"' + 'BEGIN ' + 'UPDATE files SET added = strftime("%s", "now"), ' + 'fileorder = (SELECT max(f.fileorder) + 1 FROM files f WHERE f.package=new.package) ' + 'WHERE rowid = new.rowid;' + 'END' + ) + + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "collector" (' + '"owner" INTEGER NOT NULL, ' + '"data" TEXT NOT NULL, ' + 'FOREIGN KEY(owner) REFERENCES users(uid), ' + 'PRIMARY KEY(owner) ON CONFLICT REPLACE' + ') ' + ) + + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "storage" (' + '"identifier" TEXT NOT NULL, ' + '"key" TEXT NOT NULL, ' + '"value" TEXT DEFAULT "", ' + 'PRIMARY KEY (identifier, key) ON CONFLICT REPLACE' + ')' + ) + + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "users" (' + '"uid" INTEGER PRIMARY KEY AUTOINCREMENT, ' + '"name" TEXT NOT NULL UNIQUE, ' + '"email" TEXT DEFAULT "" NOT NULL, ' + '"password" TEXT NOT NULL, ' + '"role" INTEGER DEFAULT 0 NOT NULL, ' + '"permission" INTEGER DEFAULT 0 NOT NULL, ' + '"folder" TEXT DEFAULT "" NOT NULL, ' + '"traffic" INTEGER DEFAULT -1 NOT NULL, ' + '"dllimit" INTEGER DEFAULT -1 NOT NULL, ' + '"dlquota" TEXT DEFAULT "" NOT NULL, ' + '"hddquota" INTEGER DEFAULT -1 NOT NULL, ' + '"template" TEXT DEFAULT "default" NOT NULL, ' + '"user" INTEGER DEFAULT -1 NOT NULL, ' # set by trigger to self + 'FOREIGN KEY(user) REFERENCES users(uid)' + ')' + ) + self.c.execute('CREATE INDEX IF NOT EXISTS "username_index" ON users(name)') + + self.c.execute( + 'CREATE TRIGGER IF NOT EXISTS "insert_user" AFTER INSERT ON "users"' + 'BEGIN ' + 'UPDATE users SET user = new.uid, folder=new.name ' + 'WHERE rowid = new.rowid;' + 'END' + ) + + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "settings" (' + '"plugin" TEXT NOT NULL, ' + '"user" INTEGER DEFAULT -1 NOT NULL, ' + '"config" TEXT NOT NULL, ' + 'FOREIGN KEY(user) REFERENCES users(uid), ' + 'PRIMARY KEY (plugin, user) ON CONFLICT REPLACE' + ')' + ) + + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "accounts" (' + '"plugin" TEXT NOT NULL, ' + '"loginname" TEXT NOT NULL, ' + '"owner" INTEGER NOT NULL DEFAULT -1, ' + '"activated" INTEGER DEFAULT 1, ' + '"password" TEXT DEFAULT "", ' + '"shared" INTEGER DEFAULT 0, ' + '"options" TEXT DEFAULT "", ' + 'FOREIGN KEY(owner) REFERENCES users(uid), ' + 'PRIMARY KEY (plugin, loginname, owner) ON CONFLICT REPLACE' + ')' + ) + + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "stats" (' + '"user" INTEGER NOT NULL, ' + '"plugin" TEXT NOT NULL, ' + '"time" INTEGER NOT NULL, ' + '"premium" INTEGER DEFAULT 0 NOT NULL, ' + '"amount" INTEGER DEFAULT 0 NOT NULL, ' + 'FOREIGN KEY(user) REFERENCES users(uid), ' + 'PRIMARY KEY(user, plugin, time)' + ')' + ) + self.c.execute('CREATE INDEX IF NOT EXISTS "stats_time" ON stats(time)') + + #try to lower ids + self.c.execute('SELECT max(fid) FROM files') + fid = self.c.fetchone()[0] + fid = int(fid) if fid else 0 + self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (fid, "files")) + + self.c.execute('SELECT max(pid) FROM packages') + pid = self.c.fetchone()[0] + pid = int(pid) if pid else 0 + self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (pid, "packages")) + + self.c.execute('VACUUM') + + + def createCursor(self): + return self.conn.cursor() + + @async + def commit(self): + self.conn.commit() + + @queue + def syncSave(self): + self.conn.commit() + + @async + def rollback(self): + self.conn.rollback() + + def async(self, f, *args, **kwargs): + args = (self, ) + args + job = DatabaseJob(f, *args, **kwargs) + self.jobs.put(job) + + def queue(self, f, *args, **kwargs): + args = (self, ) + args + job = DatabaseJob(f, *args, **kwargs) + self.jobs.put(job) + # only wait when db is running + if self.running.isSet(): job.wait() + return job.result + + @classmethod + def registerSub(cls, klass): + cls.subs.append(klass) + + @classmethod + def unregisterSub(cls, klass): + cls.subs.remove(klass) + + def __getattr__(self, attr): + for sub in DatabaseBackend.subs: + if hasattr(sub, attr): + return getattr(sub, attr) + raise AttributeError(attr) + +if __name__ == "__main__": + db = DatabaseBackend() + db.setup() + + class Test(): + @queue + def insert(db): + c = db.createCursor() + for i in range(1000): + c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", ("foo", i, "bar")) + + @async + def insert2(db): + c = db.createCursor() + for i in range(1000 * 1000): + c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", ("foo", i, "bar")) + + @queue + def select(db): + c = db.createCursor() + for i in range(10): + res = c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", ("foo", i)) + print res.fetchone() + + @queue + def error(db): + c = db.createCursor() + print "a" + c.execute("SELECT myerror FROM storage WHERE identifier=? AND key=?", ("foo", i)) + print "e" + + db.registerSub(Test) + from time import time + + start = time() + for i in range(100): + db.insert() + end = time() + print end - start + + start = time() + db.insert2() + end = time() + print end - start + + db.error() + diff --git a/pyload/database/FileDatabase.py b/pyload/database/FileDatabase.py new file mode 100644 index 000000000..7b39cfa47 --- /dev/null +++ b/pyload/database/FileDatabase.py @@ -0,0 +1,448 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +############################################################################### +# Copyright(c) 2008-2012 pyLoad Team +# http://www.pyload.org +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### + +from new_collections import OrderedDict + +from pyload.Api import DownloadInfo, FileInfo, PackageInfo, PackageStats, DownloadState as DS, state_string +from pyload.database import DatabaseMethods, queue, async, inner + +zero_stats = PackageStats(0, 0, 0, 0) + + +class FileMethods(DatabaseMethods): + + @queue + def filecount(self): + """returns number of files, currently only used for debugging""" + self.c.execute("SELECT COUNT(*) FROM files") + return self.c.fetchone()[0] + + @queue + def downloadstats(self, user=None): + """ number of downloads and size """ + if user is None: + self.c.execute("SELECT COUNT(*), SUM(f.size) FROM files f WHERE dlstatus != 0") + else: + self.c.execute( + "SELECT COUNT(*), SUM(f.size) FROM files f, packages p WHERE f.package = p.pid AND dlstatus != 0", + user) + + r = self.c.fetchone() + # sum is None when no elements are added + return (r[0], r[1] if r[1] is not None else 0) if r else (0, 0) + + @queue + def queuestats(self, user=None): + """ number and size of files in queue not finished yet""" + # status not in NA, finished, skipped + if user is None: + self.c.execute("SELECT COUNT(*), SUM(f.size) FROM files f WHERE dlstatus NOT IN (0,5,6)") + else: + self.c.execute( + "SELECT COUNT(*), SUM(f.size) FROM files f, package p WHERE f.package = p.pid AND p.owner=? AND dlstatus NOT IN (0,5,6)", + user) + + r = self.c.fetchone() + return (r[0], r[1] if r[1] is not None else 0) if r else (0, 0) + + + # TODO: multi user? + @queue + def processcount(self, fid=-1, user=None): + """ number of files which have to be processed """ + # status in online, queued, starting, waiting, downloading + self.c.execute("SELECT COUNT(*), SUM(size) FROM files WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", (fid, )) + return self.c.fetchone()[0] + + @queue + def processstats(self, user=None): + if user is None: + self.c.execute("SELECT COUNT(*), SUM(size) FROM files WHERE dlstatus IN (2,3,8,9,10)") + else: + self.c.execute( + "SELECT COUNT(*), SUM(f.size) FROM files f, packages p WHERE f.package = p.pid AND dlstatus IN (2,3,8,9,10)", + user) + r = self.c.fetchone() + return (r[0], r[1] if r[1] is not None else 0) if r else (0, 0) + + @queue + def addLink(self, url, name, plugin, package, owner): + # mark file status initially as missing, dlstatus - queued + self.c.execute('INSERT INTO files(url, name, plugin, status, dlstatus, package, owner) VALUES(?,?,?,1,3,?,?)', + (url, name, plugin, package, owner)) + return self.c.lastrowid + + @async + def addLinks(self, links, package, owner): + """ links is a list of tuples (url, plugin)""" + links = [(x[0], x[0], x[1], package, owner) for x in links] + self.c.executemany( + 'INSERT INTO files(url, name, plugin, status, dlstatus, package, owner) VALUES(?,?,?,1,3,?,?)', + links) + + @queue + def addFile(self, name, size, media, package, owner): + # file status - ok, dl status NA + self.c.execute('INSERT INTO files(name, size, media, package, owner) VALUES(?,?,?,?,?)', + (name, size, media, package, owner)) + return self.c.lastrowid + + @queue + def addPackage(self, name, folder, root, password, site, comment, status, owner): + self.c.execute( + 'INSERT INTO packages(name, folder, root, password, site, comment, status, owner) VALUES(?,?,?,?,?,?,?,?)' + , (name, folder, root, password, site, comment, status, owner)) + return self.c.lastrowid + + @async + def deletePackage(self, pid, owner=None): + # order updated by trigger, as well as links deleted + if owner is None: + self.c.execute('DELETE FROM packages WHERE pid=?', (pid,)) + else: + self.c.execute('DELETE FROM packages WHERE pid=? AND owner=?', (pid, owner)) + + @async + def deleteFile(self, fid, order, package, owner=None): + """ To delete a file order and package of it is needed """ + if owner is None: + self.c.execute('DELETE FROM files WHERE fid=?', (fid,)) + self.c.execute('UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=?', + (order, package)) + else: + self.c.execute('DELETE FROM files WHERE fid=? AND owner=?', (fid, owner)) + self.c.execute('UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=? AND owner=?', + (order, package, owner)) + + @async + def saveCollector(self, owner, data): + """ simply save the json string to database """ + self.c.execute("INSERT INTO collector(owner, data) VALUES (?,?)", (owner, data)) + + @queue + def retrieveCollector(self, owner): + """ retrieve the saved string """ + self.c.execute('SELECT data FROM collector WHERE owner=?', (owner,)) + r = self.c.fetchone() + if not r: return None + return r[0] + + @async + def deleteCollector(self, owner): + """ drop saved user collector """ + self.c.execute('DELETE FROM collector WHERE owner=?', (owner,)) + + @queue + def getAllFiles(self, package=None, search=None, state=None, owner=None): + """ Return dict with file information + + :param package: optional package to filter out + :param search: or search string for file name + :param unfinished: filter by dlstatus not finished + :param owner: only specific owner + """ + qry = ('SELECT fid, name, owner, size, status, media, added, fileorder, ' + 'url, plugin, hash, dlstatus, error, package FROM files WHERE ') + + arg = [] + + if state is not None and state != DS.All: + qry += 'dlstatus IN (%s) AND ' % state_string(state) + if owner is not None: + qry += 'owner=? AND ' + arg.append(owner) + + if package is not None: + arg.append(package) + qry += 'package=? AND ' + if search is not None: + search = "%%%s%%" % search.strip("%") + arg.append(search) + qry += "name LIKE ? " + + # make qry valid + if qry.endswith("WHERE "): qry = qry[:-6] + if qry.endswith("AND "): qry = qry[:-4] + + self.c.execute(qry + "ORDER BY package, fileorder", arg) + + data = OrderedDict() + for r in self.c: + f = FileInfo(r[0], r[1], r[13], r[2], r[3], r[4], r[5], r[6], r[7]) + if r[11] > 0: # dl status != NA + f.download = DownloadInfo(r[8], r[9], r[10], r[11], self.manager.statusMsg[r[11]], r[12]) + + data[r[0]] = f + + return data + + @queue + def getMatchingFilenames(self, pattern, owner=None): + """ Return matching file names for pattern, useful for search suggestions """ + qry = 'SELECT name FROM files WHERE name LIKE ?' + args = ["%%%s%%" % pattern.strip("%")] + if owner: + qry += " AND owner=?" + args.append(owner) + + self.c.execute(qry, args) + return [r[0] for r in self.c] + + @queue + def getAllPackages(self, root=None, owner=None, tags=None): + """ Return dict with package information + + :param root: optional root to filter + :param owner: optional user id + :param tags: optional tag list + """ + qry = ( + 'SELECT pid, name, folder, root, owner, site, comment, password, added, tags, status, shared, packageorder ' + 'FROM packages%s ORDER BY root, packageorder') + + if root is None: + stats = self.getPackageStats(owner=owner) + if owner is None: + self.c.execute(qry % "") + else: + self.c.execute(qry % " WHERE owner=?", (owner,)) + else: + stats = self.getPackageStats(root=root, owner=owner) + if owner is None: + self.c.execute(qry % ' WHERE root=? OR pid=?', (root, root)) + else: + self.c.execute(qry % ' WHERE (root=? OR pid=?) AND owner=?', (root, root, owner)) + + data = OrderedDict() + for r in self.c: + data[r[0]] = PackageInfo( + r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9].split(","), r[10], r[11], r[12], + stats.get(r[0], zero_stats) + ) + + return data + + @inner + def getPackageStats(self, pid=None, root=None, owner=None): + qry = ("SELECT p.pid, SUM(f.size) AS sizetotal, COUNT(f.fid) AS linkstotal, sizedone, linksdone " + "FROM packages p JOIN files f ON p.pid = f.package AND f.dlstatus > 0 %(sub)s LEFT OUTER JOIN " + "(SELECT p.pid AS pid, SUM(f.size) AS sizedone, COUNT(f.fid) AS linksdone " + "FROM packages p JOIN files f ON p.pid = f.package %(sub)s AND f.dlstatus in (5,6) GROUP BY p.pid) s ON s.pid = p.pid " + "GROUP BY p.pid") + + # status in (finished, skipped, processing) + + if root is not None: + self.c.execute(qry % {"sub": "AND (p.root=:root OR p.pid=:root)"}, locals()) + elif pid is not None: + self.c.execute(qry % {"sub": "AND p.pid=:pid"}, locals()) + elif owner is not None: + self.c.execute(qry % {"sub": "AND p.owner=:owner"}, locals()) + else: + self.c.execute(qry % {"sub": ""}) + + data = {} + for r in self.c: + data[r[0]] = PackageStats( + r[2] if r[2] else 0, + r[4] if r[4] else 0, + int(r[1]) if r[1] else 0, + int(r[3]) if r[3] else 0, + ) + + return data + + @queue + def getStatsForPackage(self, pid): + return self.getPackageStats(pid=pid)[pid] + + @queue + def getFileInfo(self, fid, force=False): + """get data for specific file, when force is true download info will be appended""" + self.c.execute('SELECT fid, name, owner, size, status, media, added, fileorder, ' + 'url, plugin, hash, dlstatus, error, package FROM files ' + 'WHERE fid=?', (fid,)) + r = self.c.fetchone() + if not r: + return None + else: + f = FileInfo(r[0], r[1], r[13], r[2], r[3], r[4], r[5], r[6], r[7]) + if r[11] > 0 or force: + f.download = DownloadInfo(r[8], r[9], r[10], r[11], self.manager.statusMsg[r[11]], r[12]) + + return f + + @queue + def getPackageInfo(self, pid, stats=True): + """get data for a specific package, optionally with package stats""" + if stats: + stats = self.getPackageStats(pid=pid) + + self.c.execute( + 'SELECT pid, name, folder, root, owner, site, comment, password, added, tags, status, shared, packageorder ' + 'FROM packages WHERE pid=?', (pid,)) + + r = self.c.fetchone() + if not r: + return None + else: + return PackageInfo( + r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9].split(","), r[10], r[11], r[12], + stats.get(r[0], zero_stats) if stats else None + ) + + # TODO: does this need owner? + @async + def updateLinkInfo(self, data): + """ data is list of tuples (name, size, status,[ hash,] url)""" + # status in (NA, Offline, Online, Queued, TempOffline) + if data and len(data[0]) == 4: + self.c.executemany('UPDATE files SET name=?, size=?, dlstatus=? WHERE url=? AND dlstatus IN (0,1,2,3,11)', + data) + else: + self.c.executemany( + 'UPDATE files SET name=?, size=?, dlstatus=?, hash=? WHERE url=? AND dlstatus IN (0,1,2,3,11)', data) + + @async + def updateFile(self, f): + self.c.execute('UPDATE files SET name=?, size=?, status=?,' + 'media=?, url=?, hash=?, dlstatus=?, error=? WHERE fid=?', + (f.name, f.size, f.filestatus, f.media, f.url, + f.hash, f.status, f.error, f.fid)) + + @async + def updatePackage(self, p): + self.c.execute( + 'UPDATE packages SET name=?, folder=?, site=?, comment=?, password=?, tags=?, status=?, shared=? WHERE pid=?', + (p.name, p.folder, p.site, p.comment, p.password, ",".join(p.tags), p.status, p.shared, p.pid)) + + # TODO: most modifying methods needs owner argument to avoid checking beforehand + @async + def orderPackage(self, pid, root, oldorder, order): + if oldorder > order: # package moved upwards + self.c.execute( + 'UPDATE packages SET packageorder=packageorder+1 WHERE packageorder >= ? AND packageorder < ? AND root=? AND packageorder >= 0' + , (order, oldorder, root)) + elif oldorder < order: # moved downwards + self.c.execute( + 'UPDATE packages SET packageorder=packageorder-1 WHERE packageorder <= ? AND packageorder > ? AND root=? AND packageorder >= 0' + , (order, oldorder, root)) + + self.c.execute('UPDATE packages SET packageorder=? WHERE pid=?', (order, pid)) + + @async + def orderFiles(self, pid, fids, oldorder, order): + diff = len(fids) + data = [] + + if oldorder > order: # moved upwards + self.c.execute('UPDATE files SET fileorder=fileorder+? WHERE fileorder >= ? AND fileorder < ? AND package=?' + , (diff, order, oldorder, pid)) + data = [(order + i, fid) for i, fid in enumerate(fids)] + elif oldorder < order: + self.c.execute( + 'UPDATE files SET fileorder=fileorder-? WHERE fileorder <= ? AND fileorder >= ? AND package=?' + , (diff, order, oldorder + diff, pid)) + data = [(order - diff + i + 1, fid) for i, fid in enumerate(fids)] + + self.c.executemany('UPDATE files SET fileorder=? WHERE fid=?', data) + + @async + def moveFiles(self, pid, fids, package): + self.c.execute('SELECT max(fileorder) FROM files WHERE package=?', (package,)) + r = self.c.fetchone() + order = (r[0] if r[0] else 0) + 1 + + self.c.execute('UPDATE files SET fileorder=fileorder-? WHERE fileorder > ? AND package=?', + (len(fids), order, pid)) + + data = [(package, order + i, fid) for i, fid in enumerate(fids)] + self.c.executemany('UPDATE files SET package=?, fileorder=? WHERE fid=?', data) + + @async + def movePackage(self, root, order, pid, dpid): + self.c.execute('SELECT max(packageorder) FROM packages WHERE root=?', (dpid,)) + r = self.c.fetchone() + max = (r[0] if r[0] else 0) + 1 + + self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND root=?', + (order, root)) + + self.c.execute('UPDATE packages SET root=?, packageorder=? WHERE pid=?', (dpid, max, pid)) + + @async + def restartFile(self, fid): + # status -> queued + self.c.execute('UPDATE files SET dlstatus=3, error="" WHERE fid=?', (fid,)) + + @async + def restartPackage(self, pid): + # status -> queued + self.c.execute('UPDATE files SET status=3 WHERE package=?', (pid,)) + + + # TODO: multi user approach + @queue + def getJob(self, occ): + """return pyfile ids, which are suitable for download and don't use a occupied plugin""" + cmd = "(%s)" % ", ".join(["'%s'" % x for x in occ]) + #TODO + + # dlstatus in online, queued | package status = ok + cmd = ("SELECT f.fid FROM files as f INNER JOIN packages as p ON f.package=p.pid " + "WHERE f.plugin NOT IN %s AND f.dlstatus IN (2,3) AND p.status=0 " + "ORDER BY p.packageorder ASC, f.fileorder ASC LIMIT 5") % cmd + + self.c.execute(cmd) + + return [x[0] for x in self.c] + + @queue + def getUnfinished(self, pid): + """return list of max length 3 ids with pyfiles in package not finished or processed""" + + # status in finished, skipped, processing + self.c.execute("SELECT fid FROM files WHERE package=? AND dlstatus NOT IN (5, 6, 14) LIMIT 3", (pid,)) + return [r[0] for r in self.c] + + @queue + def restartFailed(self, owner): + # status=queued, where status in failed, aborted, temp offline + self.c.execute("UPDATE files SET dlstatus=3, error='' WHERE dlstatus IN (7, 11, 12)") + + @queue + def findDuplicates(self, id, folder, filename): + """ checks if filename exists with different id and same package, dlstatus = finished """ + # TODO: also check root of package + self.c.execute( + "SELECT f.plugin FROM files f INNER JOIN packages as p ON f.package=p.pid AND p.folder=? WHERE f.fid!=? AND f.dlstatus=5 AND f.name=?" + , (folder, id, filename)) + return self.c.fetchone() + + @queue + def purgeLinks(self): + # fstatus = missing + self.c.execute("DELETE FROM files WHERE status == 1") + + @queue + def purgeAll(self): # only used for debugging + self.c.execute("DELETE FROM packages") + self.c.execute("DELETE FROM files") + self.c.execute("DELETE FROM collector") + + +FileMethods.register() \ No newline at end of file diff --git a/pyload/database/StatisticDatabase.py b/pyload/database/StatisticDatabase.py new file mode 100644 index 000000000..d5f9658f2 --- /dev/null +++ b/pyload/database/StatisticDatabase.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pyload.database import DatabaseMethods, queue, async, inner + +# TODO + +class StatisticMethods(DatabaseMethods): + pass + + + +StatisticMethods.register() \ No newline at end of file diff --git a/pyload/database/StorageDatabase.py b/pyload/database/StorageDatabase.py new file mode 100644 index 000000000..2d4c8a9c7 --- /dev/null +++ b/pyload/database/StorageDatabase.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" + +from pyload.database import DatabaseBackend, queue + +class StorageMethods(): + @queue + def setStorage(db, identifier, key, value): + db.c.execute("SELECT id FROM storage WHERE identifier=? AND key=?", (identifier, key)) + if db.c.fetchone() is not None: + db.c.execute("UPDATE storage SET value=? WHERE identifier=? AND key=?", (value, identifier, key)) + else: + db.c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", (identifier, key, value)) + + @queue + def getStorage(db, identifier, key=None): + if key is not None: + db.c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", (identifier, key)) + row = db.c.fetchone() + if row is not None: + return row[0] + else: + db.c.execute("SELECT key, value FROM storage WHERE identifier=?", (identifier, )) + d = {} + for row in db.c: + d[row[0]] = row[1] + return d + + @queue + def delStorage(db, identifier, key): + db.c.execute("DELETE FROM storage WHERE identifier=? AND key=?", (identifier, key)) + +DatabaseBackend.registerSub(StorageMethods) diff --git a/pyload/database/UserDatabase.py b/pyload/database/UserDatabase.py new file mode 100644 index 000000000..3dd06c912 --- /dev/null +++ b/pyload/database/UserDatabase.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- + +############################################################################### +# Copyright(c) 2008-2012 pyLoad Team +# http://www.pyload.org +# +# This file is part of pyLoad. +# pyLoad is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### + +from hashlib import sha1 +from string import letters, digits +from random import choice + +alphnum = letters+digits + +from pyload.Api import UserData + +from DatabaseBackend import DatabaseMethods, queue, async + +def random_salt(): + return "".join(choice(alphnum) for x in range(0,5)) + +class UserMethods(DatabaseMethods): + + @queue + def addUser(self, user, password): + salt = random_salt() + h = sha1(salt + password) + password = salt + h.hexdigest() + + self.c.execute('SELECT name FROM users WHERE name=?', (user, )) + if self.c.fetchone() is not None: + self.c.execute('UPDATE users SET password=? WHERE name=?', (password, user)) + else: + self.c.execute('INSERT INTO users (name, password) VALUES (?, ?)', (user, password)) + + @queue + def getUserData(self, name=None, uid=None): + qry = ('SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' + 'hddquota, user, template FROM "users" WHERE ') + + if name is not None: + self.c.execute(qry + "name=?", (name,)) + r = self.c.fetchone() + if r: + return UserData(*r) + + elif uid is not None: + self.c.execute(qry + "uid=?", (uid,)) + r = self.c.fetchone() + if r: + return UserData(*r) + + return None + + @queue + def getAllUserData(self): + self.c.execute('SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' + 'hddquota, user, template FROM "users"') + user = {} + for r in self.c: + user[r[0]] = UserData(*r) + + return user + + + @queue + def checkAuth(self, user, password): + self.c.execute('SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' + 'hddquota, user, template, password FROM "users" WHERE name=?', (user, )) + r = self.c.fetchone() + if not r: + return None + salt = r[-1][:5] + pw = r[-1][5:] + h = sha1(salt + password) + if h.hexdigest() == pw: + return UserData(*r[:-1]) + else: + return None + + @queue #TODO + def changePassword(self, user, oldpw, newpw): + self.c.execute('SELECT rowid, name, password FROM users WHERE name=?', (user, )) + r = self.c.fetchone() + if not r: + return False + + salt = r[2][:5] + pw = r[2][5:] + h = sha1(salt + oldpw) + if h.hexdigest() == pw: + salt = random_salt() + h = sha1(salt + newpw) + password = salt + h.hexdigest() + + self.c.execute("UPDATE users SET password=? WHERE name=?", (password, user)) + return True + + return False + + @async + def setPermission(self, user, perms): + self.c.execute("UPDATE users SET permission=? WHERE name=?", (perms, user)) + + @async + def setRole(self, user, role): + self.c.execute("UPDATE users SET role=? WHERE name=?", (role, user)) + + # TODO update methods + + @async + def removeUser(self, uid=None): + # deletes user and all associated accounts + self.c.execute('DELETE FROM users WHERE user=?', (uid, )) + +UserMethods.register() diff --git a/pyload/database/__init__.py b/pyload/database/__init__.py new file mode 100644 index 000000000..d3f97fb53 --- /dev/null +++ b/pyload/database/__init__.py @@ -0,0 +1,8 @@ +from DatabaseBackend import DatabaseMethods, DatabaseBackend, queue, async, inner + +from FileDatabase import FileMethods +from UserDatabase import UserMethods +from StorageDatabase import StorageMethods +from AccountDatabase import AccountMethods +from ConfigDatabase import ConfigMethods +from StatisticDatabase import StatisticMethods \ No newline at end of file diff --git a/pyload/datatypes/PyFile.py b/pyload/datatypes/PyFile.py new file mode 100644 index 000000000..ba5208795 --- /dev/null +++ b/pyload/datatypes/PyFile.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +############################################################################### +# Copyright(c) 2008-2012 pyLoad Team +# http://www.pyload.org +# +# This file is part of pyLoad. +# pyLoad is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### + +from time import sleep, time +from ReadWriteLock import ReadWriteLock + +from pyload.Api import ProgressInfo, DownloadProgress, FileInfo, DownloadInfo, DownloadStatus +from pyload.utils import lock, read_lock + +statusMap = { + "none": 0, + "offline": 1, + "online": 2, + "queued": 3, + "paused": 4, + "finished": 5, + "skipped": 6, + "failed": 7, + "starting": 8, + "waiting": 9, + "downloading": 10, + "temp. offline": 11, + "aborted": 12, + "decrypting": 13, + "processing": 14, + "custom": 15, + "unknown": 16, +} + + +class PyFile(object): + """ + Represents a file object at runtime + """ + __slots__ = ("m", "fid", "_name", "_size", "filestatus", "media", "added", "fileorder", + "url", "pluginname", "hash", "status", "error", "packageid", "ownerid", + "lock", "plugin", "waitUntil", "abort", "statusname", + "reconnected", "pluginclass") + + @staticmethod + def fromInfoData(m, info): + f = PyFile(m, info.fid, info.name, info.size, info.status, info.media, info.added, info.fileorder, + "", "", "", DownloadStatus.NA, "", info.package, info.owner) + if info.download: + f.url = info.download.url + f.pluginname = info.download.plugin + f.hash = info.download.hash + f.status = info.download.status + f.error = info.download.error + + return f + + def __init__(self, manager, fid, name, size, filestatus, media, added, fileorder, + url, pluginname, hash, status, error, package, owner): + + self.m = manager + + self.fid = int(fid) + self._name = name + self._size = size + self.filestatus = filestatus + self.media = media + self.added = added + self.fileorder = fileorder + self.url = url + self.pluginname = pluginname + self.hash = hash + self.status = status + self.error = error + self.ownerid = owner + self.packageid = package + # database information ends here + + self.lock = ReadWriteLock() + + self.plugin = None + + self.waitUntil = 0 # time() + time to wait + + # status attributes + self.abort = False + self.reconnected = False + self.statusname = None + + + @property + def id(self): + self.m.core.log.debug("Deprecated attr .id, use .fid instead") + return self.fid + + def setSize(self, value): + self._size = int(value) + + # will convert all sizes to ints + size = property(lambda self: self._size, setSize) + + def getName(self): + try: + if self.plugin.req.name: + return self.plugin.req.name + else: + return self._name + except: + return self._name + + def setName(self, name): + """ Only set unicode or utf8 strings as name """ + if type(name) == str: + name = name.decode("utf8") + + self._name = name + + name = property(getName, setName) + + def __repr__(self): + return "" % (self.id, self.name, self.pluginname) + + @lock + def initPlugin(self): + """ inits plugin instance """ + if not self.plugin: + self.pluginclass = self.m.core.pluginManager.getPlugin(self.pluginname) + self.plugin = self.pluginclass(self) + + @read_lock + def hasPlugin(self): + """Thread safe way to determine this file has initialized plugin attribute""" + return hasattr(self, "plugin") and self.plugin + + def package(self): + """ return package instance""" + return self.m.getPackage(self.packageid) + + def setStatus(self, status): + self.status = statusMap[status] + # needs to sync so status is written to database + self.sync() + + def setCustomStatus(self, msg, status="processing"): + self.statusname = msg + self.setStatus(status) + + def getStatusName(self): + if self.status not in (13, 14) or not self.statusname: + return self.m.statusMsg[self.status] + else: + return self.statusname + + def hasStatus(self, status): + return statusMap[status] == self.status + + def sync(self): + """sync PyFile instance with database""" + self.m.updateFile(self) + + @lock + def release(self): + """sync and remove from cache""" + if hasattr(self, "plugin") and self.plugin: + self.plugin.clean() + del self.plugin + + self.m.releaseFile(self.fid) + + + def toInfoData(self): + return FileInfo(self.fid, self.getName(), self.packageid, self.ownerid, self.getSize(), self.filestatus, + self.media, self.added, self.fileorder, DownloadInfo( + self.url, self.pluginname, self.hash, self.status, self.getStatusName(), self.error + ) + ) + + def getPath(self): + pass + + def move(self, pid): + pass + + @read_lock + def abortDownload(self): + """abort pyfile if possible""" + # TODO: abort timeout, currently dead locks + while self.id in self.m.core.threadManager.processingIds(): + self.abort = True + if self.plugin and self.plugin.req: + self.plugin.req.abortDownloads() + sleep(0.1) + + self.abort = False + if self.plugin and self.plugin.req: + self.plugin.req.abortDownloads() + + self.release() + + def finishIfDone(self): + """set status to finish and release file if every thread is finished with it""" + + if self.id in self.m.core.threadManager.processingIds(): + return False + + self.setStatus("finished") + self.release() + self.m.checkAllLinksFinished() + return True + + def checkIfProcessed(self): + self.m.checkAllLinksProcessed(self.id) + + def getSpeed(self): + """ calculates speed """ + try: + return self.plugin.req.speed + except: + return 0 + + def getETA(self): + """ gets established time of arrival / or waiting time""" + try: + if self.status == DownloadStatus.Waiting: + return self.waitUntil - time() + + return self.getBytesLeft() / self.getSpeed() + except: + return 0 + + def getBytesArrived(self): + """ gets bytes arrived """ + try: + return self.plugin.req.arrived + except: + return 0 + + def getBytesLeft(self): + """ gets bytes left """ + try: + return self.plugin.req.size - self.plugin.req.arrived + except: + return 0 + + def getSize(self): + """ get size of download """ + try: + if self.plugin.req.size: + return self.plugin.req.size + else: + return self.size + except: + return self.size + + def getProgressInfo(self): + return ProgressInfo(self.pluginname, self.name, self.getStatusName(), self.getETA(), + self.getBytesArrived(), self.getSize(), + DownloadProgress(self.fid, self.packageid, self.getSpeed(), self.status)) diff --git a/pyload/datatypes/PyPackage.py b/pyload/datatypes/PyPackage.py new file mode 100644 index 000000000..d23ae88e7 --- /dev/null +++ b/pyload/datatypes/PyPackage.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +############################################################################### +# Copyright(c) 2008-2012 pyLoad Team +# http://www.pyload.org +# +# This file is part of pyLoad. +# pyLoad is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### + +from time import time + +from pyload.Api import PackageInfo, PackageStatus +from pyload.utils.fs import join + +class PyPackage: + """ + Represents a package object at runtime + """ + + @staticmethod + def fromInfoData(m, info): + return PyPackage(m, info.pid, info.name, info.folder, info.root, info.owner, + info.site, info.comment, info.password, info.added, info.tags, info.status, info.shared, info.packageorder) + + def __init__(self, manager, pid, name, folder, root, owner, site, comment, password, added, tags, status, + shared, packageorder): + self.m = manager + + self.pid = pid + self.name = name + self.folder = folder + self.root = root + self.ownerid = owner + self.site = site + self.comment = comment + self.password = password + self.added = added + self.tags = tags + self.status = status + self.shared = shared + self.packageorder = packageorder + self.timestamp = time() + + #: Finish event already fired + self.setFinished = False + + @property + def id(self): + self.m.core.log.debug("Deprecated package attr .id, use .pid instead") + return self.pid + + def isStale(self): + return self.timestamp + 30 * 60 > time() + + def toInfoData(self): + return PackageInfo(self.pid, self.name, self.folder, self.root, self.ownerid, self.site, + self.comment, self.password, self.added, self.tags, self.status, self.shared, self.packageorder + ) + + def getChildren(self): + """get information about contained links""" + return self.m.getPackageData(self.pid)["links"] + + def getPath(self, name=""): + self.timestamp = time() + return join(self.m.getPackage(self.root).getPath(), self.folder, name) + + def sync(self): + """sync with db""" + self.m.updatePackage(self) + + def release(self): + """sync and delete from cache""" + self.sync() + self.m.releasePackage(self.id) + + def delete(self): + self.m.deletePackage(self.id) + + def deleteIfEmpty(self): + """ True if deleted """ + if not len(self.getChildren()): + self.delete() + return True + return False + + def notifyChange(self): + self.m.core.eventManager.dispatchEvent("packageUpdated", self.id) + + +class RootPackage(PyPackage): + def __init__(self, m, owner): + PyPackage.__init__(self, m, -1, "root", "", owner, -2, "", "", "", 0, [], PackageStatus.Ok, False, 0) + + def getPath(self, name=""): + return join(self.m.core.config["general"]["download_folder"], name) + + # no database operations + def sync(self): + pass + + def delete(self): + pass + + def release(self): + pass \ No newline at end of file diff --git a/pyload/datatypes/User.py b/pyload/datatypes/User.py new file mode 100644 index 000000000..31c9a55cc --- /dev/null +++ b/pyload/datatypes/User.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +############################################################################### +# Copyright(c) 2008-2012 pyLoad Team +# http://www.pyload.org +# +# This file is part of pyLoad. +# pyLoad is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### + + +from pyload.Api import UserData, Permission, Role +from pyload.utils import bits_set + +#TODO: activate user +#noinspection PyUnresolvedReferences +class User(UserData): + + @staticmethod + def fromUserData(api, user): + return User(api, user.uid, user.name, user.email, user.role, user.permission, user.folder, + user.traffic, user.dllimit, user.dlquota, user.hddquota, user.user, user.templateName) + + def __init__(self, api, *args, **kwargs): + UserData.__init__(self, *args, **kwargs) + self.api = api + + + def toUserData(self): + # TODO + return UserData() + + def hasPermission(self, perms): + """ Accepts permission bit or name """ + if isinstance(perms, basestring) and hasattr(Permission, perms): + perms = getattr(Permission, perms) + + return bits_set(perms, self.permission) + + def hasRole(self, role): + if isinstance(role, basestring) and hasattr(Role, role): + role = getattr(Role, role) + + return self.role == role + + def isAdmin(self): + return self.hasRole(Role.Admin) + + @property + def primary(self): + """ Primary user id, Internal user handle used for most operations + Secondary user account share id with primary user. Only Admins have no primary id. """ + if self.hasRole(Role.Admin): + return None + return self.user if self.user else self.uid \ No newline at end of file diff --git a/pyload/datatypes/__init__.py b/pyload/datatypes/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyload/interaction/EventManager.py b/pyload/interaction/EventManager.py new file mode 100644 index 000000000..7d37ca6b9 --- /dev/null +++ b/pyload/interaction/EventManager.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- + +from threading import Lock +from traceback import print_exc + +class EventManager: + """ + Handles all event-related tasks, also stores an event queue for clients, so they can retrieve them later. + + **Known Events:** + Most addon methods exist as events. These are some additional known events. + + ===================== ================ =========================================================== + Name Arguments Description + ===================== ================ =========================================================== + event eventName, *args Called for every event, with eventName and original args + download:preparing fid A download was just queued and will be prepared now. + download:start fid A plugin will immediately start the download afterwards. + download:allProcessed All links were handled, pyLoad would idle afterwards. + download:allFinished All downloads in the queue are finished. + config:changed sec, opt, value The config was changed. + ===================== ================ =========================================================== + + | Notes: + | download:allProcessed is *always* called before download:allFinished. + """ + + def __init__(self, core): + self.core = core + self.log = core.log + + # uuid : list of events + self.clients = {} + self.events = {"event": []} + + self.lock = Lock() + + def getEvents(self, uuid): + """ Get accumulated events for uuid since last call, this also registers a new client """ + if uuid not in self.clients: + self.clients[uuid] = Client() + return self.clients[uuid].get() + + def addEvent(self, event, func): + """Adds an event listener for event name""" + if event in self.events: + if func in self.events[event]: + self.log.debug("Function already registered %s" % func) + else: + self.events[event].append(func) + else: + self.events[event] = [func] + + def removeEvent(self, event, func): + """removes previously added event listener""" + if event in self.events: + self.events[event].remove(func) + + def removeFromEvents(self, func): + """ Removes func from all known events """ + for name, events in self.events.iteritems(): + if func in events: + events.remove(func) + + def dispatchEvent(self, event, *args): + """dispatches event with args""" + for f in self.events["event"]: + try: + f(event, *args) + except Exception, e: + self.log.warning("Error calling event handler %s: %s, %s, %s" + % ("event", f, args, str(e))) + if self.core.debug: + print_exc() + + if event in self.events: + for f in self.events[event]: + try: + f(*args) + except Exception, e: + self.log.warning("Error calling event handler %s: %s, %s, %s" + % (event, f, args, str(e))) + if self.core.debug: + print_exc() \ No newline at end of file diff --git a/pyload/interaction/InteractionManager.py b/pyload/interaction/InteractionManager.py new file mode 100644 index 000000000..9c5449b31 --- /dev/null +++ b/pyload/interaction/InteractionManager.py @@ -0,0 +1,166 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" +from threading import Lock +from time import time +from base64 import standard_b64encode + +from new_collections import OrderedDict + +from pyload.utils import lock, bits_set +from pyload.Api import Interaction as IA +from pyload.Api import InputType, Input + +from InteractionTask import InteractionTask + +class InteractionManager: + """ + Class that gives ability to interact with the user. + Arbitrary tasks with predefined output and input types can be set off. + """ + + # number of seconds a client is classified as active + CLIENT_THRESHOLD = 60 + NOTIFICATION_TIMEOUT = 60 * 60 * 30 + MAX_NOTIFICATIONS = 50 + + def __init__(self, core): + self.lock = Lock() + self.core = core + self.tasks = OrderedDict() #task store, for all outgoing tasks + + self.last_clients = {} + self.ids = 0 #uniue interaction ids + + def isClientConnected(self, user): + return self.last_clients.get(user, 0) + self.CLIENT_THRESHOLD > time() + + @lock + def work(self): + # old notifications will be removed + for n in [k for k, v in self.tasks.iteritems() if v.timedOut()]: + del self.tasks[n] + + # keep notifications count limited + n = [k for k,v in self.tasks.iteritems() if v.type == IA.Notification] + n.reverse() + for v in n[:self.MAX_NOTIFICATIONS]: + del self.tasks[v] + + @lock + def createNotification(self, title, content, desc="", plugin="", owner=None): + """ Creates and queues a new Notification + + :param title: short title + :param content: text content + :param desc: short form of the notification + :param plugin: plugin name + :return: :class:`InteractionTask` + """ + task = InteractionTask(self.ids, IA.Notification, Input(InputType.Text, content), "", title, desc, plugin, + owner=owner) + self.ids += 1 + self.queueTask(task) + return task + + @lock + def createQueryTask(self, input, desc, default="", plugin="", owner=None): + # input type was given, create a input widget + if type(input) == int: + input = Input(input) + if not isinstance(input, Input): + raise TypeError("'Input' class expected not '%s'" % type(input)) + + task = InteractionTask(self.ids, IA.Query, input, default, _("Query"), desc, plugin, owner=owner) + self.ids += 1 + self.queueTask(task) + return task + + @lock + def createCaptchaTask(self, img, format, filename, plugin="", type=InputType.Text, owner=None): + """ Createss a new captcha task. + + :param img: image content (not base encoded) + :param format: img format + :param type: :class:`InputType` + :return: + """ + if type == 'textual': + type = InputType.Text + elif type == 'positional': + type = InputType.Click + + input = Input(type, [standard_b64encode(img), format, filename]) + + #todo: title desc plugin + task = InteractionTask(self.ids, IA.Captcha, input, + None, _("Captcha request"), _("Please solve the captcha."), plugin, owner=owner) + + self.ids += 1 + self.queueTask(task) + return task + + @lock + def removeTask(self, task): + if task.iid in self.tasks: + del self.tasks[task.iid] + self.core.evm.dispatchEvent("interaction:deleted", task.iid) + + @lock + def getTaskByID(self, iid): + return self.tasks.get(iid, None) + + @lock + def getTasks(self, user, mode=IA.All): + # update last active clients + self.last_clients[user] = time() + + # filter current mode + tasks = [t for t in self.tasks.itervalues() if mode == IA.All or bits_set(t.type, mode)] + # filter correct user / or shared + tasks = [t for t in tasks if user is None or user == t.owner or t.shared] + + return tasks + + def isTaskWaiting(self, user, mode=IA.All): + tasks = [t for t in self.getTasks(user, mode) if not t.type == IA.Notification or not t.seen] + return len(tasks) > 0 + + def queueTask(self, task): + cli = self.isClientConnected(task.owner) + + # set waiting times based on threshold + if cli: + task.setWaiting(self.CLIENT_THRESHOLD) + else: # TODO: higher threshold after client connects? + task.setWaiting(self.CLIENT_THRESHOLD / 3) + + if task.type == IA.Notification: + task.setWaiting(self.NOTIFICATION_TIMEOUT) # notifications are valid for 30h + + for plugin in self.core.addonManager.activePlugins(): + try: + plugin.newInteractionTask(task) + except: + self.core.print_exc() + + self.tasks[task.iid] = task + self.core.evm.dispatchEvent("interaction:added", task) + + +if __name__ == "__main__": + it = InteractionTask() \ No newline at end of file diff --git a/pyload/interaction/InteractionTask.py b/pyload/interaction/InteractionTask.py new file mode 100644 index 000000000..b404aa6ce --- /dev/null +++ b/pyload/interaction/InteractionTask.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +from time import time + +from pyload.Api import InteractionTask as BaseInteractionTask +from pyload.Api import Interaction, InputType, Input + +#noinspection PyUnresolvedReferences +class InteractionTask(BaseInteractionTask): + """ + General Interaction Task extends ITask defined by api with additional fields and methods. + """ + #: Plugins can put needed data here + storage = None + #: Timestamp when task expires + wait_until = 0 + #: The received result + result = None + #: List of registered handles + handler = None + #: Error Message + error = None + #: Timeout locked + locked = False + #: A task that was retrieved counts as seen + seen = False + #: A task that is relevant to every user + shared = False + #: primary uid of the owner + owner = None + + def __init__(self, *args, **kwargs): + if 'owner' in kwargs: + self.owner = kwargs['owner'] + del kwargs['owner'] + if 'shared' in kwargs: + self.shared = kwargs['shared'] + del kwargs['shared'] + + BaseInteractionTask.__init__(self, *args, **kwargs) + + # additional internal attributes + self.storage = {} + self.handler = [] + self.wait_until = 0 + + def convertResult(self, value): + #TODO: convert based on input/output + return value + + def getResult(self): + return self.result + + def setShared(self): + """ enable shared mode, should not be reversed""" + self.shared = True + + def setResult(self, value): + self.result = self.convertResult(value) + + def setWaiting(self, sec, lock=False): + """ sets waiting in seconds from now, < 0 can be used as infinitive """ + if not self.locked: + if sec < 0: + self.wait_until = -1 + else: + self.wait_until = max(time() + sec, self.wait_until) + + if lock: self.locked = True + + def isWaiting(self): + if self.result or self.error or self.timedOut(): + return False + + return True + + def timedOut(self): + return time() > self.wait_until > -1 + + def correct(self): + [x.taskCorrect(self) for x in self.handler] + + def invalid(self): + [x.taskInvalid(self) for x in self.handler] \ No newline at end of file diff --git a/pyload/interaction/__init__.py b/pyload/interaction/__init__.py new file mode 100644 index 000000000..de6d13128 --- /dev/null +++ b/pyload/interaction/__init__.py @@ -0,0 +1,2 @@ +__author__ = 'christian' + \ No newline at end of file diff --git a/pyload/lib/Getch.py b/pyload/lib/Getch.py new file mode 100644 index 000000000..22b7ea7f8 --- /dev/null +++ b/pyload/lib/Getch.py @@ -0,0 +1,76 @@ +class Getch: + """ + Gets a single character from standard input. Does not echo to + the screen. + """ + + def __init__(self): + try: + self.impl = _GetchWindows() + except ImportError: + try: + self.impl = _GetchMacCarbon() + except(AttributeError, ImportError): + self.impl = _GetchUnix() + + def __call__(self): return self.impl() + + +class _GetchUnix: + def __init__(self): + import tty + import sys + + def __call__(self): + import sys + import tty + import termios + + fd = sys.stdin.fileno() + old_settings = termios.tcgetattr(fd) + try: + tty.setraw(sys.stdin.fileno()) + ch = sys.stdin.read(1) + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) + return ch + + +class _GetchWindows: + def __init__(self): + import msvcrt + + def __call__(self): + import msvcrt + + return msvcrt.getch() + +class _GetchMacCarbon: + """ + A function which returns the current ASCII key that is down; + if no ASCII key is down, the null string is returned. The + page http://www.mactech.com/macintosh-c/chap02-1.html was + very helpful in figuring out how to do this. + """ + + def __init__(self): + import Carbon + Carbon.Evt #see if it has this (in Unix, it doesn't) + + def __call__(self): + import Carbon + + if Carbon.Evt.EventAvail(0x0008)[0] == 0: # 0x0008 is the keyDownMask + return '' + else: + # + # The event contains the following info: + # (what,msg,when,where,mod)=Carbon.Evt.GetNextEvent(0x0008)[1] + # + # The message (msg) contains the ASCII char which is + # extracted with the 0x000000FF charCodeMask; this + # number is converted to an ASCII character with chr() and + # returned + # + (what, msg, when, where, mod) = Carbon.Evt.GetNextEvent(0x0008)[1] + return chr(msg) \ No newline at end of file diff --git a/pyload/lib/ReadWriteLock.py b/pyload/lib/ReadWriteLock.py new file mode 100644 index 000000000..cc82f3d48 --- /dev/null +++ b/pyload/lib/ReadWriteLock.py @@ -0,0 +1,232 @@ +# -*- coding: iso-8859-15 -*- +"""locks.py - Read-Write lock thread lock implementation + +See the class documentation for more info. + +Copyright (C) 2007, Heiko Wundram. +Released under the BSD-license. + +http://code.activestate.com/recipes/502283-read-write-lock-class-rlock-like/ +""" + +# Imports +# ------- + +from threading import Condition, Lock, currentThread +from time import time + + +# Read write lock +# --------------- + +class ReadWriteLock(object): + """Read-Write lock class. A read-write lock differs from a standard + threading.RLock() by allowing multiple threads to simultaneously hold a + read lock, while allowing only a single thread to hold a write lock at the + same point of time. + + When a read lock is requested while a write lock is held, the reader + is blocked; when a write lock is requested while another write lock is + held or there are read locks, the writer is blocked. + + Writers are always preferred by this implementation: if there are blocked + threads waiting for a write lock, current readers may request more read + locks (which they eventually should free, as they starve the waiting + writers otherwise), but a new thread requesting a read lock will not + be granted one, and block. This might mean starvation for readers if + two writer threads interweave their calls to acquireWrite() without + leaving a window only for readers. + + In case a current reader requests a write lock, this can and will be + satisfied without giving up the read locks first, but, only one thread + may perform this kind of lock upgrade, as a deadlock would otherwise + occur. After the write lock has been granted, the thread will hold a + full write lock, and not be downgraded after the upgrading call to + acquireWrite() has been match by a corresponding release(). + """ + + def __init__(self): + """Initialize this read-write lock.""" + + # Condition variable, used to signal waiters of a change in object + # state. + self.__condition = Condition(Lock()) + + # Initialize with no writers. + self.__writer = None + self.__upgradewritercount = 0 + self.__pendingwriters = [] + + # Initialize with no readers. + self.__readers = {} + + def acquire(self, blocking=True, timeout=None, shared=False): + if shared: + self.acquireRead(timeout) + else: + self.acquireWrite(timeout) + + def acquireRead(self, timeout=None): + """Acquire a read lock for the current thread, waiting at most + timeout seconds or doing a non-blocking check in case timeout is <= 0. + + In case timeout is None, the call to acquireRead blocks until the + lock request can be serviced. + + In case the timeout expires before the lock could be serviced, a + RuntimeError is thrown.""" + + if timeout is not None: + endtime = time() + timeout + me = currentThread() + self.__condition.acquire() + try: + if self.__writer is me: + # If we are the writer, grant a new read lock, always. + self.__writercount += 1 + return + while True: + if self.__writer is None: + # Only test anything if there is no current writer. + if self.__upgradewritercount or self.__pendingwriters: + if me in self.__readers: + # Only grant a read lock if we already have one + # in case writers are waiting for their turn. + # This means that writers can't easily get starved + # (but see below, readers can). + self.__readers[me] += 1 + return + # No, we aren't a reader (yet), wait for our turn. + else: + # Grant a new read lock, always, in case there are + # no pending writers (and no writer). + self.__readers[me] = self.__readers.get(me, 0) + 1 + return + if timeout is not None: + remaining = endtime - time() + if remaining <= 0: + # Timeout has expired, signal caller of this. + raise RuntimeError("Acquiring read lock timed out") + self.__condition.wait(remaining) + else: + self.__condition.wait() + finally: + self.__condition.release() + + def acquireWrite(self, timeout=None): + """Acquire a write lock for the current thread, waiting at most + timeout seconds or doing a non-blocking check in case timeout is <= 0. + + In case the write lock cannot be serviced due to the deadlock + condition mentioned above, a ValueError is raised. + + In case timeout is None, the call to acquireWrite blocks until the + lock request can be serviced. + + In case the timeout expires before the lock could be serviced, a + RuntimeError is thrown.""" + + if timeout is not None: + endtime = time() + timeout + me, upgradewriter = currentThread(), False + self.__condition.acquire() + try: + if self.__writer is me: + # If we are the writer, grant a new write lock, always. + self.__writercount += 1 + return + elif me in self.__readers: + # If we are a reader, no need to add us to pendingwriters, + # we get the upgradewriter slot. + if self.__upgradewritercount: + # If we are a reader and want to upgrade, and someone + # else also wants to upgrade, there is no way we can do + # this except if one of us releases all his read locks. + # Signal this to user. + raise ValueError( + "Inevitable dead lock, denying write lock" + ) + upgradewriter = True + self.__upgradewritercount = self.__readers.pop(me) + else: + # We aren't a reader, so add us to the pending writers queue + # for synchronization with the readers. + self.__pendingwriters.append(me) + while True: + if not self.__readers and self.__writer is None: + # Only test anything if there are no readers and writers. + if self.__upgradewritercount: + if upgradewriter: + # There is a writer to upgrade, and it's us. Take + # the write lock. + self.__writer = me + self.__writercount = self.__upgradewritercount + 1 + self.__upgradewritercount = 0 + return + # There is a writer to upgrade, but it's not us. + # Always leave the upgrade writer the advance slot, + # because he presumes he'll get a write lock directly + # from a previously held read lock. + elif self.__pendingwriters[0] is me: + # If there are no readers and writers, it's always + # fine for us to take the writer slot, removing us + # from the pending writers queue. + # This might mean starvation for readers, though. + self.__writer = me + self.__writercount = 1 + self.__pendingwriters = self.__pendingwriters[1:] + return + if timeout is not None: + remaining = endtime - time() + if remaining <= 0: + # Timeout has expired, signal caller of this. + if upgradewriter: + # Put us back on the reader queue. No need to + # signal anyone of this change, because no other + # writer could've taken our spot before we got + # here (because of remaining readers), as the test + # for proper conditions is at the start of the + # loop, not at the end. + self.__readers[me] = self.__upgradewritercount + self.__upgradewritercount = 0 + else: + # We were a simple pending writer, just remove us + # from the FIFO list. + self.__pendingwriters.remove(me) + raise RuntimeError("Acquiring write lock timed out") + self.__condition.wait(remaining) + else: + self.__condition.wait() + finally: + self.__condition.release() + + def release(self): + """Release the currently held lock. + + In case the current thread holds no lock, a ValueError is thrown.""" + + me = currentThread() + self.__condition.acquire() + try: + if self.__writer is me: + # We are the writer, take one nesting depth away. + self.__writercount -= 1 + if not self.__writercount: + # No more write locks; take our writer position away and + # notify waiters of the new circumstances. + self.__writer = None + self.__condition.notifyAll() + elif me in self.__readers: + # We are a reader currently, take one nesting depth away. + self.__readers[me] -= 1 + if not self.__readers[me]: + # No more read locks, take our reader position away. + del self.__readers[me] + if not self.__readers: + # No more readers, notify waiters of the new + # circumstances. + self.__condition.notifyAll() + else: + raise ValueError("Trying to release unheld lock") + finally: + self.__condition.release() diff --git a/pyload/lib/SafeEval.py b/pyload/lib/SafeEval.py new file mode 100644 index 000000000..8fc57f261 --- /dev/null +++ b/pyload/lib/SafeEval.py @@ -0,0 +1,47 @@ +## {{{ http://code.activestate.com/recipes/286134/ (r3) (modified) +import dis + +_const_codes = map(dis.opmap.__getitem__, [ + 'POP_TOP','ROT_TWO','ROT_THREE','ROT_FOUR','DUP_TOP', + 'BUILD_LIST','BUILD_MAP','BUILD_TUPLE', + 'LOAD_CONST','RETURN_VALUE','STORE_SUBSCR' + ]) + + +_load_names = ['False', 'True', 'null', 'true', 'false'] + +_locals = {'null': None, 'true': True, 'false': False} + +def _get_opcodes(codeobj): + i = 0 + opcodes = [] + s = codeobj.co_code + names = codeobj.co_names + while i < len(s): + code = ord(s[i]) + opcodes.append(code) + if code >= dis.HAVE_ARGUMENT: + i += 3 + else: + i += 1 + return opcodes, names + +def test_expr(expr, allowed_codes): + try: + c = compile(expr, "", "eval") + except: + raise ValueError, "%s is not a valid expression" % expr + codes, names = _get_opcodes(c) + for code in codes: + if code not in allowed_codes: + for n in names: + if n not in _load_names: + raise ValueError, "opcode %s not allowed" % dis.opname[code] + return c + + +def const_eval(expr): + c = test_expr(expr, _const_codes) + return eval(c, None, _locals) + +## end of http://code.activestate.com/recipes/286134/ }}} diff --git a/pyload/lib/__init__.py b/pyload/lib/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyload/lib/beaker/__init__.py b/pyload/lib/beaker/__init__.py new file mode 100644 index 000000000..792d60054 --- /dev/null +++ b/pyload/lib/beaker/__init__.py @@ -0,0 +1 @@ +# diff --git a/pyload/lib/beaker/cache.py b/pyload/lib/beaker/cache.py new file mode 100644 index 000000000..4a96537ff --- /dev/null +++ b/pyload/lib/beaker/cache.py @@ -0,0 +1,459 @@ +"""Cache object + +The Cache object is used to manage a set of cache files and their +associated backend. The backends can be rotated on the fly by +specifying an alternate type when used. + +Advanced users can add new backends in beaker.backends + +""" + +import warnings + +import beaker.container as container +import beaker.util as util +from beaker.exceptions import BeakerException, InvalidCacheBackendError + +import beaker.ext.memcached as memcached +import beaker.ext.database as database +import beaker.ext.sqla as sqla +import beaker.ext.google as google + +# Initialize the basic available backends +clsmap = { + 'memory':container.MemoryNamespaceManager, + 'dbm':container.DBMNamespaceManager, + 'file':container.FileNamespaceManager, + 'ext:memcached':memcached.MemcachedNamespaceManager, + 'ext:database':database.DatabaseNamespaceManager, + 'ext:sqla': sqla.SqlaNamespaceManager, + 'ext:google': google.GoogleNamespaceManager, + } + +# Initialize the cache region dict +cache_regions = {} +cache_managers = {} + +try: + import pkg_resources + + # Load up the additional entry point defined backends + for entry_point in pkg_resources.iter_entry_points('beaker.backends'): + try: + NamespaceManager = entry_point.load() + name = entry_point.name + if name in clsmap: + raise BeakerException("NamespaceManager name conflict,'%s' " + "already loaded" % name) + clsmap[name] = NamespaceManager + except (InvalidCacheBackendError, SyntaxError): + # Ignore invalid backends + pass + except: + import sys + from pkg_resources import DistributionNotFound + # Warn when there's a problem loading a NamespaceManager + if not isinstance(sys.exc_info()[1], DistributionNotFound): + import traceback + from StringIO import StringIO + tb = StringIO() + traceback.print_exc(file=tb) + warnings.warn("Unable to load NamespaceManager entry point: '%s': " + "%s" % (entry_point, tb.getvalue()), RuntimeWarning, + 2) +except ImportError: + pass + + + + +def cache_region(region, *deco_args): + """Decorate a function to cache itself using a cache region + + The region decorator requires arguments if there are more than + 2 of the same named function, in the same module. This is + because the namespace used for the functions cache is based on + the functions name and the module. + + + Example:: + + # Add cache region settings to beaker: + beaker.cache.cache_regions.update(dict_of_config_region_options)) + + @cache_region('short_term', 'some_data') + def populate_things(search_term, limit, offset): + return load_the_data(search_term, limit, offset) + + return load('rabbits', 20, 0) + + .. note:: + + The function being decorated must only be called with + positional arguments. + + """ + cache = [None] + + def decorate(func): + namespace = util.func_namespace(func) + def cached(*args): + reg = cache_regions[region] + if not reg.get('enabled', True): + return func(*args) + + if not cache[0]: + if region not in cache_regions: + raise BeakerException('Cache region not configured: %s' % region) + cache[0] = Cache._get_cache(namespace, reg) + + cache_key = " ".join(map(str, deco_args + args)) + def go(): + return func(*args) + + return cache[0].get_value(cache_key, createfunc=go) + cached._arg_namespace = namespace + cached._arg_region = region + return cached + return decorate + + +def region_invalidate(namespace, region, *args): + """Invalidate a cache region namespace or decorated function + + This function only invalidates cache spaces created with the + cache_region decorator. + + :param namespace: Either the namespace of the result to invalidate, or the + cached function reference + + :param region: The region the function was cached to. If the function was + cached to a single region then this argument can be None + + :param args: Arguments that were used to differentiate the cached + function as well as the arguments passed to the decorated + function + + Example:: + + # Add cache region settings to beaker: + beaker.cache.cache_regions.update(dict_of_config_region_options)) + + def populate_things(invalidate=False): + + @cache_region('short_term', 'some_data') + def load(search_term, limit, offset): + return load_the_data(search_term, limit, offset) + + # If the results should be invalidated first + if invalidate: + region_invalidate(load, None, 'some_data', + 'rabbits', 20, 0) + return load('rabbits', 20, 0) + + """ + if callable(namespace): + if not region: + region = namespace._arg_region + namespace = namespace._arg_namespace + + if not region: + raise BeakerException("Region or callable function " + "namespace is required") + else: + region = cache_regions[region] + + cache = Cache._get_cache(namespace, region) + cache_key = " ".join(str(x) for x in args) + cache.remove_value(cache_key) + + +class Cache(object): + """Front-end to the containment API implementing a data cache. + + :param namespace: the namespace of this Cache + + :param type: type of cache to use + + :param expire: seconds to keep cached data + + :param expiretime: seconds to keep cached data (legacy support) + + :param starttime: time when cache was cache was + + """ + def __init__(self, namespace, type='memory', expiretime=None, + starttime=None, expire=None, **nsargs): + try: + cls = clsmap[type] + if isinstance(cls, InvalidCacheBackendError): + raise cls + except KeyError: + raise TypeError("Unknown cache implementation %r" % type) + + self.namespace = cls(namespace, **nsargs) + self.expiretime = expiretime or expire + self.starttime = starttime + self.nsargs = nsargs + + @classmethod + def _get_cache(cls, namespace, kw): + key = namespace + str(kw) + try: + return cache_managers[key] + except KeyError: + cache_managers[key] = cache = cls(namespace, **kw) + return cache + + def put(self, key, value, **kw): + self._get_value(key, **kw).set_value(value) + set_value = put + + def get(self, key, **kw): + """Retrieve a cached value from the container""" + return self._get_value(key, **kw).get_value() + get_value = get + + def remove_value(self, key, **kw): + mycontainer = self._get_value(key, **kw) + if mycontainer.has_current_value(): + mycontainer.clear_value() + remove = remove_value + + def _get_value(self, key, **kw): + if isinstance(key, unicode): + key = key.encode('ascii', 'backslashreplace') + + if 'type' in kw: + return self._legacy_get_value(key, **kw) + + kw.setdefault('expiretime', self.expiretime) + kw.setdefault('starttime', self.starttime) + + return container.Value(key, self.namespace, **kw) + + @util.deprecated("Specifying a " + "'type' and other namespace configuration with cache.get()/put()/etc. " + "is deprecated. Specify 'type' and other namespace configuration to " + "cache_manager.get_cache() and/or the Cache constructor instead.") + def _legacy_get_value(self, key, type, **kw): + expiretime = kw.pop('expiretime', self.expiretime) + starttime = kw.pop('starttime', None) + createfunc = kw.pop('createfunc', None) + kwargs = self.nsargs.copy() + kwargs.update(kw) + c = Cache(self.namespace.namespace, type=type, **kwargs) + return c._get_value(key, expiretime=expiretime, createfunc=createfunc, + starttime=starttime) + + def clear(self): + """Clear all the values from the namespace""" + self.namespace.remove() + + # dict interface + def __getitem__(self, key): + return self.get(key) + + def __contains__(self, key): + return self._get_value(key).has_current_value() + + def has_key(self, key): + return key in self + + def __delitem__(self, key): + self.remove_value(key) + + def __setitem__(self, key, value): + self.put(key, value) + + +class CacheManager(object): + def __init__(self, **kwargs): + """Initialize a CacheManager object with a set of options + + Options should be parsed with the + :func:`~beaker.util.parse_cache_config_options` function to + ensure only valid options are used. + + """ + self.kwargs = kwargs + self.regions = kwargs.pop('cache_regions', {}) + + # Add these regions to the module global + cache_regions.update(self.regions) + + def get_cache(self, name, **kwargs): + kw = self.kwargs.copy() + kw.update(kwargs) + return Cache._get_cache(name, kw) + + def get_cache_region(self, name, region): + if region not in self.regions: + raise BeakerException('Cache region not configured: %s' % region) + kw = self.regions[region] + return Cache._get_cache(name, kw) + + def region(self, region, *args): + """Decorate a function to cache itself using a cache region + + The region decorator requires arguments if there are more than + 2 of the same named function, in the same module. This is + because the namespace used for the functions cache is based on + the functions name and the module. + + + Example:: + + # Assuming a cache object is available like: + cache = CacheManager(dict_of_config_options) + + + def populate_things(): + + @cache.region('short_term', 'some_data') + def load(search_term, limit, offset): + return load_the_data(search_term, limit, offset) + + return load('rabbits', 20, 0) + + .. note:: + + The function being decorated must only be called with + positional arguments. + + """ + return cache_region(region, *args) + + def region_invalidate(self, namespace, region, *args): + """Invalidate a cache region namespace or decorated function + + This function only invalidates cache spaces created with the + cache_region decorator. + + :param namespace: Either the namespace of the result to invalidate, or the + name of the cached function + + :param region: The region the function was cached to. If the function was + cached to a single region then this argument can be None + + :param args: Arguments that were used to differentiate the cached + function as well as the arguments passed to the decorated + function + + Example:: + + # Assuming a cache object is available like: + cache = CacheManager(dict_of_config_options) + + def populate_things(invalidate=False): + + @cache.region('short_term', 'some_data') + def load(search_term, limit, offset): + return load_the_data(search_term, limit, offset) + + # If the results should be invalidated first + if invalidate: + cache.region_invalidate(load, None, 'some_data', + 'rabbits', 20, 0) + return load('rabbits', 20, 0) + + + """ + return region_invalidate(namespace, region, *args) + if callable(namespace): + if not region: + region = namespace._arg_region + namespace = namespace._arg_namespace + + if not region: + raise BeakerException("Region or callable function " + "namespace is required") + else: + region = self.regions[region] + + cache = self.get_cache(namespace, **region) + cache_key = " ".join(str(x) for x in args) + cache.remove_value(cache_key) + + def cache(self, *args, **kwargs): + """Decorate a function to cache itself with supplied parameters + + :param args: Used to make the key unique for this function, as in region() + above. + + :param kwargs: Parameters to be passed to get_cache(), will override defaults + + Example:: + + # Assuming a cache object is available like: + cache = CacheManager(dict_of_config_options) + + + def populate_things(): + + @cache.cache('mycache', expire=15) + def load(search_term, limit, offset): + return load_the_data(search_term, limit, offset) + + return load('rabbits', 20, 0) + + .. note:: + + The function being decorated must only be called with + positional arguments. + + """ + cache = [None] + key = " ".join(str(x) for x in args) + + def decorate(func): + namespace = util.func_namespace(func) + def cached(*args): + if not cache[0]: + cache[0] = self.get_cache(namespace, **kwargs) + cache_key = key + " " + " ".join(str(x) for x in args) + def go(): + return func(*args) + return cache[0].get_value(cache_key, createfunc=go) + cached._arg_namespace = namespace + return cached + return decorate + + def invalidate(self, func, *args, **kwargs): + """Invalidate a cache decorated function + + This function only invalidates cache spaces created with the + cache decorator. + + :param func: Decorated function to invalidate + + :param args: Used to make the key unique for this function, as in region() + above. + + :param kwargs: Parameters that were passed for use by get_cache(), note that + this is only required if a ``type`` was specified for the + function + + Example:: + + # Assuming a cache object is available like: + cache = CacheManager(dict_of_config_options) + + + def populate_things(invalidate=False): + + @cache.cache('mycache', type="file", expire=15) + def load(search_term, limit, offset): + return load_the_data(search_term, limit, offset) + + # If the results should be invalidated first + if invalidate: + cache.invalidate(load, 'mycache', 'rabbits', 20, 0, type="file") + return load('rabbits', 20, 0) + + """ + namespace = func._arg_namespace + + cache = self.get_cache(namespace, **kwargs) + cache_key = " ".join(str(x) for x in args) + cache.remove_value(cache_key) diff --git a/pyload/lib/beaker/container.py b/pyload/lib/beaker/container.py new file mode 100644 index 000000000..515e97af6 --- /dev/null +++ b/pyload/lib/beaker/container.py @@ -0,0 +1,633 @@ +"""Container and Namespace classes""" +import anydbm +import cPickle +import logging +import os +import time + +import beaker.util as util +from beaker.exceptions import CreationAbortedError, MissingCacheParameter +from beaker.synchronization import _threading, file_synchronizer, \ + mutex_synchronizer, NameLock, null_synchronizer + +__all__ = ['Value', 'Container', 'ContainerContext', + 'MemoryContainer', 'DBMContainer', 'NamespaceManager', + 'MemoryNamespaceManager', 'DBMNamespaceManager', 'FileContainer', + 'OpenResourceNamespaceManager', + 'FileNamespaceManager', 'CreationAbortedError'] + + +logger = logging.getLogger('beaker.container') +if logger.isEnabledFor(logging.DEBUG): + debug = logger.debug +else: + def debug(message, *args): + pass + + +class NamespaceManager(object): + """Handles dictionary operations and locking for a namespace of + values. + + The implementation for setting and retrieving the namespace data is + handled by subclasses. + + NamespaceManager may be used alone, or may be privately accessed by + one or more Container objects. Container objects provide per-key + services like expiration times and automatic recreation of values. + + Multiple NamespaceManagers created with a particular name will all + share access to the same underlying datasource and will attempt to + synchronize against a common mutex object. The scope of this + sharing may be within a single process or across multiple + processes, depending on the type of NamespaceManager used. + + The NamespaceManager itself is generally threadsafe, except in the + case of the DBMNamespaceManager in conjunction with the gdbm dbm + implementation. + + """ + + @classmethod + def _init_dependencies(cls): + pass + + def __init__(self, namespace): + self._init_dependencies() + self.namespace = namespace + + def get_creation_lock(self, key): + raise NotImplementedError() + + def do_remove(self): + raise NotImplementedError() + + def acquire_read_lock(self): + pass + + def release_read_lock(self): + pass + + def acquire_write_lock(self, wait=True): + return True + + def release_write_lock(self): + pass + + def has_key(self, key): + return self.__contains__(key) + + def __getitem__(self, key): + raise NotImplementedError() + + def __setitem__(self, key, value): + raise NotImplementedError() + + def set_value(self, key, value, expiretime=None): + """Optional set_value() method called by Value. + + Allows an expiretime to be passed, for namespace + implementations which can prune their collections + using expiretime. + + """ + self[key] = value + + def __contains__(self, key): + raise NotImplementedError() + + def __delitem__(self, key): + raise NotImplementedError() + + def keys(self): + raise NotImplementedError() + + def remove(self): + self.do_remove() + + +class OpenResourceNamespaceManager(NamespaceManager): + """A NamespaceManager where read/write operations require opening/ + closing of a resource which is possibly mutexed. + + """ + def __init__(self, namespace): + NamespaceManager.__init__(self, namespace) + self.access_lock = self.get_access_lock() + self.openers = 0 + self.mutex = _threading.Lock() + + def get_access_lock(self): + raise NotImplementedError() + + def do_open(self, flags): + raise NotImplementedError() + + def do_close(self): + raise NotImplementedError() + + def acquire_read_lock(self): + self.access_lock.acquire_read_lock() + try: + self.open('r', checkcount = True) + except: + self.access_lock.release_read_lock() + raise + + def release_read_lock(self): + try: + self.close(checkcount = True) + finally: + self.access_lock.release_read_lock() + + def acquire_write_lock(self, wait=True): + r = self.access_lock.acquire_write_lock(wait) + try: + if (wait or r): + self.open('c', checkcount = True) + return r + except: + self.access_lock.release_write_lock() + raise + + def release_write_lock(self): + try: + self.close(checkcount=True) + finally: + self.access_lock.release_write_lock() + + def open(self, flags, checkcount=False): + self.mutex.acquire() + try: + if checkcount: + if self.openers == 0: + self.do_open(flags) + self.openers += 1 + else: + self.do_open(flags) + self.openers = 1 + finally: + self.mutex.release() + + def close(self, checkcount=False): + self.mutex.acquire() + try: + if checkcount: + self.openers -= 1 + if self.openers == 0: + self.do_close() + else: + if self.openers > 0: + self.do_close() + self.openers = 0 + finally: + self.mutex.release() + + def remove(self): + self.access_lock.acquire_write_lock() + try: + self.close(checkcount=False) + self.do_remove() + finally: + self.access_lock.release_write_lock() + +class Value(object): + __slots__ = 'key', 'createfunc', 'expiretime', 'expire_argument', 'starttime', 'storedtime',\ + 'namespace' + + def __init__(self, key, namespace, createfunc=None, expiretime=None, starttime=None): + self.key = key + self.createfunc = createfunc + self.expire_argument = expiretime + self.starttime = starttime + self.storedtime = -1 + self.namespace = namespace + + def has_value(self): + """return true if the container has a value stored. + + This is regardless of it being expired or not. + + """ + self.namespace.acquire_read_lock() + try: + return self.namespace.has_key(self.key) + finally: + self.namespace.release_read_lock() + + def can_have_value(self): + return self.has_current_value() or self.createfunc is not None + + def has_current_value(self): + self.namespace.acquire_read_lock() + try: + has_value = self.namespace.has_key(self.key) + if has_value: + try: + stored, expired, value = self._get_value() + return not self._is_expired(stored, expired) + except KeyError: + pass + return False + finally: + self.namespace.release_read_lock() + + def _is_expired(self, storedtime, expiretime): + """Return true if this container's value is expired.""" + return ( + ( + self.starttime is not None and + storedtime < self.starttime + ) + or + ( + expiretime is not None and + time.time() >= expiretime + storedtime + ) + ) + + def get_value(self): + self.namespace.acquire_read_lock() + try: + has_value = self.has_value() + if has_value: + try: + stored, expired, value = self._get_value() + if not self._is_expired(stored, expired): + return value + except KeyError: + # guard against un-mutexed backends raising KeyError + has_value = False + + if not self.createfunc: + raise KeyError(self.key) + finally: + self.namespace.release_read_lock() + + has_createlock = False + creation_lock = self.namespace.get_creation_lock(self.key) + if has_value: + if not creation_lock.acquire(wait=False): + debug("get_value returning old value while new one is created") + return value + else: + debug("lock_creatfunc (didnt wait)") + has_createlock = True + + if not has_createlock: + debug("lock_createfunc (waiting)") + creation_lock.acquire() + debug("lock_createfunc (waited)") + + try: + # see if someone created the value already + self.namespace.acquire_read_lock() + try: + if self.has_value(): + try: + stored, expired, value = self._get_value() + if not self._is_expired(stored, expired): + return value + except KeyError: + # guard against un-mutexed backends raising KeyError + pass + finally: + self.namespace.release_read_lock() + + debug("get_value creating new value") + v = self.createfunc() + self.set_value(v) + return v + finally: + creation_lock.release() + debug("released create lock") + + def _get_value(self): + value = self.namespace[self.key] + try: + stored, expired, value = value + except ValueError: + if not len(value) == 2: + raise + # Old format: upgrade + stored, value = value + expired = self.expire_argument + debug("get_value upgrading time %r expire time %r", stored, self.expire_argument) + self.namespace.release_read_lock() + self.set_value(value, stored) + self.namespace.acquire_read_lock() + except TypeError: + # occurs when the value is None. memcached + # may yank the rug from under us in which case + # that's the result + raise KeyError(self.key) + return stored, expired, value + + def set_value(self, value, storedtime=None): + self.namespace.acquire_write_lock() + try: + if storedtime is None: + storedtime = time.time() + debug("set_value stored time %r expire time %r", storedtime, self.expire_argument) + self.namespace.set_value(self.key, (storedtime, self.expire_argument, value)) + finally: + self.namespace.release_write_lock() + + def clear_value(self): + self.namespace.acquire_write_lock() + try: + debug("clear_value") + if self.namespace.has_key(self.key): + try: + del self.namespace[self.key] + except KeyError: + # guard against un-mutexed backends raising KeyError + pass + self.storedtime = -1 + finally: + self.namespace.release_write_lock() + +class AbstractDictionaryNSManager(NamespaceManager): + """A subclassable NamespaceManager that places data in a dictionary. + + Subclasses should provide a "dictionary" attribute or descriptor + which returns a dict-like object. The dictionary will store keys + that are local to the "namespace" attribute of this manager, so + ensure that the dictionary will not be used by any other namespace. + + e.g.:: + + import collections + cached_data = collections.defaultdict(dict) + + class MyDictionaryManager(AbstractDictionaryNSManager): + def __init__(self, namespace): + AbstractDictionaryNSManager.__init__(self, namespace) + self.dictionary = cached_data[self.namespace] + + The above stores data in a global dictionary called "cached_data", + which is structured as a dictionary of dictionaries, keyed + first on namespace name to a sub-dictionary, then on actual + cache key to value. + + """ + + def get_creation_lock(self, key): + return NameLock( + identifier="memorynamespace/funclock/%s/%s" % (self.namespace, key), + reentrant=True + ) + + def __getitem__(self, key): + return self.dictionary[key] + + def __contains__(self, key): + return self.dictionary.__contains__(key) + + def has_key(self, key): + return self.dictionary.__contains__(key) + + def __setitem__(self, key, value): + self.dictionary[key] = value + + def __delitem__(self, key): + del self.dictionary[key] + + def do_remove(self): + self.dictionary.clear() + + def keys(self): + return self.dictionary.keys() + +class MemoryNamespaceManager(AbstractDictionaryNSManager): + namespaces = util.SyncDict() + + def __init__(self, namespace, **kwargs): + AbstractDictionaryNSManager.__init__(self, namespace) + self.dictionary = MemoryNamespaceManager.namespaces.get(self.namespace, + dict) + +class DBMNamespaceManager(OpenResourceNamespaceManager): + def __init__(self, namespace, dbmmodule=None, data_dir=None, + dbm_dir=None, lock_dir=None, digest_filenames=True, **kwargs): + self.digest_filenames = digest_filenames + + if not dbm_dir and not data_dir: + raise MissingCacheParameter("data_dir or dbm_dir is required") + elif dbm_dir: + self.dbm_dir = dbm_dir + else: + self.dbm_dir = data_dir + "/container_dbm" + util.verify_directory(self.dbm_dir) + + if not lock_dir and not data_dir: + raise MissingCacheParameter("data_dir or lock_dir is required") + elif lock_dir: + self.lock_dir = lock_dir + else: + self.lock_dir = data_dir + "/container_dbm_lock" + util.verify_directory(self.lock_dir) + + self.dbmmodule = dbmmodule or anydbm + + self.dbm = None + OpenResourceNamespaceManager.__init__(self, namespace) + + self.file = util.encoded_path(root= self.dbm_dir, + identifiers=[self.namespace], + extension='.dbm', + digest_filenames=self.digest_filenames) + + debug("data file %s", self.file) + self._checkfile() + + def get_access_lock(self): + return file_synchronizer(identifier=self.namespace, + lock_dir=self.lock_dir) + + def get_creation_lock(self, key): + return file_synchronizer( + identifier = "dbmcontainer/funclock/%s" % self.namespace, + lock_dir=self.lock_dir + ) + + def file_exists(self, file): + if os.access(file, os.F_OK): + return True + else: + for ext in ('db', 'dat', 'pag', 'dir'): + if os.access(file + os.extsep + ext, os.F_OK): + return True + + return False + + def _checkfile(self): + if not self.file_exists(self.file): + g = self.dbmmodule.open(self.file, 'c') + g.close() + + def get_filenames(self): + list = [] + if os.access(self.file, os.F_OK): + list.append(self.file) + + for ext in ('pag', 'dir', 'db', 'dat'): + if os.access(self.file + os.extsep + ext, os.F_OK): + list.append(self.file + os.extsep + ext) + return list + + def do_open(self, flags): + debug("opening dbm file %s", self.file) + try: + self.dbm = self.dbmmodule.open(self.file, flags) + except: + self._checkfile() + self.dbm = self.dbmmodule.open(self.file, flags) + + def do_close(self): + if self.dbm is not None: + debug("closing dbm file %s", self.file) + self.dbm.close() + + def do_remove(self): + for f in self.get_filenames(): + os.remove(f) + + def __getitem__(self, key): + return cPickle.loads(self.dbm[key]) + + def __contains__(self, key): + return self.dbm.has_key(key) + + def __setitem__(self, key, value): + self.dbm[key] = cPickle.dumps(value) + + def __delitem__(self, key): + del self.dbm[key] + + def keys(self): + return self.dbm.keys() + + +class FileNamespaceManager(OpenResourceNamespaceManager): + def __init__(self, namespace, data_dir=None, file_dir=None, lock_dir=None, + digest_filenames=True, **kwargs): + self.digest_filenames = digest_filenames + + if not file_dir and not data_dir: + raise MissingCacheParameter("data_dir or file_dir is required") + elif file_dir: + self.file_dir = file_dir + else: + self.file_dir = data_dir + "/container_file" + util.verify_directory(self.file_dir) + + if not lock_dir and not data_dir: + raise MissingCacheParameter("data_dir or lock_dir is required") + elif lock_dir: + self.lock_dir = lock_dir + else: + self.lock_dir = data_dir + "/container_file_lock" + util.verify_directory(self.lock_dir) + OpenResourceNamespaceManager.__init__(self, namespace) + + self.file = util.encoded_path(root=self.file_dir, + identifiers=[self.namespace], + extension='.cache', + digest_filenames=self.digest_filenames) + self.hash = {} + + debug("data file %s", self.file) + + def get_access_lock(self): + return file_synchronizer(identifier=self.namespace, + lock_dir=self.lock_dir) + + def get_creation_lock(self, key): + return file_synchronizer( + identifier = "filecontainer/funclock/%s" % self.namespace, + lock_dir = self.lock_dir + ) + + def file_exists(self, file): + return os.access(file, os.F_OK) + + def do_open(self, flags): + if self.file_exists(self.file): + fh = open(self.file, 'rb') + try: + self.hash = cPickle.load(fh) + except (IOError, OSError, EOFError, cPickle.PickleError, ValueError): + pass + fh.close() + + self.flags = flags + + def do_close(self): + if self.flags == 'c' or self.flags == 'w': + fh = open(self.file, 'wb') + cPickle.dump(self.hash, fh) + fh.close() + + self.hash = {} + self.flags = None + + def do_remove(self): + try: + os.remove(self.file) + except OSError, err: + # for instance, because we haven't yet used this cache, + # but client code has asked for a clear() operation... + pass + self.hash = {} + + def __getitem__(self, key): + return self.hash[key] + + def __contains__(self, key): + return self.hash.has_key(key) + + def __setitem__(self, key, value): + self.hash[key] = value + + def __delitem__(self, key): + del self.hash[key] + + def keys(self): + return self.hash.keys() + + +#### legacy stuff to support the old "Container" class interface + +namespace_classes = {} + +ContainerContext = dict + +class ContainerMeta(type): + def __init__(cls, classname, bases, dict_): + namespace_classes[cls] = cls.namespace_class + return type.__init__(cls, classname, bases, dict_) + def __call__(self, key, context, namespace, createfunc=None, + expiretime=None, starttime=None, **kwargs): + if namespace in context: + ns = context[namespace] + else: + nscls = namespace_classes[self] + context[namespace] = ns = nscls(namespace, **kwargs) + return Value(key, ns, createfunc=createfunc, + expiretime=expiretime, starttime=starttime) + +class Container(object): + __metaclass__ = ContainerMeta + namespace_class = NamespaceManager + +class FileContainer(Container): + namespace_class = FileNamespaceManager + +class MemoryContainer(Container): + namespace_class = MemoryNamespaceManager + +class DBMContainer(Container): + namespace_class = DBMNamespaceManager + +DbmContainer = DBMContainer diff --git a/pyload/lib/beaker/converters.py b/pyload/lib/beaker/converters.py new file mode 100644 index 000000000..f0ad34963 --- /dev/null +++ b/pyload/lib/beaker/converters.py @@ -0,0 +1,26 @@ +# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org) +# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php +def asbool(obj): + if isinstance(obj, (str, unicode)): + obj = obj.strip().lower() + if obj in ['true', 'yes', 'on', 'y', 't', '1']: + return True + elif obj in ['false', 'no', 'off', 'n', 'f', '0']: + return False + else: + raise ValueError( + "String is not true/false: %r" % obj) + return bool(obj) + +def aslist(obj, sep=None, strip=True): + if isinstance(obj, (str, unicode)): + lst = obj.split(sep) + if strip: + lst = [v.strip() for v in lst] + return lst + elif isinstance(obj, (list, tuple)): + return obj + elif obj is None: + return [] + else: + return [obj] diff --git a/pyload/lib/beaker/crypto/__init__.py b/pyload/lib/beaker/crypto/__init__.py new file mode 100644 index 000000000..3e26b0c13 --- /dev/null +++ b/pyload/lib/beaker/crypto/__init__.py @@ -0,0 +1,40 @@ +from warnings import warn + +from beaker.crypto.pbkdf2 import PBKDF2, strxor +from beaker.crypto.util import hmac, sha1, hmac_sha1, md5 +from beaker import util + +keyLength = None + +if util.jython: + try: + from beaker.crypto.jcecrypto import getKeyLength, aesEncrypt + keyLength = getKeyLength() + except ImportError: + pass +else: + try: + from beaker.crypto.pycrypto import getKeyLength, aesEncrypt, aesDecrypt + keyLength = getKeyLength() + except ImportError: + pass + +if not keyLength: + has_aes = False +else: + has_aes = True + +if has_aes and keyLength < 32: + warn('Crypto implementation only supports key lengths up to %d bits. ' + 'Generated session cookies may be incompatible with other ' + 'environments' % (keyLength * 8)) + + +def generateCryptoKeys(master_key, salt, iterations): + # NB: We XOR parts of the keystream into the randomly-generated parts, just + # in case os.urandom() isn't as random as it should be. Note that if + # os.urandom() returns truly random data, this will have no effect on the + # overall security. + keystream = PBKDF2(master_key, salt, iterations=iterations) + cipher_key = keystream.read(keyLength) + return cipher_key diff --git a/pyload/lib/beaker/crypto/jcecrypto.py b/pyload/lib/beaker/crypto/jcecrypto.py new file mode 100644 index 000000000..4062d513e --- /dev/null +++ b/pyload/lib/beaker/crypto/jcecrypto.py @@ -0,0 +1,30 @@ +""" +Encryption module that uses the Java Cryptography Extensions (JCE). + +Note that in default installations of the Java Runtime Environment, the +maximum key length is limited to 128 bits due to US export +restrictions. This makes the generated keys incompatible with the ones +generated by pycryptopp, which has no such restrictions. To fix this, +download the "Unlimited Strength Jurisdiction Policy Files" from Sun, +which will allow encryption using 256 bit AES keys. +""" +from javax.crypto import Cipher +from javax.crypto.spec import SecretKeySpec, IvParameterSpec + +import jarray + +# Initialization vector filled with zeros +_iv = IvParameterSpec(jarray.zeros(16, 'b')) + +def aesEncrypt(data, key): + cipher = Cipher.getInstance('AES/CTR/NoPadding') + skeySpec = SecretKeySpec(key, 'AES') + cipher.init(Cipher.ENCRYPT_MODE, skeySpec, _iv) + return cipher.doFinal(data).tostring() + +# magic. +aesDecrypt = aesEncrypt + +def getKeyLength(): + maxlen = Cipher.getMaxAllowedKeyLength('AES/CTR/NoPadding') + return min(maxlen, 256) / 8 diff --git a/pyload/lib/beaker/crypto/pbkdf2.py b/pyload/lib/beaker/crypto/pbkdf2.py new file mode 100644 index 000000000..96dc5fbb2 --- /dev/null +++ b/pyload/lib/beaker/crypto/pbkdf2.py @@ -0,0 +1,342 @@ +#!/usr/bin/python +# -*- coding: ascii -*- +########################################################################### +# PBKDF2.py - PKCS#5 v2.0 Password-Based Key Derivation +# +# Copyright (C) 2007 Dwayne C. Litzenberger +# All rights reserved. +# +# Permission to use, copy, modify, and distribute this software and its +# documentation for any purpose and without fee is hereby granted, +# provided that the above copyright notice appear in all copies and that +# both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR PROVIDES THIS SOFTWARE ``AS IS'' AND ANY EXPRESSED OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Country of origin: Canada +# +########################################################################### +# Sample PBKDF2 usage: +# from Crypto.Cipher import AES +# from PBKDF2 import PBKDF2 +# import os +# +# salt = os.urandom(8) # 64-bit salt +# key = PBKDF2("This passphrase is a secret.", salt).read(32) # 256-bit key +# iv = os.urandom(16) # 128-bit IV +# cipher = AES.new(key, AES.MODE_CBC, iv) +# ... +# +# Sample crypt() usage: +# from PBKDF2 import crypt +# pwhash = crypt("secret") +# alleged_pw = raw_input("Enter password: ") +# if pwhash == crypt(alleged_pw, pwhash): +# print "Password good" +# else: +# print "Invalid password" +# +########################################################################### +# History: +# +# 2007-07-27 Dwayne C. Litzenberger +# - Initial Release (v1.0) +# +# 2007-07-31 Dwayne C. Litzenberger +# - Bugfix release (v1.1) +# - SECURITY: The PyCrypto XOR cipher (used, if available, in the _strxor +# function in the previous release) silently truncates all keys to 64 +# bytes. The way it was used in the previous release, this would only be +# problem if the pseudorandom function that returned values larger than +# 64 bytes (so SHA1, SHA256 and SHA512 are fine), but I don't like +# anything that silently reduces the security margin from what is +# expected. +# +########################################################################### + +__version__ = "1.1" + +from struct import pack +from binascii import b2a_hex +from random import randint + +from base64 import b64encode + +from beaker.crypto.util import hmac as HMAC, hmac_sha1 as SHA1 + +def strxor(a, b): + return "".join([chr(ord(x) ^ ord(y)) for (x, y) in zip(a, b)]) + +class PBKDF2(object): + """PBKDF2.py : PKCS#5 v2.0 Password-Based Key Derivation + + This implementation takes a passphrase and a salt (and optionally an + iteration count, a digest module, and a MAC module) and provides a + file-like object from which an arbitrarily-sized key can be read. + + If the passphrase and/or salt are unicode objects, they are encoded as + UTF-8 before they are processed. + + The idea behind PBKDF2 is to derive a cryptographic key from a + passphrase and a salt. + + PBKDF2 may also be used as a strong salted password hash. The + 'crypt' function is provided for that purpose. + + Remember: Keys generated using PBKDF2 are only as strong as the + passphrases they are derived from. + """ + + def __init__(self, passphrase, salt, iterations=1000, + digestmodule=SHA1, macmodule=HMAC): + if not callable(macmodule): + macmodule = macmodule.new + self.__macmodule = macmodule + self.__digestmodule = digestmodule + self._setup(passphrase, salt, iterations, self._pseudorandom) + + def _pseudorandom(self, key, msg): + """Pseudorandom function. e.g. HMAC-SHA1""" + return self.__macmodule(key=key, msg=msg, + digestmod=self.__digestmodule).digest() + + def read(self, bytes): + """Read the specified number of key bytes.""" + if self.closed: + raise ValueError("file-like object is closed") + + size = len(self.__buf) + blocks = [self.__buf] + i = self.__blockNum + while size < bytes: + i += 1 + if i > 0xffffffff: + # We could return "" here, but + raise OverflowError("derived key too long") + block = self.__f(i) + blocks.append(block) + size += len(block) + buf = "".join(blocks) + retval = buf[:bytes] + self.__buf = buf[bytes:] + self.__blockNum = i + return retval + + def __f(self, i): + # i must fit within 32 bits + assert (1 <= i <= 0xffffffff) + U = self.__prf(self.__passphrase, self.__salt + pack("!L", i)) + result = U + for j in xrange(2, 1+self.__iterations): + U = self.__prf(self.__passphrase, U) + result = strxor(result, U) + return result + + def hexread(self, octets): + """Read the specified number of octets. Return them as hexadecimal. + + Note that len(obj.hexread(n)) == 2*n. + """ + return b2a_hex(self.read(octets)) + + def _setup(self, passphrase, salt, iterations, prf): + # Sanity checks: + + # passphrase and salt must be str or unicode (in the latter + # case, we convert to UTF-8) + if isinstance(passphrase, unicode): + passphrase = passphrase.encode("UTF-8") + if not isinstance(passphrase, str): + raise TypeError("passphrase must be str or unicode") + if isinstance(salt, unicode): + salt = salt.encode("UTF-8") + if not isinstance(salt, str): + raise TypeError("salt must be str or unicode") + + # iterations must be an integer >= 1 + if not isinstance(iterations, (int, long)): + raise TypeError("iterations must be an integer") + if iterations < 1: + raise ValueError("iterations must be at least 1") + + # prf must be callable + if not callable(prf): + raise TypeError("prf must be callable") + + self.__passphrase = passphrase + self.__salt = salt + self.__iterations = iterations + self.__prf = prf + self.__blockNum = 0 + self.__buf = "" + self.closed = False + + def close(self): + """Close the stream.""" + if not self.closed: + del self.__passphrase + del self.__salt + del self.__iterations + del self.__prf + del self.__blockNum + del self.__buf + self.closed = True + +def crypt(word, salt=None, iterations=None): + """PBKDF2-based unix crypt(3) replacement. + + The number of iterations specified in the salt overrides the 'iterations' + parameter. + + The effective hash length is 192 bits. + """ + + # Generate a (pseudo-)random salt if the user hasn't provided one. + if salt is None: + salt = _makesalt() + + # salt must be a string or the us-ascii subset of unicode + if isinstance(salt, unicode): + salt = salt.encode("us-ascii") + if not isinstance(salt, str): + raise TypeError("salt must be a string") + + # word must be a string or unicode (in the latter case, we convert to UTF-8) + if isinstance(word, unicode): + word = word.encode("UTF-8") + if not isinstance(word, str): + raise TypeError("word must be a string or unicode") + + # Try to extract the real salt and iteration count from the salt + if salt.startswith("$p5k2$"): + (iterations, salt, dummy) = salt.split("$")[2:5] + if iterations == "": + iterations = 400 + else: + converted = int(iterations, 16) + if iterations != "%x" % converted: # lowercase hex, minimum digits + raise ValueError("Invalid salt") + iterations = converted + if not (iterations >= 1): + raise ValueError("Invalid salt") + + # Make sure the salt matches the allowed character set + allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789./" + for ch in salt: + if ch not in allowed: + raise ValueError("Illegal character %r in salt" % (ch,)) + + if iterations is None or iterations == 400: + iterations = 400 + salt = "$p5k2$$" + salt + else: + salt = "$p5k2$%x$%s" % (iterations, salt) + rawhash = PBKDF2(word, salt, iterations).read(24) + return salt + "$" + b64encode(rawhash, "./") + +# Add crypt as a static method of the PBKDF2 class +# This makes it easier to do "from PBKDF2 import PBKDF2" and still use +# crypt. +PBKDF2.crypt = staticmethod(crypt) + +def _makesalt(): + """Return a 48-bit pseudorandom salt for crypt(). + + This function is not suitable for generating cryptographic secrets. + """ + binarysalt = "".join([pack("@H", randint(0, 0xffff)) for i in range(3)]) + return b64encode(binarysalt, "./") + +def test_pbkdf2(): + """Module self-test""" + from binascii import a2b_hex + + # + # Test vectors from RFC 3962 + # + + # Test 1 + result = PBKDF2("password", "ATHENA.MIT.EDUraeburn", 1).read(16) + expected = a2b_hex("cdedb5281bb2f801565a1122b2563515") + if result != expected: + raise RuntimeError("self-test failed") + + # Test 2 + result = PBKDF2("password", "ATHENA.MIT.EDUraeburn", 1200).hexread(32) + expected = ("5c08eb61fdf71e4e4ec3cf6ba1f5512b" + "a7e52ddbc5e5142f708a31e2e62b1e13") + if result != expected: + raise RuntimeError("self-test failed") + + # Test 3 + result = PBKDF2("X"*64, "pass phrase equals block size", 1200).hexread(32) + expected = ("139c30c0966bc32ba55fdbf212530ac9" + "c5ec59f1a452f5cc9ad940fea0598ed1") + if result != expected: + raise RuntimeError("self-test failed") + + # Test 4 + result = PBKDF2("X"*65, "pass phrase exceeds block size", 1200).hexread(32) + expected = ("9ccad6d468770cd51b10e6a68721be61" + "1a8b4d282601db3b36be9246915ec82a") + if result != expected: + raise RuntimeError("self-test failed") + + # + # Other test vectors + # + + # Chunked read + f = PBKDF2("kickstart", "workbench", 256) + result = f.read(17) + result += f.read(17) + result += f.read(1) + result += f.read(2) + result += f.read(3) + expected = PBKDF2("kickstart", "workbench", 256).read(40) + if result != expected: + raise RuntimeError("self-test failed") + + # + # crypt() test vectors + # + + # crypt 1 + result = crypt("cloadm", "exec") + expected = '$p5k2$$exec$r1EWMCMk7Rlv3L/RNcFXviDefYa0hlql' + if result != expected: + raise RuntimeError("self-test failed") + + # crypt 2 + result = crypt("gnu", '$p5k2$c$u9HvcT4d$.....') + expected = '$p5k2$c$u9HvcT4d$Sd1gwSVCLZYAuqZ25piRnbBEoAesaa/g' + if result != expected: + raise RuntimeError("self-test failed") + + # crypt 3 + result = crypt("dcl", "tUsch7fU", iterations=13) + expected = "$p5k2$d$tUsch7fU$nqDkaxMDOFBeJsTSfABsyn.PYUXilHwL" + if result != expected: + raise RuntimeError("self-test failed") + + # crypt 4 (unicode) + result = crypt(u'\u0399\u03c9\u03b1\u03bd\u03bd\u03b7\u03c2', + '$p5k2$$KosHgqNo$9mjN8gqjt02hDoP0c2J0ABtLIwtot8cQ') + expected = '$p5k2$$KosHgqNo$9mjN8gqjt02hDoP0c2J0ABtLIwtot8cQ' + if result != expected: + raise RuntimeError("self-test failed") + +if __name__ == '__main__': + test_pbkdf2() + +# vim:set ts=4 sw=4 sts=4 expandtab: diff --git a/pyload/lib/beaker/crypto/pycrypto.py b/pyload/lib/beaker/crypto/pycrypto.py new file mode 100644 index 000000000..a3eb4d9db --- /dev/null +++ b/pyload/lib/beaker/crypto/pycrypto.py @@ -0,0 +1,31 @@ +"""Encryption module that uses pycryptopp or pycrypto""" +try: + # Pycryptopp is preferred over Crypto because Crypto has had + # various periods of not being maintained, and pycryptopp uses + # the Crypto++ library which is generally considered the 'gold standard' + # of crypto implementations + from pycryptopp.cipher import aes + + def aesEncrypt(data, key): + cipher = aes.AES(key) + return cipher.process(data) + + # magic. + aesDecrypt = aesEncrypt + +except ImportError: + from Crypto.Cipher import AES + + def aesEncrypt(data, key): + cipher = AES.new(key) + + data = data + (" " * (16 - (len(data) % 16))) + return cipher.encrypt(data) + + def aesDecrypt(data, key): + cipher = AES.new(key) + + return cipher.decrypt(data).rstrip() + +def getKeyLength(): + return 32 diff --git a/pyload/lib/beaker/crypto/util.py b/pyload/lib/beaker/crypto/util.py new file mode 100644 index 000000000..d97e8ce6f --- /dev/null +++ b/pyload/lib/beaker/crypto/util.py @@ -0,0 +1,30 @@ +from warnings import warn +from beaker import util + + +try: + # Use PyCrypto (if available) + from Crypto.Hash import HMAC as hmac, SHA as hmac_sha1 + sha1 = hmac_sha1.new + +except ImportError: + + # PyCrypto not available. Use the Python standard library. + import hmac + + # When using the stdlib, we have to make sure the hmac version and sha + # version are compatible + if util.py24: + from sha import sha as sha1 + import sha as hmac_sha1 + else: + # NOTE: We have to use the callable with hashlib (hashlib.sha1), + # otherwise hmac only accepts the sha module object itself + from hashlib import sha1 + hmac_sha1 = sha1 + + +if util.py24: + from md5 import md5 +else: + from hashlib import md5 diff --git a/pyload/lib/beaker/exceptions.py b/pyload/lib/beaker/exceptions.py new file mode 100644 index 000000000..cc0eed286 --- /dev/null +++ b/pyload/lib/beaker/exceptions.py @@ -0,0 +1,24 @@ +"""Beaker exception classes""" + +class BeakerException(Exception): + pass + + +class CreationAbortedError(Exception): + """Deprecated.""" + + +class InvalidCacheBackendError(BeakerException, ImportError): + pass + + +class MissingCacheParameter(BeakerException): + pass + + +class LockError(BeakerException): + pass + + +class InvalidCryptoBackendError(BeakerException): + pass diff --git a/pyload/lib/beaker/ext/__init__.py b/pyload/lib/beaker/ext/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyload/lib/beaker/ext/database.py b/pyload/lib/beaker/ext/database.py new file mode 100644 index 000000000..701e6f7d2 --- /dev/null +++ b/pyload/lib/beaker/ext/database.py @@ -0,0 +1,165 @@ +import cPickle +import logging +import pickle +from datetime import datetime + +from beaker.container import OpenResourceNamespaceManager, Container +from beaker.exceptions import InvalidCacheBackendError, MissingCacheParameter +from beaker.synchronization import file_synchronizer, null_synchronizer +from beaker.util import verify_directory, SyncDict + +log = logging.getLogger(__name__) + +sa = None +pool = None +types = None + +class DatabaseNamespaceManager(OpenResourceNamespaceManager): + metadatas = SyncDict() + tables = SyncDict() + + @classmethod + def _init_dependencies(cls): + global sa, pool, types + if sa is not None: + return + try: + import sqlalchemy as sa + import sqlalchemy.pool as pool + from sqlalchemy import types + except ImportError: + raise InvalidCacheBackendError("Database cache backend requires " + "the 'sqlalchemy' library") + + def __init__(self, namespace, url=None, sa_opts=None, optimistic=False, + table_name='beaker_cache', data_dir=None, lock_dir=None, + **params): + """Creates a database namespace manager + + ``url`` + SQLAlchemy compliant db url + ``sa_opts`` + A dictionary of SQLAlchemy keyword options to initialize the engine + with. + ``optimistic`` + Use optimistic session locking, note that this will result in an + additional select when updating a cache value to compare version + numbers. + ``table_name`` + The table name to use in the database for the cache. + """ + OpenResourceNamespaceManager.__init__(self, namespace) + + if sa_opts is None: + sa_opts = params + + if lock_dir: + self.lock_dir = lock_dir + elif data_dir: + self.lock_dir = data_dir + "/container_db_lock" + if self.lock_dir: + verify_directory(self.lock_dir) + + # Check to see if the table's been created before + url = url or sa_opts['sa.url'] + table_key = url + table_name + def make_cache(): + # Check to see if we have a connection pool open already + meta_key = url + table_name + def make_meta(): + # SQLAlchemy pops the url, this ensures it sticks around + # later + sa_opts['sa.url'] = url + engine = sa.engine_from_config(sa_opts, 'sa.') + meta = sa.MetaData() + meta.bind = engine + return meta + meta = DatabaseNamespaceManager.metadatas.get(meta_key, make_meta) + # Create the table object and cache it now + cache = sa.Table(table_name, meta, + sa.Column('id', types.Integer, primary_key=True), + sa.Column('namespace', types.String(255), nullable=False), + sa.Column('accessed', types.DateTime, nullable=False), + sa.Column('created', types.DateTime, nullable=False), + sa.Column('data', types.PickleType, nullable=False), + sa.UniqueConstraint('namespace') + ) + cache.create(checkfirst=True) + return cache + self.hash = {} + self._is_new = False + self.loaded = False + self.cache = DatabaseNamespaceManager.tables.get(table_key, make_cache) + + def get_access_lock(self): + return null_synchronizer() + + def get_creation_lock(self, key): + return file_synchronizer( + identifier ="databasecontainer/funclock/%s" % self.namespace, + lock_dir = self.lock_dir) + + def do_open(self, flags): + # If we already loaded the data, don't bother loading it again + if self.loaded: + self.flags = flags + return + + cache = self.cache + result = sa.select([cache.c.data], + cache.c.namespace==self.namespace + ).execute().fetchone() + if not result: + self._is_new = True + self.hash = {} + else: + self._is_new = False + try: + self.hash = result['data'] + except (IOError, OSError, EOFError, cPickle.PickleError, + pickle.PickleError): + log.debug("Couln't load pickle data, creating new storage") + self.hash = {} + self._is_new = True + self.flags = flags + self.loaded = True + + def do_close(self): + if self.flags is not None and (self.flags == 'c' or self.flags == 'w'): + cache = self.cache + if self._is_new: + cache.insert().execute(namespace=self.namespace, data=self.hash, + accessed=datetime.now(), + created=datetime.now()) + self._is_new = False + else: + cache.update(cache.c.namespace==self.namespace).execute( + data=self.hash, accessed=datetime.now()) + self.flags = None + + def do_remove(self): + cache = self.cache + cache.delete(cache.c.namespace==self.namespace).execute() + self.hash = {} + + # We can retain the fact that we did a load attempt, but since the + # file is gone this will be a new namespace should it be saved. + self._is_new = True + + def __getitem__(self, key): + return self.hash[key] + + def __contains__(self, key): + return self.hash.has_key(key) + + def __setitem__(self, key, value): + self.hash[key] = value + + def __delitem__(self, key): + del self.hash[key] + + def keys(self): + return self.hash.keys() + +class DatabaseContainer(Container): + namespace_manager = DatabaseNamespaceManager diff --git a/pyload/lib/beaker/ext/google.py b/pyload/lib/beaker/ext/google.py new file mode 100644 index 000000000..dd8380d7f --- /dev/null +++ b/pyload/lib/beaker/ext/google.py @@ -0,0 +1,120 @@ +import cPickle +import logging +from datetime import datetime + +from beaker.container import OpenResourceNamespaceManager, Container +from beaker.exceptions import InvalidCacheBackendError +from beaker.synchronization import null_synchronizer + +log = logging.getLogger(__name__) + +db = None + +class GoogleNamespaceManager(OpenResourceNamespaceManager): + tables = {} + + @classmethod + def _init_dependencies(cls): + global db + if db is not None: + return + try: + db = __import__('google.appengine.ext.db').appengine.ext.db + except ImportError: + raise InvalidCacheBackendError("Datastore cache backend requires the " + "'google.appengine.ext' library") + + def __init__(self, namespace, table_name='beaker_cache', **params): + """Creates a datastore namespace manager""" + OpenResourceNamespaceManager.__init__(self, namespace) + + def make_cache(): + table_dict = dict(created=db.DateTimeProperty(), + accessed=db.DateTimeProperty(), + data=db.BlobProperty()) + table = type(table_name, (db.Model,), table_dict) + return table + self.table_name = table_name + self.cache = GoogleNamespaceManager.tables.setdefault(table_name, make_cache()) + self.hash = {} + self._is_new = False + self.loaded = False + self.log_debug = logging.DEBUG >= log.getEffectiveLevel() + + # Google wants namespaces to start with letters, change the namespace + # to start with a letter + self.namespace = 'p%s' % self.namespace + + def get_access_lock(self): + return null_synchronizer() + + def get_creation_lock(self, key): + # this is weird, should probably be present + return null_synchronizer() + + def do_open(self, flags): + # If we already loaded the data, don't bother loading it again + if self.loaded: + self.flags = flags + return + + item = self.cache.get_by_key_name(self.namespace) + + if not item: + self._is_new = True + self.hash = {} + else: + self._is_new = False + try: + self.hash = cPickle.loads(str(item.data)) + except (IOError, OSError, EOFError, cPickle.PickleError): + if self.log_debug: + log.debug("Couln't load pickle data, creating new storage") + self.hash = {} + self._is_new = True + self.flags = flags + self.loaded = True + + def do_close(self): + if self.flags is not None and (self.flags == 'c' or self.flags == 'w'): + if self._is_new: + item = self.cache(key_name=self.namespace) + item.data = cPickle.dumps(self.hash) + item.created = datetime.now() + item.accessed = datetime.now() + item.put() + self._is_new = False + else: + item = self.cache.get_by_key_name(self.namespace) + item.data = cPickle.dumps(self.hash) + item.accessed = datetime.now() + item.put() + self.flags = None + + def do_remove(self): + item = self.cache.get_by_key_name(self.namespace) + item.delete() + self.hash = {} + + # We can retain the fact that we did a load attempt, but since the + # file is gone this will be a new namespace should it be saved. + self._is_new = True + + def __getitem__(self, key): + return self.hash[key] + + def __contains__(self, key): + return self.hash.has_key(key) + + def __setitem__(self, key, value): + self.hash[key] = value + + def __delitem__(self, key): + del self.hash[key] + + def keys(self): + return self.hash.keys() + + +class GoogleContainer(Container): + namespace_class = GoogleNamespaceManager diff --git a/pyload/lib/beaker/ext/memcached.py b/pyload/lib/beaker/ext/memcached.py new file mode 100644 index 000000000..96516953f --- /dev/null +++ b/pyload/lib/beaker/ext/memcached.py @@ -0,0 +1,82 @@ +from beaker.container import NamespaceManager, Container +from beaker.exceptions import InvalidCacheBackendError, MissingCacheParameter +from beaker.synchronization import file_synchronizer, null_synchronizer +from beaker.util import verify_directory, SyncDict +import warnings + +memcache = None + +class MemcachedNamespaceManager(NamespaceManager): + clients = SyncDict() + + @classmethod + def _init_dependencies(cls): + global memcache + if memcache is not None: + return + try: + import pylibmc as memcache + except ImportError: + try: + import cmemcache as memcache + warnings.warn("cmemcache is known to have serious " + "concurrency issues; consider using 'memcache' or 'pylibmc'") + except ImportError: + try: + import memcache + except ImportError: + raise InvalidCacheBackendError("Memcached cache backend requires either " + "the 'memcache' or 'cmemcache' library") + + def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params): + NamespaceManager.__init__(self, namespace) + + if not url: + raise MissingCacheParameter("url is required") + + if lock_dir: + self.lock_dir = lock_dir + elif data_dir: + self.lock_dir = data_dir + "/container_mcd_lock" + if self.lock_dir: + verify_directory(self.lock_dir) + + self.mc = MemcachedNamespaceManager.clients.get(url, memcache.Client, url.split(';')) + + def get_creation_lock(self, key): + return file_synchronizer( + identifier="memcachedcontainer/funclock/%s" % self.namespace,lock_dir = self.lock_dir) + + def _format_key(self, key): + return self.namespace + '_' + key.replace(' ', '\302\267') + + def __getitem__(self, key): + return self.mc.get(self._format_key(key)) + + def __contains__(self, key): + value = self.mc.get(self._format_key(key)) + return value is not None + + def has_key(self, key): + return key in self + + def set_value(self, key, value, expiretime=None): + if expiretime: + self.mc.set(self._format_key(key), value, time=expiretime) + else: + self.mc.set(self._format_key(key), value) + + def __setitem__(self, key, value): + self.set_value(key, value) + + def __delitem__(self, key): + self.mc.delete(self._format_key(key)) + + def do_remove(self): + self.mc.flush_all() + + def keys(self): + raise NotImplementedError("Memcache caching does not support iteration of all cache keys") + +class MemcachedContainer(Container): + namespace_class = MemcachedNamespaceManager diff --git a/pyload/lib/beaker/ext/sqla.py b/pyload/lib/beaker/ext/sqla.py new file mode 100644 index 000000000..8c79633c1 --- /dev/null +++ b/pyload/lib/beaker/ext/sqla.py @@ -0,0 +1,133 @@ +import cPickle +import logging +import pickle +from datetime import datetime + +from beaker.container import OpenResourceNamespaceManager, Container +from beaker.exceptions import InvalidCacheBackendError, MissingCacheParameter +from beaker.synchronization import file_synchronizer, null_synchronizer +from beaker.util import verify_directory, SyncDict + + +log = logging.getLogger(__name__) + +sa = None + +class SqlaNamespaceManager(OpenResourceNamespaceManager): + binds = SyncDict() + tables = SyncDict() + + @classmethod + def _init_dependencies(cls): + global sa + if sa is not None: + return + try: + import sqlalchemy as sa + except ImportError: + raise InvalidCacheBackendError("SQLAlchemy, which is required by " + "this backend, is not installed") + + def __init__(self, namespace, bind, table, data_dir=None, lock_dir=None, + **kwargs): + """Create a namespace manager for use with a database table via + SQLAlchemy. + + ``bind`` + SQLAlchemy ``Engine`` or ``Connection`` object + + ``table`` + SQLAlchemy ``Table`` object in which to store namespace data. + This should usually be something created by ``make_cache_table``. + """ + OpenResourceNamespaceManager.__init__(self, namespace) + + if lock_dir: + self.lock_dir = lock_dir + elif data_dir: + self.lock_dir = data_dir + "/container_db_lock" + if self.lock_dir: + verify_directory(self.lock_dir) + + self.bind = self.__class__.binds.get(str(bind.url), lambda: bind) + self.table = self.__class__.tables.get('%s:%s' % (bind.url, table.name), + lambda: table) + self.hash = {} + self._is_new = False + self.loaded = False + + def get_access_lock(self): + return null_synchronizer() + + def get_creation_lock(self, key): + return file_synchronizer( + identifier ="databasecontainer/funclock/%s" % self.namespace, + lock_dir=self.lock_dir) + + def do_open(self, flags): + if self.loaded: + self.flags = flags + return + select = sa.select([self.table.c.data], + (self.table.c.namespace == self.namespace)) + result = self.bind.execute(select).fetchone() + if not result: + self._is_new = True + self.hash = {} + else: + self._is_new = False + try: + self.hash = result['data'] + except (IOError, OSError, EOFError, cPickle.PickleError, + pickle.PickleError): + log.debug("Couln't load pickle data, creating new storage") + self.hash = {} + self._is_new = True + self.flags = flags + self.loaded = True + + def do_close(self): + if self.flags is not None and (self.flags == 'c' or self.flags == 'w'): + if self._is_new: + insert = self.table.insert() + self.bind.execute(insert, namespace=self.namespace, data=self.hash, + accessed=datetime.now(), created=datetime.now()) + self._is_new = False + else: + update = self.table.update(self.table.c.namespace == self.namespace) + self.bind.execute(update, data=self.hash, accessed=datetime.now()) + self.flags = None + + def do_remove(self): + delete = self.table.delete(self.table.c.namespace == self.namespace) + self.bind.execute(delete) + self.hash = {} + self._is_new = True + + def __getitem__(self, key): + return self.hash[key] + + def __contains__(self, key): + return self.hash.has_key(key) + + def __setitem__(self, key, value): + self.hash[key] = value + + def __delitem__(self, key): + del self.hash[key] + + def keys(self): + return self.hash.keys() + + +class SqlaContainer(Container): + namespace_manager = SqlaNamespaceManager + +def make_cache_table(metadata, table_name='beaker_cache'): + """Return a ``Table`` object suitable for storing cached values for the + namespace manager. Do not create the table.""" + return sa.Table(table_name, metadata, + sa.Column('namespace', sa.String(255), primary_key=True), + sa.Column('accessed', sa.DateTime, nullable=False), + sa.Column('created', sa.DateTime, nullable=False), + sa.Column('data', sa.PickleType, nullable=False)) diff --git a/pyload/lib/beaker/middleware.py b/pyload/lib/beaker/middleware.py new file mode 100644 index 000000000..7ba88b37d --- /dev/null +++ b/pyload/lib/beaker/middleware.py @@ -0,0 +1,165 @@ +import warnings + +try: + from paste.registry import StackedObjectProxy + beaker_session = StackedObjectProxy(name="Beaker Session") + beaker_cache = StackedObjectProxy(name="Cache Manager") +except: + beaker_cache = None + beaker_session = None + +from beaker.cache import CacheManager +from beaker.session import Session, SessionObject +from beaker.util import coerce_cache_params, coerce_session_params, \ + parse_cache_config_options + + +class CacheMiddleware(object): + cache = beaker_cache + + def __init__(self, app, config=None, environ_key='beaker.cache', **kwargs): + """Initialize the Cache Middleware + + The Cache middleware will make a Cache instance available + every request under the ``environ['beaker.cache']`` key by + default. The location in environ can be changed by setting + ``environ_key``. + + ``config`` + dict All settings should be prefixed by 'cache.'. This + method of passing variables is intended for Paste and other + setups that accumulate multiple component settings in a + single dictionary. If config contains *no cache. prefixed + args*, then *all* of the config options will be used to + intialize the Cache objects. + + ``environ_key`` + Location where the Cache instance will keyed in the WSGI + environ + + ``**kwargs`` + All keyword arguments are assumed to be cache settings and + will override any settings found in ``config`` + + """ + self.app = app + config = config or {} + + self.options = {} + + # Update the options with the parsed config + self.options.update(parse_cache_config_options(config)) + + # Add any options from kwargs, but leave out the defaults this + # time + self.options.update( + parse_cache_config_options(kwargs, include_defaults=False)) + + # Assume all keys are intended for cache if none are prefixed with + # 'cache.' + if not self.options and config: + self.options = config + + self.options.update(kwargs) + self.cache_manager = CacheManager(**self.options) + self.environ_key = environ_key + + def __call__(self, environ, start_response): + if environ.get('paste.registry'): + if environ['paste.registry'].reglist: + environ['paste.registry'].register(self.cache, + self.cache_manager) + environ[self.environ_key] = self.cache_manager + return self.app(environ, start_response) + + +class SessionMiddleware(object): + session = beaker_session + + def __init__(self, wrap_app, config=None, environ_key='beaker.session', + **kwargs): + """Initialize the Session Middleware + + The Session middleware will make a lazy session instance + available every request under the ``environ['beaker.session']`` + key by default. The location in environ can be changed by + setting ``environ_key``. + + ``config`` + dict All settings should be prefixed by 'session.'. This + method of passing variables is intended for Paste and other + setups that accumulate multiple component settings in a + single dictionary. If config contains *no cache. prefixed + args*, then *all* of the config options will be used to + intialize the Cache objects. + + ``environ_key`` + Location where the Session instance will keyed in the WSGI + environ + + ``**kwargs`` + All keyword arguments are assumed to be session settings and + will override any settings found in ``config`` + + """ + config = config or {} + + # Load up the default params + self.options = dict(invalidate_corrupt=True, type=None, + data_dir=None, key='beaker.session.id', + timeout=None, secret=None, log_file=None) + + # Pull out any config args meant for beaker session. if there are any + for dct in [config, kwargs]: + for key, val in dct.iteritems(): + if key.startswith('beaker.session.'): + self.options[key[15:]] = val + if key.startswith('session.'): + self.options[key[8:]] = val + if key.startswith('session_'): + warnings.warn('Session options should start with session. ' + 'instead of session_.', DeprecationWarning, 2) + self.options[key[8:]] = val + + # Coerce and validate session params + coerce_session_params(self.options) + + # Assume all keys are intended for cache if none are prefixed with + # 'cache.' + if not self.options and config: + self.options = config + + self.options.update(kwargs) + self.wrap_app = wrap_app + self.environ_key = environ_key + + def __call__(self, environ, start_response): + session = SessionObject(environ, **self.options) + if environ.get('paste.registry'): + if environ['paste.registry'].reglist: + environ['paste.registry'].register(self.session, session) + environ[self.environ_key] = session + environ['beaker.get_session'] = self._get_session + + def session_start_response(status, headers, exc_info = None): + if session.accessed(): + session.persist() + if session.__dict__['_headers']['set_cookie']: + cookie = session.__dict__['_headers']['cookie_out'] + if cookie: + headers.append(('Set-cookie', cookie)) + return start_response(status, headers, exc_info) + return self.wrap_app(environ, session_start_response) + + def _get_session(self): + return Session({}, use_cookies=False, **self.options) + + +def session_filter_factory(global_conf, **kwargs): + def filter(app): + return SessionMiddleware(app, global_conf, **kwargs) + return filter + + +def session_filter_app_factory(app, global_conf, **kwargs): + return SessionMiddleware(app, global_conf, **kwargs) diff --git a/pyload/lib/beaker/session.py b/pyload/lib/beaker/session.py new file mode 100644 index 000000000..7d465530b --- /dev/null +++ b/pyload/lib/beaker/session.py @@ -0,0 +1,618 @@ +import Cookie +import os +import random +import time +from datetime import datetime, timedelta + +from beaker.crypto import hmac as HMAC, hmac_sha1 as SHA1, md5 +from beaker.util import pickle + +from beaker import crypto +from beaker.cache import clsmap +from beaker.exceptions import BeakerException, InvalidCryptoBackendError +from base64 import b64encode, b64decode + + +__all__ = ['SignedCookie', 'Session'] + +getpid = hasattr(os, 'getpid') and os.getpid or (lambda : '') + +class SignedCookie(Cookie.BaseCookie): + """Extends python cookie to give digital signature support""" + def __init__(self, secret, input=None): + self.secret = secret + Cookie.BaseCookie.__init__(self, input) + + def value_decode(self, val): + val = val.strip('"') + sig = HMAC.new(self.secret, val[40:], SHA1).hexdigest() + + # Avoid timing attacks + invalid_bits = 0 + input_sig = val[:40] + if len(sig) != len(input_sig): + return None, val + + for a, b in zip(sig, input_sig): + invalid_bits += a != b + + if invalid_bits: + return None, val + else: + return val[40:], val + + def value_encode(self, val): + sig = HMAC.new(self.secret, val, SHA1).hexdigest() + return str(val), ("%s%s" % (sig, val)) + + +class Session(dict): + """Session object that uses container package for storage. + + ``key`` + The name the cookie should be set to. + ``timeout`` + How long session data is considered valid. This is used + regardless of the cookie being present or not to determine + whether session data is still valid. + ``cookie_domain`` + Domain to use for the cookie. + ``secure`` + Whether or not the cookie should only be sent over SSL. + """ + def __init__(self, request, id=None, invalidate_corrupt=False, + use_cookies=True, type=None, data_dir=None, + key='beaker.session.id', timeout=None, cookie_expires=True, + cookie_domain=None, secret=None, secure=False, + namespace_class=None, **namespace_args): + if not type: + if data_dir: + self.type = 'file' + else: + self.type = 'memory' + else: + self.type = type + + self.namespace_class = namespace_class or clsmap[self.type] + + self.namespace_args = namespace_args + + self.request = request + self.data_dir = data_dir + self.key = key + + self.timeout = timeout + self.use_cookies = use_cookies + self.cookie_expires = cookie_expires + + # Default cookie domain/path + self._domain = cookie_domain + self._path = '/' + self.was_invalidated = False + self.secret = secret + self.secure = secure + self.id = id + self.accessed_dict = {} + + if self.use_cookies: + cookieheader = request.get('cookie', '') + if secret: + try: + self.cookie = SignedCookie(secret, input=cookieheader) + except Cookie.CookieError: + self.cookie = SignedCookie(secret, input=None) + else: + self.cookie = Cookie.SimpleCookie(input=cookieheader) + + if not self.id and self.key in self.cookie: + self.id = self.cookie[self.key].value + + self.is_new = self.id is None + if self.is_new: + self._create_id() + self['_accessed_time'] = self['_creation_time'] = time.time() + else: + try: + self.load() + except: + if invalidate_corrupt: + self.invalidate() + else: + raise + + def _create_id(self): + self.id = md5( + md5("%f%s%f%s" % (time.time(), id({}), random.random(), + getpid())).hexdigest(), + ).hexdigest() + self.is_new = True + self.last_accessed = None + if self.use_cookies: + self.cookie[self.key] = self.id + if self._domain: + self.cookie[self.key]['domain'] = self._domain + if self.secure: + self.cookie[self.key]['secure'] = True + self.cookie[self.key]['path'] = self._path + if self.cookie_expires is not True: + if self.cookie_expires is False: + expires = datetime.fromtimestamp( 0x7FFFFFFF ) + elif isinstance(self.cookie_expires, timedelta): + expires = datetime.today() + self.cookie_expires + elif isinstance(self.cookie_expires, datetime): + expires = self.cookie_expires + else: + raise ValueError("Invalid argument for cookie_expires: %s" + % repr(self.cookie_expires)) + self.cookie[self.key]['expires'] = \ + expires.strftime("%a, %d-%b-%Y %H:%M:%S GMT" ) + self.request['cookie_out'] = self.cookie[self.key].output(header='') + self.request['set_cookie'] = False + + def created(self): + return self['_creation_time'] + created = property(created) + + def _set_domain(self, domain): + self['_domain'] = domain + self.cookie[self.key]['domain'] = domain + self.request['cookie_out'] = self.cookie[self.key].output(header='') + self.request['set_cookie'] = True + + def _get_domain(self): + return self._domain + + domain = property(_get_domain, _set_domain) + + def _set_path(self, path): + self['_path'] = path + self.cookie[self.key]['path'] = path + self.request['cookie_out'] = self.cookie[self.key].output(header='') + self.request['set_cookie'] = True + + def _get_path(self): + return self._path + + path = property(_get_path, _set_path) + + def _delete_cookie(self): + self.request['set_cookie'] = True + self.cookie[self.key] = self.id + if self._domain: + self.cookie[self.key]['domain'] = self._domain + if self.secure: + self.cookie[self.key]['secure'] = True + self.cookie[self.key]['path'] = '/' + expires = datetime.today().replace(year=2003) + self.cookie[self.key]['expires'] = \ + expires.strftime("%a, %d-%b-%Y %H:%M:%S GMT" ) + self.request['cookie_out'] = self.cookie[self.key].output(header='') + self.request['set_cookie'] = True + + def delete(self): + """Deletes the session from the persistent storage, and sends + an expired cookie out""" + if self.use_cookies: + self._delete_cookie() + self.clear() + + def invalidate(self): + """Invalidates this session, creates a new session id, returns + to the is_new state""" + self.clear() + self.was_invalidated = True + self._create_id() + self.load() + + def load(self): + "Loads the data from this session from persistent storage" + self.namespace = self.namespace_class(self.id, + data_dir=self.data_dir, digest_filenames=False, + **self.namespace_args) + now = time.time() + self.request['set_cookie'] = True + + self.namespace.acquire_read_lock() + timed_out = False + try: + self.clear() + try: + session_data = self.namespace['session'] + + # Memcached always returns a key, its None when its not + # present + if session_data is None: + session_data = { + '_creation_time':now, + '_accessed_time':now + } + self.is_new = True + except (KeyError, TypeError): + session_data = { + '_creation_time':now, + '_accessed_time':now + } + self.is_new = True + + if self.timeout is not None and \ + now - session_data['_accessed_time'] > self.timeout: + timed_out= True + else: + # Properly set the last_accessed time, which is different + # than the *currently* _accessed_time + if self.is_new or '_accessed_time' not in session_data: + self.last_accessed = None + else: + self.last_accessed = session_data['_accessed_time'] + + # Update the current _accessed_time + session_data['_accessed_time'] = now + self.update(session_data) + self.accessed_dict = session_data.copy() + finally: + self.namespace.release_read_lock() + if timed_out: + self.invalidate() + + def save(self, accessed_only=False): + """Saves the data for this session to persistent storage + + If accessed_only is True, then only the original data loaded + at the beginning of the request will be saved, with the updated + last accessed time. + + """ + # Look to see if its a new session that was only accessed + # Don't save it under that case + if accessed_only and self.is_new: + return None + + if not hasattr(self, 'namespace'): + self.namespace = self.namespace_class( + self.id, + data_dir=self.data_dir, + digest_filenames=False, + **self.namespace_args) + + self.namespace.acquire_write_lock() + try: + if accessed_only: + data = dict(self.accessed_dict.items()) + else: + data = dict(self.items()) + + # Save the data + if not data and 'session' in self.namespace: + del self.namespace['session'] + else: + self.namespace['session'] = data + finally: + self.namespace.release_write_lock() + if self.is_new: + self.request['set_cookie'] = True + + def revert(self): + """Revert the session to its original state from its first + access in the request""" + self.clear() + self.update(self.accessed_dict) + + # TODO: I think both these methods should be removed. They're from + # the original mod_python code i was ripping off but they really + # have no use here. + def lock(self): + """Locks this session against other processes/threads. This is + automatic when load/save is called. + + ***use with caution*** and always with a corresponding 'unlock' + inside a "finally:" block, as a stray lock typically cannot be + unlocked without shutting down the whole application. + + """ + self.namespace.acquire_write_lock() + + def unlock(self): + """Unlocks this session against other processes/threads. This + is automatic when load/save is called. + + ***use with caution*** and always within a "finally:" block, as + a stray lock typically cannot be unlocked without shutting down + the whole application. + + """ + self.namespace.release_write_lock() + +class CookieSession(Session): + """Pure cookie-based session + + Options recognized when using cookie-based sessions are slightly + more restricted than general sessions. + + ``key`` + The name the cookie should be set to. + ``timeout`` + How long session data is considered valid. This is used + regardless of the cookie being present or not to determine + whether session data is still valid. + ``encrypt_key`` + The key to use for the session encryption, if not provided the + session will not be encrypted. + ``validate_key`` + The key used to sign the encrypted session + ``cookie_domain`` + Domain to use for the cookie. + ``secure`` + Whether or not the cookie should only be sent over SSL. + + """ + def __init__(self, request, key='beaker.session.id', timeout=None, + cookie_expires=True, cookie_domain=None, encrypt_key=None, + validate_key=None, secure=False, **kwargs): + + if not crypto.has_aes and encrypt_key: + raise InvalidCryptoBackendError("No AES library is installed, can't generate " + "encrypted cookie-only Session.") + + self.request = request + self.key = key + self.timeout = timeout + self.cookie_expires = cookie_expires + self.encrypt_key = encrypt_key + self.validate_key = validate_key + self.request['set_cookie'] = False + self.secure = secure + self._domain = cookie_domain + self._path = '/' + + try: + cookieheader = request['cookie'] + except KeyError: + cookieheader = '' + + if validate_key is None: + raise BeakerException("No validate_key specified for Cookie only " + "Session.") + + try: + self.cookie = SignedCookie(validate_key, input=cookieheader) + except Cookie.CookieError: + self.cookie = SignedCookie(validate_key, input=None) + + self['_id'] = self._make_id() + self.is_new = True + + # If we have a cookie, load it + if self.key in self.cookie and self.cookie[self.key].value is not None: + self.is_new = False + try: + self.update(self._decrypt_data()) + except: + pass + if self.timeout is not None and time.time() - \ + self['_accessed_time'] > self.timeout: + self.clear() + self.accessed_dict = self.copy() + self._create_cookie() + + def created(self): + return self['_creation_time'] + created = property(created) + + def id(self): + return self['_id'] + id = property(id) + + def _set_domain(self, domain): + self['_domain'] = domain + self._domain = domain + + def _get_domain(self): + return self._domain + + domain = property(_get_domain, _set_domain) + + def _set_path(self, path): + self['_path'] = path + self._path = path + + def _get_path(self): + return self._path + + path = property(_get_path, _set_path) + + def _encrypt_data(self): + """Serialize, encipher, and base64 the session dict""" + if self.encrypt_key: + nonce = b64encode(os.urandom(40))[:8] + encrypt_key = crypto.generateCryptoKeys(self.encrypt_key, + self.validate_key + nonce, 1) + data = pickle.dumps(self.copy(), 2) + return nonce + b64encode(crypto.aesEncrypt(data, encrypt_key)) + else: + data = pickle.dumps(self.copy(), 2) + return b64encode(data) + + def _decrypt_data(self): + """Bas64, decipher, then un-serialize the data for the session + dict""" + if self.encrypt_key: + nonce = self.cookie[self.key].value[:8] + encrypt_key = crypto.generateCryptoKeys(self.encrypt_key, + self.validate_key + nonce, 1) + payload = b64decode(self.cookie[self.key].value[8:]) + data = crypto.aesDecrypt(payload, encrypt_key) + return pickle.loads(data) + else: + data = b64decode(self.cookie[self.key].value) + return pickle.loads(data) + + def _make_id(self): + return md5(md5( + "%f%s%f%s" % (time.time(), id({}), random.random(), getpid()) + ).hexdigest() + ).hexdigest() + + def save(self, accessed_only=False): + """Saves the data for this session to persistent storage""" + if accessed_only and self.is_new: + return + if accessed_only: + self.clear() + self.update(self.accessed_dict) + self._create_cookie() + + def expire(self): + """Delete the 'expires' attribute on this Session, if any.""" + + self.pop('_expires', None) + + def _create_cookie(self): + if '_creation_time' not in self: + self['_creation_time'] = time.time() + if '_id' not in self: + self['_id'] = self._make_id() + self['_accessed_time'] = time.time() + + if self.cookie_expires is not True: + if self.cookie_expires is False: + expires = datetime.fromtimestamp( 0x7FFFFFFF ) + elif isinstance(self.cookie_expires, timedelta): + expires = datetime.today() + self.cookie_expires + elif isinstance(self.cookie_expires, datetime): + expires = self.cookie_expires + else: + raise ValueError("Invalid argument for cookie_expires: %s" + % repr(self.cookie_expires)) + self['_expires'] = expires + elif '_expires' in self: + expires = self['_expires'] + else: + expires = None + + val = self._encrypt_data() + if len(val) > 4064: + raise BeakerException("Cookie value is too long to store") + + self.cookie[self.key] = val + if '_domain' in self: + self.cookie[self.key]['domain'] = self['_domain'] + elif self._domain: + self.cookie[self.key]['domain'] = self._domain + if self.secure: + self.cookie[self.key]['secure'] = True + + self.cookie[self.key]['path'] = self.get('_path', '/') + + if expires: + self.cookie[self.key]['expires'] = \ + expires.strftime("%a, %d-%b-%Y %H:%M:%S GMT" ) + self.request['cookie_out'] = self.cookie[self.key].output(header='') + self.request['set_cookie'] = True + + def delete(self): + """Delete the cookie, and clear the session""" + # Send a delete cookie request + self._delete_cookie() + self.clear() + + def invalidate(self): + """Clear the contents and start a new session""" + self.delete() + self['_id'] = self._make_id() + + +class SessionObject(object): + """Session proxy/lazy creator + + This object proxies access to the actual session object, so that in + the case that the session hasn't been used before, it will be + setup. This avoid creating and loading the session from persistent + storage unless its actually used during the request. + + """ + def __init__(self, environ, **params): + self.__dict__['_params'] = params + self.__dict__['_environ'] = environ + self.__dict__['_sess'] = None + self.__dict__['_headers'] = [] + + def _session(self): + """Lazy initial creation of session object""" + if self.__dict__['_sess'] is None: + params = self.__dict__['_params'] + environ = self.__dict__['_environ'] + self.__dict__['_headers'] = req = {'cookie_out':None} + req['cookie'] = environ.get('HTTP_COOKIE') + if params.get('type') == 'cookie': + self.__dict__['_sess'] = CookieSession(req, **params) + else: + self.__dict__['_sess'] = Session(req, use_cookies=True, + **params) + return self.__dict__['_sess'] + + def __getattr__(self, attr): + return getattr(self._session(), attr) + + def __setattr__(self, attr, value): + setattr(self._session(), attr, value) + + def __delattr__(self, name): + self._session().__delattr__(name) + + def __getitem__(self, key): + return self._session()[key] + + def __setitem__(self, key, value): + self._session()[key] = value + + def __delitem__(self, key): + self._session().__delitem__(key) + + def __repr__(self): + return self._session().__repr__() + + def __iter__(self): + """Only works for proxying to a dict""" + return iter(self._session().keys()) + + def __contains__(self, key): + return self._session().has_key(key) + + def get_by_id(self, id): + """Loads a session given a session ID""" + params = self.__dict__['_params'] + session = Session({}, use_cookies=False, id=id, **params) + if session.is_new: + return None + return session + + def save(self): + self.__dict__['_dirty'] = True + + def delete(self): + self.__dict__['_dirty'] = True + self._session().delete() + + def persist(self): + """Persist the session to the storage + + If its set to autosave, then the entire session will be saved + regardless of if save() has been called. Otherwise, just the + accessed time will be updated if save() was not called, or + the session will be saved if save() was called. + + """ + if self.__dict__['_params'].get('auto'): + self._session().save() + else: + if self.__dict__.get('_dirty'): + self._session().save() + else: + self._session().save(accessed_only=True) + + def dirty(self): + return self.__dict__.get('_dirty', False) + + def accessed(self): + """Returns whether or not the session has been accessed""" + return self.__dict__['_sess'] is not None diff --git a/pyload/lib/beaker/synchronization.py b/pyload/lib/beaker/synchronization.py new file mode 100644 index 000000000..761303707 --- /dev/null +++ b/pyload/lib/beaker/synchronization.py @@ -0,0 +1,381 @@ +"""Synchronization functions. + +File- and mutex-based mutual exclusion synchronizers are provided, +as well as a name-based mutex which locks within an application +based on a string name. + +""" + +import os +import sys +import tempfile + +try: + import threading as _threading +except ImportError: + import dummy_threading as _threading + +# check for fcntl module +try: + sys.getwindowsversion() + has_flock = False +except: + try: + import fcntl + has_flock = True + except ImportError: + has_flock = False + +from beaker import util +from beaker.exceptions import LockError + +__all__ = ["file_synchronizer", "mutex_synchronizer", "null_synchronizer", + "NameLock", "_threading"] + + +class NameLock(object): + """a proxy for an RLock object that is stored in a name based + registry. + + Multiple threads can get a reference to the same RLock based on the + name alone, and synchronize operations related to that name. + + """ + locks = util.WeakValuedRegistry() + + class NLContainer(object): + def __init__(self, reentrant): + if reentrant: + self.lock = _threading.RLock() + else: + self.lock = _threading.Lock() + def __call__(self): + return self.lock + + def __init__(self, identifier = None, reentrant = False): + if identifier is None: + self._lock = NameLock.NLContainer(reentrant) + else: + self._lock = NameLock.locks.get(identifier, NameLock.NLContainer, + reentrant) + + def acquire(self, wait = True): + return self._lock().acquire(wait) + + def release(self): + self._lock().release() + + +_synchronizers = util.WeakValuedRegistry() +def _synchronizer(identifier, cls, **kwargs): + return _synchronizers.sync_get((identifier, cls), cls, identifier, **kwargs) + + +def file_synchronizer(identifier, **kwargs): + if not has_flock or 'lock_dir' not in kwargs: + return mutex_synchronizer(identifier) + else: + return _synchronizer(identifier, FileSynchronizer, **kwargs) + + +def mutex_synchronizer(identifier, **kwargs): + return _synchronizer(identifier, ConditionSynchronizer, **kwargs) + + +class null_synchronizer(object): + def acquire_write_lock(self, wait=True): + return True + def acquire_read_lock(self): + pass + def release_write_lock(self): + pass + def release_read_lock(self): + pass + acquire = acquire_write_lock + release = release_write_lock + + +class SynchronizerImpl(object): + def __init__(self): + self._state = util.ThreadLocal() + + class SyncState(object): + __slots__ = 'reentrantcount', 'writing', 'reading' + + def __init__(self): + self.reentrantcount = 0 + self.writing = False + self.reading = False + + def state(self): + if not self._state.has(): + state = SynchronizerImpl.SyncState() + self._state.put(state) + return state + else: + return self._state.get() + state = property(state) + + def release_read_lock(self): + state = self.state + + if state.writing: + raise LockError("lock is in writing state") + if not state.reading: + raise LockError("lock is not in reading state") + + if state.reentrantcount == 1: + self.do_release_read_lock() + state.reading = False + + state.reentrantcount -= 1 + + def acquire_read_lock(self, wait = True): + state = self.state + + if state.writing: + raise LockError("lock is in writing state") + + if state.reentrantcount == 0: + x = self.do_acquire_read_lock(wait) + if (wait or x): + state.reentrantcount += 1 + state.reading = True + return x + elif state.reading: + state.reentrantcount += 1 + return True + + def release_write_lock(self): + state = self.state + + if state.reading: + raise LockError("lock is in reading state") + if not state.writing: + raise LockError("lock is not in writing state") + + if state.reentrantcount == 1: + self.do_release_write_lock() + state.writing = False + + state.reentrantcount -= 1 + + release = release_write_lock + + def acquire_write_lock(self, wait = True): + state = self.state + + if state.reading: + raise LockError("lock is in reading state") + + if state.reentrantcount == 0: + x = self.do_acquire_write_lock(wait) + if (wait or x): + state.reentrantcount += 1 + state.writing = True + return x + elif state.writing: + state.reentrantcount += 1 + return True + + acquire = acquire_write_lock + + def do_release_read_lock(self): + raise NotImplementedError() + + def do_acquire_read_lock(self): + raise NotImplementedError() + + def do_release_write_lock(self): + raise NotImplementedError() + + def do_acquire_write_lock(self): + raise NotImplementedError() + + +class FileSynchronizer(SynchronizerImpl): + """a synchronizer which locks using flock(). + + Adapted for Python/multithreads from Apache::Session::Lock::File, + http://search.cpan.org/src/CWEST/Apache-Session-1.81/Session/Lock/File.pm + + This module does not unlink temporary files, + because it interferes with proper locking. This can cause + problems on certain systems (Linux) whose file systems (ext2) do not + perform well with lots of files in one directory. To prevent this + you should use a script to clean out old files from your lock directory. + + """ + def __init__(self, identifier, lock_dir): + super(FileSynchronizer, self).__init__() + self._filedescriptor = util.ThreadLocal() + + if lock_dir is None: + lock_dir = tempfile.gettempdir() + else: + lock_dir = lock_dir + + self.filename = util.encoded_path( + lock_dir, + [identifier], + extension='.lock' + ) + + def _filedesc(self): + return self._filedescriptor.get() + _filedesc = property(_filedesc) + + def _open(self, mode): + filedescriptor = self._filedesc + if filedescriptor is None: + filedescriptor = os.open(self.filename, mode) + self._filedescriptor.put(filedescriptor) + return filedescriptor + + def do_acquire_read_lock(self, wait): + filedescriptor = self._open(os.O_CREAT | os.O_RDONLY) + if not wait: + try: + fcntl.flock(filedescriptor, fcntl.LOCK_SH | fcntl.LOCK_NB) + return True + except IOError: + os.close(filedescriptor) + self._filedescriptor.remove() + return False + else: + fcntl.flock(filedescriptor, fcntl.LOCK_SH) + return True + + def do_acquire_write_lock(self, wait): + filedescriptor = self._open(os.O_CREAT | os.O_WRONLY) + if not wait: + try: + fcntl.flock(filedescriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + return True + except IOError: + os.close(filedescriptor) + self._filedescriptor.remove() + return False + else: + fcntl.flock(filedescriptor, fcntl.LOCK_EX) + return True + + def do_release_read_lock(self): + self._release_all_locks() + + def do_release_write_lock(self): + self._release_all_locks() + + def _release_all_locks(self): + filedescriptor = self._filedesc + if filedescriptor is not None: + fcntl.flock(filedescriptor, fcntl.LOCK_UN) + os.close(filedescriptor) + self._filedescriptor.remove() + + +class ConditionSynchronizer(SynchronizerImpl): + """a synchronizer using a Condition.""" + + def __init__(self, identifier): + super(ConditionSynchronizer, self).__init__() + + # counts how many asynchronous methods are executing + self.async = 0 + + # pointer to thread that is the current sync operation + self.current_sync_operation = None + + # condition object to lock on + self.condition = _threading.Condition(_threading.Lock()) + + def do_acquire_read_lock(self, wait = True): + self.condition.acquire() + try: + # see if a synchronous operation is waiting to start + # or is already running, in which case we wait (or just + # give up and return) + if wait: + while self.current_sync_operation is not None: + self.condition.wait() + else: + if self.current_sync_operation is not None: + return False + + self.async += 1 + finally: + self.condition.release() + + if not wait: + return True + + def do_release_read_lock(self): + self.condition.acquire() + try: + self.async -= 1 + + # check if we are the last asynchronous reader thread + # out the door. + if self.async == 0: + # yes. so if a sync operation is waiting, notifyAll to wake + # it up + if self.current_sync_operation is not None: + self.condition.notifyAll() + elif self.async < 0: + raise LockError("Synchronizer error - too many " + "release_read_locks called") + finally: + self.condition.release() + + def do_acquire_write_lock(self, wait = True): + self.condition.acquire() + try: + # here, we are not a synchronous reader, and after returning, + # assuming waiting or immediate availability, we will be. + + if wait: + # if another sync is working, wait + while self.current_sync_operation is not None: + self.condition.wait() + else: + # if another sync is working, + # we dont want to wait, so forget it + if self.current_sync_operation is not None: + return False + + # establish ourselves as the current sync + # this indicates to other read/write operations + # that they should wait until this is None again + self.current_sync_operation = _threading.currentThread() + + # now wait again for asyncs to finish + if self.async > 0: + if wait: + # wait + self.condition.wait() + else: + # we dont want to wait, so forget it + self.current_sync_operation = None + return False + finally: + self.condition.release() + + if not wait: + return True + + def do_release_write_lock(self): + self.condition.acquire() + try: + if self.current_sync_operation is not _threading.currentThread(): + raise LockError("Synchronizer error - current thread doesnt " + "have the write lock") + + # reset the current sync operation so + # another can get it + self.current_sync_operation = None + + # tell everyone to get ready + self.condition.notifyAll() + finally: + # everyone go !! + self.condition.release() diff --git a/pyload/lib/beaker/util.py b/pyload/lib/beaker/util.py new file mode 100644 index 000000000..04c9617c5 --- /dev/null +++ b/pyload/lib/beaker/util.py @@ -0,0 +1,302 @@ +"""Beaker utilities""" + +try: + import thread as _thread + import threading as _threading +except ImportError: + import dummy_thread as _thread + import dummy_threading as _threading + +from datetime import datetime, timedelta +import os +import string +import types +import weakref +import warnings +import sys + +py3k = getattr(sys, 'py3kwarning', False) or sys.version_info >= (3, 0) +py24 = sys.version_info < (2,5) +jython = sys.platform.startswith('java') + +if py3k or jython: + import pickle +else: + import cPickle as pickle + +from beaker.converters import asbool +from threading import local as _tlocal + + +__all__ = ["ThreadLocal", "Registry", "WeakValuedRegistry", "SyncDict", + "encoded_path", "verify_directory"] + + +def verify_directory(dir): + """verifies and creates a directory. tries to + ignore collisions with other threads and processes.""" + + tries = 0 + while not os.access(dir, os.F_OK): + try: + tries += 1 + os.makedirs(dir) + except: + if tries > 5: + raise + + +def deprecated(message): + def wrapper(fn): + def deprecated_method(*args, **kargs): + warnings.warn(message, DeprecationWarning, 2) + return fn(*args, **kargs) + # TODO: use decorator ? functools.wrapper ? + deprecated_method.__name__ = fn.__name__ + deprecated_method.__doc__ = "%s\n\n%s" % (message, fn.__doc__) + return deprecated_method + return wrapper + +class ThreadLocal(object): + """stores a value on a per-thread basis""" + + __slots__ = '_tlocal' + + def __init__(self): + self._tlocal = _tlocal() + + def put(self, value): + self._tlocal.value = value + + def has(self): + return hasattr(self._tlocal, 'value') + + def get(self, default=None): + return getattr(self._tlocal, 'value', default) + + def remove(self): + del self._tlocal.value + +class SyncDict(object): + """ + An efficient/threadsafe singleton map algorithm, a.k.a. + "get a value based on this key, and create if not found or not + valid" paradigm: + + exists && isvalid ? get : create + + Designed to work with weakref dictionaries to expect items + to asynchronously disappear from the dictionary. + + Use python 2.3.3 or greater ! a major bug was just fixed in Nov. + 2003 that was driving me nuts with garbage collection/weakrefs in + this section. + + """ + def __init__(self): + self.mutex = _thread.allocate_lock() + self.dict = {} + + def get(self, key, createfunc, *args, **kwargs): + try: + if self.has_key(key): + return self.dict[key] + else: + return self.sync_get(key, createfunc, *args, **kwargs) + except KeyError: + return self.sync_get(key, createfunc, *args, **kwargs) + + def sync_get(self, key, createfunc, *args, **kwargs): + self.mutex.acquire() + try: + try: + if self.has_key(key): + return self.dict[key] + else: + return self._create(key, createfunc, *args, **kwargs) + except KeyError: + return self._create(key, createfunc, *args, **kwargs) + finally: + self.mutex.release() + + def _create(self, key, createfunc, *args, **kwargs): + self[key] = obj = createfunc(*args, **kwargs) + return obj + + def has_key(self, key): + return self.dict.has_key(key) + + def __contains__(self, key): + return self.dict.__contains__(key) + def __getitem__(self, key): + return self.dict.__getitem__(key) + def __setitem__(self, key, value): + self.dict.__setitem__(key, value) + def __delitem__(self, key): + return self.dict.__delitem__(key) + def clear(self): + self.dict.clear() + + +class WeakValuedRegistry(SyncDict): + def __init__(self): + self.mutex = _threading.RLock() + self.dict = weakref.WeakValueDictionary() + +sha1 = None +def encoded_path(root, identifiers, extension = ".enc", depth = 3, + digest_filenames=True): + + """Generate a unique file-accessible path from the given list of + identifiers starting at the given root directory.""" + ident = "_".join(identifiers) + + global sha1 + if sha1 is None: + from beaker.crypto import sha1 + + if digest_filenames: + if py3k: + ident = sha1(ident.encode('utf-8')).hexdigest() + else: + ident = sha1(ident).hexdigest() + + ident = os.path.basename(ident) + + tokens = [] + for d in range(1, depth): + tokens.append(ident[0:d]) + + dir = os.path.join(root, *tokens) + verify_directory(dir) + + return os.path.join(dir, ident + extension) + + +def verify_options(opt, types, error): + if not isinstance(opt, types): + if not isinstance(types, tuple): + types = (types,) + coerced = False + for typ in types: + try: + if typ in (list, tuple): + opt = [x.strip() for x in opt.split(',')] + else: + if typ == bool: + typ = asbool + opt = typ(opt) + coerced = True + except: + pass + if coerced: + break + if not coerced: + raise Exception(error) + elif isinstance(opt, str) and not opt.strip(): + raise Exception("Empty strings are invalid for: %s" % error) + return opt + + +def verify_rules(params, ruleset): + for key, types, message in ruleset: + if key in params: + params[key] = verify_options(params[key], types, message) + return params + + +def coerce_session_params(params): + rules = [ + ('data_dir', (str, types.NoneType), "data_dir must be a string " + "referring to a directory."), + ('lock_dir', (str, types.NoneType), "lock_dir must be a string referring to a " + "directory."), + ('type', (str, types.NoneType), "Session type must be a string."), + ('cookie_expires', (bool, datetime, timedelta), "Cookie expires was " + "not a boolean, datetime, or timedelta instance."), + ('cookie_domain', (str, types.NoneType), "Cookie domain must be a " + "string."), + ('id', (str,), "Session id must be a string."), + ('key', (str,), "Session key must be a string."), + ('secret', (str, types.NoneType), "Session secret must be a string."), + ('validate_key', (str, types.NoneType), "Session encrypt_key must be " + "a string."), + ('encrypt_key', (str, types.NoneType), "Session validate_key must be " + "a string."), + ('secure', (bool, types.NoneType), "Session secure must be a boolean."), + ('timeout', (int, types.NoneType), "Session timeout must be an " + "integer."), + ('auto', (bool, types.NoneType), "Session is created if accessed."), + ] + return verify_rules(params, rules) + + +def coerce_cache_params(params): + rules = [ + ('data_dir', (str, types.NoneType), "data_dir must be a string " + "referring to a directory."), + ('lock_dir', (str, types.NoneType), "lock_dir must be a string referring to a " + "directory."), + ('type', (str,), "Cache type must be a string."), + ('enabled', (bool, types.NoneType), "enabled must be true/false " + "if present."), + ('expire', (int, types.NoneType), "expire must be an integer representing " + "how many seconds the cache is valid for"), + ('regions', (list, tuple, types.NoneType), "Regions must be a " + "comma seperated list of valid regions") + ] + return verify_rules(params, rules) + + +def parse_cache_config_options(config, include_defaults=True): + """Parse configuration options and validate for use with the + CacheManager""" + + # Load default cache options + if include_defaults: + options= dict(type='memory', data_dir=None, expire=None, + log_file=None) + else: + options = {} + for key, val in config.iteritems(): + if key.startswith('beaker.cache.'): + options[key[13:]] = val + if key.startswith('cache.'): + options[key[6:]] = val + coerce_cache_params(options) + + # Set cache to enabled if not turned off + if 'enabled' not in options: + options['enabled'] = True + + # Configure region dict if regions are available + regions = options.pop('regions', None) + if regions: + region_configs = {} + for region in regions: + # Setup the default cache options + region_options = dict(data_dir=options.get('data_dir'), + lock_dir=options.get('lock_dir'), + type=options.get('type'), + enabled=options['enabled'], + expire=options.get('expire')) + region_len = len(region) + 1 + for key in options.keys(): + if key.startswith('%s.' % region): + region_options[key[region_len:]] = options.pop(key) + coerce_cache_params(region_options) + region_configs[region] = region_options + options['cache_regions'] = region_configs + return options + +def func_namespace(func): + """Generates a unique namespace for a function""" + kls = None + if hasattr(func, 'im_func'): + kls = func.im_class + func = func.im_func + + if kls: + return '%s.%s' % (kls.__module__, kls.__name__) + else: + return '%s.%s' % (func.__module__, func.__name__) diff --git a/pyload/lib/bottle.py b/pyload/lib/bottle.py new file mode 100644 index 000000000..b00bda1c9 --- /dev/null +++ b/pyload/lib/bottle.py @@ -0,0 +1,3251 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Bottle is a fast and simple micro-framework for small web applications. It +offers request dispatching (Routes) with url parameter support, templates, +a built-in HTTP Server and adapters for many third party WSGI/HTTP-server and +template engines - all in a single file and with no dependencies other than the +Python Standard Library. + +Homepage and documentation: http://bottlepy.org/ + +Copyright (c) 2012, Marcel Hellkamp. +License: MIT (see LICENSE for details) +""" + +from __future__ import with_statement + +__author__ = 'Marcel Hellkamp' +__version__ = '0.11.4' +__license__ = 'MIT' + +# The gevent server adapter needs to patch some modules before they are imported +# This is why we parse the commandline parameters here but handle them later +if __name__ == '__main__': + from optparse import OptionParser + _cmd_parser = OptionParser(usage="usage: %prog [options] package.module:app") + _opt = _cmd_parser.add_option + _opt("--version", action="store_true", help="show version number.") + _opt("-b", "--bind", metavar="ADDRESS", help="bind socket to ADDRESS.") + _opt("-s", "--server", default='wsgiref', help="use SERVER as backend.") + _opt("-p", "--plugin", action="append", help="install additional plugin/s.") + _opt("--debug", action="store_true", help="start server in debug mode.") + _opt("--reload", action="store_true", help="auto-reload on file changes.") + _cmd_options, _cmd_args = _cmd_parser.parse_args() + if _cmd_options.server and _cmd_options.server.startswith('gevent'): + import gevent.monkey; gevent.monkey.patch_all() + +import base64, cgi, email.utils, functools, hmac, imp, itertools, mimetypes,\ + os, re, subprocess, sys, tempfile, threading, time, urllib, warnings + +from datetime import date as datedate, datetime, timedelta +from tempfile import TemporaryFile +from traceback import format_exc, print_exc + +try: from json import dumps as json_dumps, loads as json_lds +except ImportError: # pragma: no cover + try: from simplejson import dumps as json_dumps, loads as json_lds + except ImportError: + try: from django.utils.simplejson import dumps as json_dumps, loads as json_lds + except ImportError: + def json_dumps(data): + raise ImportError("JSON support requires Python 2.6 or simplejson.") + json_lds = json_dumps + + + +# We now try to fix 2.5/2.6/3.1/3.2 incompatibilities. +# It ain't pretty but it works... Sorry for the mess. + +py = sys.version_info +py3k = py >= (3,0,0) +py25 = py < (2,6,0) +py31 = (3,1,0) <= py < (3,2,0) + +# Workaround for the missing "as" keyword in py3k. +def _e(): return sys.exc_info()[1] + +# Workaround for the "print is a keyword/function" Python 2/3 dilemma +# and a fallback for mod_wsgi (resticts stdout/err attribute access) +try: + _stdout, _stderr = sys.stdout.write, sys.stderr.write +except IOError: + _stdout = lambda x: sys.stdout.write(x) + _stderr = lambda x: sys.stderr.write(x) + +# Lots of stdlib and builtin differences. +if py3k: + import http.client as httplib + import _thread as thread + from urllib.parse import urljoin, SplitResult as UrlSplitResult + from urllib.parse import urlencode, quote as urlquote, unquote as urlunquote + urlunquote = functools.partial(urlunquote, encoding='latin1') + from http.cookies import SimpleCookie + from collections import MutableMapping as DictMixin + import pickle + from io import BytesIO + basestring = str + unicode = str + json_loads = lambda s: json_lds(touni(s)) + callable = lambda x: hasattr(x, '__call__') + imap = map +else: # 2.x + import httplib + import thread + from urlparse import urljoin, SplitResult as UrlSplitResult + from urllib import urlencode, quote as urlquote, unquote as urlunquote + from Cookie import SimpleCookie + from itertools import imap + import cPickle as pickle + from StringIO import StringIO as BytesIO + if py25: + from UserDict import DictMixin + def next(it): return it.next() + bytes = str + else: # 2.6, 2.7 + from collections import MutableMapping as DictMixin + json_loads = json_lds + +# Some helpers for string/byte handling +def tob(s, enc='utf8'): + return s.encode(enc) if isinstance(s, unicode) else bytes(s) +def touni(s, enc='utf8', err='strict'): + return s.decode(enc, err) if isinstance(s, bytes) else unicode(s) +tonat = touni if py3k else tob + +# 3.2 fixes cgi.FieldStorage to accept bytes (which makes a lot of sense). +# 3.1 needs a workaround. +if py31: + from io import TextIOWrapper + class NCTextIOWrapper(TextIOWrapper): + def close(self): pass # Keep wrapped buffer open. + +# File uploads (which are implemented as empty FiledStorage instances...) +# have a negative truth value. That makes no sense, here is a fix. +class FieldStorage(cgi.FieldStorage): + def __nonzero__(self): return bool(self.list or self.file) + if py3k: __bool__ = __nonzero__ + +# A bug in functools causes it to break if the wrapper is an instance method +def update_wrapper(wrapper, wrapped, *a, **ka): + try: functools.update_wrapper(wrapper, wrapped, *a, **ka) + except AttributeError: pass + + + +# These helpers are used at module level and need to be defined first. +# And yes, I know PEP-8, but sometimes a lower-case classname makes more sense. + +def depr(message): + warnings.warn(message, DeprecationWarning, stacklevel=3) + +def makelist(data): # This is just to handy + if isinstance(data, (tuple, list, set, dict)): return list(data) + elif data: return [data] + else: return [] + + +class DictProperty(object): + ''' Property that maps to a key in a local dict-like attribute. ''' + def __init__(self, attr, key=None, read_only=False): + self.attr, self.key, self.read_only = attr, key, read_only + + def __call__(self, func): + functools.update_wrapper(self, func, updated=[]) + self.getter, self.key = func, self.key or func.__name__ + return self + + def __get__(self, obj, cls): + if obj is None: return self + key, storage = self.key, getattr(obj, self.attr) + if key not in storage: storage[key] = self.getter(obj) + return storage[key] + + def __set__(self, obj, value): + if self.read_only: raise AttributeError("Read-Only property.") + getattr(obj, self.attr)[self.key] = value + + def __delete__(self, obj): + if self.read_only: raise AttributeError("Read-Only property.") + del getattr(obj, self.attr)[self.key] + + +class cached_property(object): + ''' A property that is only computed once per instance and then replaces + itself with an ordinary attribute. Deleting the attribute resets the + property. ''' + + def __init__(self, func): + self.func = func + + def __get__(self, obj, cls): + if obj is None: return self + value = obj.__dict__[self.func.__name__] = self.func(obj) + return value + + +class lazy_attribute(object): + ''' A property that caches itself to the class object. ''' + def __init__(self, func): + functools.update_wrapper(self, func, updated=[]) + self.getter = func + + def __get__(self, obj, cls): + value = self.getter(cls) + setattr(cls, self.__name__, value) + return value + + + + + + +############################################################################### +# Exceptions and Events ######################################################## +############################################################################### + + +class BottleException(Exception): + """ A base class for exceptions used by bottle. """ + pass + + + + + + +############################################################################### +# Routing ###################################################################### +############################################################################### + + +class RouteError(BottleException): + """ This is a base class for all routing related exceptions """ + + +class RouteReset(BottleException): + """ If raised by a plugin or request handler, the route is reset and all + plugins are re-applied. """ + +class RouterUnknownModeError(RouteError): pass + + +class RouteSyntaxError(RouteError): + """ The route parser found something not supported by this router """ + + +class RouteBuildError(RouteError): + """ The route could not been built """ + + +class Router(object): + ''' A Router is an ordered collection of route->target pairs. It is used to + efficiently match WSGI requests against a number of routes and return + the first target that satisfies the request. The target may be anything, + usually a string, ID or callable object. A route consists of a path-rule + and a HTTP method. + + The path-rule is either a static path (e.g. `/contact`) or a dynamic + path that contains wildcards (e.g. `/wiki/`). The wildcard syntax + and details on the matching order are described in docs:`routing`. + ''' + + default_pattern = '[^/]+' + default_filter = 're' + #: Sorry for the mess. It works. Trust me. + rule_syntax = re.compile('(\\\\*)'\ + '(?:(?::([a-zA-Z_][a-zA-Z_0-9]*)?()(?:#(.*?)#)?)'\ + '|(?:<([a-zA-Z_][a-zA-Z_0-9]*)?(?::([a-zA-Z_]*)'\ + '(?::((?:\\\\.|[^\\\\>]+)+)?)?)?>))') + + def __init__(self, strict=False): + self.rules = {} # A {rule: Rule} mapping + self.builder = {} # A rule/name->build_info mapping + self.static = {} # Cache for static routes: {path: {method: target}} + self.dynamic = [] # Cache for dynamic routes. See _compile() + #: If true, static routes are no longer checked first. + self.strict_order = strict + self.filters = {'re': self.re_filter, 'int': self.int_filter, + 'float': self.float_filter, 'path': self.path_filter} + + def re_filter(self, conf): + return conf or self.default_pattern, None, None + + def int_filter(self, conf): + return r'-?\d+', int, lambda x: str(int(x)) + + def float_filter(self, conf): + return r'-?[\d.]+', float, lambda x: str(float(x)) + + def path_filter(self, conf): + return r'.+?', None, None + + def add_filter(self, name, func): + ''' Add a filter. The provided function is called with the configuration + string as parameter and must return a (regexp, to_python, to_url) tuple. + The first element is a string, the last two are callables or None. ''' + self.filters[name] = func + + def parse_rule(self, rule): + ''' Parses a rule into a (name, filter, conf) token stream. If mode is + None, name contains a static rule part. ''' + offset, prefix = 0, '' + for match in self.rule_syntax.finditer(rule): + prefix += rule[offset:match.start()] + g = match.groups() + if len(g[0])%2: # Escaped wildcard + prefix += match.group(0)[len(g[0]):] + offset = match.end() + continue + if prefix: yield prefix, None, None + name, filtr, conf = g[1:4] if not g[2] is None else g[4:7] + if not filtr: filtr = self.default_filter + yield name, filtr, conf or None + offset, prefix = match.end(), '' + if offset <= len(rule) or prefix: + yield prefix+rule[offset:], None, None + + def add(self, rule, method, target, name=None): + ''' Add a new route or replace the target for an existing route. ''' + if rule in self.rules: + self.rules[rule][method] = target + if name: self.builder[name] = self.builder[rule] + return + + target = self.rules[rule] = {method: target} + + # Build pattern and other structures for dynamic routes + anons = 0 # Number of anonymous wildcards + pattern = '' # Regular expression pattern + filters = [] # Lists of wildcard input filters + builder = [] # Data structure for the URL builder + is_static = True + for key, mode, conf in self.parse_rule(rule): + if mode: + is_static = False + mask, in_filter, out_filter = self.filters[mode](conf) + if key: + pattern += '(?P<%s>%s)' % (key, mask) + else: + pattern += '(?:%s)' % mask + key = 'anon%d' % anons; anons += 1 + if in_filter: filters.append((key, in_filter)) + builder.append((key, out_filter or str)) + elif key: + pattern += re.escape(key) + builder.append((None, key)) + self.builder[rule] = builder + if name: self.builder[name] = builder + + if is_static and not self.strict_order: + self.static[self.build(rule)] = target + return + + def fpat_sub(m): + return m.group(0) if len(m.group(1)) % 2 else m.group(1) + '(?:' + flat_pattern = re.sub(r'(\\*)(\(\?P<[^>]*>|\((?!\?))', fpat_sub, pattern) + + try: + re_match = re.compile('^(%s)$' % pattern).match + except re.error: + raise RouteSyntaxError("Could not add Route: %s (%s)" % (rule, _e())) + + def match(path): + """ Return an url-argument dictionary. """ + url_args = re_match(path).groupdict() + for name, wildcard_filter in filters: + try: + url_args[name] = wildcard_filter(url_args[name]) + except ValueError: + raise HTTPError(400, 'Path has wrong format.') + return url_args + + try: + combined = '%s|(^%s$)' % (self.dynamic[-1][0].pattern, flat_pattern) + self.dynamic[-1] = (re.compile(combined), self.dynamic[-1][1]) + self.dynamic[-1][1].append((match, target)) + except (AssertionError, IndexError): # AssertionError: Too many groups + self.dynamic.append((re.compile('(^%s$)' % flat_pattern), + [(match, target)])) + return match + + def build(self, _name, *anons, **query): + ''' Build an URL by filling the wildcards in a rule. ''' + builder = self.builder.get(_name) + if not builder: raise RouteBuildError("No route with that name.", _name) + try: + for i, value in enumerate(anons): query['anon%d'%i] = value + url = ''.join([f(query.pop(n)) if n else f for (n,f) in builder]) + return url if not query else url+'?'+urlencode(query) + except KeyError: + raise RouteBuildError('Missing URL argument: %r' % _e().args[0]) + + def match(self, environ): + ''' Return a (target, url_agrs) tuple or raise HTTPError(400/404/405). ''' + path, targets, urlargs = environ['PATH_INFO'] or '/', None, {} + if path in self.static: + targets = self.static[path] + else: + for combined, rules in self.dynamic: + match = combined.match(path) + if not match: continue + getargs, targets = rules[match.lastindex - 1] + urlargs = getargs(path) if getargs else {} + break + + if not targets: + raise HTTPError(404, "Not found: " + repr(environ['PATH_INFO'])) + method = environ['REQUEST_METHOD'].upper() + if method in targets: + return targets[method], urlargs + if method == 'HEAD' and 'GET' in targets: + return targets['GET'], urlargs + if 'ANY' in targets: + return targets['ANY'], urlargs + allowed = [verb for verb in targets if verb != 'ANY'] + if 'GET' in allowed and 'HEAD' not in allowed: + allowed.append('HEAD') + raise HTTPError(405, "Method not allowed.", Allow=",".join(allowed)) + + +class Route(object): + ''' This class wraps a route callback along with route specific metadata and + configuration and applies Plugins on demand. It is also responsible for + turing an URL path rule into a regular expression usable by the Router. + ''' + + def __init__(self, app, rule, method, callback, name=None, + plugins=None, skiplist=None, **config): + #: The application this route is installed to. + self.app = app + #: The path-rule string (e.g. ``/wiki/:page``). + self.rule = rule + #: The HTTP method as a string (e.g. ``GET``). + self.method = method + #: The original callback with no plugins applied. Useful for introspection. + self.callback = callback + #: The name of the route (if specified) or ``None``. + self.name = name or None + #: A list of route-specific plugins (see :meth:`Bottle.route`). + self.plugins = plugins or [] + #: A list of plugins to not apply to this route (see :meth:`Bottle.route`). + self.skiplist = skiplist or [] + #: Additional keyword arguments passed to the :meth:`Bottle.route` + #: decorator are stored in this dictionary. Used for route-specific + #: plugin configuration and meta-data. + self.config = ConfigDict(config) + + def __call__(self, *a, **ka): + depr("Some APIs changed to return Route() instances instead of"\ + " callables. Make sure to use the Route.call method and not to"\ + " call Route instances directly.") + return self.call(*a, **ka) + + @cached_property + def call(self): + ''' The route callback with all plugins applied. This property is + created on demand and then cached to speed up subsequent requests.''' + return self._make_callback() + + def reset(self): + ''' Forget any cached values. The next time :attr:`call` is accessed, + all plugins are re-applied. ''' + self.__dict__.pop('call', None) + + def prepare(self): + ''' Do all on-demand work immediately (useful for debugging).''' + self.call + + @property + def _context(self): + depr('Switch to Plugin API v2 and access the Route object directly.') + return dict(rule=self.rule, method=self.method, callback=self.callback, + name=self.name, app=self.app, config=self.config, + apply=self.plugins, skip=self.skiplist) + + def all_plugins(self): + ''' Yield all Plugins affecting this route. ''' + unique = set() + for p in reversed(self.app.plugins + self.plugins): + if True in self.skiplist: break + name = getattr(p, 'name', False) + if name and (name in self.skiplist or name in unique): continue + if p in self.skiplist or type(p) in self.skiplist: continue + if name: unique.add(name) + yield p + + def _make_callback(self): + callback = self.callback + for plugin in self.all_plugins(): + try: + if hasattr(plugin, 'apply'): + api = getattr(plugin, 'api', 1) + context = self if api > 1 else self._context + callback = plugin.apply(callback, context) + else: + callback = plugin(callback) + except RouteReset: # Try again with changed configuration. + return self._make_callback() + if not callback is self.callback: + update_wrapper(callback, self.callback) + return callback + + def __repr__(self): + return '<%s %r %r>' % (self.method, self.rule, self.callback) + + + + + + +############################################################################### +# Application Object ########################################################### +############################################################################### + + +class Bottle(object): + """ Each Bottle object represents a single, distinct web application and + consists of routes, callbacks, plugins, resources and configuration. + Instances are callable WSGI applications. + + :param catchall: If true (default), handle all exceptions. Turn off to + let debugging middleware handle exceptions. + """ + + def __init__(self, catchall=True, autojson=True): + #: If true, most exceptions are caught and returned as :exc:`HTTPError` + self.catchall = catchall + + #: A :class:`ResourceManager` for application files + self.resources = ResourceManager() + + #: A :class:`ConfigDict` for app specific configuration. + self.config = ConfigDict() + self.config.autojson = autojson + + self.routes = [] # List of installed :class:`Route` instances. + self.router = Router() # Maps requests to :class:`Route` instances. + self.error_handler = {} + + # Core plugins + self.plugins = [] # List of installed plugins. + self.hooks = HooksPlugin() + self.install(self.hooks) + if self.config.autojson: + self.install(JSONPlugin()) + self.install(TemplatePlugin()) + + + def mount(self, prefix, app, **options): + ''' Mount an application (:class:`Bottle` or plain WSGI) to a specific + URL prefix. Example:: + + root_app.mount('/admin/', admin_app) + + :param prefix: path prefix or `mount-point`. If it ends in a slash, + that slash is mandatory. + :param app: an instance of :class:`Bottle` or a WSGI application. + + All other parameters are passed to the underlying :meth:`route` call. + ''' + if isinstance(app, basestring): + prefix, app = app, prefix + depr('Parameter order of Bottle.mount() changed.') # 0.10 + + segments = [p for p in prefix.split('/') if p] + if not segments: raise ValueError('Empty path prefix.') + path_depth = len(segments) + + def mountpoint_wrapper(): + try: + request.path_shift(path_depth) + rs = BaseResponse([], 200) + def start_response(status, header): + rs.status = status + for name, value in header: rs.add_header(name, value) + return rs.body.append + body = app(request.environ, start_response) + body = itertools.chain(rs.body, body) + return HTTPResponse(body, rs.status_code, **rs.headers) + finally: + request.path_shift(-path_depth) + + options.setdefault('skip', True) + options.setdefault('method', 'ANY') + options.setdefault('mountpoint', {'prefix': prefix, 'target': app}) + options['callback'] = mountpoint_wrapper + + self.route('/%s/<:re:.*>' % '/'.join(segments), **options) + if not prefix.endswith('/'): + self.route('/' + '/'.join(segments), **options) + + def merge(self, routes): + ''' Merge the routes of another :class:`Bottle` application or a list of + :class:`Route` objects into this application. The routes keep their + 'owner', meaning that the :data:`Route.app` attribute is not + changed. ''' + if isinstance(routes, Bottle): + routes = routes.routes + for route in routes: + self.add_route(route) + + def install(self, plugin): + ''' Add a plugin to the list of plugins and prepare it for being + applied to all routes of this application. A plugin may be a simple + decorator or an object that implements the :class:`Plugin` API. + ''' + if hasattr(plugin, 'setup'): plugin.setup(self) + if not callable(plugin) and not hasattr(plugin, 'apply'): + raise TypeError("Plugins must be callable or implement .apply()") + self.plugins.append(plugin) + self.reset() + return plugin + + def uninstall(self, plugin): + ''' Uninstall plugins. Pass an instance to remove a specific plugin, a type + object to remove all plugins that match that type, a string to remove + all plugins with a matching ``name`` attribute or ``True`` to remove all + plugins. Return the list of removed plugins. ''' + removed, remove = [], plugin + for i, plugin in list(enumerate(self.plugins))[::-1]: + if remove is True or remove is plugin or remove is type(plugin) \ + or getattr(plugin, 'name', True) == remove: + removed.append(plugin) + del self.plugins[i] + if hasattr(plugin, 'close'): plugin.close() + if removed: self.reset() + return removed + + def run(self, **kwargs): + ''' Calls :func:`run` with the same parameters. ''' + run(self, **kwargs) + + def reset(self, route=None): + ''' Reset all routes (force plugins to be re-applied) and clear all + caches. If an ID or route object is given, only that specific route + is affected. ''' + if route is None: routes = self.routes + elif isinstance(route, Route): routes = [route] + else: routes = [self.routes[route]] + for route in routes: route.reset() + if DEBUG: + for route in routes: route.prepare() + self.hooks.trigger('app_reset') + + def close(self): + ''' Close the application and all installed plugins. ''' + for plugin in self.plugins: + if hasattr(plugin, 'close'): plugin.close() + self.stopped = True + + def match(self, environ): + """ Search for a matching route and return a (:class:`Route` , urlargs) + tuple. The second value is a dictionary with parameters extracted + from the URL. Raise :exc:`HTTPError` (404/405) on a non-match.""" + return self.router.match(environ) + + def get_url(self, routename, **kargs): + """ Return a string that matches a named route """ + scriptname = request.environ.get('SCRIPT_NAME', '').strip('/') + '/' + location = self.router.build(routename, **kargs).lstrip('/') + return urljoin(urljoin('/', scriptname), location) + + def add_route(self, route): + ''' Add a route object, but do not change the :data:`Route.app` + attribute.''' + self.routes.append(route) + self.router.add(route.rule, route.method, route, name=route.name) + if DEBUG: route.prepare() + + def route(self, path=None, method='GET', callback=None, name=None, + apply=None, skip=None, **config): + """ A decorator to bind a function to a request URL. Example:: + + @app.route('/hello/:name') + def hello(name): + return 'Hello %s' % name + + The ``:name`` part is a wildcard. See :class:`Router` for syntax + details. + + :param path: Request path or a list of paths to listen to. If no + path is specified, it is automatically generated from the + signature of the function. + :param method: HTTP method (`GET`, `POST`, `PUT`, ...) or a list of + methods to listen to. (default: `GET`) + :param callback: An optional shortcut to avoid the decorator + syntax. ``route(..., callback=func)`` equals ``route(...)(func)`` + :param name: The name for this route. (default: None) + :param apply: A decorator or plugin or a list of plugins. These are + applied to the route callback in addition to installed plugins. + :param skip: A list of plugins, plugin classes or names. Matching + plugins are not installed to this route. ``True`` skips all. + + Any additional keyword arguments are stored as route-specific + configuration and passed to plugins (see :meth:`Plugin.apply`). + """ + if callable(path): path, callback = None, path + plugins = makelist(apply) + skiplist = makelist(skip) + def decorator(callback): + # TODO: Documentation and tests + if isinstance(callback, basestring): callback = load(callback) + for rule in makelist(path) or yieldroutes(callback): + for verb in makelist(method): + verb = verb.upper() + route = Route(self, rule, verb, callback, name=name, + plugins=plugins, skiplist=skiplist, **config) + self.add_route(route) + return callback + return decorator(callback) if callback else decorator + + def get(self, path=None, method='GET', **options): + """ Equals :meth:`route`. """ + return self.route(path, method, **options) + + def post(self, path=None, method='POST', **options): + """ Equals :meth:`route` with a ``POST`` method parameter. """ + return self.route(path, method, **options) + + def put(self, path=None, method='PUT', **options): + """ Equals :meth:`route` with a ``PUT`` method parameter. """ + return self.route(path, method, **options) + + def delete(self, path=None, method='DELETE', **options): + """ Equals :meth:`route` with a ``DELETE`` method parameter. """ + return self.route(path, method, **options) + + def error(self, code=500): + """ Decorator: Register an output handler for a HTTP error code""" + def wrapper(handler): + self.error_handler[int(code)] = handler + return handler + return wrapper + + def hook(self, name): + """ Return a decorator that attaches a callback to a hook. Three hooks + are currently implemented: + + - before_request: Executed once before each request + - after_request: Executed once after each request + - app_reset: Called whenever :meth:`reset` is called. + """ + def wrapper(func): + self.hooks.add(name, func) + return func + return wrapper + + def handle(self, path, method='GET'): + """ (deprecated) Execute the first matching route callback and return + the result. :exc:`HTTPResponse` exceptions are caught and returned. + If :attr:`Bottle.catchall` is true, other exceptions are caught as + well and returned as :exc:`HTTPError` instances (500). + """ + depr("This method will change semantics in 0.10. Try to avoid it.") + if isinstance(path, dict): + return self._handle(path) + return self._handle({'PATH_INFO': path, 'REQUEST_METHOD': method.upper()}) + + def default_error_handler(self, res): + return tob(template(ERROR_PAGE_TEMPLATE, e=res)) + + def _handle(self, environ): + try: + environ['bottle.app'] = self + request.bind(environ) + response.bind() + route, args = self.router.match(environ) + environ['route.handle'] = route + environ['bottle.route'] = route + environ['route.url_args'] = args + return route.call(**args) + except HTTPResponse: + return _e() + except RouteReset: + route.reset() + return self._handle(environ) + except (KeyboardInterrupt, SystemExit, MemoryError): + raise + except Exception: + if not self.catchall: raise + stacktrace = format_exc() + environ['wsgi.errors'].write(stacktrace) + return HTTPError(500, "Internal Server Error", _e(), stacktrace) + + def _cast(self, out, peek=None): + """ Try to convert the parameter into something WSGI compatible and set + correct HTTP headers when possible. + Support: False, str, unicode, dict, HTTPResponse, HTTPError, file-like, + iterable of strings and iterable of unicodes + """ + + # Empty output is done here + if not out: + if 'Content-Length' not in response: + response['Content-Length'] = 0 + return [] + # Join lists of byte or unicode strings. Mixed lists are NOT supported + if isinstance(out, (tuple, list))\ + and isinstance(out[0], (bytes, unicode)): + out = out[0][0:0].join(out) # b'abc'[0:0] -> b'' + # Encode unicode strings + if isinstance(out, unicode): + out = out.encode(response.charset) + # Byte Strings are just returned + if isinstance(out, bytes): + if 'Content-Length' not in response: + response['Content-Length'] = len(out) + return [out] + # HTTPError or HTTPException (recursive, because they may wrap anything) + # TODO: Handle these explicitly in handle() or make them iterable. + if isinstance(out, HTTPError): + out.apply(response) + out = self.error_handler.get(out.status_code, self.default_error_handler)(out) + return self._cast(out) + if isinstance(out, HTTPResponse): + out.apply(response) + return self._cast(out.body) + + # File-like objects. + if hasattr(out, 'read'): + if 'wsgi.file_wrapper' in request.environ: + return request.environ['wsgi.file_wrapper'](out) + elif hasattr(out, 'close') or not hasattr(out, '__iter__'): + return WSGIFileWrapper(out) + + # Handle Iterables. We peek into them to detect their inner type. + try: + out = iter(out) + first = next(out) + while not first: + first = next(out) + except StopIteration: + return self._cast('') + except HTTPResponse: + first = _e() + except (KeyboardInterrupt, SystemExit, MemoryError): + raise + except Exception: + if not self.catchall: raise + first = HTTPError(500, 'Unhandled exception', _e(), format_exc()) + + # These are the inner types allowed in iterator or generator objects. + if isinstance(first, HTTPResponse): + return self._cast(first) + if isinstance(first, bytes): + return itertools.chain([first], out) + if isinstance(first, unicode): + return imap(lambda x: x.encode(response.charset), + itertools.chain([first], out)) + return self._cast(HTTPError(500, 'Unsupported response type: %s'\ + % type(first))) + + def wsgi(self, environ, start_response): + """ The bottle WSGI-interface. """ + try: + out = self._cast(self._handle(environ)) + # rfc2616 section 4.3 + if response._status_code in (100, 101, 204, 304)\ + or environ['REQUEST_METHOD'] == 'HEAD': + if hasattr(out, 'close'): out.close() + out = [] + start_response(response._status_line, response.headerlist) + return out + except (KeyboardInterrupt, SystemExit, MemoryError): + raise + except Exception: + if not self.catchall: raise + err = '

        Critical error while processing request: %s

        ' \ + % html_escape(environ.get('PATH_INFO', '/')) + if DEBUG: + err += '

        Error:

        \n
        \n%s\n
        \n' \ + '

        Traceback:

        \n
        \n%s\n
        \n' \ + % (html_escape(repr(_e())), html_escape(format_exc())) + environ['wsgi.errors'].write(err) + headers = [('Content-Type', 'text/html; charset=UTF-8')] + start_response('500 INTERNAL SERVER ERROR', headers) + return [tob(err)] + + def __call__(self, environ, start_response): + ''' Each instance of :class:'Bottle' is a WSGI application. ''' + return self.wsgi(environ, start_response) + + + + + + +############################################################################### +# HTTP and WSGI Tools ########################################################## +############################################################################### + + +class BaseRequest(object): + """ A wrapper for WSGI environment dictionaries that adds a lot of + convenient access methods and properties. Most of them are read-only. + + Adding new attributes to a request actually adds them to the environ + dictionary (as 'bottle.request.ext.'). This is the recommended + way to store and access request-specific data. + """ + + __slots__ = ('environ') + + #: Maximum size of memory buffer for :attr:`body` in bytes. + MEMFILE_MAX = 102400 + #: Maximum number pr GET or POST parameters per request + MAX_PARAMS = 100 + + def __init__(self, environ=None): + """ Wrap a WSGI environ dictionary. """ + #: The wrapped WSGI environ dictionary. This is the only real attribute. + #: All other attributes actually are read-only properties. + self.environ = {} if environ is None else environ + self.environ['bottle.request'] = self + + @DictProperty('environ', 'bottle.app', read_only=True) + def app(self): + ''' Bottle application handling this request. ''' + raise RuntimeError('This request is not connected to an application.') + + @property + def path(self): + ''' The value of ``PATH_INFO`` with exactly one prefixed slash (to fix + broken clients and avoid the "empty path" edge case). ''' + return '/' + self.environ.get('PATH_INFO','').lstrip('/') + + @property + def method(self): + ''' The ``REQUEST_METHOD`` value as an uppercase string. ''' + return self.environ.get('REQUEST_METHOD', 'GET').upper() + + @DictProperty('environ', 'bottle.request.headers', read_only=True) + def headers(self): + ''' A :class:`WSGIHeaderDict` that provides case-insensitive access to + HTTP request headers. ''' + return WSGIHeaderDict(self.environ) + + def get_header(self, name, default=None): + ''' Return the value of a request header, or a given default value. ''' + return self.headers.get(name, default) + + @DictProperty('environ', 'bottle.request.cookies', read_only=True) + def cookies(self): + """ Cookies parsed into a :class:`FormsDict`. Signed cookies are NOT + decoded. Use :meth:`get_cookie` if you expect signed cookies. """ + cookies = SimpleCookie(self.environ.get('HTTP_COOKIE','')) + cookies = list(cookies.values())[:self.MAX_PARAMS] + return FormsDict((c.key, c.value) for c in cookies) + + def get_cookie(self, key, default=None, secret=None): + """ Return the content of a cookie. To read a `Signed Cookie`, the + `secret` must match the one used to create the cookie (see + :meth:`BaseResponse.set_cookie`). If anything goes wrong (missing + cookie or wrong signature), return a default value. """ + value = self.cookies.get(key) + if secret and value: + dec = cookie_decode(value, secret) # (key, value) tuple or None + return dec[1] if dec and dec[0] == key else default + return value or default + + @DictProperty('environ', 'bottle.request.query', read_only=True) + def query(self): + ''' The :attr:`query_string` parsed into a :class:`FormsDict`. These + values are sometimes called "URL arguments" or "GET parameters", but + not to be confused with "URL wildcards" as they are provided by the + :class:`Router`. ''' + get = self.environ['bottle.get'] = FormsDict() + pairs = _parse_qsl(self.environ.get('QUERY_STRING', '')) + for key, value in pairs[:self.MAX_PARAMS]: + get[key] = value + return get + + @DictProperty('environ', 'bottle.request.forms', read_only=True) + def forms(self): + """ Form values parsed from an `url-encoded` or `multipart/form-data` + encoded POST or PUT request body. The result is retuned as a + :class:`FormsDict`. All keys and values are strings. File uploads + are stored separately in :attr:`files`. """ + forms = FormsDict() + for name, item in self.POST.allitems(): + if not hasattr(item, 'filename'): + forms[name] = item + return forms + + @DictProperty('environ', 'bottle.request.params', read_only=True) + def params(self): + """ A :class:`FormsDict` with the combined values of :attr:`query` and + :attr:`forms`. File uploads are stored in :attr:`files`. """ + params = FormsDict() + for key, value in self.query.allitems(): + params[key] = value + for key, value in self.forms.allitems(): + params[key] = value + return params + + @DictProperty('environ', 'bottle.request.files', read_only=True) + def files(self): + """ File uploads parsed from an `url-encoded` or `multipart/form-data` + encoded POST or PUT request body. The values are instances of + :class:`cgi.FieldStorage`. The most important attributes are: + + filename + The filename, if specified; otherwise None; this is the client + side filename, *not* the file name on which it is stored (that's + a temporary file you don't deal with) + file + The file(-like) object from which you can read the data. + value + The value as a *string*; for file uploads, this transparently + reads the file every time you request the value. Do not do this + on big files. + """ + files = FormsDict() + for name, item in self.POST.allitems(): + if hasattr(item, 'filename'): + files[name] = item + return files + + @DictProperty('environ', 'bottle.request.json', read_only=True) + def json(self): + ''' If the ``Content-Type`` header is ``application/json``, this + property holds the parsed content of the request body. Only requests + smaller than :attr:`MEMFILE_MAX` are processed to avoid memory + exhaustion. ''' + if 'application/json' in self.environ.get('CONTENT_TYPE', '') \ + and 0 < self.content_length < self.MEMFILE_MAX: + return json_loads(self.body.read(self.MEMFILE_MAX)) + return None + + @DictProperty('environ', 'bottle.request.body', read_only=True) + def _body(self): + maxread = max(0, self.content_length) + stream = self.environ['wsgi.input'] + body = BytesIO() if maxread < self.MEMFILE_MAX else TemporaryFile(mode='w+b') + while maxread > 0: + part = stream.read(min(maxread, self.MEMFILE_MAX)) + if not part: break + body.write(part) + maxread -= len(part) + self.environ['wsgi.input'] = body + body.seek(0) + return body + + @property + def body(self): + """ The HTTP request body as a seek-able file-like object. Depending on + :attr:`MEMFILE_MAX`, this is either a temporary file or a + :class:`io.BytesIO` instance. Accessing this property for the first + time reads and replaces the ``wsgi.input`` environ variable. + Subsequent accesses just do a `seek(0)` on the file object. """ + self._body.seek(0) + return self._body + + #: An alias for :attr:`query`. + GET = query + + @DictProperty('environ', 'bottle.request.post', read_only=True) + def POST(self): + """ The values of :attr:`forms` and :attr:`files` combined into a single + :class:`FormsDict`. Values are either strings (form values) or + instances of :class:`cgi.FieldStorage` (file uploads). + """ + post = FormsDict() + # We default to application/x-www-form-urlencoded for everything that + # is not multipart and take the fast path (also: 3.1 workaround) + if not self.content_type.startswith('multipart/'): + maxlen = max(0, min(self.content_length, self.MEMFILE_MAX)) + pairs = _parse_qsl(tonat(self.body.read(maxlen), 'latin1')) + for key, value in pairs[:self.MAX_PARAMS]: + post[key] = value + return post + + safe_env = {'QUERY_STRING':''} # Build a safe environment for cgi + for key in ('REQUEST_METHOD', 'CONTENT_TYPE', 'CONTENT_LENGTH'): + if key in self.environ: safe_env[key] = self.environ[key] + args = dict(fp=self.body, environ=safe_env, keep_blank_values=True) + if py31: + args['fp'] = NCTextIOWrapper(args['fp'], encoding='ISO-8859-1', + newline='\n') + elif py3k: + args['encoding'] = 'ISO-8859-1' + data = FieldStorage(**args) + for item in (data.list or [])[:self.MAX_PARAMS]: + post[item.name] = item if item.filename else item.value + return post + + @property + def COOKIES(self): + ''' Alias for :attr:`cookies` (deprecated). ''' + depr('BaseRequest.COOKIES was renamed to BaseRequest.cookies (lowercase).') + return self.cookies + + @property + def url(self): + """ The full request URI including hostname and scheme. If your app + lives behind a reverse proxy or load balancer and you get confusing + results, make sure that the ``X-Forwarded-Host`` header is set + correctly. """ + return self.urlparts.geturl() + + @DictProperty('environ', 'bottle.request.urlparts', read_only=True) + def urlparts(self): + ''' The :attr:`url` string as an :class:`urlparse.SplitResult` tuple. + The tuple contains (scheme, host, path, query_string and fragment), + but the fragment is always empty because it is not visible to the + server. ''' + env = self.environ + http = env.get('HTTP_X_FORWARDED_PROTO') or env.get('wsgi.url_scheme', 'http') + host = env.get('HTTP_X_FORWARDED_HOST') or env.get('HTTP_HOST') + if not host: + # HTTP 1.1 requires a Host-header. This is for HTTP/1.0 clients. + host = env.get('SERVER_NAME', '127.0.0.1') + port = env.get('SERVER_PORT') + if port and port != ('80' if http == 'http' else '443'): + host += ':' + port + path = urlquote(self.fullpath) + return UrlSplitResult(http, host, path, env.get('QUERY_STRING'), '') + + @property + def fullpath(self): + """ Request path including :attr:`script_name` (if present). """ + return urljoin(self.script_name, self.path.lstrip('/')) + + @property + def query_string(self): + """ The raw :attr:`query` part of the URL (everything in between ``?`` + and ``#``) as a string. """ + return self.environ.get('QUERY_STRING', '') + + @property + def script_name(self): + ''' The initial portion of the URL's `path` that was removed by a higher + level (server or routing middleware) before the application was + called. This script path is returned with leading and tailing + slashes. ''' + script_name = self.environ.get('SCRIPT_NAME', '').strip('/') + return '/' + script_name + '/' if script_name else '/' + + def path_shift(self, shift=1): + ''' Shift path segments from :attr:`path` to :attr:`script_name` and + vice versa. + + :param shift: The number of path segments to shift. May be negative + to change the shift direction. (default: 1) + ''' + script = self.environ.get('SCRIPT_NAME','/') + self['SCRIPT_NAME'], self['PATH_INFO'] = path_shift(script, self.path, shift) + + @property + def content_length(self): + ''' The request body length as an integer. The client is responsible to + set this header. Otherwise, the real length of the body is unknown + and -1 is returned. In this case, :attr:`body` will be empty. ''' + return int(self.environ.get('CONTENT_LENGTH') or -1) + + @property + def content_type(self): + ''' The Content-Type header as a lowercase-string (default: empty). ''' + return self.environ.get('CONTENT_TYPE', '').lower() + + @property + def is_xhr(self): + ''' True if the request was triggered by a XMLHttpRequest. This only + works with JavaScript libraries that support the `X-Requested-With` + header (most of the popular libraries do). ''' + requested_with = self.environ.get('HTTP_X_REQUESTED_WITH','') + return requested_with.lower() == 'xmlhttprequest' + + @property + def is_ajax(self): + ''' Alias for :attr:`is_xhr`. "Ajax" is not the right term. ''' + return self.is_xhr + + @property + def auth(self): + """ HTTP authentication data as a (user, password) tuple. This + implementation currently supports basic (not digest) authentication + only. If the authentication happened at a higher level (e.g. in the + front web-server or a middleware), the password field is None, but + the user field is looked up from the ``REMOTE_USER`` environ + variable. On any errors, None is returned. """ + basic = parse_auth(self.environ.get('HTTP_AUTHORIZATION','')) + if basic: return basic + ruser = self.environ.get('REMOTE_USER') + if ruser: return (ruser, None) + return None + + @property + def remote_route(self): + """ A list of all IPs that were involved in this request, starting with + the client IP and followed by zero or more proxies. This does only + work if all proxies support the ```X-Forwarded-For`` header. Note + that this information can be forged by malicious clients. """ + proxy = self.environ.get('HTTP_X_FORWARDED_FOR') + if proxy: return [ip.strip() for ip in proxy.split(',')] + remote = self.environ.get('REMOTE_ADDR') + return [remote] if remote else [] + + @property + def remote_addr(self): + """ The client IP as a string. Note that this information can be forged + by malicious clients. """ + route = self.remote_route + return route[0] if route else None + + def copy(self): + """ Return a new :class:`Request` with a shallow :attr:`environ` copy. """ + return Request(self.environ.copy()) + + def get(self, value, default=None): return self.environ.get(value, default) + def __getitem__(self, key): return self.environ[key] + def __delitem__(self, key): self[key] = ""; del(self.environ[key]) + def __iter__(self): return iter(self.environ) + def __len__(self): return len(self.environ) + def keys(self): return self.environ.keys() + def __setitem__(self, key, value): + """ Change an environ value and clear all caches that depend on it. """ + + if self.environ.get('bottle.request.readonly'): + raise KeyError('The environ dictionary is read-only.') + + self.environ[key] = value + todelete = () + + if key == 'wsgi.input': + todelete = ('body', 'forms', 'files', 'params', 'post', 'json') + elif key == 'QUERY_STRING': + todelete = ('query', 'params') + elif key.startswith('HTTP_'): + todelete = ('headers', 'cookies') + + for key in todelete: + self.environ.pop('bottle.request.'+key, None) + + def __repr__(self): + return '<%s: %s %s>' % (self.__class__.__name__, self.method, self.url) + + def __getattr__(self, name): + ''' Search in self.environ for additional user defined attributes. ''' + try: + var = self.environ['bottle.request.ext.%s'%name] + return var.__get__(self) if hasattr(var, '__get__') else var + except KeyError: + raise AttributeError('Attribute %r not defined.' % name) + + def __setattr__(self, name, value): + if name == 'environ': return object.__setattr__(self, name, value) + self.environ['bottle.request.ext.%s'%name] = value + + + + +def _hkey(s): + return s.title().replace('_','-') + + +class HeaderProperty(object): + def __init__(self, name, reader=None, writer=str, default=''): + self.name, self.default = name, default + self.reader, self.writer = reader, writer + self.__doc__ = 'Current value of the %r header.' % name.title() + + def __get__(self, obj, cls): + if obj is None: return self + value = obj.headers.get(self.name, self.default) + return self.reader(value) if self.reader else value + + def __set__(self, obj, value): + obj.headers[self.name] = self.writer(value) + + def __delete__(self, obj): + del obj.headers[self.name] + + +class BaseResponse(object): + """ Storage class for a response body as well as headers and cookies. + + This class does support dict-like case-insensitive item-access to + headers, but is NOT a dict. Most notably, iterating over a response + yields parts of the body and not the headers. + """ + + default_status = 200 + default_content_type = 'text/html; charset=UTF-8' + + # Header blacklist for specific response codes + # (rfc2616 section 10.2.3 and 10.3.5) + bad_headers = { + 204: set(('Content-Type',)), + 304: set(('Allow', 'Content-Encoding', 'Content-Language', + 'Content-Length', 'Content-Range', 'Content-Type', + 'Content-Md5', 'Last-Modified'))} + + def __init__(self, body='', status=None, **headers): + self._cookies = None + self._headers = {'Content-Type': [self.default_content_type]} + self.body = body + self.status = status or self.default_status + if headers: + for name, value in headers.items(): + self[name] = value + + def copy(self): + ''' Returns a copy of self. ''' + copy = Response() + copy.status = self.status + copy._headers = dict((k, v[:]) for (k, v) in self._headers.items()) + return copy + + def __iter__(self): + return iter(self.body) + + def close(self): + if hasattr(self.body, 'close'): + self.body.close() + + @property + def status_line(self): + ''' The HTTP status line as a string (e.g. ``404 Not Found``).''' + return self._status_line + + @property + def status_code(self): + ''' The HTTP status code as an integer (e.g. 404).''' + return self._status_code + + def _set_status(self, status): + if isinstance(status, int): + code, status = status, _HTTP_STATUS_LINES.get(status) + elif ' ' in status: + status = status.strip() + code = int(status.split()[0]) + else: + raise ValueError('String status line without a reason phrase.') + if not 100 <= code <= 999: raise ValueError('Status code out of range.') + self._status_code = code + self._status_line = str(status or ('%d Unknown' % code)) + + def _get_status(self): + return self._status_line + + status = property(_get_status, _set_status, None, + ''' A writeable property to change the HTTP response status. It accepts + either a numeric code (100-999) or a string with a custom reason + phrase (e.g. "404 Brain not found"). Both :data:`status_line` and + :data:`status_code` are updated accordingly. The return value is + always a status string. ''') + del _get_status, _set_status + + @property + def headers(self): + ''' An instance of :class:`HeaderDict`, a case-insensitive dict-like + view on the response headers. ''' + hdict = HeaderDict() + hdict.dict = self._headers + return hdict + + def __contains__(self, name): return _hkey(name) in self._headers + def __delitem__(self, name): del self._headers[_hkey(name)] + def __getitem__(self, name): return self._headers[_hkey(name)][-1] + def __setitem__(self, name, value): self._headers[_hkey(name)] = [str(value)] + + def get_header(self, name, default=None): + ''' Return the value of a previously defined header. If there is no + header with that name, return a default value. ''' + return self._headers.get(_hkey(name), [default])[-1] + + def set_header(self, name, value): + ''' Create a new response header, replacing any previously defined + headers with the same name. ''' + self._headers[_hkey(name)] = [str(value)] + + def add_header(self, name, value): + ''' Add an additional response header, not removing duplicates. ''' + self._headers.setdefault(_hkey(name), []).append(str(value)) + + def iter_headers(self): + ''' Yield (header, value) tuples, skipping headers that are not + allowed with the current response status code. ''' + return self.headerlist + + def wsgiheader(self): + depr('The wsgiheader method is deprecated. See headerlist.') #0.10 + return self.headerlist + + @property + def headerlist(self): + ''' WSGI conform list of (header, value) tuples. ''' + out = [] + headers = self._headers.items() + if self._status_code in self.bad_headers: + bad_headers = self.bad_headers[self._status_code] + headers = [h for h in headers if h[0] not in bad_headers] + out += [(name, val) for name, vals in headers for val in vals] + if self._cookies: + for c in self._cookies.values(): + out.append(('Set-Cookie', c.OutputString())) + return out + + content_type = HeaderProperty('Content-Type') + content_length = HeaderProperty('Content-Length', reader=int) + + @property + def charset(self): + """ Return the charset specified in the content-type header (default: utf8). """ + if 'charset=' in self.content_type: + return self.content_type.split('charset=')[-1].split(';')[0].strip() + return 'UTF-8' + + @property + def COOKIES(self): + """ A dict-like SimpleCookie instance. This should not be used directly. + See :meth:`set_cookie`. """ + depr('The COOKIES dict is deprecated. Use `set_cookie()` instead.') # 0.10 + if not self._cookies: + self._cookies = SimpleCookie() + return self._cookies + + def set_cookie(self, name, value, secret=None, **options): + ''' Create a new cookie or replace an old one. If the `secret` parameter is + set, create a `Signed Cookie` (described below). + + :param name: the name of the cookie. + :param value: the value of the cookie. + :param secret: a signature key required for signed cookies. + + Additionally, this method accepts all RFC 2109 attributes that are + supported by :class:`cookie.Morsel`, including: + + :param max_age: maximum age in seconds. (default: None) + :param expires: a datetime object or UNIX timestamp. (default: None) + :param domain: the domain that is allowed to read the cookie. + (default: current domain) + :param path: limits the cookie to a given path (default: current path) + :param secure: limit the cookie to HTTPS connections (default: off). + :param httponly: prevents client-side javascript to read this cookie + (default: off, requires Python 2.6 or newer). + + If neither `expires` nor `max_age` is set (default), the cookie will + expire at the end of the browser session (as soon as the browser + window is closed). + + Signed cookies may store any pickle-able object and are + cryptographically signed to prevent manipulation. Keep in mind that + cookies are limited to 4kb in most browsers. + + Warning: Signed cookies are not encrypted (the client can still see + the content) and not copy-protected (the client can restore an old + cookie). The main intention is to make pickling and unpickling + save, not to store secret information at client side. + ''' + if not self._cookies: + self._cookies = SimpleCookie() + + if secret: + value = touni(cookie_encode((name, value), secret)) + elif not isinstance(value, basestring): + raise TypeError('Secret key missing for non-string Cookie.') + + if len(value) > 4096: raise ValueError('Cookie value to long.') + self._cookies[name] = value + + for key, value in options.items(): + if key == 'max_age': + if isinstance(value, timedelta): + value = value.seconds + value.days * 24 * 3600 + if key == 'expires': + if isinstance(value, (datedate, datetime)): + value = value.timetuple() + elif isinstance(value, (int, float)): + value = time.gmtime(value) + value = time.strftime("%a, %d %b %Y %H:%M:%S GMT", value) + self._cookies[name][key.replace('_', '-')] = value + + def delete_cookie(self, key, **kwargs): + ''' Delete a cookie. Be sure to use the same `domain` and `path` + settings as used to create the cookie. ''' + kwargs['max_age'] = -1 + kwargs['expires'] = 0 + self.set_cookie(key, '', **kwargs) + + def __repr__(self): + out = '' + for name, value in self.headerlist: + out += '%s: %s\n' % (name.title(), value.strip()) + return out + +#: Thread-local storage for :class:`LocalRequest` and :class:`LocalResponse` +#: attributes. +_lctx = threading.local() + +def local_property(name): + def fget(self): + try: + return getattr(_lctx, name) + except AttributeError: + raise RuntimeError("Request context not initialized.") + def fset(self, value): setattr(_lctx, name, value) + def fdel(self): delattr(_lctx, name) + return property(fget, fset, fdel, + 'Thread-local property stored in :data:`_lctx.%s`' % name) + + +class LocalRequest(BaseRequest): + ''' A thread-local subclass of :class:`BaseRequest` with a different + set of attribues for each thread. There is usually only one global + instance of this class (:data:`request`). If accessed during a + request/response cycle, this instance always refers to the *current* + request (even on a multithreaded server). ''' + bind = BaseRequest.__init__ + environ = local_property('request_environ') + + +class LocalResponse(BaseResponse): + ''' A thread-local subclass of :class:`BaseResponse` with a different + set of attribues for each thread. There is usually only one global + instance of this class (:data:`response`). Its attributes are used + to build the HTTP response at the end of the request/response cycle. + ''' + bind = BaseResponse.__init__ + _status_line = local_property('response_status_line') + _status_code = local_property('response_status_code') + _cookies = local_property('response_cookies') + _headers = local_property('response_headers') + body = local_property('response_body') + +Request = BaseRequest +Response = BaseResponse + +class HTTPResponse(Response, BottleException): + def __init__(self, body='', status=None, header=None, **headers): + if header or 'output' in headers: + depr('Call signature changed (for the better)') + if header: headers.update(header) + if 'output' in headers: body = headers.pop('output') + super(HTTPResponse, self).__init__(body, status, **headers) + + def apply(self, response): + response._status_code = self._status_code + response._status_line = self._status_line + response._headers = self._headers + response._cookies = self._cookies + response.body = self.body + + def _output(self, value=None): + depr('Use HTTPResponse.body instead of HTTPResponse.output') + if value is None: return self.body + self.body = value + + output = property(_output, _output, doc='Alias for .body') + +class HTTPError(HTTPResponse): + default_status = 500 + def __init__(self, status=None, body=None, exception=None, traceback=None, header=None, **headers): + self.exception = exception + self.traceback = traceback + super(HTTPError, self).__init__(body, status, header, **headers) + + + + + +############################################################################### +# Plugins ###################################################################### +############################################################################### + +class PluginError(BottleException): pass + +class JSONPlugin(object): + name = 'json' + api = 2 + + def __init__(self, json_dumps=json_dumps): + self.json_dumps = json_dumps + + def apply(self, callback, route): + dumps = self.json_dumps + if not dumps: return callback + def wrapper(*a, **ka): + rv = callback(*a, **ka) + if isinstance(rv, dict): + #Attempt to serialize, raises exception on failure + json_response = dumps(rv) + #Set content type only if serialization succesful + response.content_type = 'application/json' + return json_response + return rv + return wrapper + + +class HooksPlugin(object): + name = 'hooks' + api = 2 + + _names = 'before_request', 'after_request', 'app_reset' + + def __init__(self): + self.hooks = dict((name, []) for name in self._names) + self.app = None + + def _empty(self): + return not (self.hooks['before_request'] or self.hooks['after_request']) + + def setup(self, app): + self.app = app + + def add(self, name, func): + ''' Attach a callback to a hook. ''' + was_empty = self._empty() + self.hooks.setdefault(name, []).append(func) + if self.app and was_empty and not self._empty(): self.app.reset() + + def remove(self, name, func): + ''' Remove a callback from a hook. ''' + was_empty = self._empty() + if name in self.hooks and func in self.hooks[name]: + self.hooks[name].remove(func) + if self.app and not was_empty and self._empty(): self.app.reset() + + def trigger(self, name, *a, **ka): + ''' Trigger a hook and return a list of results. ''' + hooks = self.hooks[name] + if ka.pop('reversed', False): hooks = hooks[::-1] + return [hook(*a, **ka) for hook in hooks] + + def apply(self, callback, route): + if self._empty(): return callback + def wrapper(*a, **ka): + self.trigger('before_request') + rv = callback(*a, **ka) + self.trigger('after_request', reversed=True) + return rv + return wrapper + + +class TemplatePlugin(object): + ''' This plugin applies the :func:`view` decorator to all routes with a + `template` config parameter. If the parameter is a tuple, the second + element must be a dict with additional options (e.g. `template_engine`) + or default variables for the template. ''' + name = 'template' + api = 2 + + def apply(self, callback, route): + conf = route.config.get('template') + if isinstance(conf, (tuple, list)) and len(conf) == 2: + return view(conf[0], **conf[1])(callback) + elif isinstance(conf, str) and 'template_opts' in route.config: + depr('The `template_opts` parameter is deprecated.') #0.9 + return view(conf, **route.config['template_opts'])(callback) + elif isinstance(conf, str): + return view(conf)(callback) + else: + return callback + + +#: Not a plugin, but part of the plugin API. TODO: Find a better place. +class _ImportRedirect(object): + def __init__(self, name, impmask): + ''' Create a virtual package that redirects imports (see PEP 302). ''' + self.name = name + self.impmask = impmask + self.module = sys.modules.setdefault(name, imp.new_module(name)) + self.module.__dict__.update({'__file__': __file__, '__path__': [], + '__all__': [], '__loader__': self}) + sys.meta_path.append(self) + + def find_module(self, fullname, path=None): + if '.' not in fullname: return + packname, modname = fullname.rsplit('.', 1) + if packname != self.name: return + return self + + def load_module(self, fullname): + if fullname in sys.modules: return sys.modules[fullname] + packname, modname = fullname.rsplit('.', 1) + realname = self.impmask % modname + __import__(realname) + module = sys.modules[fullname] = sys.modules[realname] + setattr(self.module, modname, module) + module.__loader__ = self + return module + + + + + + +############################################################################### +# Common Utilities ############################################################# +############################################################################### + + +class MultiDict(DictMixin): + """ This dict stores multiple values per key, but behaves exactly like a + normal dict in that it returns only the newest value for any given key. + There are special methods available to access the full list of values. + """ + + def __init__(self, *a, **k): + self.dict = dict((k, [v]) for (k, v) in dict(*a, **k).items()) + + def __len__(self): return len(self.dict) + def __iter__(self): return iter(self.dict) + def __contains__(self, key): return key in self.dict + def __delitem__(self, key): del self.dict[key] + def __getitem__(self, key): return self.dict[key][-1] + def __setitem__(self, key, value): self.append(key, value) + def keys(self): return self.dict.keys() + + if py3k: + def values(self): return (v[-1] for v in self.dict.values()) + def items(self): return ((k, v[-1]) for k, v in self.dict.items()) + def allitems(self): + return ((k, v) for k, vl in self.dict.items() for v in vl) + iterkeys = keys + itervalues = values + iteritems = items + iterallitems = allitems + + else: + def values(self): return [v[-1] for v in self.dict.values()] + def items(self): return [(k, v[-1]) for k, v in self.dict.items()] + def iterkeys(self): return self.dict.iterkeys() + def itervalues(self): return (v[-1] for v in self.dict.itervalues()) + def iteritems(self): + return ((k, v[-1]) for k, v in self.dict.iteritems()) + def iterallitems(self): + return ((k, v) for k, vl in self.dict.iteritems() for v in vl) + def allitems(self): + return [(k, v) for k, vl in self.dict.iteritems() for v in vl] + + def get(self, key, default=None, index=-1, type=None): + ''' Return the most recent value for a key. + + :param default: The default value to be returned if the key is not + present or the type conversion fails. + :param index: An index for the list of available values. + :param type: If defined, this callable is used to cast the value + into a specific type. Exception are suppressed and result in + the default value to be returned. + ''' + try: + val = self.dict[key][index] + return type(val) if type else val + except Exception: + pass + return default + + def append(self, key, value): + ''' Add a new value to the list of values for this key. ''' + self.dict.setdefault(key, []).append(value) + + def replace(self, key, value): + ''' Replace the list of values with a single value. ''' + self.dict[key] = [value] + + def getall(self, key): + ''' Return a (possibly empty) list of values for a key. ''' + return self.dict.get(key) or [] + + #: Aliases for WTForms to mimic other multi-dict APIs (Django) + getone = get + getlist = getall + + + +class FormsDict(MultiDict): + ''' This :class:`MultiDict` subclass is used to store request form data. + Additionally to the normal dict-like item access methods (which return + unmodified data as native strings), this container also supports + attribute-like access to its values. Attributes are automatically de- + or recoded to match :attr:`input_encoding` (default: 'utf8'). Missing + attributes default to an empty string. ''' + + #: Encoding used for attribute values. + input_encoding = 'utf8' + #: If true (default), unicode strings are first encoded with `latin1` + #: and then decoded to match :attr:`input_encoding`. + recode_unicode = True + + def _fix(self, s, encoding=None): + if isinstance(s, unicode) and self.recode_unicode: # Python 3 WSGI + s = s.encode('latin1') + if isinstance(s, bytes): # Python 2 WSGI + return s.decode(encoding or self.input_encoding) + return s + + def decode(self, encoding=None): + ''' Returns a copy with all keys and values de- or recoded to match + :attr:`input_encoding`. Some libraries (e.g. WTForms) want a + unicode dictionary. ''' + copy = FormsDict() + enc = copy.input_encoding = encoding or self.input_encoding + copy.recode_unicode = False + for key, value in self.allitems(): + copy.append(self._fix(key, enc), self._fix(value, enc)) + return copy + + def getunicode(self, name, default=None, encoding=None): + try: + return self._fix(self[name], encoding) + except (UnicodeError, KeyError): + return default + + def __getattr__(self, name, default=unicode()): + # Without this guard, pickle generates a cryptic TypeError: + if name.startswith('__') and name.endswith('__'): + return super(FormsDict, self).__getattr__(name) + return self.getunicode(name, default=default) + + +class HeaderDict(MultiDict): + """ A case-insensitive version of :class:`MultiDict` that defaults to + replace the old value instead of appending it. """ + + def __init__(self, *a, **ka): + self.dict = {} + if a or ka: self.update(*a, **ka) + + def __contains__(self, key): return _hkey(key) in self.dict + def __delitem__(self, key): del self.dict[_hkey(key)] + def __getitem__(self, key): return self.dict[_hkey(key)][-1] + def __setitem__(self, key, value): self.dict[_hkey(key)] = [str(value)] + def append(self, key, value): + self.dict.setdefault(_hkey(key), []).append(str(value)) + def replace(self, key, value): self.dict[_hkey(key)] = [str(value)] + def getall(self, key): return self.dict.get(_hkey(key)) or [] + def get(self, key, default=None, index=-1): + return MultiDict.get(self, _hkey(key), default, index) + def filter(self, names): + for name in [_hkey(n) for n in names]: + if name in self.dict: + del self.dict[name] + + +class WSGIHeaderDict(DictMixin): + ''' This dict-like class wraps a WSGI environ dict and provides convenient + access to HTTP_* fields. Keys and values are native strings + (2.x bytes or 3.x unicode) and keys are case-insensitive. If the WSGI + environment contains non-native string values, these are de- or encoded + using a lossless 'latin1' character set. + + The API will remain stable even on changes to the relevant PEPs. + Currently PEP 333, 444 and 3333 are supported. (PEP 444 is the only one + that uses non-native strings.) + ''' + #: List of keys that do not have a ``HTTP_`` prefix. + cgikeys = ('CONTENT_TYPE', 'CONTENT_LENGTH') + + def __init__(self, environ): + self.environ = environ + + def _ekey(self, key): + ''' Translate header field name to CGI/WSGI environ key. ''' + key = key.replace('-','_').upper() + if key in self.cgikeys: + return key + return 'HTTP_' + key + + def raw(self, key, default=None): + ''' Return the header value as is (may be bytes or unicode). ''' + return self.environ.get(self._ekey(key), default) + + def __getitem__(self, key): + return tonat(self.environ[self._ekey(key)], 'latin1') + + def __setitem__(self, key, value): + raise TypeError("%s is read-only." % self.__class__) + + def __delitem__(self, key): + raise TypeError("%s is read-only." % self.__class__) + + def __iter__(self): + for key in self.environ: + if key[:5] == 'HTTP_': + yield key[5:].replace('_', '-').title() + elif key in self.cgikeys: + yield key.replace('_', '-').title() + + def keys(self): return [x for x in self] + def __len__(self): return len(self.keys()) + def __contains__(self, key): return self._ekey(key) in self.environ + + +class ConfigDict(dict): + ''' A dict-subclass with some extras: You can access keys like attributes. + Uppercase attributes create new ConfigDicts and act as name-spaces. + Other missing attributes return None. Calling a ConfigDict updates its + values and returns itself. + + >>> cfg = ConfigDict() + >>> cfg.Namespace.value = 5 + >>> cfg.OtherNamespace(a=1, b=2) + >>> cfg + {'Namespace': {'value': 5}, 'OtherNamespace': {'a': 1, 'b': 2}} + ''' + + def __getattr__(self, key): + if key not in self and key[0].isupper(): + self[key] = ConfigDict() + return self.get(key) + + def __setattr__(self, key, value): + if hasattr(dict, key): + raise AttributeError('Read-only attribute.') + if key in self and self[key] and isinstance(self[key], ConfigDict): + raise AttributeError('Non-empty namespace attribute.') + self[key] = value + + def __delattr__(self, key): + if key in self: del self[key] + + def __call__(self, *a, **ka): + for key, value in dict(*a, **ka).items(): setattr(self, key, value) + return self + + +class AppStack(list): + """ A stack-like list. Calling it returns the head of the stack. """ + + def __call__(self): + """ Return the current default application. """ + return self[-1] + + def push(self, value=None): + """ Add a new :class:`Bottle` instance to the stack """ + if not isinstance(value, Bottle): + value = Bottle() + self.append(value) + return value + + +class WSGIFileWrapper(object): + + def __init__(self, fp, buffer_size=1024*64): + self.fp, self.buffer_size = fp, buffer_size + for attr in ('fileno', 'close', 'read', 'readlines', 'tell', 'seek'): + if hasattr(fp, attr): setattr(self, attr, getattr(fp, attr)) + + def __iter__(self): + buff, read = self.buffer_size, self.read + while True: + part = read(buff) + if not part: return + yield part + + +class ResourceManager(object): + ''' This class manages a list of search paths and helps to find and open + application-bound resources (files). + + :param base: default value for :meth:`add_path` calls. + :param opener: callable used to open resources. + :param cachemode: controls which lookups are cached. One of 'all', + 'found' or 'none'. + ''' + + def __init__(self, base='./', opener=open, cachemode='all'): + self.opener = open + self.base = base + self.cachemode = cachemode + + #: A list of search paths. See :meth:`add_path` for details. + self.path = [] + #: A cache for resolved paths. ``res.cache.clear()`` clears the cache. + self.cache = {} + + def add_path(self, path, base=None, index=None, create=False): + ''' Add a new path to the list of search paths. Return False if the + path does not exist. + + :param path: The new search path. Relative paths are turned into + an absolute and normalized form. If the path looks like a file + (not ending in `/`), the filename is stripped off. + :param base: Path used to absolutize relative search paths. + Defaults to :attr:`base` which defaults to ``os.getcwd()``. + :param index: Position within the list of search paths. Defaults + to last index (appends to the list). + + The `base` parameter makes it easy to reference files installed + along with a python module or package:: + + res.add_path('./resources/', __file__) + ''' + base = os.path.abspath(os.path.dirname(base or self.base)) + path = os.path.abspath(os.path.join(base, os.path.dirname(path))) + path += os.sep + if path in self.path: + self.path.remove(path) + if create and not os.path.isdir(path): + os.makedirs(path) + if index is None: + self.path.append(path) + else: + self.path.insert(index, path) + self.cache.clear() + return os.path.exists(path) + + def __iter__(self): + ''' Iterate over all existing files in all registered paths. ''' + search = self.path[:] + while search: + path = search.pop() + if not os.path.isdir(path): continue + for name in os.listdir(path): + full = os.path.join(path, name) + if os.path.isdir(full): search.append(full) + else: yield full + + def lookup(self, name): + ''' Search for a resource and return an absolute file path, or `None`. + + The :attr:`path` list is searched in order. The first match is + returend. Symlinks are followed. The result is cached to speed up + future lookups. ''' + if name not in self.cache or DEBUG: + for path in self.path: + fpath = os.path.join(path, name) + if os.path.isfile(fpath): + if self.cachemode in ('all', 'found'): + self.cache[name] = fpath + return fpath + if self.cachemode == 'all': + self.cache[name] = None + return self.cache[name] + + def open(self, name, mode='r', *args, **kwargs): + ''' Find a resource and return a file object, or raise IOError. ''' + fname = self.lookup(name) + if not fname: raise IOError("Resource %r not found." % name) + return self.opener(name, mode=mode, *args, **kwargs) + + + + + + +############################################################################### +# Application Helper ########################################################### +############################################################################### + + +def abort(code=500, text='Unknown Error: Application stopped.'): + """ Aborts execution and causes a HTTP error. """ + raise HTTPError(code, text) + + +def redirect(url, code=None): + """ Aborts execution and causes a 303 or 302 redirect, depending on + the HTTP protocol version. """ + if code is None: + code = 303 if request.get('SERVER_PROTOCOL') == "HTTP/1.1" else 302 + location = urljoin(request.url, url) + res = HTTPResponse("", status=code, Location=location) + if response._cookies: + res._cookies = response._cookies + raise res + + +def _file_iter_range(fp, offset, bytes, maxread=1024*1024): + ''' Yield chunks from a range in a file. No chunk is bigger than maxread.''' + fp.seek(offset) + while bytes > 0: + part = fp.read(min(bytes, maxread)) + if not part: break + bytes -= len(part) + yield part + + +def static_file(filename, root, mimetype='auto', download=False): + """ Open a file in a safe way and return :exc:`HTTPResponse` with status + code 200, 305, 401 or 404. Set Content-Type, Content-Encoding, + Content-Length and Last-Modified header. Obey If-Modified-Since header + and HEAD requests. + """ + root = os.path.abspath(root) + os.sep + filename = os.path.abspath(os.path.join(root, filename.strip('/\\'))) + headers = dict() + + if not filename.startswith(root): + return HTTPError(403, "Access denied.") + if not os.path.exists(filename) or not os.path.isfile(filename): + return HTTPError(404, "File does not exist.") + if not os.access(filename, os.R_OK): + return HTTPError(403, "You do not have permission to access this file.") + + if mimetype == 'auto': + mimetype, encoding = mimetypes.guess_type(filename) + if mimetype: headers['Content-Type'] = mimetype + if encoding: headers['Content-Encoding'] = encoding + elif mimetype: + headers['Content-Type'] = mimetype + + if download: + download = os.path.basename(filename if download == True else download) + headers['Content-Disposition'] = 'attachment; filename="%s"' % download + + stats = os.stat(filename) + headers['Content-Length'] = clen = stats.st_size + lm = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(stats.st_mtime)) + headers['Last-Modified'] = lm + + ims = request.environ.get('HTTP_IF_MODIFIED_SINCE') + if ims: + ims = parse_date(ims.split(";")[0].strip()) + if ims is not None and ims >= int(stats.st_mtime): + headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) + return HTTPResponse(status=304, **headers) + + body = '' if request.method == 'HEAD' else open(filename, 'rb') + + headers["Accept-Ranges"] = "bytes" + ranges = request.environ.get('HTTP_RANGE') + if 'HTTP_RANGE' in request.environ: + ranges = list(parse_range_header(request.environ['HTTP_RANGE'], clen)) + if not ranges: + return HTTPError(416, "Requested Range Not Satisfiable") + offset, end = ranges[0] + headers["Content-Range"] = "bytes %d-%d/%d" % (offset, end-1, clen) + headers["Content-Length"] = str(end-offset) + if body: body = _file_iter_range(body, offset, end-offset) + return HTTPResponse(body, status=206, **headers) + return HTTPResponse(body, **headers) + + + + + + +############################################################################### +# HTTP Utilities and MISC (TODO) ############################################### +############################################################################### + + +def debug(mode=True): + """ Change the debug level. + There is only one debug level supported at the moment.""" + global DEBUG + DEBUG = bool(mode) + + +def parse_date(ims): + """ Parse rfc1123, rfc850 and asctime timestamps and return UTC epoch. """ + try: + ts = email.utils.parsedate_tz(ims) + return time.mktime(ts[:8] + (0,)) - (ts[9] or 0) - time.timezone + except (TypeError, ValueError, IndexError, OverflowError): + return None + + +def parse_auth(header): + """ Parse rfc2617 HTTP authentication header string (basic) and return (user,pass) tuple or None""" + try: + method, data = header.split(None, 1) + if method.lower() == 'basic': + user, pwd = touni(base64.b64decode(tob(data))).split(':',1) + return user, pwd + except (KeyError, ValueError): + return None + +def parse_range_header(header, maxlen=0): + ''' Yield (start, end) ranges parsed from a HTTP Range header. Skip + unsatisfiable ranges. The end index is non-inclusive.''' + if not header or header[:6] != 'bytes=': return + ranges = [r.split('-', 1) for r in header[6:].split(',') if '-' in r] + for start, end in ranges: + try: + if not start: # bytes=-100 -> last 100 bytes + start, end = max(0, maxlen-int(end)), maxlen + elif not end: # bytes=100- -> all but the first 99 bytes + start, end = int(start), maxlen + else: # bytes=100-200 -> bytes 100-200 (inclusive) + start, end = int(start), min(int(end)+1, maxlen) + if 0 <= start < end <= maxlen: + yield start, end + except ValueError: + pass + +def _parse_qsl(qs): + r = [] + for pair in qs.replace(';','&').split('&'): + if not pair: continue + nv = pair.split('=', 1) + if len(nv) != 2: nv.append('') + key = urlunquote(nv[0].replace('+', ' ')) + value = urlunquote(nv[1].replace('+', ' ')) + r.append((key, value)) + return r + +def _lscmp(a, b): + ''' Compares two strings in a cryptographically safe way: + Runtime is not affected by length of common prefix. ''' + return not sum(0 if x==y else 1 for x, y in zip(a, b)) and len(a) == len(b) + + +def cookie_encode(data, key): + ''' Encode and sign a pickle-able object. Return a (byte) string ''' + msg = base64.b64encode(pickle.dumps(data, -1)) + sig = base64.b64encode(hmac.new(tob(key), msg).digest()) + return tob('!') + sig + tob('?') + msg + + +def cookie_decode(data, key): + ''' Verify and decode an encoded string. Return an object or None.''' + data = tob(data) + if cookie_is_encoded(data): + sig, msg = data.split(tob('?'), 1) + if _lscmp(sig[1:], base64.b64encode(hmac.new(tob(key), msg).digest())): + return pickle.loads(base64.b64decode(msg)) + return None + + +def cookie_is_encoded(data): + ''' Return True if the argument looks like a encoded cookie.''' + return bool(data.startswith(tob('!')) and tob('?') in data) + + +def html_escape(string): + ''' Escape HTML special characters ``&<>`` and quotes ``'"``. ''' + return string.replace('&','&').replace('<','<').replace('>','>')\ + .replace('"','"').replace("'",''') + + +def html_quote(string): + ''' Escape and quote a string to be used as an HTTP attribute.''' + return '"%s"' % html_escape(string).replace('\n','%#10;')\ + .replace('\r',' ').replace('\t',' ') + + +def yieldroutes(func): + """ Return a generator for routes that match the signature (name, args) + of the func parameter. This may yield more than one route if the function + takes optional keyword arguments. The output is best described by example:: + + a() -> '/a' + b(x, y) -> '/b/:x/:y' + c(x, y=5) -> '/c/:x' and '/c/:x/:y' + d(x=5, y=6) -> '/d' and '/d/:x' and '/d/:x/:y' + """ + import inspect # Expensive module. Only import if necessary. + path = '/' + func.__name__.replace('__','/').lstrip('/') + spec = inspect.getargspec(func) + argc = len(spec[0]) - len(spec[3] or []) + path += ('/:%s' * argc) % tuple(spec[0][:argc]) + yield path + for arg in spec[0][argc:]: + path += '/:%s' % arg + yield path + + +def path_shift(script_name, path_info, shift=1): + ''' Shift path fragments from PATH_INFO to SCRIPT_NAME and vice versa. + + :return: The modified paths. + :param script_name: The SCRIPT_NAME path. + :param script_name: The PATH_INFO path. + :param shift: The number of path fragments to shift. May be negative to + change the shift direction. (default: 1) + ''' + if shift == 0: return script_name, path_info + pathlist = path_info.strip('/').split('/') + scriptlist = script_name.strip('/').split('/') + if pathlist and pathlist[0] == '': pathlist = [] + if scriptlist and scriptlist[0] == '': scriptlist = [] + if shift > 0 and shift <= len(pathlist): + moved = pathlist[:shift] + scriptlist = scriptlist + moved + pathlist = pathlist[shift:] + elif shift < 0 and shift >= -len(scriptlist): + moved = scriptlist[shift:] + pathlist = moved + pathlist + scriptlist = scriptlist[:shift] + else: + empty = 'SCRIPT_NAME' if shift < 0 else 'PATH_INFO' + raise AssertionError("Cannot shift. Nothing left from %s" % empty) + new_script_name = '/' + '/'.join(scriptlist) + new_path_info = '/' + '/'.join(pathlist) + if path_info.endswith('/') and pathlist: new_path_info += '/' + return new_script_name, new_path_info + + +def validate(**vkargs): + """ + Validates and manipulates keyword arguments by user defined callables. + Handles ValueError and missing arguments by raising HTTPError(403). + """ + depr('Use route wildcard filters instead.') + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kargs): + for key, value in vkargs.items(): + if key not in kargs: + abort(403, 'Missing parameter: %s' % key) + try: + kargs[key] = value(kargs[key]) + except ValueError: + abort(403, 'Wrong parameter format for: %s' % key) + return func(*args, **kargs) + return wrapper + return decorator + + +def auth_basic(check, realm="private", text="Access denied"): + ''' Callback decorator to require HTTP auth (basic). + TODO: Add route(check_auth=...) parameter. ''' + def decorator(func): + def wrapper(*a, **ka): + user, password = request.auth or (None, None) + if user is None or not check(user, password): + response.headers['WWW-Authenticate'] = 'Basic realm="%s"' % realm + return HTTPError(401, text) + return func(*a, **ka) + return wrapper + return decorator + + +# Shortcuts for common Bottle methods. +# They all refer to the current default application. + +def make_default_app_wrapper(name): + ''' Return a callable that relays calls to the current default app. ''' + @functools.wraps(getattr(Bottle, name)) + def wrapper(*a, **ka): + return getattr(app(), name)(*a, **ka) + return wrapper + +route = make_default_app_wrapper('route') +get = make_default_app_wrapper('get') +post = make_default_app_wrapper('post') +put = make_default_app_wrapper('put') +delete = make_default_app_wrapper('delete') +error = make_default_app_wrapper('error') +mount = make_default_app_wrapper('mount') +hook = make_default_app_wrapper('hook') +install = make_default_app_wrapper('install') +uninstall = make_default_app_wrapper('uninstall') +url = make_default_app_wrapper('get_url') + + + + + + + +############################################################################### +# Server Adapter ############################################################### +############################################################################### + + +class ServerAdapter(object): + quiet = False + def __init__(self, host='127.0.0.1', port=8080, **config): + self.options = config + self.host = host + self.port = int(port) + + def run(self, handler): # pragma: no cover + pass + + def __repr__(self): + args = ', '.join(['%s=%s'%(k,repr(v)) for k, v in self.options.items()]) + return "%s(%s)" % (self.__class__.__name__, args) + + +class CGIServer(ServerAdapter): + quiet = True + def run(self, handler): # pragma: no cover + from wsgiref.handlers import CGIHandler + def fixed_environ(environ, start_response): + environ.setdefault('PATH_INFO', '') + return handler(environ, start_response) + CGIHandler().run(fixed_environ) + + +class FlupFCGIServer(ServerAdapter): + def run(self, handler): # pragma: no cover + import flup.server.fcgi + self.options.setdefault('bindAddress', (self.host, self.port)) + flup.server.fcgi.WSGIServer(handler, **self.options).run() + + +class WSGIRefServer(ServerAdapter): + def run(self, handler): # pragma: no cover + from wsgiref.simple_server import make_server, WSGIRequestHandler + if self.quiet: + class QuietHandler(WSGIRequestHandler): + def log_request(*args, **kw): pass + self.options['handler_class'] = QuietHandler + srv = make_server(self.host, self.port, handler, **self.options) + srv.serve_forever() + + +class CherryPyServer(ServerAdapter): + def run(self, handler): # pragma: no cover + from cherrypy import wsgiserver + server = wsgiserver.CherryPyWSGIServer((self.host, self.port), handler) + try: + server.start() + finally: + server.stop() + + +class WaitressServer(ServerAdapter): + def run(self, handler): + from waitress import serve + serve(handler, host=self.host, port=self.port) + + +class PasteServer(ServerAdapter): + def run(self, handler): # pragma: no cover + from paste import httpserver + if not self.quiet: + from paste.translogger import TransLogger + handler = TransLogger(handler) + httpserver.serve(handler, host=self.host, port=str(self.port), + **self.options) + + +class MeinheldServer(ServerAdapter): + def run(self, handler): + from meinheld import server + server.listen((self.host, self.port)) + server.run(handler) + + +class FapwsServer(ServerAdapter): + """ Extremely fast webserver using libev. See http://www.fapws.org/ """ + def run(self, handler): # pragma: no cover + import fapws._evwsgi as evwsgi + from fapws import base, config + port = self.port + if float(config.SERVER_IDENT[-2:]) > 0.4: + # fapws3 silently changed its API in 0.5 + port = str(port) + evwsgi.start(self.host, port) + # fapws3 never releases the GIL. Complain upstream. I tried. No luck. + if 'BOTTLE_CHILD' in os.environ and not self.quiet: + _stderr("WARNING: Auto-reloading does not work with Fapws3.\n") + _stderr(" (Fapws3 breaks python thread support)\n") + evwsgi.set_base_module(base) + def app(environ, start_response): + environ['wsgi.multiprocess'] = False + return handler(environ, start_response) + evwsgi.wsgi_cb(('', app)) + evwsgi.run() + + +class TornadoServer(ServerAdapter): + """ The super hyped asynchronous server by facebook. Untested. """ + def run(self, handler): # pragma: no cover + import tornado.wsgi, tornado.httpserver, tornado.ioloop + container = tornado.wsgi.WSGIContainer(handler) + server = tornado.httpserver.HTTPServer(container) + server.listen(port=self.port) + tornado.ioloop.IOLoop.instance().start() + + +class AppEngineServer(ServerAdapter): + """ Adapter for Google App Engine. """ + quiet = True + def run(self, handler): + from google.appengine.ext.webapp import util + # A main() function in the handler script enables 'App Caching'. + # Lets makes sure it is there. This _really_ improves performance. + module = sys.modules.get('__main__') + if module and not hasattr(module, 'main'): + module.main = lambda: util.run_wsgi_app(handler) + util.run_wsgi_app(handler) + + +class TwistedServer(ServerAdapter): + """ Untested. """ + def run(self, handler): + from twisted.web import server, wsgi + from twisted.python.threadpool import ThreadPool + from twisted.internet import reactor + thread_pool = ThreadPool() + thread_pool.start() + reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop) + factory = server.Site(wsgi.WSGIResource(reactor, thread_pool, handler)) + reactor.listenTCP(self.port, factory, interface=self.host) + reactor.run() + + +class DieselServer(ServerAdapter): + """ Untested. """ + def run(self, handler): + from diesel.protocols.wsgi import WSGIApplication + app = WSGIApplication(handler, port=self.port) + app.run() + + +class GeventServer(ServerAdapter): + """ Untested. Options: + + * `fast` (default: False) uses libevent's http server, but has some + issues: No streaming, no pipelining, no SSL. + """ + def run(self, handler): + from gevent import wsgi, pywsgi, local + if not isinstance(_lctx, local.local): + msg = "Bottle requires gevent.monkey.patch_all() (before import)" + raise RuntimeError(msg) + if not self.options.get('fast'): wsgi = pywsgi + log = None if self.quiet else 'default' + wsgi.WSGIServer((self.host, self.port), handler, log=log).serve_forever() + + +class GunicornServer(ServerAdapter): + """ Untested. See http://gunicorn.org/configure.html for options. """ + def run(self, handler): + from gunicorn.app.base import Application + + config = {'bind': "%s:%d" % (self.host, int(self.port))} + config.update(self.options) + + class GunicornApplication(Application): + def init(self, parser, opts, args): + return config + + def load(self): + return handler + + GunicornApplication().run() + + +class EventletServer(ServerAdapter): + """ Untested """ + def run(self, handler): + from eventlet import wsgi, listen + try: + wsgi.server(listen((self.host, self.port)), handler, + log_output=(not self.quiet)) + except TypeError: + # Fallback, if we have old version of eventlet + wsgi.server(listen((self.host, self.port)), handler) + + +class RocketServer(ServerAdapter): + """ Untested. """ + def run(self, handler): + from rocket import Rocket + server = Rocket((self.host, self.port), 'wsgi', { 'wsgi_app' : handler }) + server.start() + + +class BjoernServer(ServerAdapter): + """ Fast server written in C: https://github.com/jonashaag/bjoern """ + def run(self, handler): + from bjoern import run + run(handler, self.host, self.port) + + +class AutoServer(ServerAdapter): + """ Untested. """ + adapters = [WaitressServer, PasteServer, TwistedServer, CherryPyServer, WSGIRefServer] + def run(self, handler): + for sa in self.adapters: + try: + return sa(self.host, self.port, **self.options).run(handler) + except ImportError: + pass + +server_names = { + 'cgi': CGIServer, + 'flup': FlupFCGIServer, + 'wsgiref': WSGIRefServer, + 'waitress': WaitressServer, + 'cherrypy': CherryPyServer, + 'paste': PasteServer, + 'fapws3': FapwsServer, + 'tornado': TornadoServer, + 'gae': AppEngineServer, + 'twisted': TwistedServer, + 'diesel': DieselServer, + 'meinheld': MeinheldServer, + 'gunicorn': GunicornServer, + 'eventlet': EventletServer, + 'gevent': GeventServer, + 'rocket': RocketServer, + 'bjoern' : BjoernServer, + 'auto': AutoServer, +} + + + + + + +############################################################################### +# Application Control ########################################################## +############################################################################### + + +def load(target, **namespace): + """ Import a module or fetch an object from a module. + + * ``package.module`` returns `module` as a module object. + * ``pack.mod:name`` returns the module variable `name` from `pack.mod`. + * ``pack.mod:func()`` calls `pack.mod.func()` and returns the result. + + The last form accepts not only function calls, but any type of + expression. Keyword arguments passed to this function are available as + local variables. Example: ``import_string('re:compile(x)', x='[a-z]')`` + """ + module, target = target.split(":", 1) if ':' in target else (target, None) + if module not in sys.modules: __import__(module) + if not target: return sys.modules[module] + if target.isalnum(): return getattr(sys.modules[module], target) + package_name = module.split('.')[0] + namespace[package_name] = sys.modules[package_name] + return eval('%s.%s' % (module, target), namespace) + + +def load_app(target): + """ Load a bottle application from a module and make sure that the import + does not affect the current default application, but returns a separate + application object. See :func:`load` for the target parameter. """ + global NORUN; NORUN, nr_old = True, NORUN + try: + tmp = default_app.push() # Create a new "default application" + rv = load(target) # Import the target module + return rv if callable(rv) else tmp + finally: + default_app.remove(tmp) # Remove the temporary added default application + NORUN = nr_old + +_debug = debug +def run(app=None, server='wsgiref', host='127.0.0.1', port=8080, + interval=1, reloader=False, quiet=False, plugins=None, + debug=False, **kargs): + """ Start a server instance. This method blocks until the server terminates. + + :param app: WSGI application or target string supported by + :func:`load_app`. (default: :func:`default_app`) + :param server: Server adapter to use. See :data:`server_names` keys + for valid names or pass a :class:`ServerAdapter` subclass. + (default: `wsgiref`) + :param host: Server address to bind to. Pass ``0.0.0.0`` to listens on + all interfaces including the external one. (default: 127.0.0.1) + :param port: Server port to bind to. Values below 1024 require root + privileges. (default: 8080) + :param reloader: Start auto-reloading server? (default: False) + :param interval: Auto-reloader interval in seconds (default: 1) + :param quiet: Suppress output to stdout and stderr? (default: False) + :param options: Options passed to the server adapter. + """ + if NORUN: return + if reloader and not os.environ.get('BOTTLE_CHILD'): + try: + lockfile = None + fd, lockfile = tempfile.mkstemp(prefix='bottle.', suffix='.lock') + os.close(fd) # We only need this file to exist. We never write to it + while os.path.exists(lockfile): + args = [sys.executable] + sys.argv + environ = os.environ.copy() + environ['BOTTLE_CHILD'] = 'true' + environ['BOTTLE_LOCKFILE'] = lockfile + p = subprocess.Popen(args, env=environ) + while p.poll() is None: # Busy wait... + os.utime(lockfile, None) # I am alive! + time.sleep(interval) + if p.poll() != 3: + if os.path.exists(lockfile): os.unlink(lockfile) + sys.exit(p.poll()) + except KeyboardInterrupt: + pass + finally: + if os.path.exists(lockfile): + os.unlink(lockfile) + return + + try: + _debug(debug) + app = app or default_app() + if isinstance(app, basestring): + app = load_app(app) + if not callable(app): + raise ValueError("Application is not callable: %r" % app) + + for plugin in plugins or []: + app.install(plugin) + + if server in server_names: + server = server_names.get(server) + if isinstance(server, basestring): + server = load(server) + if isinstance(server, type): + server = server(host=host, port=port, **kargs) + if not isinstance(server, ServerAdapter): + raise ValueError("Unknown or unsupported server: %r" % server) + + server.quiet = server.quiet or quiet + if not server.quiet: + _stderr("Bottle v%s server starting up (using %s)...\n" % (__version__, repr(server))) + _stderr("Listening on http://%s:%d/\n" % (server.host, server.port)) + _stderr("Hit Ctrl-C to quit.\n\n") + + if reloader: + lockfile = os.environ.get('BOTTLE_LOCKFILE') + bgcheck = FileCheckerThread(lockfile, interval) + with bgcheck: + server.run(app) + if bgcheck.status == 'reload': + sys.exit(3) + else: + server.run(app) + except KeyboardInterrupt: + pass + except (SystemExit, MemoryError): + raise + except: + if not reloader: raise + if not getattr(server, 'quiet', quiet): + print_exc() + time.sleep(interval) + sys.exit(3) + + + +class FileCheckerThread(threading.Thread): + ''' Interrupt main-thread as soon as a changed module file is detected, + the lockfile gets deleted or gets to old. ''' + + def __init__(self, lockfile, interval): + threading.Thread.__init__(self) + self.lockfile, self.interval = lockfile, interval + #: Is one of 'reload', 'error' or 'exit' + self.status = None + + def run(self): + exists = os.path.exists + mtime = lambda path: os.stat(path).st_mtime + files = dict() + + for module in list(sys.modules.values()): + path = getattr(module, '__file__', '') + if path[-4:] in ('.pyo', '.pyc'): path = path[:-1] + if path and exists(path): files[path] = mtime(path) + + while not self.status: + if not exists(self.lockfile)\ + or mtime(self.lockfile) < time.time() - self.interval - 5: + self.status = 'error' + thread.interrupt_main() + for path, lmtime in list(files.items()): + if not exists(path) or mtime(path) > lmtime: + self.status = 'reload' + thread.interrupt_main() + break + time.sleep(self.interval) + + def __enter__(self): + self.start() + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.status: self.status = 'exit' # silent exit + self.join() + return exc_type is not None and issubclass(exc_type, KeyboardInterrupt) + + + + + +############################################################################### +# Template Adapters ############################################################ +############################################################################### + + +class TemplateError(HTTPError): + def __init__(self, message): + HTTPError.__init__(self, 500, message) + + +class BaseTemplate(object): + """ Base class and minimal API for template adapters """ + extensions = ['tpl','html','thtml','stpl'] + settings = {} #used in prepare() + defaults = {} #used in render() + + def __init__(self, source=None, name=None, lookup=[], encoding='utf8', **settings): + """ Create a new template. + If the source parameter (str or buffer) is missing, the name argument + is used to guess a template filename. Subclasses can assume that + self.source and/or self.filename are set. Both are strings. + The lookup, encoding and settings parameters are stored as instance + variables. + The lookup parameter stores a list containing directory paths. + The encoding parameter should be used to decode byte strings or files. + The settings parameter contains a dict for engine-specific settings. + """ + self.name = name + self.source = source.read() if hasattr(source, 'read') else source + self.filename = source.filename if hasattr(source, 'filename') else None + self.lookup = [os.path.abspath(x) for x in lookup] + self.encoding = encoding + self.settings = self.settings.copy() # Copy from class variable + self.settings.update(settings) # Apply + if not self.source and self.name: + self.filename = self.search(self.name, self.lookup) + if not self.filename: + raise TemplateError('Template %s not found.' % repr(name)) + if not self.source and not self.filename: + raise TemplateError('No template specified.') + self.prepare(**self.settings) + + @classmethod + def search(cls, name, lookup=[]): + """ Search name in all directories specified in lookup. + First without, then with common extensions. Return first hit. """ + if not lookup: + depr('The template lookup path list should not be empty.') + lookup = ['.'] + + if os.path.isabs(name) and os.path.isfile(name): + depr('Absolute template path names are deprecated.') + return os.path.abspath(name) + + for spath in lookup: + spath = os.path.abspath(spath) + os.sep + fname = os.path.abspath(os.path.join(spath, name)) + if not fname.startswith(spath): continue + if os.path.isfile(fname): return fname + for ext in cls.extensions: + if os.path.isfile('%s.%s' % (fname, ext)): + return '%s.%s' % (fname, ext) + + @classmethod + def global_config(cls, key, *args): + ''' This reads or sets the global settings stored in class.settings. ''' + if args: + cls.settings = cls.settings.copy() # Make settings local to class + cls.settings[key] = args[0] + else: + return cls.settings[key] + + def prepare(self, **options): + """ Run preparations (parsing, caching, ...). + It should be possible to call this again to refresh a template or to + update settings. + """ + raise NotImplementedError + + def render(self, *args, **kwargs): + """ Render the template with the specified local variables and return + a single byte or unicode string. If it is a byte string, the encoding + must match self.encoding. This method must be thread-safe! + Local variables may be provided in dictionaries (*args) + or directly, as keywords (**kwargs). + """ + raise NotImplementedError + + +class MakoTemplate(BaseTemplate): + def prepare(self, **options): + from mako.template import Template + from mako.lookup import TemplateLookup + options.update({'input_encoding':self.encoding}) + options.setdefault('format_exceptions', bool(DEBUG)) + lookup = TemplateLookup(directories=self.lookup, **options) + if self.source: + self.tpl = Template(self.source, lookup=lookup, **options) + else: + self.tpl = Template(uri=self.name, filename=self.filename, lookup=lookup, **options) + + def render(self, *args, **kwargs): + for dictarg in args: kwargs.update(dictarg) + _defaults = self.defaults.copy() + _defaults.update(kwargs) + return self.tpl.render(**_defaults) + + +class CheetahTemplate(BaseTemplate): + def prepare(self, **options): + from Cheetah.Template import Template + self.context = threading.local() + self.context.vars = {} + options['searchList'] = [self.context.vars] + if self.source: + self.tpl = Template(source=self.source, **options) + else: + self.tpl = Template(file=self.filename, **options) + + def render(self, *args, **kwargs): + for dictarg in args: kwargs.update(dictarg) + self.context.vars.update(self.defaults) + self.context.vars.update(kwargs) + out = str(self.tpl) + self.context.vars.clear() + return out + + +class Jinja2Template(BaseTemplate): + def prepare(self, filters=None, tests=None, **kwargs): + from jinja2 import Environment, FunctionLoader + if 'prefix' in kwargs: # TODO: to be removed after a while + raise RuntimeError('The keyword argument `prefix` has been removed. ' + 'Use the full jinja2 environment name line_statement_prefix instead.') + self.env = Environment(loader=FunctionLoader(self.loader), **kwargs) + if filters: self.env.filters.update(filters) + if tests: self.env.tests.update(tests) + if self.source: + self.tpl = self.env.from_string(self.source) + else: + self.tpl = self.env.get_template(self.filename) + + def render(self, *args, **kwargs): + for dictarg in args: kwargs.update(dictarg) + _defaults = self.defaults.copy() + _defaults.update(kwargs) + return self.tpl.render(**_defaults) + + def loader(self, name): + fname = self.search(name, self.lookup) + if not fname: return + with open(fname, "rb") as f: + return f.read().decode(self.encoding) + + +class SimpleTALTemplate(BaseTemplate): + ''' Deprecated, do not use. ''' + def prepare(self, **options): + depr('The SimpleTAL template handler is deprecated'\ + ' and will be removed in 0.12') + from simpletal import simpleTAL + if self.source: + self.tpl = simpleTAL.compileHTMLTemplate(self.source) + else: + with open(self.filename, 'rb') as fp: + self.tpl = simpleTAL.compileHTMLTemplate(tonat(fp.read())) + + def render(self, *args, **kwargs): + from simpletal import simpleTALES + for dictarg in args: kwargs.update(dictarg) + context = simpleTALES.Context() + for k,v in self.defaults.items(): + context.addGlobal(k, v) + for k,v in kwargs.items(): + context.addGlobal(k, v) + output = StringIO() + self.tpl.expand(context, output) + return output.getvalue() + + +class SimpleTemplate(BaseTemplate): + blocks = ('if', 'elif', 'else', 'try', 'except', 'finally', 'for', 'while', + 'with', 'def', 'class') + dedent_blocks = ('elif', 'else', 'except', 'finally') + + @lazy_attribute + def re_pytokens(cls): + ''' This matches comments and all kinds of quoted strings but does + NOT match comments (#...) within quoted strings. (trust me) ''' + return re.compile(r''' + (''(?!')|""(?!")|'{6}|"{6} # Empty strings (all 4 types) + |'(?:[^\\']|\\.)+?' # Single quotes (') + |"(?:[^\\"]|\\.)+?" # Double quotes (") + |'{3}(?:[^\\]|\\.|\n)+?'{3} # Triple-quoted strings (') + |"{3}(?:[^\\]|\\.|\n)+?"{3} # Triple-quoted strings (") + |\#.* # Comments + )''', re.VERBOSE) + + def prepare(self, escape_func=html_escape, noescape=False, **kwargs): + self.cache = {} + enc = self.encoding + self._str = lambda x: touni(x, enc) + self._escape = lambda x: escape_func(touni(x, enc)) + if noescape: + self._str, self._escape = self._escape, self._str + + @classmethod + def split_comment(cls, code): + """ Removes comments (#...) from python code. """ + if '#' not in code: return code + #: Remove comments only (leave quoted strings as they are) + subf = lambda m: '' if m.group(0)[0]=='#' else m.group(0) + return re.sub(cls.re_pytokens, subf, code) + + @cached_property + def co(self): + return compile(self.code, self.filename or '', 'exec') + + @cached_property + def code(self): + stack = [] # Current Code indentation + lineno = 0 # Current line of code + ptrbuffer = [] # Buffer for printable strings and token tuple instances + codebuffer = [] # Buffer for generated python code + multiline = dedent = oneline = False + template = self.source or open(self.filename, 'rb').read() + + def yield_tokens(line): + for i, part in enumerate(re.split(r'\{\{(.*?)\}\}', line)): + if i % 2: + if part.startswith('!'): yield 'RAW', part[1:] + else: yield 'CMD', part + else: yield 'TXT', part + + def flush(): # Flush the ptrbuffer + if not ptrbuffer: return + cline = '' + for line in ptrbuffer: + for token, value in line: + if token == 'TXT': cline += repr(value) + elif token == 'RAW': cline += '_str(%s)' % value + elif token == 'CMD': cline += '_escape(%s)' % value + cline += ', ' + cline = cline[:-2] + '\\\n' + cline = cline[:-2] + if cline[:-1].endswith('\\\\\\\\\\n'): + cline = cline[:-7] + cline[-1] # 'nobr\\\\\n' --> 'nobr' + cline = '_printlist([' + cline + '])' + del ptrbuffer[:] # Do this before calling code() again + code(cline) + + def code(stmt): + for line in stmt.splitlines(): + codebuffer.append(' ' * len(stack) + line.strip()) + + for line in template.splitlines(True): + lineno += 1 + line = touni(line, self.encoding) + sline = line.lstrip() + if lineno <= 2: + m = re.match(r"%\s*#.*coding[:=]\s*([-\w.]+)", sline) + if m: self.encoding = m.group(1) + if m: line = line.replace('coding','coding (removed)') + if sline and sline[0] == '%' and sline[:2] != '%%': + line = line.split('%',1)[1].lstrip() # Full line following the % + cline = self.split_comment(line).strip() + cmd = re.split(r'[^a-zA-Z0-9_]', cline)[0] + flush() # You are actually reading this? Good luck, it's a mess :) + if cmd in self.blocks or multiline: + cmd = multiline or cmd + dedent = cmd in self.dedent_blocks # "else:" + if dedent and not oneline and not multiline: + cmd = stack.pop() + code(line) + oneline = not cline.endswith(':') # "if 1: pass" + multiline = cmd if cline.endswith('\\') else False + if not oneline and not multiline: + stack.append(cmd) + elif cmd == 'end' and stack: + code('#end(%s) %s' % (stack.pop(), line.strip()[3:])) + elif cmd == 'include': + p = cline.split(None, 2)[1:] + if len(p) == 2: + code("_=_include(%s, _stdout, %s)" % (repr(p[0]), p[1])) + elif p: + code("_=_include(%s, _stdout)" % repr(p[0])) + else: # Empty %include -> reverse of %rebase + code("_printlist(_base)") + elif cmd == 'rebase': + p = cline.split(None, 2)[1:] + if len(p) == 2: + code("globals()['_rebase']=(%s, dict(%s))" % (repr(p[0]), p[1])) + elif p: + code("globals()['_rebase']=(%s, {})" % repr(p[0])) + else: + code(line) + else: # Line starting with text (not '%') or '%%' (escaped) + if line.strip().startswith('%%'): + line = line.replace('%%', '%', 1) + ptrbuffer.append(yield_tokens(line)) + flush() + return '\n'.join(codebuffer) + '\n' + + def subtemplate(self, _name, _stdout, *args, **kwargs): + for dictarg in args: kwargs.update(dictarg) + if _name not in self.cache: + self.cache[_name] = self.__class__(name=_name, lookup=self.lookup) + return self.cache[_name].execute(_stdout, kwargs) + + def execute(self, _stdout, *args, **kwargs): + for dictarg in args: kwargs.update(dictarg) + env = self.defaults.copy() + env.update({'_stdout': _stdout, '_printlist': _stdout.extend, + '_include': self.subtemplate, '_str': self._str, + '_escape': self._escape, 'get': env.get, + 'setdefault': env.setdefault, 'defined': env.__contains__}) + env.update(kwargs) + eval(self.co, env) + if '_rebase' in env: + subtpl, rargs = env['_rebase'] + rargs['_base'] = _stdout[:] #copy stdout + del _stdout[:] # clear stdout + return self.subtemplate(subtpl,_stdout,rargs) + return env + + def render(self, *args, **kwargs): + """ Render the template using keyword arguments as local variables. """ + for dictarg in args: kwargs.update(dictarg) + stdout = [] + self.execute(stdout, kwargs) + return ''.join(stdout) + + +def template(*args, **kwargs): + ''' + Get a rendered template as a string iterator. + You can use a name, a filename or a template string as first parameter. + Template rendering arguments can be passed as dictionaries + or directly (as keyword arguments). + ''' + tpl = args[0] if args else None + adapter = kwargs.pop('template_adapter', SimpleTemplate) + lookup = kwargs.pop('template_lookup', TEMPLATE_PATH) + tplid = (id(lookup), tpl) + if tplid not in TEMPLATES or DEBUG: + settings = kwargs.pop('template_settings', {}) + if isinstance(tpl, adapter): + TEMPLATES[tplid] = tpl + if settings: TEMPLATES[tplid].prepare(**settings) + elif "\n" in tpl or "{" in tpl or "%" in tpl or '$' in tpl: + TEMPLATES[tplid] = adapter(source=tpl, lookup=lookup, **settings) + else: + TEMPLATES[tplid] = adapter(name=tpl, lookup=lookup, **settings) + if not TEMPLATES[tplid]: + abort(500, 'Template (%s) not found' % tpl) + for dictarg in args[1:]: kwargs.update(dictarg) + return TEMPLATES[tplid].render(kwargs) + +mako_template = functools.partial(template, template_adapter=MakoTemplate) +cheetah_template = functools.partial(template, template_adapter=CheetahTemplate) +jinja2_template = functools.partial(template, template_adapter=Jinja2Template) +simpletal_template = functools.partial(template, template_adapter=SimpleTALTemplate) + + +def view(tpl_name, **defaults): + ''' Decorator: renders a template for a handler. + The handler can control its behavior like that: + + - return a dict of template vars to fill out the template + - return something other than a dict and the view decorator will not + process the template, but return the handler result as is. + This includes returning a HTTPResponse(dict) to get, + for instance, JSON with autojson or other castfilters. + ''' + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + result = func(*args, **kwargs) + if isinstance(result, (dict, DictMixin)): + tplvars = defaults.copy() + tplvars.update(result) + return template(tpl_name, **tplvars) + return result + return wrapper + return decorator + +mako_view = functools.partial(view, template_adapter=MakoTemplate) +cheetah_view = functools.partial(view, template_adapter=CheetahTemplate) +jinja2_view = functools.partial(view, template_adapter=Jinja2Template) +simpletal_view = functools.partial(view, template_adapter=SimpleTALTemplate) + + + + + + +############################################################################### +# Constants and Globals ######################################################## +############################################################################### + + +TEMPLATE_PATH = ['./', './views/'] +TEMPLATES = {} +DEBUG = False +NORUN = False # If set, run() does nothing. Used by load_app() + +#: A dict to map HTTP status codes (e.g. 404) to phrases (e.g. 'Not Found') +HTTP_CODES = httplib.responses +HTTP_CODES[418] = "I'm a teapot" # RFC 2324 +HTTP_CODES[428] = "Precondition Required" +HTTP_CODES[429] = "Too Many Requests" +HTTP_CODES[431] = "Request Header Fields Too Large" +HTTP_CODES[511] = "Network Authentication Required" +_HTTP_STATUS_LINES = dict((k, '%d %s'%(k,v)) for (k,v) in HTTP_CODES.items()) + +#: The default template used for error pages. Override with @error() +ERROR_PAGE_TEMPLATE = """ +%%try: + %%from %s import DEBUG, HTTP_CODES, request, touni + + + + Error: {{e.status}} + + + +

        Error: {{e.status}}

        +

        Sorry, the requested URL {{repr(request.url)}} + caused an error:

        + 
        {{e.body}}
        + %%if DEBUG and e.exception: +

        Exception:

        + 
        {{repr(e.exception)}}
        + %%end + %%if DEBUG and e.traceback: +

        Traceback:

        + 
        {{e.traceback}}
        + %%end + + +%%except ImportError: + ImportError: Could not generate the error page. Please add bottle to + the import path. +%%end +""" % __name__ + +#: A thread-safe instance of :class:`LocalRequest`. If accessed from within a +#: request callback, this instance always refers to the *current* request +#: (even on a multithreaded server). +request = LocalRequest() + +#: A thread-safe instance of :class:`LocalResponse`. It is used to change the +#: HTTP response for the *current* request. +response = LocalResponse() + +#: A thread-safe namespace. Not used by Bottle. +local = threading.local() + +# Initialize app stack (create first empty Bottle app) +# BC: 0.6.4 and needed for run() +app = default_app = AppStack() +app.push() + +#: A virtual package that redirects import statements. +#: Example: ``import bottle.ext.sqlite`` actually imports `bottle_sqlite`. +ext = _ImportRedirect('bottle.ext' if __name__ == '__main__' else __name__+".ext", 'bottle_%s').module + +if __name__ == '__main__': + opt, args, parser = _cmd_options, _cmd_args, _cmd_parser + if opt.version: + _stdout('Bottle %s\n'%__version__) + sys.exit(0) + if not args: + parser.print_help() + _stderr('\nError: No application specified.\n') + sys.exit(1) + + sys.path.insert(0, '.') + sys.modules.setdefault('bottle', sys.modules['__main__']) + + host, port = (opt.bind or 'localhost'), 8080 + if ':' in host: + host, port = host.rsplit(':', 1) + + run(args[0], host=host, port=port, server=opt.server, + reloader=opt.reload, plugins=opt.plugin, debug=opt.debug) + + + + +# THE END diff --git a/pyload/lib/forwarder.py b/pyload/lib/forwarder.py new file mode 100644 index 000000000..eacb33c2b --- /dev/null +++ b/pyload/lib/forwarder.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +from sys import argv +from sys import exit + +import socket +import thread + +from traceback import print_exc + +class Forwarder(): + + def __init__(self, extip,extport=9666): + print "Start portforwarding to %s:%s" % (extip, extport) + proxy(extip, extport, 9666) + + +def proxy(*settings): + while True: + server(*settings) + +def server(*settings): + try: + dock_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + dock_socket.bind(("127.0.0.1", settings[2])) + dock_socket.listen(5) + while True: + client_socket = dock_socket.accept()[0] + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.connect((settings[0], settings[1])) + thread.start_new_thread(forward, (client_socket, server_socket)) + thread.start_new_thread(forward, (server_socket, client_socket)) + except Exception: + print_exc() + + +def forward(source, destination): + string = ' ' + while string: + string = source.recv(1024) + if string: + destination.sendall(string) + else: + #source.shutdown(socket.SHUT_RD) + destination.shutdown(socket.SHUT_WR) + +if __name__ == "__main__": + args = argv[1:] + if not args: + print "Usage: forwarder.py " + exit() + if len(args) == 1: + args.append(9666) + + f = Forwarder(args[0], int(args[1])) + \ No newline at end of file diff --git a/pyload/lib/hg_tool.py b/pyload/lib/hg_tool.py new file mode 100644 index 000000000..cd97833df --- /dev/null +++ b/pyload/lib/hg_tool.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +from subprocess import Popen, PIPE +from time import time, gmtime, strftime + +aliases = {"zoidber": "zoidberg", "zoidberg10": "zoidberg", "webmaster": "dhmh", "mast3rranan": "ranan", + "ranan2": "ranan"} +exclude = ["locale/*", "module/lib/*"] +date_format = "%Y-%m-%d" +line_re = re.compile(r" (\d+) \**", re.I) + +def add_exclude_flags(args): + for dir in exclude: + args.extend(["-X", dir]) + +# remove small percentages +def wipe(data, perc=1): + s = (sum(data.values()) * perc) / 100 + for k, v in data.items(): + if v < s: del data[k] + + return data + +# remove aliases +def de_alias(data): + for k, v in aliases.iteritems(): + if k not in data: continue + alias = aliases[k] + + if alias in data: data[alias] += data[k] + else: data[alias] = data[k] + + del data[k] + + return data + + +def output(data): + s = float(sum(data.values())) + print "Total Lines: %d" % s + for k, v in data.iteritems(): + print "%15s: %.1f%% | %d" % (k, (v * 100) / s, v) + print + + +def file_list(): + args = ["hg", "status", "-A"] + add_exclude_flags(args) + p = Popen(args, stdout=PIPE) + out, err = p.communicate() + return [x.split()[1] for x in out.splitlines() if x.split()[0] in "CMA"] + + +def hg_annotate(path): + args = ["hg", "annotate", "-u", path] + p = Popen(args, stdout=PIPE) + out, err = p.communicate() + + data = {} + + for line in out.splitlines(): + author, non, line = line.partition(":") + + # probably binary file + if author == path: return {} + + author = author.strip().lower() + if not line.strip(): continue # don't count blank lines + + if author in data: data[author] += 1 + else: data[author] = 1 + + return de_alias(data) + + +def hg_churn(days=None): + args = ["hg", "churn"] + if days: + args.append("-d") + t = time() - 60 * 60 * 24 * days + args.append("%s to %s" % (strftime(date_format, gmtime(t)), strftime(date_format))) + + add_exclude_flags(args) + p = Popen(args, stdout=PIPE) + out, err = p.communicate() + + data = {} + + for line in out.splitlines(): + m = line_re.search(line) + author = line.split()[0] + lines = int(m.group(1)) + + if "@" in author: + author, n, email = author.partition("@") + + author = author.strip().lower() + + if author in data: data[author] += lines + else: data[author] = lines + + return de_alias(data) + + +def complete_annotate(): + files = file_list() + data = {} + for f in files: + tmp = hg_annotate(f) + for k, v in tmp.iteritems(): + if k in data: data[k] += v + else: data[k] = v + + return data + + +if __name__ == "__main__": + for d in (30, 90, 180): + c = wipe(hg_churn(d)) + print "Changes in %d days:" % d + output(c) + + c = wipe(hg_churn()) + print "Total changes:" + output(c) + + print "Current source code version:" + data = wipe(complete_annotate()) + output(data) + + diff --git a/pyload/lib/mod_pywebsocket/COPYING b/pyload/lib/mod_pywebsocket/COPYING new file mode 100644 index 000000000..989d02e4c --- /dev/null +++ b/pyload/lib/mod_pywebsocket/COPYING @@ -0,0 +1,28 @@ +Copyright 2012, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pyload/lib/mod_pywebsocket/__init__.py b/pyload/lib/mod_pywebsocket/__init__.py new file mode 100644 index 000000000..454ae0c45 --- /dev/null +++ b/pyload/lib/mod_pywebsocket/__init__.py @@ -0,0 +1,197 @@ +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""WebSocket extension for Apache HTTP Server. + +mod_pywebsocket is a WebSocket extension for Apache HTTP Server +intended for testing or experimental purposes. mod_python is required. + + +Installation +============ + +0. Prepare an Apache HTTP Server for which mod_python is enabled. + +1. Specify the following Apache HTTP Server directives to suit your + configuration. + + If mod_pywebsocket is not in the Python path, specify the following. + is the directory where mod_pywebsocket is installed. + + PythonPath "sys.path+['']" + + Always specify the following. is the directory where + user-written WebSocket handlers are placed. + + PythonOption mod_pywebsocket.handler_root + PythonHeaderParserHandler mod_pywebsocket.headerparserhandler + + To limit the search for WebSocket handlers to a directory + under , configure as follows: + + PythonOption mod_pywebsocket.handler_scan + + is useful in saving scan time when + contains many non-WebSocket handler files. + + If you want to allow handlers whose canonical path is not under the root + directory (i.e. symbolic link is in root directory but its target is not), + configure as follows: + + PythonOption mod_pywebsocket.allow_handlers_outside_root_dir On + + Example snippet of httpd.conf: + (mod_pywebsocket is in /websock_lib, WebSocket handlers are in + /websock_handlers, port is 80 for ws, 443 for wss.) + + + PythonPath "sys.path+['/websock_lib']" + PythonOption mod_pywebsocket.handler_root /websock_handlers + PythonHeaderParserHandler mod_pywebsocket.headerparserhandler + + +2. Tune Apache parameters for serving WebSocket. We'd like to note that at + least TimeOut directive from core features and RequestReadTimeout + directive from mod_reqtimeout should be modified not to kill connections + in only a few seconds of idle time. + +3. Verify installation. You can use example/console.html to poke the server. + + +Writing WebSocket handlers +========================== + +When a WebSocket request comes in, the resource name +specified in the handshake is considered as if it is a file path under + and the handler defined in +/_wsh.py is invoked. + +For example, if the resource name is /example/chat, the handler defined in +/example/chat_wsh.py is invoked. + +A WebSocket handler is composed of the following three functions: + + web_socket_do_extra_handshake(request) + web_socket_transfer_data(request) + web_socket_passive_closing_handshake(request) + +where: + request: mod_python request. + +web_socket_do_extra_handshake is called during the handshake after the +headers are successfully parsed and WebSocket properties (ws_location, +ws_origin, and ws_resource) are added to request. A handler +can reject the request by raising an exception. + +A request object has the following properties that you can use during the +extra handshake (web_socket_do_extra_handshake): +- ws_resource +- ws_origin +- ws_version +- ws_location (HyBi 00 only) +- ws_extensions (HyBi 06 and later) +- ws_deflate (HyBi 06 and later) +- ws_protocol +- ws_requested_protocols (HyBi 06 and later) + +The last two are a bit tricky. See the next subsection. + + +Subprotocol Negotiation +----------------------- + +For HyBi 06 and later, ws_protocol is always set to None when +web_socket_do_extra_handshake is called. If ws_requested_protocols is not +None, you must choose one subprotocol from this list and set it to +ws_protocol. + +For HyBi 00, when web_socket_do_extra_handshake is called, +ws_protocol is set to the value given by the client in +Sec-WebSocket-Protocol header or None if +such header was not found in the opening handshake request. Finish extra +handshake with ws_protocol untouched to accept the request subprotocol. +Then, Sec-WebSocket-Protocol header will be sent to +the client in response with the same value as requested. Raise an exception +in web_socket_do_extra_handshake to reject the requested subprotocol. + + +Data Transfer +------------- + +web_socket_transfer_data is called after the handshake completed +successfully. A handler can receive/send messages from/to the client +using request. mod_pywebsocket.msgutil module provides utilities +for data transfer. + +You can receive a message by the following statement. + + message = request.ws_stream.receive_message() + +This call blocks until any complete text frame arrives, and the payload data +of the incoming frame will be stored into message. When you're using IETF +HyBi 00 or later protocol, receive_message() will return None on receiving +client-initiated closing handshake. When any error occurs, receive_message() +will raise some exception. + +You can send a message by the following statement. + + request.ws_stream.send_message(message) + + +Closing Connection +------------------ + +Executing the following statement or just return-ing from +web_socket_transfer_data cause connection close. + + request.ws_stream.close_connection() + +close_connection will wait +for closing handshake acknowledgement coming from the client. When it +couldn't receive a valid acknowledgement, raises an exception. + +web_socket_passive_closing_handshake is called after the server receives +incoming closing frame from the client peer immediately. You can specify +code and reason by return values. They are sent as a outgoing closing frame +from the server. A request object has the following properties that you can +use in web_socket_passive_closing_handshake. +- ws_close_code +- ws_close_reason + + +Threading +--------- + +A WebSocket handler must be thread-safe if the server (Apache or +standalone.py) is configured to use threads. +""" + + +# vi:sts=4 sw=4 et tw=72 diff --git a/pyload/lib/mod_pywebsocket/_stream_base.py b/pyload/lib/mod_pywebsocket/_stream_base.py new file mode 100644 index 000000000..60fb33d2c --- /dev/null +++ b/pyload/lib/mod_pywebsocket/_stream_base.py @@ -0,0 +1,165 @@ +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Base stream class. +""" + + +# Note: request.connection.write/read are used in this module, even though +# mod_python document says that they should be used only in connection +# handlers. Unfortunately, we have no other options. For example, +# request.write/read are not suitable because they don't allow direct raw bytes +# writing/reading. + + +from mod_pywebsocket import util + + +# Exceptions + + +class ConnectionTerminatedException(Exception): + """This exception will be raised when a connection is terminated + unexpectedly. + """ + + pass + + +class InvalidFrameException(ConnectionTerminatedException): + """This exception will be raised when we received an invalid frame we + cannot parse. + """ + + pass + + +class BadOperationException(Exception): + """This exception will be raised when send_message() is called on + server-terminated connection or receive_message() is called on + client-terminated connection. + """ + + pass + + +class UnsupportedFrameException(Exception): + """This exception will be raised when we receive a frame with flag, opcode + we cannot handle. Handlers can just catch and ignore this exception and + call receive_message() again to continue processing the next frame. + """ + + pass + + +class InvalidUTF8Exception(Exception): + """This exception will be raised when we receive a text frame which + contains invalid UTF-8 strings. + """ + + pass + + +class StreamBase(object): + """Base stream class.""" + + def __init__(self, request): + """Construct an instance. + + Args: + request: mod_python request. + """ + + self._logger = util.get_class_logger(self) + + self._request = request + + def _read(self, length): + """Reads length bytes from connection. In case we catch any exception, + prepends remote address to the exception message and raise again. + + Raises: + ConnectionTerminatedException: when read returns empty string. + """ + + bytes = self._request.connection.read(length) + if not bytes: + raise ConnectionTerminatedException( + 'Receiving %d byte failed. Peer (%r) closed connection' % + (length, (self._request.connection.remote_addr,))) + return bytes + + def _write(self, bytes): + """Writes given bytes to connection. In case we catch any exception, + prepends remote address to the exception message and raise again. + """ + + try: + self._request.connection.write(bytes) + except Exception, e: + util.prepend_message_to_exception( + 'Failed to send message to %r: ' % + (self._request.connection.remote_addr,), + e) + raise + + def receive_bytes(self, length): + """Receives multiple bytes. Retries read when we couldn't receive the + specified amount. + + Raises: + ConnectionTerminatedException: when read returns empty string. + """ + + bytes = [] + while length > 0: + new_bytes = self._read(length) + bytes.append(new_bytes) + length -= len(new_bytes) + return ''.join(bytes) + + def _read_until(self, delim_char): + """Reads bytes until we encounter delim_char. The result will not + contain delim_char. + + Raises: + ConnectionTerminatedException: when read returns empty string. + """ + + bytes = [] + while True: + ch = self._read(1) + if ch == delim_char: + break + bytes.append(ch) + return ''.join(bytes) + + +# vi:sts=4 sw=4 et diff --git a/pyload/lib/mod_pywebsocket/_stream_hixie75.py b/pyload/lib/mod_pywebsocket/_stream_hixie75.py new file mode 100644 index 000000000..94cf5b31b --- /dev/null +++ b/pyload/lib/mod_pywebsocket/_stream_hixie75.py @@ -0,0 +1,229 @@ +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""This file provides a class for parsing/building frames of the WebSocket +protocol version HyBi 00 and Hixie 75. + +Specification: +- HyBi 00 http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-00 +- Hixie 75 http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-75 +""" + + +from mod_pywebsocket import common +from mod_pywebsocket._stream_base import BadOperationException +from mod_pywebsocket._stream_base import ConnectionTerminatedException +from mod_pywebsocket._stream_base import InvalidFrameException +from mod_pywebsocket._stream_base import StreamBase +from mod_pywebsocket._stream_base import UnsupportedFrameException +from mod_pywebsocket import util + + +class StreamHixie75(StreamBase): + """A class for parsing/building frames of the WebSocket protocol version + HyBi 00 and Hixie 75. + """ + + def __init__(self, request, enable_closing_handshake=False): + """Construct an instance. + + Args: + request: mod_python request. + enable_closing_handshake: to let StreamHixie75 perform closing + handshake as specified in HyBi 00, set + this option to True. + """ + + StreamBase.__init__(self, request) + + self._logger = util.get_class_logger(self) + + self._enable_closing_handshake = enable_closing_handshake + + self._request.client_terminated = False + self._request.server_terminated = False + + def send_message(self, message, end=True, binary=False): + """Send message. + + Args: + message: unicode string to send. + binary: not used in hixie75. + + Raises: + BadOperationException: when called on a server-terminated + connection. + """ + + if not end: + raise BadOperationException( + 'StreamHixie75 doesn\'t support send_message with end=False') + + if binary: + raise BadOperationException( + 'StreamHixie75 doesn\'t support send_message with binary=True') + + if self._request.server_terminated: + raise BadOperationException( + 'Requested send_message after sending out a closing handshake') + + self._write(''.join(['\x00', message.encode('utf-8'), '\xff'])) + + def _read_payload_length_hixie75(self): + """Reads a length header in a Hixie75 version frame with length. + + Raises: + ConnectionTerminatedException: when read returns empty string. + """ + + length = 0 + while True: + b_str = self._read(1) + b = ord(b_str) + length = length * 128 + (b & 0x7f) + if (b & 0x80) == 0: + break + return length + + def receive_message(self): + """Receive a WebSocket frame and return its payload an unicode string. + + Returns: + payload unicode string in a WebSocket frame. + + Raises: + ConnectionTerminatedException: when read returns empty + string. + BadOperationException: when called on a client-terminated + connection. + """ + + if self._request.client_terminated: + raise BadOperationException( + 'Requested receive_message after receiving a closing ' + 'handshake') + + while True: + # Read 1 byte. + # mp_conn.read will block if no bytes are available. + # Timeout is controlled by TimeOut directive of Apache. + frame_type_str = self.receive_bytes(1) + frame_type = ord(frame_type_str) + if (frame_type & 0x80) == 0x80: + # The payload length is specified in the frame. + # Read and discard. + length = self._read_payload_length_hixie75() + if length > 0: + _ = self.receive_bytes(length) + # 5.3 3. 12. if /type/ is 0xFF and /length/ is 0, then set the + # /client terminated/ flag and abort these steps. + if not self._enable_closing_handshake: + continue + + if frame_type == 0xFF and length == 0: + self._request.client_terminated = True + + if self._request.server_terminated: + self._logger.debug( + 'Received ack for server-initiated closing ' + 'handshake') + return None + + self._logger.debug( + 'Received client-initiated closing handshake') + + self._send_closing_handshake() + self._logger.debug( + 'Sent ack for client-initiated closing handshake') + return None + else: + # The payload is delimited with \xff. + bytes = self._read_until('\xff') + # The WebSocket protocol section 4.4 specifies that invalid + # characters must be replaced with U+fffd REPLACEMENT + # CHARACTER. + message = bytes.decode('utf-8', 'replace') + if frame_type == 0x00: + return message + # Discard data of other types. + + def _send_closing_handshake(self): + if not self._enable_closing_handshake: + raise BadOperationException( + 'Closing handshake is not supported in Hixie 75 protocol') + + self._request.server_terminated = True + + # 5.3 the server may decide to terminate the WebSocket connection by + # running through the following steps: + # 1. send a 0xFF byte and a 0x00 byte to the client to indicate the + # start of the closing handshake. + self._write('\xff\x00') + + def close_connection(self, unused_code='', unused_reason=''): + """Closes a WebSocket connection. + + Raises: + ConnectionTerminatedException: when closing handshake was + not successfull. + """ + + if self._request.server_terminated: + self._logger.debug( + 'Requested close_connection but server is already terminated') + return + + if not self._enable_closing_handshake: + self._request.server_terminated = True + self._logger.debug('Connection closed') + return + + self._send_closing_handshake() + self._logger.debug('Sent server-initiated closing handshake') + + # TODO(ukai): 2. wait until the /client terminated/ flag has been set, + # or until a server-defined timeout expires. + # + # For now, we expect receiving closing handshake right after sending + # out closing handshake, and if we couldn't receive non-handshake + # frame, we take it as ConnectionTerminatedException. + message = self.receive_message() + if message is not None: + raise ConnectionTerminatedException( + 'Didn\'t receive valid ack for closing handshake') + # TODO: 3. close the WebSocket connection. + # note: mod_python Connection (mp_conn) doesn't have close method. + + def send_ping(self, body): + raise BadOperationException( + 'StreamHixie75 doesn\'t support send_ping') + + +# vi:sts=4 sw=4 et diff --git a/pyload/lib/mod_pywebsocket/_stream_hybi.py b/pyload/lib/mod_pywebsocket/_stream_hybi.py new file mode 100644 index 000000000..bd158fa6b --- /dev/null +++ b/pyload/lib/mod_pywebsocket/_stream_hybi.py @@ -0,0 +1,915 @@ +# Copyright 2012, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""This file provides classes and helper functions for parsing/building frames +of the WebSocket protocol (RFC 6455). + +Specification: +http://tools.ietf.org/html/rfc6455 +""" + + +from collections import deque +import logging +import os +import struct +import time + +from mod_pywebsocket import common +from mod_pywebsocket import util +from mod_pywebsocket._stream_base import BadOperationException +from mod_pywebsocket._stream_base import ConnectionTerminatedException +from mod_pywebsocket._stream_base import InvalidFrameException +from mod_pywebsocket._stream_base import InvalidUTF8Exception +from mod_pywebsocket._stream_base import StreamBase +from mod_pywebsocket._stream_base import UnsupportedFrameException + + +_NOOP_MASKER = util.NoopMasker() + + +class Frame(object): + + def __init__(self, fin=1, rsv1=0, rsv2=0, rsv3=0, + opcode=None, payload=''): + self.fin = fin + self.rsv1 = rsv1 + self.rsv2 = rsv2 + self.rsv3 = rsv3 + self.opcode = opcode + self.payload = payload + + +# Helper functions made public to be used for writing unittests for WebSocket +# clients. + + +def create_length_header(length, mask): + """Creates a length header. + + Args: + length: Frame length. Must be less than 2^63. + mask: Mask bit. Must be boolean. + + Raises: + ValueError: when bad data is given. + """ + + if mask: + mask_bit = 1 << 7 + else: + mask_bit = 0 + + if length < 0: + raise ValueError('length must be non negative integer') + elif length <= 125: + return chr(mask_bit | length) + elif length < (1 << 16): + return chr(mask_bit | 126) + struct.pack('!H', length) + elif length < (1 << 63): + return chr(mask_bit | 127) + struct.pack('!Q', length) + else: + raise ValueError('Payload is too big for one frame') + + +def create_header(opcode, payload_length, fin, rsv1, rsv2, rsv3, mask): + """Creates a frame header. + + Raises: + Exception: when bad data is given. + """ + + if opcode < 0 or 0xf < opcode: + raise ValueError('Opcode out of range') + + if payload_length < 0 or (1 << 63) <= payload_length: + raise ValueError('payload_length out of range') + + if (fin | rsv1 | rsv2 | rsv3) & ~1: + raise ValueError('FIN bit and Reserved bit parameter must be 0 or 1') + + header = '' + + first_byte = ((fin << 7) + | (rsv1 << 6) | (rsv2 << 5) | (rsv3 << 4) + | opcode) + header += chr(first_byte) + header += create_length_header(payload_length, mask) + + return header + + +def _build_frame(header, body, mask): + if not mask: + return header + body + + masking_nonce = os.urandom(4) + masker = util.RepeatedXorMasker(masking_nonce) + + return header + masking_nonce + masker.mask(body) + + +def _filter_and_format_frame_object(frame, mask, frame_filters): + for frame_filter in frame_filters: + frame_filter.filter(frame) + + header = create_header( + frame.opcode, len(frame.payload), frame.fin, + frame.rsv1, frame.rsv2, frame.rsv3, mask) + return _build_frame(header, frame.payload, mask) + + +def create_binary_frame( + message, opcode=common.OPCODE_BINARY, fin=1, mask=False, frame_filters=[]): + """Creates a simple binary frame with no extension, reserved bit.""" + + frame = Frame(fin=fin, opcode=opcode, payload=message) + return _filter_and_format_frame_object(frame, mask, frame_filters) + + +def create_text_frame( + message, opcode=common.OPCODE_TEXT, fin=1, mask=False, frame_filters=[]): + """Creates a simple text frame with no extension, reserved bit.""" + + encoded_message = message.encode('utf-8') + return create_binary_frame(encoded_message, opcode, fin, mask, + frame_filters) + + +def parse_frame(receive_bytes, logger=None, + ws_version=common.VERSION_HYBI_LATEST, + unmask_receive=True): + """Parses a frame. Returns a tuple containing each header field and + payload. + + Args: + receive_bytes: a function that reads frame data from a stream or + something similar. The function takes length of the bytes to be + read. The function must raise ConnectionTerminatedException if + there is not enough data to be read. + logger: a logging object. + ws_version: the version of WebSocket protocol. + unmask_receive: unmask received frames. When received unmasked + frame, raises InvalidFrameException. + + Raises: + ConnectionTerminatedException: when receive_bytes raises it. + InvalidFrameException: when the frame contains invalid data. + """ + + if not logger: + logger = logging.getLogger() + + logger.log(common.LOGLEVEL_FINE, 'Receive the first 2 octets of a frame') + + received = receive_bytes(2) + + first_byte = ord(received[0]) + fin = (first_byte >> 7) & 1 + rsv1 = (first_byte >> 6) & 1 + rsv2 = (first_byte >> 5) & 1 + rsv3 = (first_byte >> 4) & 1 + opcode = first_byte & 0xf + + second_byte = ord(received[1]) + mask = (second_byte >> 7) & 1 + payload_length = second_byte & 0x7f + + logger.log(common.LOGLEVEL_FINE, + 'FIN=%s, RSV1=%s, RSV2=%s, RSV3=%s, opcode=%s, ' + 'Mask=%s, Payload_length=%s', + fin, rsv1, rsv2, rsv3, opcode, mask, payload_length) + + if (mask == 1) != unmask_receive: + raise InvalidFrameException( + 'Mask bit on the received frame did\'nt match masking ' + 'configuration for received frames') + + # The HyBi and later specs disallow putting a value in 0x0-0xFFFF + # into the 8-octet extended payload length field (or 0x0-0xFD in + # 2-octet field). + valid_length_encoding = True + length_encoding_bytes = 1 + if payload_length == 127: + logger.log(common.LOGLEVEL_FINE, + 'Receive 8-octet extended payload length') + + extended_payload_length = receive_bytes(8) + payload_length = struct.unpack( + '!Q', extended_payload_length)[0] + if payload_length > 0x7FFFFFFFFFFFFFFF: + raise InvalidFrameException( + 'Extended payload length >= 2^63') + if ws_version >= 13 and payload_length < 0x10000: + valid_length_encoding = False + length_encoding_bytes = 8 + + logger.log(common.LOGLEVEL_FINE, + 'Decoded_payload_length=%s', payload_length) + elif payload_length == 126: + logger.log(common.LOGLEVEL_FINE, + 'Receive 2-octet extended payload length') + + extended_payload_length = receive_bytes(2) + payload_length = struct.unpack( + '!H', extended_payload_length)[0] + if ws_version >= 13 and payload_length < 126: + valid_length_encoding = False + length_encoding_bytes = 2 + + logger.log(common.LOGLEVEL_FINE, + 'Decoded_payload_length=%s', payload_length) + + if not valid_length_encoding: + logger.warning( + 'Payload length is not encoded using the minimal number of ' + 'bytes (%d is encoded using %d bytes)', + payload_length, + length_encoding_bytes) + + if mask == 1: + logger.log(common.LOGLEVEL_FINE, 'Receive mask') + + masking_nonce = receive_bytes(4) + masker = util.RepeatedXorMasker(masking_nonce) + + logger.log(common.LOGLEVEL_FINE, 'Mask=%r', masking_nonce) + else: + masker = _NOOP_MASKER + + logger.log(common.LOGLEVEL_FINE, 'Receive payload data') + if logger.isEnabledFor(common.LOGLEVEL_FINE): + receive_start = time.time() + + raw_payload_bytes = receive_bytes(payload_length) + + if logger.isEnabledFor(common.LOGLEVEL_FINE): + logger.log( + common.LOGLEVEL_FINE, + 'Done receiving payload data at %s MB/s', + payload_length / (time.time() - receive_start) / 1000 / 1000) + logger.log(common.LOGLEVEL_FINE, 'Unmask payload data') + + if logger.isEnabledFor(common.LOGLEVEL_FINE): + unmask_start = time.time() + + bytes = masker.mask(raw_payload_bytes) + + if logger.isEnabledFor(common.LOGLEVEL_FINE): + logger.log( + common.LOGLEVEL_FINE, + 'Done unmasking payload data at %s MB/s', + payload_length / (time.time() - unmask_start) / 1000 / 1000) + + return opcode, bytes, fin, rsv1, rsv2, rsv3 + + +class FragmentedFrameBuilder(object): + """A stateful class to send a message as fragments.""" + + def __init__(self, mask, frame_filters=[], encode_utf8=True): + """Constructs an instance.""" + + self._mask = mask + self._frame_filters = frame_filters + # This is for skipping UTF-8 encoding when building text type frames + # from compressed data. + self._encode_utf8 = encode_utf8 + + self._started = False + + # Hold opcode of the first frame in messages to verify types of other + # frames in the message are all the same. + self._opcode = common.OPCODE_TEXT + + def build(self, payload_data, end, binary): + if binary: + frame_type = common.OPCODE_BINARY + else: + frame_type = common.OPCODE_TEXT + if self._started: + if self._opcode != frame_type: + raise ValueError('Message types are different in frames for ' + 'the same message') + opcode = common.OPCODE_CONTINUATION + else: + opcode = frame_type + self._opcode = frame_type + + if end: + self._started = False + fin = 1 + else: + self._started = True + fin = 0 + + if binary or not self._encode_utf8: + return create_binary_frame( + payload_data, opcode, fin, self._mask, self._frame_filters) + else: + return create_text_frame( + payload_data, opcode, fin, self._mask, self._frame_filters) + + +def _create_control_frame(opcode, body, mask, frame_filters): + frame = Frame(opcode=opcode, payload=body) + + for frame_filter in frame_filters: + frame_filter.filter(frame) + + if len(frame.payload) > 125: + raise BadOperationException( + 'Payload data size of control frames must be 125 bytes or less') + + header = create_header( + frame.opcode, len(frame.payload), frame.fin, + frame.rsv1, frame.rsv2, frame.rsv3, mask) + return _build_frame(header, frame.payload, mask) + + +def create_ping_frame(body, mask=False, frame_filters=[]): + return _create_control_frame(common.OPCODE_PING, body, mask, frame_filters) + + +def create_pong_frame(body, mask=False, frame_filters=[]): + return _create_control_frame(common.OPCODE_PONG, body, mask, frame_filters) + + +def create_close_frame(body, mask=False, frame_filters=[]): + return _create_control_frame( + common.OPCODE_CLOSE, body, mask, frame_filters) + + +def create_closing_handshake_body(code, reason): + body = '' + if code is not None: + if (code > common.STATUS_USER_PRIVATE_MAX or + code < common.STATUS_NORMAL_CLOSURE): + raise BadOperationException('Status code is out of range') + if (code == common.STATUS_NO_STATUS_RECEIVED or + code == common.STATUS_ABNORMAL_CLOSURE or + code == common.STATUS_TLS_HANDSHAKE): + raise BadOperationException('Status code is reserved pseudo ' + 'code') + encoded_reason = reason.encode('utf-8') + body = struct.pack('!H', code) + encoded_reason + return body + + +class StreamOptions(object): + """Holds option values to configure Stream objects.""" + + def __init__(self): + """Constructs StreamOptions.""" + + # Enables deflate-stream extension. + self.deflate_stream = False + + # Filters applied to frames. + self.outgoing_frame_filters = [] + self.incoming_frame_filters = [] + + # Filters applied to messages. Control frames are not affected by them. + self.outgoing_message_filters = [] + self.incoming_message_filters = [] + + self.encode_text_message_to_utf8 = True + self.mask_send = False + self.unmask_receive = True + # RFC6455 disallows fragmented control frames, but mux extension + # relaxes the restriction. + self.allow_fragmented_control_frame = False + + +class Stream(StreamBase): + """A class for parsing/building frames of the WebSocket protocol + (RFC 6455). + """ + + def __init__(self, request, options): + """Constructs an instance. + + Args: + request: mod_python request. + """ + + StreamBase.__init__(self, request) + + self._logger = util.get_class_logger(self) + + self._options = options + + if self._options.deflate_stream: + self._logger.debug('Setup filter for deflate-stream') + self._request = util.DeflateRequest(self._request) + + self._request.client_terminated = False + self._request.server_terminated = False + + # Holds body of received fragments. + self._received_fragments = [] + # Holds the opcode of the first fragment. + self._original_opcode = None + + self._writer = FragmentedFrameBuilder( + self._options.mask_send, self._options.outgoing_frame_filters, + self._options.encode_text_message_to_utf8) + + self._ping_queue = deque() + + def _receive_frame(self): + """Receives a frame and return data in the frame as a tuple containing + each header field and payload separately. + + Raises: + ConnectionTerminatedException: when read returns empty + string. + InvalidFrameException: when the frame contains invalid data. + """ + + def _receive_bytes(length): + return self.receive_bytes(length) + + return parse_frame(receive_bytes=_receive_bytes, + logger=self._logger, + ws_version=self._request.ws_version, + unmask_receive=self._options.unmask_receive) + + def _receive_frame_as_frame_object(self): + opcode, bytes, fin, rsv1, rsv2, rsv3 = self._receive_frame() + + return Frame(fin=fin, rsv1=rsv1, rsv2=rsv2, rsv3=rsv3, + opcode=opcode, payload=bytes) + + def receive_filtered_frame(self): + """Receives a frame and applies frame filters and message filters. + The frame to be received must satisfy following conditions: + - The frame is not fragmented. + - The opcode of the frame is TEXT or BINARY. + + DO NOT USE this method except for testing purpose. + """ + + frame = self._receive_frame_as_frame_object() + if not frame.fin: + raise InvalidFrameException( + 'Segmented frames must not be received via ' + 'receive_filtered_frame()') + if (frame.opcode != common.OPCODE_TEXT and + frame.opcode != common.OPCODE_BINARY): + raise InvalidFrameException( + 'Control frames must not be received via ' + 'receive_filtered_frame()') + + for frame_filter in self._options.incoming_frame_filters: + frame_filter.filter(frame) + for message_filter in self._options.incoming_message_filters: + frame.payload = message_filter.filter(frame.payload) + return frame + + def send_message(self, message, end=True, binary=False): + """Send message. + + Args: + message: text in unicode or binary in str to send. + binary: send message as binary frame. + + Raises: + BadOperationException: when called on a server-terminated + connection or called with inconsistent message type or + binary parameter. + """ + + if self._request.server_terminated: + raise BadOperationException( + 'Requested send_message after sending out a closing handshake') + + if binary and isinstance(message, unicode): + raise BadOperationException( + 'Message for binary frame must be instance of str') + + for message_filter in self._options.outgoing_message_filters: + message = message_filter.filter(message, end, binary) + + try: + # Set this to any positive integer to limit maximum size of data in + # payload data of each frame. + MAX_PAYLOAD_DATA_SIZE = -1 + + if MAX_PAYLOAD_DATA_SIZE <= 0: + self._write(self._writer.build(message, end, binary)) + return + + bytes_written = 0 + while True: + end_for_this_frame = end + bytes_to_write = len(message) - bytes_written + if (MAX_PAYLOAD_DATA_SIZE > 0 and + bytes_to_write > MAX_PAYLOAD_DATA_SIZE): + end_for_this_frame = False + bytes_to_write = MAX_PAYLOAD_DATA_SIZE + + frame = self._writer.build( + message[bytes_written:bytes_written + bytes_to_write], + end_for_this_frame, + binary) + self._write(frame) + + bytes_written += bytes_to_write + + # This if must be placed here (the end of while block) so that + # at least one frame is sent. + if len(message) <= bytes_written: + break + except ValueError, e: + raise BadOperationException(e) + + def _get_message_from_frame(self, frame): + """Gets a message from frame. If the message is composed of fragmented + frames and the frame is not the last fragmented frame, this method + returns None. The whole message will be returned when the last + fragmented frame is passed to this method. + + Raises: + InvalidFrameException: when the frame doesn't match defragmentation + context, or the frame contains invalid data. + """ + + if frame.opcode == common.OPCODE_CONTINUATION: + if not self._received_fragments: + if frame.fin: + raise InvalidFrameException( + 'Received a termination frame but fragmentation ' + 'not started') + else: + raise InvalidFrameException( + 'Received an intermediate frame but ' + 'fragmentation not started') + + if frame.fin: + # End of fragmentation frame + self._received_fragments.append(frame.payload) + message = ''.join(self._received_fragments) + self._received_fragments = [] + return message + else: + # Intermediate frame + self._received_fragments.append(frame.payload) + return None + else: + if self._received_fragments: + if frame.fin: + raise InvalidFrameException( + 'Received an unfragmented frame without ' + 'terminating existing fragmentation') + else: + raise InvalidFrameException( + 'New fragmentation started without terminating ' + 'existing fragmentation') + + if frame.fin: + # Unfragmented frame + + self._original_opcode = frame.opcode + return frame.payload + else: + # Start of fragmentation frame + + if (not self._options.allow_fragmented_control_frame and + common.is_control_opcode(frame.opcode)): + raise InvalidFrameException( + 'Control frames must not be fragmented') + + self._original_opcode = frame.opcode + self._received_fragments.append(frame.payload) + return None + + def _process_close_message(self, message): + """Processes close message. + + Args: + message: close message. + + Raises: + InvalidFrameException: when the message is invalid. + """ + + self._request.client_terminated = True + + # Status code is optional. We can have status reason only if we + # have status code. Status reason can be empty string. So, + # allowed cases are + # - no application data: no code no reason + # - 2 octet of application data: has code but no reason + # - 3 or more octet of application data: both code and reason + if len(message) == 0: + self._logger.debug('Received close frame (empty body)') + self._request.ws_close_code = ( + common.STATUS_NO_STATUS_RECEIVED) + elif len(message) == 1: + raise InvalidFrameException( + 'If a close frame has status code, the length of ' + 'status code must be 2 octet') + elif len(message) >= 2: + self._request.ws_close_code = struct.unpack( + '!H', message[0:2])[0] + self._request.ws_close_reason = message[2:].decode( + 'utf-8', 'replace') + self._logger.debug( + 'Received close frame (code=%d, reason=%r)', + self._request.ws_close_code, + self._request.ws_close_reason) + + # Drain junk data after the close frame if necessary. + self._drain_received_data() + + if self._request.server_terminated: + self._logger.debug( + 'Received ack for server-initiated closing handshake') + return + + self._logger.debug( + 'Received client-initiated closing handshake') + + code = common.STATUS_NORMAL_CLOSURE + reason = '' + if hasattr(self._request, '_dispatcher'): + dispatcher = self._request._dispatcher + code, reason = dispatcher.passive_closing_handshake( + self._request) + if code is None and reason is not None and len(reason) > 0: + self._logger.warning( + 'Handler specified reason despite code being None') + reason = '' + if reason is None: + reason = '' + self._send_closing_handshake(code, reason) + self._logger.debug( + 'Sent ack for client-initiated closing handshake ' + '(code=%r, reason=%r)', code, reason) + + def _process_ping_message(self, message): + """Processes ping message. + + Args: + message: ping message. + """ + + try: + handler = self._request.on_ping_handler + if handler: + handler(self._request, message) + return + except AttributeError, e: + pass + self._send_pong(message) + + def _process_pong_message(self, message): + """Processes pong message. + + Args: + message: pong message. + """ + + # TODO(tyoshino): Add ping timeout handling. + + inflight_pings = deque() + + while True: + try: + expected_body = self._ping_queue.popleft() + if expected_body == message: + # inflight_pings contains pings ignored by the + # other peer. Just forget them. + self._logger.debug( + 'Ping %r is acked (%d pings were ignored)', + expected_body, len(inflight_pings)) + break + else: + inflight_pings.append(expected_body) + except IndexError, e: + # The received pong was unsolicited pong. Keep the + # ping queue as is. + self._ping_queue = inflight_pings + self._logger.debug('Received a unsolicited pong') + break + + try: + handler = self._request.on_pong_handler + if handler: + handler(self._request, message) + except AttributeError, e: + pass + + def receive_message(self): + """Receive a WebSocket frame and return its payload as a text in + unicode or a binary in str. + + Returns: + payload data of the frame + - as unicode instance if received text frame + - as str instance if received binary frame + or None iff received closing handshake. + Raises: + BadOperationException: when called on a client-terminated + connection. + ConnectionTerminatedException: when read returns empty + string. + InvalidFrameException: when the frame contains invalid + data. + UnsupportedFrameException: when the received frame has + flags, opcode we cannot handle. You can ignore this + exception and continue receiving the next frame. + """ + + if self._request.client_terminated: + raise BadOperationException( + 'Requested receive_message after receiving a closing ' + 'handshake') + + while True: + # mp_conn.read will block if no bytes are available. + # Timeout is controlled by TimeOut directive of Apache. + + frame = self._receive_frame_as_frame_object() + + # Check the constraint on the payload size for control frames + # before extension processes the frame. + # See also http://tools.ietf.org/html/rfc6455#section-5.5 + if (common.is_control_opcode(frame.opcode) and + len(frame.payload) > 125): + raise InvalidFrameException( + 'Payload data size of control frames must be 125 bytes or ' + 'less') + + for frame_filter in self._options.incoming_frame_filters: + frame_filter.filter(frame) + + if frame.rsv1 or frame.rsv2 or frame.rsv3: + raise UnsupportedFrameException( + 'Unsupported flag is set (rsv = %d%d%d)' % + (frame.rsv1, frame.rsv2, frame.rsv3)) + + message = self._get_message_from_frame(frame) + if message is None: + continue + + for message_filter in self._options.incoming_message_filters: + message = message_filter.filter(message) + + if self._original_opcode == common.OPCODE_TEXT: + # The WebSocket protocol section 4.4 specifies that invalid + # characters must be replaced with U+fffd REPLACEMENT + # CHARACTER. + try: + return message.decode('utf-8') + except UnicodeDecodeError, e: + raise InvalidUTF8Exception(e) + elif self._original_opcode == common.OPCODE_BINARY: + return message + elif self._original_opcode == common.OPCODE_CLOSE: + self._process_close_message(message) + return None + elif self._original_opcode == common.OPCODE_PING: + self._process_ping_message(message) + elif self._original_opcode == common.OPCODE_PONG: + self._process_pong_message(message) + else: + raise UnsupportedFrameException( + 'Opcode %d is not supported' % self._original_opcode) + + def _send_closing_handshake(self, code, reason): + body = create_closing_handshake_body(code, reason) + frame = create_close_frame( + body, mask=self._options.mask_send, + frame_filters=self._options.outgoing_frame_filters) + + self._request.server_terminated = True + + self._write(frame) + + def close_connection(self, code=common.STATUS_NORMAL_CLOSURE, reason=''): + """Closes a WebSocket connection. + + Args: + code: Status code for close frame. If code is None, a close + frame with empty body will be sent. + reason: string representing close reason. + Raises: + BadOperationException: when reason is specified with code None + or reason is not an instance of both str and unicode. + """ + + if self._request.server_terminated: + self._logger.debug( + 'Requested close_connection but server is already terminated') + return + + if code is None: + if reason is not None and len(reason) > 0: + raise BadOperationException( + 'close reason must not be specified if code is None') + reason = '' + else: + if not isinstance(reason, str) and not isinstance(reason, unicode): + raise BadOperationException( + 'close reason must be an instance of str or unicode') + + self._send_closing_handshake(code, reason) + self._logger.debug( + 'Sent server-initiated closing handshake (code=%r, reason=%r)', + code, reason) + + if (code == common.STATUS_GOING_AWAY or + code == common.STATUS_PROTOCOL_ERROR): + # It doesn't make sense to wait for a close frame if the reason is + # protocol error or that the server is going away. For some of + # other reasons, it might not make sense to wait for a close frame, + # but it's not clear, yet. + return + + # TODO(ukai): 2. wait until the /client terminated/ flag has been set, + # or until a server-defined timeout expires. + # + # For now, we expect receiving closing handshake right after sending + # out closing handshake. + message = self.receive_message() + if message is not None: + raise ConnectionTerminatedException( + 'Didn\'t receive valid ack for closing handshake') + # TODO: 3. close the WebSocket connection. + # note: mod_python Connection (mp_conn) doesn't have close method. + + def send_ping(self, body=''): + frame = create_ping_frame( + body, + self._options.mask_send, + self._options.outgoing_frame_filters) + self._write(frame) + + self._ping_queue.append(body) + + def _send_pong(self, body): + frame = create_pong_frame( + body, + self._options.mask_send, + self._options.outgoing_frame_filters) + self._write(frame) + + def get_last_received_opcode(self): + """Returns the opcode of the WebSocket message which the last received + frame belongs to. The return value is valid iff immediately after + receive_message call. + """ + + return self._original_opcode + + def _drain_received_data(self): + """Drains unread data in the receive buffer to avoid sending out TCP + RST packet. This is because when deflate-stream is enabled, some + DEFLATE block for flushing data may follow a close frame. If any data + remains in the receive buffer of a socket when the socket is closed, + it sends out TCP RST packet to the other peer. + + Since mod_python's mp_conn object doesn't support non-blocking read, + we perform this only when pywebsocket is running in standalone mode. + """ + + # If self._options.deflate_stream is true, self._request is + # DeflateRequest, so we can get wrapped request object by + # self._request._request. + # + # Only _StandaloneRequest has _drain_received_data method. + if (self._options.deflate_stream and + ('_drain_received_data' in dir(self._request._request))): + self._request._request._drain_received_data() + + +# vi:sts=4 sw=4 et diff --git a/pyload/lib/mod_pywebsocket/common.py b/pyload/lib/mod_pywebsocket/common.py new file mode 100644 index 000000000..2388379c0 --- /dev/null +++ b/pyload/lib/mod_pywebsocket/common.py @@ -0,0 +1,307 @@ +# Copyright 2012, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""This file must not depend on any module specific to the WebSocket protocol. +""" + + +from mod_pywebsocket import http_header_util + + +# Additional log level definitions. +LOGLEVEL_FINE = 9 + +# Constants indicating WebSocket protocol version. +VERSION_HIXIE75 = -1 +VERSION_HYBI00 = 0 +VERSION_HYBI01 = 1 +VERSION_HYBI02 = 2 +VERSION_HYBI03 = 2 +VERSION_HYBI04 = 4 +VERSION_HYBI05 = 5 +VERSION_HYBI06 = 6 +VERSION_HYBI07 = 7 +VERSION_HYBI08 = 8 +VERSION_HYBI09 = 8 +VERSION_HYBI10 = 8 +VERSION_HYBI11 = 8 +VERSION_HYBI12 = 8 +VERSION_HYBI13 = 13 +VERSION_HYBI14 = 13 +VERSION_HYBI15 = 13 +VERSION_HYBI16 = 13 +VERSION_HYBI17 = 13 + +# Constants indicating WebSocket protocol latest version. +VERSION_HYBI_LATEST = VERSION_HYBI13 + +# Port numbers +DEFAULT_WEB_SOCKET_PORT = 80 +DEFAULT_WEB_SOCKET_SECURE_PORT = 443 + +# Schemes +WEB_SOCKET_SCHEME = 'ws' +WEB_SOCKET_SECURE_SCHEME = 'wss' + +# Frame opcodes defined in the spec. +OPCODE_CONTINUATION = 0x0 +OPCODE_TEXT = 0x1 +OPCODE_BINARY = 0x2 +OPCODE_CLOSE = 0x8 +OPCODE_PING = 0x9 +OPCODE_PONG = 0xa + +# UUIDs used by HyBi 04 and later opening handshake and frame masking. +WEBSOCKET_ACCEPT_UUID = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11' + +# Opening handshake header names and expected values. +UPGRADE_HEADER = 'Upgrade' +WEBSOCKET_UPGRADE_TYPE = 'websocket' +WEBSOCKET_UPGRADE_TYPE_HIXIE75 = 'WebSocket' +CONNECTION_HEADER = 'Connection' +UPGRADE_CONNECTION_TYPE = 'Upgrade' +HOST_HEADER = 'Host' +ORIGIN_HEADER = 'Origin' +SEC_WEBSOCKET_ORIGIN_HEADER = 'Sec-WebSocket-Origin' +SEC_WEBSOCKET_KEY_HEADER = 'Sec-WebSocket-Key' +SEC_WEBSOCKET_ACCEPT_HEADER = 'Sec-WebSocket-Accept' +SEC_WEBSOCKET_VERSION_HEADER = 'Sec-WebSocket-Version' +SEC_WEBSOCKET_PROTOCOL_HEADER = 'Sec-WebSocket-Protocol' +SEC_WEBSOCKET_EXTENSIONS_HEADER = 'Sec-WebSocket-Extensions' +SEC_WEBSOCKET_DRAFT_HEADER = 'Sec-WebSocket-Draft' +SEC_WEBSOCKET_KEY1_HEADER = 'Sec-WebSocket-Key1' +SEC_WEBSOCKET_KEY2_HEADER = 'Sec-WebSocket-Key2' +SEC_WEBSOCKET_LOCATION_HEADER = 'Sec-WebSocket-Location' + +# Extensions +DEFLATE_STREAM_EXTENSION = 'deflate-stream' +DEFLATE_FRAME_EXTENSION = 'deflate-frame' +PERFRAME_COMPRESSION_EXTENSION = 'perframe-compress' +PERMESSAGE_COMPRESSION_EXTENSION = 'permessage-compress' +X_WEBKIT_DEFLATE_FRAME_EXTENSION = 'x-webkit-deflate-frame' +X_WEBKIT_PERMESSAGE_COMPRESSION_EXTENSION = 'x-webkit-permessage-compress' +MUX_EXTENSION = 'mux_DO_NOT_USE' + +# Status codes +# Code STATUS_NO_STATUS_RECEIVED, STATUS_ABNORMAL_CLOSURE, and +# STATUS_TLS_HANDSHAKE are pseudo codes to indicate specific error cases. +# Could not be used for codes in actual closing frames. +# Application level errors must use codes in the range +# STATUS_USER_REGISTERED_BASE to STATUS_USER_PRIVATE_MAX. The codes in the +# range STATUS_USER_REGISTERED_BASE to STATUS_USER_REGISTERED_MAX are managed +# by IANA. Usually application must define user protocol level errors in the +# range STATUS_USER_PRIVATE_BASE to STATUS_USER_PRIVATE_MAX. +STATUS_NORMAL_CLOSURE = 1000 +STATUS_GOING_AWAY = 1001 +STATUS_PROTOCOL_ERROR = 1002 +STATUS_UNSUPPORTED_DATA = 1003 +STATUS_NO_STATUS_RECEIVED = 1005 +STATUS_ABNORMAL_CLOSURE = 1006 +STATUS_INVALID_FRAME_PAYLOAD_DATA = 1007 +STATUS_POLICY_VIOLATION = 1008 +STATUS_MESSAGE_TOO_BIG = 1009 +STATUS_MANDATORY_EXTENSION = 1010 +STATUS_INTERNAL_ENDPOINT_ERROR = 1011 +STATUS_TLS_HANDSHAKE = 1015 +STATUS_USER_REGISTERED_BASE = 3000 +STATUS_USER_REGISTERED_MAX = 3999 +STATUS_USER_PRIVATE_BASE = 4000 +STATUS_USER_PRIVATE_MAX = 4999 +# Following definitions are aliases to keep compatibility. Applications must +# not use these obsoleted definitions anymore. +STATUS_NORMAL = STATUS_NORMAL_CLOSURE +STATUS_UNSUPPORTED = STATUS_UNSUPPORTED_DATA +STATUS_CODE_NOT_AVAILABLE = STATUS_NO_STATUS_RECEIVED +STATUS_ABNORMAL_CLOSE = STATUS_ABNORMAL_CLOSURE +STATUS_INVALID_FRAME_PAYLOAD = STATUS_INVALID_FRAME_PAYLOAD_DATA +STATUS_MANDATORY_EXT = STATUS_MANDATORY_EXTENSION + +# HTTP status codes +HTTP_STATUS_BAD_REQUEST = 400 +HTTP_STATUS_FORBIDDEN = 403 +HTTP_STATUS_NOT_FOUND = 404 + + +def is_control_opcode(opcode): + return (opcode >> 3) == 1 + + +class ExtensionParameter(object): + """Holds information about an extension which is exchanged on extension + negotiation in opening handshake. + """ + + def __init__(self, name): + self._name = name + # TODO(tyoshino): Change the data structure to more efficient one such + # as dict when the spec changes to say like + # - Parameter names must be unique + # - The order of parameters is not significant + self._parameters = [] + + def name(self): + return self._name + + def add_parameter(self, name, value): + self._parameters.append((name, value)) + + def get_parameters(self): + return self._parameters + + def get_parameter_names(self): + return [name for name, unused_value in self._parameters] + + def has_parameter(self, name): + for param_name, param_value in self._parameters: + if param_name == name: + return True + return False + + def get_parameter_value(self, name): + for param_name, param_value in self._parameters: + if param_name == name: + return param_value + + +class ExtensionParsingException(Exception): + def __init__(self, name): + super(ExtensionParsingException, self).__init__(name) + + +def _parse_extension_param(state, definition, allow_quoted_string): + param_name = http_header_util.consume_token(state) + + if param_name is None: + raise ExtensionParsingException('No valid parameter name found') + + http_header_util.consume_lwses(state) + + if not http_header_util.consume_string(state, '='): + definition.add_parameter(param_name, None) + return + + http_header_util.consume_lwses(state) + + if allow_quoted_string: + # TODO(toyoshim): Add code to validate that parsed param_value is token + param_value = http_header_util.consume_token_or_quoted_string(state) + else: + param_value = http_header_util.consume_token(state) + if param_value is None: + raise ExtensionParsingException( + 'No valid parameter value found on the right-hand side of ' + 'parameter %r' % param_name) + + definition.add_parameter(param_name, param_value) + + +def _parse_extension(state, allow_quoted_string): + extension_token = http_header_util.consume_token(state) + if extension_token is None: + return None + + extension = ExtensionParameter(extension_token) + + while True: + http_header_util.consume_lwses(state) + + if not http_header_util.consume_string(state, ';'): + break + + http_header_util.consume_lwses(state) + + try: + _parse_extension_param(state, extension, allow_quoted_string) + except ExtensionParsingException, e: + raise ExtensionParsingException( + 'Failed to parse parameter for %r (%r)' % + (extension_token, e)) + + return extension + + +def parse_extensions(data, allow_quoted_string=False): + """Parses Sec-WebSocket-Extensions header value returns a list of + ExtensionParameter objects. + + Leading LWSes must be trimmed. + """ + + state = http_header_util.ParsingState(data) + + extension_list = [] + while True: + extension = _parse_extension(state, allow_quoted_string) + if extension is not None: + extension_list.append(extension) + + http_header_util.consume_lwses(state) + + if http_header_util.peek(state) is None: + break + + if not http_header_util.consume_string(state, ','): + raise ExtensionParsingException( + 'Failed to parse Sec-WebSocket-Extensions header: ' + 'Expected a comma but found %r' % + http_header_util.peek(state)) + + http_header_util.consume_lwses(state) + + if len(extension_list) == 0: + raise ExtensionParsingException( + 'No valid extension entry found') + + return extension_list + + +def format_extension(extension): + """Formats an ExtensionParameter object.""" + + formatted_params = [extension.name()] + for param_name, param_value in extension.get_parameters(): + if param_value is None: + formatted_params.append(param_name) + else: + quoted_value = http_header_util.quote_if_necessary(param_value) + formatted_params.append('%s=%s' % (param_name, quoted_value)) + return '; '.join(formatted_params) + + +def format_extensions(extension_list): + """Formats a list of ExtensionParameter objects.""" + + formatted_extension_list = [] + for extension in extension_list: + formatted_extension_list.append(format_extension(extension)) + return ', '.join(formatted_extension_list) + + +# vi:sts=4 sw=4 et diff --git a/pyload/lib/mod_pywebsocket/dispatch.py b/pyload/lib/mod_pywebsocket/dispatch.py new file mode 100644 index 000000000..25905f180 --- /dev/null +++ b/pyload/lib/mod_pywebsocket/dispatch.py @@ -0,0 +1,387 @@ +# Copyright 2012, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Dispatch WebSocket request. +""" + + +import logging +import os +import re + +from mod_pywebsocket import common +from mod_pywebsocket import handshake +from mod_pywebsocket import msgutil +from mod_pywebsocket import mux +from mod_pywebsocket import stream +from mod_pywebsocket import util + + +_SOURCE_PATH_PATTERN = re.compile(r'(?i)_wsh\.py$') +_SOURCE_SUFFIX = '_wsh.py' +_DO_EXTRA_HANDSHAKE_HANDLER_NAME = 'web_socket_do_extra_handshake' +_TRANSFER_DATA_HANDLER_NAME = 'web_socket_transfer_data' +_PASSIVE_CLOSING_HANDSHAKE_HANDLER_NAME = ( + 'web_socket_passive_closing_handshake') + + +class DispatchException(Exception): + """Exception in dispatching WebSocket request.""" + + def __init__(self, name, status=common.HTTP_STATUS_NOT_FOUND): + super(DispatchException, self).__init__(name) + self.status = status + + +def _default_passive_closing_handshake_handler(request): + """Default web_socket_passive_closing_handshake handler.""" + + return common.STATUS_NORMAL_CLOSURE, '' + + +def _normalize_path(path): + """Normalize path. + + Args: + path: the path to normalize. + + Path is converted to the absolute path. + The input path can use either '\\' or '/' as the separator. + The normalized path always uses '/' regardless of the platform. + """ + + path = path.replace('\\', os.path.sep) + path = os.path.realpath(path) + path = path.replace('\\', '/') + return path + + +def _create_path_to_resource_converter(base_dir): + """Returns a function that converts the path of a WebSocket handler source + file to a resource string by removing the path to the base directory from + its head, removing _SOURCE_SUFFIX from its tail, and replacing path + separators in it with '/'. + + Args: + base_dir: the path to the base directory. + """ + + base_dir = _normalize_path(base_dir) + + base_len = len(base_dir) + suffix_len = len(_SOURCE_SUFFIX) + + def converter(path): + if not path.endswith(_SOURCE_SUFFIX): + return None + # _normalize_path must not be used because resolving symlink breaks + # following path check. + path = path.replace('\\', '/') + if not path.startswith(base_dir): + return None + return path[base_len:-suffix_len] + + return converter + + +def _enumerate_handler_file_paths(directory): + """Returns a generator that enumerates WebSocket Handler source file names + in the given directory. + """ + + for root, unused_dirs, files in os.walk(directory): + for base in files: + path = os.path.join(root, base) + if _SOURCE_PATH_PATTERN.search(path): + yield path + + +class _HandlerSuite(object): + """A handler suite holder class.""" + + def __init__(self, do_extra_handshake, transfer_data, + passive_closing_handshake): + self.do_extra_handshake = do_extra_handshake + self.transfer_data = transfer_data + self.passive_closing_handshake = passive_closing_handshake + + +def _source_handler_file(handler_definition): + """Source a handler definition string. + + Args: + handler_definition: a string containing Python statements that define + handler functions. + """ + + global_dic = {} + try: + exec handler_definition in global_dic + except Exception: + raise DispatchException('Error in sourcing handler:' + + util.get_stack_trace()) + passive_closing_handshake_handler = None + try: + passive_closing_handshake_handler = _extract_handler( + global_dic, _PASSIVE_CLOSING_HANDSHAKE_HANDLER_NAME) + except Exception: + passive_closing_handshake_handler = ( + _default_passive_closing_handshake_handler) + return _HandlerSuite( + _extract_handler(global_dic, _DO_EXTRA_HANDSHAKE_HANDLER_NAME), + _extract_handler(global_dic, _TRANSFER_DATA_HANDLER_NAME), + passive_closing_handshake_handler) + + +def _extract_handler(dic, name): + """Extracts a callable with the specified name from the given dictionary + dic. + """ + + if name not in dic: + raise DispatchException('%s is not defined.' % name) + handler = dic[name] + if not callable(handler): + raise DispatchException('%s is not callable.' % name) + return handler + + +class Dispatcher(object): + """Dispatches WebSocket requests. + + This class maintains a map from resource name to handlers. + """ + + def __init__( + self, root_dir, scan_dir=None, + allow_handlers_outside_root_dir=True): + """Construct an instance. + + Args: + root_dir: The directory where handler definition files are + placed. + scan_dir: The directory where handler definition files are + searched. scan_dir must be a directory under root_dir, + including root_dir itself. If scan_dir is None, + root_dir is used as scan_dir. scan_dir can be useful + in saving scan time when root_dir contains many + subdirectories. + allow_handlers_outside_root_dir: Scans handler files even if their + canonical path is not under root_dir. + """ + + self._logger = util.get_class_logger(self) + + self._handler_suite_map = {} + self._source_warnings = [] + if scan_dir is None: + scan_dir = root_dir + if not os.path.realpath(scan_dir).startswith( + os.path.realpath(root_dir)): + raise DispatchException('scan_dir:%s must be a directory under ' + 'root_dir:%s.' % (scan_dir, root_dir)) + self._source_handler_files_in_dir( + root_dir, scan_dir, allow_handlers_outside_root_dir) + + def add_resource_path_alias(self, + alias_resource_path, existing_resource_path): + """Add resource path alias. + + Once added, request to alias_resource_path would be handled by + handler registered for existing_resource_path. + + Args: + alias_resource_path: alias resource path + existing_resource_path: existing resource path + """ + try: + handler_suite = self._handler_suite_map[existing_resource_path] + self._handler_suite_map[alias_resource_path] = handler_suite + except KeyError: + raise DispatchException('No handler for: %r' % + existing_resource_path) + + def source_warnings(self): + """Return warnings in sourcing handlers.""" + + return self._source_warnings + + def do_extra_handshake(self, request): + """Do extra checking in WebSocket handshake. + + Select a handler based on request.uri and call its + web_socket_do_extra_handshake function. + + Args: + request: mod_python request. + + Raises: + DispatchException: when handler was not found + AbortedByUserException: when user handler abort connection + HandshakeException: when opening handshake failed + """ + + handler_suite = self.get_handler_suite(request.ws_resource) + if handler_suite is None: + raise DispatchException('No handler for: %r' % request.ws_resource) + do_extra_handshake_ = handler_suite.do_extra_handshake + try: + do_extra_handshake_(request) + except handshake.AbortedByUserException, e: + raise + except Exception, e: + util.prepend_message_to_exception( + '%s raised exception for %s: ' % ( + _DO_EXTRA_HANDSHAKE_HANDLER_NAME, + request.ws_resource), + e) + raise handshake.HandshakeException(e, common.HTTP_STATUS_FORBIDDEN) + + def transfer_data(self, request): + """Let a handler transfer_data with a WebSocket client. + + Select a handler based on request.ws_resource and call its + web_socket_transfer_data function. + + Args: + request: mod_python request. + + Raises: + DispatchException: when handler was not found + AbortedByUserException: when user handler abort connection + """ + + # TODO(tyoshino): Terminate underlying TCP connection if possible. + try: + if mux.use_mux(request): + mux.start(request, self) + else: + handler_suite = self.get_handler_suite(request.ws_resource) + if handler_suite is None: + raise DispatchException('No handler for: %r' % + request.ws_resource) + transfer_data_ = handler_suite.transfer_data + transfer_data_(request) + + if not request.server_terminated: + request.ws_stream.close_connection() + # Catch non-critical exceptions the handler didn't handle. + except handshake.AbortedByUserException, e: + self._logger.debug('%s', e) + raise + except msgutil.BadOperationException, e: + self._logger.debug('%s', e) + request.ws_stream.close_connection(common.STATUS_ABNORMAL_CLOSURE) + except msgutil.InvalidFrameException, e: + # InvalidFrameException must be caught before + # ConnectionTerminatedException that catches InvalidFrameException. + self._logger.debug('%s', e) + request.ws_stream.close_connection(common.STATUS_PROTOCOL_ERROR) + except msgutil.UnsupportedFrameException, e: + self._logger.debug('%s', e) + request.ws_stream.close_connection(common.STATUS_UNSUPPORTED_DATA) + except stream.InvalidUTF8Exception, e: + self._logger.debug('%s', e) + request.ws_stream.close_connection( + common.STATUS_INVALID_FRAME_PAYLOAD_DATA) + except msgutil.ConnectionTerminatedException, e: + self._logger.debug('%s', e) + except Exception, e: + util.prepend_message_to_exception( + '%s raised exception for %s: ' % ( + _TRANSFER_DATA_HANDLER_NAME, request.ws_resource), + e) + raise + + def passive_closing_handshake(self, request): + """Prepare code and reason for responding client initiated closing + handshake. + """ + + handler_suite = self.get_handler_suite(request.ws_resource) + if handler_suite is None: + return _default_passive_closing_handshake_handler(request) + return handler_suite.passive_closing_handshake(request) + + def get_handler_suite(self, resource): + """Retrieves two handlers (one for extra handshake processing, and one + for data transfer) for the given request as a HandlerSuite object. + """ + + fragment = None + if '#' in resource: + resource, fragment = resource.split('#', 1) + if '?' in resource: + resource = resource.split('?', 1)[0] + handler_suite = self._handler_suite_map.get(resource) + if handler_suite and fragment: + raise DispatchException('Fragment identifiers MUST NOT be used on ' + 'WebSocket URIs', + common.HTTP_STATUS_BAD_REQUEST) + return handler_suite + + def _source_handler_files_in_dir( + self, root_dir, scan_dir, allow_handlers_outside_root_dir): + """Source all the handler source files in the scan_dir directory. + + The resource path is determined relative to root_dir. + """ + + # We build a map from resource to handler code assuming that there's + # only one path from root_dir to scan_dir and it can be obtained by + # comparing realpath of them. + + # Here we cannot use abspath. See + # https://bugs.webkit.org/show_bug.cgi?id=31603 + + convert = _create_path_to_resource_converter(root_dir) + scan_realpath = os.path.realpath(scan_dir) + root_realpath = os.path.realpath(root_dir) + for path in _enumerate_handler_file_paths(scan_realpath): + if (not allow_handlers_outside_root_dir and + (not os.path.realpath(path).startswith(root_realpath))): + self._logger.debug( + 'Canonical path of %s is not under root directory' % + path) + continue + try: + handler_suite = _source_handler_file(open(path).read()) + except DispatchException, e: + self._source_warnings.append('%s: %s' % (path, e)) + continue + resource = convert(path) + if resource is None: + self._logger.debug( + 'Path to resource conversion on %s failed' % path) + else: + self._handler_suite_map[convert(path)] = handler_suite + + +# vi:sts=4 sw=4 et diff --git a/pyload/lib/mod_pywebsocket/extensions.py b/pyload/lib/mod_pywebsocket/extensions.py new file mode 100644 index 000000000..03dbf9ee1 --- /dev/null +++ b/pyload/lib/mod_pywebsocket/extensions.py @@ -0,0 +1,727 @@ +# Copyright 2012, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +from mod_pywebsocket import common +from mod_pywebsocket import util +from mod_pywebsocket.http_header_util import quote_if_necessary + + +_available_processors = {} + + +class ExtensionProcessorInterface(object): + + def name(self): + return None + + def get_extension_response(self): + return None + + def setup_stream_options(self, stream_options): + pass + + +class DeflateStreamExtensionProcessor(ExtensionProcessorInterface): + """WebSocket DEFLATE stream extension processor. + + Specification: + Section 9.2.1 in + http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-10 + """ + + def __init__(self, request): + self._logger = util.get_class_logger(self) + + self._request = request + + def name(self): + return common.DEFLATE_STREAM_EXTENSION + + def get_extension_response(self): + if len(self._request.get_parameter_names()) != 0: + return None + + self._logger.debug( + 'Enable %s extension', common.DEFLATE_STREAM_EXTENSION) + + return common.ExtensionParameter(common.DEFLATE_STREAM_EXTENSION) + + def setup_stream_options(self, stream_options): + stream_options.deflate_stream = True + + +_available_processors[common.DEFLATE_STREAM_EXTENSION] = ( + DeflateStreamExtensionProcessor) + + +def _log_compression_ratio(logger, original_bytes, total_original_bytes, + filtered_bytes, total_filtered_bytes): + # Print inf when ratio is not available. + ratio = float('inf') + average_ratio = float('inf') + if original_bytes != 0: + ratio = float(filtered_bytes) / original_bytes + if total_original_bytes != 0: + average_ratio = ( + float(total_filtered_bytes) / total_original_bytes) + logger.debug('Outgoing compress ratio: %f (average: %f)' % + (ratio, average_ratio)) + + +def _log_decompression_ratio(logger, received_bytes, total_received_bytes, + filtered_bytes, total_filtered_bytes): + # Print inf when ratio is not available. + ratio = float('inf') + average_ratio = float('inf') + if received_bytes != 0: + ratio = float(received_bytes) / filtered_bytes + if total_filtered_bytes != 0: + average_ratio = ( + float(total_received_bytes) / total_filtered_bytes) + logger.debug('Incoming compress ratio: %f (average: %f)' % + (ratio, average_ratio)) + + +class DeflateFrameExtensionProcessor(ExtensionProcessorInterface): + """WebSocket Per-frame DEFLATE extension processor. + + Specification: + http://tools.ietf.org/html/draft-tyoshino-hybi-websocket-perframe-deflate + """ + + _WINDOW_BITS_PARAM = 'max_window_bits' + _NO_CONTEXT_TAKEOVER_PARAM = 'no_context_takeover' + + def __init__(self, request): + self._logger = util.get_class_logger(self) + + self._request = request + + self._response_window_bits = None + self._response_no_context_takeover = False + self._bfinal = False + + # Counters for statistics. + + # Total number of outgoing bytes supplied to this filter. + self._total_outgoing_payload_bytes = 0 + # Total number of bytes sent to the network after applying this filter. + self._total_filtered_outgoing_payload_bytes = 0 + + # Total number of bytes received from the network. + self._total_incoming_payload_bytes = 0 + # Total number of incoming bytes obtained after applying this filter. + self._total_filtered_incoming_payload_bytes = 0 + + def name(self): + return common.DEFLATE_FRAME_EXTENSION + + def get_extension_response(self): + # Any unknown parameter will be just ignored. + + window_bits = self._request.get_parameter_value( + self._WINDOW_BITS_PARAM) + no_context_takeover = self._request.has_parameter( + self._NO_CONTEXT_TAKEOVER_PARAM) + if (no_context_takeover and + self._request.get_parameter_value( + self._NO_CONTEXT_TAKEOVER_PARAM) is not None): + return None + + if window_bits is not None: + try: + window_bits = int(window_bits) + except ValueError, e: + return None + if window_bits < 8 or window_bits > 15: + return None + + self._deflater = util._RFC1979Deflater( + window_bits, no_context_takeover) + + self._inflater = util._RFC1979Inflater() + + self._compress_outgoing = True + + response = common.ExtensionParameter(self._request.name()) + + if self._response_window_bits is not None: + response.add_parameter( + self._WINDOW_BITS_PARAM, str(self._response_window_bits)) + if self._response_no_context_takeover: + response.add_parameter( + self._NO_CONTEXT_TAKEOVER_PARAM, None) + + self._logger.debug( + 'Enable %s extension (' + 'request: window_bits=%s; no_context_takeover=%r, ' + 'response: window_wbits=%s; no_context_takeover=%r)' % + (self._request.name(), + window_bits, + no_context_takeover, + self._response_window_bits, + self._response_no_context_takeover)) + + return response + + def setup_stream_options(self, stream_options): + + class _OutgoingFilter(object): + + def __init__(self, parent): + self._parent = parent + + def filter(self, frame): + self._parent._outgoing_filter(frame) + + class _IncomingFilter(object): + + def __init__(self, parent): + self._parent = parent + + def filter(self, frame): + self._parent._incoming_filter(frame) + + stream_options.outgoing_frame_filters.append( + _OutgoingFilter(self)) + stream_options.incoming_frame_filters.insert( + 0, _IncomingFilter(self)) + + def set_response_window_bits(self, value): + self._response_window_bits = value + + def set_response_no_context_takeover(self, value): + self._response_no_context_takeover = value + + def set_bfinal(self, value): + self._bfinal = value + + def enable_outgoing_compression(self): + self._compress_outgoing = True + + def disable_outgoing_compression(self): + self._compress_outgoing = False + + def _outgoing_filter(self, frame): + """Transform outgoing frames. This method is called only by + an _OutgoingFilter instance. + """ + + original_payload_size = len(frame.payload) + self._total_outgoing_payload_bytes += original_payload_size + + if (not self._compress_outgoing or + common.is_control_opcode(frame.opcode)): + self._total_filtered_outgoing_payload_bytes += ( + original_payload_size) + return + + frame.payload = self._deflater.filter( + frame.payload, bfinal=self._bfinal) + frame.rsv1 = 1 + + filtered_payload_size = len(frame.payload) + self._total_filtered_outgoing_payload_bytes += filtered_payload_size + + _log_compression_ratio(self._logger, original_payload_size, + self._total_outgoing_payload_bytes, + filtered_payload_size, + self._total_filtered_outgoing_payload_bytes) + + def _incoming_filter(self, frame): + """Transform incoming frames. This method is called only by + an _IncomingFilter instance. + """ + + received_payload_size = len(frame.payload) + self._total_incoming_payload_bytes += received_payload_size + + if frame.rsv1 != 1 or common.is_control_opcode(frame.opcode): + self._total_filtered_incoming_payload_bytes += ( + received_payload_size) + return + + frame.payload = self._inflater.filter(frame.payload) + frame.rsv1 = 0 + + filtered_payload_size = len(frame.payload) + self._total_filtered_incoming_payload_bytes += filtered_payload_size + + _log_decompression_ratio(self._logger, received_payload_size, + self._total_incoming_payload_bytes, + filtered_payload_size, + self._total_filtered_incoming_payload_bytes) + + +_available_processors[common.DEFLATE_FRAME_EXTENSION] = ( + DeflateFrameExtensionProcessor) + + +# Adding vendor-prefixed deflate-frame extension. +# TODO(bashi): Remove this after WebKit stops using vendor prefix. +_available_processors[common.X_WEBKIT_DEFLATE_FRAME_EXTENSION] = ( + DeflateFrameExtensionProcessor) + + +def _parse_compression_method(data): + """Parses the value of "method" extension parameter.""" + + return common.parse_extensions(data, allow_quoted_string=True) + + +def _create_accepted_method_desc(method_name, method_params): + """Creates accepted-method-desc from given method name and parameters""" + + extension = common.ExtensionParameter(method_name) + for name, value in method_params: + extension.add_parameter(name, value) + return common.format_extension(extension) + + +class CompressionExtensionProcessorBase(ExtensionProcessorInterface): + """Base class for Per-frame and Per-message compression extension.""" + + _METHOD_PARAM = 'method' + + def __init__(self, request): + self._logger = util.get_class_logger(self) + self._request = request + self._compression_method_name = None + self._compression_processor = None + self._compression_processor_hook = None + + def name(self): + return '' + + def _lookup_compression_processor(self, method_desc): + return None + + def _get_compression_processor_response(self): + """Looks up the compression processor based on the self._request and + returns the compression processor's response. + """ + + method_list = self._request.get_parameter_value(self._METHOD_PARAM) + if method_list is None: + return None + methods = _parse_compression_method(method_list) + if methods is None: + return None + comression_processor = None + # The current implementation tries only the first method that matches + # supported algorithm. Following methods aren't tried even if the + # first one is rejected. + # TODO(bashi): Need to clarify this behavior. + for method_desc in methods: + compression_processor = self._lookup_compression_processor( + method_desc) + if compression_processor is not None: + self._compression_method_name = method_desc.name() + break + if compression_processor is None: + return None + + if self._compression_processor_hook: + self._compression_processor_hook(compression_processor) + + processor_response = compression_processor.get_extension_response() + if processor_response is None: + return None + self._compression_processor = compression_processor + return processor_response + + def get_extension_response(self): + processor_response = self._get_compression_processor_response() + if processor_response is None: + return None + + response = common.ExtensionParameter(self._request.name()) + accepted_method_desc = _create_accepted_method_desc( + self._compression_method_name, + processor_response.get_parameters()) + response.add_parameter(self._METHOD_PARAM, accepted_method_desc) + self._logger.debug( + 'Enable %s extension (method: %s)' % + (self._request.name(), self._compression_method_name)) + return response + + def setup_stream_options(self, stream_options): + if self._compression_processor is None: + return + self._compression_processor.setup_stream_options(stream_options) + + def set_compression_processor_hook(self, hook): + self._compression_processor_hook = hook + + def get_compression_processor(self): + return self._compression_processor + + +class PerFrameCompressionExtensionProcessor(CompressionExtensionProcessorBase): + """WebSocket Per-frame compression extension processor. + + Specification: + http://tools.ietf.org/html/draft-ietf-hybi-websocket-perframe-compression + """ + + _DEFLATE_METHOD = 'deflate' + + def __init__(self, request): + CompressionExtensionProcessorBase.__init__(self, request) + + def name(self): + return common.PERFRAME_COMPRESSION_EXTENSION + + def _lookup_compression_processor(self, method_desc): + if method_desc.name() == self._DEFLATE_METHOD: + return DeflateFrameExtensionProcessor(method_desc) + return None + + +_available_processors[common.PERFRAME_COMPRESSION_EXTENSION] = ( + PerFrameCompressionExtensionProcessor) + + +class DeflateMessageProcessor(ExtensionProcessorInterface): + """Per-message deflate processor.""" + + _S2C_MAX_WINDOW_BITS_PARAM = 's2c_max_window_bits' + _S2C_NO_CONTEXT_TAKEOVER_PARAM = 's2c_no_context_takeover' + _C2S_MAX_WINDOW_BITS_PARAM = 'c2s_max_window_bits' + _C2S_NO_CONTEXT_TAKEOVER_PARAM = 'c2s_no_context_takeover' + + def __init__(self, request): + self._request = request + self._logger = util.get_class_logger(self) + + self._c2s_max_window_bits = None + self._c2s_no_context_takeover = False + self._bfinal = False + + self._compress_outgoing_enabled = False + + # True if a message is fragmented and compression is ongoing. + self._compress_ongoing = False + + # Counters for statistics. + + # Total number of outgoing bytes supplied to this filter. + self._total_outgoing_payload_bytes = 0 + # Total number of bytes sent to the network after applying this filter. + self._total_filtered_outgoing_payload_bytes = 0 + + # Total number of bytes received from the network. + self._total_incoming_payload_bytes = 0 + # Total number of incoming bytes obtained after applying this filter. + self._total_filtered_incoming_payload_bytes = 0 + + def name(self): + return 'deflate' + + def get_extension_response(self): + # Any unknown parameter will be just ignored. + + s2c_max_window_bits = self._request.get_parameter_value( + self._S2C_MAX_WINDOW_BITS_PARAM) + if s2c_max_window_bits is not None: + try: + s2c_max_window_bits = int(s2c_max_window_bits) + except ValueError, e: + return None + if s2c_max_window_bits < 8 or s2c_max_window_bits > 15: + return None + + s2c_no_context_takeover = self._request.has_parameter( + self._S2C_NO_CONTEXT_TAKEOVER_PARAM) + if (s2c_no_context_takeover and + self._request.get_parameter_value( + self._S2C_NO_CONTEXT_TAKEOVER_PARAM) is not None): + return None + + self._deflater = util._RFC1979Deflater( + s2c_max_window_bits, s2c_no_context_takeover) + + self._inflater = util._RFC1979Inflater() + + self._compress_outgoing_enabled = True + + response = common.ExtensionParameter(self._request.name()) + + if s2c_max_window_bits is not None: + response.add_parameter( + self._S2C_MAX_WINDOW_BITS_PARAM, str(s2c_max_window_bits)) + + if s2c_no_context_takeover: + response.add_parameter( + self._S2C_NO_CONTEXT_TAKEOVER_PARAM, None) + + if self._c2s_max_window_bits is not None: + response.add_parameter( + self._C2S_MAX_WINDOW_BITS_PARAM, + str(self._c2s_max_window_bits)) + if self._c2s_no_context_takeover: + response.add_parameter( + self._C2S_NO_CONTEXT_TAKEOVER_PARAM, None) + + self._logger.debug( + 'Enable %s extension (' + 'request: s2c_max_window_bits=%s; s2c_no_context_takeover=%r, ' + 'response: c2s_max_window_bits=%s; c2s_no_context_takeover=%r)' % + (self._request.name(), + s2c_max_window_bits, + s2c_no_context_takeover, + self._c2s_max_window_bits, + self._c2s_no_context_takeover)) + + return response + + def setup_stream_options(self, stream_options): + class _OutgoingMessageFilter(object): + + def __init__(self, parent): + self._parent = parent + + def filter(self, message, end=True, binary=False): + return self._parent._process_outgoing_message( + message, end, binary) + + class _IncomingMessageFilter(object): + + def __init__(self, parent): + self._parent = parent + self._decompress_next_message = False + + def decompress_next_message(self): + self._decompress_next_message = True + + def filter(self, message): + message = self._parent._process_incoming_message( + message, self._decompress_next_message) + self._decompress_next_message = False + return message + + self._outgoing_message_filter = _OutgoingMessageFilter(self) + self._incoming_message_filter = _IncomingMessageFilter(self) + stream_options.outgoing_message_filters.append( + self._outgoing_message_filter) + stream_options.incoming_message_filters.append( + self._incoming_message_filter) + + class _OutgoingFrameFilter(object): + + def __init__(self, parent): + self._parent = parent + self._set_compression_bit = False + + def set_compression_bit(self): + self._set_compression_bit = True + + def filter(self, frame): + self._parent._process_outgoing_frame( + frame, self._set_compression_bit) + self._set_compression_bit = False + + class _IncomingFrameFilter(object): + + def __init__(self, parent): + self._parent = parent + + def filter(self, frame): + self._parent._process_incoming_frame(frame) + + self._outgoing_frame_filter = _OutgoingFrameFilter(self) + self._incoming_frame_filter = _IncomingFrameFilter(self) + stream_options.outgoing_frame_filters.append( + self._outgoing_frame_filter) + stream_options.incoming_frame_filters.append( + self._incoming_frame_filter) + + stream_options.encode_text_message_to_utf8 = False + + def set_c2s_max_window_bits(self, value): + self._c2s_max_window_bits = value + + def set_c2s_no_context_takeover(self, value): + self._c2s_no_context_takeover = value + + def set_bfinal(self, value): + self._bfinal = value + + def enable_outgoing_compression(self): + self._compress_outgoing_enabled = True + + def disable_outgoing_compression(self): + self._compress_outgoing_enabled = False + + def _process_incoming_message(self, message, decompress): + if not decompress: + return message + + received_payload_size = len(message) + self._total_incoming_payload_bytes += received_payload_size + + message = self._inflater.filter(message) + + filtered_payload_size = len(message) + self._total_filtered_incoming_payload_bytes += filtered_payload_size + + _log_decompression_ratio(self._logger, received_payload_size, + self._total_incoming_payload_bytes, + filtered_payload_size, + self._total_filtered_incoming_payload_bytes) + + return message + + def _process_outgoing_message(self, message, end, binary): + if not binary: + message = message.encode('utf-8') + + if not self._compress_outgoing_enabled: + return message + + original_payload_size = len(message) + self._total_outgoing_payload_bytes += original_payload_size + + message = self._deflater.filter( + message, flush=end, bfinal=self._bfinal) + + filtered_payload_size = len(message) + self._total_filtered_outgoing_payload_bytes += filtered_payload_size + + _log_compression_ratio(self._logger, original_payload_size, + self._total_outgoing_payload_bytes, + filtered_payload_size, + self._total_filtered_outgoing_payload_bytes) + + if not self._compress_ongoing: + self._outgoing_frame_filter.set_compression_bit() + self._compress_ongoing = not end + return message + + def _process_incoming_frame(self, frame): + if frame.rsv1 == 1 and not common.is_control_opcode(frame.opcode): + self._incoming_message_filter.decompress_next_message() + frame.rsv1 = 0 + + def _process_outgoing_frame(self, frame, compression_bit): + if (not compression_bit or + common.is_control_opcode(frame.opcode)): + return + + frame.rsv1 = 1 + + +class PerMessageCompressionExtensionProcessor( + CompressionExtensionProcessorBase): + """WebSocket Per-message compression extension processor. + + Specification: + http://tools.ietf.org/html/draft-ietf-hybi-permessage-compression + """ + + _DEFLATE_METHOD = 'deflate' + + def __init__(self, request): + CompressionExtensionProcessorBase.__init__(self, request) + + def name(self): + return common.PERMESSAGE_COMPRESSION_EXTENSION + + def _lookup_compression_processor(self, method_desc): + if method_desc.name() == self._DEFLATE_METHOD: + return DeflateMessageProcessor(method_desc) + return None + + +_available_processors[common.PERMESSAGE_COMPRESSION_EXTENSION] = ( + PerMessageCompressionExtensionProcessor) + + +# Adding vendor-prefixed permessage-compress extension. +# TODO(bashi): Remove this after WebKit stops using vendor prefix. +_available_processors[common.X_WEBKIT_PERMESSAGE_COMPRESSION_EXTENSION] = ( + PerMessageCompressionExtensionProcessor) + + +class MuxExtensionProcessor(ExtensionProcessorInterface): + """WebSocket multiplexing extension processor.""" + + _QUOTA_PARAM = 'quota' + + def __init__(self, request): + self._request = request + + def name(self): + return common.MUX_EXTENSION + + def get_extension_response(self, ws_request, + logical_channel_extensions): + # Mux extension cannot be used after extensions that depend on + # frame boundary, extension data field, or any reserved bits + # which are attributed to each frame. + for extension in logical_channel_extensions: + name = extension.name() + if (name == common.PERFRAME_COMPRESSION_EXTENSION or + name == common.DEFLATE_FRAME_EXTENSION or + name == common.X_WEBKIT_DEFLATE_FRAME_EXTENSION): + return None + + quota = self._request.get_parameter_value(self._QUOTA_PARAM) + if quota is None: + ws_request.mux_quota = 0 + else: + try: + quota = int(quota) + except ValueError, e: + return None + if quota < 0 or quota >= 2 ** 32: + return None + ws_request.mux_quota = quota + + ws_request.mux = True + ws_request.mux_extensions = logical_channel_extensions + return common.ExtensionParameter(common.MUX_EXTENSION) + + def setup_stream_options(self, stream_options): + pass + + +_available_processors[common.MUX_EXTENSION] = MuxExtensionProcessor + + +def get_extension_processor(extension_request): + global _available_processors + processor_class = _available_processors.get(extension_request.name()) + if processor_class is None: + return None + return processor_class(extension_request) + + +# vi:sts=4 sw=4 et diff --git a/pyload/lib/mod_pywebsocket/handshake/__init__.py b/pyload/lib/mod_pywebsocket/handshake/__init__.py new file mode 100644 index 000000000..194f6b395 --- /dev/null +++ b/pyload/lib/mod_pywebsocket/handshake/__init__.py @@ -0,0 +1,110 @@ +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""WebSocket opening handshake processor. This class try to apply available +opening handshake processors for each protocol version until a connection is +successfully established. +""" + + +import logging + +from mod_pywebsocket import common +from mod_pywebsocket.handshake import hybi00 +from mod_pywebsocket.handshake import hybi +# Export AbortedByUserException, HandshakeException, and VersionException +# symbol from this module. +from mod_pywebsocket.handshake._base import AbortedByUserException +from mod_pywebsocket.handshake._base import HandshakeException +from mod_pywebsocket.handshake._base import VersionException + + +_LOGGER = logging.getLogger(__name__) + + +def do_handshake(request, dispatcher, allowDraft75=False, strict=False): + """Performs WebSocket handshake. + + Args: + request: mod_python request. + dispatcher: Dispatcher (dispatch.Dispatcher). + allowDraft75: obsolete argument. ignored. + strict: obsolete argument. ignored. + + Handshaker will add attributes such as ws_resource in performing + handshake. + """ + + _LOGGER.debug('Client\'s opening handshake resource: %r', request.uri) + # To print mimetools.Message as escaped one-line string, we converts + # headers_in to dict object. Without conversion, if we use %r, it just + # prints the type and address, and if we use %s, it prints the original + # header string as multiple lines. + # + # Both mimetools.Message and MpTable_Type of mod_python can be + # converted to dict. + # + # mimetools.Message.__str__ returns the original header string. + # dict(mimetools.Message object) returns the map from header names to + # header values. While MpTable_Type doesn't have such __str__ but just + # __repr__ which formats itself as well as dictionary object. + _LOGGER.debug( + 'Client\'s opening handshake headers: %r', dict(request.headers_in)) + + handshakers = [] + handshakers.append( + ('RFC 6455', hybi.Handshaker(request, dispatcher))) + handshakers.append( + ('HyBi 00', hybi00.Handshaker(request, dispatcher))) + + for name, handshaker in handshakers: + _LOGGER.debug('Trying protocol version %s', name) + try: + handshaker.do_handshake() + _LOGGER.info('Established (%s protocol)', name) + return + except HandshakeException, e: + _LOGGER.debug( + 'Failed to complete opening handshake as %s protocol: %r', + name, e) + if e.status: + raise e + except AbortedByUserException, e: + raise + except VersionException, e: + raise + + # TODO(toyoshim): Add a test to cover the case all handshakers fail. + raise HandshakeException( + 'Failed to complete opening handshake for all available protocols', + status=common.HTTP_STATUS_BAD_REQUEST) + + +# vi:sts=4 sw=4 et diff --git a/pyload/lib/mod_pywebsocket/handshake/_base.py b/pyload/lib/mod_pywebsocket/handshake/_base.py new file mode 100644 index 000000000..e5c94ca90 --- /dev/null +++ b/pyload/lib/mod_pywebsocket/handshake/_base.py @@ -0,0 +1,226 @@ +# Copyright 2012, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Common functions and exceptions used by WebSocket opening handshake +processors. +""" + + +from mod_pywebsocket import common +from mod_pywebsocket import http_header_util + + +class AbortedByUserException(Exception): + """Exception for aborting a connection intentionally. + + If this exception is raised in do_extra_handshake handler, the connection + will be abandoned. No other WebSocket or HTTP(S) handler will be invoked. + + If this exception is raised in transfer_data_handler, the connection will + be closed without closing handshake. No other WebSocket or HTTP(S) handler + will be invoked. + """ + + pass + + +class HandshakeException(Exception): + """This exception will be raised when an error occurred while processing + WebSocket initial handshake. + """ + + def __init__(self, name, status=None): + super(HandshakeException, self).__init__(name) + self.status = status + + +class VersionException(Exception): + """This exception will be raised when a version of client request does not + match with version the server supports. + """ + + def __init__(self, name, supported_versions=''): + """Construct an instance. + + Args: + supported_version: a str object to show supported hybi versions. + (e.g. '8, 13') + """ + super(VersionException, self).__init__(name) + self.supported_versions = supported_versions + + +def get_default_port(is_secure): + if is_secure: + return common.DEFAULT_WEB_SOCKET_SECURE_PORT + else: + return common.DEFAULT_WEB_SOCKET_PORT + + +def validate_subprotocol(subprotocol, hixie): + """Validate a value in the Sec-WebSocket-Protocol field. + + See + - RFC 6455: Section 4.1., 4.2.2., and 4.3. + - HyBi 00: Section 4.1. Opening handshake + + Args: + hixie: if True, checks if characters in subprotocol are in range + between U+0020 and U+007E. It's required by HyBi 00 but not by + RFC 6455. + """ + + if not subprotocol: + raise HandshakeException('Invalid subprotocol name: empty') + if hixie: + # Parameter should be in the range U+0020 to U+007E. + for c in subprotocol: + if not 0x20 <= ord(c) <= 0x7e: + raise HandshakeException( + 'Illegal character in subprotocol name: %r' % c) + else: + # Parameter should be encoded HTTP token. + state = http_header_util.ParsingState(subprotocol) + token = http_header_util.consume_token(state) + rest = http_header_util.peek(state) + # If |rest| is not None, |subprotocol| is not one token or invalid. If + # |rest| is None, |token| must not be None because |subprotocol| is + # concatenation of |token| and |rest| and is not None. + if rest is not None: + raise HandshakeException('Invalid non-token string in subprotocol ' + 'name: %r' % rest) + + +def parse_host_header(request): + fields = request.headers_in['Host'].split(':', 1) + if len(fields) == 1: + return fields[0], get_default_port(request.is_https()) + try: + return fields[0], int(fields[1]) + except ValueError, e: + raise HandshakeException('Invalid port number format: %r' % e) + + +def format_header(name, value): + return '%s: %s\r\n' % (name, value) + + +def build_location(request): + """Build WebSocket location for request.""" + location_parts = [] + if request.is_https(): + location_parts.append(common.WEB_SOCKET_SECURE_SCHEME) + else: + location_parts.append(common.WEB_SOCKET_SCHEME) + location_parts.append('://') + host, port = parse_host_header(request) + connection_port = request.connection.local_addr[1] + if port != connection_port: + raise HandshakeException('Header/connection port mismatch: %d/%d' % + (port, connection_port)) + location_parts.append(host) + if (port != get_default_port(request.is_https())): + location_parts.append(':') + location_parts.append(str(port)) + location_parts.append(request.uri) + return ''.join(location_parts) + + +def get_mandatory_header(request, key): + value = request.headers_in.get(key) + if value is None: + raise HandshakeException('Header %s is not defined' % key) + return value + + +def validate_mandatory_header(request, key, expected_value, fail_status=None): + value = get_mandatory_header(request, key) + + if value.lower() != expected_value.lower(): + raise HandshakeException( + 'Expected %r for header %s but found %r (case-insensitive)' % + (expected_value, key, value), status=fail_status) + + +def check_request_line(request): + # 5.1 1. The three character UTF-8 string "GET". + # 5.1 2. A UTF-8-encoded U+0020 SPACE character (0x20 byte). + if request.method != 'GET': + raise HandshakeException('Method is not GET: %r' % request.method) + + if request.protocol != 'HTTP/1.1': + raise HandshakeException('Version is not HTTP/1.1: %r' % + request.protocol) + + +def check_header_lines(request, mandatory_headers): + check_request_line(request) + + # The expected field names, and the meaning of their corresponding + # values, are as follows. + # |Upgrade| and |Connection| + for key, expected_value in mandatory_headers: + validate_mandatory_header(request, key, expected_value) + + +def parse_token_list(data): + """Parses a header value which follows 1#token and returns parsed elements + as a list of strings. + + Leading LWSes must be trimmed. + """ + + state = http_header_util.ParsingState(data) + + token_list = [] + + while True: + token = http_header_util.consume_token(state) + if token is not None: + token_list.append(token) + + http_header_util.consume_lwses(state) + + if http_header_util.peek(state) is None: + break + + if not http_header_util.consume_string(state, ','): + raise HandshakeException( + 'Expected a comma but found %r' % http_header_util.peek(state)) + + http_header_util.consume_lwses(state) + + if len(token_list) == 0: + raise HandshakeException('No valid token found') + + return token_list + + +# vi:sts=4 sw=4 et diff --git a/pyload/lib/mod_pywebsocket/handshake/hybi.py b/pyload/lib/mod_pywebsocket/handshake/hybi.py new file mode 100644 index 000000000..fc0e2a096 --- /dev/null +++ b/pyload/lib/mod_pywebsocket/handshake/hybi.py @@ -0,0 +1,404 @@ +# Copyright 2012, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""This file provides the opening handshake processor for the WebSocket +protocol (RFC 6455). + +Specification: +http://tools.ietf.org/html/rfc6455 +""" + + +# Note: request.connection.write is used in this module, even though mod_python +# document says that it should be used only in connection handlers. +# Unfortunately, we have no other options. For example, request.write is not +# suitable because it doesn't allow direct raw bytes writing. + + +import base64 +import logging +import os +import re + +from mod_pywebsocket import common +from mod_pywebsocket.extensions import get_extension_processor +from mod_pywebsocket.handshake._base import check_request_line +from mod_pywebsocket.handshake._base import format_header +from mod_pywebsocket.handshake._base import get_mandatory_header +from mod_pywebsocket.handshake._base import HandshakeException +from mod_pywebsocket.handshake._base import parse_token_list +from mod_pywebsocket.handshake._base import validate_mandatory_header +from mod_pywebsocket.handshake._base import validate_subprotocol +from mod_pywebsocket.handshake._base import VersionException +from mod_pywebsocket.stream import Stream +from mod_pywebsocket.stream import StreamOptions +from mod_pywebsocket import util + + +# Used to validate the value in the Sec-WebSocket-Key header strictly. RFC 4648 +# disallows non-zero padding, so the character right before == must be any of +# A, Q, g and w. +_SEC_WEBSOCKET_KEY_REGEX = re.compile('^[+/0-9A-Za-z]{21}[AQgw]==$') + +# Defining aliases for values used frequently. +_VERSION_HYBI08 = common.VERSION_HYBI08 +_VERSION_HYBI08_STRING = str(_VERSION_HYBI08) +_VERSION_LATEST = common.VERSION_HYBI_LATEST +_VERSION_LATEST_STRING = str(_VERSION_LATEST) +_SUPPORTED_VERSIONS = [ + _VERSION_LATEST, + _VERSION_HYBI08, +] + + +def compute_accept(key): + """Computes value for the Sec-WebSocket-Accept header from value of the + Sec-WebSocket-Key header. + """ + + accept_binary = util.sha1_hash( + key + common.WEBSOCKET_ACCEPT_UUID).digest() + accept = base64.b64encode(accept_binary) + + return (accept, accept_binary) + + +class Handshaker(object): + """Opening handshake processor for the WebSocket protocol (RFC 6455).""" + + def __init__(self, request, dispatcher): + """Construct an instance. + + Args: + request: mod_python request. + dispatcher: Dispatcher (dispatch.Dispatcher). + + Handshaker will add attributes such as ws_resource during handshake. + """ + + self._logger = util.get_class_logger(self) + + self._request = request + self._dispatcher = dispatcher + + def _validate_connection_header(self): + connection = get_mandatory_header( + self._request, common.CONNECTION_HEADER) + + try: + connection_tokens = parse_token_list(connection) + except HandshakeException, e: + raise HandshakeException( + 'Failed to parse %s: %s' % (common.CONNECTION_HEADER, e)) + + connection_is_valid = False + for token in connection_tokens: + if token.lower() == common.UPGRADE_CONNECTION_TYPE.lower(): + connection_is_valid = True + break + if not connection_is_valid: + raise HandshakeException( + '%s header doesn\'t contain "%s"' % + (common.CONNECTION_HEADER, common.UPGRADE_CONNECTION_TYPE)) + + def do_handshake(self): + self._request.ws_close_code = None + self._request.ws_close_reason = None + + # Parsing. + + check_request_line(self._request) + + validate_mandatory_header( + self._request, + common.UPGRADE_HEADER, + common.WEBSOCKET_UPGRADE_TYPE) + + self._validate_connection_header() + + self._request.ws_resource = self._request.uri + + unused_host = get_mandatory_header(self._request, common.HOST_HEADER) + + self._request.ws_version = self._check_version() + + # This handshake must be based on latest hybi. We are responsible to + # fallback to HTTP on handshake failure as latest hybi handshake + # specifies. + try: + self._get_origin() + self._set_protocol() + self._parse_extensions() + + # Key validation, response generation. + + key = self._get_key() + (accept, accept_binary) = compute_accept(key) + self._logger.debug( + '%s: %r (%s)', + common.SEC_WEBSOCKET_ACCEPT_HEADER, + accept, + util.hexify(accept_binary)) + + self._logger.debug('Protocol version is RFC 6455') + + # Setup extension processors. + + processors = [] + if self._request.ws_requested_extensions is not None: + for extension_request in self._request.ws_requested_extensions: + processor = get_extension_processor(extension_request) + # Unknown extension requests are just ignored. + if processor is not None: + processors.append(processor) + self._request.ws_extension_processors = processors + + # Extra handshake handler may modify/remove processors. + self._dispatcher.do_extra_handshake(self._request) + processors = filter(lambda processor: processor is not None, + self._request.ws_extension_processors) + + accepted_extensions = [] + + # We need to take care of mux extension here. Extensions that + # are placed before mux should be applied to logical channels. + mux_index = -1 + for i, processor in enumerate(processors): + if processor.name() == common.MUX_EXTENSION: + mux_index = i + break + if mux_index >= 0: + mux_processor = processors[mux_index] + logical_channel_processors = processors[:mux_index] + processors = processors[mux_index+1:] + + for processor in logical_channel_processors: + extension_response = processor.get_extension_response() + if extension_response is None: + # Rejected. + continue + accepted_extensions.append(extension_response) + # Pass a shallow copy of accepted_extensions as extensions for + # logical channels. + mux_response = mux_processor.get_extension_response( + self._request, accepted_extensions[:]) + if mux_response is not None: + accepted_extensions.append(mux_response) + + stream_options = StreamOptions() + + # When there is mux extension, here, |processors| contain only + # prosessors for extensions placed after mux. + for processor in processors: + + extension_response = processor.get_extension_response() + if extension_response is None: + # Rejected. + continue + + accepted_extensions.append(extension_response) + + processor.setup_stream_options(stream_options) + + if len(accepted_extensions) > 0: + self._request.ws_extensions = accepted_extensions + self._logger.debug( + 'Extensions accepted: %r', + map(common.ExtensionParameter.name, accepted_extensions)) + else: + self._request.ws_extensions = None + + self._request.ws_stream = self._create_stream(stream_options) + + if self._request.ws_requested_protocols is not None: + if self._request.ws_protocol is None: + raise HandshakeException( + 'do_extra_handshake must choose one subprotocol from ' + 'ws_requested_protocols and set it to ws_protocol') + validate_subprotocol(self._request.ws_protocol, hixie=False) + + self._logger.debug( + 'Subprotocol accepted: %r', + self._request.ws_protocol) + else: + if self._request.ws_protocol is not None: + raise HandshakeException( + 'ws_protocol must be None when the client didn\'t ' + 'request any subprotocol') + + self._send_handshake(accept) + except HandshakeException, e: + if not e.status: + # Fallback to 400 bad request by default. + e.status = common.HTTP_STATUS_BAD_REQUEST + raise e + + def _get_origin(self): + if self._request.ws_version is _VERSION_HYBI08: + origin_header = common.SEC_WEBSOCKET_ORIGIN_HEADER + else: + origin_header = common.ORIGIN_HEADER + origin = self._request.headers_in.get(origin_header) + if origin is None: + self._logger.debug('Client request does not have origin header') + self._request.ws_origin = origin + + def _check_version(self): + version = get_mandatory_header(self._request, + common.SEC_WEBSOCKET_VERSION_HEADER) + if version == _VERSION_HYBI08_STRING: + return _VERSION_HYBI08 + if version == _VERSION_LATEST_STRING: + return _VERSION_LATEST + + if version.find(',') >= 0: + raise HandshakeException( + 'Multiple versions (%r) are not allowed for header %s' % + (version, common.SEC_WEBSOCKET_VERSION_HEADER), + status=common.HTTP_STATUS_BAD_REQUEST) + raise VersionException( + 'Unsupported version %r for header %s' % + (version, common.SEC_WEBSOCKET_VERSION_HEADER), + supported_versions=', '.join(map(str, _SUPPORTED_VERSIONS))) + + def _set_protocol(self): + self._request.ws_protocol = None + + protocol_header = self._request.headers_in.get( + common.SEC_WEBSOCKET_PROTOCOL_HEADER) + + if protocol_header is None: + self._request.ws_requested_protocols = None + return + + self._request.ws_requested_protocols = parse_token_list( + protocol_header) + self._logger.debug('Subprotocols requested: %r', + self._request.ws_requested_protocols) + + def _parse_extensions(self): + extensions_header = self._request.headers_in.get( + common.SEC_WEBSOCKET_EXTENSIONS_HEADER) + if not extensions_header: + self._request.ws_requested_extensions = None + return + + if self._request.ws_version is common.VERSION_HYBI08: + allow_quoted_string=False + else: + allow_quoted_string=True + try: + self._request.ws_requested_extensions = common.parse_extensions( + extensions_header, allow_quoted_string=allow_quoted_string) + except common.ExtensionParsingException, e: + raise HandshakeException( + 'Failed to parse Sec-WebSocket-Extensions header: %r' % e) + + self._logger.debug( + 'Extensions requested: %r', + map(common.ExtensionParameter.name, + self._request.ws_requested_extensions)) + + def _validate_key(self, key): + if key.find(',') >= 0: + raise HandshakeException('Request has multiple %s header lines or ' + 'contains illegal character \',\': %r' % + (common.SEC_WEBSOCKET_KEY_HEADER, key)) + + # Validate + key_is_valid = False + try: + # Validate key by quick regex match before parsing by base64 + # module. Because base64 module skips invalid characters, we have + # to do this in advance to make this server strictly reject illegal + # keys. + if _SEC_WEBSOCKET_KEY_REGEX.match(key): + decoded_key = base64.b64decode(key) + if len(decoded_key) == 16: + key_is_valid = True + except TypeError, e: + pass + + if not key_is_valid: + raise HandshakeException( + 'Illegal value for header %s: %r' % + (common.SEC_WEBSOCKET_KEY_HEADER, key)) + + return decoded_key + + def _get_key(self): + key = get_mandatory_header( + self._request, common.SEC_WEBSOCKET_KEY_HEADER) + + decoded_key = self._validate_key(key) + + self._logger.debug( + '%s: %r (%s)', + common.SEC_WEBSOCKET_KEY_HEADER, + key, + util.hexify(decoded_key)) + + return key + + def _create_stream(self, stream_options): + return Stream(self._request, stream_options) + + def _create_handshake_response(self, accept): + response = [] + + response.append('HTTP/1.1 101 Switching Protocols\r\n') + + response.append(format_header( + common.UPGRADE_HEADER, common.WEBSOCKET_UPGRADE_TYPE)) + response.append(format_header( + common.CONNECTION_HEADER, common.UPGRADE_CONNECTION_TYPE)) + response.append(format_header( + common.SEC_WEBSOCKET_ACCEPT_HEADER, accept)) + if self._request.ws_protocol is not None: + response.append(format_header( + common.SEC_WEBSOCKET_PROTOCOL_HEADER, + self._request.ws_protocol)) + if (self._request.ws_extensions is not None and + len(self._request.ws_extensions) != 0): + response.append(format_header( + common.SEC_WEBSOCKET_EXTENSIONS_HEADER, + common.format_extensions(self._request.ws_extensions))) + response.append('\r\n') + + return ''.join(response) + + def _send_handshake(self, accept): + raw_response = self._create_handshake_response(accept) + self._request.connection.write(raw_response) + self._logger.debug('Sent server\'s opening handshake: %r', + raw_response) + + +# vi:sts=4 sw=4 et diff --git a/pyload/lib/mod_pywebsocket/handshake/hybi00.py b/pyload/lib/mod_pywebsocket/handshake/hybi00.py new file mode 100644 index 000000000..cc6f8dc43 --- /dev/null +++ b/pyload/lib/mod_pywebsocket/handshake/hybi00.py @@ -0,0 +1,242 @@ +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""This file provides the opening handshake processor for the WebSocket +protocol version HyBi 00. + +Specification: +http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-00 +""" + + +# Note: request.connection.write/read are used in this module, even though +# mod_python document says that they should be used only in connection +# handlers. Unfortunately, we have no other options. For example, +# request.write/read are not suitable because they don't allow direct raw bytes +# writing/reading. + + +import logging +import re +import struct + +from mod_pywebsocket import common +from mod_pywebsocket.stream import StreamHixie75 +from mod_pywebsocket import util +from mod_pywebsocket.handshake._base import HandshakeException +from mod_pywebsocket.handshake._base import build_location +from mod_pywebsocket.handshake._base import check_header_lines +from mod_pywebsocket.handshake._base import format_header +from mod_pywebsocket.handshake._base import get_mandatory_header +from mod_pywebsocket.handshake._base import validate_subprotocol + + +_MANDATORY_HEADERS = [ + # key, expected value or None + [common.UPGRADE_HEADER, common.WEBSOCKET_UPGRADE_TYPE_HIXIE75], + [common.CONNECTION_HEADER, common.UPGRADE_CONNECTION_TYPE], +] + + +class Handshaker(object): + """Opening handshake processor for the WebSocket protocol version HyBi 00. + """ + + def __init__(self, request, dispatcher): + """Construct an instance. + + Args: + request: mod_python request. + dispatcher: Dispatcher (dispatch.Dispatcher). + + Handshaker will add attributes such as ws_resource in performing + handshake. + """ + + self._logger = util.get_class_logger(self) + + self._request = request + self._dispatcher = dispatcher + + def do_handshake(self): + """Perform WebSocket Handshake. + + On _request, we set + ws_resource, ws_protocol, ws_location, ws_origin, ws_challenge, + ws_challenge_md5: WebSocket handshake information. + ws_stream: Frame generation/parsing class. + ws_version: Protocol version. + + Raises: + HandshakeException: when any error happened in parsing the opening + handshake request. + """ + + # 5.1 Reading the client's opening handshake. + # dispatcher sets it in self._request. + check_header_lines(self._request, _MANDATORY_HEADERS) + self._set_resource() + self._set_subprotocol() + self._set_location() + self._set_origin() + self._set_challenge_response() + self._set_protocol_version() + + self._dispatcher.do_extra_handshake(self._request) + + self._send_handshake() + + def _set_resource(self): + self._request.ws_resource = self._request.uri + + def _set_subprotocol(self): + # |Sec-WebSocket-Protocol| + subprotocol = self._request.headers_in.get( + common.SEC_WEBSOCKET_PROTOCOL_HEADER) + if subprotocol is not None: + validate_subprotocol(subprotocol, hixie=True) + self._request.ws_protocol = subprotocol + + def _set_location(self): + # |Host| + host = self._request.headers_in.get(common.HOST_HEADER) + if host is not None: + self._request.ws_location = build_location(self._request) + # TODO(ukai): check host is this host. + + def _set_origin(self): + # |Origin| + origin = self._request.headers_in.get(common.ORIGIN_HEADER) + if origin is not None: + self._request.ws_origin = origin + + def _set_protocol_version(self): + # |Sec-WebSocket-Draft| + draft = self._request.headers_in.get(common.SEC_WEBSOCKET_DRAFT_HEADER) + if draft is not None and draft != '0': + raise HandshakeException('Illegal value for %s: %s' % + (common.SEC_WEBSOCKET_DRAFT_HEADER, + draft)) + + self._logger.debug('Protocol version is HyBi 00') + self._request.ws_version = common.VERSION_HYBI00 + self._request.ws_stream = StreamHixie75(self._request, True) + + def _set_challenge_response(self): + # 5.2 4-8. + self._request.ws_challenge = self._get_challenge() + # 5.2 9. let /response/ be the MD5 finterprint of /challenge/ + self._request.ws_challenge_md5 = util.md5_hash( + self._request.ws_challenge).digest() + self._logger.debug( + 'Challenge: %r (%s)', + self._request.ws_challenge, + util.hexify(self._request.ws_challenge)) + self._logger.debug( + 'Challenge response: %r (%s)', + self._request.ws_challenge_md5, + util.hexify(self._request.ws_challenge_md5)) + + def _get_key_value(self, key_field): + key_value = get_mandatory_header(self._request, key_field) + + self._logger.debug('%s: %r', key_field, key_value) + + # 5.2 4. let /key-number_n/ be the digits (characters in the range + # U+0030 DIGIT ZERO (0) to U+0039 DIGIT NINE (9)) in /key_n/, + # interpreted as a base ten integer, ignoring all other characters + # in /key_n/. + try: + key_number = int(re.sub("\\D", "", key_value)) + except: + raise HandshakeException('%s field contains no digit' % key_field) + # 5.2 5. let /spaces_n/ be the number of U+0020 SPACE characters + # in /key_n/. + spaces = re.subn(" ", "", key_value)[1] + if spaces == 0: + raise HandshakeException('%s field contains no space' % key_field) + + self._logger.debug( + '%s: Key-number is %d and number of spaces is %d', + key_field, key_number, spaces) + + # 5.2 6. if /key-number_n/ is not an integral multiple of /spaces_n/ + # then abort the WebSocket connection. + if key_number % spaces != 0: + raise HandshakeException( + '%s: Key-number (%d) is not an integral multiple of spaces ' + '(%d)' % (key_field, key_number, spaces)) + # 5.2 7. let /part_n/ be /key-number_n/ divided by /spaces_n/. + part = key_number / spaces + self._logger.debug('%s: Part is %d', key_field, part) + return part + + def _get_challenge(self): + # 5.2 4-7. + key1 = self._get_key_value(common.SEC_WEBSOCKET_KEY1_HEADER) + key2 = self._get_key_value(common.SEC_WEBSOCKET_KEY2_HEADER) + # 5.2 8. let /challenge/ be the concatenation of /part_1/, + challenge = '' + challenge += struct.pack('!I', key1) # network byteorder int + challenge += struct.pack('!I', key2) # network byteorder int + challenge += self._request.connection.read(8) + return challenge + + def _send_handshake(self): + response = [] + + # 5.2 10. send the following line. + response.append('HTTP/1.1 101 WebSocket Protocol Handshake\r\n') + + # 5.2 11. send the following fields to the client. + response.append(format_header( + common.UPGRADE_HEADER, common.WEBSOCKET_UPGRADE_TYPE_HIXIE75)) + response.append(format_header( + common.CONNECTION_HEADER, common.UPGRADE_CONNECTION_TYPE)) + response.append(format_header( + common.SEC_WEBSOCKET_LOCATION_HEADER, self._request.ws_location)) + response.append(format_header( + common.SEC_WEBSOCKET_ORIGIN_HEADER, self._request.ws_origin)) + if self._request.ws_protocol: + response.append(format_header( + common.SEC_WEBSOCKET_PROTOCOL_HEADER, + self._request.ws_protocol)) + # 5.2 12. send two bytes 0x0D 0x0A. + response.append('\r\n') + # 5.2 13. send /response/ + response.append(self._request.ws_challenge_md5) + + raw_response = ''.join(response) + self._request.connection.write(raw_response) + self._logger.debug('Sent server\'s opening handshake: %r', + raw_response) + + +# vi:sts=4 sw=4 et diff --git a/pyload/lib/mod_pywebsocket/headerparserhandler.py b/pyload/lib/mod_pywebsocket/headerparserhandler.py new file mode 100644 index 000000000..2cc62de04 --- /dev/null +++ b/pyload/lib/mod_pywebsocket/headerparserhandler.py @@ -0,0 +1,244 @@ +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""PythonHeaderParserHandler for mod_pywebsocket. + +Apache HTTP Server and mod_python must be configured such that this +function is called to handle WebSocket request. +""" + + +import logging + +from mod_python import apache + +from mod_pywebsocket import common +from mod_pywebsocket import dispatch +from mod_pywebsocket import handshake +from mod_pywebsocket import util + + +# PythonOption to specify the handler root directory. +_PYOPT_HANDLER_ROOT = 'mod_pywebsocket.handler_root' + +# PythonOption to specify the handler scan directory. +# This must be a directory under the root directory. +# The default is the root directory. +_PYOPT_HANDLER_SCAN = 'mod_pywebsocket.handler_scan' + +# PythonOption to allow handlers whose canonical path is +# not under the root directory. It's disallowed by default. +# Set this option with value of 'yes' to allow. +_PYOPT_ALLOW_HANDLERS_OUTSIDE_ROOT = ( + 'mod_pywebsocket.allow_handlers_outside_root_dir') +# Map from values to their meanings. 'Yes' and 'No' are allowed just for +# compatibility. +_PYOPT_ALLOW_HANDLERS_OUTSIDE_ROOT_DEFINITION = { + 'off': False, 'no': False, 'on': True, 'yes': True} + +# (Obsolete option. Ignored.) +# PythonOption to specify to allow handshake defined in Hixie 75 version +# protocol. The default is None (Off) +_PYOPT_ALLOW_DRAFT75 = 'mod_pywebsocket.allow_draft75' +# Map from values to their meanings. +_PYOPT_ALLOW_DRAFT75_DEFINITION = {'off': False, 'on': True} + + +class ApacheLogHandler(logging.Handler): + """Wrapper logging.Handler to emit log message to apache's error.log.""" + + _LEVELS = { + logging.DEBUG: apache.APLOG_DEBUG, + logging.INFO: apache.APLOG_INFO, + logging.WARNING: apache.APLOG_WARNING, + logging.ERROR: apache.APLOG_ERR, + logging.CRITICAL: apache.APLOG_CRIT, + } + + def __init__(self, request=None): + logging.Handler.__init__(self) + self._log_error = apache.log_error + if request is not None: + self._log_error = request.log_error + + # Time and level will be printed by Apache. + self._formatter = logging.Formatter('%(name)s: %(message)s') + + def emit(self, record): + apache_level = apache.APLOG_DEBUG + if record.levelno in ApacheLogHandler._LEVELS: + apache_level = ApacheLogHandler._LEVELS[record.levelno] + + msg = self._formatter.format(record) + + # "server" parameter must be passed to have "level" parameter work. + # If only "level" parameter is passed, nothing shows up on Apache's + # log. However, at this point, we cannot get the server object of the + # virtual host which will process WebSocket requests. The only server + # object we can get here is apache.main_server. But Wherever (server + # configuration context or virtual host context) we put + # PythonHeaderParserHandler directive, apache.main_server just points + # the main server instance (not any of virtual server instance). Then, + # Apache follows LogLevel directive in the server configuration context + # to filter logs. So, we need to specify LogLevel in the server + # configuration context. Even if we specify "LogLevel debug" in the + # virtual host context which actually handles WebSocket connections, + # DEBUG level logs never show up unless "LogLevel debug" is specified + # in the server configuration context. + # + # TODO(tyoshino): Provide logging methods on request object. When + # request is mp_request object (when used together with Apache), the + # methods call request.log_error indirectly. When request is + # _StandaloneRequest, the methods call Python's logging facility which + # we create in standalone.py. + self._log_error(msg, apache_level, apache.main_server) + + +def _configure_logging(): + logger = logging.getLogger() + # Logs are filtered by Apache based on LogLevel directive in Apache + # configuration file. We must just pass logs for all levels to + # ApacheLogHandler. + logger.setLevel(logging.DEBUG) + logger.addHandler(ApacheLogHandler()) + + +_configure_logging() + +_LOGGER = logging.getLogger(__name__) + + +def _parse_option(name, value, definition): + if value is None: + return False + + meaning = definition.get(value.lower()) + if meaning is None: + raise Exception('Invalid value for PythonOption %s: %r' % + (name, value)) + return meaning + + +def _create_dispatcher(): + _LOGGER.info('Initializing Dispatcher') + + options = apache.main_server.get_options() + + handler_root = options.get(_PYOPT_HANDLER_ROOT, None) + if not handler_root: + raise Exception('PythonOption %s is not defined' % _PYOPT_HANDLER_ROOT, + apache.APLOG_ERR) + + handler_scan = options.get(_PYOPT_HANDLER_SCAN, handler_root) + + allow_handlers_outside_root = _parse_option( + _PYOPT_ALLOW_HANDLERS_OUTSIDE_ROOT, + options.get(_PYOPT_ALLOW_HANDLERS_OUTSIDE_ROOT), + _PYOPT_ALLOW_HANDLERS_OUTSIDE_ROOT_DEFINITION) + + dispatcher = dispatch.Dispatcher( + handler_root, handler_scan, allow_handlers_outside_root) + + for warning in dispatcher.source_warnings(): + apache.log_error('mod_pywebsocket: %s' % warning, apache.APLOG_WARNING) + + return dispatcher + + +# Initialize +_dispatcher = _create_dispatcher() + + +def headerparserhandler(request): + """Handle request. + + Args: + request: mod_python request. + + This function is named headerparserhandler because it is the default + name for a PythonHeaderParserHandler. + """ + + handshake_is_done = False + try: + # Fallback to default http handler for request paths for which + # we don't have request handlers. + if not _dispatcher.get_handler_suite(request.uri): + request.log_error('No handler for resource: %r' % request.uri, + apache.APLOG_INFO) + request.log_error('Fallback to Apache', apache.APLOG_INFO) + return apache.DECLINED + except dispatch.DispatchException, e: + request.log_error('mod_pywebsocket: %s' % e, apache.APLOG_INFO) + if not handshake_is_done: + return e.status + + try: + allow_draft75 = _parse_option( + _PYOPT_ALLOW_DRAFT75, + apache.main_server.get_options().get(_PYOPT_ALLOW_DRAFT75), + _PYOPT_ALLOW_DRAFT75_DEFINITION) + + try: + handshake.do_handshake( + request, _dispatcher, allowDraft75=allow_draft75) + except handshake.VersionException, e: + request.log_error('mod_pywebsocket: %s' % e, apache.APLOG_INFO) + request.err_headers_out.add(common.SEC_WEBSOCKET_VERSION_HEADER, + e.supported_versions) + return apache.HTTP_BAD_REQUEST + except handshake.HandshakeException, e: + # Handshake for ws/wss failed. + # Send http response with error status. + request.log_error('mod_pywebsocket: %s' % e, apache.APLOG_INFO) + return e.status + + handshake_is_done = True + request._dispatcher = _dispatcher + _dispatcher.transfer_data(request) + except handshake.AbortedByUserException, e: + request.log_error('mod_pywebsocket: %s' % e, apache.APLOG_INFO) + except Exception, e: + # DispatchException can also be thrown if something is wrong in + # pywebsocket code. It's caught here, then. + + request.log_error('mod_pywebsocket: %s\n%s' % + (e, util.get_stack_trace()), + apache.APLOG_ERR) + # Unknown exceptions before handshake mean Apache must handle its + # request with another handler. + if not handshake_is_done: + return apache.DECLINED + # Set assbackwards to suppress response header generation by Apache. + request.assbackwards = 1 + return apache.DONE # Return DONE such that no other handlers are invoked. + + +# vi:sts=4 sw=4 et diff --git a/pyload/lib/mod_pywebsocket/http_header_util.py b/pyload/lib/mod_pywebsocket/http_header_util.py new file mode 100644 index 000000000..b77465393 --- /dev/null +++ b/pyload/lib/mod_pywebsocket/http_header_util.py @@ -0,0 +1,263 @@ +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Utilities for parsing and formatting headers that follow the grammar defined +in HTTP RFC http://www.ietf.org/rfc/rfc2616.txt. +""" + + +import urlparse + + +_SEPARATORS = '()<>@,;:\\"/[]?={} \t' + + +def _is_char(c): + """Returns true iff c is in CHAR as specified in HTTP RFC.""" + + return ord(c) <= 127 + + +def _is_ctl(c): + """Returns true iff c is in CTL as specified in HTTP RFC.""" + + return ord(c) <= 31 or ord(c) == 127 + + +class ParsingState(object): + + def __init__(self, data): + self.data = data + self.head = 0 + + +def peek(state, pos=0): + """Peeks the character at pos from the head of data.""" + + if state.head + pos >= len(state.data): + return None + + return state.data[state.head + pos] + + +def consume(state, amount=1): + """Consumes specified amount of bytes from the head and returns the + consumed bytes. If there's not enough bytes to consume, returns None. + """ + + if state.head + amount > len(state.data): + return None + + result = state.data[state.head:state.head + amount] + state.head = state.head + amount + return result + + +def consume_string(state, expected): + """Given a parsing state and a expected string, consumes the string from + the head. Returns True if consumed successfully. Otherwise, returns + False. + """ + + pos = 0 + + for c in expected: + if c != peek(state, pos): + return False + pos += 1 + + consume(state, pos) + return True + + +def consume_lws(state): + """Consumes a LWS from the head. Returns True if any LWS is consumed. + Otherwise, returns False. + + LWS = [CRLF] 1*( SP | HT ) + """ + + original_head = state.head + + consume_string(state, '\r\n') + + pos = 0 + + while True: + c = peek(state, pos) + if c == ' ' or c == '\t': + pos += 1 + else: + if pos == 0: + state.head = original_head + return False + else: + consume(state, pos) + return True + + +def consume_lwses(state): + """Consumes *LWS from the head.""" + + while consume_lws(state): + pass + + +def consume_token(state): + """Consumes a token from the head. Returns the token or None if no token + was found. + """ + + pos = 0 + + while True: + c = peek(state, pos) + if c is None or c in _SEPARATORS or _is_ctl(c) or not _is_char(c): + if pos == 0: + return None + + return consume(state, pos) + else: + pos += 1 + + +def consume_token_or_quoted_string(state): + """Consumes a token or a quoted-string, and returns the token or unquoted + string. If no token or quoted-string was found, returns None. + """ + + original_head = state.head + + if not consume_string(state, '"'): + return consume_token(state) + + result = [] + + expect_quoted_pair = False + + while True: + if not expect_quoted_pair and consume_lws(state): + result.append(' ') + continue + + c = consume(state) + if c is None: + # quoted-string is not enclosed with double quotation + state.head = original_head + return None + elif expect_quoted_pair: + expect_quoted_pair = False + if _is_char(c): + result.append(c) + else: + # Non CHAR character found in quoted-pair + state.head = original_head + return None + elif c == '\\': + expect_quoted_pair = True + elif c == '"': + return ''.join(result) + elif _is_ctl(c): + # Invalid character %r found in qdtext + state.head = original_head + return None + else: + result.append(c) + + +def quote_if_necessary(s): + """Quotes arbitrary string into quoted-string.""" + + quote = False + if s == '': + return '""' + + result = [] + for c in s: + if c == '"' or c in _SEPARATORS or _is_ctl(c) or not _is_char(c): + quote = True + + if c == '"' or _is_ctl(c): + result.append('\\' + c) + else: + result.append(c) + + if quote: + return '"' + ''.join(result) + '"' + else: + return ''.join(result) + + +def parse_uri(uri): + """Parse absolute URI then return host, port and resource.""" + + parsed = urlparse.urlsplit(uri) + if parsed.scheme != 'wss' and parsed.scheme != 'ws': + # |uri| must be a relative URI. + # TODO(toyoshim): Should validate |uri|. + return None, None, uri + + if parsed.hostname is None: + return None, None, None + + port = None + try: + port = parsed.port + except ValueError, e: + # port property cause ValueError on invalid null port description like + # 'ws://host:/path'. + return None, None, None + + if port is None: + if parsed.scheme == 'ws': + port = 80 + else: + port = 443 + + path = parsed.path + if not path: + path += '/' + if parsed.query: + path += '?' + parsed.query + if parsed.fragment: + path += '#' + parsed.fragment + + return parsed.hostname, port, path + + +try: + urlparse.uses_netloc.index('ws') +except ValueError, e: + # urlparse in Python2.5.1 doesn't have 'ws' and 'wss' entries. + urlparse.uses_netloc.append('ws') + urlparse.uses_netloc.append('wss') + + +# vi:sts=4 sw=4 et diff --git a/pyload/lib/mod_pywebsocket/memorizingfile.py b/pyload/lib/mod_pywebsocket/memorizingfile.py new file mode 100644 index 000000000..4d4cd9585 --- /dev/null +++ b/pyload/lib/mod_pywebsocket/memorizingfile.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Memorizing file. + +A memorizing file wraps a file and memorizes lines read by readline. +""" + + +import sys + + +class MemorizingFile(object): + """MemorizingFile wraps a file and memorizes lines read by readline. + + Note that data read by other methods are not memorized. This behavior + is good enough for memorizing lines SimpleHTTPServer reads before + the control reaches WebSocketRequestHandler. + """ + + def __init__(self, file_, max_memorized_lines=sys.maxint): + """Construct an instance. + + Args: + file_: the file object to wrap. + max_memorized_lines: the maximum number of lines to memorize. + Only the first max_memorized_lines are memorized. + Default: sys.maxint. + """ + + self._file = file_ + self._memorized_lines = [] + self._max_memorized_lines = max_memorized_lines + self._buffered = False + self._buffered_line = None + + def __getattribute__(self, name): + if name in ('_file', '_memorized_lines', '_max_memorized_lines', + '_buffered', '_buffered_line', 'readline', + 'get_memorized_lines'): + return object.__getattribute__(self, name) + return self._file.__getattribute__(name) + + def readline(self, size=-1): + """Override file.readline and memorize the line read. + + Note that even if size is specified and smaller than actual size, + the whole line will be read out from underlying file object by + subsequent readline calls. + """ + + if self._buffered: + line = self._buffered_line + self._buffered = False + else: + line = self._file.readline() + if line and len(self._memorized_lines) < self._max_memorized_lines: + self._memorized_lines.append(line) + if size >= 0 and size < len(line): + self._buffered = True + self._buffered_line = line[size:] + return line[:size] + return line + + def get_memorized_lines(self): + """Get lines memorized so far.""" + return self._memorized_lines + + +# vi:sts=4 sw=4 et diff --git a/pyload/lib/mod_pywebsocket/msgutil.py b/pyload/lib/mod_pywebsocket/msgutil.py new file mode 100644 index 000000000..4c1a0114b --- /dev/null +++ b/pyload/lib/mod_pywebsocket/msgutil.py @@ -0,0 +1,219 @@ +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Message related utilities. + +Note: request.connection.write/read are used in this module, even though +mod_python document says that they should be used only in connection +handlers. Unfortunately, we have no other options. For example, +request.write/read are not suitable because they don't allow direct raw +bytes writing/reading. +""" + + +import Queue +import threading + + +# Export Exception symbols from msgutil for backward compatibility +from mod_pywebsocket._stream_base import ConnectionTerminatedException +from mod_pywebsocket._stream_base import InvalidFrameException +from mod_pywebsocket._stream_base import BadOperationException +from mod_pywebsocket._stream_base import UnsupportedFrameException + + +# An API for handler to send/receive WebSocket messages. +def close_connection(request): + """Close connection. + + Args: + request: mod_python request. + """ + request.ws_stream.close_connection() + + +def send_message(request, payload_data, end=True, binary=False): + """Send a message (or part of a message). + + Args: + request: mod_python request. + payload_data: unicode text or str binary to send. + end: True to terminate a message. + False to send payload_data as part of a message that is to be + terminated by next or later send_message call with end=True. + binary: send payload_data as binary frame(s). + Raises: + BadOperationException: when server already terminated. + """ + request.ws_stream.send_message(payload_data, end, binary) + + +def receive_message(request): + """Receive a WebSocket frame and return its payload as a text in + unicode or a binary in str. + + Args: + request: mod_python request. + Raises: + InvalidFrameException: when client send invalid frame. + UnsupportedFrameException: when client send unsupported frame e.g. some + of reserved bit is set but no extension can + recognize it. + InvalidUTF8Exception: when client send a text frame containing any + invalid UTF-8 string. + ConnectionTerminatedException: when the connection is closed + unexpectedly. + BadOperationException: when client already terminated. + """ + return request.ws_stream.receive_message() + + +def send_ping(request, body=''): + request.ws_stream.send_ping(body) + + +class MessageReceiver(threading.Thread): + """This class receives messages from the client. + + This class provides three ways to receive messages: blocking, + non-blocking, and via callback. Callback has the highest precedence. + + Note: This class should not be used with the standalone server for wss + because pyOpenSSL used by the server raises a fatal error if the socket + is accessed from multiple threads. + """ + + def __init__(self, request, onmessage=None): + """Construct an instance. + + Args: + request: mod_python request. + onmessage: a function to be called when a message is received. + May be None. If not None, the function is called on + another thread. In that case, MessageReceiver.receive + and MessageReceiver.receive_nowait are useless + because they will never return any messages. + """ + + threading.Thread.__init__(self) + self._request = request + self._queue = Queue.Queue() + self._onmessage = onmessage + self._stop_requested = False + self.setDaemon(True) + self.start() + + def run(self): + try: + while not self._stop_requested: + message = receive_message(self._request) + if self._onmessage: + self._onmessage(message) + else: + self._queue.put(message) + finally: + close_connection(self._request) + + def receive(self): + """ Receive a message from the channel, blocking. + + Returns: + message as a unicode string. + """ + return self._queue.get() + + def receive_nowait(self): + """ Receive a message from the channel, non-blocking. + + Returns: + message as a unicode string if available. None otherwise. + """ + try: + message = self._queue.get_nowait() + except Queue.Empty: + message = None + return message + + def stop(self): + """Request to stop this instance. + + The instance will be stopped after receiving the next message. + This method may not be very useful, but there is no clean way + in Python to forcefully stop a running thread. + """ + self._stop_requested = True + + +class MessageSender(threading.Thread): + """This class sends messages to the client. + + This class provides both synchronous and asynchronous ways to send + messages. + + Note: This class should not be used with the standalone server for wss + because pyOpenSSL used by the server raises a fatal error if the socket + is accessed from multiple threads. + """ + + def __init__(self, request): + """Construct an instance. + + Args: + request: mod_python request. + """ + threading.Thread.__init__(self) + self._request = request + self._queue = Queue.Queue() + self.setDaemon(True) + self.start() + + def run(self): + while True: + message, condition = self._queue.get() + condition.acquire() + send_message(self._request, message) + condition.notify() + condition.release() + + def send(self, message): + """Send a message, blocking.""" + + condition = threading.Condition() + condition.acquire() + self._queue.put((message, condition)) + condition.wait() + + def send_nowait(self, message): + """Send a message, non-blocking.""" + + self._queue.put((message, threading.Condition())) + + +# vi:sts=4 sw=4 et diff --git a/pyload/lib/mod_pywebsocket/mux.py b/pyload/lib/mod_pywebsocket/mux.py new file mode 100644 index 000000000..f0bdd2461 --- /dev/null +++ b/pyload/lib/mod_pywebsocket/mux.py @@ -0,0 +1,1636 @@ +# Copyright 2012, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""This file provides classes and helper functions for multiplexing extension. + +Specification: +http://tools.ietf.org/html/draft-ietf-hybi-websocket-multiplexing-06 +""" + + +import collections +import copy +import email +import email.parser +import logging +import math +import struct +import threading +import traceback + +from mod_pywebsocket import common +from mod_pywebsocket import handshake +from mod_pywebsocket import util +from mod_pywebsocket._stream_base import BadOperationException +from mod_pywebsocket._stream_base import ConnectionTerminatedException +from mod_pywebsocket._stream_hybi import Frame +from mod_pywebsocket._stream_hybi import Stream +from mod_pywebsocket._stream_hybi import StreamOptions +from mod_pywebsocket._stream_hybi import create_binary_frame +from mod_pywebsocket._stream_hybi import create_closing_handshake_body +from mod_pywebsocket._stream_hybi import create_header +from mod_pywebsocket._stream_hybi import create_length_header +from mod_pywebsocket._stream_hybi import parse_frame +from mod_pywebsocket.handshake import hybi + + +_CONTROL_CHANNEL_ID = 0 +_DEFAULT_CHANNEL_ID = 1 + +_MUX_OPCODE_ADD_CHANNEL_REQUEST = 0 +_MUX_OPCODE_ADD_CHANNEL_RESPONSE = 1 +_MUX_OPCODE_FLOW_CONTROL = 2 +_MUX_OPCODE_DROP_CHANNEL = 3 +_MUX_OPCODE_NEW_CHANNEL_SLOT = 4 + +_MAX_CHANNEL_ID = 2 ** 29 - 1 + +_INITIAL_NUMBER_OF_CHANNEL_SLOTS = 64 +_INITIAL_QUOTA_FOR_CLIENT = 8 * 1024 + +_HANDSHAKE_ENCODING_IDENTITY = 0 +_HANDSHAKE_ENCODING_DELTA = 1 + +# We need only these status code for now. +_HTTP_BAD_RESPONSE_MESSAGES = { + common.HTTP_STATUS_BAD_REQUEST: 'Bad Request', +} + +# DropChannel reason code +# TODO(bashi): Define all reason code defined in -05 draft. +_DROP_CODE_NORMAL_CLOSURE = 1000 + +_DROP_CODE_INVALID_ENCAPSULATING_MESSAGE = 2001 +_DROP_CODE_CHANNEL_ID_TRUNCATED = 2002 +_DROP_CODE_ENCAPSULATED_FRAME_IS_TRUNCATED = 2003 +_DROP_CODE_UNKNOWN_MUX_OPCODE = 2004 +_DROP_CODE_INVALID_MUX_CONTROL_BLOCK = 2005 +_DROP_CODE_CHANNEL_ALREADY_EXISTS = 2006 +_DROP_CODE_NEW_CHANNEL_SLOT_VIOLATION = 2007 + +_DROP_CODE_UNKNOWN_REQUEST_ENCODING = 3002 +_DROP_CODE_SEND_QUOTA_VIOLATION = 3005 +_DROP_CODE_ACKNOWLEDGED = 3008 + + +class MuxUnexpectedException(Exception): + """Exception in handling multiplexing extension.""" + pass + + +# Temporary +class MuxNotImplementedException(Exception): + """Raised when a flow enters unimplemented code path.""" + pass + + +class LogicalConnectionClosedException(Exception): + """Raised when logical connection is gracefully closed.""" + pass + + +class PhysicalConnectionError(Exception): + """Raised when there is a physical connection error.""" + def __init__(self, drop_code, message=''): + super(PhysicalConnectionError, self).__init__( + 'code=%d, message=%r' % (drop_code, message)) + self.drop_code = drop_code + self.message = message + + +class LogicalChannelError(Exception): + """Raised when there is a logical channel error.""" + def __init__(self, channel_id, drop_code, message=''): + super(LogicalChannelError, self).__init__( + 'channel_id=%d, code=%d, message=%r' % ( + channel_id, drop_code, message)) + self.channel_id = channel_id + self.drop_code = drop_code + self.message = message + + +def _encode_channel_id(channel_id): + if channel_id < 0: + raise ValueError('Channel id %d must not be negative' % channel_id) + + if channel_id < 2 ** 7: + return chr(channel_id) + if channel_id < 2 ** 14: + return struct.pack('!H', 0x8000 + channel_id) + if channel_id < 2 ** 21: + first = chr(0xc0 + (channel_id >> 16)) + return first + struct.pack('!H', channel_id & 0xffff) + if channel_id < 2 ** 29: + return struct.pack('!L', 0xe0000000 + channel_id) + + raise ValueError('Channel id %d is too large' % channel_id) + + +def _encode_number(number): + return create_length_header(number, False) + + +def _create_add_channel_response(channel_id, encoded_handshake, + encoding=0, rejected=False, + outer_frame_mask=False): + if encoding != 0 and encoding != 1: + raise ValueError('Invalid encoding %d' % encoding) + + first_byte = ((_MUX_OPCODE_ADD_CHANNEL_RESPONSE << 5) | + (rejected << 4) | encoding) + block = (chr(first_byte) + + _encode_channel_id(channel_id) + + _encode_number(len(encoded_handshake)) + + encoded_handshake) + payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block + return create_binary_frame(payload, mask=outer_frame_mask) + + +def _create_drop_channel(channel_id, code=None, message='', + outer_frame_mask=False): + if len(message) > 0 and code is None: + raise ValueError('Code must be specified if message is specified') + + first_byte = _MUX_OPCODE_DROP_CHANNEL << 5 + block = chr(first_byte) + _encode_channel_id(channel_id) + if code is None: + block += _encode_number(0) # Reason size + else: + reason = struct.pack('!H', code) + message + reason_size = _encode_number(len(reason)) + block += reason_size + reason + + payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block + return create_binary_frame(payload, mask=outer_frame_mask) + + +def _create_flow_control(channel_id, replenished_quota, + outer_frame_mask=False): + first_byte = _MUX_OPCODE_FLOW_CONTROL << 5 + block = (chr(first_byte) + + _encode_channel_id(channel_id) + + _encode_number(replenished_quota)) + payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block + return create_binary_frame(payload, mask=outer_frame_mask) + + +def _create_new_channel_slot(slots, send_quota, outer_frame_mask=False): + if slots < 0 or send_quota < 0: + raise ValueError('slots and send_quota must be non-negative.') + first_byte = _MUX_OPCODE_NEW_CHANNEL_SLOT << 5 + block = (chr(first_byte) + + _encode_number(slots) + + _encode_number(send_quota)) + payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block + return create_binary_frame(payload, mask=outer_frame_mask) + + +def _create_fallback_new_channel_slot(outer_frame_mask=False): + first_byte = (_MUX_OPCODE_NEW_CHANNEL_SLOT << 5) | 1 # Set the F flag + block = (chr(first_byte) + _encode_number(0) + _encode_number(0)) + payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block + return create_binary_frame(payload, mask=outer_frame_mask) + + +def _parse_request_text(request_text): + request_line, header_lines = request_text.split('\r\n', 1) + + words = request_line.split(' ') + if len(words) != 3: + raise ValueError('Bad Request-Line syntax %r' % request_line) + [command, path, version] = words + if version != 'HTTP/1.1': + raise ValueError('Bad request version %r' % version) + + # email.parser.Parser() parses RFC 2822 (RFC 822) style headers. + # RFC 6455 refers RFC 2616 for handshake parsing, and RFC 2616 refers + # RFC 822. + headers = email.parser.Parser().parsestr(header_lines) + return command, path, version, headers + + +class _ControlBlock(object): + """A structure that holds parsing result of multiplexing control block. + Control block specific attributes will be added by _MuxFramePayloadParser. + (e.g. encoded_handshake will be added for AddChannelRequest and + AddChannelResponse) + """ + + def __init__(self, opcode): + self.opcode = opcode + + +class _MuxFramePayloadParser(object): + """A class that parses multiplexed frame payload.""" + + def __init__(self, payload): + self._data = payload + self._read_position = 0 + self._logger = util.get_class_logger(self) + + def read_channel_id(self): + """Reads channel id. + + Raises: + ValueError: when the payload doesn't contain + valid channel id. + """ + + remaining_length = len(self._data) - self._read_position + pos = self._read_position + if remaining_length == 0: + raise ValueError('Invalid channel id format') + + channel_id = ord(self._data[pos]) + channel_id_length = 1 + if channel_id & 0xe0 == 0xe0: + if remaining_length < 4: + raise ValueError('Invalid channel id format') + channel_id = struct.unpack('!L', + self._data[pos:pos+4])[0] & 0x1fffffff + channel_id_length = 4 + elif channel_id & 0xc0 == 0xc0: + if remaining_length < 3: + raise ValueError('Invalid channel id format') + channel_id = (((channel_id & 0x1f) << 16) + + struct.unpack('!H', self._data[pos+1:pos+3])[0]) + channel_id_length = 3 + elif channel_id & 0x80 == 0x80: + if remaining_length < 2: + raise ValueError('Invalid channel id format') + channel_id = struct.unpack('!H', + self._data[pos:pos+2])[0] & 0x3fff + channel_id_length = 2 + self._read_position += channel_id_length + + return channel_id + + def read_inner_frame(self): + """Reads an inner frame. + + Raises: + PhysicalConnectionError: when the inner frame is invalid. + """ + + if len(self._data) == self._read_position: + raise PhysicalConnectionError( + _DROP_CODE_ENCAPSULATED_FRAME_IS_TRUNCATED) + + bits = ord(self._data[self._read_position]) + self._read_position += 1 + fin = (bits & 0x80) == 0x80 + rsv1 = (bits & 0x40) == 0x40 + rsv2 = (bits & 0x20) == 0x20 + rsv3 = (bits & 0x10) == 0x10 + opcode = bits & 0xf + payload = self.remaining_data() + # Consume rest of the message which is payload data of the original + # frame. + self._read_position = len(self._data) + return fin, rsv1, rsv2, rsv3, opcode, payload + + def _read_number(self): + if self._read_position + 1 > len(self._data): + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Cannot read the first byte of number field') + + number = ord(self._data[self._read_position]) + if number & 0x80 == 0x80: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'The most significant bit of the first byte of number should ' + 'be unset') + self._read_position += 1 + pos = self._read_position + if number == 127: + if pos + 8 > len(self._data): + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Invalid number field') + self._read_position += 8 + number = struct.unpack('!Q', self._data[pos:pos+8])[0] + if number > 0x7FFFFFFFFFFFFFFF: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Encoded number >= 2^63') + if number <= 0xFFFF: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + '%d should not be encoded by 9 bytes encoding' % number) + return number + if number == 126: + if pos + 2 > len(self._data): + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Invalid number field') + self._read_position += 2 + number = struct.unpack('!H', self._data[pos:pos+2])[0] + if number <= 125: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + '%d should not be encoded by 3 bytes encoding' % number) + return number + + def _read_size_and_contents(self): + """Reads data that consists of followings: + - the size of the contents encoded the same way as payload length + of the WebSocket Protocol with 1 bit padding at the head. + - the contents. + """ + + size = self._read_number() + pos = self._read_position + if pos + size > len(self._data): + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Cannot read %d bytes data' % size) + + self._read_position += size + return self._data[pos:pos+size] + + def _read_add_channel_request(self, first_byte, control_block): + reserved = (first_byte >> 2) & 0x7 + if reserved != 0: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Reserved bits must be unset') + + # Invalid encoding will be handled by MuxHandler. + encoding = first_byte & 0x3 + try: + control_block.channel_id = self.read_channel_id() + except ValueError, e: + raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) + control_block.encoding = encoding + encoded_handshake = self._read_size_and_contents() + control_block.encoded_handshake = encoded_handshake + return control_block + + def _read_add_channel_response(self, first_byte, control_block): + reserved = (first_byte >> 2) & 0x3 + if reserved != 0: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Reserved bits must be unset') + + control_block.accepted = (first_byte >> 4) & 1 + control_block.encoding = first_byte & 0x3 + try: + control_block.channel_id = self.read_channel_id() + except ValueError, e: + raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) + control_block.encoded_handshake = self._read_size_and_contents() + return control_block + + def _read_flow_control(self, first_byte, control_block): + reserved = first_byte & 0x1f + if reserved != 0: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Reserved bits must be unset') + + try: + control_block.channel_id = self.read_channel_id() + except ValueError, e: + raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) + control_block.send_quota = self._read_number() + return control_block + + def _read_drop_channel(self, first_byte, control_block): + reserved = first_byte & 0x1f + if reserved != 0: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Reserved bits must be unset') + + try: + control_block.channel_id = self.read_channel_id() + except ValueError, e: + raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) + reason = self._read_size_and_contents() + if len(reason) == 0: + control_block.drop_code = None + control_block.drop_message = '' + elif len(reason) >= 2: + control_block.drop_code = struct.unpack('!H', reason[:2])[0] + control_block.drop_message = reason[2:] + else: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Received DropChannel that conains only 1-byte reason') + return control_block + + def _read_new_channel_slot(self, first_byte, control_block): + reserved = first_byte & 0x1e + if reserved != 0: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Reserved bits must be unset') + control_block.fallback = first_byte & 1 + control_block.slots = self._read_number() + control_block.send_quota = self._read_number() + return control_block + + def read_control_blocks(self): + """Reads control block(s). + + Raises: + PhysicalConnectionError: when the payload contains invalid control + block(s). + StopIteration: when no control blocks left. + """ + + while self._read_position < len(self._data): + first_byte = ord(self._data[self._read_position]) + self._read_position += 1 + opcode = (first_byte >> 5) & 0x7 + control_block = _ControlBlock(opcode=opcode) + if opcode == _MUX_OPCODE_ADD_CHANNEL_REQUEST: + yield self._read_add_channel_request(first_byte, control_block) + elif opcode == _MUX_OPCODE_ADD_CHANNEL_RESPONSE: + yield self._read_add_channel_response( + first_byte, control_block) + elif opcode == _MUX_OPCODE_FLOW_CONTROL: + yield self._read_flow_control(first_byte, control_block) + elif opcode == _MUX_OPCODE_DROP_CHANNEL: + yield self._read_drop_channel(first_byte, control_block) + elif opcode == _MUX_OPCODE_NEW_CHANNEL_SLOT: + yield self._read_new_channel_slot(first_byte, control_block) + else: + raise PhysicalConnectionError( + _DROP_CODE_UNKNOWN_MUX_OPCODE, + 'Invalid opcode %d' % opcode) + + assert self._read_position == len(self._data) + raise StopIteration + + def remaining_data(self): + """Returns remaining data.""" + + return self._data[self._read_position:] + + +class _LogicalRequest(object): + """Mimics mod_python request.""" + + def __init__(self, channel_id, command, path, protocol, headers, + connection): + """Constructs an instance. + + Args: + channel_id: the channel id of the logical channel. + command: HTTP request command. + path: HTTP request path. + headers: HTTP headers. + connection: _LogicalConnection instance. + """ + + self.channel_id = channel_id + self.method = command + self.uri = path + self.protocol = protocol + self.headers_in = headers + self.connection = connection + self.server_terminated = False + self.client_terminated = False + + def is_https(self): + """Mimics request.is_https(). Returns False because this method is + used only by old protocols (hixie and hybi00). + """ + + return False + + +class _LogicalConnection(object): + """Mimics mod_python mp_conn.""" + + # For details, see the comment of set_read_state(). + STATE_ACTIVE = 1 + STATE_GRACEFULLY_CLOSED = 2 + STATE_TERMINATED = 3 + + def __init__(self, mux_handler, channel_id): + """Constructs an instance. + + Args: + mux_handler: _MuxHandler instance. + channel_id: channel id of this connection. + """ + + self._mux_handler = mux_handler + self._channel_id = channel_id + self._incoming_data = '' + self._write_condition = threading.Condition() + self._waiting_write_completion = False + self._read_condition = threading.Condition() + self._read_state = self.STATE_ACTIVE + + def get_local_addr(self): + """Getter to mimic mp_conn.local_addr.""" + + return self._mux_handler.physical_connection.get_local_addr() + local_addr = property(get_local_addr) + + def get_remote_addr(self): + """Getter to mimic mp_conn.remote_addr.""" + + return self._mux_handler.physical_connection.get_remote_addr() + remote_addr = property(get_remote_addr) + + def get_memorized_lines(self): + """Gets memorized lines. Not supported.""" + + raise MuxUnexpectedException('_LogicalConnection does not support ' + 'get_memorized_lines') + + def write(self, data): + """Writes data. mux_handler sends data asynchronously. The caller will + be suspended until write done. + + Args: + data: data to be written. + + Raises: + MuxUnexpectedException: when called before finishing the previous + write. + """ + + try: + self._write_condition.acquire() + if self._waiting_write_completion: + raise MuxUnexpectedException( + 'Logical connection %d is already waiting the completion ' + 'of write' % self._channel_id) + + self._waiting_write_completion = True + self._mux_handler.send_data(self._channel_id, data) + self._write_condition.wait() + finally: + self._write_condition.release() + + def write_control_data(self, data): + """Writes data via the control channel. Don't wait finishing write + because this method can be called by mux dispatcher. + + Args: + data: data to be written. + """ + + self._mux_handler.send_control_data(data) + + def notify_write_done(self): + """Called when sending data is completed.""" + + try: + self._write_condition.acquire() + if not self._waiting_write_completion: + raise MuxUnexpectedException( + 'Invalid call of notify_write_done for logical connection' + ' %d' % self._channel_id) + self._waiting_write_completion = False + self._write_condition.notify() + finally: + self._write_condition.release() + + def append_frame_data(self, frame_data): + """Appends incoming frame data. Called when mux_handler dispatches + frame data to the corresponding application. + + Args: + frame_data: incoming frame data. + """ + + self._read_condition.acquire() + self._incoming_data += frame_data + self._read_condition.notify() + self._read_condition.release() + + def read(self, length): + """Reads data. Blocks until enough data has arrived via physical + connection. + + Args: + length: length of data to be read. + Raises: + LogicalConnectionClosedException: when closing handshake for this + logical channel has been received. + ConnectionTerminatedException: when the physical connection has + closed, or an error is caused on the reader thread. + """ + + self._read_condition.acquire() + while (self._read_state == self.STATE_ACTIVE and + len(self._incoming_data) < length): + self._read_condition.wait() + + try: + if self._read_state == self.STATE_GRACEFULLY_CLOSED: + raise LogicalConnectionClosedException( + 'Logical channel %d has closed.' % self._channel_id) + elif self._read_state == self.STATE_TERMINATED: + raise ConnectionTerminatedException( + 'Receiving %d byte failed. Logical channel (%d) closed' % + (length, self._channel_id)) + + value = self._incoming_data[:length] + self._incoming_data = self._incoming_data[length:] + finally: + self._read_condition.release() + + return value + + def set_read_state(self, new_state): + """Sets the state of this connection. Called when an event for this + connection has occurred. + + Args: + new_state: state to be set. new_state must be one of followings: + - STATE_GRACEFULLY_CLOSED: when closing handshake for this + connection has been received. + - STATE_TERMINATED: when the physical connection has closed or + DropChannel of this connection has received. + """ + + self._read_condition.acquire() + self._read_state = new_state + self._read_condition.notify() + self._read_condition.release() + + +class _LogicalStream(Stream): + """Mimics the Stream class. This class interprets multiplexed WebSocket + frames. + """ + + def __init__(self, request, send_quota, receive_quota): + """Constructs an instance. + + Args: + request: _LogicalRequest instance. + send_quota: Initial send quota. + receive_quota: Initial receive quota. + """ + + # TODO(bashi): Support frame filters. + stream_options = StreamOptions() + # Physical stream is responsible for masking. + stream_options.unmask_receive = False + # Control frames can be fragmented on logical channel. + stream_options.allow_fragmented_control_frame = True + Stream.__init__(self, request, stream_options) + self._send_quota = send_quota + self._send_quota_condition = threading.Condition() + self._receive_quota = receive_quota + self._write_inner_frame_semaphore = threading.Semaphore() + + def _create_inner_frame(self, opcode, payload, end=True): + # TODO(bashi): Support extensions that use reserved bits. + first_byte = (end << 7) | opcode + return (_encode_channel_id(self._request.channel_id) + + chr(first_byte) + payload) + + def _write_inner_frame(self, opcode, payload, end=True): + payload_length = len(payload) + write_position = 0 + + try: + # An inner frame will be fragmented if there is no enough send + # quota. This semaphore ensures that fragmented inner frames are + # sent in order on the logical channel. + # Note that frames that come from other logical channels or + # multiplexing control blocks can be inserted between fragmented + # inner frames on the physical channel. + self._write_inner_frame_semaphore.acquire() + while write_position < payload_length: + try: + self._send_quota_condition.acquire() + while self._send_quota == 0: + self._logger.debug( + 'No quota. Waiting FlowControl message for %d.' % + self._request.channel_id) + self._send_quota_condition.wait() + + remaining = payload_length - write_position + write_length = min(self._send_quota, remaining) + inner_frame_end = ( + end and + (write_position + write_length == payload_length)) + + inner_frame = self._create_inner_frame( + opcode, + payload[write_position:write_position+write_length], + inner_frame_end) + frame_data = self._writer.build( + inner_frame, end=True, binary=True) + self._send_quota -= write_length + self._logger.debug('Consumed quota=%d, remaining=%d' % + (write_length, self._send_quota)) + finally: + self._send_quota_condition.release() + + # Writing data will block the worker so we need to release + # _send_quota_condition before writing. + self._logger.debug('Sending inner frame: %r' % frame_data) + self._request.connection.write(frame_data) + write_position += write_length + + opcode = common.OPCODE_CONTINUATION + + except ValueError, e: + raise BadOperationException(e) + finally: + self._write_inner_frame_semaphore.release() + + def replenish_send_quota(self, send_quota): + """Replenish send quota.""" + + self._send_quota_condition.acquire() + self._send_quota += send_quota + self._logger.debug('Replenished send quota for channel id %d: %d' % + (self._request.channel_id, self._send_quota)) + self._send_quota_condition.notify() + self._send_quota_condition.release() + + def consume_receive_quota(self, amount): + """Consumes receive quota. Returns False on failure.""" + + if self._receive_quota < amount: + self._logger.debug('Violate quota on channel id %d: %d < %d' % + (self._request.channel_id, + self._receive_quota, amount)) + return False + self._receive_quota -= amount + return True + + def send_message(self, message, end=True, binary=False): + """Override Stream.send_message.""" + + if self._request.server_terminated: + raise BadOperationException( + 'Requested send_message after sending out a closing handshake') + + if binary and isinstance(message, unicode): + raise BadOperationException( + 'Message for binary frame must be instance of str') + + if binary: + opcode = common.OPCODE_BINARY + else: + opcode = common.OPCODE_TEXT + message = message.encode('utf-8') + + self._write_inner_frame(opcode, message, end) + + def _receive_frame(self): + """Overrides Stream._receive_frame. + + In addition to call Stream._receive_frame, this method adds the amount + of payload to receiving quota and sends FlowControl to the client. + We need to do it here because Stream.receive_message() handles + control frames internally. + """ + + opcode, payload, fin, rsv1, rsv2, rsv3 = Stream._receive_frame(self) + amount = len(payload) + self._receive_quota += amount + frame_data = _create_flow_control(self._request.channel_id, + amount) + self._logger.debug('Sending flow control for %d, replenished=%d' % + (self._request.channel_id, amount)) + self._request.connection.write_control_data(frame_data) + return opcode, payload, fin, rsv1, rsv2, rsv3 + + def receive_message(self): + """Overrides Stream.receive_message.""" + + # Just call Stream.receive_message(), but catch + # LogicalConnectionClosedException, which is raised when the logical + # connection has closed gracefully. + try: + return Stream.receive_message(self) + except LogicalConnectionClosedException, e: + self._logger.debug('%s', e) + return None + + def _send_closing_handshake(self, code, reason): + """Overrides Stream._send_closing_handshake.""" + + body = create_closing_handshake_body(code, reason) + self._logger.debug('Sending closing handshake for %d: (%r, %r)' % + (self._request.channel_id, code, reason)) + self._write_inner_frame(common.OPCODE_CLOSE, body, end=True) + + self._request.server_terminated = True + + def send_ping(self, body=''): + """Overrides Stream.send_ping""" + + self._logger.debug('Sending ping on logical channel %d: %r' % + (self._request.channel_id, body)) + self._write_inner_frame(common.OPCODE_PING, body, end=True) + + self._ping_queue.append(body) + + def _send_pong(self, body): + """Overrides Stream._send_pong""" + + self._logger.debug('Sending pong on logical channel %d: %r' % + (self._request.channel_id, body)) + self._write_inner_frame(common.OPCODE_PONG, body, end=True) + + def close_connection(self, code=common.STATUS_NORMAL_CLOSURE, reason=''): + """Overrides Stream.close_connection.""" + + # TODO(bashi): Implement + self._logger.debug('Closing logical connection %d' % + self._request.channel_id) + self._request.server_terminated = True + + def _drain_received_data(self): + """Overrides Stream._drain_received_data. Nothing need to be done for + logical channel. + """ + + pass + + +class _OutgoingData(object): + """A structure that holds data to be sent via physical connection and + origin of the data. + """ + + def __init__(self, channel_id, data): + self.channel_id = channel_id + self.data = data + + +class _PhysicalConnectionWriter(threading.Thread): + """A thread that is responsible for writing data to physical connection. + + TODO(bashi): Make sure there is no thread-safety problem when the reader + thread reads data from the same socket at a time. + """ + + def __init__(self, mux_handler): + """Constructs an instance. + + Args: + mux_handler: _MuxHandler instance. + """ + + threading.Thread.__init__(self) + self._logger = util.get_class_logger(self) + self._mux_handler = mux_handler + self.setDaemon(True) + self._stop_requested = False + self._deque = collections.deque() + self._deque_condition = threading.Condition() + + def put_outgoing_data(self, data): + """Puts outgoing data. + + Args: + data: _OutgoingData instance. + + Raises: + BadOperationException: when the thread has been requested to + terminate. + """ + + try: + self._deque_condition.acquire() + if self._stop_requested: + raise BadOperationException('Cannot write data anymore') + + self._deque.append(data) + self._deque_condition.notify() + finally: + self._deque_condition.release() + + def _write_data(self, outgoing_data): + try: + self._mux_handler.physical_connection.write(outgoing_data.data) + except Exception, e: + util.prepend_message_to_exception( + 'Failed to send message to %r: ' % + (self._mux_handler.physical_connection.remote_addr,), e) + raise + + # TODO(bashi): It would be better to block the thread that sends + # control data as well. + if outgoing_data.channel_id != _CONTROL_CHANNEL_ID: + self._mux_handler.notify_write_done(outgoing_data.channel_id) + + def run(self): + self._deque_condition.acquire() + while not self._stop_requested: + if len(self._deque) == 0: + self._deque_condition.wait() + continue + + outgoing_data = self._deque.popleft() + self._deque_condition.release() + self._write_data(outgoing_data) + self._deque_condition.acquire() + + # Flush deque + try: + while len(self._deque) > 0: + outgoing_data = self._deque.popleft() + self._write_data(outgoing_data) + finally: + self._deque_condition.release() + + def stop(self): + """Stops the writer thread.""" + + self._deque_condition.acquire() + self._stop_requested = True + self._deque_condition.notify() + self._deque_condition.release() + + +class _PhysicalConnectionReader(threading.Thread): + """A thread that is responsible for reading data from physical connection. + """ + + def __init__(self, mux_handler): + """Constructs an instance. + + Args: + mux_handler: _MuxHandler instance. + """ + + threading.Thread.__init__(self) + self._logger = util.get_class_logger(self) + self._mux_handler = mux_handler + self.setDaemon(True) + + def run(self): + while True: + try: + physical_stream = self._mux_handler.physical_stream + message = physical_stream.receive_message() + if message is None: + break + # Below happens only when a data message is received. + opcode = physical_stream.get_last_received_opcode() + if opcode != common.OPCODE_BINARY: + self._mux_handler.fail_physical_connection( + _DROP_CODE_INVALID_ENCAPSULATING_MESSAGE, + 'Received a text message on physical connection') + break + + except ConnectionTerminatedException, e: + self._logger.debug('%s', e) + break + + try: + self._mux_handler.dispatch_message(message) + except PhysicalConnectionError, e: + self._mux_handler.fail_physical_connection( + e.drop_code, e.message) + break + except LogicalChannelError, e: + self._mux_handler.fail_logical_channel( + e.channel_id, e.drop_code, e.message) + except Exception, e: + self._logger.debug(traceback.format_exc()) + break + + self._mux_handler.notify_reader_done() + + +class _Worker(threading.Thread): + """A thread that is responsible for running the corresponding application + handler. + """ + + def __init__(self, mux_handler, request): + """Constructs an instance. + + Args: + mux_handler: _MuxHandler instance. + request: _LogicalRequest instance. + """ + + threading.Thread.__init__(self) + self._logger = util.get_class_logger(self) + self._mux_handler = mux_handler + self._request = request + self.setDaemon(True) + + def run(self): + self._logger.debug('Logical channel worker started. (id=%d)' % + self._request.channel_id) + try: + # Non-critical exceptions will be handled by dispatcher. + self._mux_handler.dispatcher.transfer_data(self._request) + finally: + self._mux_handler.notify_worker_done(self._request.channel_id) + + +class _MuxHandshaker(hybi.Handshaker): + """Opening handshake processor for multiplexing.""" + + _DUMMY_WEBSOCKET_KEY = 'dGhlIHNhbXBsZSBub25jZQ==' + + def __init__(self, request, dispatcher, send_quota, receive_quota): + """Constructs an instance. + Args: + request: _LogicalRequest instance. + dispatcher: Dispatcher instance (dispatch.Dispatcher). + send_quota: Initial send quota. + receive_quota: Initial receive quota. + """ + + hybi.Handshaker.__init__(self, request, dispatcher) + self._send_quota = send_quota + self._receive_quota = receive_quota + + # Append headers which should not be included in handshake field of + # AddChannelRequest. + # TODO(bashi): Make sure whether we should raise exception when + # these headers are included already. + request.headers_in[common.UPGRADE_HEADER] = ( + common.WEBSOCKET_UPGRADE_TYPE) + request.headers_in[common.CONNECTION_HEADER] = ( + common.UPGRADE_CONNECTION_TYPE) + request.headers_in[common.SEC_WEBSOCKET_VERSION_HEADER] = ( + str(common.VERSION_HYBI_LATEST)) + request.headers_in[common.SEC_WEBSOCKET_KEY_HEADER] = ( + self._DUMMY_WEBSOCKET_KEY) + + def _create_stream(self, stream_options): + """Override hybi.Handshaker._create_stream.""" + + self._logger.debug('Creating logical stream for %d' % + self._request.channel_id) + return _LogicalStream(self._request, self._send_quota, + self._receive_quota) + + def _create_handshake_response(self, accept): + """Override hybi._create_handshake_response.""" + + response = [] + + response.append('HTTP/1.1 101 Switching Protocols\r\n') + + # Upgrade, Connection and Sec-WebSocket-Accept should be excluded. + if self._request.ws_protocol is not None: + response.append('%s: %s\r\n' % ( + common.SEC_WEBSOCKET_PROTOCOL_HEADER, + self._request.ws_protocol)) + if (self._request.ws_extensions is not None and + len(self._request.ws_extensions) != 0): + response.append('%s: %s\r\n' % ( + common.SEC_WEBSOCKET_EXTENSIONS_HEADER, + common.format_extensions(self._request.ws_extensions))) + response.append('\r\n') + + return ''.join(response) + + def _send_handshake(self, accept): + """Override hybi.Handshaker._send_handshake.""" + + # Don't send handshake response for the default channel + if self._request.channel_id == _DEFAULT_CHANNEL_ID: + return + + handshake_response = self._create_handshake_response(accept) + frame_data = _create_add_channel_response( + self._request.channel_id, + handshake_response) + self._logger.debug('Sending handshake response for %d: %r' % + (self._request.channel_id, frame_data)) + self._request.connection.write_control_data(frame_data) + + +class _LogicalChannelData(object): + """A structure that holds information about logical channel. + """ + + def __init__(self, request, worker): + self.request = request + self.worker = worker + self.drop_code = _DROP_CODE_NORMAL_CLOSURE + self.drop_message = '' + + +class _HandshakeDeltaBase(object): + """A class that holds information for delta-encoded handshake.""" + + def __init__(self, headers): + self._headers = headers + + def create_headers(self, delta=None): + """Creates request headers for an AddChannelRequest that has + delta-encoded handshake. + + Args: + delta: headers should be overridden. + """ + + headers = copy.copy(self._headers) + if delta: + for key, value in delta.items(): + # The spec requires that a header with an empty value is + # removed from the delta base. + if len(value) == 0 and headers.has_key(key): + del headers[key] + else: + headers[key] = value + # TODO(bashi): Support extensions + headers['Sec-WebSocket-Extensions'] = '' + return headers + + +class _MuxHandler(object): + """Multiplexing handler. When a handler starts, it launches three + threads; the reader thread, the writer thread, and a worker thread. + + The reader thread reads data from the physical stream, i.e., the + ws_stream object of the underlying websocket connection. The reader + thread interprets multiplexed frames and dispatches them to logical + channels. Methods of this class are mostly called by the reader thread. + + The writer thread sends multiplexed frames which are created by + logical channels via the physical connection. + + The worker thread launched at the starting point handles the + "Implicitly Opened Connection". If multiplexing handler receives + an AddChannelRequest and accepts it, the handler will launch a new worker + thread and dispatch the request to it. + """ + + def __init__(self, request, dispatcher): + """Constructs an instance. + + Args: + request: mod_python request of the physical connection. + dispatcher: Dispatcher instance (dispatch.Dispatcher). + """ + + self.original_request = request + self.dispatcher = dispatcher + self.physical_connection = request.connection + self.physical_stream = request.ws_stream + self._logger = util.get_class_logger(self) + self._logical_channels = {} + self._logical_channels_condition = threading.Condition() + # Holds client's initial quota + self._channel_slots = collections.deque() + self._handshake_base = None + self._worker_done_notify_received = False + self._reader = None + self._writer = None + + def start(self): + """Starts the handler. + + Raises: + MuxUnexpectedException: when the handler already started, or when + opening handshake of the default channel fails. + """ + + if self._reader or self._writer: + raise MuxUnexpectedException('MuxHandler already started') + + self._reader = _PhysicalConnectionReader(self) + self._writer = _PhysicalConnectionWriter(self) + self._reader.start() + self._writer.start() + + # Create "Implicitly Opened Connection". + logical_connection = _LogicalConnection(self, _DEFAULT_CHANNEL_ID) + self._handshake_base = _HandshakeDeltaBase( + self.original_request.headers_in) + logical_request = _LogicalRequest( + _DEFAULT_CHANNEL_ID, + self.original_request.method, + self.original_request.uri, + self.original_request.protocol, + self._handshake_base.create_headers(), + logical_connection) + # Client's send quota for the implicitly opened connection is zero, + # but we will send FlowControl later so set the initial quota to + # _INITIAL_QUOTA_FOR_CLIENT. + self._channel_slots.append(_INITIAL_QUOTA_FOR_CLIENT) + if not self._do_handshake_for_logical_request( + logical_request, send_quota=self.original_request.mux_quota): + raise MuxUnexpectedException( + 'Failed handshake on the default channel id') + self._add_logical_channel(logical_request) + + # Send FlowControl for the implicitly opened connection. + frame_data = _create_flow_control(_DEFAULT_CHANNEL_ID, + _INITIAL_QUOTA_FOR_CLIENT) + logical_request.connection.write_control_data(frame_data) + + def add_channel_slots(self, slots, send_quota): + """Adds channel slots. + + Args: + slots: number of slots to be added. + send_quota: initial send quota for slots. + """ + + self._channel_slots.extend([send_quota] * slots) + # Send NewChannelSlot to client. + frame_data = _create_new_channel_slot(slots, send_quota) + self.send_control_data(frame_data) + + def wait_until_done(self, timeout=None): + """Waits until all workers are done. Returns False when timeout has + occurred. Returns True on success. + + Args: + timeout: timeout in sec. + """ + + self._logical_channels_condition.acquire() + try: + while len(self._logical_channels) > 0: + self._logger.debug('Waiting workers(%d)...' % + len(self._logical_channels)) + self._worker_done_notify_received = False + self._logical_channels_condition.wait(timeout) + if not self._worker_done_notify_received: + self._logger.debug('Waiting worker(s) timed out') + return False + + finally: + self._logical_channels_condition.release() + + # Flush pending outgoing data + self._writer.stop() + self._writer.join() + + return True + + def notify_write_done(self, channel_id): + """Called by the writer thread when a write operation has done. + + Args: + channel_id: objective channel id. + """ + + try: + self._logical_channels_condition.acquire() + if channel_id in self._logical_channels: + channel_data = self._logical_channels[channel_id] + channel_data.request.connection.notify_write_done() + else: + self._logger.debug('Seems that logical channel for %d has gone' + % channel_id) + finally: + self._logical_channels_condition.release() + + def send_control_data(self, data): + """Sends data via the control channel. + + Args: + data: data to be sent. + """ + + self._writer.put_outgoing_data(_OutgoingData( + channel_id=_CONTROL_CHANNEL_ID, data=data)) + + def send_data(self, channel_id, data): + """Sends data via given logical channel. This method is called by + worker threads. + + Args: + channel_id: objective channel id. + data: data to be sent. + """ + + self._writer.put_outgoing_data(_OutgoingData( + channel_id=channel_id, data=data)) + + def _send_drop_channel(self, channel_id, code=None, message=''): + frame_data = _create_drop_channel(channel_id, code, message) + self._logger.debug( + 'Sending drop channel for channel id %d' % channel_id) + self.send_control_data(frame_data) + + def _send_error_add_channel_response(self, channel_id, status=None): + if status is None: + status = common.HTTP_STATUS_BAD_REQUEST + + if status in _HTTP_BAD_RESPONSE_MESSAGES: + message = _HTTP_BAD_RESPONSE_MESSAGES[status] + else: + self._logger.debug('Response message for %d is not found' % status) + message = '???' + + response = 'HTTP/1.1 %d %s\r\n\r\n' % (status, message) + frame_data = _create_add_channel_response(channel_id, + encoded_handshake=response, + encoding=0, rejected=True) + self.send_control_data(frame_data) + + def _create_logical_request(self, block): + if block.channel_id == _CONTROL_CHANNEL_ID: + # TODO(bashi): Raise PhysicalConnectionError with code 2006 + # instead of MuxUnexpectedException. + raise MuxUnexpectedException( + 'Received the control channel id (0) as objective channel ' + 'id for AddChannel') + + if block.encoding > _HANDSHAKE_ENCODING_DELTA: + raise PhysicalConnectionError( + _DROP_CODE_UNKNOWN_REQUEST_ENCODING) + + method, path, version, headers = _parse_request_text( + block.encoded_handshake) + if block.encoding == _HANDSHAKE_ENCODING_DELTA: + headers = self._handshake_base.create_headers(headers) + + connection = _LogicalConnection(self, block.channel_id) + request = _LogicalRequest(block.channel_id, method, path, version, + headers, connection) + return request + + def _do_handshake_for_logical_request(self, request, send_quota=0): + try: + receive_quota = self._channel_slots.popleft() + except IndexError: + raise LogicalChannelError( + request.channel_id, _DROP_CODE_NEW_CHANNEL_SLOT_VIOLATION) + + handshaker = _MuxHandshaker(request, self.dispatcher, + send_quota, receive_quota) + try: + handshaker.do_handshake() + except handshake.VersionException, e: + self._logger.info('%s', e) + self._send_error_add_channel_response( + request.channel_id, status=common.HTTP_STATUS_BAD_REQUEST) + return False + except handshake.HandshakeException, e: + # TODO(bashi): Should we _Fail the Logical Channel_ with 3001 + # instead? + self._logger.info('%s', e) + self._send_error_add_channel_response(request.channel_id, + status=e.status) + return False + except handshake.AbortedByUserException, e: + self._logger.info('%s', e) + self._send_error_add_channel_response(request.channel_id) + return False + + return True + + def _add_logical_channel(self, logical_request): + try: + self._logical_channels_condition.acquire() + if logical_request.channel_id in self._logical_channels: + self._logger.debug('Channel id %d already exists' % + logical_request.channel_id) + raise PhysicalConnectionError( + _DROP_CODE_CHANNEL_ALREADY_EXISTS, + 'Channel id %d already exists' % + logical_request.channel_id) + worker = _Worker(self, logical_request) + channel_data = _LogicalChannelData(logical_request, worker) + self._logical_channels[logical_request.channel_id] = channel_data + worker.start() + finally: + self._logical_channels_condition.release() + + def _process_add_channel_request(self, block): + try: + logical_request = self._create_logical_request(block) + except ValueError, e: + self._logger.debug('Failed to create logical request: %r' % e) + self._send_error_add_channel_response( + block.channel_id, status=common.HTTP_STATUS_BAD_REQUEST) + return + if self._do_handshake_for_logical_request(logical_request): + if block.encoding == _HANDSHAKE_ENCODING_IDENTITY: + # Update handshake base. + # TODO(bashi): Make sure this is the right place to update + # handshake base. + self._handshake_base = _HandshakeDeltaBase( + logical_request.headers_in) + self._add_logical_channel(logical_request) + else: + self._send_error_add_channel_response( + block.channel_id, status=common.HTTP_STATUS_BAD_REQUEST) + + def _process_flow_control(self, block): + try: + self._logical_channels_condition.acquire() + if not block.channel_id in self._logical_channels: + return + channel_data = self._logical_channels[block.channel_id] + channel_data.request.ws_stream.replenish_send_quota( + block.send_quota) + finally: + self._logical_channels_condition.release() + + def _process_drop_channel(self, block): + self._logger.debug( + 'DropChannel received for %d: code=%r, reason=%r' % + (block.channel_id, block.drop_code, block.drop_message)) + try: + self._logical_channels_condition.acquire() + if not block.channel_id in self._logical_channels: + return + channel_data = self._logical_channels[block.channel_id] + channel_data.drop_code = _DROP_CODE_ACKNOWLEDGED + # Close the logical channel + channel_data.request.connection.set_read_state( + _LogicalConnection.STATE_TERMINATED) + finally: + self._logical_channels_condition.release() + + def _process_control_blocks(self, parser): + for control_block in parser.read_control_blocks(): + opcode = control_block.opcode + self._logger.debug('control block received, opcode: %d' % opcode) + if opcode == _MUX_OPCODE_ADD_CHANNEL_REQUEST: + self._process_add_channel_request(control_block) + elif opcode == _MUX_OPCODE_ADD_CHANNEL_RESPONSE: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Received AddChannelResponse') + elif opcode == _MUX_OPCODE_FLOW_CONTROL: + self._process_flow_control(control_block) + elif opcode == _MUX_OPCODE_DROP_CHANNEL: + self._process_drop_channel(control_block) + elif opcode == _MUX_OPCODE_NEW_CHANNEL_SLOT: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Received NewChannelSlot') + else: + raise MuxUnexpectedException( + 'Unexpected opcode %r' % opcode) + + def _process_logical_frame(self, channel_id, parser): + self._logger.debug('Received a frame. channel id=%d' % channel_id) + try: + self._logical_channels_condition.acquire() + if not channel_id in self._logical_channels: + # We must ignore the message for an inactive channel. + return + channel_data = self._logical_channels[channel_id] + fin, rsv1, rsv2, rsv3, opcode, payload = parser.read_inner_frame() + if not channel_data.request.ws_stream.consume_receive_quota( + len(payload)): + # The client violates quota. Close logical channel. + raise LogicalChannelError( + channel_id, _DROP_CODE_SEND_QUOTA_VIOLATION) + header = create_header(opcode, len(payload), fin, rsv1, rsv2, rsv3, + mask=False) + frame_data = header + payload + channel_data.request.connection.append_frame_data(frame_data) + finally: + self._logical_channels_condition.release() + + def dispatch_message(self, message): + """Dispatches message. The reader thread calls this method. + + Args: + message: a message that contains encapsulated frame. + Raises: + PhysicalConnectionError: if the message contains physical + connection level errors. + LogicalChannelError: if the message contains logical channel + level errors. + """ + + parser = _MuxFramePayloadParser(message) + try: + channel_id = parser.read_channel_id() + except ValueError, e: + raise PhysicalConnectionError(_DROP_CODE_CHANNEL_ID_TRUNCATED) + if channel_id == _CONTROL_CHANNEL_ID: + self._process_control_blocks(parser) + else: + self._process_logical_frame(channel_id, parser) + + def notify_worker_done(self, channel_id): + """Called when a worker has finished. + + Args: + channel_id: channel id corresponded with the worker. + """ + + self._logger.debug('Worker for channel id %d terminated' % channel_id) + try: + self._logical_channels_condition.acquire() + if not channel_id in self._logical_channels: + raise MuxUnexpectedException( + 'Channel id %d not found' % channel_id) + channel_data = self._logical_channels.pop(channel_id) + finally: + self._worker_done_notify_received = True + self._logical_channels_condition.notify() + self._logical_channels_condition.release() + + if not channel_data.request.server_terminated: + self._send_drop_channel( + channel_id, code=channel_data.drop_code, + message=channel_data.drop_message) + + def notify_reader_done(self): + """This method is called by the reader thread when the reader has + finished. + """ + + # Terminate all logical connections + self._logger.debug('termiating all logical connections...') + self._logical_channels_condition.acquire() + for channel_data in self._logical_channels.values(): + try: + channel_data.request.connection.set_read_state( + _LogicalConnection.STATE_TERMINATED) + except Exception: + pass + self._logical_channels_condition.release() + + def fail_physical_connection(self, code, message): + """Fail the physical connection. + + Args: + code: drop reason code. + message: drop message. + """ + + self._logger.debug('Failing the physical connection...') + self._send_drop_channel(_CONTROL_CHANNEL_ID, code, message) + self.physical_stream.close_connection( + common.STATUS_INTERNAL_ENDPOINT_ERROR) + + def fail_logical_channel(self, channel_id, code, message): + """Fail a logical channel. + + Args: + channel_id: channel id. + code: drop reason code. + message: drop message. + """ + + self._logger.debug('Failing logical channel %d...' % channel_id) + try: + self._logical_channels_condition.acquire() + if channel_id in self._logical_channels: + channel_data = self._logical_channels[channel_id] + # Close the logical channel. notify_worker_done() will be + # called later and it will send DropChannel. + channel_data.drop_code = code + channel_data.drop_message = message + channel_data.request.connection.set_read_state( + _LogicalConnection.STATE_TERMINATED) + else: + self._send_drop_channel(channel_id, code, message) + finally: + self._logical_channels_condition.release() + + +def use_mux(request): + return hasattr(request, 'mux') and request.mux + + +def start(request, dispatcher): + mux_handler = _MuxHandler(request, dispatcher) + mux_handler.start() + + mux_handler.add_channel_slots(_INITIAL_NUMBER_OF_CHANNEL_SLOTS, + _INITIAL_QUOTA_FOR_CLIENT) + + mux_handler.wait_until_done() + + +# vi:sts=4 sw=4 et diff --git a/pyload/lib/mod_pywebsocket/standalone.py b/pyload/lib/mod_pywebsocket/standalone.py new file mode 100755 index 000000000..07a33d9c9 --- /dev/null +++ b/pyload/lib/mod_pywebsocket/standalone.py @@ -0,0 +1,998 @@ +#!/usr/bin/env python +# +# Copyright 2012, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Standalone WebSocket server. + +Use this file to launch pywebsocket without Apache HTTP Server. + + +BASIC USAGE + +Go to the src directory and run + + $ python mod_pywebsocket/standalone.py [-p ] + [-w ] + [-d ] + + is the port number to use for ws:// connection. + + is the path to the root directory of HTML files. + + is the path to the root directory of WebSocket handlers. +If not specified, will be used. See __init__.py (or +run $ pydoc mod_pywebsocket) for how to write WebSocket handlers. + +For more detail and other options, run + + $ python mod_pywebsocket/standalone.py --help + +or see _build_option_parser method below. + +For trouble shooting, adding "--log_level debug" might help you. + + +TRY DEMO + +Go to the src directory and run + + $ python standalone.py -d example + +to launch pywebsocket with the sample handler and html on port 80. Open +http://localhost/console.html, click the connect button, type something into +the text box next to the send button and click the send button. If everything +is working, you'll see the message you typed echoed by the server. + + +SUPPORTING TLS + +To support TLS, run standalone.py with -t, -k, and -c options. + + +SUPPORTING CLIENT AUTHENTICATION + +To support client authentication with TLS, run standalone.py with -t, -k, -c, +and --tls-client-auth, and --tls-client-ca options. + +E.g., $./standalone.py -d ../example -p 10443 -t -c ../test/cert/cert.pem -k +../test/cert/key.pem --tls-client-auth --tls-client-ca=../test/cert/cacert.pem + + +CONFIGURATION FILE + +You can also write a configuration file and use it by specifying the path to +the configuration file by --config option. Please write a configuration file +following the documentation of the Python ConfigParser library. Name of each +entry must be the long version argument name. E.g. to set log level to debug, +add the following line: + +log_level=debug + +For options which doesn't take value, please add some fake value. E.g. for +--tls option, add the following line: + +tls=True + +Note that tls will be enabled even if you write tls=False as the value part is +fake. + +When both a command line argument and a configuration file entry are set for +the same configuration item, the command line value will override one in the +configuration file. + + +THREADING + +This server is derived from SocketServer.ThreadingMixIn. Hence a thread is +used for each request. + + +SECURITY WARNING + +This uses CGIHTTPServer and CGIHTTPServer is not secure. +It may execute arbitrary Python code or external programs. It should not be +used outside a firewall. +""" + +import BaseHTTPServer +import CGIHTTPServer +import SimpleHTTPServer +import SocketServer +import ConfigParser +import base64 +import httplib +import logging +import logging.handlers +import optparse +import os +import re +import select +import socket +import sys +import threading +import time + +_HAS_SSL = False +_HAS_OPEN_SSL = False +try: + import ssl + _HAS_SSL = True +except ImportError: + try: + import OpenSSL.SSL + _HAS_OPEN_SSL = True + except ImportError: + pass + +from mod_pywebsocket import common +from mod_pywebsocket import dispatch +from mod_pywebsocket import handshake +from mod_pywebsocket import http_header_util +from mod_pywebsocket import memorizingfile +from mod_pywebsocket import util + + +_DEFAULT_LOG_MAX_BYTES = 1024 * 256 +_DEFAULT_LOG_BACKUP_COUNT = 5 + +_DEFAULT_REQUEST_QUEUE_SIZE = 128 + +# 1024 is practically large enough to contain WebSocket handshake lines. +_MAX_MEMORIZED_LINES = 1024 + + +class _StandaloneConnection(object): + """Mimic mod_python mp_conn.""" + + def __init__(self, request_handler): + """Construct an instance. + + Args: + request_handler: A WebSocketRequestHandler instance. + """ + + self._request_handler = request_handler + + def get_local_addr(self): + """Getter to mimic mp_conn.local_addr.""" + + return (self._request_handler.server.server_name, + self._request_handler.server.server_port) + local_addr = property(get_local_addr) + + def get_remote_addr(self): + """Getter to mimic mp_conn.remote_addr. + + Setting the property in __init__ won't work because the request + handler is not initialized yet there.""" + + return self._request_handler.client_address + remote_addr = property(get_remote_addr) + + def write(self, data): + """Mimic mp_conn.write().""" + + return self._request_handler.wfile.write(data) + + def read(self, length): + """Mimic mp_conn.read().""" + + return self._request_handler.rfile.read(length) + + def get_memorized_lines(self): + """Get memorized lines.""" + + return self._request_handler.rfile.get_memorized_lines() + + +class _StandaloneRequest(object): + """Mimic mod_python request.""" + + def __init__(self, request_handler, use_tls): + """Construct an instance. + + Args: + request_handler: A WebSocketRequestHandler instance. + """ + + self._logger = util.get_class_logger(self) + + self._request_handler = request_handler + self.connection = _StandaloneConnection(request_handler) + self._use_tls = use_tls + self.headers_in = request_handler.headers + + def get_uri(self): + """Getter to mimic request.uri.""" + + return self._request_handler.path + uri = property(get_uri) + + def get_method(self): + """Getter to mimic request.method.""" + + return self._request_handler.command + method = property(get_method) + + def get_protocol(self): + """Getter to mimic request.protocol.""" + + return self._request_handler.request_version + protocol = property(get_protocol) + + def is_https(self): + """Mimic request.is_https().""" + + return self._use_tls + + def _drain_received_data(self): + """Don't use this method from WebSocket handler. Drains unread data + in the receive buffer. + """ + + raw_socket = self._request_handler.connection + drained_data = util.drain_received_data(raw_socket) + + if drained_data: + self._logger.debug( + 'Drained data following close frame: %r', drained_data) + + +class _StandaloneSSLConnection(object): + """A wrapper class for OpenSSL.SSL.Connection to provide makefile method + which is not supported by the class. + """ + + def __init__(self, connection): + self._connection = connection + + def __getattribute__(self, name): + if name in ('_connection', 'makefile'): + return object.__getattribute__(self, name) + return self._connection.__getattribute__(name) + + def __setattr__(self, name, value): + if name in ('_connection', 'makefile'): + return object.__setattr__(self, name, value) + return self._connection.__setattr__(name, value) + + def makefile(self, mode='r', bufsize=-1): + return socket._fileobject(self._connection, mode, bufsize) + + +def _alias_handlers(dispatcher, websock_handlers_map_file): + """Set aliases specified in websock_handler_map_file in dispatcher. + + Args: + dispatcher: dispatch.Dispatcher instance + websock_handler_map_file: alias map file + """ + + fp = open(websock_handlers_map_file) + try: + for line in fp: + if line[0] == '#' or line.isspace(): + continue + m = re.match('(\S+)\s+(\S+)', line) + if not m: + logging.warning('Wrong format in map file:' + line) + continue + try: + dispatcher.add_resource_path_alias( + m.group(1), m.group(2)) + except dispatch.DispatchException, e: + logging.error(str(e)) + finally: + fp.close() + + +class WebSocketServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): + """HTTPServer specialized for WebSocket.""" + + # Overrides SocketServer.ThreadingMixIn.daemon_threads + daemon_threads = True + # Overrides BaseHTTPServer.HTTPServer.allow_reuse_address + allow_reuse_address = True + + def __init__(self, options): + """Override SocketServer.TCPServer.__init__ to set SSL enabled + socket object to self.socket before server_bind and server_activate, + if necessary. + """ + + # Share a Dispatcher among request handlers to save time for + # instantiation. Dispatcher can be shared because it is thread-safe. + options.dispatcher = dispatch.Dispatcher( + options.websock_handlers, + options.scan_dir, + options.allow_handlers_outside_root_dir) + if options.websock_handlers_map_file: + _alias_handlers(options.dispatcher, + options.websock_handlers_map_file) + warnings = options.dispatcher.source_warnings() + if warnings: + for warning in warnings: + logging.warning('mod_pywebsocket: %s' % warning) + + self._logger = util.get_class_logger(self) + + self.request_queue_size = options.request_queue_size + self.__ws_is_shut_down = threading.Event() + self.__ws_serving = False + + SocketServer.BaseServer.__init__( + self, (options.server_host, options.port), WebSocketRequestHandler) + + # Expose the options object to allow handler objects access it. We name + # it with websocket_ prefix to avoid conflict. + self.websocket_server_options = options + + self._create_sockets() + self.server_bind() + self.server_activate() + + def _create_sockets(self): + self.server_name, self.server_port = self.server_address + self._sockets = [] + if not self.server_name: + # On platforms that doesn't support IPv6, the first bind fails. + # On platforms that supports IPv6 + # - If it binds both IPv4 and IPv6 on call with AF_INET6, the + # first bind succeeds and the second fails (we'll see 'Address + # already in use' error). + # - If it binds only IPv6 on call with AF_INET6, both call are + # expected to succeed to listen both protocol. + addrinfo_array = [ + (socket.AF_INET6, socket.SOCK_STREAM, '', '', ''), + (socket.AF_INET, socket.SOCK_STREAM, '', '', '')] + else: + addrinfo_array = socket.getaddrinfo(self.server_name, + self.server_port, + socket.AF_UNSPEC, + socket.SOCK_STREAM, + socket.IPPROTO_TCP) + for addrinfo in addrinfo_array: + self._logger.info('Create socket on: %r', addrinfo) + family, socktype, proto, canonname, sockaddr = addrinfo + try: + socket_ = socket.socket(family, socktype) + except Exception, e: + self._logger.info('Skip by failure: %r', e) + continue + if self.websocket_server_options.use_tls: + if _HAS_SSL: + if self.websocket_server_options.tls_client_auth: + client_cert_ = ssl.CERT_REQUIRED + else: + client_cert_ = ssl.CERT_NONE + socket_ = ssl.wrap_socket(socket_, + keyfile=self.websocket_server_options.private_key, + certfile=self.websocket_server_options.certificate, + ssl_version=ssl.PROTOCOL_SSLv23, + ca_certs=self.websocket_server_options.tls_client_ca, + cert_reqs=client_cert_) + if _HAS_OPEN_SSL: + ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) + ctx.use_privatekey_file( + self.websocket_server_options.private_key) + ctx.use_certificate_file( + self.websocket_server_options.certificate) + socket_ = OpenSSL.SSL.Connection(ctx, socket_) + self._sockets.append((socket_, addrinfo)) + + def server_bind(self): + """Override SocketServer.TCPServer.server_bind to enable multiple + sockets bind. + """ + + failed_sockets = [] + + for socketinfo in self._sockets: + socket_, addrinfo = socketinfo + self._logger.info('Bind on: %r', addrinfo) + if self.allow_reuse_address: + socket_.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + socket_.bind(self.server_address) + except Exception, e: + self._logger.info('Skip by failure: %r', e) + socket_.close() + failed_sockets.append(socketinfo) + if self.server_address[1] == 0: + # The operating system assigns the actual port number for port + # number 0. This case, the second and later sockets should use + # the same port number. Also self.server_port is rewritten + # because it is exported, and will be used by external code. + self.server_address = ( + self.server_name, socket_.getsockname()[1]) + self.server_port = self.server_address[1] + self._logger.info('Port %r is assigned', self.server_port) + + for socketinfo in failed_sockets: + self._sockets.remove(socketinfo) + + def server_activate(self): + """Override SocketServer.TCPServer.server_activate to enable multiple + sockets listen. + """ + + failed_sockets = [] + + for socketinfo in self._sockets: + socket_, addrinfo = socketinfo + self._logger.info('Listen on: %r', addrinfo) + try: + socket_.listen(self.request_queue_size) + except Exception, e: + self._logger.info('Skip by failure: %r', e) + socket_.close() + failed_sockets.append(socketinfo) + + for socketinfo in failed_sockets: + self._sockets.remove(socketinfo) + + if len(self._sockets) == 0: + self._logger.critical( + 'No sockets activated. Use info log level to see the reason.') + + def server_close(self): + """Override SocketServer.TCPServer.server_close to enable multiple + sockets close. + """ + + for socketinfo in self._sockets: + socket_, addrinfo = socketinfo + self._logger.info('Close on: %r', addrinfo) + socket_.close() + + def fileno(self): + """Override SocketServer.TCPServer.fileno.""" + + self._logger.critical('Not supported: fileno') + return self._sockets[0][0].fileno() + + def handle_error(self, rquest, client_address): + """Override SocketServer.handle_error.""" + + self._logger.error( + 'Exception in processing request from: %r\n%s', + client_address, + util.get_stack_trace()) + # Note: client_address is a tuple. + + def get_request(self): + """Override TCPServer.get_request to wrap OpenSSL.SSL.Connection + object with _StandaloneSSLConnection to provide makefile method. We + cannot substitute OpenSSL.SSL.Connection.makefile since it's readonly + attribute. + """ + + accepted_socket, client_address = self.socket.accept() + if self.websocket_server_options.use_tls and _HAS_OPEN_SSL: + accepted_socket = _StandaloneSSLConnection(accepted_socket) + return accepted_socket, client_address + + def serve_forever(self, poll_interval=0.5): + """Override SocketServer.BaseServer.serve_forever.""" + + self.__ws_serving = True + self.__ws_is_shut_down.clear() + handle_request = self.handle_request + if hasattr(self, '_handle_request_noblock'): + handle_request = self._handle_request_noblock + else: + self._logger.warning('Fallback to blocking request handler') + try: + while self.__ws_serving: + r, w, e = select.select( + [socket_[0] for socket_ in self._sockets], + [], [], poll_interval) + for socket_ in r: + self.socket = socket_ + handle_request() + self.socket = None + finally: + self.__ws_is_shut_down.set() + + def shutdown(self): + """Override SocketServer.BaseServer.shutdown.""" + + self.__ws_serving = False + self.__ws_is_shut_down.wait() + + +class WebSocketRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler): + """CGIHTTPRequestHandler specialized for WebSocket.""" + + # Use httplib.HTTPMessage instead of mimetools.Message. + MessageClass = httplib.HTTPMessage + + def setup(self): + """Override SocketServer.StreamRequestHandler.setup to wrap rfile + with MemorizingFile. + + This method will be called by BaseRequestHandler's constructor + before calling BaseHTTPRequestHandler.handle. + BaseHTTPRequestHandler.handle will call + BaseHTTPRequestHandler.handle_one_request and it will call + WebSocketRequestHandler.parse_request. + """ + + # Call superclass's setup to prepare rfile, wfile, etc. See setup + # definition on the root class SocketServer.StreamRequestHandler to + # understand what this does. + CGIHTTPServer.CGIHTTPRequestHandler.setup(self) + + self.rfile = memorizingfile.MemorizingFile( + self.rfile, + max_memorized_lines=_MAX_MEMORIZED_LINES) + + def __init__(self, request, client_address, server): + self._logger = util.get_class_logger(self) + + self._options = server.websocket_server_options + + # Overrides CGIHTTPServerRequestHandler.cgi_directories. + self.cgi_directories = self._options.cgi_directories + # Replace CGIHTTPRequestHandler.is_executable method. + if self._options.is_executable_method is not None: + self.is_executable = self._options.is_executable_method + + # This actually calls BaseRequestHandler.__init__. + CGIHTTPServer.CGIHTTPRequestHandler.__init__( + self, request, client_address, server) + + def parse_request(self): + """Override BaseHTTPServer.BaseHTTPRequestHandler.parse_request. + + Return True to continue processing for HTTP(S), False otherwise. + + See BaseHTTPRequestHandler.handle_one_request method which calls + this method to understand how the return value will be handled. + """ + + # We hook parse_request method, but also call the original + # CGIHTTPRequestHandler.parse_request since when we return False, + # CGIHTTPRequestHandler.handle_one_request continues processing and + # it needs variables set by CGIHTTPRequestHandler.parse_request. + # + # Variables set by this method will be also used by WebSocket request + # handling (self.path, self.command, self.requestline, etc. See also + # how _StandaloneRequest's members are implemented using these + # attributes). + if not CGIHTTPServer.CGIHTTPRequestHandler.parse_request(self): + return False + + if self._options.use_basic_auth: + auth = self.headers.getheader('Authorization') + if auth != self._options.basic_auth_credential: + self.send_response(401) + self.send_header('WWW-Authenticate', + 'Basic realm="Pywebsocket"') + self.end_headers() + self._logger.info('Request basic authentication') + return True + + host, port, resource = http_header_util.parse_uri(self.path) + if resource is None: + self._logger.info('Invalid URI: %r', self.path) + self._logger.info('Fallback to CGIHTTPRequestHandler') + return True + server_options = self.server.websocket_server_options + if host is not None: + validation_host = server_options.validation_host + if validation_host is not None and host != validation_host: + self._logger.info('Invalid host: %r (expected: %r)', + host, + validation_host) + self._logger.info('Fallback to CGIHTTPRequestHandler') + return True + if port is not None: + validation_port = server_options.validation_port + if validation_port is not None and port != validation_port: + self._logger.info('Invalid port: %r (expected: %r)', + port, + validation_port) + self._logger.info('Fallback to CGIHTTPRequestHandler') + return True + self.path = resource + + request = _StandaloneRequest(self, self._options.use_tls) + + try: + # Fallback to default http handler for request paths for which + # we don't have request handlers. + if not self._options.dispatcher.get_handler_suite(self.path): + self._logger.info('No handler for resource: %r', + self.path) + self._logger.info('Fallback to CGIHTTPRequestHandler') + return True + except dispatch.DispatchException, e: + self._logger.info('%s', e) + self.send_error(e.status) + return False + + # If any Exceptions without except clause setup (including + # DispatchException) is raised below this point, it will be caught + # and logged by WebSocketServer. + + try: + try: + handshake.do_handshake( + request, + self._options.dispatcher, + allowDraft75=self._options.allow_draft75, + strict=self._options.strict) + except handshake.VersionException, e: + self._logger.info('%s', e) + self.send_response(common.HTTP_STATUS_BAD_REQUEST) + self.send_header(common.SEC_WEBSOCKET_VERSION_HEADER, + e.supported_versions) + self.end_headers() + return False + except handshake.HandshakeException, e: + # Handshake for ws(s) failed. + self._logger.info('%s', e) + self.send_error(e.status) + return False + + request._dispatcher = self._options.dispatcher + self._options.dispatcher.transfer_data(request) + except handshake.AbortedByUserException, e: + self._logger.info('%s', e) + return False + + def log_request(self, code='-', size='-'): + """Override BaseHTTPServer.log_request.""" + + self._logger.info('"%s" %s %s', + self.requestline, str(code), str(size)) + + def log_error(self, *args): + """Override BaseHTTPServer.log_error.""" + + # Despite the name, this method is for warnings than for errors. + # For example, HTTP status code is logged by this method. + self._logger.warning('%s - %s', + self.address_string(), + args[0] % args[1:]) + + def is_cgi(self): + """Test whether self.path corresponds to a CGI script. + + Add extra check that self.path doesn't contains .. + Also check if the file is a executable file or not. + If the file is not executable, it is handled as static file or dir + rather than a CGI script. + """ + + if CGIHTTPServer.CGIHTTPRequestHandler.is_cgi(self): + if '..' in self.path: + return False + # strip query parameter from request path + resource_name = self.path.split('?', 2)[0] + # convert resource_name into real path name in filesystem. + scriptfile = self.translate_path(resource_name) + if not os.path.isfile(scriptfile): + return False + if not self.is_executable(scriptfile): + return False + return True + return False + + +def _get_logger_from_class(c): + return logging.getLogger('%s.%s' % (c.__module__, c.__name__)) + + +def _configure_logging(options): + logging.addLevelName(common.LOGLEVEL_FINE, 'FINE') + + logger = logging.getLogger() + logger.setLevel(logging.getLevelName(options.log_level.upper())) + if options.log_file: + handler = logging.handlers.RotatingFileHandler( + options.log_file, 'a', options.log_max, options.log_count) + else: + handler = logging.StreamHandler() + formatter = logging.Formatter( + '[%(asctime)s] [%(levelname)s] %(name)s: %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + + deflate_log_level_name = logging.getLevelName( + options.deflate_log_level.upper()) + _get_logger_from_class(util._Deflater).setLevel( + deflate_log_level_name) + _get_logger_from_class(util._Inflater).setLevel( + deflate_log_level_name) + + +def _build_option_parser(): + parser = optparse.OptionParser() + + parser.add_option('--config', dest='config_file', type='string', + default=None, + help=('Path to configuration file. See the file comment ' + 'at the top of this file for the configuration ' + 'file format')) + parser.add_option('-H', '--server-host', '--server_host', + dest='server_host', + default='', + help='server hostname to listen to') + parser.add_option('-V', '--validation-host', '--validation_host', + dest='validation_host', + default=None, + help='server hostname to validate in absolute path.') + parser.add_option('-p', '--port', dest='port', type='int', + default=common.DEFAULT_WEB_SOCKET_PORT, + help='port to listen to') + parser.add_option('-P', '--validation-port', '--validation_port', + dest='validation_port', type='int', + default=None, + help='server port to validate in absolute path.') + parser.add_option('-w', '--websock-handlers', '--websock_handlers', + dest='websock_handlers', + default='.', + help=('The root directory of WebSocket handler files. ' + 'If the path is relative, --document-root is used ' + 'as the base.')) + parser.add_option('-m', '--websock-handlers-map-file', + '--websock_handlers_map_file', + dest='websock_handlers_map_file', + default=None, + help=('WebSocket handlers map file. ' + 'Each line consists of alias_resource_path and ' + 'existing_resource_path, separated by spaces.')) + parser.add_option('-s', '--scan-dir', '--scan_dir', dest='scan_dir', + default=None, + help=('Must be a directory under --websock-handlers. ' + 'Only handlers under this directory are scanned ' + 'and registered to the server. ' + 'Useful for saving scan time when the handler ' + 'root directory contains lots of files that are ' + 'not handler file or are handler files but you ' + 'don\'t want them to be registered. ')) + parser.add_option('--allow-handlers-outside-root-dir', + '--allow_handlers_outside_root_dir', + dest='allow_handlers_outside_root_dir', + action='store_true', + default=False, + help=('Scans WebSocket handlers even if their canonical ' + 'path is not under --websock-handlers.')) + parser.add_option('-d', '--document-root', '--document_root', + dest='document_root', default='.', + help='Document root directory.') + parser.add_option('-x', '--cgi-paths', '--cgi_paths', dest='cgi_paths', + default=None, + help=('CGI paths relative to document_root.' + 'Comma-separated. (e.g -x /cgi,/htbin) ' + 'Files under document_root/cgi_path are handled ' + 'as CGI programs. Must be executable.')) + parser.add_option('-t', '--tls', dest='use_tls', action='store_true', + default=False, help='use TLS (wss://)') + parser.add_option('-k', '--private-key', '--private_key', + dest='private_key', + default='', help='TLS private key file.') + parser.add_option('-c', '--certificate', dest='certificate', + default='', help='TLS certificate file.') + parser.add_option('--tls-client-auth', dest='tls_client_auth', + action='store_true', default=False, + help='Requires TLS client auth on every connection.') + parser.add_option('--tls-client-ca', dest='tls_client_ca', default='', + help=('Specifies a pem file which contains a set of ' + 'concatenated CA certificates which are used to ' + 'validate certificates passed from clients')) + parser.add_option('--basic-auth', dest='use_basic_auth', + action='store_true', default=False, + help='Requires Basic authentication.') + parser.add_option('--basic-auth-credential', + dest='basic_auth_credential', default='test:test', + help='Specifies the credential of basic authentication ' + 'by username:password pair (e.g. test:test).') + parser.add_option('-l', '--log-file', '--log_file', dest='log_file', + default='', help='Log file.') + # Custom log level: + # - FINE: Prints status of each frame processing step + parser.add_option('--log-level', '--log_level', type='choice', + dest='log_level', default='warn', + choices=['fine', + 'debug', 'info', 'warning', 'warn', 'error', + 'critical'], + help='Log level.') + parser.add_option('--deflate-log-level', '--deflate_log_level', + type='choice', + dest='deflate_log_level', default='warn', + choices=['debug', 'info', 'warning', 'warn', 'error', + 'critical'], + help='Log level for _Deflater and _Inflater.') + parser.add_option('--thread-monitor-interval-in-sec', + '--thread_monitor_interval_in_sec', + dest='thread_monitor_interval_in_sec', + type='int', default=-1, + help=('If positive integer is specified, run a thread ' + 'monitor to show the status of server threads ' + 'periodically in the specified inteval in ' + 'second. If non-positive integer is specified, ' + 'disable the thread monitor.')) + parser.add_option('--log-max', '--log_max', dest='log_max', type='int', + default=_DEFAULT_LOG_MAX_BYTES, + help='Log maximum bytes') + parser.add_option('--log-count', '--log_count', dest='log_count', + type='int', default=_DEFAULT_LOG_BACKUP_COUNT, + help='Log backup count') + parser.add_option('--allow-draft75', dest='allow_draft75', + action='store_true', default=False, + help='Obsolete option. Ignored.') + parser.add_option('--strict', dest='strict', action='store_true', + default=False, help='Obsolete option. Ignored.') + parser.add_option('-q', '--queue', dest='request_queue_size', type='int', + default=_DEFAULT_REQUEST_QUEUE_SIZE, + help='request queue size') + + return parser + + +class ThreadMonitor(threading.Thread): + daemon = True + + def __init__(self, interval_in_sec): + threading.Thread.__init__(self, name='ThreadMonitor') + + self._logger = util.get_class_logger(self) + + self._interval_in_sec = interval_in_sec + + def run(self): + while True: + thread_name_list = [] + for thread in threading.enumerate(): + thread_name_list.append(thread.name) + self._logger.info( + "%d active threads: %s", + threading.active_count(), + ', '.join(thread_name_list)) + time.sleep(self._interval_in_sec) + + +def _parse_args_and_config(args): + parser = _build_option_parser() + + # First, parse options without configuration file. + temporary_options, temporary_args = parser.parse_args(args=args) + if temporary_args: + logging.critical( + 'Unrecognized positional arguments: %r', temporary_args) + sys.exit(1) + + if temporary_options.config_file: + try: + config_fp = open(temporary_options.config_file, 'r') + except IOError, e: + logging.critical( + 'Failed to open configuration file %r: %r', + temporary_options.config_file, + e) + sys.exit(1) + + config_parser = ConfigParser.SafeConfigParser() + config_parser.readfp(config_fp) + config_fp.close() + + args_from_config = [] + for name, value in config_parser.items('pywebsocket'): + args_from_config.append('--' + name) + args_from_config.append(value) + if args is None: + args = args_from_config + else: + args = args_from_config + args + return parser.parse_args(args=args) + else: + return temporary_options, temporary_args + + +def _main(args=None): + """You can call this function from your own program, but please note that + this function has some side-effects that might affect your program. For + example, util.wrap_popen3_for_win use in this method replaces implementation + of os.popen3. + """ + + options, args = _parse_args_and_config(args=args) + + os.chdir(options.document_root) + + _configure_logging(options) + + # TODO(tyoshino): Clean up initialization of CGI related values. Move some + # of code here to WebSocketRequestHandler class if it's better. + options.cgi_directories = [] + options.is_executable_method = None + if options.cgi_paths: + options.cgi_directories = options.cgi_paths.split(',') + if sys.platform in ('cygwin', 'win32'): + cygwin_path = None + # For Win32 Python, it is expected that CYGWIN_PATH + # is set to a directory of cygwin binaries. + # For example, websocket_server.py in Chromium sets CYGWIN_PATH to + # full path of third_party/cygwin/bin. + if 'CYGWIN_PATH' in os.environ: + cygwin_path = os.environ['CYGWIN_PATH'] + util.wrap_popen3_for_win(cygwin_path) + + def __check_script(scriptpath): + return util.get_script_interp(scriptpath, cygwin_path) + + options.is_executable_method = __check_script + + if options.use_tls: + if not (_HAS_SSL or _HAS_OPEN_SSL): + logging.critical('TLS support requires ssl or pyOpenSSL module.') + sys.exit(1) + if not options.private_key or not options.certificate: + logging.critical( + 'To use TLS, specify private_key and certificate.') + sys.exit(1) + + if options.tls_client_auth: + if not options.use_tls: + logging.critical('TLS must be enabled for client authentication.') + sys.exit(1) + if not _HAS_SSL: + logging.critical('Client authentication requires ssl module.') + + if not options.scan_dir: + options.scan_dir = options.websock_handlers + + if options.use_basic_auth: + options.basic_auth_credential = 'Basic ' + base64.b64encode( + options.basic_auth_credential) + + try: + if options.thread_monitor_interval_in_sec > 0: + # Run a thread monitor to show the status of server threads for + # debugging. + ThreadMonitor(options.thread_monitor_interval_in_sec).start() + + server = WebSocketServer(options) + server.serve_forever() + except Exception, e: + logging.critical('mod_pywebsocket: %s' % e) + logging.critical('mod_pywebsocket: %s' % util.get_stack_trace()) + sys.exit(1) + + +if __name__ == '__main__': + _main(sys.argv[1:]) + + +# vi:sts=4 sw=4 et diff --git a/pyload/lib/mod_pywebsocket/stream.py b/pyload/lib/mod_pywebsocket/stream.py new file mode 100644 index 000000000..edc533279 --- /dev/null +++ b/pyload/lib/mod_pywebsocket/stream.py @@ -0,0 +1,57 @@ +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""This file exports public symbols. +""" + + +from mod_pywebsocket._stream_base import BadOperationException +from mod_pywebsocket._stream_base import ConnectionTerminatedException +from mod_pywebsocket._stream_base import InvalidFrameException +from mod_pywebsocket._stream_base import InvalidUTF8Exception +from mod_pywebsocket._stream_base import UnsupportedFrameException +from mod_pywebsocket._stream_hixie75 import StreamHixie75 +from mod_pywebsocket._stream_hybi import Frame +from mod_pywebsocket._stream_hybi import Stream +from mod_pywebsocket._stream_hybi import StreamOptions + +# These methods are intended to be used by WebSocket client developers to have +# their implementations receive broken data in tests. +from mod_pywebsocket._stream_hybi import create_close_frame +from mod_pywebsocket._stream_hybi import create_header +from mod_pywebsocket._stream_hybi import create_length_header +from mod_pywebsocket._stream_hybi import create_ping_frame +from mod_pywebsocket._stream_hybi import create_pong_frame +from mod_pywebsocket._stream_hybi import create_binary_frame +from mod_pywebsocket._stream_hybi import create_text_frame +from mod_pywebsocket._stream_hybi import create_closing_handshake_body + + +# vi:sts=4 sw=4 et diff --git a/pyload/lib/mod_pywebsocket/util.py b/pyload/lib/mod_pywebsocket/util.py new file mode 100644 index 000000000..7bb0b5d9e --- /dev/null +++ b/pyload/lib/mod_pywebsocket/util.py @@ -0,0 +1,515 @@ +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""WebSocket utilities. +""" + + +import array +import errno + +# Import hash classes from a module available and recommended for each Python +# version and re-export those symbol. Use sha and md5 module in Python 2.4, and +# hashlib module in Python 2.6. +try: + import hashlib + md5_hash = hashlib.md5 + sha1_hash = hashlib.sha1 +except ImportError: + import md5 + import sha + md5_hash = md5.md5 + sha1_hash = sha.sha + +import StringIO +import logging +import os +import re +import socket +import traceback +import zlib + + +def get_stack_trace(): + """Get the current stack trace as string. + + This is needed to support Python 2.3. + TODO: Remove this when we only support Python 2.4 and above. + Use traceback.format_exc instead. + """ + + out = StringIO.StringIO() + traceback.print_exc(file=out) + return out.getvalue() + + +def prepend_message_to_exception(message, exc): + """Prepend message to the exception.""" + + exc.args = (message + str(exc),) + return + + +def __translate_interp(interp, cygwin_path): + """Translate interp program path for Win32 python to run cygwin program + (e.g. perl). Note that it doesn't support path that contains space, + which is typically true for Unix, where #!-script is written. + For Win32 python, cygwin_path is a directory of cygwin binaries. + + Args: + interp: interp command line + cygwin_path: directory name of cygwin binary, or None + Returns: + translated interp command line. + """ + if not cygwin_path: + return interp + m = re.match('^[^ ]*/([^ ]+)( .*)?', interp) + if m: + cmd = os.path.join(cygwin_path, m.group(1)) + return cmd + m.group(2) + return interp + + +def get_script_interp(script_path, cygwin_path=None): + """Gets #!-interpreter command line from the script. + + It also fixes command path. When Cygwin Python is used, e.g. in WebKit, + it could run "/usr/bin/perl -wT hello.pl". + When Win32 Python is used, e.g. in Chromium, it couldn't. So, fix + "/usr/bin/perl" to "\perl.exe". + + Args: + script_path: pathname of the script + cygwin_path: directory name of cygwin binary, or None + Returns: + #!-interpreter command line, or None if it is not #!-script. + """ + fp = open(script_path) + line = fp.readline() + fp.close() + m = re.match('^#!(.*)', line) + if m: + return __translate_interp(m.group(1), cygwin_path) + return None + + +def wrap_popen3_for_win(cygwin_path): + """Wrap popen3 to support #!-script on Windows. + + Args: + cygwin_path: path for cygwin binary if command path is needed to be + translated. None if no translation required. + """ + + __orig_popen3 = os.popen3 + + def __wrap_popen3(cmd, mode='t', bufsize=-1): + cmdline = cmd.split(' ') + interp = get_script_interp(cmdline[0], cygwin_path) + if interp: + cmd = interp + ' ' + cmd + return __orig_popen3(cmd, mode, bufsize) + + os.popen3 = __wrap_popen3 + + +def hexify(s): + return ' '.join(map(lambda x: '%02x' % ord(x), s)) + + +def get_class_logger(o): + return logging.getLogger( + '%s.%s' % (o.__class__.__module__, o.__class__.__name__)) + + +class NoopMasker(object): + """A masking object that has the same interface as RepeatedXorMasker but + just returns the string passed in without making any change. + """ + + def __init__(self): + pass + + def mask(self, s): + return s + + +class RepeatedXorMasker(object): + """A masking object that applies XOR on the string given to mask method + with the masking bytes given to the constructor repeatedly. This object + remembers the position in the masking bytes the last mask method call + ended and resumes from that point on the next mask method call. + """ + + def __init__(self, mask): + self._mask = map(ord, mask) + self._mask_size = len(self._mask) + self._count = 0 + + def mask(self, s): + result = array.array('B') + result.fromstring(s) + # Use temporary local variables to eliminate the cost to access + # attributes + count = self._count + mask = self._mask + mask_size = self._mask_size + for i in xrange(len(result)): + result[i] ^= mask[count] + count = (count + 1) % mask_size + self._count = count + + return result.tostring() + + +class DeflateRequest(object): + """A wrapper class for request object to intercept send and recv to perform + deflate compression and decompression transparently. + """ + + def __init__(self, request): + self._request = request + self.connection = DeflateConnection(request.connection) + + def __getattribute__(self, name): + if name in ('_request', 'connection'): + return object.__getattribute__(self, name) + return self._request.__getattribute__(name) + + def __setattr__(self, name, value): + if name in ('_request', 'connection'): + return object.__setattr__(self, name, value) + return self._request.__setattr__(name, value) + + +# By making wbits option negative, we can suppress CMF/FLG (2 octet) and +# ADLER32 (4 octet) fields of zlib so that we can use zlib module just as +# deflate library. DICTID won't be added as far as we don't set dictionary. +# LZ77 window of 32K will be used for both compression and decompression. +# For decompression, we can just use 32K to cover any windows size. For +# compression, we use 32K so receivers must use 32K. +# +# Compression level is Z_DEFAULT_COMPRESSION. We don't have to match level +# to decode. +# +# See zconf.h, deflate.cc, inflate.cc of zlib library, and zlibmodule.c of +# Python. See also RFC1950 (ZLIB 3.3). + + +class _Deflater(object): + + def __init__(self, window_bits): + self._logger = get_class_logger(self) + + self._compress = zlib.compressobj( + zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -window_bits) + + def compress(self, bytes): + compressed_bytes = self._compress.compress(bytes) + self._logger.debug('Compress input %r', bytes) + self._logger.debug('Compress result %r', compressed_bytes) + return compressed_bytes + + def compress_and_flush(self, bytes): + compressed_bytes = self._compress.compress(bytes) + compressed_bytes += self._compress.flush(zlib.Z_SYNC_FLUSH) + self._logger.debug('Compress input %r', bytes) + self._logger.debug('Compress result %r', compressed_bytes) + return compressed_bytes + + def compress_and_finish(self, bytes): + compressed_bytes = self._compress.compress(bytes) + compressed_bytes += self._compress.flush(zlib.Z_FINISH) + self._logger.debug('Compress input %r', bytes) + self._logger.debug('Compress result %r', compressed_bytes) + return compressed_bytes + +class _Inflater(object): + + def __init__(self): + self._logger = get_class_logger(self) + + self._unconsumed = '' + + self.reset() + + def decompress(self, size): + if not (size == -1 or size > 0): + raise Exception('size must be -1 or positive') + + data = '' + + while True: + if size == -1: + data += self._decompress.decompress(self._unconsumed) + # See Python bug http://bugs.python.org/issue12050 to + # understand why the same code cannot be used for updating + # self._unconsumed for here and else block. + self._unconsumed = '' + else: + data += self._decompress.decompress( + self._unconsumed, size - len(data)) + self._unconsumed = self._decompress.unconsumed_tail + if self._decompress.unused_data: + # Encountered a last block (i.e. a block with BFINAL = 1) and + # found a new stream (unused_data). We cannot use the same + # zlib.Decompress object for the new stream. Create a new + # Decompress object to decompress the new one. + # + # It's fine to ignore unconsumed_tail if unused_data is not + # empty. + self._unconsumed = self._decompress.unused_data + self.reset() + if size >= 0 and len(data) == size: + # data is filled. Don't call decompress again. + break + else: + # Re-invoke Decompress.decompress to try to decompress all + # available bytes before invoking read which blocks until + # any new byte is available. + continue + else: + # Here, since unused_data is empty, even if unconsumed_tail is + # not empty, bytes of requested length are already in data. We + # don't have to "continue" here. + break + + if data: + self._logger.debug('Decompressed %r', data) + return data + + def append(self, data): + self._logger.debug('Appended %r', data) + self._unconsumed += data + + def reset(self): + self._logger.debug('Reset') + self._decompress = zlib.decompressobj(-zlib.MAX_WBITS) + + +# Compresses/decompresses given octets using the method introduced in RFC1979. + + +class _RFC1979Deflater(object): + """A compressor class that applies DEFLATE to given byte sequence and + flushes using the algorithm described in the RFC1979 section 2.1. + """ + + def __init__(self, window_bits, no_context_takeover): + self._deflater = None + if window_bits is None: + window_bits = zlib.MAX_WBITS + self._window_bits = window_bits + self._no_context_takeover = no_context_takeover + + def filter(self, bytes, flush=True, bfinal=False): + if self._deflater is None or (self._no_context_takeover and flush): + self._deflater = _Deflater(self._window_bits) + + if bfinal: + result = self._deflater.compress_and_finish(bytes) + # Add a padding block with BFINAL = 0 and BTYPE = 0. + result = result + chr(0) + self._deflater = None + return result + if flush: + # Strip last 4 octets which is LEN and NLEN field of a + # non-compressed block added for Z_SYNC_FLUSH. + return self._deflater.compress_and_flush(bytes)[:-4] + return self._deflater.compress(bytes) + +class _RFC1979Inflater(object): + """A decompressor class for byte sequence compressed and flushed following + the algorithm described in the RFC1979 section 2.1. + """ + + def __init__(self): + self._inflater = _Inflater() + + def filter(self, bytes): + # Restore stripped LEN and NLEN field of a non-compressed block added + # for Z_SYNC_FLUSH. + self._inflater.append(bytes + '\x00\x00\xff\xff') + return self._inflater.decompress(-1) + + +class DeflateSocket(object): + """A wrapper class for socket object to intercept send and recv to perform + deflate compression and decompression transparently. + """ + + # Size of the buffer passed to recv to receive compressed data. + _RECV_SIZE = 4096 + + def __init__(self, socket): + self._socket = socket + + self._logger = get_class_logger(self) + + self._deflater = _Deflater(zlib.MAX_WBITS) + self._inflater = _Inflater() + + def recv(self, size): + """Receives data from the socket specified on the construction up + to the specified size. Once any data is available, returns it even + if it's smaller than the specified size. + """ + + # TODO(tyoshino): Allow call with size=0. It should block until any + # decompressed data is available. + if size <= 0: + raise Exception('Non-positive size passed') + while True: + data = self._inflater.decompress(size) + if len(data) != 0: + return data + + read_data = self._socket.recv(DeflateSocket._RECV_SIZE) + if not read_data: + return '' + self._inflater.append(read_data) + + def sendall(self, bytes): + self.send(bytes) + + def send(self, bytes): + self._socket.sendall(self._deflater.compress_and_flush(bytes)) + return len(bytes) + + +class DeflateConnection(object): + """A wrapper class for request object to intercept write and read to + perform deflate compression and decompression transparently. + """ + + def __init__(self, connection): + self._connection = connection + + self._logger = get_class_logger(self) + + self._deflater = _Deflater(zlib.MAX_WBITS) + self._inflater = _Inflater() + + def get_remote_addr(self): + return self._connection.remote_addr + remote_addr = property(get_remote_addr) + + def put_bytes(self, bytes): + self.write(bytes) + + def read(self, size=-1): + """Reads at most size bytes. Blocks until there's at least one byte + available. + """ + + # TODO(tyoshino): Allow call with size=0. + if not (size == -1 or size > 0): + raise Exception('size must be -1 or positive') + + data = '' + while True: + if size == -1: + data += self._inflater.decompress(-1) + else: + data += self._inflater.decompress(size - len(data)) + + if size >= 0 and len(data) != 0: + break + + # TODO(tyoshino): Make this read efficient by some workaround. + # + # In 3.0.3 and prior of mod_python, read blocks until length bytes + # was read. We don't know the exact size to read while using + # deflate, so read byte-by-byte. + # + # _StandaloneRequest.read that ultimately performs + # socket._fileobject.read also blocks until length bytes was read + read_data = self._connection.read(1) + if not read_data: + break + self._inflater.append(read_data) + return data + + def write(self, bytes): + self._connection.write(self._deflater.compress_and_flush(bytes)) + + +def _is_ewouldblock_errno(error_number): + """Returns True iff error_number indicates that receive operation would + block. To make this portable, we check availability of errno and then + compare them. + """ + + for error_name in ['WSAEWOULDBLOCK', 'EWOULDBLOCK', 'EAGAIN']: + if (error_name in dir(errno) and + error_number == getattr(errno, error_name)): + return True + return False + + +def drain_received_data(raw_socket): + # Set the socket non-blocking. + original_timeout = raw_socket.gettimeout() + raw_socket.settimeout(0.0) + + drained_data = [] + + # Drain until the socket is closed or no data is immediately + # available for read. + while True: + try: + data = raw_socket.recv(1) + if not data: + break + drained_data.append(data) + except socket.error, e: + # e can be either a pair (errno, string) or just a string (or + # something else) telling what went wrong. We suppress only + # the errors that indicates that the socket blocks. Those + # exceptions can be parsed as a pair (errno, string). + try: + error_number, message = e + except: + # Failed to parse socket.error. + raise e + + if _is_ewouldblock_errno(error_number): + break + else: + raise e + + # Rollback timeout value. + raw_socket.settimeout(original_timeout) + + return ''.join(drained_data) + + +# vi:sts=4 sw=4 et diff --git a/pyload/lib/new_collections.py b/pyload/lib/new_collections.py new file mode 100644 index 000000000..12d05b4b9 --- /dev/null +++ b/pyload/lib/new_collections.py @@ -0,0 +1,375 @@ +## {{{ http://code.activestate.com/recipes/576693/ (r9) +# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. +# Passes Python2.7's test suite and incorporates all the latest updates. + +try: + from thread import get_ident as _get_ident +except ImportError: + from dummy_thread import get_ident as _get_ident + +try: + from _abcoll import KeysView, ValuesView, ItemsView +except ImportError: + pass + + +class OrderedDict(dict): + 'Dictionary that remembers insertion order' + # An inherited dict maps keys to values. + # The inherited dict provides __getitem__, __len__, __contains__, and get. + # The remaining methods are order-aware. + # Big-O running times for all methods are the same as for regular dictionaries. + + # The internal self.__map dictionary maps keys to links in a doubly linked list. + # The circular doubly linked list starts and ends with a sentinel element. + # The sentinel element never gets deleted (this simplifies the algorithm). + # Each link is stored as a list of length three: [PREV, NEXT, KEY]. + + def __init__(self, *args, **kwds): + '''Initialize an ordered dictionary. Signature is the same as for + regular dictionaries, but keyword arguments are not recommended + because their insertion order is arbitrary. + + ''' + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + try: + self.__root + except AttributeError: + self.__root = root = [] # sentinel node + root[:] = [root, root, None] + self.__map = {} + self.__update(*args, **kwds) + + def __setitem__(self, key, value, dict_setitem=dict.__setitem__): + 'od.__setitem__(i, y) <==> od[i]=y' + # Setting a new item creates a new link which goes at the end of the linked + # list, and the inherited dictionary is updated with the new key/value pair. + if key not in self: + root = self.__root + last = root[0] + last[1] = root[0] = self.__map[key] = [last, root, key] + dict_setitem(self, key, value) + + def __delitem__(self, key, dict_delitem=dict.__delitem__): + 'od.__delitem__(y) <==> del od[y]' + # Deleting an existing item uses self.__map to find the link which is + # then removed by updating the links in the predecessor and successor nodes. + dict_delitem(self, key) + link_prev, link_next, key = self.__map.pop(key) + link_prev[1] = link_next + link_next[0] = link_prev + + def __iter__(self): + 'od.__iter__() <==> iter(od)' + root = self.__root + curr = root[1] + while curr is not root: + yield curr[2] + curr = curr[1] + + def __reversed__(self): + 'od.__reversed__() <==> reversed(od)' + root = self.__root + curr = root[0] + while curr is not root: + yield curr[2] + curr = curr[0] + + def clear(self): + 'od.clear() -> None. Remove all items from od.' + try: + for node in self.__map.itervalues(): + del node[:] + root = self.__root + root[:] = [root, root, None] + self.__map.clear() + except AttributeError: + pass + dict.clear(self) + + def popitem(self, last=True): + '''od.popitem() -> (k, v), return and remove a (key, value) pair. + Pairs are returned in LIFO order if last is true or FIFO order if false. + + ''' + if not self: + raise KeyError('dictionary is empty') + root = self.__root + if last: + link = root[0] + link_prev = link[0] + link_prev[1] = root + root[0] = link_prev + else: + link = root[1] + link_next = link[1] + root[1] = link_next + link_next[0] = root + key = link[2] + del self.__map[key] + value = dict.pop(self, key) + return key, value + + # -- the following methods do not depend on the internal structure -- + + def keys(self): + 'od.keys() -> list of keys in od' + return list(self) + + def values(self): + 'od.values() -> list of values in od' + return [self[key] for key in self] + + def items(self): + 'od.items() -> list of (key, value) pairs in od' + return [(key, self[key]) for key in self] + + def iterkeys(self): + 'od.iterkeys() -> an iterator over the keys in od' + return iter(self) + + def itervalues(self): + 'od.itervalues -> an iterator over the values in od' + for k in self: + yield self[k] + + def iteritems(self): + 'od.iteritems -> an iterator over the (key, value) items in od' + for k in self: + yield (k, self[k]) + + def update(*args, **kwds): + '''od.update(E, **F) -> None. Update od from dict/iterable E and F. + + If E is a dict instance, does: for k in E: od[k] = E[k] + If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] + Or if E is an iterable of items, does: for k, v in E: od[k] = v + In either case, this is followed by: for k, v in F.items(): od[k] = v + + ''' + if len(args) > 2: + raise TypeError('update() takes at most 2 positional ' + 'arguments (%d given)' % (len(args),)) + elif not args: + raise TypeError('update() takes at least 1 argument (0 given)') + self = args[0] + # Make progressively weaker assumptions about "other" + other = () + if len(args) == 2: + other = args[1] + if isinstance(other, dict): + for key in other: + self[key] = other[key] + elif hasattr(other, 'keys'): + for key in other.keys(): + self[key] = other[key] + else: + for key, value in other: + self[key] = value + for key, value in kwds.items(): + self[key] = value + + __update = update # let subclasses override update without breaking __init__ + + __marker = object() + + def pop(self, key, default=__marker): + '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. + If key is not found, d is returned if given, otherwise KeyError is raised. + + ''' + if key in self: + result = self[key] + del self[key] + return result + if default is self.__marker: + raise KeyError(key) + return default + + def setdefault(self, key, default=None): + 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' + if key in self: + return self[key] + self[key] = default + return default + + def __repr__(self, _repr_running={}): + 'od.__repr__() <==> repr(od)' + call_key = id(self), _get_ident() + if call_key in _repr_running: + return '...' + _repr_running[call_key] = 1 + try: + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, self.items()) + finally: + del _repr_running[call_key] + + def __reduce__(self): + 'Return state information for pickling' + items = [[k, self[k]] for k in self] + inst_dict = vars(self).copy() + for k in vars(OrderedDict()): + inst_dict.pop(k, None) + if inst_dict: + return (self.__class__, (items,), inst_dict) + return self.__class__, (items,) + + def copy(self): + 'od.copy() -> a shallow copy of od' + return self.__class__(self) + + @classmethod + def fromkeys(cls, iterable, value=None): + '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S + and values equal to v (which defaults to None). + + ''' + d = cls() + for key in iterable: + d[key] = value + return d + + def __eq__(self, other): + '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive + while comparison to a regular mapping is order-insensitive. + + ''' + if isinstance(other, OrderedDict): + return len(self)==len(other) and self.items() == other.items() + return dict.__eq__(self, other) + + def __ne__(self, other): + return not self == other + + # -- the following methods are only used in Python 2.7 -- + + def viewkeys(self): + "od.viewkeys() -> a set-like object providing a view on od's keys" + return KeysView(self) + + def viewvalues(self): + "od.viewvalues() -> an object providing a view on od's values" + return ValuesView(self) + + def viewitems(self): + "od.viewitems() -> a set-like object providing a view on od's items" + return ItemsView(self) +## end of http://code.activestate.com/recipes/576693/ }}} + +## {{{ http://code.activestate.com/recipes/500261/ (r15) +from operator import itemgetter as _itemgetter +from keyword import iskeyword as _iskeyword +import sys as _sys + +def namedtuple(typename, field_names, verbose=False, rename=False): + """Returns a new subclass of tuple with named fields. + + >>> Point = namedtuple('Point', 'x y') + >>> Point.__doc__ # docstring for the new class + 'Point(x, y)' + >>> p = Point(11, y=22) # instantiate with positional args or keywords + >>> p[0] + p[1] # indexable like a plain tuple + 33 + >>> x, y = p # unpack like a regular tuple + >>> x, y + (11, 22) + >>> p.x + p.y # fields also accessable by name + 33 + >>> d = p._asdict() # convert to a dictionary + >>> d['x'] + 11 + >>> Point(**d) # convert from a dictionary + Point(x=11, y=22) + >>> p._replace(x=100) # _replace() is like str.replace() but targets named fields + Point(x=100, y=22) + + """ + + # Parse and validate the field names. Validation serves two purposes, + # generating informative error messages and preventing template injection attacks. + if isinstance(field_names, basestring): + field_names = field_names.replace(',', ' ').split() # names separated by whitespace and/or commas + field_names = tuple(map(str, field_names)) + if rename: + names = list(field_names) + seen = set() + for i, name in enumerate(names): + if (not min(c.isalnum() or c=='_' for c in name) or _iskeyword(name) + or not name or name[0].isdigit() or name.startswith('_') + or name in seen): + names[i] = '_%d' % i + seen.add(name) + field_names = tuple(names) + for name in (typename,) + field_names: + if not min(c.isalnum() or c=='_' for c in name): + raise ValueError('Type names and field names can only contain alphanumeric characters and underscores: %r' % name) + if _iskeyword(name): + raise ValueError('Type names and field names cannot be a keyword: %r' % name) + if name[0].isdigit(): + raise ValueError('Type names and field names cannot start with a number: %r' % name) + seen_names = set() + for name in field_names: + if name.startswith('_') and not rename: + raise ValueError('Field names cannot start with an underscore: %r' % name) + if name in seen_names: + raise ValueError('Encountered duplicate field name: %r' % name) + seen_names.add(name) + + # Create and fill-in the class template + numfields = len(field_names) + argtxt = repr(field_names).replace("'", "")[1:-1] # tuple repr without parens or quotes + reprtxt = ', '.join('%s=%%r' % name for name in field_names) + template = '''class %(typename)s(tuple): + '%(typename)s(%(argtxt)s)' \n + __slots__ = () \n + _fields = %(field_names)r \n + def __new__(_cls, %(argtxt)s): + return _tuple.__new__(_cls, (%(argtxt)s)) \n + @classmethod + def _make(cls, iterable, new=tuple.__new__, len=len): + 'Make a new %(typename)s object from a sequence or iterable' + result = new(cls, iterable) + if len(result) != %(numfields)d: + raise TypeError('Expected %(numfields)d arguments, got %%d' %% len(result)) + return result \n + def __repr__(self): + return '%(typename)s(%(reprtxt)s)' %% self \n + def _asdict(self): + 'Return a new dict which maps field names to their values' + return dict(zip(self._fields, self)) \n + def _replace(_self, **kwds): + 'Return a new %(typename)s object replacing specified fields with new values' + result = _self._make(map(kwds.pop, %(field_names)r, _self)) + if kwds: + raise ValueError('Got unexpected field names: %%r' %% kwds.keys()) + return result \n + def __getnewargs__(self): + return tuple(self) \n\n''' % locals() + for i, name in enumerate(field_names): + template += ' %s = _property(_itemgetter(%d))\n' % (name, i) + if verbose: + print template + + # Execute the template string in a temporary namespace + namespace = dict(_itemgetter=_itemgetter, __name__='namedtuple_%s' % typename, + _property=property, _tuple=tuple) + try: + exec template in namespace + except SyntaxError, e: + raise SyntaxError(e.message + ':\n' + template) + result = namespace[typename] + + # For pickling to work, the __module__ variable needs to be set to the frame + # where the named tuple is created. Bypass this step in enviroments where + # sys._getframe is not defined (Jython for example) or sys._getframe is not + # defined for arguments greater than 0 (IronPython). + try: + result.__module__ = _sys._getframe(1).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + pass + + return result +## end of http://code.activestate.com/recipes/500261/ }}} diff --git a/pyload/lib/rename_process.py b/pyload/lib/rename_process.py new file mode 100644 index 000000000..2527cef39 --- /dev/null +++ b/pyload/lib/rename_process.py @@ -0,0 +1,14 @@ +import sys + +def renameProcess(new_name): + """ Renames the process calling the function to the given name. """ + if sys.platform != 'linux2': + return False + try: + from ctypes import CDLL + libc = CDLL('libc.so.6') + libc.prctl(15, new_name, 0, 0, 0) + return True + except Exception, e: + #print "Rename process failed", e + return False diff --git a/pyload/lib/simplejson/__init__.py b/pyload/lib/simplejson/__init__.py new file mode 100644 index 000000000..ef5c0db48 --- /dev/null +++ b/pyload/lib/simplejson/__init__.py @@ -0,0 +1,466 @@ +r"""JSON (JavaScript Object Notation) is a subset of +JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data +interchange format. + +:mod:`simplejson` exposes an API familiar to users of the standard library +:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained +version of the :mod:`json` library contained in Python 2.6, but maintains +compatibility with Python 2.4 and Python 2.5 and (currently) has +significant performance advantages, even without using the optional C +extension for speedups. + +Encoding basic Python object hierarchies:: + + >>> import simplejson as json + >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) + '["foo", {"bar": ["baz", null, 1.0, 2]}]' + >>> print json.dumps("\"foo\bar") + "\"foo\bar" + >>> print json.dumps(u'\u1234') + "\u1234" + >>> print json.dumps('\\') + "\\" + >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True) + {"a": 0, "b": 0, "c": 0} + >>> from StringIO import StringIO + >>> io = StringIO() + >>> json.dump(['streaming API'], io) + >>> io.getvalue() + '["streaming API"]' + +Compact encoding:: + + >>> import simplejson as json + >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':')) + '[1,2,3,{"4":5,"6":7}]' + +Pretty printing:: + + >>> import simplejson as json + >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=' ') + >>> print '\n'.join([l.rstrip() for l in s.splitlines()]) + { + "4": 5, + "6": 7 + } + +Decoding JSON:: + + >>> import simplejson as json + >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] + >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj + True + >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' + True + >>> from StringIO import StringIO + >>> io = StringIO('["streaming API"]') + >>> json.load(io)[0] == 'streaming API' + True + +Specializing JSON object decoding:: + + >>> import simplejson as json + >>> def as_complex(dct): + ... if '__complex__' in dct: + ... return complex(dct['real'], dct['imag']) + ... return dct + ... + >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', + ... object_hook=as_complex) + (1+2j) + >>> from decimal import Decimal + >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1') + True + +Specializing JSON object encoding:: + + >>> import simplejson as json + >>> def encode_complex(obj): + ... if isinstance(obj, complex): + ... return [obj.real, obj.imag] + ... raise TypeError(repr(o) + " is not JSON serializable") + ... + >>> json.dumps(2 + 1j, default=encode_complex) + '[2.0, 1.0]' + >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) + '[2.0, 1.0]' + >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) + '[2.0, 1.0]' + + +Using simplejson.tool from the shell to validate and pretty-print:: + + $ echo '{"json":"obj"}' | python -m simplejson.tool + { + "json": "obj" + } + $ echo '{ 1.2:3.4}' | python -m simplejson.tool + Expecting property name: line 1 column 2 (char 2) +""" +__version__ = '2.2.1' +__all__ = [ + 'dump', 'dumps', 'load', 'loads', + 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', + 'OrderedDict', +] + +__author__ = 'Bob Ippolito ' + +from decimal import Decimal + +from decoder import JSONDecoder, JSONDecodeError +from encoder import JSONEncoder +def _import_OrderedDict(): + import collections + try: + return collections.OrderedDict + except AttributeError: + import ordered_dict + return ordered_dict.OrderedDict +OrderedDict = _import_OrderedDict() + +def _import_c_make_encoder(): + try: + from simplejson._speedups import make_encoder + return make_encoder + except ImportError: + return None + +_default_encoder = JSONEncoder( + skipkeys=False, + ensure_ascii=True, + check_circular=True, + allow_nan=True, + indent=None, + separators=None, + encoding='utf-8', + default=None, + use_decimal=True, + namedtuple_as_object=True, + tuple_as_array=True, +) + +def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, + allow_nan=True, cls=None, indent=None, separators=None, + encoding='utf-8', default=None, use_decimal=True, + namedtuple_as_object=True, tuple_as_array=True, + **kw): + """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a + ``.write()``-supporting file-like object). + + If ``skipkeys`` is true then ``dict`` keys that are not basic types + (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) + will be skipped instead of raising a ``TypeError``. + + If ``ensure_ascii`` is false, then the some chunks written to ``fp`` + may be ``unicode`` instances, subject to normal Python ``str`` to + ``unicode`` coercion rules. Unless ``fp.write()`` explicitly + understands ``unicode`` (as in ``codecs.getwriter()``) this is likely + to cause an error. + + If ``check_circular`` is false, then the circular reference check + for container types will be skipped and a circular reference will + result in an ``OverflowError`` (or worse). + + If ``allow_nan`` is false, then it will be a ``ValueError`` to + serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) + in strict compliance of the JSON specification, instead of using the + JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). + + If *indent* is a string, then JSON array elements and object members + will be pretty-printed with a newline followed by that string repeated + for each level of nesting. ``None`` (the default) selects the most compact + representation without any newlines. For backwards compatibility with + versions of simplejson earlier than 2.1.0, an integer is also accepted + and is converted to a string with that many spaces. + + If ``separators`` is an ``(item_separator, dict_separator)`` tuple + then it will be used instead of the default ``(', ', ': ')`` separators. + ``(',', ':')`` is the most compact JSON representation. + + ``encoding`` is the character encoding for str instances, default is UTF-8. + + ``default(obj)`` is a function that should return a serializable version + of obj or raise TypeError. The default simply raises TypeError. + + If *use_decimal* is true (default: ``True``) then decimal.Decimal + will be natively serialized to JSON with full precision. + + If *namedtuple_as_object* is true (default: ``True``), + :class:`tuple` subclasses with ``_asdict()`` methods will be encoded + as JSON objects. + + If *tuple_as_array* is true (default: ``True``), + :class:`tuple` (and subclasses) will be encoded as JSON arrays. + + To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the + ``.default()`` method to serialize additional types), specify it with + the ``cls`` kwarg. + + """ + # cached encoder + if (not skipkeys and ensure_ascii and + check_circular and allow_nan and + cls is None and indent is None and separators is None and + encoding == 'utf-8' and default is None and use_decimal + and namedtuple_as_object and tuple_as_array and not kw): + iterable = _default_encoder.iterencode(obj) + else: + if cls is None: + cls = JSONEncoder + iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, + check_circular=check_circular, allow_nan=allow_nan, indent=indent, + separators=separators, encoding=encoding, + default=default, use_decimal=use_decimal, + namedtuple_as_object=namedtuple_as_object, + tuple_as_array=tuple_as_array, + **kw).iterencode(obj) + # could accelerate with writelines in some versions of Python, at + # a debuggability cost + for chunk in iterable: + fp.write(chunk) + + +def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, + allow_nan=True, cls=None, indent=None, separators=None, + encoding='utf-8', default=None, use_decimal=True, + namedtuple_as_object=True, + tuple_as_array=True, + **kw): + """Serialize ``obj`` to a JSON formatted ``str``. + + If ``skipkeys`` is false then ``dict`` keys that are not basic types + (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) + will be skipped instead of raising a ``TypeError``. + + If ``ensure_ascii`` is false, then the return value will be a + ``unicode`` instance subject to normal Python ``str`` to ``unicode`` + coercion rules instead of being escaped to an ASCII ``str``. + + If ``check_circular`` is false, then the circular reference check + for container types will be skipped and a circular reference will + result in an ``OverflowError`` (or worse). + + If ``allow_nan`` is false, then it will be a ``ValueError`` to + serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in + strict compliance of the JSON specification, instead of using the + JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). + + If ``indent`` is a string, then JSON array elements and object members + will be pretty-printed with a newline followed by that string repeated + for each level of nesting. ``None`` (the default) selects the most compact + representation without any newlines. For backwards compatibility with + versions of simplejson earlier than 2.1.0, an integer is also accepted + and is converted to a string with that many spaces. + + If ``separators`` is an ``(item_separator, dict_separator)`` tuple + then it will be used instead of the default ``(', ', ': ')`` separators. + ``(',', ':')`` is the most compact JSON representation. + + ``encoding`` is the character encoding for str instances, default is UTF-8. + + ``default(obj)`` is a function that should return a serializable version + of obj or raise TypeError. The default simply raises TypeError. + + If *use_decimal* is true (default: ``True``) then decimal.Decimal + will be natively serialized to JSON with full precision. + + If *namedtuple_as_object* is true (default: ``True``), + :class:`tuple` subclasses with ``_asdict()`` methods will be encoded + as JSON objects. + + If *tuple_as_array* is true (default: ``True``), + :class:`tuple` (and subclasses) will be encoded as JSON arrays. + + To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the + ``.default()`` method to serialize additional types), specify it with + the ``cls`` kwarg. + + """ + # cached encoder + if (not skipkeys and ensure_ascii and + check_circular and allow_nan and + cls is None and indent is None and separators is None and + encoding == 'utf-8' and default is None and use_decimal + and namedtuple_as_object and tuple_as_array and not kw): + return _default_encoder.encode(obj) + if cls is None: + cls = JSONEncoder + return cls( + skipkeys=skipkeys, ensure_ascii=ensure_ascii, + check_circular=check_circular, allow_nan=allow_nan, indent=indent, + separators=separators, encoding=encoding, default=default, + use_decimal=use_decimal, + namedtuple_as_object=namedtuple_as_object, + tuple_as_array=tuple_as_array, + **kw).encode(obj) + + +_default_decoder = JSONDecoder(encoding=None, object_hook=None, + object_pairs_hook=None) + + +def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, object_pairs_hook=None, + use_decimal=False, namedtuple_as_object=True, tuple_as_array=True, + **kw): + """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing + a JSON document) to a Python object. + + *encoding* determines the encoding used to interpret any + :class:`str` objects decoded by this instance (``'utf-8'`` by + default). It has no effect when decoding :class:`unicode` objects. + + Note that currently only encodings that are a superset of ASCII work, + strings of other encodings should be passed in as :class:`unicode`. + + *object_hook*, if specified, will be called with the result of every + JSON object decoded and its return value will be used in place of the + given :class:`dict`. This can be used to provide custom + deserializations (e.g. to support JSON-RPC class hinting). + + *object_pairs_hook* is an optional function that will be called with + the result of any object literal decode with an ordered list of pairs. + The return value of *object_pairs_hook* will be used instead of the + :class:`dict`. This feature can be used to implement custom decoders + that rely on the order that the key and value pairs are decoded (for + example, :func:`collections.OrderedDict` will remember the order of + insertion). If *object_hook* is also defined, the *object_pairs_hook* + takes priority. + + *parse_float*, if specified, will be called with the string of every + JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser + for JSON floats (e.g. :class:`decimal.Decimal`). + + *parse_int*, if specified, will be called with the string of every + JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser + for JSON integers (e.g. :class:`float`). + + *parse_constant*, if specified, will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This + can be used to raise an exception if invalid JSON numbers are + encountered. + + If *use_decimal* is true (default: ``False``) then it implies + parse_float=decimal.Decimal for parity with ``dump``. + + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` + kwarg. + + """ + return loads(fp.read(), + encoding=encoding, cls=cls, object_hook=object_hook, + parse_float=parse_float, parse_int=parse_int, + parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, + use_decimal=use_decimal, **kw) + + +def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, object_pairs_hook=None, + use_decimal=False, **kw): + """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON + document) to a Python object. + + *encoding* determines the encoding used to interpret any + :class:`str` objects decoded by this instance (``'utf-8'`` by + default). It has no effect when decoding :class:`unicode` objects. + + Note that currently only encodings that are a superset of ASCII work, + strings of other encodings should be passed in as :class:`unicode`. + + *object_hook*, if specified, will be called with the result of every + JSON object decoded and its return value will be used in place of the + given :class:`dict`. This can be used to provide custom + deserializations (e.g. to support JSON-RPC class hinting). + + *object_pairs_hook* is an optional function that will be called with + the result of any object literal decode with an ordered list of pairs. + The return value of *object_pairs_hook* will be used instead of the + :class:`dict`. This feature can be used to implement custom decoders + that rely on the order that the key and value pairs are decoded (for + example, :func:`collections.OrderedDict` will remember the order of + insertion). If *object_hook* is also defined, the *object_pairs_hook* + takes priority. + + *parse_float*, if specified, will be called with the string of every + JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser + for JSON floats (e.g. :class:`decimal.Decimal`). + + *parse_int*, if specified, will be called with the string of every + JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser + for JSON integers (e.g. :class:`float`). + + *parse_constant*, if specified, will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This + can be used to raise an exception if invalid JSON numbers are + encountered. + + If *use_decimal* is true (default: ``False``) then it implies + parse_float=decimal.Decimal for parity with ``dump``. + + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` + kwarg. + + """ + if (cls is None and encoding is None and object_hook is None and + parse_int is None and parse_float is None and + parse_constant is None and object_pairs_hook is None + and not use_decimal and not kw): + return _default_decoder.decode(s) + if cls is None: + cls = JSONDecoder + if object_hook is not None: + kw['object_hook'] = object_hook + if object_pairs_hook is not None: + kw['object_pairs_hook'] = object_pairs_hook + if parse_float is not None: + kw['parse_float'] = parse_float + if parse_int is not None: + kw['parse_int'] = parse_int + if parse_constant is not None: + kw['parse_constant'] = parse_constant + if use_decimal: + if parse_float is not None: + raise TypeError("use_decimal=True implies parse_float=Decimal") + kw['parse_float'] = Decimal + return cls(encoding=encoding, **kw).decode(s) + + +def _toggle_speedups(enabled): + import simplejson.decoder as dec + import simplejson.encoder as enc + import simplejson.scanner as scan + c_make_encoder = _import_c_make_encoder() + if enabled: + dec.scanstring = dec.c_scanstring or dec.py_scanstring + enc.c_make_encoder = c_make_encoder + enc.encode_basestring_ascii = (enc.c_encode_basestring_ascii or + enc.py_encode_basestring_ascii) + scan.make_scanner = scan.c_make_scanner or scan.py_make_scanner + else: + dec.scanstring = dec.py_scanstring + enc.c_make_encoder = None + enc.encode_basestring_ascii = enc.py_encode_basestring_ascii + scan.make_scanner = scan.py_make_scanner + dec.make_scanner = scan.make_scanner + global _default_decoder + _default_decoder = JSONDecoder( + encoding=None, + object_hook=None, + object_pairs_hook=None, + ) + global _default_encoder + _default_encoder = JSONEncoder( + skipkeys=False, + ensure_ascii=True, + check_circular=True, + allow_nan=True, + indent=None, + separators=None, + encoding='utf-8', + default=None, + ) diff --git a/pyload/lib/simplejson/decoder.py b/pyload/lib/simplejson/decoder.py new file mode 100644 index 000000000..e5496d6e7 --- /dev/null +++ b/pyload/lib/simplejson/decoder.py @@ -0,0 +1,421 @@ +"""Implementation of JSONDecoder +""" +import re +import sys +import struct + +from simplejson.scanner import make_scanner +def _import_c_scanstring(): + try: + from simplejson._speedups import scanstring + return scanstring + except ImportError: + return None +c_scanstring = _import_c_scanstring() + +__all__ = ['JSONDecoder'] + +FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL + +def _floatconstants(): + _BYTES = '7FF80000000000007FF0000000000000'.decode('hex') + # The struct module in Python 2.4 would get frexp() out of range here + # when an endian is specified in the format string. Fixed in Python 2.5+ + if sys.byteorder != 'big': + _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1] + nan, inf = struct.unpack('dd', _BYTES) + return nan, inf, -inf + +NaN, PosInf, NegInf = _floatconstants() + + +class JSONDecodeError(ValueError): + """Subclass of ValueError with the following additional properties: + + msg: The unformatted error message + doc: The JSON document being parsed + pos: The start index of doc where parsing failed + end: The end index of doc where parsing failed (may be None) + lineno: The line corresponding to pos + colno: The column corresponding to pos + endlineno: The line corresponding to end (may be None) + endcolno: The column corresponding to end (may be None) + + """ + def __init__(self, msg, doc, pos, end=None): + ValueError.__init__(self, errmsg(msg, doc, pos, end=end)) + self.msg = msg + self.doc = doc + self.pos = pos + self.end = end + self.lineno, self.colno = linecol(doc, pos) + if end is not None: + self.endlineno, self.endcolno = linecol(doc, end) + else: + self.endlineno, self.endcolno = None, None + + +def linecol(doc, pos): + lineno = doc.count('\n', 0, pos) + 1 + if lineno == 1: + colno = pos + else: + colno = pos - doc.rindex('\n', 0, pos) + return lineno, colno + + +def errmsg(msg, doc, pos, end=None): + # Note that this function is called from _speedups + lineno, colno = linecol(doc, pos) + if end is None: + #fmt = '{0}: line {1} column {2} (char {3})' + #return fmt.format(msg, lineno, colno, pos) + fmt = '%s: line %d column %d (char %d)' + return fmt % (msg, lineno, colno, pos) + endlineno, endcolno = linecol(doc, end) + #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})' + #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end) + fmt = '%s: line %d column %d - line %d column %d (char %d - %d)' + return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end) + + +_CONSTANTS = { + '-Infinity': NegInf, + 'Infinity': PosInf, + 'NaN': NaN, +} + +STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) +BACKSLASH = { + '"': u'"', '\\': u'\\', '/': u'/', + 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', +} + +DEFAULT_ENCODING = "utf-8" + +def py_scanstring(s, end, encoding=None, strict=True, + _b=BACKSLASH, _m=STRINGCHUNK.match): + """Scan the string s for a JSON string. End is the index of the + character in s after the quote that started the JSON string. + Unescapes all valid JSON string escape sequences and raises ValueError + on attempt to decode an invalid string. If strict is False then literal + control characters are allowed in the string. + + Returns a tuple of the decoded string and the index of the character in s + after the end quote.""" + if encoding is None: + encoding = DEFAULT_ENCODING + chunks = [] + _append = chunks.append + begin = end - 1 + while 1: + chunk = _m(s, end) + if chunk is None: + raise JSONDecodeError( + "Unterminated string starting at", s, begin) + end = chunk.end() + content, terminator = chunk.groups() + # Content is contains zero or more unescaped string characters + if content: + if not isinstance(content, unicode): + content = unicode(content, encoding) + _append(content) + # Terminator is the end of string, a literal control character, + # or a backslash denoting that an escape sequence follows + if terminator == '"': + break + elif terminator != '\\': + if strict: + msg = "Invalid control character %r at" % (terminator,) + #msg = "Invalid control character {0!r} at".format(terminator) + raise JSONDecodeError(msg, s, end) + else: + _append(terminator) + continue + try: + esc = s[end] + except IndexError: + raise JSONDecodeError( + "Unterminated string starting at", s, begin) + # If not a unicode escape sequence, must be in the lookup table + if esc != 'u': + try: + char = _b[esc] + except KeyError: + msg = "Invalid \\escape: " + repr(esc) + raise JSONDecodeError(msg, s, end) + end += 1 + else: + # Unicode escape sequence + esc = s[end + 1:end + 5] + next_end = end + 5 + if len(esc) != 4: + msg = "Invalid \\uXXXX escape" + raise JSONDecodeError(msg, s, end) + uni = int(esc, 16) + # Check for surrogate pair on UCS-4 systems + if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535: + msg = "Invalid \\uXXXX\\uXXXX surrogate pair" + if not s[end + 5:end + 7] == '\\u': + raise JSONDecodeError(msg, s, end) + esc2 = s[end + 7:end + 11] + if len(esc2) != 4: + raise JSONDecodeError(msg, s, end) + uni2 = int(esc2, 16) + uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00)) + next_end += 6 + char = unichr(uni) + end = next_end + # Append the unescaped character + _append(char) + return u''.join(chunks), end + + +# Use speedup if available +scanstring = c_scanstring or py_scanstring + +WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) +WHITESPACE_STR = ' \t\n\r' + +def JSONObject((s, end), encoding, strict, scan_once, object_hook, + object_pairs_hook, memo=None, + _w=WHITESPACE.match, _ws=WHITESPACE_STR): + # Backwards compatibility + if memo is None: + memo = {} + memo_get = memo.setdefault + pairs = [] + # Use a slice to prevent IndexError from being raised, the following + # check will raise a more specific ValueError if the string is empty + nextchar = s[end:end + 1] + # Normally we expect nextchar == '"' + if nextchar != '"': + if nextchar in _ws: + end = _w(s, end).end() + nextchar = s[end:end + 1] + # Trivial empty object + if nextchar == '}': + if object_pairs_hook is not None: + result = object_pairs_hook(pairs) + return result, end + 1 + pairs = {} + if object_hook is not None: + pairs = object_hook(pairs) + return pairs, end + 1 + elif nextchar != '"': + raise JSONDecodeError("Expecting property name", s, end) + end += 1 + while True: + key, end = scanstring(s, end, encoding, strict) + key = memo_get(key, key) + + # To skip some function call overhead we optimize the fast paths where + # the JSON key separator is ": " or just ":". + if s[end:end + 1] != ':': + end = _w(s, end).end() + if s[end:end + 1] != ':': + raise JSONDecodeError("Expecting : delimiter", s, end) + + end += 1 + + try: + if s[end] in _ws: + end += 1 + if s[end] in _ws: + end = _w(s, end + 1).end() + except IndexError: + pass + + try: + value, end = scan_once(s, end) + except StopIteration: + raise JSONDecodeError("Expecting object", s, end) + pairs.append((key, value)) + + try: + nextchar = s[end] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end] + except IndexError: + nextchar = '' + end += 1 + + if nextchar == '}': + break + elif nextchar != ',': + raise JSONDecodeError("Expecting , delimiter", s, end - 1) + + try: + nextchar = s[end] + if nextchar in _ws: + end += 1 + nextchar = s[end] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end] + except IndexError: + nextchar = '' + + end += 1 + if nextchar != '"': + raise JSONDecodeError("Expecting property name", s, end - 1) + + if object_pairs_hook is not None: + result = object_pairs_hook(pairs) + return result, end + pairs = dict(pairs) + if object_hook is not None: + pairs = object_hook(pairs) + return pairs, end + +def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): + values = [] + nextchar = s[end:end + 1] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end:end + 1] + # Look-ahead for trivial empty array + if nextchar == ']': + return values, end + 1 + _append = values.append + while True: + try: + value, end = scan_once(s, end) + except StopIteration: + raise JSONDecodeError("Expecting object", s, end) + _append(value) + nextchar = s[end:end + 1] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end:end + 1] + end += 1 + if nextchar == ']': + break + elif nextchar != ',': + raise JSONDecodeError("Expecting , delimiter", s, end) + + try: + if s[end] in _ws: + end += 1 + if s[end] in _ws: + end = _w(s, end + 1).end() + except IndexError: + pass + + return values, end + +class JSONDecoder(object): + """Simple JSON decoder + + Performs the following translations in decoding by default: + + +---------------+-------------------+ + | JSON | Python | + +===============+===================+ + | object | dict | + +---------------+-------------------+ + | array | list | + +---------------+-------------------+ + | string | unicode | + +---------------+-------------------+ + | number (int) | int, long | + +---------------+-------------------+ + | number (real) | float | + +---------------+-------------------+ + | true | True | + +---------------+-------------------+ + | false | False | + +---------------+-------------------+ + | null | None | + +---------------+-------------------+ + + It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as + their corresponding ``float`` values, which is outside the JSON spec. + + """ + + def __init__(self, encoding=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, strict=True, + object_pairs_hook=None): + """ + *encoding* determines the encoding used to interpret any + :class:`str` objects decoded by this instance (``'utf-8'`` by + default). It has no effect when decoding :class:`unicode` objects. + + Note that currently only encodings that are a superset of ASCII work, + strings of other encodings should be passed in as :class:`unicode`. + + *object_hook*, if specified, will be called with the result of every + JSON object decoded and its return value will be used in place of the + given :class:`dict`. This can be used to provide custom + deserializations (e.g. to support JSON-RPC class hinting). + + *object_pairs_hook* is an optional function that will be called with + the result of any object literal decode with an ordered list of pairs. + The return value of *object_pairs_hook* will be used instead of the + :class:`dict`. This feature can be used to implement custom decoders + that rely on the order that the key and value pairs are decoded (for + example, :func:`collections.OrderedDict` will remember the order of + insertion). If *object_hook* is also defined, the *object_pairs_hook* + takes priority. + + *parse_float*, if specified, will be called with the string of every + JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser + for JSON floats (e.g. :class:`decimal.Decimal`). + + *parse_int*, if specified, will be called with the string of every + JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser + for JSON integers (e.g. :class:`float`). + + *parse_constant*, if specified, will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This + can be used to raise an exception if invalid JSON numbers are + encountered. + + *strict* controls the parser's behavior when it encounters an + invalid control character in a string. The default setting of + ``True`` means that unescaped control characters are parse errors, if + ``False`` then control characters will be allowed in strings. + + """ + self.encoding = encoding + self.object_hook = object_hook + self.object_pairs_hook = object_pairs_hook + self.parse_float = parse_float or float + self.parse_int = parse_int or int + self.parse_constant = parse_constant or _CONSTANTS.__getitem__ + self.strict = strict + self.parse_object = JSONObject + self.parse_array = JSONArray + self.parse_string = scanstring + self.memo = {} + self.scan_once = make_scanner(self) + + def decode(self, s, _w=WHITESPACE.match): + """Return the Python representation of ``s`` (a ``str`` or ``unicode`` + instance containing a JSON document) + + """ + obj, end = self.raw_decode(s, idx=_w(s, 0).end()) + end = _w(s, end).end() + if end != len(s): + raise JSONDecodeError("Extra data", s, end, len(s)) + return obj + + def raw_decode(self, s, idx=0): + """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` + beginning with a JSON document) and return a 2-tuple of the Python + representation and the index in ``s`` where the document ended. + + This can be used to decode a JSON document from a string that may + have extraneous data at the end. + + """ + try: + obj, end = self.scan_once(s, idx) + except StopIteration: + raise JSONDecodeError("No JSON object could be decoded", s, idx) + return obj, end diff --git a/pyload/lib/simplejson/encoder.py b/pyload/lib/simplejson/encoder.py new file mode 100644 index 000000000..5ec7440f1 --- /dev/null +++ b/pyload/lib/simplejson/encoder.py @@ -0,0 +1,534 @@ +"""Implementation of JSONEncoder +""" +import re +from decimal import Decimal + +def _import_speedups(): + try: + from simplejson import _speedups + return _speedups.encode_basestring_ascii, _speedups.make_encoder + except ImportError: + return None, None +c_encode_basestring_ascii, c_make_encoder = _import_speedups() + +from simplejson.decoder import PosInf + +ESCAPE = re.compile(ur'[\x00-\x1f\\"\b\f\n\r\t\u2028\u2029]') +ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') +HAS_UTF8 = re.compile(r'[\x80-\xff]') +ESCAPE_DCT = { + '\\': '\\\\', + '"': '\\"', + '\b': '\\b', + '\f': '\\f', + '\n': '\\n', + '\r': '\\r', + '\t': '\\t', + u'\u2028': '\\u2028', + u'\u2029': '\\u2029', +} +for i in range(0x20): + #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) + ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) + +FLOAT_REPR = repr + +def encode_basestring(s): + """Return a JSON representation of a Python string + + """ + if isinstance(s, str) and HAS_UTF8.search(s) is not None: + s = s.decode('utf-8') + def replace(match): + return ESCAPE_DCT[match.group(0)] + return u'"' + ESCAPE.sub(replace, s) + u'"' + + +def py_encode_basestring_ascii(s): + """Return an ASCII-only JSON representation of a Python string + + """ + if isinstance(s, str) and HAS_UTF8.search(s) is not None: + s = s.decode('utf-8') + def replace(match): + s = match.group(0) + try: + return ESCAPE_DCT[s] + except KeyError: + n = ord(s) + if n < 0x10000: + #return '\\u{0:04x}'.format(n) + return '\\u%04x' % (n,) + else: + # surrogate pair + n -= 0x10000 + s1 = 0xd800 | ((n >> 10) & 0x3ff) + s2 = 0xdc00 | (n & 0x3ff) + #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) + return '\\u%04x\\u%04x' % (s1, s2) + return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' + + +encode_basestring_ascii = ( + c_encode_basestring_ascii or py_encode_basestring_ascii) + +class JSONEncoder(object): + """Extensible JSON encoder for Python data structures. + + Supports the following objects and types by default: + + +-------------------+---------------+ + | Python | JSON | + +===================+===============+ + | dict, namedtuple | object | + +-------------------+---------------+ + | list, tuple | array | + +-------------------+---------------+ + | str, unicode | string | + +-------------------+---------------+ + | int, long, float | number | + +-------------------+---------------+ + | True | true | + +-------------------+---------------+ + | False | false | + +-------------------+---------------+ + | None | null | + +-------------------+---------------+ + + To extend this to recognize other objects, subclass and implement a + ``.default()`` method with another method that returns a serializable + object for ``o`` if possible, otherwise it should call the superclass + implementation (to raise ``TypeError``). + + """ + item_separator = ', ' + key_separator = ': ' + def __init__(self, skipkeys=False, ensure_ascii=True, + check_circular=True, allow_nan=True, sort_keys=False, + indent=None, separators=None, encoding='utf-8', default=None, + use_decimal=True, namedtuple_as_object=True, + tuple_as_array=True): + """Constructor for JSONEncoder, with sensible defaults. + + If skipkeys is false, then it is a TypeError to attempt + encoding of keys that are not str, int, long, float or None. If + skipkeys is True, such items are simply skipped. + + If ensure_ascii is true, the output is guaranteed to be str + objects with all incoming unicode characters escaped. If + ensure_ascii is false, the output will be unicode object. + + If check_circular is true, then lists, dicts, and custom encoded + objects will be checked for circular references during encoding to + prevent an infinite recursion (which would cause an OverflowError). + Otherwise, no such check takes place. + + If allow_nan is true, then NaN, Infinity, and -Infinity will be + encoded as such. This behavior is not JSON specification compliant, + but is consistent with most JavaScript based encoders and decoders. + Otherwise, it will be a ValueError to encode such floats. + + If sort_keys is true, then the output of dictionaries will be + sorted by key; this is useful for regression tests to ensure + that JSON serializations can be compared on a day-to-day basis. + + If indent is a string, then JSON array elements and object members + will be pretty-printed with a newline followed by that string repeated + for each level of nesting. ``None`` (the default) selects the most compact + representation without any newlines. For backwards compatibility with + versions of simplejson earlier than 2.1.0, an integer is also accepted + and is converted to a string with that many spaces. + + If specified, separators should be a (item_separator, key_separator) + tuple. The default is (', ', ': '). To get the most compact JSON + representation you should specify (',', ':') to eliminate whitespace. + + If specified, default is a function that gets called for objects + that can't otherwise be serialized. It should return a JSON encodable + version of the object or raise a ``TypeError``. + + If encoding is not None, then all input strings will be + transformed into unicode using that encoding prior to JSON-encoding. + The default is UTF-8. + + If use_decimal is true (not the default), ``decimal.Decimal`` will + be supported directly by the encoder. For the inverse, decode JSON + with ``parse_float=decimal.Decimal``. + + If namedtuple_as_object is true (the default), tuple subclasses with + ``_asdict()`` methods will be encoded as JSON objects. + + If tuple_as_array is true (the default), tuple (and subclasses) will + be encoded as JSON arrays. + """ + + self.skipkeys = skipkeys + self.ensure_ascii = ensure_ascii + self.check_circular = check_circular + self.allow_nan = allow_nan + self.sort_keys = sort_keys + self.use_decimal = use_decimal + self.namedtuple_as_object = namedtuple_as_object + self.tuple_as_array = tuple_as_array + if isinstance(indent, (int, long)): + indent = ' ' * indent + self.indent = indent + if separators is not None: + self.item_separator, self.key_separator = separators + elif indent is not None: + self.item_separator = ',' + if default is not None: + self.default = default + self.encoding = encoding + + def default(self, o): + """Implement this method in a subclass such that it returns + a serializable object for ``o``, or calls the base implementation + (to raise a ``TypeError``). + + For example, to support arbitrary iterators, you could + implement default like this:: + + def default(self, o): + try: + iterable = iter(o) + except TypeError: + pass + else: + return list(iterable) + return JSONEncoder.default(self, o) + + """ + raise TypeError(repr(o) + " is not JSON serializable") + + def encode(self, o): + """Return a JSON string representation of a Python data structure. + + >>> from simplejson import JSONEncoder + >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) + '{"foo": ["bar", "baz"]}' + + """ + # This is for extremely simple cases and benchmarks. + if isinstance(o, basestring): + if isinstance(o, str): + _encoding = self.encoding + if (_encoding is not None + and not (_encoding == 'utf-8')): + o = o.decode(_encoding) + if self.ensure_ascii: + return encode_basestring_ascii(o) + else: + return encode_basestring(o) + # This doesn't pass the iterator directly to ''.join() because the + # exceptions aren't as detailed. The list call should be roughly + # equivalent to the PySequence_Fast that ''.join() would do. + chunks = self.iterencode(o, _one_shot=True) + if not isinstance(chunks, (list, tuple)): + chunks = list(chunks) + if self.ensure_ascii: + return ''.join(chunks) + else: + return u''.join(chunks) + + def iterencode(self, o, _one_shot=False): + """Encode the given object and yield each string + representation as available. + + For example:: + + for chunk in JSONEncoder().iterencode(bigobject): + mysocket.write(chunk) + + """ + if self.check_circular: + markers = {} + else: + markers = None + if self.ensure_ascii: + _encoder = encode_basestring_ascii + else: + _encoder = encode_basestring + if self.encoding != 'utf-8': + def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): + if isinstance(o, str): + o = o.decode(_encoding) + return _orig_encoder(o) + + def floatstr(o, allow_nan=self.allow_nan, + _repr=FLOAT_REPR, _inf=PosInf, _neginf=-PosInf): + # Check for specials. Note that this type of test is processor + # and/or platform-specific, so do tests which don't depend on + # the internals. + + if o != o: + text = 'NaN' + elif o == _inf: + text = 'Infinity' + elif o == _neginf: + text = '-Infinity' + else: + return _repr(o) + + if not allow_nan: + raise ValueError( + "Out of range float values are not JSON compliant: " + + repr(o)) + + return text + + + key_memo = {} + if (_one_shot and c_make_encoder is not None + and self.indent is None): + _iterencode = c_make_encoder( + markers, self.default, _encoder, self.indent, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, self.allow_nan, key_memo, self.use_decimal, + self.namedtuple_as_object, self.tuple_as_array) + else: + _iterencode = _make_iterencode( + markers, self.default, _encoder, self.indent, floatstr, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, _one_shot, self.use_decimal, + self.namedtuple_as_object, self.tuple_as_array) + try: + return _iterencode(o, 0) + finally: + key_memo.clear() + + +class JSONEncoderForHTML(JSONEncoder): + """An encoder that produces JSON safe to embed in HTML. + + To embed JSON content in, say, a script tag on a web page, the + characters &, < and > should be escaped. They cannot be escaped + with the usual entities (e.g. &) because they are not expanded + within |;var)" + SHOW_PATTERN = r"title='(.*?)'" + SERIE_PATTERN = r".*Serie.*" + + def decrypt(self, pyfile): + src = self.req.load(str(pyfile.url)) + + pattern = re.compile(self.VALUES_PATTERN, re.DOTALL) + data = json_loads(re.search(pattern, src).group(1)) + + # Get package info + links = [] + Showname = re.search(self.SHOW_PATTERN, src) + if Showname: + Showname = Showname.group(1).decode("utf-8") + else: + Showname = self.pyfile.package().name + + if re.search(self.SERIE_PATTERN, src): + for Season in data: + self.logDebug("Season " + Season) + for Episode in data[Season]: + self.logDebug("Episode " + Episode) + links.extend(self.getpreferred(data[Season][Episode])) + if self.getConfig("changeNameS") == "Episode": + self.packages.append((data[Season][Episode]['info']['name'].split("»")[0], links, data[Season][Episode]['info']['name'].split("»")[0])) + links = [] + + if self.getConfig("changeNameS") == "Season": + self.packages.append((Showname + " Season " + Season, links, Showname + " Season " + Season)) + links = [] + + if self.getConfig("changeNameS") == "Show": + if links == []: + self.fail('Could not extract any links (Out of Date?)') + else: + self.packages.append((Showname, links, Showname)) + + elif self.getConfig("changeNameS") == "Packagename": + if links == []: + self.fail('Could not extract any links (Out of Date?)') + else: + self.core.files.addLinks(links, self.pyfile.package().id) + else: + for Movie in data: + links.extend(self.getpreferred(data[Movie])) + if self.getConfig("changeName") == "Movie": + if links == []: + self.fail('Could not extract any links (Out of Date?)') + else: + self.packages.append((Showname, links, Showname)) + + elif self.getConfig("changeName") == "Packagename": + if links == []: + self.fail('Could not extract any links (Out of Date?)') + else: + self.core.files.addLinks(links, self.pyfile.package().id) + + #selects the preferred hoster, after that selects any hoster (ignoring the one to ignore) + #selects only one Hoster + def getpreferred(self, hosterslist): + hosterlist = {} + if 'u' in hosterslist: + hosterlist.update(hosterslist['u']) + if ('d' in hosterslist): + hosterlist.update(hosterslist['d']) + if self.getConfig("useStreams") and 's' in hosterslist: + hosterlist.update(hosterslist['s']) + + result = [] + preferredList = self.getConfig("hosterList").strip().lower().replace('|',',').replace('.','').replace(';',',').split(',') + if self.getConfig("randomPreferred") == True: + random.shuffle(preferredList) + for preferred in preferredList: + for Hoster in hosterlist: + if preferred == Hoster.split('<')[0].strip().lower().replace('.',''): + for Part in hosterlist[Hoster]: + self.logDebug("selected " + Part[3]) + result.append(str(Part[3])) + return result + + ignorelist = self.getConfig("ignoreList").strip().lower().replace('|',',').replace('.','').replace(';',',').split(',') + if self.getConfig('hosterListMode') == "all": + for Hoster in hosterlist: + if Hoster.split('<')[0].strip().lower().replace('.','') not in ignorelist: + for Part in hosterlist[Hoster]: + self.logDebug("selected " + Part[3]) + result.append(str(Part[3])) + return result + return result + + + diff --git a/pyload/plugins/crypter/CCF.py b/pyload/plugins/crypter/CCF.py new file mode 100644 index 000000000..ab7ff1099 --- /dev/null +++ b/pyload/plugins/crypter/CCF.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +from urllib2 import build_opener + +from module.plugins.Crypter import Crypter +from module.lib.MultipartPostHandler import MultipartPostHandler + +from os import makedirs +from os.path import exists, join + +class CCF(Crypter): + __name__ = "CCF" + __version__ = "0.2" + __pattern__ = r"(?!http://).*\.ccf$" + __description__ = """CCF Container Convert Plugin""" + __author_name__ = ("Willnix") + __author_mail__ = ("Willnix@pyload.org") + + def decrypt(self, pyfile): + + infile = pyfile.url.replace("\n", "") + + opener = build_opener(MultipartPostHandler) + params = {"src": "ccf", + "filename": "test.ccf", + "upload": open(infile, "rb")} + tempdlc_content = opener.open('http://service.jdownloader.net/dlcrypt/getDLC.php', params).read() + + download_folder = self.config['general']['download_folder'] + location = download_folder #join(download_folder, self.pyfile.package().folder.decode(sys.getfilesystemencoding())) + if not exists(location): + makedirs(location) + + tempdlc_name = join(location, "tmp_%s.dlc" % pyfile.name) + tempdlc = open(tempdlc_name, "w") + tempdlc.write(re.search(r'(.*)', tempdlc_content, re.DOTALL).group(1)) + tempdlc.close() + + self.packages.append((tempdlc_name, [tempdlc_name], tempdlc_name)) + diff --git a/pyload/plugins/crypter/CrockoComFolder.py b/pyload/plugins/crypter/CrockoComFolder.py new file mode 100644 index 000000000..d727ec7ab --- /dev/null +++ b/pyload/plugins/crypter/CrockoComFolder.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- + +from module.plugins.internal.SimpleCrypter import SimpleCrypter + +class CrockoComFolder(SimpleCrypter): + __name__ = "CrockoComFolder" + __type__ = "crypter" + __pattern__ = r"http://(www\.)?crocko.com/f/.*" + __version__ = "0.01" + __description__ = """Crocko.com Folder Plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + LINK_PATTERN = r'download' \ No newline at end of file diff --git a/pyload/plugins/crypter/CryptItCom.py b/pyload/plugins/crypter/CryptItCom.py new file mode 100644 index 000000000..4935758c7 --- /dev/null +++ b/pyload/plugins/crypter/CryptItCom.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + + +import re + +from random import randint + +from module.plugins.Crypter import Crypter + + +class CryptItCom(Crypter): + __name__ = "CryptItCom" + __type__ = "container" + __pattern__ = r"http://[\w\.]*?crypt-it\.com/(s|e|d|c)/[\w]+" + __version__ = "0.1" + __description__ = """Crypt.It.com Container Plugin""" + __author_name__ = ("jeix") + __author_mail__ = ("jeix@hasnomail.de") + + def file_exists(self): + html = self.load(self.pyfile.url) + if r'
        Was ist Crypt-It
        ' in html: + return False + return True + + def decrypt(self, pyfile): + if not self.file_exists(): + self.offline() + + # @TODO parse name and password + repl_pattern = r"/(s|e|d|c)/" + url = re.sub(repl_pattern, r"/d/", self.pyfile.url) + + pyfile.name = "tmp_cryptit_%s.ccf" % randint(0,1000) + location = self.download(url) + + self.packages.append(["Crypt-it Package", [location], "Crypt-it Package"]) + \ No newline at end of file diff --git a/pyload/plugins/crypter/CzshareComFolder.py b/pyload/plugins/crypter/CzshareComFolder.py new file mode 100644 index 000000000..c240c6a70 --- /dev/null +++ b/pyload/plugins/crypter/CzshareComFolder.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- + +import re +from module.plugins.Crypter import Crypter + +class CzshareComFolder(Crypter): + __name__ = "CzshareComFolder" + __type__ = "crypter" + __pattern__ = r"http://(\w*\.)*czshare\.(com|cz)/folders/.*" + __version__ = "0.1" + __description__ = """Czshare.com Folder Plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + FOLDER_PATTERN = r'\s*\s*(.*?)
        ' + LINK_PATTERN = r'info' + #NEXT_PAGE_PATTERN = r'' + + def decrypt(self, pyfile): + html = self.load(self.pyfile.url) + + new_links = [] + found = re.search(self.FOLDER_PATTERN, html, re.DOTALL) + if found is None: self.fail("Parse error (FOLDER)") + new_links.extend(re.findall(self.LINK_PATTERN, found.group(1))) + + if new_links: + self.core.files.addLinks(new_links, self.pyfile.package().id) + else: + self.fail('Could not extract any links') \ No newline at end of file diff --git a/pyload/plugins/crypter/DDLMusicOrg.py b/pyload/plugins/crypter/DDLMusicOrg.py new file mode 100644 index 000000000..f7cc996d0 --- /dev/null +++ b/pyload/plugins/crypter/DDLMusicOrg.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +from time import sleep + +from module.plugins.Crypter import Crypter + +class DDLMusicOrg(Crypter): + __name__ = "DDLMusicOrg" + __type__ = "container" + __pattern__ = r"http://[\w\.]*?ddl-music\.org/captcha/ddlm_cr\d\.php\?\d+\?\d+" + __version__ = "0.3" + __description__ = """ddl-music.org Container Plugin""" + __author_name__ = ("mkaay") + __author_mail__ = ("mkaay@mkaay.de") + + def setup(self): + self.multiDL = False + + def decrypt(self, pyfile): + html = self.req.load(self.pyfile.url, cookies=True) + + if re.search(r"Wer dies nicht rechnen kann", html) is not None: + self.offline() + + math = re.search(r"(\d+) ([\+-]) (\d+) =\s+", htmlwithlink) + if m: + self.packages.append((self.pyfile.package().name, [m.group(1)], self.pyfile.package().folder)) + else: + self.retry() diff --git a/pyload/plugins/crypter/DataHuFolder.py b/pyload/plugins/crypter/DataHuFolder.py new file mode 100644 index 000000000..f710f60d7 --- /dev/null +++ b/pyload/plugins/crypter/DataHuFolder.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- + +############################################################################ +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU Affero General Public License as # +# published by the Free Software Foundation, either version 3 of the # +# License, or (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU Affero General Public License for more details. # +# # +# You should have received a copy of the GNU Affero General Public License # +# along with this program. If not, see . # +############################################################################ + +import re + +from module.plugins.internal.SimpleCrypter import SimpleCrypter + + +class DataHuFolder(SimpleCrypter): + __name__ = "DataHuFolder" + __type__ = "crypter" + __pattern__ = r"http://(www\.)?data.hu/dir/\w+" + __version__ = "0.03" + __description__ = """Data.hu Folder Plugin""" + __author_name__ = ("crash", "stickell") + __author_mail__ = ("l.stickell@yahoo.it") + + LINK_PATTERN = r"\1" + TITLE_PATTERN = ur'(?P<title>.+) Let\xf6lt\xe9se' + + def decrypt(self, pyfile): + self.html = self.load(pyfile.url, decode=True) + + if u'K\xe9rlek add meg a jelsz\xf3t' in self.html: # Password protected + password = self.getPassword() + if password is '': + self.fail("No password specified, please set right password on Add package form and retry") + self.logDebug('The folder is password protected', 'Using password: ' + password) + self.html = self.load(pyfile.url, post={'mappa_pass': password}, decode=True) + if u'Hib\xe1s jelsz\xf3' in self.html: # Wrong password + self.fail("Incorrect password, please set right password on Add package form and retry") + + package_name, folder_name = self.getPackageNameAndFolder() + + package_links = re.findall(self.LINK_PATTERN, self.html) + self.logDebug('Package has %d links' % len(package_links)) + + if package_links: + self.packages = [(package_name, package_links, folder_name)] + else: + self.fail('Could not extract any links') diff --git a/pyload/plugins/crypter/DdlstorageComFolder.py b/pyload/plugins/crypter/DdlstorageComFolder.py new file mode 100644 index 000000000..d536032c6 --- /dev/null +++ b/pyload/plugins/crypter/DdlstorageComFolder.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +import re +from module.plugins.Crypter import Crypter +from module.plugins.hoster.MediafireCom import checkHTMLHeader +from module.common.json_layer import json_loads + +class DdlstorageComFolder(Crypter): + __name__ = "DdlstorageComFolder" + __type__ = "crypter" + __pattern__ = r"http://(?:\w*\.)*?ddlstorage.com/folder/\w{10}" + __version__ = "0.01" + __description__ = """DDLStorage.com Folder Plugin""" + __author_name__ = ("godofdream") + __author_mail__ = ("soilfiction@gmail.com") + + FILE_URL_PATTERN = '' + + def decrypt(self, pyfile): + new_links = [] + # load and parse html + html = self.load(pyfile.url) + found = re.findall(self.FILE_URL_PATTERN, html) + self.logDebug(found) + for link in found: + # file page + new_links.append("http://www.ddlstorage.com/%s" % link) + + if new_links: + self.core.files.addLinks(new_links, self.pyfile.package().id) + else: + self.fail('Could not extract any links') diff --git a/pyload/plugins/crypter/DepositfilesComFolder.py b/pyload/plugins/crypter/DepositfilesComFolder.py new file mode 100644 index 000000000..9023b238f --- /dev/null +++ b/pyload/plugins/crypter/DepositfilesComFolder.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- + +from module.plugins.internal.SimpleCrypter import SimpleCrypter + +class DepositfilesComFolder(SimpleCrypter): + __name__ = "DepositfilesComFolder" + __type__ = "crypter" + __pattern__ = r"http://(www\.)?depositfiles.com/folders/\w+" + __version__ = "0.01" + __description__ = """Depositfiles.com Folder Plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + LINK_PATTERN = r'
        ]*>\s*' \ No newline at end of file diff --git a/pyload/plugins/crypter/Dereferer.py b/pyload/plugins/crypter/Dereferer.py new file mode 100644 index 000000000..584835e18 --- /dev/null +++ b/pyload/plugins/crypter/Dereferer.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . +""" + +import re +import urllib + +from module.plugins.Crypter import Crypter + +class Dereferer(Crypter): + __name__ = "Dereferer" + __type__ = "crypter" + __pattern__ = r'https?://([^/]+)/.*?(?P(ht|f)tps?(://|%3A%2F%2F).*)' + __version__ = "0.1" + __description__ = """Crypter for dereferers""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + def decrypt(self, pyfile): + link = re.match(self.__pattern__, self.pyfile.url).group('url') + self.core.files.addLinks([ urllib.unquote(link).rstrip('+') ], self.pyfile.package().id) diff --git a/pyload/plugins/crypter/DontKnowMe.py b/pyload/plugins/crypter/DontKnowMe.py new file mode 100644 index 000000000..dfa72df47 --- /dev/null +++ b/pyload/plugins/crypter/DontKnowMe.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +import re +import urllib + +from module.plugins.Crypter import Crypter + +class DontKnowMe(Crypter): + __name__ = "DontKnowMe" + __type__ = "crypter" + __pattern__ = r"http://dontknow.me/at/\?.+$" + __version__ = "0.1" + __description__ = """DontKnowMe""" + __author_name__ = ("selaux") + __author_mail__ = ("") + + LINK_PATTERN = r"http://dontknow.me/at/\?(.+)$" + + def decrypt(self, pyfile): + link = re.findall(self.LINK_PATTERN, self.pyfile.url)[0] + self.core.files.addLinks([ urllib.unquote(link) ], self.pyfile.package().id) diff --git a/pyload/plugins/crypter/DownloadVimeoCom.py b/pyload/plugins/crypter/DownloadVimeoCom.py new file mode 100644 index 000000000..88310915b --- /dev/null +++ b/pyload/plugins/crypter/DownloadVimeoCom.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +import HTMLParser +from module.plugins.Crypter import Crypter + +class DownloadVimeoCom(Crypter): + __name__ = 'DownloadVimeoCom' + __type__ = 'crypter' + __pattern__ = r'(?:http://vimeo\.com/\d*|http://smotri\.com/video/view/\?id=.*)' + ## The download from dailymotion failed with a 403 + __version__ = '0.1' + __description__ = """Video Download Plugin based on downloadvimeo.com""" + __author_name__ = ('4Christopher') + __author_mail__ = ('4Christopher@gmx.de') + BASE_URL = 'http://downloadvimeo.com' + + def decrypt(self, pyfile): + self.package = pyfile.package() + html = self.load('%s/generate?url=%s' % (self.BASE_URL, pyfile.url)) + h = HTMLParser.HTMLParser() + try: + f = re.search(r'cmd quality="(?P[^"]+?)">\s*?(?P[^<]*?)', html) + except: + self.logDebug('Failed to find the URL') + else: + url = h.unescape(f.group('URL')) + self.logDebug('Quality: %s, URL: %s' % (f.group('quality'), url)) + self.packages.append((self.package.name, [url], self.package.folder)) diff --git a/pyload/plugins/crypter/DuckCryptInfo.py b/pyload/plugins/crypter/DuckCryptInfo.py new file mode 100644 index 000000000..4886d24db --- /dev/null +++ b/pyload/plugins/crypter/DuckCryptInfo.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- + +import re +from module.lib.BeautifulSoup import BeautifulSoup +from module.plugins.Crypter import Crypter + +class DuckCryptInfo(Crypter): + __name__ = "DuckCryptInfo" + __type__ = "container" + __pattern__ = r"http://(?:www\.)?duckcrypt.info/(folder|wait|link)/(\w+)/?(\w*)" + __version__ = "0.02" + __description__ = """DuckCrypt.Info Container Plugin""" + __author_name__ = ("godofdream") + __author_mail__ = ("soilfiction@gmail.com") + + TIMER_PATTERN = r'(.*)' + + def decrypt(self, pyfile): + url = pyfile.url + # seems we don't need to wait + #src = self.req.load(str(url)) + #found = re.search(self.TIMER_PATTERN, src) + #if found: + # self.logDebug("Sleeping for" % found.group(1)) + # self.setWait(int(found.group(1)) ,False) + found = re.search(self.__pattern__, url) + if not found: + self.fail('Weird error in link') + if str(found.group(1)) == "link": + self.handleLink(url) + else: + self.handleFolder(found) + + + + def handleFolder(self, found): + src = self.load("http://duckcrypt.info/ajax/auth.php?hash=" + str(found.group(2))) + found = re.search(self.__pattern__, src) + self.logDebug("Redirectet to " + str(found.group(0))) + src = self.load(str(found.group(0))) + soup = BeautifulSoup(src) + cryptlinks = soup.findAll("div", attrs={"class": "folderbox"}) + self.logDebug("Redirectet to " + str(cryptlinks)) + if not cryptlinks: + self.fail('no links found - (Plugin out of date?)') + for clink in cryptlinks: + if clink.find("a"): + self.handleLink(clink.find("a")['href']) + + def handleLink(self, url): + src = self.load(url) + soup = BeautifulSoup(src) + link = soup.find("iframe")["src"] + if not link: + self.logDebug('no links found - (Plugin out of date?)') + else: + self.core.files.addLinks([link], self.pyfile.package().id) + diff --git a/pyload/plugins/crypter/EasybytezComFolder.py b/pyload/plugins/crypter/EasybytezComFolder.py new file mode 100644 index 000000000..1b887e421 --- /dev/null +++ b/pyload/plugins/crypter/EasybytezComFolder.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +############################################################################ +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU Affero General Public License as # +# published by the Free Software Foundation, either version 3 of the # +# License, or (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU Affero General Public License for more details. # +# # +# You should have received a copy of the GNU Affero General Public License # +# along with this program. If not, see . # +############################################################################ + +import re + +from module.plugins.internal.SimpleCrypter import SimpleCrypter + + +class EasybytezComFolder(SimpleCrypter): + __name__ = "EasybytezComFolder" + __type__ = "crypter" + __pattern__ = r"https?://(www\.)?easybytez\.com/users/\w+/\w+" + __version__ = "0.01" + __description__ = """Easybytez Crypter Plugin""" + __author_name__ = ("stickell") + __author_mail__ = ("l.stickell@yahoo.it") + + LINK_PATTERN = r'
        .+
        ' + TITLE_PATTERN = r'Files of (?P<title>.+) folder' + PAGES_PATTERN = r"(\d+)Next »
        \(\d+ total\)
        " + + def decrypt(self, pyfile): + self.html = self.load(pyfile.url, decode=True) + + package_name, folder_name = self.getPackageNameAndFolder() + + package_links = re.findall(self.LINK_PATTERN, self.html) + + pages = re.search(self.PAGES_PATTERN, self.html) + if pages: + pages = int(pages.group(1)) + else: + pages = 1 + + p = 2 + while p <= pages: + self.html = self.load(pyfile.url, get={'page': p}, decode=True) + package_links += re.findall(self.LINK_PATTERN, self.html) + p += 1 + + self.logDebug('Package has %d links' % len(package_links)) + + if package_links: + self.packages = [(package_name, package_links, folder_name)] + else: + self.fail('Could not extract any links') \ No newline at end of file diff --git a/pyload/plugins/crypter/EmbeduploadCom.py b/pyload/plugins/crypter/EmbeduploadCom.py new file mode 100644 index 000000000..8fd70882f --- /dev/null +++ b/pyload/plugins/crypter/EmbeduploadCom.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- + +import re +from module.plugins.Crypter import Crypter +from module.network.HTTPRequest import BadHeader + +class EmbeduploadCom(Crypter): + __name__ = "EmbeduploadCom" + __type__ = "crypter" + __pattern__ = r"http://(www\.)?embedupload.com/\?d=.*" + __version__ = "0.02" + __description__ = """EmbedUpload.com crypter""" + __config__ = [("preferedHoster", "str", "Prefered hoster list (bar-separated) ", "embedupload"), + ("ignoredHoster", "str", "Ignored hoster list (bar-separated) ", "")] + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + LINK_PATTERN = r'
        ]*>\s*' + + def decrypt(self, pyfile): + self.html = self.load(self.pyfile.url, decode=True) + tmp_links = [] + new_links = [] + + found = re.findall(self.LINK_PATTERN, self.html) + if found: + prefered_set = set(self.getConfig("preferedHoster").split('|')) + prefered_set = map(lambda s: s.lower().split('.')[0], prefered_set) + print "PF", prefered_set + tmp_links.extend([x[1] for x in found if x[0] in prefered_set]) + self.getLocation(tmp_links, new_links) + + if not new_links: + ignored_set = set(self.getConfig("ignoredHoster").split('|')) + ignored_set = map(lambda s: s.lower().split('.')[0], ignored_set) + print "IG", ignored_set + tmp_links.extend([x[1] for x in found if x[0] not in ignored_set]) + self.getLocation(tmp_links, new_links) + + if new_links: + self.core.files.addLinks(new_links, self.pyfile.package().id) + else: + self.fail('Could not extract any links') + + def getLocation(self, tmp_links, new_links): + for link in tmp_links: + try: + header = self.load(link, just_header = True) + if "location" in header: + new_links.append(header['location']) + except BadHeader: + pass + + \ No newline at end of file diff --git a/pyload/plugins/crypter/FilebeerInfoFolder.py b/pyload/plugins/crypter/FilebeerInfoFolder.py new file mode 100644 index 000000000..f45144f14 --- /dev/null +++ b/pyload/plugins/crypter/FilebeerInfoFolder.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +import re +from module.plugins.Crypter import Crypter + +class FilebeerInfoFolder(Crypter): + __name__ = "FilebeerInfoFolder" + __type__ = "crypter" + __pattern__ = r"http://(?:www\.)?filebeer\.info/(\d+~f).*" + __version__ = "0.01" + __description__ = """Filebeer.info Folder Plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + LINK_PATTERN = r'' + PAGE_COUNT_PATTERN = r'

        \s*Total Pages (\d+)' + + def decrypt(self, pyfile): + pyfile.url = re.sub(self.__pattern__, r'http://filebeer.info/\1?page=1', pyfile.url) + html = self.load(pyfile.url) + + page_count = int(re.search(self.PAGE_COUNT_PATTERN, html).group(1)) + new_links = [] + + for i in range(1, page_count + 1): + self.logInfo("Fetching links from page %i" % i) + new_links.extend(re.findall(self.LINK_PATTERN, html)) + + if i < page_count: + html = self.load("%s?page=%d" % (pyfile.url, i+1)) + + if new_links: + self.core.files.addLinks(new_links, self.pyfile.package().id) + else: + self.fail('Could not extract any links') \ No newline at end of file diff --git a/pyload/plugins/crypter/FilefactoryComFolder.py b/pyload/plugins/crypter/FilefactoryComFolder.py new file mode 100644 index 000000000..32793b491 --- /dev/null +++ b/pyload/plugins/crypter/FilefactoryComFolder.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +import re +from module.plugins.Crypter import Crypter + +class FilefactoryComFolder(Crypter): + __name__ = "FilefactoryComFolder" + __type__ = "crypter" + __pattern__ = r"(http://(www\.)?filefactory\.com/f/\w+).*" + __version__ = "0.1" + __description__ = """Filefactory.com Folder Plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + FOLDER_PATTERN = r'(.*?)
        ' + LINK_PATTERN = r'        ' + PAGINATOR_PATTERN = r'

        \s*\s*
          (.*?)
        \s*
        ' + NEXT_PAGE_PATTERN = r'
      • .*?
        • \s*
      • ' + + def decrypt(self, pyfile): + url_base = re.search(self.__pattern__, self.pyfile.url).group(1) + html = self.load(url_base) + + new_links = [] + for i in range(1,100): + self.logInfo("Fetching links from page %i" % i) + found = re.search(self.FOLDER_PATTERN, html, re.DOTALL) + if found is None: self.fail("Parse error (FOLDER)") + + new_links.extend(re.findall(self.LINK_PATTERN, found.group(1))) + + try: + paginator = re.search(self.PAGINATOR_PATTERN, html, re.DOTALL).group(1) + next_page = re.search(self.NEXT_PAGE_PATTERN, paginator).group(1) + html = self.load("%s/%s" % (url_base, next_page)) + except Exception, e: + break + else: + self.logInfo("Limit of 99 pages reached, aborting") + + if new_links: + self.core.files.addLinks(new_links, self.pyfile.package().id) + else: + self.fail('Could not extract any links') \ No newline at end of file diff --git a/pyload/plugins/crypter/FileserveComFolder.py b/pyload/plugins/crypter/FileserveComFolder.py new file mode 100644 index 000000000..9fe806971 --- /dev/null +++ b/pyload/plugins/crypter/FileserveComFolder.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +import re + +from module.plugins.Crypter import Crypter + +class FileserveComFolder(Crypter): + __name__ = "FileserveComFolder" + __type__ = "crypter" + __pattern__ = r"http://(?:www\.)?fileserve.com/list/\w+" + __version__ = "0.11" + __description__ = """FileServeCom.com Folder Plugin""" + __author_name__ = ("fionnc") + __author_mail__ = ("fionnc@gmail.com") + + FOLDER_PATTERN = r'(.*?)
        ' + LINK_PATTERN = r'        ' + + def decrypt(self, pyfile): + html = self.load(self.pyfile.url) + + new_links = [] + + folder = re.search(self.FOLDER_PATTERN, html, re.DOTALL) + if folder is None: self.fail("Parse error (FOLDER)") + + new_links.extend(re.findall(self.LINK_PATTERN, folder.group(1))) + + if new_links: + self.core.files.addLinks(map(lambda s:"http://fileserve.com%s" % s, new_links), self.pyfile.package().id) + else: + self.fail('Could not extract any links') \ No newline at end of file diff --git a/pyload/plugins/crypter/FourChanOrg.py b/pyload/plugins/crypter/FourChanOrg.py new file mode 100644 index 000000000..5c96e723d --- /dev/null +++ b/pyload/plugins/crypter/FourChanOrg.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re + +from module.plugins.Crypter import Crypter + +class FourChanOrg(Crypter): + # Based on 4chandl by Roland Beermann + # https://gist.github.com/enkore/3492599 + __name__ = "FourChanOrg" + __type__ = "container" + __version__ = "0.3" + __pattern__ = r"http://boards\.4chan.org/\w+/res/(\d+)" + __description__ = "Downloader for entire 4chan threads" + + def decrypt(self, pyfile): + pagehtml = self.load(pyfile.url) + + images = set(re.findall(r'(images\.4chan\.org/[^/]*/src/[^"<]*)', pagehtml)) + urls = [] + for image in images: + urls.append("http://" + image) + + self.core.files.addLinks(urls, self.pyfile.package().id) diff --git a/pyload/plugins/crypter/FshareVnFolder.py b/pyload/plugins/crypter/FshareVnFolder.py new file mode 100644 index 000000000..2515e7edd --- /dev/null +++ b/pyload/plugins/crypter/FshareVnFolder.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- + +from module.plugins.internal.SimpleCrypter import SimpleCrypter + +class FshareVnFolder(SimpleCrypter): + __name__ = "FshareVnFolder" + __type__ = "crypter" + __pattern__ = r"http://(www\.)?fshare.vn/folder/.*" + __version__ = "0.01" + __description__ = """Fshare.vn Folder Plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + LINK_PATTERN = r'
      • ' \ No newline at end of file diff --git a/pyload/plugins/crypter/GooGl.py b/pyload/plugins/crypter/GooGl.py new file mode 100644 index 000000000..bcb1d7494 --- /dev/null +++ b/pyload/plugins/crypter/GooGl.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +############################################################################ +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU Affero General Public License as # +# published by the Free Software Foundation, either version 3 of the # +# License, or (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU Affero General Public License for more details. # +# # +# You should have received a copy of the GNU Affero General Public License # +# along with this program. If not, see . # +############################################################################ + +from module.plugins.Crypter import Crypter +from module.common.json_layer import json_loads + + +class GooGl(Crypter): + __name__ = "GooGl" + __type__ = "crypter" + __pattern__ = r"https?://(www\.)?goo\.gl/\w+" + __version__ = "0.01" + __description__ = """Goo.gl Crypter Plugin""" + __author_name__ = ("stickell") + __author_mail__ = ("l.stickell@yahoo.it") + + API_URL = 'https://www.googleapis.com/urlshortener/v1/url' + + def decrypt(self, pyfile): + rep = self.load(self.API_URL, get={'shortUrl': pyfile.url}) + self.logDebug('JSON data: ' + rep) + rep = json_loads(rep) + + if 'longUrl' in rep: + self.core.files.addLinks([rep['longUrl']], self.pyfile.package().id) + else: + self.fail('Unable to expand shortened link') diff --git a/pyload/plugins/crypter/HoerbuchIn.py b/pyload/plugins/crypter/HoerbuchIn.py new file mode 100644 index 000000000..6f23b2eb9 --- /dev/null +++ b/pyload/plugins/crypter/HoerbuchIn.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re + +from module.plugins.Crypter import Crypter +from module.lib.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup + +class HoerbuchIn(Crypter): + __name__ = "HoerbuchIn" + __type__ = "container" + __pattern__ = r"http://(www\.)?hoerbuch\.in/(wp/horbucher/\d+/.+/|tp/out.php\?.+|protection/folder_\d+\.html)" + __version__ = "0.7" + __description__ = """Hoerbuch.in Container Plugin""" + __author_name__ = ("spoob", "mkaay") + __author_mail__ = ("spoob@pyload.org", "mkaay@mkaay.de") + + article = re.compile("http://(www\.)?hoerbuch\.in/wp/horbucher/\d+/.+/") + protection = re.compile("http://(www\.)?hoerbuch\.in/protection/folder_\d+.html") + + def decrypt(self, pyfile): + self.pyfile = pyfile + + if self.article.match(self.pyfile.url): + src = self.load(self.pyfile.url) + soup = BeautifulSoup(src, convertEntities=BeautifulStoneSoup.HTML_ENTITIES) + + abookname = soup.find("a", attrs={"rel": "bookmark"}).text + for a in soup.findAll("a", attrs={"href": self.protection}): + package = "%s (%s)" % (abookname, a.previousSibling.previousSibling.text[:-1]) + links = self.decryptFolder(a["href"]) + + self.packages.append((package, links, self.pyfile.package().folder)) + else: + links = self.decryptFolder(self.pyfile.url) + + self.packages.append((self.pyfile.package().name, links, self.pyfile.package().folder)) + + def decryptFolder(self, url): + m = self.protection.search(url) + if not m: + self.fail("Bad URL") + url = m.group(0) + + self.pyfile.url = url + src = self.req.load(url, post={"viewed": "adpg"}) + + links = [] + pattern = re.compile(r'
        ", src) + + if not len(links) > 0: + self.retry() + + self.correctCaptcha() + + cleanedLinks = [] + for link in links: + if link.startswith("http://dontknow.me/at/?"): + cleanedLink = urllib.unquote(link[23:]) + else: + cleanedLink = link + self.logDebug("Link: %s" % cleanedLink) + cleanedLinks.append(cleanedLink) + + self.logDebug("Decrypted %d links" % len(links)) + + self.pyfile.package().password = "iload.to" + self.packages.append((self.pyfile.package().name, cleanedLinks, self.pyfile.package().folder)) \ No newline at end of file diff --git a/pyload/plugins/crypter/LetitbitNetFolder.py b/pyload/plugins/crypter/LetitbitNetFolder.py new file mode 100644 index 000000000..68aad9dd7 --- /dev/null +++ b/pyload/plugins/crypter/LetitbitNetFolder.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +import re +from module.plugins.Crypter import Crypter + + +class LetitbitNetFolder(Crypter): + __name__ = "LetitbitNetFolder" + __type__ = "crypter" + __pattern__ = r"http://(?:www\.)?letitbit.net/folder/\w+" + __version__ = "0.1" + __description__ = """Letitbit.net Folder Plugin""" + __author_name__ = ("DHMH", "z00nx") + __author_mail__ = ("webmaster@pcProfil.de", "z00nx0@gmail.com") + + FOLDER_PATTERN = r'(.*)
        ' + LINK_PATTERN = r'        ' + + def decrypt(self, pyfile): + html = self.load(self.pyfile.url) + + new_links = [] + + folder = re.search(self.FOLDER_PATTERN, html, re.DOTALL) + if folder is None: + self.fail("Parse error (FOLDER)") + + new_links.extend(re.findall(self.LINK_PATTERN, folder.group(0))) + + if new_links: + self.core.files.addLinks(new_links, self.pyfile.package().id) + else: + self.fail('Could not extract any links') diff --git a/pyload/plugins/crypter/LinkList.py b/pyload/plugins/crypter/LinkList.py new file mode 100644 index 000000000..ebfa373eb --- /dev/null +++ b/pyload/plugins/crypter/LinkList.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from module.plugins.Crypter import Crypter, Package + +class LinkList(Crypter): + __name__ = "LinkList" + __version__ = "0.11" + __pattern__ = r".+\.txt$" + __description__ = """Read Link Lists in txt format""" + __author_name__ = ("spoob", "jeix") + __author_mail__ = ("spoob@pyload.org", "jeix@hasnomail.com") + + # method declaration is needed here + def decryptURL(self, url): + return Crypter.decryptURL(self, url) + + def decryptFile(self, content): + links = content.splitlines() + + curPack = "default" + packages = {curPack:[]} + + for link in links: + link = link.strip() + if not link: continue + + if link.startswith(";"): + continue + if link.startswith("[") and link.endswith("]"): + # new package + curPack = link[1:-1] + packages[curPack] = [] + continue + packages[curPack].append(link) + + # empty packages fix + delete = [] + + for key,value in packages.iteritems(): + if not value: + delete.append(key) + + for key in delete: + del packages[key] + + urls = [] + + for name, links in packages.iteritems(): + if name == "default": + urls.extend(links) + else: + urls.append(Package(name, links)) + + return urls \ No newline at end of file diff --git a/pyload/plugins/crypter/LinkSaveIn.py b/pyload/plugins/crypter/LinkSaveIn.py new file mode 100644 index 000000000..e021316bf --- /dev/null +++ b/pyload/plugins/crypter/LinkSaveIn.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- + +# +# v2.01 - hagg +# * cnl2 and web links are skipped if JS is not available (instead of failing the package) +# * only best available link source is used (priority: cnl2>rsdf>ccf>dlc>web +# + +from Crypto.Cipher import AES +from module.plugins.Crypter import Crypter +from module.unescape import unescape +import base64 +import binascii +import re + +class LinkSaveIn(Crypter): + __name__ = "LinkSaveIn" + __type__ = "crypter" + __pattern__ = r"http://(www\.)?linksave.in/(?P\w+)$" + __version__ = "2.01" + __description__ = """LinkSave.in Crypter Plugin""" + __author_name__ = ("fragonib") + __author_mail__ = ("fragonib[AT]yahoo[DOT]es") + + # Constants + _JK_KEY_ = "jk" + _CRYPTED_KEY_ = "crypted" + HOSTER_DOMAIN = "linksave.in" + + def setup(self): + self.html = None + self.fileid = None + self.captcha = False + self.package = None + self.preferred_sources = ['cnl2', 'rsdf', 'ccf', 'dlc', 'web'] + + def decrypt(self, pyfile): + + # Init + self.package = pyfile.package() + self.fileid = re.match(self.__pattern__, pyfile.url).group('id') + self.req.cj.setCookie(self.HOSTER_DOMAIN, "Linksave_Language", "english") + + # Request package + self.html = self.load(self.pyfile.url) + if not self.isOnline(): + self.offline() + + # Check for protection + if self.isPasswordProtected(): + self.unlockPasswordProtection() + self.handleErrors() + + if self.isCaptchaProtected(): + self.captcha = True + self.unlockCaptchaProtection() + self.handleErrors() + + # Get package name and folder + (package_name, folder_name) = self.getPackageInfo() + + # Extract package links + package_links = [] + for type_ in self.preferred_sources: + package_links.extend(self.handleLinkSource(type_)) + if package_links: # use only first source which provides links + break + package_links = set(package_links) + + # Pack + if package_links: + self.packages = [(package_name, package_links, folder_name)] + else: + self.fail('Could not extract any links') + + def isOnline(self): + if "Error 404 - Folder not found!" in self.html: + self.logDebug("File not found") + return False + return True + + def isPasswordProtected(self): + if re.search(r'''Captcha:" in self.html: + self.logDebug("Links are captcha protected") + return True + return False + + def unlockPasswordProtection(self): + password = self.getPassword() + self.logDebug("Submitting password [%s] for protected links" % password) + post = {"id": self.fileid, "besucherpasswort": password, 'login': 'submit'} + self.html = self.load(self.pyfile.url, post=post) + + def unlockCaptchaProtection(self): + captcha_hash = re.search(r'name="hash" value="([^"]+)', self.html).group(1) + captcha_url = re.search(r'src=".(/captcha/cap.php\?hsh=[^"]+)', self.html).group(1) + captcha_code = self.decryptCaptcha("http://linksave.in" + captcha_url, forceUser=True) + self.html = self.load(self.pyfile.url, post={"id": self.fileid, "hash": captcha_hash, "code": captcha_code}) + + def getPackageInfo(self): + name = self.pyfile.package().name + folder = self.pyfile.package().folder + self.logDebug("Defaulting to pyfile name [%s] and folder [%s] for package" % (name, folder)) + return name, folder + + def handleErrors(self): + if "The visitorpassword you have entered is wrong" in self.html: + self.logDebug("Incorrect password, please set right password on 'Edit package' form and retry") + self.fail("Incorrect password, please set right password on 'Edit package' form and retry") + + if self.captcha: + if "Wrong code. Please retry" in self.html: + self.logDebug("Invalid captcha, retrying") + self.invalidCaptcha() + self.retry() + else: + self.correctCaptcha() + + def handleLinkSource(self, type_): + if type_ == 'cnl2': + return self.handleCNL2() + elif type_ in ('rsdf', 'ccf', 'dlc'): + return self.handleContainer(type_) + elif type_ == 'web': + return self.handleWebLinks() + else: + self.fail('unknown source type "%s" (this is probably a bug)' % type_) + + def handleWebLinks(self): + package_links = [] + self.logDebug("Search for Web links") + if not self.js: + self.logDebug("no JS -> skip Web links") + else: + #@TODO: Gather paginated web links + pattern = r'(.*)', response)[-1] + jseval = self.js.eval("document = { write: function(e) { return e; } }; %s" % jscode) + dlLink = re.search(r'http://linksave\.in/dl-\w+', jseval).group(0) + self.logDebug("JsEngine returns value [%s] for redirection link" % dlLink) + response = self.load(dlLink) + link = unescape(re.search(r'') + for pattern in patterns: + rexpr = re.compile(pattern, re.DOTALL) + content = re.sub(rexpr, "", content) + return content + + def removeContainers(self,package_links): + tmp_package_links = package_links[:] + for link in tmp_package_links: + self.logDebug(link) + if ".dlc" in link or ".ccf" in link or ".rsdf" in link: + self.logDebug("Removing [%s] from package_links" % link) + package_links.remove(link) + + if len(package_links) > 0: + return package_links + else: + return tmp_package_links + + def isOnline(self): + if "Your folder does not exist" in self.cleanedHtml: + self.logDebug("File not found") + return False + return True + + def isProtected(self): + if re.search(r'''''', self.cleanedHtml): + self.logDebug("Links are protected") + return True + return False + + def getPackageInfo(self): + title_re = r'

        (?P[^<]+).*?</span>.*?</h2>' + m = re.findall(title_re, self.html, re.DOTALL) + if m is not None: + title = m[-1].strip() + name = folder = title + self.logDebug("Found name [%s] and folder [%s] in package info" % (name, folder)) + else: + name = self.package.name + folder = self.package.folder + self.logDebug("Package info not found, defaulting to pyfile name [%s] and folder [%s]" % (name, folder)) + return name, folder + + def unlockProtection(self): + + postData = {} + + form = re.search(r'''<form\ name="protected"(.*?)</form>''', self.cleanedHtml, re.DOTALL).group(1) + + # Submit package password + if "password" in form: + password = self.getPassword() + self.logDebug("Submitting password [%s] for protected links" % password) + postData['password'] = password + + # Resolve anicaptcha + if "anicaptcha" in form: + self.captcha = True + self.logDebug("Captcha protected, resolving captcha") + captchaUri = re.search(r'src="(/temp/anicaptcha/[^"]+)', form).group(1) + captcha = self.decryptCaptcha("http://ncrypt.in" + captchaUri) + self.logDebug("Captcha resolved [%s]" % captcha) + postData['captcha'] = captcha + + # Resolve recaptcha + if "recaptcha" in form: + self.captcha = True + id = re.search(r'\?k=(.*?)"', form).group(1) + self.logDebug("Resolving ReCaptcha with key [%s]" % id) + recaptcha = ReCaptcha(self) + challenge, code = recaptcha.challenge(id) + postData['recaptcha_challenge_field'] = challenge + postData['recaptcha_response_field'] = code + + # Resolve circlecaptcha + if "circlecaptcha" in form: + self.captcha = True + self.logDebug("Captcha protected") + captcha_img_url = "http://ncrypt.in/classes/captcha/circlecaptcha.php" + coords = self.decryptCaptcha(captcha_img_url, forceUser=True, imgtype="png", result_type='positional') + self.logDebug("Captcha resolved, coords [%s]" % str(coords)) + self.captcha_post_url = self.pyfile.url + + postData['circle.x'] = coords[0] + postData['circle.y'] = coords[1] + + + # Unlock protection + postData['submit_protected'] = 'Continue to folder ' + return self.load(self.pyfile.url, post=postData) + + def handleErrors(self): + + if "This password is invalid!" in self.cleanedHtml: + self.logDebug("Incorrect password, please set right password on 'Edit package' form and retry") + self.fail("Incorrect password, please set right password on 'Edit package' form and retry") + + if self.captcha: + if "The securitycheck was wrong!" in self.cleanedHtml: + self.logDebug("Invalid captcha, retrying") + self.invalidCaptcha() + self.retry() + else: + self.correctCaptcha() + + def handleWebLinks(self): + package_links = [] + self.logDebug("Handling Web links") + + pattern = r"(http://ncrypt\.in/link-.*?=)" + links = re.findall(pattern, self.html) + self.logDebug("Decrypting %d Web links" % len(links)) + for i, link in enumerate(links): + self.logDebug("Decrypting Web link %d, %s" % (i+1, link)) + try: + url = link.replace("link-", "frame-") + link = self.load(url, just_header=True)['location'] + package_links.append(link) + except Exception, detail: + self.logDebug("Error decrypting Web link %s, %s" % (link, detail)) + return package_links + + def handleContainers(self): + package_links = [] + self.logDebug("Handling Container links") + + pattern = r"/container/(rsdf|dlc|ccf)/([a-z0-9]+)" + containersLinks = re.findall(pattern, self.html) + self.logDebug("Decrypting %d Container links" % len(containersLinks)) + for containerLink in containersLinks: + link = "http://ncrypt.in/container/%s/%s.%s" % (containerLink[0], containerLink[1], containerLink[0]) + package_links.append(link) + return package_links + + def handleCNL2(self): + package_links = [] + self.logDebug("Handling CNL2 links") + + if 'cnl2_output' in self.cleanedHtml: + try: + (vcrypted, vjk) = self._getCipherParams() + for (crypted, jk) in zip(vcrypted, vjk): + package_links.extend(self._getLinks(crypted, jk)) + except: + self.fail("Unable to decrypt CNL2 links") + return package_links + + def _getCipherParams(self): + + pattern = r'<input.*?name="%s".*?value="(.*?)"' + + # Get jk + jk_re = pattern % NCryptIn._JK_KEY_ + vjk = re.findall(jk_re, self.html) + + # Get crypted + crypted_re = pattern % NCryptIn._CRYPTED_KEY_ + vcrypted = re.findall(crypted_re, self.html) + + # Log and return + self.logDebug("Detected %d crypted blocks" % len(vcrypted)) + return vcrypted, vjk + + def _getLinks(self, crypted, jk): + + # Get key + jreturn = self.js.eval("%s f()" % jk) + self.logDebug("JsEngine returns value [%s]" % jreturn) + key = binascii.unhexlify(jreturn) + + # Decode crypted + crypted = base64.standard_b64decode(crypted) + + # Decrypt + Key = key + IV = key + obj = AES.new(Key, AES.MODE_CBC, IV) + text = obj.decrypt(crypted) + + # Extract links + text = text.replace("\x00", "").replace("\r", "") + links = text.split("\n") + links = filter(lambda x: x != "", links) + + # Log and return + self.logDebug("Block has %d links" % len(links)) + return links diff --git a/pyload/plugins/crypter/NetfolderIn.py b/pyload/plugins/crypter/NetfolderIn.py new file mode 100644 index 000000000..c5c602c27 --- /dev/null +++ b/pyload/plugins/crypter/NetfolderIn.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- + +import re + +from module.plugins.internal.SimpleCrypter import SimpleCrypter + + +class NetfolderIn(SimpleCrypter): + __name__ = "NetfolderIn" + __type__ = "crypter" + __pattern__ = r"http://(?:www\.)?netfolder.in/((?P<id1>\w+)/\w+|folder.php\?folder_id=(?P<id2>\w+))" + __version__ = "0.6" + __description__ = """NetFolder Crypter Plugin""" + __author_name__ = ("RaNaN", "fragonib") + __author_mail__ = ("RaNaN@pyload.org", "fragonib[AT]yahoo[DOT]es") + + TITLE_PATTERN = r'<div class="Text">Inhalt des Ordners <span(.*)>(?P<title>.+)</span></div>' + + def decrypt(self, pyfile): + # Request package + self.html = self.load(pyfile.url) + + # Check for password protection + if self.isPasswordProtected(): + self.html = self.submitPassword() + if self.html is None: + self.fail("Incorrect password, please set right password on Add package form and retry") + + # Get package name and folder + (package_name, folder_name) = self.getPackageNameAndFolder() + + # Get package links + package_links = self.getLinks() + + # Set package + self.packages = [(package_name, package_links, folder_name)] + + def isPasswordProtected(self): + + if '<input type="password" name="password"' in self.html: + self.logDebug("Links are password protected") + return True + return False + + def submitPassword(self): + # Gather data + try: + m = re.match(self.__pattern__, self.pyfile.url) + id = max(m.group('id1'), m.group('id2')) + except AttributeError: + self.logDebug("Unable to get package id from url [%s]" % self.pyfile.url) + return + url = "http://netfolder.in/folder.php?folder_id=" + id + password = self.getPassword() + + # Submit package password + post = {'password': password, 'save': 'Absenden'} + self.logDebug("Submitting password [%s] for protected links with id [%s]" % (password, id)) + html = self.load(url, {}, post) + + # Check for invalid password + if '<div class="InPage_Error">' in html: + self.logDebug("Incorrect password, please set right password on Edit package form and retry") + return None + + return html + + def getLinks(self): + links = re.search(r'name="list" value="(.*?)"', self.html).group(1).split(",") + self.logDebug("Package has %d links" % len(links)) + return links diff --git a/pyload/plugins/crypter/OneKhDe.py b/pyload/plugins/crypter/OneKhDe.py new file mode 100644 index 000000000..c77203187 --- /dev/null +++ b/pyload/plugins/crypter/OneKhDe.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re + +from module.unescape import unescape +from module.plugins.Crypter import Crypter + +class OneKhDe(Crypter): + __name__ = "OneKhDe" + __type__ = "container" + __pattern__ = r"http://(www\.)?1kh.de/f/" + __version__ = "0.1" + __description__ = """1kh.de Container Plugin""" + __author_name__ = ("spoob") + __author_mail__ = ("spoob@pyload.org") + + def __init__(self, parent): + Crypter.__init__(self, parent) + self.parent = parent + self.html = None + + def file_exists(self): + """ returns True or False + """ + return True + + def proceed(self, url, location): + url = self.parent.url + self.html = self.req.load(url) + temp_links = [] + link_ids = re.findall(r"<a id=\"DownloadLink_(\d*)\" href=\"http://1kh.de/", self.html) + for id in link_ids: + new_link = unescape(re.search("width=\"100%\" src=\"(.*)\"></iframe>", self.req.load("http://1kh.de/l/" + id)).group(1)) + temp_links.append(new_link) + self.links = temp_links diff --git a/pyload/plugins/crypter/OronComFolder.py b/pyload/plugins/crypter/OronComFolder.py new file mode 100755 index 000000000..726371966 --- /dev/null +++ b/pyload/plugins/crypter/OronComFolder.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +import re + +from module.plugins.Crypter import Crypter + +class OronComFolder(Crypter): + __name__ = "OronComFolder" + __type__ = "crypter" + __pattern__ = r"http://(?:www\.)?oron.com/folder/\w+" + __version__ = "0.2" + __description__ = """Oron.com Folder Plugin""" + __author_name__ = ("DHMH") + __author_mail__ = ("webmaster@pcProfil.de") + + FOLDER_PATTERN = r'<table(?:.*)class="tbl"(?:.*)>(?:.*)<table(?:.*)class="tbl2"(?:.*)>(?P<body>.*)</table>(?:.*)</table>' + LINK_PATTERN = r'<a href="([^"]+)" target="_blank">' + + def decryptURL(self, url): + html = self.load(url) + + new_links = [] + + if 'No such folder exist' in html: + # Don't fail because if there's more than a folder for this package + # and only one of them fails, no urls at all will be added. + self.logWarning("Folder does not exist") + return new_links + + folder = re.search(self.FOLDER_PATTERN, html, re.DOTALL) + if folder is None: + # Don't fail because if there's more than a folder for this package + # and only one of them fails, no urls at all will be added. + self.logWarning("Parse error (FOLDER)") + return new_links + + new_links.extend(re.findall(self.LINK_PATTERN, folder.group(0))) + + if new_links: + self.logDebug("Found %d new links" % len(new_links)) + return new_links + else: + # Don't fail because if there's more than a folder for this package + # and only one of them fails, no urls at all will be added. + self.logWarning('Could not extract any links') + return new_links diff --git a/pyload/plugins/crypter/QuickshareCzFolder.py b/pyload/plugins/crypter/QuickshareCzFolder.py new file mode 100644 index 000000000..6cb049935 --- /dev/null +++ b/pyload/plugins/crypter/QuickshareCzFolder.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- + +import re +from module.plugins.Crypter import Crypter + +class QuickshareCzFolder(Crypter): + __name__ = "QuickshareCzFolder" + __type__ = "crypter" + __pattern__ = r"http://(www\.)?quickshare.cz/slozka-\d+.*" + __version__ = "0.1" + __description__ = """Quickshare.cz Folder Plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + + FOLDER_PATTERN = r'<textarea[^>]*>(.*?)</textarea>' + LINK_PATTERN = r'(http://www.quickshare.cz/\S+)' + + def decrypt(self, pyfile): + html = self.load(self.pyfile.url) + + new_links = [] + found = re.search(self.FOLDER_PATTERN, html, re.DOTALL) + if found is None: self.fail("Parse error (FOLDER)") + new_links.extend(re.findall(self.LINK_PATTERN, found.group(1))) + + if new_links: + self.core.files.addLinks(new_links, self.pyfile.package().id) + else: + self.fail('Could not extract any links') \ No newline at end of file diff --git a/pyload/plugins/crypter/RSDF.py b/pyload/plugins/crypter/RSDF.py new file mode 100644 index 000000000..cbc9864b1 --- /dev/null +++ b/pyload/plugins/crypter/RSDF.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import base64 +import binascii +import re + +from module.plugins.Crypter import Crypter + +class RSDF(Crypter): + __name__ = "RSDF" + __version__ = "0.21" + __pattern__ = r".*\.rsdf" + __description__ = """RSDF Container Decode Plugin""" + __author_name__ = ("RaNaN", "spoob") + __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org") + + + def decrypt(self, pyfile): + + from Crypto.Cipher import AES + + infile = pyfile.url.replace("\n", "") + Key = binascii.unhexlify('8C35192D964DC3182C6F84F3252239EB4A320D2500000000') + + IV = binascii.unhexlify('FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF') + IV_Cipher = AES.new(Key, AES.MODE_ECB) + IV = IV_Cipher.encrypt(IV) + + obj = AES.new(Key, AES.MODE_CFB, IV) + + rsdf = open(infile, 'r') + + data = rsdf.read() + rsdf.close() + + if re.search(r"<title>404 - Not Found", data) is None: + data = binascii.unhexlify(''.join(data.split())) + data = data.splitlines() + + links = [] + for link in data: + link = base64.b64decode(link) + link = obj.decrypt(link) + decryptedUrl = link.replace('CCF: ', '') + links.append(decryptedUrl) + + self.log.debug("%s: adding package %s with %d links" % (self.__name__,pyfile.package().name,len(links))) + self.packages.append((pyfile.package().name, links)) diff --git a/pyload/plugins/crypter/RSLayerCom.py b/pyload/plugins/crypter/RSLayerCom.py new file mode 100644 index 000000000..6e4266f2e --- /dev/null +++ b/pyload/plugins/crypter/RSLayerCom.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re + +from module.plugins.Crypter import Crypter +from module.lib.BeautifulSoup import BeautifulSoup +from module.unescape import unescape + +class RSLayerCom(Crypter): + __name__ = "RSLayerCom" + __type__ = "container" + __pattern__ = r"http://(www\.)?rs-layer.com/directory-" + __config__ = [] + __version__ = "0.2" + __description__ = """RS-Layer.com Container Plugin""" + __author_name__ = ("hzpz") + __author_mail__ = ("none") + + def decrypt(self, pyfile): + url = pyfile.url + src = self.req.load(str(url)) + + soup = BeautifulSoup(src) + captchaTag = soup.find("img", attrs={"id": "captcha_image"}) + if captchaTag: + captchaUrl = "http://rs-layer.com/" + captchaTag["src"] + self.logDebug("Captcha URL: %s" % captchaUrl) + result = self.decryptCaptcha(str(captchaUrl), imgtype="png") + captchaInput = soup.find("input", attrs={"id": "captcha"}) + self.req.lastUrl = url + src = self.req.load(str(url), post={'captcha_input': result, 'image_name': captchaTag["src"]}) + + link_ids = re.findall(r"onclick=\"getFile\(\'([0-9]{7}-.{8})\'\);changeBackgroundColor", src) + + if not len(link_ids) > 0: + self.retry() + + self.correctCaptcha() + + links = [] + for id in link_ids: + self.logDebug("ID: %s" % id) + new_link = unescape(re.search(r"' + + + def setup(self): + self.fileid = None + self.package = None + self.password = None + self.html = None + self.captcha = False + + def decrypt(self, pyfile): + + # Init + self.initPackage(pyfile) + + # Request package + self.requestPackage() + + # Check for online + if not self.isOnline(): + self.offline() + + # Check for protection + if self.isPasswordProtected(): + self.unlockPasswordProtection() + self.handleErrors() + + if self.isCaptchaProtected(): + self.captcha = True + self.unlockCaptchaProtection() + self.handleErrors() + + # Get package name and folder + (package_name, folder_name) = self.getPackageInfo() + + # Extract package links + package_links = [] + for sources in self.PREFERRED_LINK_SOURCES: + package_links.extend(self.handleLinkSource(sources)) + if package_links: # use only first source which provides links + break + package_links = set(package_links) + + # Pack + if package_links: + self.packages = [(package_name, package_links, folder_name)] + else: + self.fail('Could not extract any links') + + def initPackage(self, pyfile): + self.fileid = re.match(self.__pattern__, pyfile.url).group('id') + self.package = pyfile.package() + self.password = self.getPassword() + self.url = pyfile.url + + def requestPackage(self): + self.html = self.load(self.url, decode = True) + + def isOnline(self): + if self.OFFLINE_TOKEN in self.html: + self.logDebug("File not found") + return False + return True + + def isPasswordProtected(self): + if self.PASSWORD_TOKEN in self.html: + self.logDebug("Links are password protected") + return True + + def isCaptchaProtected(self): + if self.CAPTCHA_TOKEN in self.html: + self.logDebug("Links are captcha protected") + return True + return False + + def unlockPasswordProtection(self): + self.logDebug("Submitting password [%s] for protected links" % self.password) + passwd_url = self.PASSWORD_SUBMIT_URL + "?id=%s" % self.fileid + passwd_data = { 'id': self.fileid, 'password': self.password, 'pw': 'submit' } + self.html = self.load(passwd_url, post=passwd_data, decode=True) + + def unlockCaptchaProtection(self): + self.logDebug("Request user positional captcha resolving") + captcha_img_url = self.CAPTCHA_IMG_URL + "?id=%s" % self.fileid + coords = self.decryptCaptcha(captcha_img_url, forceUser=True, imgtype="png", result_type='positional') + self.logDebug("Captcha resolved, coords [%s]" % str(coords)) + captcha_post_url = self.CAPTCHA_SUBMIT_URL + "?id=%s" % self.fileid + captcha_post_data = { 'button.x': coords[0], 'button.y': coords[1], 'captcha': 'submit' } + self.html = self.load(captcha_post_url, post=captcha_post_data, decode=True) + + def getPackageInfo(self): + name = folder = None + + # Try to get info from web + m = re.search(self.FILE_TITLE_REGEX, self.html) + if m is not None: + title = m.group(1).strip() + if not self.FILE_NOTITLE in title: + name = folder = title + self.logDebug("Found name [%s] and folder [%s] in package info" % (name, folder)) + + # Fallback to defaults + if not name or not folder: + name = self.package.name + folder = self.package.folder + self.logDebug("Package info not found, defaulting to pyfile name [%s] and folder [%s]" % (name, folder)) + + # Return package info + return name, folder + + def handleErrors(self): + if self.PASSWORD_ERROR_ROKEN in self.html: + msg = "Incorrect password, please set right password on 'Edit package' form and retry" + self.logDebug(msg) + self.fail(msg) + + if self.captcha: + if self.CAPTCHA_ERROR_ROKEN in self.html: + self.logDebug("Invalid captcha, retrying") + self.invalidCaptcha() + self.retry() + else: + self.correctCaptcha() + + def handleLinkSource(self, source): + if source == 'cnl2': + return self.handleCNL2Links() + elif source == 'dlc': + return self.handleDLCLinks() + elif source == 'web': + return self.handleWEBLinks() + else: + self.fail('Unknown source [%s] (this is probably a bug)' % source) + + def handleCNL2Links(self): + self.logDebug("Search for CNL2 links") + package_links = [] + m = re.search(self.CNL2_FORM_REGEX, self.html, re.DOTALL) + if m is not None: + cnl2_form = m.group(1) + try: + (vcrypted, vjk) = self._getCipherParams(cnl2_form) + for (crypted, jk) in zip(vcrypted, vjk): + package_links.extend(self._getLinks(crypted, jk)) + except: + self.logDebug("Unable to decrypt CNL2 links") + return package_links + + def handleDLCLinks(self): + self.logDebug('Search for DLC links') + package_links = [] + m = re.search(self.DLC_LINK_REGEX, self.html) + if m is not None: + container_url = self.DLC_DOWNLOAD_URL + "?id=%s&dlc=1" % self.fileid + self.logDebug("Downloading DLC container link [%s]" % container_url) + try: + dlc = self.load(container_url) + dlc_filename = self.fileid + ".dlc" + dlc_filepath = os.path.join(self.config["general"]["download_folder"], dlc_filename) + f = open(dlc_filepath, "wb") + f.write(dlc) + f.close() + package_links.append(dlc_filepath) + except: + self.logDebug("Unable to download DLC container") + return package_links + + def handleWEBLinks(self): + self.logDebug("Search for WEB links") + package_links = [] + fw_params = re.findall(self.WEB_FORWARD_REGEX, self.html) + self.logDebug("Decrypting %d Web links" % len(fw_params)) + for index, fw_param in enumerate(fw_params): + try: + fw_url = self.WEB_FORWARD_URL + "?%s" % fw_param + self.logDebug("Decrypting Web link %d, %s" % (index+1, fw_url)) + fw_response = self.load(fw_url, decode=True) + dl_link = re.search(self.WEB_LINK_REGEX, fw_response).group('link') + package_links.append(dl_link) + except Exception, detail: + self.logDebug("Error decrypting Web link %s, %s" % (index, detail)) + self.setWait(4) + self.wait() + return package_links + + def _getCipherParams(self, cnl2_form): + + # Get jk + jk_re = self.CNL2_FORMINPUT_REGEX % self.CNL2_JK_KEY + vjk = re.findall(jk_re, cnl2_form, re.IGNORECASE) + + # Get crypted + crypted_re = self.CNL2_FORMINPUT_REGEX % RelinkUs.CNL2_CRYPTED_KEY + vcrypted = re.findall(crypted_re, cnl2_form, re.IGNORECASE) + + # Log and return + self.logDebug("Detected %d crypted blocks" % len(vcrypted)) + return vcrypted, vjk + + def _getLinks(self, crypted, jk): + + # Get key + jreturn = self.js.eval("%s f()" % jk) + self.logDebug("JsEngine returns value [%s]" % jreturn) + key = binascii.unhexlify(jreturn) + + # Decode crypted + crypted = base64.standard_b64decode(crypted) + + # Decrypt + Key = key + IV = key + obj = AES.new(Key, AES.MODE_CBC, IV) + text = obj.decrypt(crypted) + + # Extract links + text = text.replace("\x00", "").replace("\r", "") + links = text.split("\n") + links = filter(lambda x: x != "", links) + + # Log and return + self.logDebug("Package has %d links" % len(links)) + return links diff --git a/pyload/plugins/crypter/SecuredIn.py b/pyload/plugins/crypter/SecuredIn.py new file mode 100644 index 000000000..e41896c5f --- /dev/null +++ b/pyload/plugins/crypter/SecuredIn.py @@ -0,0 +1,334 @@ +# -*- coding: utf-8 -*- + +import re + +from module.plugins.Crypter import Crypter +from module.lib.BeautifulSoup import BeautifulSoup + +from math import ceil + +class SecuredIn(Crypter): + __name__ = "SecuredIn" + __type__ = "container" + __pattern__ = r"http://[\w\.]*?secured\.in/download-[\d]+-[\w]{8}\.html" + __version__ = "0.1" + __description__ = """secured.in Container Plugin""" + __author_name__ = ("mkaay") + __author_mail__ = ("mkaay@mkaay.de") + + def __init__(self, parent): + Crypter.__init__(self, parent) + self.parent = parent + self.html = None + self.multi_dl = False + + def file_exists(self): + return True + + def proceed(self, url, location): + links = [] + ajaxUrl = "http://secured.in/ajax-handler.php" + src = self.req.load(url, cookies=True) + soup = BeautifulSoup(src) + img = soup.find("img", attrs={"id":"captcha_img"}) + for i in range(3): + form = soup.find("form", attrs={"id":"frm_captcha"}) + captchaHash = form.find("input", attrs={"id":"captcha_hash"})["value"] + captchaUrl = "http://secured.in/%s" % img["src"] + captchaData = self.req.load(str(captchaUrl), cookies=True) + result = self.waitForCaptcha(captchaData, "jpg") + src = self.req.load(url, cookies=True, post={"captcha_key":result, "captcha_hash":captchaHash}) + soup = BeautifulSoup(src) + img = soup.find("img", attrs={"id":"captcha_img"}) + if not img: + files = soup.findAll("tr", attrs={"id":re.compile("file-\d+")}) + dlIDPattern = re.compile("accessDownload\(\d, \d+, '(.*?)', \d\)") + cypher = self.Cypher() + for cfile in files: + m = dlIDPattern.search(cfile["onclick"]) + if m: + crypted = self.req.load(ajaxUrl, cookies=True, post={"cmd":"download", "download_id":m.group(1)}) + cypher.reset() + link = cypher.cypher(crypted) + links.append(link) + break + self.links = links + + class Cypher(): + def __init__(self): + self.reset() + + def reset(self): + self.iatwbfrd = [ + 0xd1310ba6, 0x98dfb5ac, 0x2ffd72db, 0xd01adfb7, 0xb8e1afed, 0x6a267e96, 0xba7c9045, 0xf12c7f99, 0x24a19947, 0xb3916cf7, 0x0801f2e2, + 0x858efc16, 0x636920d8, 0x71574e69, 0xa458fea3, 0xf4933d7e, 0x0d95748f, 0x728eb658, 0x718bcd58, 0x82154aee, 0x7b54a41d, 0xc25a59b5, + 0x9c30d539, 0x2af26013, 0xc5d1b023, 0x286085f0, 0xca417918, 0xb8db38ef, 0x8e79dcb0, 0x603a180e, 0x6c9e0e8b, 0xb01e8a3e, 0xd71577c1, + 0xbd314b27, 0x78af2fda, 0x55605c60, 0xe65525f3, 0xaa55ab94, 0x57489862, 0x63e81440, 0x55ca396a, 0x2aab10b6, 0xb4cc5c34, 0x1141e8ce, + 0xa15486af, 0x7c72e993, 0xb3ee1411, 0x636fbc2a, 0x2ba9c55d, 0x741831f6, 0xce5c3e16, 0x9b87931e, 0xafd6ba33, 0x6c24cf5c, 0x7a325381, + 0x28958677, 0x3b8f4898, 0x6b4bb9af, 0xc4bfe81b, 0x66282193, 0x61d809cc, 0xfb21a991, 0x487cac60, 0x5dec8032, 0xef845d5d, 0xe98575b1, + 0xdc262302, 0xeb651b88, 0x23893e81, 0xd396acc5, 0x0f6d6ff3, 0x83f44239, 0x2e0b4482, 0xa4842004, 0x69c8f04a, 0x9e1f9b5e, 0x21c66842, + 0xf6e96c9a, 0x670c9c61, 0xabd388f0, 0x6a51a0d2, 0xd8542f68, 0x960fa728, 0xab5133a3, 0x6eef0b6c, 0x137a3be4, 0xba3bf050, 0x7efb2a98, + 0xa1f1651d, 0x39af0176, 0x66ca593e, 0x82430e88, 0x8cee8619, 0x456f9fb4, 0x7d84a5c3, 0x3b8b5ebe, 0xe06f75d8, 0x85c12073, 0x401a449f, + 0x56c16aa6, 0x4ed3aa62, 0x363f7706, 0x1bfedf72, 0x429b023d, 0x37d0d724, 0xd00a1248, 0xdb0fead3, 0x49f1c09b, 0x075372c9, 0x80991b7b, + 0x25d479d8, 0xf6e8def7, 0xe3fe501a, 0xb6794c3b, 0x976ce0bd, 0x04c006ba, 0xc1a94fb6, 0x409f60c4, 0x5e5c9ec2, 0x196a2463, 0x68fb6faf, + 0x3e6c53b5, 0x1339b2eb, 0x3b52ec6f, 0x6dfc511f, 0x9b30952c, 0xcc814544, 0xaf5ebd09, 0xbee3d004, 0xde334afd, 0x660f2807, 0x192e4bb3, + 0xc0cba857, 0x45c8740f, 0xd20b5f39, 0xb9d3fbdb, 0x5579c0bd, 0x1a60320a, 0xd6a100c6, 0x402c7279, 0x679f25fe, 0xfb1fa3cc, 0x8ea5e9f8, + 0xdb3222f8, 0x3c7516df, 0xfd616b15, 0x2f501ec8, 0xad0552ab, 0x323db5fa, 0xfd238760, 0x53317b48, 0x3e00df82, 0x9e5c57bb, 0xca6f8ca0, + 0x1a87562e, 0xdf1769db, 0xd542a8f6, 0x287effc3, 0xac6732c6, 0x8c4f5573, 0x695b27b0, 0xbbca58c8, 0xe1ffa35d, 0xb8f011a0, 0x10fa3d98, + 0xfd2183b8, 0x4afcb56c, 0x2dd1d35b, 0x9a53e479, 0xb6f84565, 0xd28e49bc, 0x4bfb9790, 0xe1ddf2da, 0xa4cb7e33, 0x62fb1341, 0xcee4c6e8, + 0xef20cada, 0x36774c01, 0xd07e9efe, 0x2bf11fb4, 0x95dbda4d, 0xae909198, 0xeaad8e71, 0x6b93d5a0, 0xd08ed1d0, 0xafc725e0, 0x8e3c5b2f, + 0x8e7594b7, 0x8ff6e2fb, 0xf2122b64, 0x8888b812, 0x900df01c, 0x4fad5ea0, 0x688fc31c, 0xd1cff191, 0xb3a8c1ad, 0x2f2f2218, 0xbe0e1777, + 0xea752dfe, 0x8b021fa1, 0xe5a0cc0f, 0xb56f74e8, 0x18acf3d6, 0xce89e299, 0xb4a84fe0, 0xfd13e0b7, 0x7cc43b81, 0xd2ada8d9, 0x165fa266, + 0x80957705, 0x93cc7314, 0x211a1477, 0xe6ad2065, 0x77b5fa86, 0xc75442f5, 0xfb9d35cf, 0xebcdaf0c, 0x7b3e89a0, 0xd6411bd3, 0xae1e7e49, + 0x00250e2d, 0x2071b35e, 0x226800bb, 0x57b8e0af, 0x2464369b, 0xf009b91e, 0x5563911d, 0x59dfa6aa, 0x78c14389, 0xd95a537f, 0x207d5ba2, + 0x02e5b9c5, 0x83260376, 0x6295cfa9, 0x11c81968, 0x4e734a41, 0xb3472dca, 0x7b14a94a, 0x1b510052, 0x9a532915, 0xd60f573f, 0xbc9bc6e4, + 0x2b60a476, 0x81e67400, 0x08ba6fb5, 0x571be91f, 0xf296ec6b, 0x2a0dd915, 0xb6636521, 0xe7b9f9b6, 0xff34052e, 0xc5855664, 0x53b02d5d, + 0xa99f8fa1, 0x08ba4799, 0x6e85076a + ] + + self.olkemfjq = [ + 0x243f6a88, 0x85a308d3, 0x13198a2e, 0x03707344, 0xa4093822, 0x299f31d0, 0x082efa98, 0xec4e6c89, 0x452821e6, 0x38d01377, 0xbe5466cf, + 0x34e90c6c, 0xc0ac29b7, 0xc97c50dd, 0x3f84d5b5, 0xb5470917, 0x9216d5d9, 0x8979fb1b + ] + + self.oqlaoymh = 0 + self.oqmykrna = 0 + self.pqmyzkid = 0 + self.pldmjnde = 0 + self.ldiwkqly = 0 + + self.plkodnyq = [ + 0x3a39ce37, 0xd3faf5cf, 0xabc27737, 0x5ac52d1b, 0x5cb0679e, 0x4fa33742, 0xd3822740, 0x99bc9bbe, 0xd5118e9d, 0xbf0f7315, 0xd62d1c7e, + 0xc700c47b, 0xb78c1b6b, 0x21a19045, 0xb26eb1be, 0x6a366eb4, 0x5748ab2f, 0xbc946e79, 0xc6a376d2, 0x6549c2c8, 0x530ff8ee, 0x468dde7d, + 0xd5730a1d, 0x4cd04dc6, 0x2939bbdb, 0xa9ba4650, 0xac9526e8, 0xbe5ee304, 0xa1fad5f0, 0x6a2d519a, 0x63ef8ce2, 0x9a86ee22, 0xc089c2b8, + 0x43242ef6, 0xa51e03aa, 0x9cf2d0a4, 0x83c061ba, 0x9be96a4d, 0x8fe51550, 0xba645bd6, 0x2826a2f9, 0xa73a3ae1, 0x4ba99586, 0xef5562e9, + 0xc72fefd3, 0xf752f7da, 0x3f046f69, 0x77fa0a59, 0x80e4a915, 0x87b08601, 0x9b09e6ad, 0x3b3ee593, 0xe990fd5a, 0x9e34d797, 0x2cf0b7d9, + 0x022b8b51, 0x96d5ac3a, 0x017da67d, 0xd1cf3ed6, 0x7c7d2d28, 0x1f9f25cf, 0xadf2b89b, 0x5ad6b472, 0x5a88f54c, 0xe029ac71, 0xe019a5e6, + 0x47b0acfd, 0xed93fa9b, 0xe8d3c48d, 0x283b57cc, 0xf8d56629, 0x79132e28, 0x785f0191, 0xed756055, 0xf7960e44, 0xe3d35e8c, 0x15056dd4, + 0x88f46dba, 0x03a16125, 0x0564f0bd, 0xc3eb9e15, 0x3c9057a2, 0x97271aec, 0xa93a072a, 0x1b3f6d9b, 0x1e6321f5, 0xf59c66fb, 0x26dcf319, + 0x7533d928, 0xb155fdf5, 0x03563482, 0x8aba3cbb, 0x28517711, 0xc20ad9f8, 0xabcc5167, 0xccad925f, 0x4de81751, 0x3830dc8e, 0x379d5862, + 0x9320f991, 0xea7a90c2, 0xfb3e7bce, 0x5121ce64, 0x774fbe32, 0xa8b6e37e, 0xc3293d46, 0x48de5369, 0x6413e680, 0xa2ae0810, 0xdd6db224, + 0x69852dfd, 0x09072166, 0xb39a460a, 0x6445c0dd, 0x586cdecf, 0x1c20c8ae, 0x5bbef7dd, 0x1b588d40, 0xccd2017f, 0x6bb4e3bb, 0xdda26a7e, + 0x3a59ff45, 0x3e350a44, 0xbcb4cdd5, 0x72eacea8, 0xfa6484bb, 0x8d6612ae, 0xbf3c6f47, 0xd29be463, 0x542f5d9e, 0xaec2771b, 0xf64e6370, + 0x740e0d8d, 0xe75b1357, 0xf8721671, 0xaf537d5d, 0x4040cb08, 0x4eb4e2cc, 0x34d2466a, 0x0115af84, 0xe1b00428, 0x95983a1d, 0x06b89fb4, + 0xce6ea048, 0x6f3f3b82, 0x3520ab82, 0x011a1d4b, 0x277227f8, 0x611560b1, 0xe7933fdc, 0xbb3a792b, 0x344525bd, 0xa08839e1, 0x51ce794b, + 0x2f32c9b7, 0xa01fbac9, 0xe01cc87e, 0xbcc7d1f6, 0xcf0111c3, 0xa1e8aac7, 0x1a908749, 0xd44fbd9a, 0xd0dadecb, 0xd50ada38, 0x0339c32a, + 0xc6913667, 0x8df9317c, 0xe0b12b4f, 0xf79e59b7, 0x43f5bb3a, 0xf2d519ff, 0x27d9459c, 0xbf97222c, 0x15e6fc2a, 0x0f91fc71, 0x9b941525, + 0xfae59361, 0xceb69ceb, 0xc2a86459, 0x12baa8d1, 0xb6c1075e, 0xe3056a0c, 0x10d25065, 0xcb03a442, 0xe0ec6e0e, 0x1698db3b, 0x4c98a0be, + 0x3278e964, 0x9f1f9532, 0xe0d392df, 0xd3a0342b, 0x8971f21e, 0x1b0a7441, 0x4ba3348c, 0xc5be7120, 0xc37632d8, 0xdf359f8d, 0x9b992f2e, + 0xe60b6f47, 0x0fe3f11d, 0xe54cda54, 0x1edad891, 0xce6279cf, 0xcd3e7e6f, 0x1618b166, 0xfd2c1d05, 0x848fd2c5, 0xf6fb2299, 0xf523f357, + 0xa6327623, 0x93a83531, 0x56cccd02, 0xacf08162, 0x5a75ebb5, 0x6e163697, 0x88d273cc, 0xde966292, 0x81b949d0, 0x4c50901b, 0x71c65614, + 0xe6c6c7bd, 0x327a140a, 0x45e1d006, 0xc3f27b9a, 0xc9aa53fd, 0x62a80f00, 0xbb25bfe2, 0x35bdd2f6, 0x71126905, 0xb2040222, 0xb6cbcf7c, + 0xcd769c2b, 0x53113ec0, 0x1640e3d3, 0x38abbd60, 0x2547adf0, 0xba38209c, 0xf746ce76, 0x77afa1c5, 0x20756060, 0x85cbfe4e, 0x8ae88dd8, + 0x7aaaf9b0, 0x4cf9aa7e, 0x1948c25c, 0x02fb8a8c, 0x01c36ae4, 0xd6ebe1f9, 0x90d4f869, 0xa65cdea0, 0x3f09252d, 0xc208e69f, 0xb74e6132, + 0xce77e25b, 0x578fdfe3, 0x3ac372e6 + ] + + self.pnjzokye = None + + self.thdlpsmy = [ + 0xe93d5a68, 0x948140f7, 0xf64c261c, 0x94692934, 0x411520f7, 0x7602d4f7, 0xbcf46b2e, 0xd4a20068, 0xd4082471, 0x3320f46a, 0x43b7d4b7, + 0x500061af, 0x1e39f62e, 0x97244546, 0x14214f74, 0xbf8b8840, 0x4d95fc1d, 0x96b591af, 0x70f4ddd3, 0x66a02f45, 0xbfbc09ec, 0x03bd9785, + 0x7fac6dd0, 0x31cb8504, 0x96eb27b3, 0x55fd3941, 0xda2547e6, 0xabca0a9a, 0x28507825, 0x530429f4, 0x0a2c86da, 0xe9b66dfb, 0x68dc1462, + 0xd7486900, 0x680ec0a4, 0x27a18dee, 0x4f3ffea2, 0xe887ad8c, 0xb58ce006, 0x7af4d6b6, 0xaace1e7c, 0xd3375fec, 0xce78a399, 0x406b2a42, + 0x20fe9e35, 0xd9f385b9, 0xee39d7ab, 0x3b124e8b, 0x1dc9faf7, 0x4b6d1856, 0x26a36631, 0xeae397b2, 0x3a6efa74, 0xdd5b4332, 0x6841e7f7, + 0xca7820fb, 0xfb0af54e, 0xd8feb397, 0x454056ac, 0xba489527, 0x55533a3a, 0x20838d87, 0xfe6ba9b7, 0xd096954b, 0x55a867bc, 0xa1159a58, + 0xcca92963, 0x99e1db33, 0xa62a4a56, 0x3f3125f9, 0x5ef47e1c, 0x9029317c, 0xfdf8e802, 0x04272f70, 0x80bb155c, 0x05282ce3, 0x95c11548, + 0xe4c66d22, 0x48c1133f, 0xc70f86dc, 0x07f9c9ee, 0x41041f0f, 0x404779a4, 0x5d886e17, 0x325f51eb, 0xd59bc0d1, 0xf2bcc18f, 0x41113564, + 0x257b7834, 0x602a9c60, 0xdff8e8a3, 0x1f636c1b, 0x0e12b4c2, 0x02e1329e, 0xaf664fd1, 0xcad18115, 0x6b2395e0, 0x333e92e1, 0x3b240b62, + 0xeebeb922, 0x85b2a20e, 0xe6ba0d99, 0xde720c8c, 0x2da2f728, 0xd0127845, 0x95b794fd, 0x647d0862, 0xe7ccf5f0, 0x5449a36f, 0x877d48fa, + 0xc39dfd27, 0xf33e8d1e, 0x0a476341, 0x992eff74, 0x3a6f6eab, 0xf4f8fd37, 0xa812dc60, 0xa1ebddf8, 0x991be14c, 0xdb6e6b0d, 0xc67b5510, + 0x6d672c37, 0x2765d43b, 0xdcd0e804, 0xf1290dc7, 0xcc00ffa3, 0xb5390f92, 0x690fed0b, 0x667b9ffb, 0xcedb7d9c, 0xa091cf0b, 0xd9155ea3, + 0xbb132f88, 0x515bad24, 0x7b9479bf, 0x763bd6eb, 0x37392eb3, 0xcc115979, 0x8026e297, 0xf42e312d, 0x6842ada7, 0xc66a2b3b, 0x12754ccc, + 0x782ef11c, 0x6a124237, 0xb79251e7, 0x06a1bbe6, 0x4bfb6350, 0x1a6b1018, 0x11caedfa, 0x3d25bdd8, 0xe2e1c3c9, 0x44421659, 0x0a121386, + 0xd90cec6e, 0xd5abea2a, 0x64af674e, 0xda86a85f, 0xbebfe988, 0x64e4c3fe, 0x9dbc8057, 0xf0f7c086, 0x60787bf8, 0x6003604d, 0xd1fd8346, + 0xf6381fb0, 0x7745ae04, 0xd736fccc, 0x83426b33, 0xf01eab71, 0xb0804187, 0x3c005e5f, 0x77a057be, 0xbde8ae24, 0x55464299, 0xbf582e61, + 0x4e58f48f, 0xf2ddfda2, 0xf474ef38, 0x8789bdc2, 0x5366f9c3, 0xc8b38e74, 0xb475f255, 0x46fcd9b9, 0x7aeb2661, 0x8b1ddf84, 0x846a0e79, + 0x915f95e2, 0x466e598e, 0x20b45770, 0x8cd55591, 0xc902de4c, 0xb90bace1, 0xbb8205d0, 0x11a86248, 0x7574a99e, 0xb77f19b6, 0xe0a9dc09, + 0x662d09a1, 0xc4324633, 0xe85a1f02, 0x09f0be8c, 0x4a99a025, 0x1d6efe10, 0x1ab93d1d, 0x0ba5a4df, 0xa186f20f, 0x2868f169, 0xdcb7da83, + 0x573906fe, 0xa1e2ce9b, 0x4fcd7f52, 0x50115e01, 0xa70683fa, 0xa002b5c4, 0x0de6d027, 0x9af88c27, 0x773f8641, 0xc3604c06, 0x61a806b5, + 0xf0177a28, 0xc0f586e0, 0x006058aa, 0x30dc7d62, 0x11e69ed7, 0x2338ea63, 0x53c2dd94, 0xc2c21634, 0xbbcbee56, 0x90bcb6de, 0xebfc7da1, + 0xce591d76, 0x6f05e409, 0x4b7c0188, 0x39720a3d, 0x7c927c24, 0x86e3725f, 0x724d9db9, 0x1ac15bb4, 0xd39eb8fc, 0xed545578, 0x08fca5b5, + 0xd83d7cd3, 0x4dad0fc4, 0x1e50ef5e, 0xb161e6f8, 0xa28514d9, 0x6c51133c, 0x6fd5c7e7, 0x56e14ec4, 0x362abfce, 0xddc6c837, 0xd79a3234, + 0x92638212, 0x670efa8e, 0x406000e0 + ] + + self.ybghjtik = [ + 0x4b7a70e9, 0xb5b32944, 0xdb75092e, 0xc4192623, 0xad6ea6b0, 0x49a7df7d, 0x9cee60b8, 0x8fedb266, 0xecaa8c71, 0x699a17ff, 0x5664526c, + 0xc2b19ee1, 0x193602a5, 0x75094c29, 0xa0591340, 0xe4183a3e, 0x3f54989a, 0x5b429d65, 0x6b8fe4d6, 0x99f73fd6, 0xa1d29c07, 0xefe830f5, + 0x4d2d38e6, 0xf0255dc1, 0x4cdd2086, 0x8470eb26, 0x6382e9c6, 0x021ecc5e, 0x09686b3f, 0x3ebaefc9, 0x3c971814, 0x6b6a70a1, 0x687f3584, + 0x52a0e286, 0xb79c5305, 0xaa500737, 0x3e07841c, 0x7fdeae5c, 0x8e7d44ec, 0x5716f2b8, 0xb03ada37, 0xf0500c0d, 0xf01c1f04, 0x0200b3ff, + 0xae0cf51a, 0x3cb574b2, 0x25837a58, 0xdc0921bd, 0xd19113f9, 0x7ca92ff6, 0x94324773, 0x22f54701, 0x3ae5e581, 0x37c2dadc, 0xc8b57634, + 0x9af3dda7, 0xa9446146, 0x0fd0030e, 0xecc8c73e, 0xa4751e41, 0xe238cd99, 0x3bea0e2f, 0x3280bba1, 0x183eb331, 0x4e548b38, 0x4f6db908, + 0x6f420d03, 0xf60a04bf, 0x2cb81290, 0x24977c79, 0x5679b072, 0xbcaf89af, 0xde9a771f, 0xd9930810, 0xb38bae12, 0xdccf3f2e, 0x5512721f, + 0x2e6b7124, 0x501adde6, 0x9f84cd87, 0x7a584718, 0x7408da17, 0xbc9f9abc, 0xe94b7d8c, 0xec7aec3a, 0xdb851dfa, 0x63094366, 0xc464c3d2, + 0xef1c1847, 0x3215d908, 0xdd433b37, 0x24c2ba16, 0x12a14d43, 0x2a65c451, 0x50940002, 0x133ae4dd, 0x71dff89e, 0x10314e55, 0x81ac77d6, + 0x5f11199b, 0x043556f1, 0xd7a3c76b, 0x3c11183b, 0x5924a509, 0xf28fe6ed, 0x97f1fbfa, 0x9ebabf2c, 0x1e153c6e, 0x86e34570, 0xeae96fb1, + 0x860e5e0a, 0x5a3e2ab3, 0x771fe71c, 0x4e3d06fa, 0x2965dcb9, 0x99e71d0f, 0x803e89d6, 0x5266c825, 0x2e4cc978, 0x9c10b36a, 0xc6150eba, + 0x94e2ea78, 0xa5fc3c53, 0x1e0a2df4, 0xf2f74ea7, 0x361d2b3d, 0x1939260f, 0x19c27960, 0x5223a708, 0xf71312b6, 0xebadfe6e, 0xeac31f66, + 0xe3bc4595, 0xa67bc883, 0xb17f37d1, 0x018cff28, 0xc332ddef, 0xbe6c5aa5, 0x65582185, 0x68ab9802, 0xeecea50f, 0xdb2f953b, 0x2aef7dad, + 0x5b6e2f84, 0x1521b628, 0x29076170, 0xecdd4775, 0x619f1510, 0x13cca830, 0xeb61bd96, 0x0334fe1e, 0xaa0363cf, 0xb5735c90, 0x4c70a239, + 0xd59e9e0b, 0xcbaade14, 0xeecc86bc, 0x60622ca7, 0x9cab5cab, 0xb2f3846e, 0x648b1eaf, 0x19bdf0ca, 0xa02369b9, 0x655abb50, 0x40685a32, + 0x3c2ab4b3, 0x319ee9d5, 0xc021b8f7, 0x9b540b19, 0x875fa099, 0x95f7997e, 0x623d7da8, 0xf837889a, 0x97e32d77, 0x11ed935f, 0x16681281, + 0x0e358829, 0xc7e61fd6, 0x96dedfa1, 0x7858ba99, 0x57f584a5, 0x1b227263, 0x9b83c3ff, 0x1ac24696, 0xcdb30aeb, 0x532e3054, 0x8fd948e4, + 0x6dbc3128, 0x58ebf2ef, 0x34c6ffea, 0xfe28ed61, 0xee7c3c73, 0x5d4a14d9, 0xe864b7e3, 0x42105d14, 0x203e13e0, 0x45eee2b6, 0xa3aaabea, + 0xdb6c4f15, 0xfacb4fd0, 0xc742f442, 0xef6abbb5, 0x654f3b1d, 0x41cd2105, 0xd81e799e, 0x86854dc7, 0xe44b476a, 0x3d816250, 0xcf62a1f2, + 0x5b8d2646, 0xfc8883a0, 0xc1c7b6a3, 0x7f1524c3, 0x69cb7492, 0x47848a0b, 0x5692b285, 0x095bbf00, 0xad19489d, 0x1462b174, 0x23820e00, + 0x58428d2a, 0x0c55f5ea, 0x1dadf43e, 0x233f7061, 0x3372f092, 0x8d937e41, 0xd65fecf1, 0x6c223bdb, 0x7cde3759, 0xcbee7460, 0x4085f2a7, + 0xce77326e, 0xa6078084, 0x19f8509e, 0xe8efd855, 0x61d99735, 0xa969a7aa, 0xc50c06c2, 0x5a04abfc, 0x800bcadc, 0x9e447a2e, 0xc3453484, + 0xfdd56705, 0x0e1e9ec9, 0xdb73dbd3, 0x105588cd, 0x675fda79, 0xe3674340, 0xc5c43465, 0x713e38d8, 0x3d28f89e, 0xf16dff20, 0x153e21e7, + 0x8fb03d4a, 0xe6e39f2b, 0xdb83adf7 + ] + + def cypher(self, code): + return self.lskdqpyr(code, "") + + def lskdqpyr(self, alokfmth, yoaksjdh): + if self.pnjzokye is None or self.pnjzokye.lower() == yoaksjdh: + self.yoliukev(yoaksjdh) + self.pnjzokye = yoaksjdh + alokfmth = self.plaomtje(alokfmth) + ykaiumgp = "" + alokijuh = len(alokfmth) + lokimyas = self.ylomiktb(alokfmth[0:8]) + palsiuzt = lokimyas[0] + tzghbndf = lokimyas[1] + awsedrft = [None, None] + for kiujzhqa in range(8, alokijuh, 8): + lokimyas = self.ylomiktb(alokfmth[kiujzhqa:kiujzhqa+8]) + awsedrft[0] = lokimyas[0] + awsedrft[1] = lokimyas[1] + lokimyas = self.okaqnhlp(lokimyas[0], lokimyas[1]) + lokimyas[0] ^= palsiuzt + lokimyas[1] ^= tzghbndf + palsiuzt = awsedrft[0] + tzghbndf = awsedrft[1] + ykaiumgp += self.ykijmtkd(lokimyas) + return ykaiumgp + + def okaqnhlp(self, lahgrnvp, trenlpys): + ujhaqylw = 0 + for yalmhopr in range(17, 1, -1): + lahgrnvp ^= self.ldiwkqly[yalmhopr] + trenlpys ^= (self.oqlaoymh[lahgrnvp >> 24 & 0xff] + self.oqmykrna[lahgrnvp >> 16 & 0xff] ^ self.pqmyzkid[lahgrnvp >> 8 & 0xff]) + self.pldmjnde[lahgrnvp & 0xff] + ujhaqylw = lahgrnvp + lahgrnvp = trenlpys + trenlpys = ujhaqylw + ujhaqylw = lahgrnvp + lahgrnvp = trenlpys + trenlpys = ujhaqylw + trenlpys ^= self.ldiwkqly[1] + lahgrnvp ^= self.ldiwkqly[0] + return [lahgrnvp, trenlpys] + + def plaomtje(self, yoiumqpy): + qkailkzt = "" + xoliuzem = 0 + lyomiujt = 0 + yploemju = -1 + for i in range(0, len(yoiumqpy)): + yploamzu = ord(yoiumqpy[i]) + if ord('A') <= yploamzu <= ord('Z'): + xoliuzem = ord(yoiumqpy[i]) - 65 + elif ord('a') <= yploamzu <= ord('z'): + xoliuzem = ord(yoiumqpy[i]) - 97 + 26 + elif ord('0') <= yploamzu <= ord('9'): + xoliuzem = ord(yoiumqpy[i]) - 48 + 52 + elif yploamzu == ord('+'): + xoliuzem = 62 + elif yploamzu == ord('/'): + xoliuzem = 63 + else: + continue + yploemju += 1 + + lxkdmizj = 0 + switch = yploemju % 4 + if switch == 0: + lyomiujt = xoliuzem + continue + elif switch == 1: + lxkdmizj = lyomiujt << 2 | xoliuzem >> 4 + lyomiujt = xoliuzem & 0x0F + elif switch == 2: + lxkdmizj = lyomiujt << 4 | xoliuzem >> 2 + lyomiujt = xoliuzem & 0x03 + elif switch == 3: + lxkdmizj = lyomiujt << 6 | xoliuzem >> 0 + lyomiujt = xoliuzem & 0x00 + qkailkzt += unichr(lxkdmizj) + return qkailkzt + + def qmyjuila(self, oqlamykt, yalkionj): + dolizmvw = 0 + for iumswkya in range(0, 16): + oqlamykt ^= self.ldiwkqly[iumswkya] + yalkionj ^= (self.oqlaoymh[oqlamykt >> 24 & 0xff] + self.oqmykrna[oqlamykt >> 16 & 0xff] ^ self.pqmyzkid[oqlamykt >> 8 & 0xff]) + self.pldmjnde[oqlamykt & 0xff] + dolizmvw = oqlamykt + oqlamykt = yalkionj + yalkionj = dolizmvw + dolizmvw = oqlamykt + oqlamykt = yalkionj + yalkionj = dolizmvw + yalkionj ^= self.ldiwkqly[16] + oqlamykt ^= self.ldiwkqly[17] + return [oqlamykt, yalkionj] + + def ykijmtkd(self, yoirlkqw): + loipamyu = len(yoirlkqw) + yoirlkqwchar = [] + for ymujtnbq in range(0, loipamyu): + yoir = [yoirlkqw[ymujtnbq] >> 24 & 0xff, yoirlkqw[ymujtnbq] >> 16 & 0xff, yoirlkqw[ymujtnbq] >> 8 & 0xff, yoirlkqw[ymujtnbq] & 0xff] + for c in yoir: + yoirlkqwchar.append(chr(c)) + return "".join(yoirlkqwchar) + + def ylomiktb(self, lofiuzmq): + plokimqw = int(ceil(len(lofiuzmq) / 4.0)) + lopkisdq = [] + for ypoqlktz in range(0, plokimqw): + lopkisdq.append(ord(lofiuzmq[(ypoqlktz << 2) + 3]) + (ord(lofiuzmq[(ypoqlktz << 2) + 2]) << 8) + (ord(lofiuzmq[(ypoqlktz << 2) + 1]) << 16) + (ord(lofiuzmq[(ypoqlktz << 2)]) << 24)) + return lopkisdq + + def yoliukev(self, kaiumylq): + self.oqlaoymh = self.iatwbfrd + self.oqmykrna = self.ybghjtik + self.pqmyzkid = self.thdlpsmy + self.pldmjnde = self.plkodnyq + + yaqpolft = [0 for i in range(len(kaiumylq))] + + yaqwsedr = 0 + btzqwsay = 0 + while yaqwsedr < len(kaiumylq): + wlqoakmy = 0 + for lopiuztr in range(0, 4): + wlqoakmy = wlqoakmy << 8 | ord(kaiumylq[yaqwsedr % len(kaiumylq)]) + yaqwsedr += 1 + yaqpolft[btzqwsay] = wlqoakmy + btzqwsay += 1 + self.ldiwkqly = [] + for btzqwsay in range(0, 18): + self.ldiwkqly.append(self.olkemfjq[btzqwsay]) + yalopiuq = [0, 0] + for btzqwsay in range(0, 18, 2): + yalopiuq = self.qmyjuila(yalopiuq[0], yalopiuq[1]) + self.ldiwkqly[btzqwsay] = yalopiuq[0] + self.ldiwkqly[btzqwsay + 1] = yalopiuq[1] + for btzqwsay in range(0, 256, 2): + yalopiuq = self.qmyjuila(yalopiuq[0], yalopiuq[1]) + self.oqlaoymh[btzqwsay] = yalopiuq[0] + self.oqlaoymh[btzqwsay + 1] = yalopiuq[1] + for btzqwsay in range(0, 256, 2): + yalopiuq = self.qmyjuila(yalopiuq[0], yalopiuq[1]) + self.oqmykrna[btzqwsay] = yalopiuq[0] + self.oqmykrna[btzqwsay + 1] = yalopiuq[1] + for btzqwsay in range(0, 256, 2): + yalopiuq = self.qmyjuila(yalopiuq[0], yalopiuq[1]) + self.pqmyzkid[btzqwsay] = yalopiuq[0] + self.pqmyzkid[btzqwsay + 1] = yalopiuq[1] + for btzqwsay in range(0, 256, 2): + yalopiuq = self.qmyjuila(yalopiuq[0], yalopiuq[1]) + self.pldmjnde[btzqwsay] = yalopiuq[0] + self.pldmjnde[btzqwsay + 1] = yalopiuq[1] + diff --git a/pyload/plugins/crypter/SerienjunkiesOrg.py b/pyload/plugins/crypter/SerienjunkiesOrg.py new file mode 100644 index 000000000..3fcc12e36 --- /dev/null +++ b/pyload/plugins/crypter/SerienjunkiesOrg.py @@ -0,0 +1,321 @@ +# -*- coding: utf-8 -*- + +import re +from time import sleep +import random +from module.plugins.Crypter import Crypter +from module.lib.BeautifulSoup import BeautifulSoup +from module.unescape import unescape + +class SerienjunkiesOrg(Crypter): + __name__ = "SerienjunkiesOrg" + __type__ = "container" + __pattern__ = r"http://.*?(serienjunkies.org|dokujunkies.org)/.*?" + __version__ = "0.38" + __config__ = [ + ("changeNameSJ", "Packagename;Show;Season;Format;Episode", "Take SJ.org name", "Show"), + ("changeNameDJ", "Packagename;Show;Format;Episode", "Take DJ.org name", "Show"), + ("randomPreferred", "bool", "Randomize Preferred-List", False), + ("hosterListMode", "OnlyOne;OnlyPreferred(One);OnlyPreferred(All);All", "Use for hosters (if supported)", "All"), + ("hosterList", "str", "Preferred Hoster list (comma separated)", "RapidshareCom,UploadedTo,NetloadIn,FilefactoryCom,FreakshareNet,FilebaseTo,HotfileCom,DepositfilesCom,EasyshareCom,KickloadCom"), + ("ignoreList", "str", "Ignored Hoster list (comma separated)", "MegauploadCom") + ] + __description__ = """serienjunkies.org Container Plugin""" + __author_name__ = ("mkaay", "godofdream") + __author_mail__ = ("mkaay@mkaay.de", "soilfiction@gmail.com") + + + def setup(self): + self.multiDL = False + + def getSJSrc(self, url): + src = self.req.load(str(url)) + if "This website is not available in your country" in src: + self.fail("Not available in your country") + if not src.find("Enter Serienjunkies") == -1: + sleep(1) + src = self.req.load(str(url)) + return src + + def handleShow(self, url): + src = self.getSJSrc(url) + soup = BeautifulSoup(src) + packageName = self.pyfile.package().name + if self.getConfig("changeNameSJ") == "Show": + found = unescape(soup.find("h2").find("a").string.split(' –')[0]) + if found: + packageName = found + + nav = soup.find("div", attrs={"id": "scb"}) + + package_links = [] + for a in nav.findAll("a"): + if self.getConfig("changeNameSJ") == "Show": + package_links.append(a["href"]) + else: + package_links.append(a["href"] + "#hasName") + if self.getConfig("changeNameSJ") == "Show": + self.packages.append((packageName, package_links, packageName)) + else: + self.core.files.addLinks(package_links, self.pyfile.package().id) + + + def handleSeason(self, url): + src = self.getSJSrc(url) + soup = BeautifulSoup(src) + post = soup.find("div", attrs={"class": "post-content"}) + ps = post.findAll("p") + + seasonName = unescape(soup.find("a", attrs={"rel": "bookmark"}).string).replace("–", "-") + groups = {} + gid = -1 + for p in ps: + if re.search("Sprache|Format", str(p)): + var = p.findAll("strong") + opts = {"Sprache": "", "Format": ""} + for v in var: + n = unescape(v.string).strip() + n = re.sub(r"^([:]?)(.*?)([:]?)$", r'\2', n) + if n.strip() not in opts: + continue + val = v.nextSibling + if not val: + continue + val = val.replace("|", "").strip() + val = re.sub(r"^([:]?)(.*?)([:]?)$", r'\2', val) + opts[n.strip()] = val.strip() + gid += 1 + groups[gid] = {} + groups[gid]["ep"] = {} + groups[gid]["opts"] = opts + elif re.search("Download:", str(p)): + parts = str(p).split("
        ") + if re.search("", parts[0]): + ename = re.search('(.*?)',parts[0]).group(1).strip().decode("utf-8").replace("–", "-") + groups[gid]["ep"][ename] = {} + parts.remove(parts[0]) + for part in parts: + hostername = re.search(" \| ([-a-zA-Z0-9]+\.\w+)",part) + if hostername: + hostername = hostername.group(1) + groups[gid]["ep"][ename][hostername] = [] + links = re.findall('href="(.*?)"',part) + for link in links: + groups[gid]["ep"][ename][hostername].append(link + "#hasName") + + links = [] + for g in groups.values(): + for ename in g["ep"]: + links.extend(self.getpreferred(g["ep"][ename])) + if self.getConfig("changeNameSJ") == "Episode": + self.packages.append((ename, links, ename)) + links = [] + package = "%s (%s, %s)" % (seasonName, g["opts"]["Format"], g["opts"]["Sprache"]) + if self.getConfig("changeNameSJ") == "Format": + self.packages.append((package, links, package)) + links = [] + if (self.getConfig("changeNameSJ") == "Packagename") or re.search("#hasName", url): + self.core.files.addLinks(links, self.pyfile.package().id) + elif (self.getConfig("changeNameSJ") == "Season") or not re.search("#hasName", url): + self.packages.append((seasonName, links, seasonName)) + + def handleEpisode(self, url): + src = self.getSJSrc(url) + if not src.find( + "Du hast das Download-Limit überschritten! Bitte versuche es später nocheinmal.") == -1: + self.fail(_("Downloadlimit reached")) + else: + soup = BeautifulSoup(src) + form = soup.find("form") + h1 = soup.find("h1") + + if h1.get("class") == "wrap": + captchaTag = soup.find(attrs={"src": re.compile("^/secure/")}) + if not captchaTag: + sleep(5) + self.retry() + + captchaUrl = "http://download.serienjunkies.org" + captchaTag["src"] + result = self.decryptCaptcha(str(captchaUrl), imgtype="png") + sinp = form.find(attrs={"name": "s"}) + + self.req.lastURL = str(url) + sj = self.load(str(url), post={'s': sinp["value"], 'c': result, 'action': "Download"}) + + soup = BeautifulSoup(sj) + rawLinks = soup.findAll(attrs={"action": re.compile("^http://download.serienjunkies.org/")}) + + if not len(rawLinks) > 0: + sleep(1) + self.retry() + return + + self.correctCaptcha() + + links = [] + for link in rawLinks: + frameUrl = link["action"].replace("/go-", "/frame/go-") + links.append(self.handleFrame(frameUrl)) + if re.search("#hasName", url) or ((self.getConfig("changeNameSJ") == "Packagename") and (self.getConfig("changeNameDJ") == "Packagename")): + self.core.files.addLinks(links, self.pyfile.package().id) + else: + if h1.text[2] == "_": + eName = h1.text[3:] + else: + eName = h1.text + self.packages.append((eName, links, eName)) + + + def handleOldStyleLink(self, url): + sj = self.req.load(str(url)) + soup = BeautifulSoup(sj) + form = soup.find("form", attrs={"action": re.compile("^http://serienjunkies.org")}) + captchaTag = form.find(attrs={"src": re.compile("^/safe/secure/")}) + captchaUrl = "http://serienjunkies.org" + captchaTag["src"] + result = self.decryptCaptcha(str(captchaUrl)) + url = form["action"] + sinp = form.find(attrs={"name": "s"}) + + self.req.load(str(url), post={'s': sinp["value"], 'c': result, 'dl.start': "Download"}, cookies=False, + just_header=True) + decrypted = self.req.lastEffectiveURL + if decrypted == str(url): + self.retry() + self.core.files.addLinks([decrypted], self.pyfile.package().id) + + def handleFrame(self, url): + self.req.load(str(url)) + return self.req.lastEffectiveURL + + def handleShowDJ(self, url): + src = self.getSJSrc(url) + soup = BeautifulSoup(src) + post = soup.find("div", attrs={"id": "page_post"}) + ps = post.findAll("p") + found = unescape(soup.find("h2").find("a").string.split(' –')[0]) + if found: + seasonName = found + + groups = {} + gid = -1 + for p in ps: + if re.search("Sprache|Format", str(p)): + var = p.findAll("strong") + opts = {"Sprache": "", "Format": ""} + for v in var: + n = unescape(v.string).strip() + n = re.sub(r"^([:]?)(.*?)([:]?)$", r'\2', n) + if n.strip() not in opts: + continue + val = v.nextSibling + if not val: + continue + val = val.replace("|", "").strip() + val = re.sub(r"^([:]?)(.*?)([:]?)$", r'\2', val) + opts[n.strip()] = val.strip() + gid += 1 + groups[gid] = {} + groups[gid]["ep"] = {} + groups[gid]["opts"] = opts + elif re.search("Download:", str(p)): + parts = str(p).split("
        ") + if re.search("", parts[0]): + ename = re.search('(.*?)',parts[0]).group(1).strip().decode("utf-8").replace("–", "-") + groups[gid]["ep"][ename] = {} + parts.remove(parts[0]) + for part in parts: + hostername = re.search(" \| ([-a-zA-Z0-9]+\.\w+)",part) + if hostername: + hostername = hostername.group(1) + groups[gid]["ep"][ename][hostername] = [] + links = re.findall('href="(.*?)"',part) + for link in links: + groups[gid]["ep"][ename][hostername].append(link + "#hasName") + + links = [] + for g in groups.values(): + for ename in g["ep"]: + links.extend(self.getpreferred(g["ep"][ename])) + if self.getConfig("changeNameDJ") == "Episode": + self.packages.append((ename, links, ename)) + links = [] + package = "%s (%s, %s)" % (seasonName, g["opts"]["Format"], g["opts"]["Sprache"]) + if self.getConfig("changeNameDJ") == "Format": + self.packages.append((package, links, package)) + links = [] + if (self.getConfig("changeNameDJ") == "Packagename") or re.search("#hasName", url): + self.core.files.addLinks(links, self.pyfile.package().id) + elif (self.getConfig("changeNameDJ") == "Show") or not re.search("#hasName", url): + self.packages.append((seasonName, links, seasonName)) + + + + + + + + def handleCategoryDJ(self, url): + package_links = [] + src = self.getSJSrc(url) + soup = BeautifulSoup(src) + content = soup.find("div", attrs={"id": "content"}) + for a in content.findAll("a", attrs={"rel": "bookmark"}): + package_links.append(a["href"]) + self.core.files.addLinks(package_links, self.pyfile.package().id) + + def decrypt(self, pyfile): + showPattern = re.compile("^http://serienjunkies.org/serie/(.*)/$") + seasonPattern = re.compile("^http://serienjunkies.org/.*?/(.*)/$") + episodePattern = re.compile("^http://download.serienjunkies.org/f-.*?.html(#hasName)?$") + oldStyleLink = re.compile("^http://serienjunkies.org/safe/(.*)$") + categoryPatternDJ = re.compile("^http://dokujunkies.org/.*?(.*)$") + showPatternDJ = re.compile("^http://dokujunkies.org/.*?/(.*)\.html(#hasName)?$") + framePattern = re.compile("^http://download.(serienjunkies.org|dokujunkies.org)/frame/go-.*?/$") + url = pyfile.url + if framePattern.match(url): + self.packages.append((self.pyfile.package().name, [self.handleFrame(url)], self.pyfile.package().name)) + elif episodePattern.match(url): + self.handleEpisode(url) + elif oldStyleLink.match(url): + self.handleOldStyleLink(url) + elif showPattern.match(url): + self.handleShow(url) + elif showPatternDJ.match(url): + self.handleShowDJ(url) + elif seasonPattern.match(url): + self.handleSeason(url) + elif categoryPatternDJ.match(url): + self.handleCategoryDJ(url) + + #selects the preferred hoster, after that selects any hoster (ignoring the one to ignore) + def getpreferred(self, hosterlist): + + result = [] + preferredList = self.getConfig("hosterList").strip().lower().replace('|',',').replace('.','').replace(';',',').split(',') + if (self.getConfig("randomPreferred") == True) and (self.getConfig("hosterListMode") in ["OnlyOne","OnlyPreferred(One)"]) : + random.shuffle(preferredList) + # we don't want hosters be read two times + hosterlist2 = hosterlist.copy() + + for preferred in preferredList: + for Hoster in hosterlist: + if preferred == Hoster.lower().replace('.',''): + for Part in hosterlist[Hoster]: + self.logDebug("selected " + Part) + result.append(str(Part)) + del(hosterlist2[Hoster]) + if (self.getConfig("hosterListMode") in ["OnlyOne","OnlyPreferred(One)"]): + return result + + + ignorelist = self.getConfig("ignoreList").strip().lower().replace('|',',').replace('.','').replace(';',',').split(',') + if self.getConfig('hosterListMode') in ["OnlyOne","All"]: + for Hoster in hosterlist2: + if Hoster.strip().lower().replace('.','') not in ignorelist: + for Part in hosterlist2[Hoster]: + self.logDebug("selected2 " + Part) + result.append(str(Part)) + + if self.getConfig('hosterListMode') == "OnlyOne": + return result + return result diff --git a/pyload/plugins/crypter/ShareLinksBiz.py b/pyload/plugins/crypter/ShareLinksBiz.py new file mode 100644 index 000000000..b0e735896 --- /dev/null +++ b/pyload/plugins/crypter/ShareLinksBiz.py @@ -0,0 +1,269 @@ +# -*- coding: utf-8 -*- + +from Crypto.Cipher import AES +from module.plugins.Crypter import Crypter +from module.plugins.ReCaptcha import ReCaptcha +import base64 +import binascii +import re + + +class ShareLinksBiz(Crypter): + __name__ = "ShareLinksBiz" + __type__ = "crypter" + __pattern__ = r"(?Phttp://[\w\.]*?(share-links|s2l)\.biz)/(?P_?[0-9a-z]+)(/.*)?" + __version__ = "1.12" + __description__ = """Share-Links.biz Crypter""" + __author_name__ = ("fragonib") + __author_mail__ = ("fragonib[AT]yahoo[DOT]es") + + + def setup(self): + self.baseUrl = None + self.fileId = None + self.package = None + self.html = None + self.captcha = False + + def decrypt(self, pyfile): + + # Init + self.initFile(pyfile) + + # Request package + url = self.baseUrl + '/' + self.fileId + self.html = self.load(url, decode=True) + + # Unblock server (load all images) + self.unblockServer() + + # Check for protection + if self.isPasswordProtected(): + self.unlockPasswordProtection() + self.handleErrors() + + if self.isCaptchaProtected(): + self.captcha = True + self.unlockCaptchaProtection() + self.handleErrors() + + # Extract package links + package_links = [] + package_links.extend(self.handleWebLinks()) + package_links.extend(self.handleContainers()) + package_links.extend(self.handleCNL2()) + package_links = set(package_links) + + # Get package info + package_name, package_folder = self.getPackageInfo() + + # Pack + self.packages = [(package_name, package_links, package_folder)] + + def initFile(self, pyfile): + url = pyfile.url + if 's2l.biz' in url: + url = self.load(url, just_header=True)['location'] + self.baseUrl = re.search(self.__pattern__, url).group(1) + self.fileId = re.match(self.__pattern__, url).group('id') + self.package = pyfile.package() + + def isOnline(self): + if "No usable content was found" in self.html: + self.logDebug("File not found") + return False + return True + + def isPasswordProtected(self): + if re.search(r'''''', self.html): + self.logDebug("Links are protected") + return True + return False + + def isCaptchaProtected(self): + if '=x1 and x<=x2) and (y>=y1 and y<=y2): + return href + + def handleErrors(self): + if "The inserted password was wrong" in self.html: + self.logDebug("Incorrect password, please set right password on 'Edit package' form and retry") + self.fail("Incorrect password, please set right password on 'Edit package' form and retry") + + if self.captcha: + if "Your choice was wrong" in self.html: + self.logDebug("Invalid captcha, retrying") + self.invalidCaptcha() + self.setWait(5) + self.wait() + self.retry() + else: + self.correctCaptcha() + + def getPackageInfo(self): + name = folder = None + + # Extract from web package header + title_re = r'

        (.*)

        ' + m = re.search(title_re, self.html, re.DOTALL) + if m is not None: + title = m.group(1).strip() + if 'unnamed' not in title: + name = folder = title + self.logDebug("Found name [%s] and folder [%s] in package info" % (name, folder)) + + # Fallback to defaults + if not name or not folder: + name = self.package.name + folder = self.package.folder + self.logDebug("Package info not found, defaulting to pyfile name [%s] and folder [%s]" % (name, folder)) + + # Return package info + return name, folder + + def handleWebLinks(self): + package_links = [] + self.logDebug("Handling Web links") + + #@TODO: Gather paginated web links + pattern = r"javascript:_get\('(.*?)', \d+, ''\)" + ids = re.findall(pattern, self.html) + self.logDebug("Decrypting %d Web links" % len(ids)) + for i, id in enumerate(ids): + try: + self.logDebug("Decrypting Web link %d, [%s]" % (i+1, id)) + dwLink = self.baseUrl + "/get/lnk/" + id + response = self.load(dwLink) + code = re.search(r'frm/(\d+)', response).group(1) + fwLink = self.baseUrl + "/get/frm/" + code + response = self.load(fwLink) + jscode = re.search(r'', response, re.DOTALL).group(1) + jscode = self.js.eval("f = %s" % jscode) + jslauncher = "window=''; parent={frames:{Main:{location:{href:''}}},location:''}; %s; parent.frames.Main.location.href" + dlLink = self.js.eval(jslauncher % jscode) + self.logDebug("JsEngine returns value [%s] for redirection link" % dlLink) + package_links.append(dlLink) + except Exception, detail: + self.logDebug("Error decrypting Web link [%s], %s" % (id, detail)) + return package_links + + def handleContainers(self): + package_links = [] + self.logDebug("Handling Container links") + + pattern = r"javascript:_get\('(.*?)', 0, '(rsdf|ccf|dlc)'\)" + containersLinks = re.findall(pattern, self.html) + self.logDebug("Decrypting %d Container links" % len(containersLinks)) + for containerLink in containersLinks: + link = "%s/get/%s/%s" % (self.baseUrl, containerLink[1], containerLink[0]) + package_links.append(link) + return package_links + + def handleCNL2(self): + package_links = [] + self.logDebug("Handling CNL2 links") + + if '/lib/cnl2/ClicknLoad.swf' in self.html: + try: + (crypted, jk) = self._getCipherParams() + package_links.extend(self._getLinks(crypted, jk)) + except: + self.fail("Unable to decrypt CNL2 links") + return package_links + + def _getCipherParams(self): + + # Request CNL2 + code = re.search(r'ClicknLoad.swf\?code=(.*?)"', self.html).group(1) + url = "%s/get/cnl2/%s" % (self.baseUrl, code) + response = self.load(url) + params = response.split(";;") + + # Get jk + strlist = list(base64.standard_b64decode(params[1])) + strlist.reverse() + jk = ''.join(strlist) + + # Get crypted + strlist = list(base64.standard_b64decode(params[2])) + strlist.reverse() + crypted = ''.join(strlist) + + # Log and return + return crypted, jk + + def _getLinks(self, crypted, jk): + + # Get key + jreturn = self.js.eval("%s f()" % jk) + self.logDebug("JsEngine returns value [%s]" % jreturn) + key = binascii.unhexlify(jreturn) + + # Decode crypted + crypted = base64.standard_b64decode(crypted) + + # Decrypt + Key = key + IV = key + obj = AES.new(Key, AES.MODE_CBC, IV) + text = obj.decrypt(crypted) + + # Extract links + text = text.replace("\x00", "").replace("\r", "") + links = text.split("\n") + links = filter(lambda x: x != "", links) + + # Log and return + self.logDebug("Block has %d links" % len(links)) + return links \ No newline at end of file diff --git a/pyload/plugins/crypter/ShareRapidComFolder.py b/pyload/plugins/crypter/ShareRapidComFolder.py new file mode 100644 index 000000000..cb7f37525 --- /dev/null +++ b/pyload/plugins/crypter/ShareRapidComFolder.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- + +from module.plugins.internal.SimpleCrypter import SimpleCrypter + +class ShareRapidComFolder(SimpleCrypter): + __name__ = "ShareRapidComFolder" + __type__ = "crypter" + __pattern__ = r"http://(?:www\.)?((share(-?rapid\.(biz|com|cz|info|eu|net|org|pl|sk)|-(central|credit|free|net)\.cz|-ms\.net)|(s-?rapid|rapids)\.(cz|sk))|(e-stahuj|mediatack|premium-rapidshare|rapidshare-premium|qiuck)\.cz|kadzet\.com|stahuj-zdarma\.eu|strelci\.net|universal-share\.com)/(slozka/.+)" + __version__ = "0.01" + __description__ = """Share-Rapid.com Folder Plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + LINK_PATTERN = r']*>        ' \ No newline at end of file diff --git a/pyload/plugins/crypter/SpeedLoadOrgFolder.py b/pyload/plugins/crypter/SpeedLoadOrgFolder.py new file mode 100644 index 000000000..f85ede6f3 --- /dev/null +++ b/pyload/plugins/crypter/SpeedLoadOrgFolder.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- + +############################################################################ +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU Affero General Public License as # +# published by the Free Software Foundation, either version 3 of the # +# License, or (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU Affero General Public License for more details. # +# # +# You should have received a copy of the GNU Affero General Public License # +# along with this program. If not, see . # +############################################################################ + +from module.plugins.internal.SimpleCrypter import SimpleCrypter + +class SpeedLoadOrgFolder(SimpleCrypter): + __name__ = "SpeedLoadOrgFolder" + __type__ = "crypter" + __pattern__ = r"http://(www\.)?speedload\.org/(\d+~f$|folder/\d+/)" + __version__ = "0.2" + __description__ = """Speedload Crypter Plugin""" + __author_name__ = ("stickell") + __author_mail__ = ("l.stickell@yahoo.it") + + LINK_PATTERN = r'
        Files of: (?P[^<]+) folder' diff --git a/pyload/plugins/crypter/StealthTo.py b/pyload/plugins/crypter/StealthTo.py new file mode 100644 index 000000000..cf7a79e9b --- /dev/null +++ b/pyload/plugins/crypter/StealthTo.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re + +from module.plugins.Crypter import Crypter + +class StealthTo(Crypter): + __name__ = "StealthTo" + __type__ = "container" + __pattern__ = r"http://(www\.)?stealth.to/folder/" + __version__ = "0.1" + __description__ = """Stealth.to Container Plugin""" + __author_name__ = ("spoob") + __author_mail__ = ("spoob@pyload.org") + + def __init__(self, parent): + Crypter.__init__(self, parent) + self.parent = parent + self.html = None + + def file_exists(self): + """ returns True or False + """ + return True + + def proceed(self, url, location): + url = self.parent.url + self.html = self.req.load(url, cookies=True) + temp_links = [] + ids = [] + ats = [] # authenticity_token + inputs = re.findall(r"(<(input|form)[^>]+)", self.html) + for input in inputs: + if re.search(r"name=\"authenticity_token\"",input[0]): + ats.append(re.search(r"value=\"([^\"]+)", input[0]).group(1)) + if re.search(r"name=\"id\"",input[0]): + ids.append(re.search(r"value=\"([^\"]+)", input[0]).group(1)) + + for i in range(0, len(ids)): + self.req.load(url + "/web", post={"authenticity_token": ats[i], "id": str(ids[i]), "link": ("download_" + str(ids[i]))}, cookies=True) + new_html = self.req.load(url + "/web", post={"authenticity_token": ats[i], "id": str(ids[i]), "link": "1"}, cookies=True) + temp_links.append(re.search(r"iframe src=\"(.*)\" frameborder", new_html).group(1)) + + self.links = temp_links diff --git a/pyload/plugins/crypter/TrailerzoneInfo.py b/pyload/plugins/crypter/TrailerzoneInfo.py new file mode 100644 index 000000000..43a4fcce5 --- /dev/null +++ b/pyload/plugins/crypter/TrailerzoneInfo.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +import re +from module.plugins.Crypter import Crypter + +class TrailerzoneInfo(Crypter): + __name__ = "TrailerzoneInfo" + __type__ = "crypter" + __pattern__ = r"http://(www\.)?trailerzone.info/.*?" + __version__ = "0.02" + __description__ = """TrailerZone.info Crypter Plugin""" + __author_name__ = ("godofdream") + __author_mail__ = ("soilfiction@gmail.com") + + JS_KEY_PATTERN = r"' + + def process(self, pyfile): + self.html = self.load(pyfile.url, decode = True) + self.getFileInfo() + + # parse js variables + self.jsvars = dict((x, y.strip("'")) for x,y in re.findall(r"var (\w+) = ([0-9.]+|'[^']*')", self.html)) + self.logDebug(self.jsvars) + pyfile.name = self.jsvars['ID3'] + + # determine download type - free or premium + if self.premium: + if 'UU_prihlasen' in self.jsvars: + if self.jsvars['UU_prihlasen'] == '0': + self.logWarning('User not logged in') + self.relogin(self.user) + self.retry() + elif float(self.jsvars['UU_kredit']) < float(self.jsvars['kredit_odecet']): + self.logWarning('Not enough credit left') + self.premium = False + + if self.premium: + self.handlePremium() + else: + self.handleFree() + + check = self.checkDownload({"err": re.compile(r"\AChyba!")}, max_size=100) + if check == "err": + self.fail("File not found or plugin defect") + + def handleFree(self): + # get download url + download_url = '%s/download.php' % self.jsvars['server'] + data = dict((x, self.jsvars[x]) for x in self.jsvars if x in ('ID1', 'ID2', 'ID3', 'ID4')) + self.logDebug("FREE URL1:" + download_url, data) + + self.req.http.c.setopt(FOLLOWLOCATION, 0) + self.load(download_url, post=data) + self.header = self.req.http.header + self.req.http.c.setopt(FOLLOWLOCATION, 1) + + found = re.search("Location\s*:\s*(.*)", self.header, re.I) + if not found: self.fail('File not found') + download_url = found.group(1) + self.logDebug("FREE URL2:" + download_url) + + # check errors + found = re.search(r'/chyba/(\d+)', download_url) + if found: + if found.group(1) == '1': + self.retry(max_tries=60, wait_time=120, reason="This IP is already downloading") + elif found.group(1) == '2': + self.retry(max_tries=60, wait_time=60, reason="No free slots available") + else: + self.fail('Error %d' % found.group(1)) + + # download file + self.download(download_url) + + def handlePremium(self): + download_url = '%s/download_premium.php' % self.jsvars['server'] + data = dict((x, self.jsvars[x]) for x in self.jsvars if x in ('ID1', 'ID2', 'ID4', 'ID5')) + self.logDebug("PREMIUM URL:" + download_url, data) + self.download(download_url, get=data) + +getInfo = create_getInfo(QuickshareCz) diff --git a/pyload/plugins/hoster/RapidgatorNet.py b/pyload/plugins/hoster/RapidgatorNet.py new file mode 100644 index 000000000..d2ca77e0f --- /dev/null +++ b/pyload/plugins/hoster/RapidgatorNet.py @@ -0,0 +1,192 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: zoidberg +""" + +import re +from pycurl import HTTPHEADER +from random import random + +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo +from module.plugins.internal.CaptchaService import ReCaptcha, SolveMedia, AdsCaptcha +from module.common.json_layer import json_loads +from module.network.HTTPRequest import BadHeader + +class RapidgatorNet(SimpleHoster): + __name__ = "RapidgatorNet" + __type__ = "hoster" + __pattern__ = r"http://(?:www\.)?(rapidgator.net)/file/(\w+)" + __version__ = "0.17" + __description__ = """rapidgator.net""" + __author_name__ = ("zoidberg", "chrox", "stickell") + + API_URL = 'http://rapidgator.net/api/file' + + FILE_INFO_PATTERN = r'Downloading:(\s*<[^>]*>)*\s*(?P.*?)(\s*<[^>]*>)*\s*File size:\s*(?P.*?)' + FILE_OFFLINE_PATTERN = r'File not found' + + JSVARS_PATTERN = r"\s+var\s*(startTimerUrl|getDownloadUrl|captchaUrl|fid|secs)\s*=\s*'?(.*?)'?;" + DOWNLOAD_LINK_PATTERN = r"return '(http[^']+)';\s*}\s*}\s*}\);" + RECAPTCHA_KEY_PATTERN = r'"http://api.recaptcha.net/challenge?k=(.*?)"' + ADSCAPTCHA_SRC_PATTERN = r'(http://api.adscaptcha.com/Get.aspx[^"\']*)' + SOLVEMEDIA_PATTERN = r'http:\/\/api\.solvemedia\.com\/papi\/challenge\.script\?k=(.*?)"' + + def setup(self): + self.resumeDownload = False + self.multiDL = False + self.sid = None + self.chunkLimit = 1 + self.req.setOption("timeout", 120) + + def process(self, pyfile): + if self.account: + self.sid = self.account.getAccountData(self.user).get('SID', None) + + if self.sid: + self.handlePremium() + else: + self.handleFree() + + def getAPIResponse(self, cmd): + try: + json = self.load('%s/%s' % (self.API_URL, cmd), + get = {'sid': self.sid, + 'url': self.pyfile.url}, decode = True) + self.logDebug('API:%s' % cmd, json, "SID: %s" % self.sid) + json = json_loads(json) + status = json['response_status'] + msg = json['response_details'] + except BadHeader, e: + self.logError('API:%s' % cmd, e, "SID: %s" % self.sid) + status = e.code + msg = e + + if status == 200: + return json['response'] + elif status == 423: + self.account.empty(self.user) + self.retry() + else: + self.account.relogin(self.user) + self.retry(wait_time=60) + + def handlePremium(self): + #self.logDebug("ACCOUNT_DATA", self.account.getAccountData(self.user)) + self.api_data = self.getAPIResponse('info') + self.api_data['md5'] = self.api_data['hash'] + self.pyfile.name = self.api_data['filename'] + self.pyfile.size = self.api_data['size'] + url = self.getAPIResponse('download')['url'] + self.multiDL = True + self.download(url) + + def handleFree(self): + self.html = self.load(self.pyfile.url, decode = True) + self.getFileInfo() + + if "You can download files up to 500 MB in free mode" in self.html \ + or "This file can be downloaded by premium only" in self.html: + self.fail("Premium account needed for download") + + self.checkWait() + + jsvars = dict(re.findall(self.JSVARS_PATTERN, self.html)) + self.logDebug(jsvars) + + self.req.http.lastURL = self.pyfile.url + self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) + + url = "http://rapidgator.net%s?fid=%s" % (jsvars.get('startTimerUrl', '/download/AjaxStartTimer'), jsvars["fid"]) + jsvars.update(self.getJsonResponse(url)) + + self.setWait(int(jsvars.get('secs', 30)) + 1, False) + self.wait() + + url = "http://rapidgator.net%s?sid=%s" % (jsvars.get('getDownloadUrl', '/download/AjaxGetDownload'), jsvars["sid"]) + jsvars.update(self.getJsonResponse(url)) + + self.req.http.lastURL = self.pyfile.url + self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With:"]) + + url = "http://rapidgator.net%s" % jsvars.get('captchaUrl', '/download/captcha') + self.html = self.load(url) + found = re.search(self.ADSCAPTCHA_SRC_PATTERN, self.html) + if found: + captcha_key = found.group(1) + captcha = AdsCaptcha(self) + else: + found = re.search(self.RECAPTCHA_KEY_PATTERN, self.html) + if found: + captcha_key = found.group(1) + captcha = ReCaptcha(self) + + else: + found = re.search(self.SOLVEMEDIA_PATTERN, self.html) + if found: + captcha_key = found.group(1) + captcha = SolveMedia(self) + else: + self.parseError("Captcha") + + for i in range(5): + self.checkWait() + captcha_challenge, captcha_response = captcha.challenge(captcha_key) + + self.html = self.load(url, post={ + "DownloadCaptchaForm[captcha]": "", + "adcopy_challenge": captcha_challenge, + "adcopy_response": captcha_response + }) + + if 'The verification code is incorrect' in self.html: + self.invalidCaptcha() + else: + self.correctCaptcha() + break + else: + self.fail("No valid captcha solution received") + + found = re.search(self.DOWNLOAD_LINK_PATTERN, self.html) + if not found: + self.parseError("download link") + download_url = found.group(1) + self.logDebug(download_url) + self.download(download_url) + + def checkWait(self): + found = re.search(r"(?:Delay between downloads must be not less than|Try again in)\s*(\d+)\s*(hour|min)", self.html) + if found: + wait_time = int(found.group(1)) * {"hour": 60, "min": 1}[found.group(2)] + else: + found = re.search(r"You have reached your (daily|hourly) downloads limit", self.html) + if found: + wait_time = 60 + else: + return + + self.logDebug("Waiting %d minutes" % wait_time) + self.setWait(wait_time * 60, True) + self.wait() + self.retry(max_tries = 24) + + def getJsonResponse(self, url): + response = self.load(url, decode = True) + if not response.startswith('{'): + self.retry() + self.logDebug(url, response) + return json_loads(response) + +getInfo = create_getInfo(RapidgatorNet) diff --git a/pyload/plugins/hoster/RapidshareCom.py b/pyload/plugins/hoster/RapidshareCom.py new file mode 100644 index 000000000..6aacd684e --- /dev/null +++ b/pyload/plugins/hoster/RapidshareCom.py @@ -0,0 +1,225 @@ + +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# v1.36 +# * fixed call checkfiles subroutine +# v1.35 +# * fixed rs-urls in handleFree(..) and freeWait(..) +# * removed getInfo(..) function as it was not used anywhere (in this file) +# * removed some (old?) comment blocks + +import re + +from module.network.RequestFactory import getURL +from module.plugins.Hoster import Hoster + +def getInfo(urls): + ids = "" + names = "" + + p = re.compile(RapidshareCom.__pattern__) + + for url in urls: + r = p.search(url) + if r.group("name"): + ids+= ","+r.group("id") + names+= ","+r.group("name") + elif r.group("name_new"): + ids+= ","+r.group("id_new") + names+= ","+r.group("name_new") + + url = "http://api.rapidshare.com/cgi-bin/rsapi.cgi?sub=checkfiles&files=%s&filenames=%s" % (ids[1:], names[1:]) + + api = getURL(url) + result = [] + i = 0 + for res in api.split(): + tmp = res.split(",") + if tmp[4] in ("0", "4", "5"): status = 1 + elif tmp[4] == "1": status = 2 + else: status = 3 + + result.append( (tmp[1], tmp[2], status, urls[i]) ) + i += 1 + + yield result + + +class RapidshareCom(Hoster): + __name__ = "RapidshareCom" + __type__ = "hoster" + __pattern__ = r"https?://[\w\.]*?rapidshare.com/(?:files/(?P\d*?)/(?P[^?]+)|#!download\|(?:\w+)\|(?P\d+)\|(?P[^|]+))" + __version__ = "1.38" + __description__ = """Rapidshare.com Download Hoster""" + __config__ = [("server", "Cogent;Deutsche Telekom;Level(3);Level(3) #2;GlobalCrossing;Level(3) #3;Teleglobe;GlobalCrossing #2;TeliaSonera #2;Teleglobe #2;TeliaSonera #3;TeliaSonera", "Preferred Server", "None")] + __author_name__ = ("spoob", "RaNaN", "mkaay") + __author_mail__ = ("spoob@pyload.org", "ranan@pyload.org", "mkaay@mkaay.de") + + def setup(self): + self.html = None + self.no_download = True + self.api_data = None + self.offset = 0 + self.dl_dict = {} + + self.id = None + self.name = None + + self.chunkLimit = -1 if self.premium else 1 + self.multiDL = self.resumeDownload = self.premium + + def process(self, pyfile): + self.url = self.pyfile.url + self.prepare() + + def prepare(self): + m = re.search(self.__pattern__, self.url) + + if m.group("name"): + self.id = m.group("id") + self.name = m.group("name") + else: + self.id = m.group("id_new") + self.name = m.group("name_new") + + self.download_api_data() + if self.api_data["status"] == "1": + self.pyfile.name = self.get_file_name() + + if self.premium: + self.handlePremium() + else: + self.handleFree() + + elif self.api_data["status"] == "2": + self.log.info(_("Rapidshare: Traffic Share (direct download)")) + self.pyfile.name = self.get_file_name() + + self.download(self.pyfile.url, get={"directstart":1}) + + elif self.api_data["status"] in ("0","4","5"): + self.offline() + elif self.api_data["status"] == "3": + self.tempOffline() + else: + self.fail("Unknown response code.") + + def handleFree(self): + + while self.no_download: + self.dl_dict = self.freeWait() + + #tmp = "#!download|%(server)s|%(id)s|%(name)s|%(size)s" + download = "http://%(host)s/cgi-bin/rsapi.cgi?sub=download&editparentlocation=0&bin=1&fileid=%(id)s&filename=%(name)s&dlauth=%(auth)s" % self.dl_dict + + self.log.debug("RS API Request: %s" % download) + self.download(download, ref=False) + + check = self.checkDownload({"ip" : "You need RapidPro to download more files from your IP address", + "auth" : "Download auth invalid"}) + if check == "ip": + self.setWait(60) + self.log.info(_("Already downloading from this ip address, waiting 60 seconds")) + self.wait() + self.handleFree() + elif check == "auth": + self.log.info(_("Invalid Auth Code, download will be restarted")) + self.offset += 5 + self.handleFree() + + def handlePremium(self): + info = self.account.getAccountInfo(True) + self.log.debug("%s: Use Premium Account" % self.__name__) + url = self.api_data["mirror"] + self.download(url, get={"directstart":1}) + + + def download_api_data(self, force=False): + """ + http://images.rapidshare.com/apidoc.txt + """ + if self.api_data and not force: + return + api_url_base = "http://api.rapidshare.com/cgi-bin/rsapi.cgi" + api_param_file = {"sub": "checkfiles", "incmd5": "1", "files": self.id, "filenames": self.name} + src = self.load(api_url_base, cookies=False, get=api_param_file).strip() + self.log.debug("RS INFO API: %s" % src) + if src.startswith("ERROR"): + return + fields = src.split(",") + """ + status codes: + 0=File not found + 1=File OK (Anonymous downloading) + 3=Server down + 4=File marked as illegal + 5=Anonymous file locked, because it has more than 10 downloads already + 50+n=File OK (TrafficShare direct download type "n" without any logging.) + 100+n=File OK (TrafficShare direct download type "n" with logging. Read our privacy policy to see what is logged.) + """ + self.api_data = {"fileid": fields[0], "filename": fields[1], "size": int(fields[2]), "serverid": fields[3], + "status": fields[4], "shorthost": fields[5], "checksum": fields[6].strip().lower()} + + if int(self.api_data["status"]) > 100: + self.api_data["status"] = str(int(self.api_data["status"]) - 100) + elif int(self.api_data["status"]) > 50: + self.api_data["status"] = str(int(self.api_data["status"]) - 50) + + self.api_data["mirror"] = "http://rs%(serverid)s%(shorthost)s.rapidshare.com/files/%(fileid)s/%(filename)s" % self.api_data + + def freeWait(self): + """downloads html with the important information + """ + self.no_download = True + + id = self.id + name = self.name + + prepare = "https://api.rapidshare.com/cgi-bin/rsapi.cgi?sub=download&fileid=%(id)s&filename=%(name)s&try=1&cbf=RSAPIDispatcher&cbid=1" % {"name": name, "id" : id} + + self.log.debug("RS API Request: %s" % prepare) + result = self.load(prepare, ref=False) + self.log.debug("RS API Result: %s" % result) + + between_wait = re.search("You need to wait (\d+) seconds", result) + + if "You need RapidPro to download more files from your IP address" in result: + self.setWait(60) + self.log.info(_("Already downloading from this ip address, waiting 60 seconds")) + self.wait() + elif "Too many users downloading from this server right now" in result or "All free download slots are full" in result: + self.setWait(120) + self.log.info(_("RapidShareCom: No free slots")) + self.wait() + elif "This file is too big to download it for free" in result: + self.fail(_("You need a premium account for this file")) + elif "Filename invalid." in result: + self.fail(_("Filename reported invalid")) + elif between_wait: + self.setWait(int(between_wait.group(1))) + self.wantReconnect = True + self.wait() + else: + self.no_download = False + + tmp, info = result.split(":") + data = info.split(",") + + dl_dict = {"id": id, + "name": name, + "host": data[0], + "auth": data[1], + "server": self.api_data["serverid"], + "size": self.api_data["size"] + } + self.setWait(int(data[2])+2+self.offset) + self.wait() + + return dl_dict + + + def get_file_name(self): + if self.api_data["filename"]: + return self.api_data["filename"] + return self.url.split("/")[-1] \ No newline at end of file diff --git a/pyload/plugins/hoster/RarefileNet.py b/pyload/plugins/hoster/RarefileNet.py new file mode 100644 index 000000000..a0f5930b5 --- /dev/null +++ b/pyload/plugins/hoster/RarefileNet.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +import re + +from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo +from module.utils import html_unescape + + +class RarefileNet(XFileSharingPro): + __name__ = "RarefileNet" + __type__ = "hoster" + __pattern__ = r"http://(?:\w*\.)*rarefile.net/\w{12}" + __version__ = "0.03" + __description__ = """Rarefile.net hoster plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + FILE_NAME_PATTERN = r'(?P.*?)' + FILE_SIZE_PATTERN = r'Size : (?P.+?) ' + DIRECT_LINK_PATTERN = r'(?P=link)' + HOSTER_NAME = "rarefile.net" + + def setup(self): + self.resumeDownload = self.multiDL = self.premium + + def handleCaptcha(self, inputs): + captcha_div = re.search(r'Enter code.*?(.*?)
        ', self.html, re.S).group(1) + self.logDebug(captcha_div) + numerals = re.findall('(\d)', html_unescape(captcha_div)) + inputs['code'] = "".join([a[1] for a in sorted(numerals, key = lambda num: int(num[0]))]) + self.logDebug("CAPTCHA", inputs['code'], numerals) + return 3 + +getInfo = create_getInfo(RarefileNet) diff --git a/pyload/plugins/hoster/RealdebridCom.py b/pyload/plugins/hoster/RealdebridCom.py new file mode 100644 index 000000000..3c796232e --- /dev/null +++ b/pyload/plugins/hoster/RealdebridCom.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +from time import time +from urllib import quote, unquote +from random import randrange + +from module.utils import parseFileSize, remove_chars +from module.common.json_layer import json_loads +from module.plugins.Hoster import Hoster + +class RealdebridCom(Hoster): + __name__ = "RealdebridCom" + __version__ = "0.49" + __type__ = "hoster" + + __pattern__ = r"https?://.*real-debrid\..*" + __description__ = """Real-Debrid.com hoster plugin""" + __author_name__ = ("Devirex, Hazzard") + __author_mail__ = ("naibaf_11@yahoo.de") + + def getFilename(self, url): + try: + name = unquote(url.rsplit("/", 1)[1]) + except IndexError: + name = "Unknown_Filename..." + if not name or name.endswith(".."): #incomplete filename, append random stuff + name += "%s.tmp" % randrange(100,999) + return name + + def init(self): + self.tries = 0 + self.chunkLimit = 3 + self.resumeDownload = True + + + def process(self, pyfile): + if not self.account: + self.logError(_("Please enter your Real-debrid account or deactivate this plugin")) + self.fail("No Real-debrid account provided") + + self.log.debug("Real-Debrid: Old URL: %s" % pyfile.url) + if re.match(self.__pattern__, pyfile.url): + new_url = pyfile.url + else: + password = self.getPassword().splitlines() + if not password: password = "" + else: password = password[0] + + url = "http://real-debrid.com/ajax/unrestrict.php?lang=en&link=%s&password=%s&time=%s" % (quote(pyfile.url, ""), password, int(time()*1000)) + page = self.load(url) + data = json_loads(page) + + self.logDebug("Returned Data: %s" % data) + + if data["error"] != 0: + if data["message"] == "Your file is unavailable on the hoster.": + self.offline() + else: + self.logWarning(data["message"]) + self.tempOffline() + else: + if self.pyfile.name is not None and self.pyfile.name.endswith('.tmp') and data["file_name"]: + self.pyfile.name = data["file_name"] + self.pyfile.size = parseFileSize(data["file_size"]) + new_url = data['generated_links'][0][-1] + + if self.getConfig("https"): + new_url = new_url.replace("http://", "https://") + else: + new_url = new_url.replace("https://", "http://") + + self.log.debug("Real-Debrid: New URL: %s" % new_url) + + if pyfile.name.startswith("http") or pyfile.name.startswith("Unknown") or pyfile.name.endswith('..'): + #only use when name wasnt already set + pyfile.name = self.getFilename(new_url) + + self.download(new_url, disposition=True) + + check = self.checkDownload( + {"error": "An error occured while processing your request"}) + + if check == "error": + #usual this download can safely be retried + self.retry(reason="An error occured while generating link.", wait_time=60) + diff --git a/pyload/plugins/hoster/RedtubeCom.py b/pyload/plugins/hoster/RedtubeCom.py new file mode 100644 index 000000000..c2083e679 --- /dev/null +++ b/pyload/plugins/hoster/RedtubeCom.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +from module.plugins.Hoster import Hoster +from module.unescape import unescape + +class RedtubeCom(Hoster): + __name__ = "RedtubeCom" + __type__ = "hoster" + __pattern__ = r'http://[\w\.]*?redtube\.com/\d+' + __version__ = "0.2" + __description__ = """Redtube.com Download Hoster""" + __author_name__ = ("jeix") + __author_mail__ = ("jeix@hasnomail.de") + + def process(self, pyfile): + self.download_html() + if not self.file_exists(): + self.offline() + + pyfile.name = self.get_file_name() + self.download(self.get_file_url()) + + def download_html(self): + url = self.pyfile.url + self.html = self.load(url) + + def get_file_url(self): + """ returns the absolute downloadable filepath + """ + if self.html is None: + self.download_html() + + file_url = unescape(re.search(r'hashlink=(http.*?)"', self.html).group(1)) + + return file_url + + def get_file_name(self): + if self.html is None: + self.download_html() + + name = re.search('(.*?)- RedTube - Free Porn Videos', self.html).group(1).strip() + ".flv" + return name + + def file_exists(self): + """ returns True or False + """ + if self.html is None: + self.download_html() + + if re.search(r'This video has been removed.', self.html) is not None: + return False + else: + return True + diff --git a/pyload/plugins/hoster/RehostTo.py b/pyload/plugins/hoster/RehostTo.py new file mode 100644 index 000000000..141dcb8c8 --- /dev/null +++ b/pyload/plugins/hoster/RehostTo.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from urllib import quote, unquote +from module.plugins.Hoster import Hoster + +class RehostTo(Hoster): + __name__ = "RehostTo" + __version__ = "0.11" + __type__ = "hoster" + __pattern__ = r"https?://.*rehost.to\..*" + __description__ = """rehost.com hoster plugin""" + __author_name__ = ("RaNaN") + __author_mail__ = ("RaNaN@pyload.org") + + def getFilename(self, url): + return unquote(url.rsplit("/", 1)[1]) + + def setup(self): + self.chunkLimit = 1 + self.resumeDownload = True + + def process(self, pyfile): + if not self.account: + self.log.error(_("Please enter your rehost.to account or deactivate this plugin")) + self.fail("No rehost.to account provided") + + data = self.account.getAccountInfo(self.user) + long_ses = data["long_ses"] + + self.log.debug("Rehost.to: Old URL: %s" % pyfile.url) + new_url = "http://rehost.to/process_download.php?user=cookie&pass=%s&dl=%s" % (long_ses, quote(pyfile.url, "")) + + #raise timeout to 2min + self.req.setOption("timeout", 120) + + self.download(new_url, disposition=True) \ No newline at end of file diff --git a/pyload/plugins/hoster/ReloadCc.py b/pyload/plugins/hoster/ReloadCc.py new file mode 100644 index 000000000..7dc6d9bb6 --- /dev/null +++ b/pyload/plugins/hoster/ReloadCc.py @@ -0,0 +1,103 @@ +from module.plugins.Hoster import Hoster + +from module.common.json_layer import json_loads + +from module.network.HTTPRequest import BadHeader + +class ReloadCc(Hoster): + __name__ = "ReloadCc" + __version__ = "0.4" + __type__ = "hoster" + __description__ = """Reload.Cc hoster plugin""" + + # Since we want to allow the user to specify the list of hoster to use we let MultiHoster.coreReady create the regex patterns for us using getHosters in our ReloadCc hook. + __pattern__ = None + + __author_name__ = ("Reload Team") + __author_mail__ = ("hello@reload.cc") + + def process(self, pyfile): + # Check account + if not self.account or not self.account.canUse(): + self.logError("Please enter a valid reload.cc account or deactivate this plugin") + self.fail("No valid reload.cc account provided") + + # In some cases hostsers do not supply us with a filename at download, so we are going to set a fall back filename (e.g. for freakshare or xfileshare) + self.pyfile.name = self.pyfile.name.split('/').pop() # Remove everthing before last slash + + # Correction for automatic assigned filename: Removing html at end if needed + suffix_to_remove = ["html", "htm", "php", "php3", "asp", "shtm", "shtml", "cfml", "cfm"] + temp = self.pyfile.name.split('.') + if temp.pop() in suffix_to_remove: + self.pyfile.name = ".".join(temp) + + # Get account data + (user, data) = self.account.selectAccount() + + query_params = dict( + via='pyload', + v=1, + user=user, + uri=self.pyfile.url + ) + + try: + query_params.update(dict(hash=self.account.infos[user]['pwdhash'])) + except Exception: + query_params.update(dict(pwd=data['password'])) + + try: + answer = self.load("http://api.reload.cc/dl", get=query_params) + except BadHeader, e: + if e.code == 400: + self.fail("The URI is not supported by Reload.cc.") + elif e.code == 401: + self.fail("Wrong username or password") + elif e.code == 402: + self.fail("Your account is inactive. A payment is required for downloading!") + elif e.code == 403: + self.fail("Your account is disabled. Please contact the Reload.cc support!") + elif e.code == 409: + self.logWarning("The hoster seems to be a limited hoster and you've used your daily traffic for this hoster: %s" % self.pyfile.url) + # Wait for 6 hours and retry up to 4 times => one day + self.retry(max_retries=4, wait_time=(3600 * 6), reason="Limited hoster traffic limit exceeded") + elif e.code == 429: + self.retry(max_retries=5, wait_time=120, reason="Too many concurrent connections") # Too many connections, wait 2 minutes and try again + elif e.code == 503: + self.retry(wait_time=600, reason="Reload.cc is currently in maintenance mode! Please check again later.") # Retry in 10 minutes + else: + self.fail("Internal error within Reload.cc. Please contact the Reload.cc support for further information.") + return + + data = json_loads(answer) + + # Check status and decide what to do + status = data.get('status', None) + if status == "ok": + conn_limit = data.get('msg', 0) + # API says these connections are limited + # Make sure this limit is used - the download will fail if not + if conn_limit > 0: + try: + self.limitDL = int(conn_limit) + except ValueError: + self.limitDL = 1 + else: + self.limitDL = 0 + + try: + self.download(data['link'], disposition=True) + except BadHeader, e: + if e.code == 404: + self.fail("File Not Found") + elif e.code == 412: + self.fail("File access password is wrong") + elif e.code == 417: + self.fail("Password required for file access") + elif e.code == 429: + self.retry(max_retries=5, wait_time=120, reason="Too many concurrent connections") # Too many connections, wait 2 minutes and try again + else: + self.fail("Internal error within Reload.cc. Please contact the Reload.cc support for further information.") + return + else: + self.fail("Internal error within Reload.cc. Please contact the Reload.cc support for further information.") diff --git a/pyload/plugins/hoster/RyushareCom.py b/pyload/plugins/hoster/RyushareCom.py new file mode 100644 index 000000000..7bfe4e8fe --- /dev/null +++ b/pyload/plugins/hoster/RyushareCom.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo +import re + + +class RyushareCom(XFileSharingPro): + __name__ = "RyushareCom" + __type__ = "hoster" + __pattern__ = r"http://(?:\w*\.)*?ryushare.com/\w{11,}" + __version__ = "0.11" + __description__ = """ryushare.com hoster plugin""" + __author_name__ = ("zoidberg", "stickell") + __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it") + + HOSTER_NAME = "ryushare.com" + + WAIT_PATTERN = r'(?:You have to|Please) wait (?:(?P\d+) minutes, )?(?:)?(?P\d+)(?:)? seconds' + DIRECT_LINK_PATTERN = r'Click here to download' + + def setup(self): + self.resumeDownload = self.multiDL = True + if not self.premium: + self.limitDL = 2 + # Up to 3 chunks allowed in free downloads. Unknown for premium + self.chunkLimit = 3 + + def getDownloadLink(self): + self.html = self.load(self.pyfile.url) + action, inputs = self.parseHtmlForm(input_names={"op": re.compile("^download")}) + if inputs.has_key('method_premium'): + del inputs['method_premium'] + + self.html = self.load(self.pyfile.url, post = inputs) + action, inputs = self.parseHtmlForm('F1') + + for i in xrange(10): + self.logInfo('Attempt to detect direct link #%d' % i) + + # Wait + if 'You have reached the download-limit!!!' in self.html: + self.setWait(3600, True) + else: + m = re.search(self.WAIT_PATTERN, self.html).groupdict('0') + waittime = int(m['min']) * 60 + int(m['sec']) + self.setWait(waittime) + self.wait() + + self.html = self.load(self.pyfile.url, post = inputs) + if 'Click here to download' in self.html: + m = re.search(self.DIRECT_LINK_PATTERN, self.html) + return m.group(1) + + self.parseError('No direct link within 10 retries') + +getInfo = create_getInfo(RyushareCom) diff --git a/pyload/plugins/hoster/SecureUploadEu.py b/pyload/plugins/hoster/SecureUploadEu.py new file mode 100644 index 000000000..b9a900d96 --- /dev/null +++ b/pyload/plugins/hoster/SecureUploadEu.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo + + +class SecureUploadEu(XFileSharingPro): + __name__ = "SecureUploadEu" + __type__ = "hoster" + __pattern__ = r"http://(www\.)?secureupload\.eu/(\w){12}(/\w+)" + __version__ = "0.01" + __description__ = """SecureUpload.eu hoster plugin""" + __author_name__ = ("z00nx") + __author_mail__ = ("z00nx0@gmail.com") + + HOSTER_NAME = "secureupload.eu" + FILE_INFO_PATTERN = '

        Downloading (?P[^<]+) \((?P[^<]+)\)

        ' + FILE_OFFLINE_PATTERN = 'The file was removed|File Not Found' + +getInfo = create_getInfo(SecureUploadEu) diff --git a/pyload/plugins/hoster/SendmywayCom.py b/pyload/plugins/hoster/SendmywayCom.py new file mode 100644 index 000000000..fcbac850a --- /dev/null +++ b/pyload/plugins/hoster/SendmywayCom.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo + + +class SendmywayCom(XFileSharingPro): + __name__ = "SendmywayCom" + __type__ = "hoster" + __pattern__ = r"http://(?:\w*\.)*?sendmyway.com/\w{12}" + __version__ = "0.01" + __description__ = """SendMyWay hoster plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + FILE_NAME_PATTERN = r'

        <.*?>\s*(?P.+)' + FILE_SIZE_PATTERN = r'\((?P\d+) bytes\)' + HOSTER_NAME = "sendmyway.com" + +getInfo = create_getInfo(SendmywayCom) diff --git a/pyload/plugins/hoster/SendspaceCom.py b/pyload/plugins/hoster/SendspaceCom.py new file mode 100644 index 000000000..22abaff56 --- /dev/null +++ b/pyload/plugins/hoster/SendspaceCom.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: zoidberg +""" + +import re +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo + +class SendspaceCom(SimpleHoster): + __name__ = "SendspaceCom" + __type__ = "hoster" + __pattern__ = r"http://(www\.)?sendspace.com/file/.*" + __version__ = "0.13" + __description__ = """sendspace.com plugin - free only""" + __author_name__ = ("zoidberg") + + DOWNLOAD_URL_PATTERN = r'\s*<(?:b|strong)>(?P[^<]+)\s*File Size:\s*(?P[0-9.]+)(?P[kKMG])i?B\s*

        ' + FILE_OFFLINE_PATTERN = r'
        Sorry, the file you requested is not available.
        ' + CAPTCHA_PATTERN = r'' + USER_CAPTCHA_PATTERN = r'' + + def handleFree(self): + params = {} + for i in range(3): + found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) + if found: + if params.has_key('captcha_hash'): self.correctCaptcha() + download_url = found.group(1) + break + + found = re.search(self.CAPTCHA_PATTERN, self.html) + if found: + if params.has_key('captcha_hash'): self.invalidCaptcha() + captcha_url1 = "http://www.sendspace.com/" + found.group(1) + found = re.search(self.USER_CAPTCHA_PATTERN, self.html) + captcha_url2 = "http://www.sendspace.com/" + found.group(1) + params = {'captcha_hash' : found.group(2), + 'captcha_submit': 'Verify', + 'captcha_answer': self.decryptCaptcha(captcha_url1) + " " + self.decryptCaptcha(captcha_url2) + } + else: + params = {'download': "Regular Download"} + + self.logDebug(params) + self.html = self.load(self.pyfile.url, post = params) + else: + self.fail("Download link not found") + + self.logDebug("Download URL: %s" % download_url) + self.download(download_url) + +create_getInfo(SendspaceCom) \ No newline at end of file diff --git a/pyload/plugins/hoster/Share4webCom.py b/pyload/plugins/hoster/Share4webCom.py new file mode 100644 index 000000000..ef9c2acf8 --- /dev/null +++ b/pyload/plugins/hoster/Share4webCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from module.plugins.hoster.UnibytesCom import UnibytesCom +from module.plugins.internal.SimpleHoster import create_getInfo + +class Share4webCom(UnibytesCom): + __name__ = "Share4webCom" + __type__ = "hoster" + __pattern__ = r"http://(www\.)?share4web\.com/get/\w+" + __version__ = "0.1" + __description__ = """Share4web.com""" + __author_name__ = ("zoidberg") + + DOMAIN = 'http://www.share4web.com' + +getInfo = create_getInfo(UnibytesCom) \ No newline at end of file diff --git a/pyload/plugins/hoster/Share76Com.py b/pyload/plugins/hoster/Share76Com.py new file mode 100644 index 000000000..db850cb73 --- /dev/null +++ b/pyload/plugins/hoster/Share76Com.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo + +class Share76Com(XFileSharingPro): + __name__ = "Share76Com" + __type__ = "hoster" + __pattern__ = r"http://(?:\w*\.)*?share76.com/\w{12}" + __version__ = "0.03" + __description__ = """share76.com hoster plugin""" + __author_name__ = ("me") + + FILE_INFO_PATTERN = r'

        \s*File:\s*]*>(?P[^>]+)\s*\[]*>(?P[0-9.]+) (?P[kKMG])i?B\]

        ' + HOSTER_NAME = "share76.com" + + def setup(self): + self.resumeDownload = self.multiDL = self.premium + self.chunkLimit = 1 + +getInfo = create_getInfo(Share76Com) diff --git a/pyload/plugins/hoster/ShareFilesCo.py b/pyload/plugins/hoster/ShareFilesCo.py new file mode 100644 index 000000000..ee44b0a1f --- /dev/null +++ b/pyload/plugins/hoster/ShareFilesCo.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo +import re + +class ShareFilesCo(XFileSharingPro): + __name__ = "ShareFilesCo" + __type__ = "hoster" + __pattern__ = r"http://(www\.)?sharefiles\.co/\w{12}" + __version__ = "0.01" + __description__ = """Sharefiles.co hoster plugin""" + __author_name__ = ("stickell") + __author_mail__ = ("l.stickell@yahoo.it") + + HOSTER_NAME = "sharefiles.co" + + def startDownload(self, link): + link = link.strip() + if link.startswith('http://adf.ly'): + link = re.sub('http://adf.ly/\d+/', '', link) + if self.captcha: self.correctCaptcha() + self.logDebug('DIRECT LINK: %s' % link) + self.download(link) + +getInfo = create_getInfo(ShareFilesCo) diff --git a/pyload/plugins/hoster/ShareRapidCom.py b/pyload/plugins/hoster/ShareRapidCom.py new file mode 100644 index 000000000..5a08fed1f --- /dev/null +++ b/pyload/plugins/hoster/ShareRapidCom.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +from pycurl import HTTPHEADER +from module.network.RequestFactory import getRequest, getURL +from module.network.HTTPRequest import BadHeader +from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo +from module.common.json_layer import json_loads + +def checkFile(url): + response = getURL("http://share-rapid.com/checkfiles.php", post = {"files": url}, decode = True) + info = json_loads(response) + + if "error" in info: + if info['error'] == False: + info['name'] = info['filename'] + info['status'] = 2 + elif info['msg'] == "Not found": + info['status'] = 1 #offline + elif info['msg'] == "Service Unavailable": + info['status'] = 6 #temp.offline + + return info + +def getInfo(urls): + for url in urls: + info = checkFile(url) + if "filename" in info: + yield info['name'], info['size'], info['status'], url + else: + file_info = (url, 0, 3, url) + h = getRequest() + try: + h.c.setopt(HTTPHEADER, ["Accept: text/html"]) + html = h.load(url, cookies = True, decode = True) + file_info = parseFileInfo(ShareRapidCom, url, html) + finally: + h.close() + yield file_info + +class ShareRapidCom(SimpleHoster): + __name__ = "ShareRapidCom" + __type__ = "hoster" + __pattern__ = r"http://(?:www\.)?((share(-?rapid\.(biz|com|cz|info|eu|net|org|pl|sk)|-(central|credit|free|net)\.cz|-ms\.net)|(s-?rapid|rapids)\.(cz|sk))|(e-stahuj|mediatack|premium-rapidshare|rapidshare-premium|qiuck)\.cz|kadzet\.com|stahuj-zdarma\.eu|strelci\.net|universal-share\.com)/stahuj/(\w+)" + __version__ = "0.52" + __description__ = """Share-rapid.com plugin - premium only""" + __author_name__ = ("MikyWoW", "zoidberg") + __author_mail__ = ("MikyWoW@seznam.cz", "zoidberg@mujmail.cz") + + FILE_NAME_PATTERN = r']*>]*>(?:]*>)?(?P[^<]+)' + FILE_SIZE_PATTERN = r'Velikost:\s*\s*(?P[0-9.]+) (?P[kKMG])i?B' + FILE_OFFLINE_PATTERN = ur'Nastala chyba 404|Soubor byl smazán' + + DOWNLOAD_URL_PATTERN = r'        ([^<]+)' + ERR_LOGIN_PATTERN = ur'
        Stahování je přístupné pouze přihlášeným uživatelům' + ERR_CREDIT_PATTERN = ur'
        Stahování zdarma je možné jen přes náš' + + FILE_URL_REPLACEMENTS = [(__pattern__, r'http://share-rapid.com/stahuj/\1')] + + def setup(self): + self.chunkLimit = 1 + self.resumeDownload = True + + def process(self, pyfile): + if not self.account: self.fail("User not logged in") + + self.info = checkFile(pyfile.url) + self.logDebug(self.info) + + pyfile.status = self.info['status'] + + if pyfile.status == 2: + pyfile.name = self.info['name'] + pyfile.size = self.info['size'] + elif pyfile.status == 1: + self.offline() + elif pyfile.status == 6: + self.tempOffline() + else: + self.fail("Unexpected file status") + + url = "http://share-rapid.com/stahuj/%s" % self.info['filepath'] + try: + self.html = self.load(url, decode=True) + except BadHeader, e: + self.account.relogin(self.user) + self.retry(3, 0, str(e)) + + found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) + if found is not None: + link = found.group(1) + self.logDebug("Premium link: %s" % link) + + self.check_data = {"size": pyfile.size} + self.download(link) + else: + if re.search(self.ERR_LOGIN_PATTERN, self.html): + self.relogin(self.user) + self.retry(3,0,"User login failed") + elif re.search(self.ERR_CREDIT_PATTERN, self.html): + self.fail("Not enough credit left") + else: + self.fail("Download link not found") diff --git a/pyload/plugins/hoster/SharebeesCom.py b/pyload/plugins/hoster/SharebeesCom.py new file mode 100644 index 000000000..f5bacc5b0 --- /dev/null +++ b/pyload/plugins/hoster/SharebeesCom.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo + + +class SharebeesCom(XFileSharingPro): + __name__ = "SharebeesCom" + __type__ = "hoster" + __pattern__ = r"http://(?:\w*\.)*?sharebees.com/\w{12}" + __version__ = "0.01" + __description__ = """ShareBees hoster plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + FILE_NAME_PATTERN = r'

        <.*?>\s*(?P.+)' + FILE_SIZE_PATTERN = r'\((?P\d+) bytes\)' + FORM_PATTERN = 'F1' + HOSTER_NAME = "sharebees.com" + +getInfo = create_getInfo(SharebeesCom) diff --git a/pyload/plugins/hoster/ShareonlineBiz.py b/pyload/plugins/hoster/ShareonlineBiz.py new file mode 100644 index 000000000..e1867168b --- /dev/null +++ b/pyload/plugins/hoster/ShareonlineBiz.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +from base64 import b64decode +import hashlib +import random +from time import time, sleep + +from module.plugins.Hoster import Hoster +from module.network.RequestFactory import getURL +from module.plugins.Plugin import chunks +from module.plugins.ReCaptcha import ReCaptcha as _ReCaptcha + +def getInfo(urls): + api_url_base = "http://api.share-online.biz/linkcheck.php" + + for chunk in chunks(urls, 90): + api_param_file = {"links": "\n".join(x.replace("http://www.share-online.biz/dl/","").rstrip("/") for x in chunk)} #api only supports old style links + src = getURL(api_url_base, post=api_param_file, decode=True) + result = [] + for i, res in enumerate(src.split("\n")): + if not res: + continue + fields = res.split(";") + + if fields[1] == "OK": + status = 2 + elif fields[1] in ("DELETED", "NOT FOUND"): + status = 1 + else: + status = 3 + + result.append((fields[2], int(fields[3]), status, chunk[i])) + yield result + +#suppress ocr plugin +class ReCaptcha(_ReCaptcha): + def result(self, server, challenge): + return self.plugin.decryptCaptcha("%simage"%server, get={"c":challenge}, cookies=True, forceUser=True, imgtype="jpg") + +class ShareonlineBiz(Hoster): + __name__ = "ShareonlineBiz" + __type__ = "hoster" + __pattern__ = r"http://[\w\.]*?(share\-online\.biz|egoshare\.com)/(download.php\?id\=|dl/)[\w]+" + __version__ = "0.36" + __description__ = """Shareonline.biz Download Hoster""" + __author_name__ = ("spoob", "mkaay", "zoidberg") + __author_mail__ = ("spoob@pyload.org", "mkaay@mkaay.de", "zoidberg@mujmail.cz") + + ERROR_INFO_PATTERN = r'

        Information:

        \s*
        \s*(.*?)' + + def setup(self): + # range request not working? + # api supports resume, only one chunk + # website isn't supporting resuming in first place + self.file_id = re.search(r"(id\=|/dl/)([a-zA-Z0-9]+)", self.pyfile.url).group(2) + self.pyfile.url = "http://www.share-online.biz/dl/" + self.file_id + + self.resumeDownload = self.premium + self.multiDL = False + #self.chunkLimit = 1 + + self.check_data = None + + def process(self, pyfile): + if self.premium: + self.handleAPIPremium() + #web-download fallback removed - didn't work anyway + else: + self.handleFree() + + """ + check = self.checkDownload({"failure": re.compile(self.ERROR_INFO_PATTERN)}) + if check == "failure": + try: + self.retry(reason = self.lastCheck.group(1).decode("utf8")) + except: + self.retry(reason = "Unknown error") + """ + + if self.api_data: + self.check_data = {"size": int(self.api_data['size']), "md5": self.api_data['md5']} + + def downloadAPIData(self): + api_url_base = "http://api.share-online.biz/linkcheck.php?md5=1" + api_param_file = {"links": self.pyfile.url.replace("http://www.share-online.biz/dl/","")} #api only supports old style links + src = self.load(api_url_base, cookies=False, post=api_param_file, decode=True) + + fields = src.split(";") + self.api_data = {"fileid": fields[0], + "status": fields[1]} + if not self.api_data["status"] == "OK": + self.offline() + self.api_data["filename"] = fields[2] + self.api_data["size"] = fields[3] # in bytes + self.api_data["md5"] = fields[4].strip().lower().replace("\n\n", "") # md5 + + def handleFree(self): + self.downloadAPIData() + self.pyfile.name = self.api_data["filename"] + self.pyfile.size = int(self.api_data["size"]) + + self.html = self.load(self.pyfile.url, cookies = True) #refer, stuff + self.setWait(3) + self.wait() + + self.html = self.load("%s/free/" % self.pyfile.url, post={"dl_free":"1", "choice": "free"}, decode = True) + self.checkErrors() + + found = re.search(r'var wait=(\d+);', self.html) + + recaptcha = ReCaptcha(self) + for i in range(5): + challenge, response = recaptcha.challenge("6LdatrsSAAAAAHZrB70txiV5p-8Iv8BtVxlTtjKX") + self.setWait(int(found.group(1)) if found else 30) + response = self.load("%s/free/captcha/%d" % (self.pyfile.url, int(time() * 1000)), post = { + 'dl_free': '1', + 'recaptcha_challenge_field': challenge, + 'recaptcha_response_field': response}) + + if not response == '0': + break + + else: self.fail("No valid captcha solution received") + + download_url = response.decode("base64") + self.logDebug(download_url) + if not download_url.startswith("http://"): + self.parseError("download url") + + self.wait() + self.download(download_url) + # check download + check = self.checkDownload({ + "cookie": re.compile(r'
        Share-Online") + }) + if check == "cookie": + self.retry(5, 60, "Cookie failure") + elif check == "fail": + self.retry(5, 300, "Download failed") + + def checkErrors(self): + found = re.search(r"/failure/(.*?)/1", self.req.lastEffectiveURL) + if found: + err = found.group(1) + found = re.search(self.ERROR_INFO_PATTERN, self.html) + msg = found.group(1) if found else "" + self.logError(err, msg or "Unknown error occurred") + + if err in ('freelimit', 'size', 'proxy'): + self.fail(msg or "Premium account needed") + if err in ('invalid'): + self.fail(msg or "File not available") + elif err in ('server'): + self.setWait(600, False) + elif err in ('expired'): + self.setWait(30, False) + else: + self.setWait(300, True) + + self.wait() + self.retry(max_tries=25, reason = msg) + + def handleAPIPremium(self): #should be working better + self.account.getAccountInfo(self.user, True) + src = self.load("http://api.share-online.biz/account.php", + {"username": self.user, "password": self.account.accounts[self.user]["password"], "act": "download", "lid": self.file_id}) + + self.api_data = dlinfo = {} + for line in src.splitlines(): + key, value = line.split(": ") + dlinfo[key.lower()] = value + + self.logDebug(dlinfo) + if not dlinfo["status"] == "online": + self.offline() + + self.pyfile.name = dlinfo["name"] + self.pyfile.size = int(dlinfo["size"]) + + dlLink = dlinfo["url"] + if dlLink == "server_under_maintenance": + self.tempoffline() + else: + self.multiDL = True + self.download(dlLink) + + def checksum(self, local_file): + if self.api_data and "md5" in self.api_data and self.api_data["md5"]: + h = hashlib.md5() + f = open(local_file, "rb") + h.update(f.read()) + f.close() + hexd = h.hexdigest() + if hexd == self.api_data["md5"]: + return True, 0 + else: + return False, 1 + else: + self.logWarning("MD5 checksum missing") + return True, 5 diff --git a/pyload/plugins/hoster/ShareplaceCom.py b/pyload/plugins/hoster/ShareplaceCom.py new file mode 100644 index 000000000..1d4441925 --- /dev/null +++ b/pyload/plugins/hoster/ShareplaceCom.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +import urllib +from module.plugins.Hoster import Hoster + +class ShareplaceCom(Hoster): + __name__ = "ShareplaceCom" + __type__ = "hoster" + __pattern__ = r"(http://)?(www\.)?shareplace\.(com|org)/\?[a-zA-Z0-9]+" + __version__ = "0.1" + __description__ = """Shareplace.com Download Hoster""" + __author_name__ = ("ACCakut, based on YourfilesTo by jeix and skydancer") + __author_mail__ = ("none") + + def setup(self): + self.html = None + self.multiDL = True + + def process(self,pyfile): + self.pyfile = pyfile + self.prepare() + self.download(self.get_file_url()) + + def prepare(self): + if not self.file_exists(): + self.offline() + + self.pyfile.name = self.get_file_name() + + wait_time = self.get_waiting_time() + self.setWait(wait_time) + self.log.debug("%s: Waiting %d seconds." % (self.__name__,wait_time)) + self.wait() + + def get_waiting_time(self): + if self.html is None: + self.download_html() + + #var zzipitime = 15; + m = re.search(r'var zzipitime = (\d+);', self.html) + if m: + sec = int(m.group(1)) + else: + sec = 0 + + return sec + + def download_html(self): + url = re.sub("shareplace.com\/\?", "shareplace.com//index1.php/?a=", self.pyfile.url) + self.html = self.load(url, decode=True) + + def get_file_url(self): + """ returns the absolute downloadable filepath + """ + url = re.search(r"var beer = '(.*?)';", self.html) + if url: + url = url.group(1) + url = urllib.unquote(url.replace("http://http:/", "").replace("vvvvvvvvv", "").replace("lllllllll", "").replace("teletubbies", "")) + self.logDebug("URL: %s" % url) + return url + else: + self.fail("absolute filepath could not be found. offline? ") + + def get_file_name(self): + if self.html is None: + self.download_html() + + return re.search("\s*(.*?)\s*", self.html).group(1) + + def file_exists(self): + """ returns True or False + """ + if self.html is None: + self.download_html() + + if re.search(r"HTTP Status 404", self.html) is not None: + return False + else: + return True + + + diff --git a/pyload/plugins/hoster/ShragleCom.py b/pyload/plugins/hoster/ShragleCom.py new file mode 100644 index 000000000..99f9f2366 --- /dev/null +++ b/pyload/plugins/hoster/ShragleCom.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +from pycurl import FOLLOWLOCATION + +from module.plugins.Hoster import Hoster +from module.plugins.internal.SimpleHoster import parseHtmlForm +from module.plugins.ReCaptcha import ReCaptcha +from module.network.RequestFactory import getURL + +API_KEY = "078e5ca290d728fd874121030efb4a0d" + +def parseFileInfo(self, url): + file_id = re.match(self.__pattern__, url).group('ID') + + data = getURL( + "http://www.cloudnator.com/api.php?key=%s&action=getStatus&fileID=%s" % (API_KEY, file_id), + decode = True + ).split() + + if len(data) == 4: + name, size, md5, status = data + size = int(size) + + if hasattr(self, "check_data"): + self.checkdata = {"size": size, "md5": md5} + + return name, size, 2 if status == "0" else 1, url + else: + return url, 0, 1, url + +def getInfo(urls): + for url in urls: + file_info = parseFileInfo(ShragleCom, url) + yield file_info + +class ShragleCom(Hoster): + __name__ = "ShragleCom" + __type__ = "hoster" + __pattern__ = r"http://(?:www.)?(cloudnator|shragle).com/files/(?P.*?)/" + __version__ = "0.21" + __description__ = """Cloudnator.com (Shragle.com) Download PLugin""" + __author_name__ = ("RaNaN", "zoidberg") + __author_mail__ = ("RaNaN@pyload.org", "zoidberg@mujmail.cz") + + def setup(self): + self.html = None + self.multiDL = False + self.check_data = None + + def process(self, pyfile): + #get file status and info + self.pyfile.name, self.pyfile.size, status = parseFileInfo(self, pyfile.url)[:3] + if status != 2: + self.offline() + + self.handleFree() + + def handleFree(self): + self.html = self.load(self.pyfile.url) + + #get wait time + found = re.search('\s*var\sdownloadWait\s=\s(\d+);', self.html) + self.setWait(int(found.group(1)) if found else 30) + + #parse download form + action, inputs = parseHtmlForm('id="download', self.html) + + #solve captcha + found = re.search('recaptcha/api/(?:challenge|noscript)?k=(.+?)', self.html) + captcha_key = found.group(1) if found else "6LdEFb0SAAAAAAwM70vnYo2AkiVkCx-xmfniatHz" + + recaptcha = ReCaptcha(self) + + inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(captcha_key) + self.wait() + + #validate + self.req.http.c.setopt(FOLLOWLOCATION, 0) + self.html = self.load(action, post = inputs) + + found = re.search(r"Location\s*:\s*(\S*)", self.req.http.header, re.I) + if found: + self.correctCaptcha() + download_url = found.group(1) + else: + if "Sicherheitscode falsch" in self.html: + self.invalidCaptcha() + self.retry(max_tries = 5, reason = "Invalid captcha") + else: + self.fail("Invalid session") + + #download + self.req.http.c.setopt(FOLLOWLOCATION, 1) + self.download(download_url) + + check = self.checkDownload({ + "ip_blocked": re.compile(r'
        \w+)" + __version__ = "1.01" + __description__ = """Speedload.org hoster plugin""" + __author_name__ = ("stickell") + __author_mail__ = ("l.stickell@yahoo.it") + + FILE_NAME_PATTERN = r'Filename:(?P[^<]+)' + FILE_SIZE_PATTERN = r'Size:[\w. ]+\((?P\d+) bytes\)' + + HOSTER_NAME = "speedload.org" + + def handlePremium(self): + self.download(self.pyfile.url, post = self.getPostParameters()) + +getInfo = create_getInfo(SpeedLoadOrg) diff --git a/pyload/plugins/hoster/SpeedfileCz.py b/pyload/plugins/hoster/SpeedfileCz.py new file mode 100644 index 000000000..bfd316dfa --- /dev/null +++ b/pyload/plugins/hoster/SpeedfileCz.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: zoidberg +""" + +import re +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo + +class SpeedfileCz(SimpleHoster): + __name__ = "SpeedFileCz" + __type__ = "hoster" + __pattern__ = r"http://speedfile.cz/.*" + __version__ = "0.31" + __description__ = """speedfile.cz""" + __author_name__ = ("zoidberg") + + FILE_NAME_PATTERN = r'' + FILE_SIZE_PATTERN = r'(?P[0-9.]+) (?P[kKMG])i?B' + URL_PATTERN = r'' + FILE_OFFLINE_PATTERN = r'Speedfile \| 404' + WAIT_PATTERN = r'"requestedAt":(\d+),"allowedAt":(\d+),"adUri"' + + def setup(self): + self.multiDL = False + + def process(self, pyfile): + self.html = self.load(pyfile.url, decode=True) + + if re.search(self.FILE_OFFLINE_PATTERN, self.html): + self.offline() + + found = re.search(self.FILE_NAME_PATTERN, self.html) + if found is None: + self.fail("Parse error (NAME)") + pyfile.name = found.group(1) + + found = re.search(self.URL_PATTERN, self.html) + if found is None: + self.fail("Parse error (URL)") + download_url = "http://speedfile.cz/" + found.group(1) + + self.html = self.load(download_url) + self.logDebug(self.html) + found = re.search(self.WAIT_PATTERN, self.html) + if found is None: + self.fail("Parse error (WAIT)") + self.setWait(int(found.group(2)) - int(found.group(1))) + self.wait() + + self.download(download_url) + +create_getInfo(SpeedfileCz) \ No newline at end of file diff --git a/pyload/plugins/hoster/StreamCz.py b/pyload/plugins/hoster/StreamCz.py new file mode 100644 index 000000000..ca1033502 --- /dev/null +++ b/pyload/plugins/hoster/StreamCz.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: zoidberg +""" + +import re +from module.plugins.Hoster import Hoster +from module.network.RequestFactory import getURL + +def getInfo(urls): + result = [] + + for url in urls: + + html = getURL(url) + if re.search(StreamCz.FILE_OFFLINE_PATTERN, html): + # File offline + result.append((url, 0, 1, url)) + else: + result.append((url, 0, 2, url)) + yield result + +class StreamCz(Hoster): + __name__ = "StreamCz" + __type__ = "hoster" + __pattern__ = r"http://www.stream.cz/[^/]+/\d+.*" + __version__ = "0.1" + __description__ = """stream.cz""" + __author_name__ = ("zoidberg") + + FILE_OFFLINE_PATTERN = r'<h1 class="commonTitle">Str.nku nebylo mo.n. nal.zt \(404\)</h1>' + FILE_NAME_PATTERN = r'<link rel="video_src" href="http://www.stream.cz/\w+/(\d+)-([^"]+)" />' + CDN_PATTERN = r'<param name="flashvars" value="[^"]*&id=(?P<ID>\d+)(?:&cdnLQ=(?P<cdnLQ>\d*))?(?:&cdnHQ=(?P<cdnHQ>\d*))?(?:&cdnHD=(?P<cdnHD>\d*))?&' + + def setup(self): + self.multiDL = True + self.resumeDownload = True + + def process(self, pyfile): + + self.html = self.load(pyfile.url, decode=True) + + if re.search(self.FILE_OFFLINE_PATTERN, self.html): + self.offline() + + found = re.search(self.CDN_PATTERN, self.html) + if found is None: self.fail("Parse error (CDN)") + cdn = found.groupdict() + self.logDebug(cdn) + for cdnkey in ("cdnHD", "cdnHQ", "cdnLQ"): + if cdn.has_key(cdnkey) and cdn[cdnkey] > '': + cdnid = cdn[cdnkey] + break + else: + self.fail("Stream URL not found") + + found = re.search(self.FILE_NAME_PATTERN, self.html) + if found is None: self.fail("Parse error (NAME)") + pyfile.name = "%s-%s.%s.mp4" % (found.group(2), found.group(1), cdnkey[-2:]) + + download_url = "http://cdn-dispatcher.stream.cz/?id=" + cdnid + self.logInfo("STREAM (%s): %s" % (cdnkey[-2:], download_url)) + self.download(download_url) diff --git a/pyload/plugins/hoster/TurbobitNet.py b/pyload/plugins/hoster/TurbobitNet.py new file mode 100644 index 000000000..b429d5510 --- /dev/null +++ b/pyload/plugins/hoster/TurbobitNet.py @@ -0,0 +1,170 @@ +# -*- coding: utf-8 -*- +""" + Copyright (C) 2012 pyLoad team + Copyright (C) 2012 JD-Team support@jdownloader.org + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: zoidberg +""" + +import re +import random +from urllib import quote +from binascii import hexlify, unhexlify +from Crypto.Cipher import ARC4 +import time + +from module.network.RequestFactory import getURL +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, timestamp +from module.plugins.ReCaptcha import ReCaptcha + +from pycurl import HTTPHEADER + +class TurbobitNet(SimpleHoster): + __name__ = "TurbobitNet" + __type__ = "hoster" + __pattern__ = r"http://(?:\w*\.)?(turbobit.net|unextfiles.com)/(?:download/free/)?(?P<ID>\w+).*" + __version__ = "0.08" + __description__ = """Turbobit.net plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + FILE_INFO_PATTERN = r"<span class='file-icon1[^>]*>(?P<N>[^<]+)</span>\s*\((?P<S>[^\)]+)\)\s*</h1>" #long filenames are shortened + FILE_NAME_PATTERN = r'<meta name="keywords" content="\s+(?P<N>[^,]+)' #full name but missing on page2 + FILE_OFFLINE_PATTERN = r'<h2>File Not Found</h2>|html\(\'File was not found' + FILE_URL_REPLACEMENTS = [(r"http://(?:\w*\.)?(turbobit.net|unextfiles.com)/(?:download/free/)?(?P<ID>\w+).*", "http://turbobit.net/\g<ID>.html")] + SH_COOKIES = [("turbobit.net", "user_lang", "en")] + + CAPTCHA_KEY_PATTERN = r'src="http://api\.recaptcha\.net/challenge\?k=([^"]+)"' + DOWNLOAD_URL_PATTERN = r'(?P<url>/download/redirect/[^"\']+)' + LIMIT_WAIT_PATTERN = r'<div id="time-limit-text">\s*.*?<span id=\'timeout\'>(\d+)</span>' + CAPTCHA_SRC_PATTERN = r'<img alt="Captcha" src="(.*?)"' + + def handleFree(self): + self.url = "http://turbobit.net/download/free/%s" % self.file_info['ID'] + self.html = self.load(self.url) + + rtUpdate = self.getRtUpdate() + + self.solveCaptcha() + self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) + self.url = self.getDownloadUrl(rtUpdate) + + self.wait() + self.html = self.load(self.url) + self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With:"]) + self.downloadFile() + + def solveCaptcha(self): + for i in range(5): + found = re.search(self.LIMIT_WAIT_PATTERN, self.html) + if found: + wait_time = int(found.group(1)) + self.setWait(wait_time, wait_time > 60) + self.wait() + self.retry() + + action, inputs = self.parseHtmlForm("action='#'") + if not inputs: self.parseError("captcha form") + self.logDebug(inputs) + + if inputs['captcha_type'] == 'recaptcha': + recaptcha = ReCaptcha(self) + found = re.search(self.CAPTCHA_KEY_PATTERN, self.html) + captcha_key = found.group(1) if found else '6LcTGLoSAAAAAHCWY9TTIrQfjUlxu6kZlTYP50_c' + inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(captcha_key) + else: + found = re.search(self.CAPTCHA_SRC_PATTERN, self.html) + if not found: self.parseError('captcha') + captcha_url = found.group(1) + inputs['captcha_response'] = self.decryptCaptcha(captcha_url) + + self.logDebug(inputs) + self.html = self.load(self.url, post = inputs) + + if not "<div class='download-timer-header'>" in self.html: + self.invalidCaptcha() + else: + self.correctCaptcha() + break + else: self.fail("Invalid captcha") + + def getRtUpdate(self): + rtUpdate = self.getStorage("rtUpdate") + if not rtUpdate: + if self.getStorage("version") != self.__version__ or int(self.getStorage("timestamp", 0)) + 86400000 < timestamp(): + # that's right, we are even using jdownloader updates + rtUpdate = getURL("http://update0.jdownloader.org/pluginstuff/tbupdate.js") + rtUpdate = self.decrypt(rtUpdate.splitlines()[1]) + # but we still need to fix the syntax to work with other engines than rhino + rtUpdate = re.sub(r'for each\(var (\w+) in(\[[^\]]+\])\)\{',r'zza=\2;for(var zzi=0;zzi<zza.length;zzi++){\1=zza[zzi];',rtUpdate) + rtUpdate = re.sub(r"for\((\w+)=",r"for(var \1=", rtUpdate) + + self.logDebug("rtUpdate") + self.setStorage("rtUpdate", rtUpdate) + self.setStorage("timestamp", timestamp()) + self.setStorage("version", self.__version__) + else: + self.logError("Unable to download, wait for update...") + self.tempOffline() + + return rtUpdate + + def getDownloadUrl(self, rtUpdate): + self.req.http.lastURL = self.url + + found = re.search("(/\w+/timeout\.js\?\w+=)([^\"\'<>]+)", self.html) + url = "http://turbobit.net%s%s" % (found.groups() if found else ('/files/timeout.js?ver=', ''.join(random.choice('0123456789ABCDEF') for x in range(32)))) + fun = self.load(url) + + self.setWait(65, False) + + for b in [1,3]: + self.jscode = "var id = \'%s\';var b = %d;var inn = \'%s\';%sout" % (self.file_info['ID'], b, quote(fun), rtUpdate) + + try: + out = self.js.eval(self.jscode) + self.logDebug("URL", self.js.engine, out) + if out.startswith('/download/'): + return "http://turbobit.net%s" % out.strip() + except Exception, e: + self.logError(e) + else: + if self.retries >= 2: + # retry with updated js + self.delStorage("rtUpdate") + self.retry() + + def decrypt(self, data): + cipher = ARC4.new(hexlify('E\x15\xa1\x9e\xa3M\xa0\xc6\xa0\x84\xb6H\x83\xa8o\xa0')) + return unhexlify(cipher.encrypt(unhexlify(data))) + + def getLocalTimeString(self): + lt = time.localtime() + tz = time.altzone if lt.tm_isdst else time.timezone + return "%s GMT%+03d%02d" % (time.strftime("%a %b %d %Y %H:%M:%S", lt), -tz // 3600, tz % 3600) + + def handlePremium(self): + self.logDebug("Premium download as user %s" % self.user) + self.downloadFile() + + def downloadFile(self): + found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) + if not found: self.parseError("download link") + self.url = "http://turbobit.net" + found.group('url') + self.logDebug(self.url) + self.download(self.url) + +getInfo = create_getInfo(TurbobitNet) diff --git a/pyload/plugins/hoster/TurbouploadCom.py b/pyload/plugins/hoster/TurbouploadCom.py new file mode 100644 index 000000000..6e81c6319 --- /dev/null +++ b/pyload/plugins/hoster/TurbouploadCom.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: zoidberg +""" + +import re +from module.plugins.internal.DeadHoster import DeadHoster as EasybytezCom, create_getInfo +#from module.plugins.internal.SimpleHoster import create_getInfo +#from module.plugins.hoster.EasybytezCom import EasybytezCom + +class TurbouploadCom(EasybytezCom): + __name__ = "TurbouploadCom" + __type__ = "hoster" + __pattern__ = r"http://(?:\w*\.)?turboupload.com/(\w+).*" + __version__ = "0.02" + __description__ = """turboupload.com""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + # shares code with EasybytezCom + + DIRECT_LINK_PATTERN = r'<a href="(http://turboupload.com/files/[^"]+)">\1</a>' + + def handleFree(self): + self.html = self.load(self.pyfile.url, post = self.getPostParameters(), ref = True, cookies = True) + found = re.search(self.DIRECT_LINK_PATTERN, self.html) + if not found: self.parseError('Download Link') + url = found.group(1) + self.logDebug('URL: ' + url) + self.download(url) + +getInfo = create_getInfo(TurbouploadCom) \ No newline at end of file diff --git a/pyload/plugins/hoster/TusfilesNet.py b/pyload/plugins/hoster/TusfilesNet.py new file mode 100644 index 000000000..517df8561 --- /dev/null +++ b/pyload/plugins/hoster/TusfilesNet.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo + +class TusfilesNet(XFileSharingPro): + __name__ = "TusfilesNet" + __type__ = "hoster" + __pattern__ = r"http://(www\.)?tusfiles\.net/\w{12}" + __version__ = "0.01" + __description__ = """Tusfiles.net hoster plugin""" + __author_name__ = ("stickell") + __author_mail__ = ("l.stickell@yahoo.it") + + FILE_INFO_PATTERN = r'<li>(?P<N>[^<]+)</li>\s+<li><b>Size:</b> <small>(?P<S>[\d.]+) (?P<U>\w+)</small></li>' + FILE_OFFLINE_PATTERN = r'The file you were looking for could not be found' + + HOSTER_NAME = "tusfiles.net" + +getInfo = create_getInfo(TusfilesNet) diff --git a/pyload/plugins/hoster/TwoSharedCom.py b/pyload/plugins/hoster/TwoSharedCom.py new file mode 100644 index 000000000..8401e0cb0 --- /dev/null +++ b/pyload/plugins/hoster/TwoSharedCom.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo +import re + +class TwoSharedCom(SimpleHoster): + __name__ = "TwoSharedCom" + __type__ = "hoster" + __pattern__ = r"http://[\w\.]*?2shared.com/(account/)?(download|get|file|document|photo|video|audio)/.*" + __version__ = "0.11" + __description__ = """2Shared Download Hoster""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + FILE_NAME_PATTERN = r'<h1>(?P<N>.*)</h1>' + FILE_SIZE_PATTERN = r'<span class="dtitle">File size:</span>\s*(?P<S>[0-9,.]+) (?P<U>[kKMG])i?B' + FILE_OFFLINE_PATTERN = r'The file link that you requested is not valid\.|This file was deleted\.' + DOWNLOAD_URL_PATTERN = r"window.location ='([^']+)';" + + def setup(self): + self.resumeDownload = self.multiDL = True + + def handleFree(self): + found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) + if not found: self.parseError('Download link') + link = found.group(1) + self.logDebug("Download URL %s" % link) + + self.download(link) + +getInfo = create_getInfo(TwoSharedCom) + \ No newline at end of file diff --git a/pyload/plugins/hoster/UlozTo.py b/pyload/plugins/hoster/UlozTo.py new file mode 100644 index 000000000..5c38fdaad --- /dev/null +++ b/pyload/plugins/hoster/UlozTo.py @@ -0,0 +1,156 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: zoidberg +""" + +import re +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo + +def convertDecimalPrefix(m): + # decimal prefixes used in filesize and traffic + return ("%%.%df" % {'k':3,'M':6,'G':9}[m.group(2)] % float(m.group(1))).replace('.','') + +class UlozTo(SimpleHoster): + __name__ = "UlozTo" + __type__ = "hoster" + __pattern__ = r"http://(\w*\.)?(uloz\.to|ulozto\.(cz|sk|net)|bagruj.cz|zachowajto.pl)/(?:live/)?(?P<id>\w+/[^/?]*)" + __version__ = "0.92" + __description__ = """uloz.to""" + __author_name__ = ("zoidberg") + + FILE_NAME_PATTERN = r'<a href="#download" class="jsShowDownload">(?P<N>[^<]+)</a>' + FILE_SIZE_PATTERN = r'<span id="fileSize">.*?(?P<S>[0-9.]+\s[kMG]?B)</span>' + FILE_INFO_PATTERN = r'<p>File <strong>(?P<N>[^<]+)</strong> is password protected</p>' + FILE_OFFLINE_PATTERN = r'<title>404 - Page not found|

        File (has been deleted|was banned)

        ' + FILE_SIZE_REPLACEMENTS = [('([0-9.]+)\s([kMG])B', convertDecimalPrefix)] + FILE_URL_REPLACEMENTS = [(r"(?<=http://)([^/]+)", "www.ulozto.net")] + + PASSWD_PATTERN = r'
        ' + VIPLINK_PATTERN = r'' + FREE_URL_PATTERN = r'
        \s*
      • Error rewriting the text.
        • '), + "offline": re.compile(self.FILE_OFFLINE_PATTERN), + "passwd": self.PASSWD_PATTERN, + "server_error": 'src="http://img.ulozto.cz/error403/vykricnik.jpg"', #paralell dl, server overload etc. + "not_found": "Ulož.to" + }) + + if check == "wrong_captcha": + self.delStorage("captcha_id") + self.delStorage("captcha_text") + self.invalidCaptcha() + self.retry(reason="Wrong captcha code") + elif check == "offline": + self.offline() + elif check == "passwd": + self.fail("Wrong password") + elif check == "server_error": + self.logError("Server error, try downloading later") + self.multiDL = False + self.setWait(3600, True) + self.wait() + self.retry() + elif check == "not_found": + self.fail("Server error - file not downloadable") + +getInfo = create_getInfo(UlozTo) diff --git a/pyload/plugins/hoster/UloziskoSk.py b/pyload/plugins/hoster/UloziskoSk.py new file mode 100644 index 000000000..c607e7a5b --- /dev/null +++ b/pyload/plugins/hoster/UloziskoSk.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: zoidberg +""" + +import re +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, PluginParseError + +class UloziskoSk(SimpleHoster): + __name__ = "UloziskoSk" + __type__ = "hoster" + __pattern__ = r"http://(\w*\.)?ulozisko.sk/.*" + __version__ = "0.23" + __description__ = """Ulozisko.sk""" + __author_name__ = ("zoidberg") + + URL_PATTERN = r'' + ID_PATTERN = r'' + FILE_NAME_PATTERN = r'
        (?P[^<]+)
        ' + FILE_SIZE_PATTERN = ur'Veľkosť súboru: (?P[0-9.]+) (?P[kKMG])i?B
        ' + CAPTCHA_PATTERN = r'' + FILE_OFFLINE_PATTERN = ur'Zadaný súbor neexistuje z jedného z nasledujúcich dôvodov:' + IMG_PATTERN = ur'PRE ZVÄČŠENIE KLIKNITE NA OBRÁZOK
        ' + + def process(self, pyfile): + self.html = self.load(pyfile.url, decode=True) + self.getFileInfo() + + found = re.search(self.IMG_PATTERN, self.html) + if found: + url = "http://ulozisko.sk" + found.group(1) + self.download(url) + else: + self.handleFree() + + def handleFree(self): + found = re.search(self.URL_PATTERN, self.html) + if found is None: raise PluginParseError('URL') + parsed_url = 'http://www.ulozisko.sk' + found.group(1) + + found = re.search(self.ID_PATTERN, self.html) + if found is None: raise PluginParseError('ID') + id = found.group(1) + + self.logDebug('URL:' + parsed_url + ' ID:' + id) + + found = re.search(self.CAPTCHA_PATTERN, self.html) + if found is None: raise PluginParseError('CAPTCHA') + captcha_url = 'http://www.ulozisko.sk' + found.group(1) + + captcha = self.decryptCaptcha(captcha_url, cookies=True) + + self.logDebug('CAPTCHA_URL:' + captcha_url + ' CAPTCHA:' + captcha) + + self.download(parsed_url, post={ + "antispam": captcha, + "id": id, + "name": self.pyfile.name, + "but": "++++STIAHNI+S%DABOR++++" + }) + +getInfo = create_getInfo(UloziskoSk) diff --git a/pyload/plugins/hoster/UnibytesCom.py b/pyload/plugins/hoster/UnibytesCom.py new file mode 100644 index 000000000..3c8552271 --- /dev/null +++ b/pyload/plugins/hoster/UnibytesCom.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: zoidberg +""" + +import re +from pycurl import FOLLOWLOCATION +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo + +class UnibytesCom(SimpleHoster): + __name__ = "UnibytesCom" + __type__ = "hoster" + __pattern__ = r"http://(www\.)?unibytes\.com/[a-zA-Z0-9-._ ]{11}B" + __version__ = "0.1" + __description__ = """UniBytes.com""" + __author_name__ = ("zoidberg") + + FILE_INFO_PATTERN = r']*?id="fileName"[^>]*>(?P[^>]+)\s*\((?P\d.*?)\)' + DOMAIN = 'http://www.unibytes.com' + + WAIT_PATTERN = r'Wait for (\d+) sec' + DOWNLOAD_LINK_PATTERN = r'Download' + + def handleFree(self): + action, post_data = self.parseHtmlForm('id="startForm"') + self.req.http.c.setopt(FOLLOWLOCATION, 0) + + for i in range(8): + self.logDebug(action, post_data) + self.html = self.load(self.DOMAIN + action, post = post_data) + + found = re.search(r'location:\s*(\S+)', self.req.http.header, re.I) + if found: + url = found.group(1) + break + + if '>Somebody else is already downloading using your IP-address<' in self.html: + self.setWait(600, True) + self.wait() + self.retry() + + if post_data['step'] == 'last': + found = re.search(self.DOWNLOAD_LINK_PATTERN, self.html) + if found: + url = found.group(1) + self.correctCaptcha() + break + else: + self.invalidCaptcha() + + last_step = post_data['step'] + action, post_data = self.parseHtmlForm('id="stepForm"') + + if last_step == 'timer': + found = re.search(self.WAIT_PATTERN, self.html) + self.setWait(int(found.group(1)) if found else 60, False) + self.wait() + elif last_step in ('captcha', 'last'): + post_data['captcha'] = self.decryptCaptcha(self.DOMAIN + '/captcha.jpg') + else: + self.fail("No valid captcha code entered") + + self.logDebug('Download link: ' + url) + self.req.http.c.setopt(FOLLOWLOCATION, 1) + self.download(url) + +getInfo = create_getInfo(UnibytesCom) \ No newline at end of file diff --git a/pyload/plugins/hoster/UploadStationCom.py b/pyload/plugins/hoster/UploadStationCom.py new file mode 100644 index 000000000..96dc7b577 --- /dev/null +++ b/pyload/plugins/hoster/UploadStationCom.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +from module.plugins.hoster.FileserveCom import FileserveCom, checkFile +from module.plugins.Plugin import chunks + +class UploadStationCom(FileserveCom): + __name__ = "UploadStationCom" + __type__ = "hoster" + __pattern__ = r"http://(?:www\.)?uploadstation\.com/file/(?P[A-Za-z0-9]+)" + __version__ = "0.51" + __description__ = """UploadStation.Com File Download Hoster""" + __author_name__ = ("fragonib", "zoidberg") + __author_mail__ = ("fragonib[AT]yahoo[DOT]es", "zoidberg@mujmail.cz") + + URLS = ['http://www.uploadstation.com/file/', 'http://www.uploadstation.com/check-links.php', 'http://www.uploadstation.com/checkReCaptcha.php'] + LINKCHECK_TR = r'
        (.*?)\t{9}
        ' + LINKCHECK_TD = r'
        (?:<[^>]*>| )*([^<]*)' + + LONG_WAIT_PATTERN = r'

        You have to wait (\d+) (\w+) to download the next file\.

        ' + +def getInfo(urls): + for chunk in chunks(urls, 100): yield checkFile(UploadStationCom, chunk) \ No newline at end of file diff --git a/pyload/plugins/hoster/UploadedTo.py b/pyload/plugins/hoster/UploadedTo.py new file mode 100644 index 000000000..5aa08891e --- /dev/null +++ b/pyload/plugins/hoster/UploadedTo.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 -*- + +# Test links (random.bin): +# http://ul.to/044yug9o +# http://ul.to/gzfhd0xs + +import re +from time import sleep + +from module.utils import html_unescape, parseFileSize, chunks + +from module.plugins.Hoster import Hoster +from module.network.RequestFactory import getURL +from module.plugins.ReCaptcha import ReCaptcha + +key = "bGhGMkllZXByd2VEZnU5Y2NXbHhYVlZ5cEE1bkEzRUw=".decode('base64') + + +def getID(url): + """ returns id from file url""" + m = re.match(UploadedTo.__pattern__, url) + return m.group('ID') + + +def getAPIData(urls): + post = {"apikey": key} + + idMap = {} + + for i, url in enumerate(urls): + id = getID(url) + post["id_%s" % i] = id + idMap[id] = url + + for i in xrange(5): + api = unicode(getURL("http://uploaded.net/api/filemultiple", post=post, decode=False), 'iso-8859-1') + if api != "can't find request": + break + else: + sleep(3) + + result = {} + + if api: + for line in api.splitlines(): + data = line.split(",", 4) + if data[1] in idMap: + result[data[1]] = (data[0], data[2], data[4], data[3], idMap[data[1]]) + + return result + + +def parseFileInfo(self, url='', html=''): + if not html and hasattr(self, "html"): html = self.html + name, size, status, found, fileid = url, 0, 3, None, None + + if re.search(self.FILE_OFFLINE_PATTERN, html): + # File offline + status = 1 + else: + found = re.search(self.FILE_INFO_PATTERN, html) + if found: + name, fileid = html_unescape(found.group('N')), found.group('ID') + size = parseFileSize(found.group('S')) + status = 2 + + return name, size, status, fileid + + +def getInfo(urls): + for chunk in chunks(urls, 80): + result = [] + + api = getAPIData(chunk) + + for data in api.itervalues(): + if data[0] == "online": + result.append((html_unescape(data[2]), data[1], 2, data[4])) + + elif data[0] == "offline": + result.append((data[4], 0, 1, data[4])) + + yield result + + +class UploadedTo(Hoster): + __name__ = "UploadedTo" + __type__ = "hoster" + __pattern__ = r"https?://[\w\.-]*?(uploaded\.(to|net)|ul\.to)(/file/|/?\?id=|.*?&id=|/)(?P\w+)" + __version__ = "0.70" + __description__ = """Uploaded.net Download Hoster""" + __author_name__ = ("spoob", "mkaay", "zoidberg", "netpok", "stickell") + __author_mail__ = ("spoob@pyload.org", "mkaay@mkaay.de", "zoidberg@mujmail.cz", "netpok@gmail.com", "l.stickell@yahoo.it") + + FILE_INFO_PATTERN = r'(?P[^<]+)  \s*]*>(?P[^<]+)' + FILE_OFFLINE_PATTERN = r'Error: 404' + DL_LIMIT_PATTERN = "You have reached the max. number of possible free downloads for this hour" + + def setup(self): + self.html = None + self.multiDL = False + self.resumeDownload = False + self.url = False + self.chunkLimit = 1 # critical problems with more chunks + if self.account: + self.premium = self.account.getAccountInfo(self.user)["premium"] + if self.premium: + self.multiDL = True + self.resumeDownload = True + + self.fileID = getID(self.pyfile.url) + self.pyfile.url = "http://uploaded.net/file/%s" % self.fileID + + def process(self, pyfile): + self.load("http://uploaded.net/language/en", just_header=True) + + api = getAPIData([pyfile.url]) + + # TODO: fallback to parse from site, because api sometimes delivers wrong status codes + + if not api: + self.logWarning("No response for API call") + + self.html = unicode(self.load(pyfile.url, decode=False), 'iso-8859-1') + name, size, status, self.fileID = parseFileInfo(self) + self.logDebug(name, size, status, self.fileID) + if status == 1: + self.offline() + elif status == 2: + pyfile.name, pyfile.size = name, size + else: + self.fail('Parse error - file info') + elif api == 'Access denied': + self.fail(_("API key invalid")) + + else: + if self.fileID not in api: + self.offline() + + self.data = api[self.fileID] + if self.data[0] != "online": + self.offline() + + pyfile.name = html_unescape(self.data[2]) + + # self.pyfile.name = self.get_file_name() + + if self.premium: + self.handlePremium() + else: + self.handleFree() + + + def handlePremium(self): + info = self.account.getAccountInfo(self.user, True) + self.log.debug("%(name)s: Use Premium Account (%(left)sGB left)" % {"name": self.__name__, + "left": info["trafficleft"] / 1024 / 1024}) + if int(self.data[1]) / 1024 > info["trafficleft"]: + self.log.info(_("%s: Not enough traffic left" % self.__name__)) + self.account.empty(self.user) + self.resetAccount() + self.fail(_("Traffic exceeded")) + + header = self.load("http://uploaded.net/file/%s" % self.fileID, just_header=True) + if "location" in header: + #Direct download + print "Direct Download: " + header['location'] + self.download(header['location']) + else: + #Indirect download + self.html = self.load("http://uploaded.net/file/%s" % self.fileID) + found = re.search(r'
        (\d+) seconds", self.html) + if not found: + self.fail("File not downloadable for free users") + self.setWait(int(found.group(1))) + + js = self.load("http://uploaded.net/js/download.js", decode=True) + + challengeId = re.search(r'Recaptcha\.create\("([^"]+)', js) + + url = "http://uploaded.net/io/ticket/captcha/%s" % self.fileID + downloadURL = "" + + for i in range(5): + #self.req.lastURL = str(self.url) + re_captcha = ReCaptcha(self) + challenge, result = re_captcha.challenge(challengeId.group(1)) + options = {"recaptcha_challenge_field": challenge, "recaptcha_response_field": result} + self.wait() + + result = self.load(url, post=options) + self.logDebug("result: %s" % result) + + if "limit-size" in result: + self.fail("File too big for free download") + elif "limit-slot" in result: # Temporary restriction so just wait a bit + self.setWait(30 * 60, True) + self.wait() + self.retry() + elif "limit-parallel" in result: + self.fail("Cannot download in parallel") + elif self.DL_LIMIT_PATTERN in result: # limit-dl + self.setWait(3 * 60 * 60, True) + self.wait() + self.retry() + elif 'err:"captcha"' in result: + self.logError("ul.net captcha is disabled") + self.invalidCaptcha() + elif "type:'download'" in result: + self.correctCaptcha() + downloadURL = re.search("url:'([^']+)", result).group(1) + break + else: + self.fail("Unknown error '%s'") + + if not downloadURL: + self.fail("No Download url retrieved/all captcha attempts failed") + + self.download(downloadURL, disposition=True) + check = self.checkDownload({"limit-dl": self.DL_LIMIT_PATTERN}) + if check == "limit-dl": + self.setWait(3 * 60 * 60, True) + self.wait() + self.retry() diff --git a/pyload/plugins/hoster/UploadheroCom.py b/pyload/plugins/hoster/UploadheroCom.py new file mode 100644 index 000000000..502f849af --- /dev/null +++ b/pyload/plugins/hoster/UploadheroCom.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: zoidberg +""" + +import re +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo + +class UploadheroCom(SimpleHoster): + __name__ = "UploadheroCom" + __type__ = "hoster" + __pattern__ = r"http://(?:www\.)?uploadhero\.com?/dl/\w+" + __version__ = "0.13" + __description__ = """UploadHero.com plugin""" + __author_name__ = ("mcmyst", "zoidberg") + __author_mail__ = ("mcmyst@hotmail.fr", "zoidberg@mujmail.cz") + + SH_COOKIES = [("http://uploadhero.com", "lang", "en")] + FILE_NAME_PATTERN = r'
        (?P.*?)
        ' + FILE_SIZE_PATTERN = r'Taille du fichier : (?P.*?)' + FILE_OFFLINE_PATTERN = r'

        |

        Le lien du fichier ci-dessus n\'existe plus.' + + DOWNLOAD_URL_PATTERN = r'(\d+).*\s*(\d+)' + + CAPTCHA_PATTERN = r'"(/captchadl\.php\?[a-z0-9]+)"' + FREE_URL_PATTERN = r'var magicomfg = \'"/]+)"' + + def handleFree(self): + self.checkErrors() + + found = re.search(self.CAPTCHA_PATTERN, self.html) + if not found: self.parseError("Captcha URL") + captcha_url = "http://uploadhero.com" + found.group(1) + + for i in range(5): + captcha = self.decryptCaptcha(captcha_url) + self.html = self.load(self.pyfile.url, get = {"code": captcha}) + found = re.search(self.FREE_URL_PATTERN, self.html) + if found: + self.correctCaptcha() + download_url = found.group(1) or found.group(2) + break + else: + self.invalidCaptcha() + else: + self.fail("No valid captcha code entered") + + self.download(download_url) + + def handlePremium(self): + self.log.debug("%s: Use Premium Account" % self.__name__) + self.html = self.load(self.pyfile.url) + link = re.search(self.DOWNLOAD_URL_PATTERN, self.html).group(1) + self.log.debug("Downloading link : '%s'" % link) + self.download(link) + + def checkErrors(self): + found = re.search(self.IP_BLOCKED_PATTERN, self.html) + if found: + self.html = self.load("http://uploadhero.com%s" % found.group(1)) + + found = re.search(self.IP_WAIT_PATTERN, self.html) + wait_time = (int(found.group(1)) * 60 + int(found.group(2))) if found else 300 + self.setWait(wait_time, True) + self.wait() + self.retry() + +getInfo = create_getInfo(UploadheroCom) diff --git a/pyload/plugins/hoster/UploadingCom.py b/pyload/plugins/hoster/UploadingCom.py new file mode 100644 index 000000000..84040eb4d --- /dev/null +++ b/pyload/plugins/hoster/UploadingCom.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: jeix +""" + +import re +from pycurl import HTTPHEADER +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, timestamp +from module.common.json_layer import json_loads + +class UploadingCom(SimpleHoster): + __name__ = "UploadingCom" + __type__ = "hoster" + __pattern__ = r"http://(?:www\.)?uploading\.com/files/(?:get/)?(?P[\w\d]+)" + __version__ = "0.32" + __description__ = """Uploading.Com File Download Hoster""" + __author_name__ = ("jeix", "mkaay", "zoidberg") + __author_mail__ = ("jeix@hasnomail.de", "mkaay@mkaay.de", "zoidberg@mujmail.cz") + + FILE_NAME_PATTERN = r'Download (?P<N>.*?) for free on uploading.com' + FILE_SIZE_PATTERN = r'File size: (?P.*?)' + FILE_OFFLINE_PATTERN = r'The requested file is not found' + + def process(self, pyfile): + # set lang to english + self.req.cj.setCookie("uploading.com", "lang", "1") + self.req.cj.setCookie("uploading.com", "language", "1") + self.req.cj.setCookie("uploading.com", "setlang", "en") + self.req.cj.setCookie("uploading.com", "_lang", "en") + + if not "/get/" in self.pyfile.url: + self.pyfile.url = self.pyfile.url.replace("/files", "/files/get") + + self.html = self.load(pyfile.url, decode = True) + self.file_info = self.getFileInfo() + + if self.premium: + self.handlePremium() + else: + self.handleFree() + + def handlePremium(self): + postData = {'action': 'get_link', + 'code': self.file_info['ID'], + 'pass': 'undefined'} + + self.html = self.load('http://uploading.com/files/get/?JsHttpRequest=%d-xml' % timestamp(), post=postData) + url = re.search(r'"link"\s*:\s*"(.*?)"', self.html) + if url: + url = url.group(1).replace("\\/", "/") + self.download(url) + + raise Exception("Plugin defect.") + + def handleFree(self): + found = re.search('

        ((Daily )?Download Limit)

        ', self.html) + if found: + self.pyfile.error = found.group(1) + self.logWarning(self.pyfile.error) + self.retry(max_tries=6, wait_time = 21600 if found.group(2) else 900, reason = self.pyfile.error) + + ajax_url = "http://uploading.com/files/get/?ajax" + self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) + self.req.http.lastURL = self.pyfile.url + + response = json_loads(self.load(ajax_url, post = {'action': 'second_page', 'code': self.file_info['ID']})) + if 'answer' in response and 'wait_time' in response['answer']: + wait_time = int(response['answer']['wait_time']) + self.log.info("%s: Waiting %d seconds." % (self.__name__, wait_time)) + self.setWait(wait_time) + self.wait() + else: + self.pluginParseError("AJAX/WAIT") + + response = json_loads(self.load(ajax_url, post = {'action': 'get_link', 'code': self.file_info['ID'], 'pass': 'false'})) + if 'answer' in response and 'link' in response['answer']: + url = response['answer']['link'] + else: + self.pluginParseError("AJAX/URL") + + self.html = self.load(url) + found = re.search(r'\s*Download File\s*]*>(?P[^>]+)\s*[^\(]*\((?P[^\)]+)\)' + FILE_OFFLINE_PATTERN = r'
        File Not Found
        ' + HOSTER_NAME = "uptobox.com" + + def setup(self): + self.resumeDownload = self.multiDL = self.premium + self.chunkLimit = 1 + +getInfo = create_getInfo(UptoboxCom) \ No newline at end of file diff --git a/pyload/plugins/hoster/VeehdCom.py b/pyload/plugins/hoster/VeehdCom.py new file mode 100644 index 000000000..23048b831 --- /dev/null +++ b/pyload/plugins/hoster/VeehdCom.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import re +from module.plugins.Hoster import Hoster + +class VeehdCom(Hoster): + __name__ = 'VeehdCom' + __type__ = 'hoster' + __pattern__ = r'http://veehd\.com/video/\d+_\S+' + __config__ = [ + ('filename_spaces', 'bool', "Allow spaces in filename", 'False'), + ('replacement_char', 'str', "Filename replacement character", '_'), + ] + __version__ = '0.2' + __description__ = """Veehd.com Download Hoster""" + __author_name__ = ('cat') + __author_mail__ = ('cat@pyload') + + def _debug(self, msg): + self.log.debug('[%s] %s' % (self.__name__, msg)) + + def setup(self): + self.html = None + self.multiDL = True + self.req.canContinue = True + + def process(self, pyfile): + self.download_html() + if not self.file_exists(): + self.offline() + + pyfile.name = self.get_file_name() + self.download(self.get_file_url()) + + def download_html(self): + url = self.pyfile.url + self._debug("Requesting page: %s" % (repr(url),)) + self.html = self.load(url) + + def file_exists(self): + if self.html is None: + self.download_html() + + if 'Veehd' in self.html: + return False + return True + + def get_file_name(self): + if self.html is None: + self.download_html() + + match = re.search(r']*>([^<]+) on Veehd', self.html) + if not match: + self.fail("video title not found") + name = match.group(1) + + # replace unwanted characters in filename + if self.getConf('filename_spaces'): + pattern = '[^0-9A-Za-z\.\ ]+' + else: + pattern = '[^0-9A-Za-z\.]+' + + name = re.sub(pattern, self.getConf('replacement_char'), + name) + return name + '.avi' + + def get_file_url(self): + """ returns the absolute downloadable filepath + """ + if self.html is None: + self.download_html() + + match = re.search(r'. + + @author: zoidberg +""" + +#similar to coolshare.cz (down) + +import re +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo +from module.network.HTTPRequest import BadHeader +from module.utils import html_unescape + +class WarserverCz(SimpleHoster): + __name__ = "WarserverCz" + __type__ = "hoster" + __pattern__ = r"http://(?:\w*\.)?warserver.cz/stahnout/(?P\d+)/.+" + __version__ = "0.12" + __description__ = """Warserver.cz""" + __author_name__ = ("zoidberg") + + FILE_NAME_PATTERN = r'(?P[^<]+)' + FILE_SIZE_PATTERN = r'
      • Velikost: (?P[^<]+)' + FILE_OFFLINE_PATTERN = r'

        Soubor nenalezen

        ' + + PREMIUM_URL_PATTERN = r'href="(http://[^/]+/dwn-premium.php.*?)"' + DOMAIN = "http://csd01.coolshare.cz" + + DOMAIN = "http://s01.warserver.cz" + + def handleFree(self): + try: + self.download("%s/dwn-free.php?fid=%s" % (self.DOMAIN, self.file_info['ID'])) + except BadHeader, e: + self.logError(e) + if e.code == 403: + self.longWait(60,60) + else: raise + self.checkDownloadedFile() + + def handlePremium(self): + found = re.search(self.PREMIUM_URL_PATTERN, self.html) + if not found: self.parseError("Premium URL") + url = html_unescape(found.group(1)) + self.logDebug("Premium URL: " + url) + if not url.startswith("http://"): self.resetAccount() + self.download(url) + self.checkDownloadedFile() + + def checkDownloadedFile(self): + check = self.checkDownload({ + "offline": ">404 Not Found<" + }) + + if check == "offline": + self.offline() + +getInfo = create_getInfo(WarserverCz) \ No newline at end of file diff --git a/pyload/plugins/hoster/WebshareCz.py b/pyload/plugins/hoster/WebshareCz.py new file mode 100644 index 000000000..195e65a93 --- /dev/null +++ b/pyload/plugins/hoster/WebshareCz.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: zoidberg +""" + +import re +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo +from module.network.HTTPRequest import BadHeader + +class WebshareCz(SimpleHoster): + __name__ = "WebshareCz" + __type__ = "hoster" + __pattern__ = r"http://(\w+\.)?webshare.cz/(stahnout/)?(?P\w{10})-.+" + __version__ = "0.12" + __description__ = """WebShare.cz""" + __author_name__ = ("zoidberg") + + FILE_NAME_PATTERN = r'

        Stahujete soubor:

        \s*
        (?P[^<]+)
        ' + FILE_SIZE_PATTERN = r'

        Velikost souboru je:

        \s*
        (?P[^<]+)
        ' + FILE_OFFLINE_PATTERN = r'

        Soubor ".*?" nebyl nalezen.

        ' + + DOWNLOAD_LINK_PATTERN = r'id="download_link" href="(?P.*?)"' + + def setup(self): + self.multiDL = True + + def handleFree(self): + url_a = re.search(r"(var l.*)", self.html).group(1) + url_b = re.search(r"(var keyStr.*)", self.html).group(1) + url = self.js.eval("%s\n%s\ndec(l)" % (url_a, url_b)) + + self.logDebug('Download link: ' + url) + self.download(url) + +getInfo = create_getInfo(WebshareCz) \ No newline at end of file diff --git a/pyload/plugins/hoster/WrzucTo.py b/pyload/plugins/hoster/WrzucTo.py new file mode 100644 index 000000000..4a5e89f22 --- /dev/null +++ b/pyload/plugins/hoster/WrzucTo.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: zoidberg +""" + +import re +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo +from pycurl import HTTPHEADER + +class WrzucTo(SimpleHoster): + __name__ = "WrzucTo" + __type__ = "hoster" + __pattern__ = r"http://(?:\w+\.)*?wrzuc\.to/([a-zA-Z0-9]+(\.wt|\.html)|(\w+/?linki/[a-zA-Z0-9]+))" + __version__ = "0.01" + __description__ = """Wrzuc.to plugin - free only""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + SH_COOKIES = [("http://www.wrzuc.to", "language", "en")] + FILE_SIZE_PATTERN = r'class="info">\s*\s*(?P.*?)' + FILE_NAME_PATTERN = r'id="file_info">\s*(?P.*?)' + + def setup(self): + self.multiDL = True + + def handleFree(self): + data = dict(re.findall(r'(md5|file): "(.*?)"', self.html)) + if len(data) != 2: self.parseError('File ID') + + self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) + self.req.http.lastURL = self.pyfile.url + self.load("http://www.wrzuc.to/ajax/server/prepair", post = {"md5": data['md5']}) + + self.req.http.lastURL = self.pyfile.url + self.html = self.load("http://www.wrzuc.to/ajax/server/download_link", post = {"file": data['file']}) + + data.update(re.findall(r'"(download_link|server_id)":"(.*?)"', self.html)) + if len(data) != 4: self.parseError('Download URL') + + download_url = "http://%s.wrzuc.to/pobierz/%s" % (data['server_id'], data['download_link']) + self.logDebug("Download URL: %s" % download_url) + self.download(download_url) + +getInfo = create_getInfo(WrzucTo) + diff --git a/pyload/plugins/hoster/WuploadCom.py b/pyload/plugins/hoster/WuploadCom.py new file mode 100644 index 000000000..1a0eb442b --- /dev/null +++ b/pyload/plugins/hoster/WuploadCom.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +import string +from urllib import unquote + +from module.plugins.Hoster import Hoster +from module.plugins.ReCaptcha import ReCaptcha +from module.plugins.Plugin import chunks + +from module.network.RequestFactory import getURL +from module.common.json_layer import json_loads + + +def getInfo(urls): + for chunk in chunks(urls, 20): + result = [] + ids = dict() + for url in chunk: + id = getId(url) + if id: + ids[id] = url + else: + result.append((None, 0, 1, url)) + + if len(ids) > 0: + check_url = "http://api.wupload.com/link?method=getInfo&format=json&ids=" + ",".join(ids.keys()) + response = json_loads(getURL(check_url).decode("utf8", "ignore")) + for item in response["FSApi_Link"]["getInfo"]["response"]["links"]: + if item["status"] != "AVAILABLE": + result.append((None, 0, 1, ids[str(item["id"])])) + else: + result.append((unquote(item["filename"]), item["size"], 2, ids[str(item["id"])])) + yield result + + +def getId(url): + match = re.search(WuploadCom.FILE_ID_PATTERN, url) + if match: + return string.replace(match.group("id"), "/", "-") + else: + return None + + +class WuploadCom(Hoster): + __name__ = "WuploadCom" + __type__ = "hoster" + __pattern__ = r"http://[\w\.]*?wupload\..*?/file/(([a-z][0-9]+/)?[0-9]+)(/.*)?" + __version__ = "0.20" + __description__ = """Wupload com""" + __author_name__ = ("jeix", "paulking") + __author_mail__ = ("jeix@hasnomail.de", "") + + API_ADDRESS = "http://api.wupload.com" + URL_DOMAIN_PATTERN = r'(?P.*?)(?P.wupload\..+?)(?P/.*)' + FILE_ID_PATTERN = r'/file/(?P([a-z][0-9]+/)?[0-9]+)(/.*)?' + FILE_LINK_PATTERN = r'

        Download Link

        \s*]*>([^<]+)' + OVR_KILL_LINK_PATTERN = r'

        Delete Link

        \s*]*>([^<]+)' + CAPTCHA_URL_PATTERN = r'(http://[^"\']+?/captchas?/[^"\']+)' + RECAPTCHA_URL_PATTERN = r'http://[^"\']+?recaptcha[^"\']+?\?k=([^"\']+)"' + CAPTCHA_DIV_PATTERN = r'Enter code.*?(.*?)

        Download Now' + WAIT_TIME_PATTERN = r'countDownDelay = (?P\d+)' + WAIT_TM_PATTERN = r"name='tm' value='(.*?)' />" + WAIT_TM_HASH_PATTERN = r"name='tm_hash' value='(.*?)' />" + CAPTCHA_TYPE1_PATTERN = r'Recaptcha.create\("(.*?)",' + CAPTCHA_TYPE2_PATTERN = r'id="recaptcha_image"> 300: + self.wantReconnect = True + + self.setWait(wait) + self.logDebug("Waiting %d seconds." % wait) + self.wait() + + tm = re.search(self.WAIT_TM_PATTERN, self.html) + tm_hash = re.search(self.WAIT_TM_HASH_PATTERN, self.html) + + if tm and tm_hash: + tm = tm.group(1) + tm_hash = tm_hash.group(1) + self.html = self.load(url, post={"tm": tm, "tm_hash": tm_hash}) + self.handleErrors() + break + else: + self.html = self.load(url) + self.handleErrors() + waitSearch = re.search(self.WAIT_TIME_PATTERN, self.html) + + def handleErrors(self): + if "This file is available for premium users only." in self.html: + self.fail("need premium account for file") + + if "The file that you're trying to download is larger than" in self.html: + self.fail("need premium account for file") + + if "Free users may only download 1 file at a time" in self.html: + self.fail("only 1 file at a time for free users") + + if "Free user can not download files" in self.html: + self.fail("need premium account for file") + + if "Download session in progress" in self.html: + self.fail("already downloading") + + if "This file is password protected" in self.html: + self.fail("This file is password protected") + + if "An Error Occurred" in self.html: + self.fail("A server error occured.") + + if "This file was deleted" in self.html: + self.offline() diff --git a/pyload/plugins/hoster/X7To.py b/pyload/plugins/hoster/X7To.py new file mode 100644 index 000000000..965d84543 --- /dev/null +++ b/pyload/plugins/hoster/X7To.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +import re + +from module.plugins.Hoster import Hoster + +from module.network.RequestFactory import getURL + +def getInfo(urls): + yield [(url, 0, 1, url) for url in urls] + + +class X7To(Hoster): + __name__ = "X7To" + __type__ = "hoster" + __pattern__ = r"http://(?:www.)?x7.to/" + __version__ = "0.3" + __description__ = """X7.To File Download Hoster""" + __author_name__ = ("ernieb") + __author_mail__ = ("ernieb") + + FILE_INFO_PATTERN=r'', self.html) + if overLimit: + self.logDebug("over limit, falling back to free") + self.handleFree() + else: + realurl = re.search(r'', self.html) + if realurl: + realurl = realurl.group(1) + self.logDebug("premium url found %s" % realurl) + else: + self.logDebug("premium link not found") + self.download(realurl) + + def handleFree(self): + # find file id + file_id = re.search(r"var dlID = '(.*?)'", self.html) + if not file_id: + self.fail("Free download id not found") + + file_url = "http://x7.to/james/ticket/dl/" + file_id.group(1) + self.logDebug("download id %s" % file_id.group(1)) + + self.html = self.load(file_url, ref=False, decode=True) + + # deal with errors + if "limit-dl" in self.html: + self.logDebug("Limit reached ... waiting") + self.setWait(900,True) + self.wait() + self.retry() + + if "limit-parallel" in self.html: + self.fail("Cannot download in parallel") + + # no waiting required, go to download + waitCheck = re.search(r"wait:(\d*),", self.html) + if waitCheck: + waitCheck = int(waitCheck.group(1)) + self.setWait(waitCheck) + self.wait() + + urlCheck = re.search(r"url:'(.*?)'", self.html) + url = None + if urlCheck: + url = urlCheck.group(1) + self.logDebug("free url found %s" % url) + + if url: + try: + self.download(url) + except: + self.logDebug("downloading url failed: %s" % url) + else: + self.fail("Free download url found") diff --git a/pyload/plugins/hoster/XFileSharingPro.py b/pyload/plugins/hoster/XFileSharingPro.py new file mode 100644 index 000000000..1120a2a8b --- /dev/null +++ b/pyload/plugins/hoster/XFileSharingPro.py @@ -0,0 +1,314 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: zoidberg +""" + +import re +from random import random +from urllib import unquote +from urlparse import urlparse +from pycurl import FOLLOWLOCATION, LOW_SPEED_TIME +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, PluginParseError +from module.plugins.ReCaptcha import ReCaptcha +from module.plugins.internal.CaptchaService import SolveMedia, AdsCaptcha +from module.utils import html_unescape + +class XFileSharingPro(SimpleHoster): + """ + Common base for XFileSharingPro hosters like EasybytezCom, CramitIn, FiledinoCom... + Some hosters may work straight away when added to __pattern__ + However, most of them will NOT work because they are either down or running a customized version + """ + __name__ = "XFileSharingPro" + __type__ = "hoster" + __pattern__ = r"^unmatchable$" + __version__ = "0.17" + __description__ = """XFileSharingPro common hoster base""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + FILE_NAME_PATTERN = r'[^<]+ \((?P[^<]+)\)' + FILE_INFO_PATTERN = r'Filename:(?P[^<]+)\s*.*?\((?P[^<]+)\)' + FILE_OFFLINE_PATTERN = r'<(b|h[1-6])>File Not Found' + + WAIT_PATTERN = r'.*?>(\d+)' + LONG_WAIT_PATTERN = r'(?P\d+(?=\s*hour))?.*?(?P\d+(?=\s*minute))?.*?(?P\d+(?=\s*second))?' + OVR_DOWNLOAD_LINK_PATTERN = r'

        • ' + SOLVEMEDIA_PATTERN = r'http:\/\/api\.solvemedia\.com\/papi\/challenge\.script\?k=(.*?)"' + ERROR_PATTERN = r'class=["\']err["\'][^>]*>(.*?)"," ",self.errmsg)) + + if 'wait' in self.errmsg: + wait_time = sum([int(v) * {"hour": 3600, "minute": 60, "second": 1}[u] for v, u in re.findall('(\d+)\s*(hour|minute|second)?', self.errmsg)]) + self.setWait(wait_time, True) + self.wait() + elif 'captcha' in self.errmsg: + self.invalidCaptcha() + elif 'premium' in self.errmsg and 'require' in self.errmsg: + self.fail("File can be downloaded by premium users only") + elif 'limit' in self.errmsg: + self.setWait(3600, True) + self.wait() + self.retry(25) + elif 'countdown' in self.errmsg or 'Expired session' in self.errmsg: + self.retry(3) + elif 'maintenance' in self.errmsg: + self.tempOffline() + elif 'download files up to' in self.errmsg: + self.fail("File too large for free download") + else: + self.fail(self.errmsg) + + else: + self.errmsg = None + + return self.errmsg + + def getPostParameters(self): + for i in range(3): + if not self.errmsg: self.checkErrors() + + if hasattr(self,"FORM_PATTERN"): + action, inputs = self.parseHtmlForm(self.FORM_PATTERN) + else: + action, inputs = self.parseHtmlForm(input_names={"op": re.compile("^download")}) + + if not inputs: + action, inputs = self.parseHtmlForm('F1') + if not inputs: + if self.errmsg: + self.retry() + else: + self.parseError("Form not found") + + self.logDebug(self.HOSTER_NAME, inputs) + + if 'op' in inputs and inputs['op'] in ('download2', 'download3'): + if "password" in inputs: + if self.passwords: + inputs['password'] = self.passwords.pop(0) + else: + self.fail("No or invalid passport") + + if not self.premium: + found = re.search(self.WAIT_PATTERN, self.html) + if found: + wait_time = int(found.group(1)) + 1 + self.setWait(wait_time, False) + else: + wait_time = 0 + + self.captcha = self.handleCaptcha(inputs) + + if wait_time: self.wait() + + self.errmsg = None + return inputs + + else: + inputs['referer'] = self.pyfile.url + + if self.premium: + inputs['method_premium'] = "Premium Download" + if 'method_free' in inputs: del inputs['method_free'] + else: + inputs['method_free'] = "Free Download" + if 'method_premium' in inputs: del inputs['method_premium'] + + self.html = self.load(self.pyfile.url, post = inputs, ref = True) + self.errmsg = None + + else: self.parseError('FORM: %s' % (inputs['op'] if 'op' in inputs else 'UNKNOWN')) + + def handleCaptcha(self, inputs): + found = re.search(self.RECAPTCHA_URL_PATTERN, self.html) + if found: + recaptcha_key = unquote(found.group(1)) + self.logDebug("RECAPTCHA KEY: %s" % recaptcha_key) + recaptcha = ReCaptcha(self) + inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(recaptcha_key) + return 1 + else: + found = re.search(self.CAPTCHA_URL_PATTERN, self.html) + if found: + captcha_url = found.group(1) + inputs['code'] = self.decryptCaptcha(captcha_url) + return 2 + else: + found = re.search(self.CAPTCHA_DIV_PATTERN, self.html, re.S) + if found: + captcha_div = found.group(1) + self.logDebug(captcha_div) + numerals = re.findall('(\d)', html_unescape(captcha_div)) + inputs['code'] = "".join([a[1] for a in sorted(numerals, key = lambda num: int(num[0]))]) + self.logDebug("CAPTCHA", inputs['code'], numerals) + return 3 + else: + found = re.search(self.SOLVEMEDIA_PATTERN, self.html) + if found: + captcha_key = found.group(1) + captcha = SolveMedia(self) + inputs['adcopy_challenge'], inputs['adcopy_response'] = captcha.challenge(captcha_key) + return 4 + return 0 + +getInfo = create_getInfo(XFileSharingPro) diff --git a/pyload/plugins/hoster/XHamsterCom.py b/pyload/plugins/hoster/XHamsterCom.py new file mode 100644 index 000000000..0779a78e6 --- /dev/null +++ b/pyload/plugins/hoster/XHamsterCom.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +from module.plugins.Hoster import Hoster +from urllib import unquote +from module.common.json_layer import json_loads + +def clean_json(json_expr): + json_expr = re.sub('[\n\r]', '', json_expr) + json_expr = re.sub(' +', '', json_expr) + json_expr = re.sub('\'','"', json_expr) + + return json_expr + +class XHamsterCom(Hoster): + __name__ = "XHamsterCom" + __type__ = "hoster" + __pattern__ = r"http://(www\.)?xhamster\.com/movies/.+" + __version__ = "0.1" + __config__ = [("type", ".mp4;.flv", "Preferred type", ".mp4")] + __description__ = """XHamster.com Video Download Hoster""" + + def setup(self): + self.html = None + + def process(self, pyfile): + self.pyfile = pyfile + + if not self.file_exists(): + self.offline() + + if self.getConfig("type"): + self.desired_fmt = self.getConf("type") + + self.pyfile.name = self.get_file_name() + self.desired_fmt + self.download(self.get_file_url()) + + def download_html(self): + url = self.pyfile.url + self.html = self.load(url) + + def get_file_url(self): + """ returns the absolute downloadable filepath + """ + if self.html is None: + self.download_html() + + flashvar_pattern = re.compile('flashvars = ({.*?});', re.DOTALL) + json_flashvar=flashvar_pattern.search(self.html) + + if json_flashvar is None: + self.fail("Parse error (flashvars)") + + j = clean_json(json_flashvar.group(1)) + flashvars = json_loads(j) + + if flashvars["srv"]: + srv_url = flashvars["srv"] + '/' + else: + self.fail("Parse error (srv_url)") + + if flashvars["url_mode"]: + url_mode = flashvars["url_mode"] + else: + self.fail("Parse error (url_mode)") + + + if self.desired_fmt == ".mp4": + file_url = re.search(r"(.*?) - xHamster\.com" + file_name = re.search(file_name_pattern, self.html) + if file_name is None: + file_name_pattern = r"

        (.*)

        " + file_name = re.search(file_name_pattern, self.html) + if file_name is None: + file_name_pattern = r"http://[www.]+xhamster\.com/movies/.*/(.*?)\.html?" + file_name = re.search(file_name_pattern, self.pyfile.url) + if file_name is None: + file_name_pattern = r"
        (.*)
        " + file_name = re.search(file_name_pattern, self.html) + if file_name is None: + return "Unknown" + + return file_name.group(1) + + def file_exists(self): + """ returns True or False + """ + if self.html is None: + self.download_html() + if re.search(r"(.*Video not found.*)", self.html) is not None: + return False + else: + return True diff --git a/pyload/plugins/hoster/XVideosCom.py b/pyload/plugins/hoster/XVideosCom.py new file mode 100644 index 000000000..b7f3f7b58 --- /dev/null +++ b/pyload/plugins/hoster/XVideosCom.py @@ -0,0 +1,19 @@ + +import re +import urllib + +from module.plugins.Hoster import Hoster + +class XVideosCom(Hoster): + __name__ = "XVideos.com" + __version__ = "0.1" + __pattern__ = r"http://www\.xvideos\.com/video([0-9]+)/.*" + __config__ = [] + + def process(self, pyfile): + site = self.load(pyfile.url) + pyfile.name = "%s (%s).flv" %( + re.search(r"

        ([^<]+). + + @author: jeix +""" + +from os.path import join +from os.path import exists +from os import makedirs +import re +import sys +import time +import socket, struct +from select import select +from module.utils import save_join + +from module.plugins.Hoster import Hoster + + +class Xdcc(Hoster): + __name__ = "Xdcc" + __version__ = "0.3" + __pattern__ = r'xdcc://.*?(/#?.*?)?/.*?/#?\d+/?' # xdcc://irc.Abjects.net/#channel/[XDCC]|Shit/#0004/ + __type__ = "hoster" + __config__ = [ + ("nick", "str", "Nickname", "pyload"), + ("ident", "str", "Ident", "pyloadident"), + ("realname", "str", "Realname", "pyloadreal") + ] + __description__ = """A Plugin that allows you to download from an IRC XDCC bot""" + __author_name__ = ("jeix") + __author_mail__ = ("jeix@hasnomail.com") + + def setup(self): + self.debug = 0 #0,1,2 + self.timeout = 30 + self.multiDL = False + + + + def process(self, pyfile): + # change request type + self.req = pyfile.m.core.requestFactory.getRequest(self.__name__, type="XDCC") + + self.pyfile = pyfile + for i in range(0,3): + try: + nmn = self.doDownload(pyfile.url) + self.log.debug("%s: Download of %s finished." % (self.__name__, nmn)) + return + except socket.error, e: + if hasattr(e, "errno"): + errno = e.errno + else: + errno = e.args[0] + + if errno in (10054,): + self.log.debug("XDCC: Server blocked our ip, retry in 5 min") + self.setWait(300) + self.wait() + continue + + self.fail("Failed due to socket errors. Code: %d" % errno) + + self.fail("Server blocked our ip, retry again later manually") + + + def doDownload(self, url): + self.pyfile.setStatus("waiting") # real link + + download_folder = self.config['general']['download_folder'] + location = join(download_folder, self.pyfile.package().folder.decode(sys.getfilesystemencoding())) + if not exists(location): + makedirs(location) + + m = re.search(r'xdcc://(.*?)/#?(.*?)/(.*?)/#?(\d+)/?', url) + server = m.group(1) + chan = m.group(2) + bot = m.group(3) + pack = m.group(4) + nick = self.getConf('nick') + ident = self.getConf('ident') + real = self.getConf('realname') + + temp = server.split(':') + ln = len(temp) + if ln == 2: + host, port = temp + elif ln == 1: + host, port = temp[0], 6667 + else: + self.fail("Invalid hostname for IRC Server (%s)" % server) + + + ####################### + # CONNECT TO IRC AND IDLE FOR REAL LINK + dl_time = time.time() + + sock = socket.socket() + sock.connect((host, int(port))) + if nick == "pyload": + nick = "pyload-%d" % (time.time() % 1000) # last 3 digits + sock.send("NICK %s\r\n" % nick) + sock.send("USER %s %s bla :%s\r\n" % (ident, host, real)) + time.sleep(3) + sock.send("JOIN #%s\r\n" % chan) + sock.send("PRIVMSG %s :xdcc send #%s\r\n" % (bot, pack)) + + # IRC recv loop + readbuffer = "" + done = False + retry = None + m = None + while True: + + # done is set if we got our real link + if done: + break + + if retry: + if time.time() > retry: + retry = None + dl_time = time.time() + sock.send("PRIVMSG %s :xdcc send #%s\r\n" % (bot, pack)) + + else: + if (dl_time + self.timeout) < time.time(): # todo: add in config + sock.send("QUIT :byebye\r\n") + sock.close() + self.fail("XDCC Bot did not answer") + + + fdset = select([sock], [], [], 0) + if sock not in fdset[0]: + continue + + readbuffer += sock.recv(1024) + temp = readbuffer.split("\n") + readbuffer = temp.pop() + + for line in temp: + if self.debug is 2: print "*> " + unicode(line, errors='ignore') + line = line.rstrip() + first = line.split() + + if first[0] == "PING": + sock.send("PONG %s\r\n" % first[1]) + + if first[0] == "ERROR": + self.fail("IRC-Error: %s" % line) + + msg = line.split(None, 3) + if len(msg) != 4: + continue + + msg = { \ + "origin":msg[0][1:], \ + "action":msg[1], \ + "target":msg[2], \ + "text" :msg[3][1:] \ + } + + + if nick == msg["target"][0:len(nick)] and "PRIVMSG" == msg["action"]: + if msg["text"] == "\x01VERSION\x01": + self.log.debug("XDCC: Sending CTCP VERSION.") + sock.send("NOTICE %s :%s\r\n" % (msg['origin'], "pyLoad! IRC Interface")) + elif msg["text"] == "\x01TIME\x01": + self.log.debug("Sending CTCP TIME.") + sock.send("NOTICE %s :%d\r\n" % (msg['origin'], time.time())) + elif msg["text"] == "\x01LAG\x01": + pass # don't know how to answer + + if not (bot == msg["origin"][0:len(bot)] + and nick == msg["target"][0:len(nick)] + and msg["action"] in ("PRIVMSG", "NOTICE")): + continue + + if self.debug is 1: + print "%s: %s" % (msg["origin"], msg["text"]) + + if "You already requested that pack" in msg["text"]: + retry = time.time() + 300 + + if "you must be on a known channel to request a pack" in msg["text"]: + self.fail("Wrong channel") + + m = re.match('\x01DCC SEND (.*?) (\d+) (\d+)(?: (\d+))?\x01', msg["text"]) + if m: + done = True + + # get connection data + ip = socket.inet_ntoa(struct.pack('L', socket.ntohl(int(m.group(2))))) + port = int(m.group(3)) + packname = m.group(1) + + if len(m.groups()) > 3: + self.req.filesize = int(m.group(4)) + + self.pyfile.name = packname + filename = save_join(location, packname) + self.log.info("XDCC: Downloading %s from %s:%d" % (packname, ip, port)) + + self.pyfile.setStatus("downloading") + newname = self.req.download(ip, port, filename, sock, self.pyfile.setProgress) + if newname and newname != filename: + self.log.info("%(name)s saved as %(newname)s" % {"name": self.pyfile.name, "newname": newname}) + filename = newname + + # kill IRC socket + # sock.send("QUIT :byebye\r\n") + sock.close() + + self.lastDownload = filename + return self.lastDownload + diff --git a/pyload/plugins/hoster/XvidstageCom.py b/pyload/plugins/hoster/XvidstageCom.py new file mode 100644 index 000000000..4962c05af --- /dev/null +++ b/pyload/plugins/hoster/XvidstageCom.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: 4Christopher +""" + +import re +import HTMLParser + +from module.plugins.Hoster import Hoster +from module.network.RequestFactory import getURL + + +def setup(self): + self.wantReconnect = False + self.resumeDownload = True + self.multiDL = True + + +def getInfo(urls): + result = [] + + for url in urls: + result.append(parseFileInfo(url, getInfoMode=True)) + yield result + + +def parseFileInfo(url, getInfoMode=False): + html = getURL(url) + info = {"name": url, "size": 0, "status": 3} + try: + info['name'] = re.search(r'(?:Filename|Dateiname):]*?>(.*?)<', html).group(1) + info['size'] = re.search(r'(?:Size|Größe):.*? \((\d+?) bytes\)', html).group(1) + except: ## The file is offline + info['status'] = 1 + else: + info['status'] = 2 + + if getInfoMode: + return info['name'], info['size'], info['status'], url + else: + return info['name'], info['size'], info['status'], html + + +class XvidstageCom(Hoster): + __name__ = 'XvidstageCom' + __version__ = '0.4' + __pattern__ = r'http://(?:www.)?xvidstage.com/(?P[0-9A-Za-z]+)' + __type__ = 'hoster' + __description__ = """A Plugin that allows you to download files from http://xvidstage.com""" + __author_name__ = ('4Christopher') + __author_mail__ = ('4Christopher@gmx.de') + + + def process(self, pyfile): + pyfile.name, pyfile.size, pyfile.status, self.html = parseFileInfo(pyfile.url) + self.logDebug('Name: %s' % pyfile.name) + if pyfile.status == 1: ## offline + self.offline() + self.id = re.search(self.__pattern__, pyfile.url).group('id') + + wait_sec = int(re.search(r'countdown_str">.+?>(\d+?)<', self.html).group(1)) + self.setWait(wait_sec, reconnect=False) + self.logDebug('Waiting %d seconds before submitting the captcha' % wait_sec) + self.wait() + + rand = re.search(r'', self.html).group(1) + self.logDebug('rand: %s, id: %s' % (rand, self.id)) + self.html = self.req.load(pyfile.url, + post={'op': 'download2', 'id': self.id, 'rand': rand, 'code': self.get_captcha()}) + file_url = re.search(r'(?P=url)', self.html).group('url') + try: + hours_file_available = int( + re.search(r'This direct link will be available for your IP next (?P\d+?) hours', + self.html).group('hours')) + self.logDebug( + 'You have %d hours to download this file with your current IP address.' % hours_file_available) + except: + self.logDebug('Failed') + self.logDebug('Download file: %s' % file_url) + self.download(file_url) + check = self.checkDownload({'empty': re.compile(r'^$')}) + + if check == 'empty': + self.logInfo('Downloaded File was empty') + # self.retry() + + def get_captcha(self): + ## 1 … + cap_chars = {} + for pad_left, char in re.findall(r"position:absolute;padding-left:(\d+?)px;.*?;'>(.*?)<", self.html): + cap_chars[int(pad_left)] = char + + h = HTMLParser.HTMLParser() + ## Sorting after padding-left + captcha = '' + for pad_left in sorted(cap_chars): + captcha += h.unescape(cap_chars[pad_left]) + + self.logDebug('The captcha is: %s' % captcha) + return captcha diff --git a/pyload/plugins/hoster/YibaishiwuCom.py b/pyload/plugins/hoster/YibaishiwuCom.py new file mode 100644 index 000000000..901225944 --- /dev/null +++ b/pyload/plugins/hoster/YibaishiwuCom.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: zoidberg +""" + +import re +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo +from module.common.json_layer import json_loads + +class YibaishiwuCom(SimpleHoster): + __name__ = "YibaishiwuCom" + __type__ = "hoster" + __pattern__ = r"http://(?:www\.)?(?:u\.)?115.com/file/(?P\w+)" + __version__ = "0.12" + __description__ = """115.com""" + __author_name__ = ("zoidberg") + + FILE_NAME_PATTERN = r"file_name: '(?P[^']+)'" + FILE_SIZE_PATTERN = r"file_size: '(?P[^']+)'" + FILE_OFFLINE_PATTERN = ur'

        哎呀！提取码不存在！不妨搜搜看吧！

        ' + + AJAX_URL_PATTERN = r'(/\?ct=(pickcode|download)[^"\']+)' + + def handleFree(self): + found = re.search(self.AJAX_URL_PATTERN, self.html) + if not found: self.parseError("AJAX URL") + url = found.group(1) + self.logDebug(('FREEUSER' if found.group(2) == 'download' else 'GUEST') + ' URL', url) + + response = json_loads(self.load("http://115.com" + url, decode = False)) + for mirror in (response['urls'] if 'urls' in response else response['data'] if 'data' in response else []): + try: + url = mirror['url'].replace('\\','') + self.logDebug("Trying URL: " + url) + self.download(url) + break + except: + continue + else: self.fail('No working link found') + +getInfo = create_getInfo(YibaishiwuCom) diff --git a/pyload/plugins/hoster/YoupornCom.py b/pyload/plugins/hoster/YoupornCom.py new file mode 100644 index 000000000..b17a4ef80 --- /dev/null +++ b/pyload/plugins/hoster/YoupornCom.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +from module.plugins.Hoster import Hoster + +class YoupornCom(Hoster): + __name__ = "YoupornCom" + __type__ = "hoster" + __pattern__ = r"http://(www\.)?youporn\.com/watch/.+" + __version__ = "0.2" + __description__ = """Youporn.com Video Download Hoster""" + __author_name__ = ("willnix") + __author_mail__ = ("willnix@pyload.org") + + def setup(self): + self.html = None + + def process(self, pyfile): + self.pyfile = pyfile + + if not self.file_exists(): + self.offline() + + self.pyfile.name = self.get_file_name() + self.download(self.get_file_url()) + + def download_html(self): + url = self.pyfile.url + self.html = self.load(url, post={"user_choice":"Enter"}, cookies=False) + + def get_file_url(self): + """ returns the absolute downloadable filepath + """ + if self.html is None: + self.download_html() + + file_url = re.search(r'(http://download\.youporn\.com/download/\d+\?save=1)">', self.html).group(1) + return file_url + + def get_file_name(self): + if self.html is None: + self.download_html() + + file_name_pattern = r"(.*) - Free Porn Videos - YouPorn" + return re.search(file_name_pattern, self.html).group(1).replace("&", "&").replace("/","") + '.flv' + + def file_exists(self): + """ returns True or False + """ + if self.html is None: + self.download_html() + if re.search(r"(.*invalid video_id.*)", self.html) is not None: + return False + else: + return True diff --git a/pyload/plugins/hoster/YourfilesTo.py b/pyload/plugins/hoster/YourfilesTo.py new file mode 100644 index 000000000..7a8f327ca --- /dev/null +++ b/pyload/plugins/hoster/YourfilesTo.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +import urllib +from module.plugins.Hoster import Hoster + +class YourfilesTo(Hoster): + __name__ = "YourfilesTo" + __type__ = "hoster" + __pattern__ = r"(http://)?(www\.)?yourfiles\.(to|biz)/\?d=[a-zA-Z0-9]+" + __version__ = "0.2" + __description__ = """Youfiles.to Download Hoster""" + __author_name__ = ("jeix", "skydancer") + __author_mail__ = ("jeix@hasnomail.de", "skydancer@hasnomail.de") + + def setup(self): + self.html = None + self.multiDL = True + + def process(self,pyfile): + self.pyfile = pyfile + self.prepare() + self.download(self.get_file_url()) + + def prepare(self): + if not self.file_exists(): + self.offline() + + self.pyfile.name = self.get_file_name() + + wait_time = self.get_waiting_time() + self.setWait(wait_time) + self.log.debug("%s: Waiting %d seconds." % (self.__name__,wait_time)) + self.wait() + + def get_waiting_time(self): + if self.html is None: + self.download_html() + + #var zzipitime = 15; + m = re.search(r'var zzipitime = (\d+);', self.html) + if m: + sec = int(m.group(1)) + else: + sec = 0 + + return sec + + def download_html(self): + url = self.pyfile.url + self.html = self.load(url) + + def get_file_url(self): + """ returns the absolute downloadable filepath + """ + url = re.search(r"var bla = '(.*?)';", self.html) + if url: + url = url.group(1) + url = urllib.unquote(url.replace("http://http:/http://", "http://").replace("dumdidum", "")) + return url + else: + self.fail("absolute filepath could not be found. offline? ") + + def get_file_name(self): + if self.html is None: + self.download_html() + + return re.search("(.*)", self.html).group(1) + + def file_exists(self): + """ returns True or False + """ + if self.html is None: + self.download_html() + + if re.search(r"HTTP Status 404", self.html) is not None: + return False + else: + return True + + + diff --git a/pyload/plugins/hoster/YoutubeCom.py b/pyload/plugins/hoster/YoutubeCom.py new file mode 100644 index 000000000..70db597cf --- /dev/null +++ b/pyload/plugins/hoster/YoutubeCom.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +import subprocess +import os +import os.path +from urllib import unquote + +from module.utils import html_unescape +from module.plugins.Hoster import Hoster + +def which(program): + """Works exactly like the unix command which + + Courtesy of http://stackoverflow.com/a/377028/675646""" + def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + fpath, fname = os.path.split(program) + if fpath: + if is_exe(program): + return program + else: + for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + exe_file = os.path.join(path, program) + if is_exe(exe_file): + return exe_file + + return None + +class YoutubeCom(Hoster): + __name__ = "YoutubeCom" + __type__ = "hoster" + __pattern__ = r"https?://(?:[^/]*?)youtube\.com/watch.*?[?&]v=.*" + __version__ = "0.32" + __config__ = [("quality", "sd;hd;fullhd;240p;360p;480p;720p;1080p;3072p", "Quality Setting", "hd"), + ("fmt", "int", "FMT/ITAG Number (5-102, 0 for auto)", 0), + (".mp4", "bool", "Allow .mp4", True), + (".flv", "bool", "Allow .flv", True), + (".webm", "bool", "Allow .webm", False), + (".3gp", "bool", "Allow .3gp", False), + ("3d", "bool", "Prefer 3D", False)] + __description__ = """Youtube.com Video Download Hoster""" + __author_name__ = ("spoob", "zoidberg") + __author_mail__ = ("spoob@pyload.org", "zoidberg@mujmail.cz") + + # name, width, height, quality ranking, 3D + formats = {5: (".flv", 400, 240, 1, False), + 6: (".flv", 640, 400, 4, False), + 17: (".3gp", 176, 144, 0, False), + 18: (".mp4", 480, 360, 2, False), + 22: (".mp4", 1280, 720, 8, False), + 43: (".webm", 640, 360, 3, False), + 34: (".flv", 640, 360, 4, False), + 35: (".flv", 854, 480, 6, False), + 36: (".3gp", 400, 240, 1, False), + 37: (".mp4", 1920, 1080, 9, False), + 38: (".mp4", 4096, 3072, 10, False), + 44: (".webm", 854, 480, 5, False), + 45: (".webm", 1280, 720, 7, False), + 46: (".webm", 1920, 1080, 9, False), + 82: (".mp4", 640, 360, 3, True), + 83: (".mp4", 400, 240, 1, True), + 84: (".mp4", 1280, 720, 8, True), + 85: (".mp4", 1920, 1080, 9, True), + 100: (".webm", 640, 360, 3, True), + 101: (".webm", 640, 360, 4, True), + 102: (".webm", 1280, 720, 8, True) + } + + def setup(self): + self.resumeDownload = self.multiDL = True + + def process(self, pyfile): + html = self.load(pyfile.url, decode=True) + + if '

        ' in html: + self.offline() + + if "We have been receiving a large volume of requests from your network." in html: + self.tempOffline() + + #get config + use3d = self.getConf("3d") + if use3d: + quality = {"sd":82,"hd":84,"fullhd":85,"240p":83,"360p":82,"480p":82,"720p":84,"1080p":85,"3072p":85} + else: + quality = {"sd":18,"hd":22,"fullhd":37,"240p":5,"360p":18,"480p":35,"720p":22,"1080p":37,"3072p":38} + desired_fmt = self.getConf("fmt") + if desired_fmt and desired_fmt not in self.formats: + self.logWarning("FMT %d unknown - using default." % desired_fmt) + desired_fmt = 0 + if not desired_fmt: + desired_fmt = quality.get(self.getConf("quality"), 18) + + #parse available streams + streams = re.search(r'"url_encoded_fmt_stream_map": "(.*?)",', html).group(1) + streams = [x.split('\u0026') for x in streams.split(',')] + streams = [dict((y.split('=',1)) for y in x) for x in streams] + streams = [(int(x['itag']), "%s&signature=%s" % (unquote(x['url']), x['sig'])) for x in streams] + #self.logDebug("Found links: %s" % streams) + self.logDebug("AVAILABLE STREAMS: %s" % [x[0] for x in streams]) + + #build dictionary of supported itags (3D/2D) + allowed = lambda x: self.getConfig(self.formats[x][0]) + streams = [x for x in streams if x[0] in self.formats and allowed(x[0])] + if not streams: + self.fail("No available stream meets your preferences") + fmt_dict = dict([x for x in streams if self.formats[x[0]][4] == use3d] or streams) + + self.logDebug("DESIRED STREAM: ITAG:%d (%s) %sfound, %sallowed" % + (desired_fmt, + "%s %dx%d Q:%d 3D:%s" % self.formats[desired_fmt], + "" if desired_fmt in fmt_dict else "NOT ", + "" if allowed(desired_fmt) else "NOT ") + ) + + #return fmt nearest to quality index + if desired_fmt in fmt_dict and allowed(desired_fmt): + fmt = desired_fmt + else: + sel = lambda x: self.formats[x][3] #select quality index + comp = lambda x, y: abs(sel(x) - sel(y)) + + self.logDebug("Choosing nearest fmt: %s" % [(x, allowed(x), comp(x, desired_fmt)) for x in fmt_dict.keys()]) + fmt = reduce(lambda x, y: x if comp(x, desired_fmt) <= comp(y, desired_fmt) and + sel(x) > sel(y) else y, fmt_dict.keys()) + + self.logDebug("Chosen fmt: %s" % fmt) + url = fmt_dict[fmt] + self.logDebug("URL: %s" % url) + + #set file name + file_suffix = self.formats[fmt][0] if fmt in self.formats else ".flv" + file_name_pattern = '' + name = re.search(file_name_pattern, html).group(1).replace("/", "") + pyfile.name = html_unescape(name) + + time = re.search(r"t=((\d+)m)?(\d+)s", pyfile.url) + ffmpeg = which("ffmpeg") + if ffmpeg and time: + m, s = time.groups()[1:] + if not m: + m = "0" + + pyfile.name += " (starting at %s:%s)" % (m, s) + pyfile.name += file_suffix + + filename = self.download(url) + + if ffmpeg and time: + inputfile = filename + "_" + os.rename(filename, inputfile) + + subprocess.call([ + ffmpeg, + "-ss", "00:%s:%s" % (m, s), + "-i", inputfile, + "-vcodec", "copy", + "-acodec", "copy", + filename]) + os.remove(inputfile) diff --git a/pyload/plugins/hoster/ZDF.py b/pyload/plugins/hoster/ZDF.py new file mode 100644 index 000000000..ea45f4fd8 --- /dev/null +++ b/pyload/plugins/hoster/ZDF.py @@ -0,0 +1,46 @@ + +import re +from xml.etree.ElementTree import fromstring + +from module.plugins.Hoster import Hoster + +XML_API = "http://www.zdf.de/ZDFmediathek/xmlservice/web/beitragsDetails?id=%i" + +class ZDF(Hoster): + # Based on zdfm by Roland Beermann + # http://github.com/enkore/zdfm/ + __name__ = "ZDF Mediathek" + __version__ = "0.7" + __pattern__ = r"http://www\.zdf\.de/ZDFmediathek/[^0-9]*([0-9]+)[^0-9]*" + __config__ = [] + + @staticmethod + def video_key(video): + return ( + int(video.findtext("videoBitrate", "0")), + any(f.text == "progressive" for f in video.iter("facet")), + ) + + @staticmethod + def video_valid(video): + return (video.findtext("url").startswith("http") and video.findtext("url").endswith(".mp4")) + + @staticmethod + def get_id(url): + return int(re.search(r"[^0-9]*([0-9]+)[^0-9]*", url).group(1)) + + def process(self, pyfile): + xml = fromstring(self.load(XML_API % self.get_id(pyfile.url))) + + status = xml.findtext("./status/statuscode") + if status != "ok": + self.fail("Error retrieving manifest.") + + video = xml.find("video") + title = video.findtext("information/title") + + pyfile.name = title + + target_url = sorted((v for v in video.iter("formitaet") if self.video_valid(v)), key=self.video_key)[-1].findtext("url") + + self.download(target_url) diff --git a/pyload/plugins/hoster/ZeveraCom.py b/pyload/plugins/hoster/ZeveraCom.py new file mode 100644 index 000000000..8be725d2f --- /dev/null +++ b/pyload/plugins/hoster/ZeveraCom.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from module.plugins.Hoster import Hoster +from module.utils import html_unescape +from urllib import quote, unquote +from time import sleep + +class ZeveraCom(Hoster): + __name__ = "ZeveraCom" + __version__ = "0.20" + __type__ = "hoster" + __pattern__ = r"http://zevera.com/.*" + __description__ = """zevera.com hoster plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + def setup(self): + self.resumeDownload = self.multiDL = True + self.chunkLimit = 1 + + def process(self, pyfile): + if not self.account: + self.logError(_("Please enter your zevera.com account or deactivate this plugin")) + self.fail("No zevera.com account provided") + + self.logDebug("zevera.com: Old URL: %s" % pyfile.url) + + if self.account.getAPIData(self.req, cmd = "checklink", olink = pyfile.url) != "Alive": + self.fail("Offline or not downloadable - contact Zevera support") + + header = self.account.getAPIData(self.req, just_header = True, cmd="generatedownloaddirect", olink = pyfile.url) + if not "location" in header: + self.fail("Unable to initialize download - contact Zevera support") + + self.download(header['location'], disposition = True) + + check = self.checkDownload({"error" : 'action="ErrorDownload.aspx'}) + if check == "error": + self.fail("Error response received - contact Zevera support") + + """ + # BitAPI not used - defunct, probably abandoned by Zevera + + api_url = "http://zevera.com/API.ashx" + + def process(self, pyfile): + if not self.account: + self.logError(_("Please enter your zevera.com account or deactivate this plugin")) + self.fail("No zevera.com account provided") + + self.logDebug("zevera.com: Old URL: %s" % pyfile.url) + + last_size = retries = 0 + olink = self.pyfile.url #quote(self.pyfile.url.encode('utf_8')) + + for i in range(100): + self.retData = self.account.loadAPIRequest(self.req, cmd = 'download_request', olink = olink) + self.checkAPIErrors(self.retData) + + if self.retData['FileInfo']['StatusID'] == 100: + break + elif self.retData['FileInfo']['StatusID'] == 99: + self.fail('Failed to initialize download (99)') + else: + if self.retData['FileInfo']['Progress']['BytesReceived'] <= last_size: + if retries >= 6: + self.fail('Failed to initialize download (%d)' % self.retData['FileInfo']['StatusID'] ) + retries += 1 + else: + retries = 0 + + last_size = self.retData['FileInfo']['Progress']['BytesReceived'] + + self.setWait(self.retData['Update_Wait']) + self.wait() + + pyfile.name = self.retData['FileInfo']['RealFileName'] + pyfile.size = self.retData['FileInfo']['FileSizeInBytes'] + + self.retData = self.account.loadAPIRequest(self.req, cmd = 'download_start', FileID = self.retData['FileInfo']['FileID']) + self.checkAPIErrors(self.retData) + + self.download(self.api_url, get = { + 'cmd': "open_stream", + 'login': self.account.loginname, + 'pass': self.account.password, + 'FileID': self.retData['FileInfo']['FileID'], + 'startBytes': 0 + } + ) + + def checkAPIErrors(self, retData): + if not retData: + self.fail('Unknown API response') + + if retData['ErrorCode']: + self.logError(retData['ErrorCode'], retData['ErrorMessage']) + #self.fail('ERROR: ' + retData['ErrorMessage']) + + if self.pyfile.size / 1024000 > retData['AccountInfo']['AvailableTODAYTrafficForUseInMBytes']: + self.logWarning("Not enough data left to download the file") + + def crazyDecode(self, ustring): + # accepts decoded ie. unicode string - API response is double-quoted, double-utf8-encoded + # no idea what the proper order of calling these functions would be :-/ + return html_unescape(unquote(unquote(ustring.replace('@DELIMITER@','#'))).encode('raw_unicode_escape').decode('utf-8')) + """ \ No newline at end of file diff --git a/pyload/plugins/hoster/ZippyshareCom.py b/pyload/plugins/hoster/ZippyshareCom.py new file mode 100644 index 000000000..84974e7ba --- /dev/null +++ b/pyload/plugins/hoster/ZippyshareCom.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re, subprocess, tempfile, os +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, timestamp +from module.plugins.ReCaptcha import ReCaptcha +from module.common.json_layer import json_loads + +class ZippyshareCom(SimpleHoster): + __name__ = "ZippyshareCom" + __type__ = "hoster" + __pattern__ = r"(?Phttp://www\d{0,2}\.zippyshare.com)/v(?:/|iew.jsp.*key=)(?P\d+)" + __version__ = "0.37" + __description__ = """Zippyshare.com Download Hoster""" + __author_name__ = ("spoob", "zoidberg") + __author_mail__ = ("spoob@pyload.org", "zoidberg@mujmail.cz") + __config__ = [("swfdump_path", "string", "Path to swfdump", "")] + + FILE_NAME_PATTERN = r'>Name:\s*]*>(?P[^<]+)
        ' + FILE_SIZE_PATTERN = r'>Size:\s*]*>(?P[0-9.,]+) (?P[kKMG]+)i?B
        ' + FILE_INFO_PATTERN = r'document\.getElementById\(\'dlbutton\'\)\.href = "[^;]*/(?P[^"]+)";' + FILE_OFFLINE_PATTERN = r'>File does not exist on this server

        ' + + DOWNLOAD_URL_PATTERN = r" + + + + + + +
        +
        +
        + +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        + © 2008-2013
        + The pyLoad Team
        +
        +
        +
        +

        Powered by

        +
        + Bootstrap
        +
        + +
        +

        pyLoad

        +
        + dsfdsf
        +
        + +
        +

        Community

        +
        + asd
        +
        + +
        +

        Development

        +
        + asd
        +
        +
        +
        +
        + + + diff --git a/pyload/web/app/scripts/app.js b/pyload/web/app/scripts/app.js new file mode 100644 index 000000000..427cb1bc8 --- /dev/null +++ b/pyload/web/app/scripts/app.js @@ -0,0 +1,105 @@ +/* + * Global Application Object + * Contains all necessary logic shared across views + */ +define([ + + // Libraries. + 'jquery', + 'underscore', + 'backbone', + 'utils/initHB', + 'utils/animations', + 'utils/lazyRequire', + 'utils/dialogs', + 'marionette', + 'bootstrap', + 'animate' + +], function($, _, Backbone, Handlebars) { + 'use strict'; + + Backbone.Marionette.TemplateCache.prototype.compileTemplate = function(rawTemplate) { + return Handlebars.compile(rawTemplate); + }; + + // TODO: configurable root + var App = new Backbone.Marionette.Application({ + root: '/' + }); + + App.addRegions({ + header: '#header', + notification: '#notification-area', + selection: '#selection-area', + content: '#content', + actionbar: '#actionbar' + }); + + App.navigate = function(url) { + return Backbone.history.navigate(url, true); + }; + + App.apiUrl = function(path) { + var url = window.hostProtocol + window.hostAddress + ':' + window.hostPort + window.pathPrefix + path; + return url; + }; + + // Add Global Helper functions + // Generates options dict that can be used for xhr requests + App.apiRequest = function(method, data, options) { + options || (options = {}); + options.url = App.apiUrl('api/' + method); + options.dataType = 'json'; + + if (data) { + options.type = 'POST'; + options.data = {}; + // Convert arguments to json + _.keys(data).map(function(key) { + options.data[key] = JSON.stringify(data[key]); + }); + } + + return options; + }; + + App.setTitle = function(name) { + var title = window.document.title; + var newTitle; + // page name separator + var index = title.indexOf('-'); + if (index >= 0) + newTitle = name + ' - ' + title.substr(index + 2, title.length); + else + newTitle = name + ' - ' + title; + + window.document.title = newTitle; + }; + + App.openWebSocket = function(path) { + // TODO + return new WebSocket(window.wsAddress.replace('%s', window.hostAddress) + path); + }; + + App.on('initialize:after', function() { +// TODO pushState variable + Backbone.history.start({ + pushState: false, + root: App.root + }); + + // All links should be handled by backbone + $(document).on('click', 'a[data-nav]', function(evt) { + var href = { prop: $(this).prop('href'), attr: $(this).attr('href') }; + var root = location.protocol + '//' + location.host + App.root; + if (href.prop.slice(0, root.length) === root) { + evt.preventDefault(); + Backbone.history.navigate(href.attr, true); + } + }); + }); + + // Returns the app object to be available to other modules through require.js. + return App; +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/collections/AccountList.js b/pyload/web/app/scripts/collections/AccountList.js new file mode 100644 index 000000000..bfc2af5a3 --- /dev/null +++ b/pyload/web/app/scripts/collections/AccountList.js @@ -0,0 +1,24 @@ +define(['jquery', 'backbone', 'underscore', 'app', 'models/Account'], function($, Backbone, _, App, Account) { + 'use strict'; + + return Backbone.Collection.extend({ + + model: Account, + + comparator: function(account) { + return account.get('plugin'); + }, + + initialize: function() { + + }, + + fetch: function(options) { + // TODO: refresh options? + options = App.apiRequest('getAccounts/false', null, options); + return Backbone.Collection.prototype.fetch.call(this, options); + } + + }); + +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/collections/FileList.js b/pyload/web/app/scripts/collections/FileList.js new file mode 100644 index 000000000..873f4c0e3 --- /dev/null +++ b/pyload/web/app/scripts/collections/FileList.js @@ -0,0 +1,18 @@ +define(['jquery', 'backbone', 'underscore', 'models/File'], function($, Backbone, _, File) { + 'use strict'; + + return Backbone.Collection.extend({ + + model: File, + + comparator: function(file) { + return file.get('fileorder'); + }, + + initialize: function() { + + } + + }); + +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/collections/InteractionList.js b/pyload/web/app/scripts/collections/InteractionList.js new file mode 100644 index 000000000..24f8b9248 --- /dev/null +++ b/pyload/web/app/scripts/collections/InteractionList.js @@ -0,0 +1,49 @@ +define(['jquery', 'backbone', 'underscore', 'app', 'models/InteractionTask'], + function($, Backbone, _, App, InteractionTask) { + 'use strict'; + + return Backbone.Collection.extend({ + + model: InteractionTask, + + comparator: function(task) { + return task.get('iid'); + }, + + fetch: function(options) { + options = App.apiRequest('getInteractionTasks/0', null, options); + var self = this; + options.success = function(data) { + self.set(data); + }; + + return $.ajax(options); + }, + + toJSON: function() { + var data = {queries: 0, notifications: 0}; + + this.map(function(task) { + if (task.isNotification()) + data.notifications++; + else + data.queries++; + }); + + return data; + }, + + // a task is waiting for attention (no notification) + hasTaskWaiting: function() { + var tasks = 0; + this.map(function(task) { + if (!task.isNotification()) + tasks++; + }); + + return tasks > 0; + } + + }); + + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/collections/PackageList.js b/pyload/web/app/scripts/collections/PackageList.js new file mode 100644 index 000000000..7bee861a4 --- /dev/null +++ b/pyload/web/app/scripts/collections/PackageList.js @@ -0,0 +1,16 @@ +define(['jquery', 'backbone', 'underscore', 'models/Package'], function($, Backbone, _, Package) { + 'use strict'; + + return Backbone.Collection.extend({ + + model: Package, + + comparator: function(pack) { + return pack.get('packageorder'); + }, + + initialize: function() { + } + + }); +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/collections/ProgressList.js b/pyload/web/app/scripts/collections/ProgressList.js new file mode 100644 index 000000000..51849d8de --- /dev/null +++ b/pyload/web/app/scripts/collections/ProgressList.js @@ -0,0 +1,18 @@ +define(['jquery', 'backbone', 'underscore', 'models/Progress'], function($, Backbone, _, Progress) { + 'use strict'; + + return Backbone.Collection.extend({ + + model: Progress, + + comparator: function(progress) { + return progress.get('eta'); + }, + + initialize: function() { + + } + + }); + +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/config.js b/pyload/web/app/scripts/config.js new file mode 100644 index 000000000..398d97e11 --- /dev/null +++ b/pyload/web/app/scripts/config.js @@ -0,0 +1,73 @@ +// Sets the require.js configuration for your application. +'use strict'; +require.config({ + + deps: ['default'], + + paths: { + + jquery: '../components/jquery/jquery', + flot: '../components/flot/jquery.flot', + transit: '../components/jquery.transit/jquery.transit', + animate: '../components/jquery.animate-enhanced/scripts/src/jquery.animate-enhanced', + cookie: '../components/jquery.cookie/jquery.cookie', + omniwindow: 'vendor/jquery.omniwindow', + select2: '../components/select2/select2', + bootstrap: 'vendor/bootstrap-2.3.2', + underscore: '../components/underscore/underscore', + backbone: '../components/backbone/backbone', + marionette: '../components/backbone.marionette/lib/backbone.marionette', +// handlebars: '../components/handlebars.js/dist/handlebars', + handlebars: 'vendor/Handlebars-1.0rc1', + jed: '../components/jed/jed', + + // TODO: Two hbs dependencies could be replaced + i18nprecompile: '../components/require-handlebars-plugin/hbs/i18nprecompile', + json2: '../components/require-handlebars-plugin/hbs/json2', + + // Plugins + text: '../components/requirejs-text/text', + hbs: '../components/require-handlebars-plugin/hbs', + + // Shortcut + tpl: '../templates/default' + }, + + hbs: { + disableI18n: true, + helperPathCallback: // Callback to determine the path to look for helpers + function(name) { + // Some helpers are accumulated into one file + if (name.indexOf('file') === 0) + name = 'fileHelper'; + + return 'helpers/' + name; + }, + templateExtension: 'html' + }, + + // Sets the configuration for your third party scripts that are not AMD compatible + shim: { + underscore: { + exports: '_' + }, + + backbone: { + deps: ['underscore', 'jquery'], + exports: 'Backbone' + }, + + marionette: ['backbone'], +// handlebars: { +// exports: 'Handlebars' +// }, + + flot: ['jquery'], + transit: ['jquery'], + cookie: ['jquery'], + omniwindow: ['jquery'], + select2: ['jquery'], + bootstrap: ['jquery'], + animate: ['jquery'] + } +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/controller.js b/pyload/web/app/scripts/controller.js new file mode 100644 index 000000000..05237914d --- /dev/null +++ b/pyload/web/app/scripts/controller.js @@ -0,0 +1,67 @@ +define([ + 'app', + 'backbone', + + // Views + 'views/headerView', + 'views/notificationView', + 'views/dashboard/dashboardView', + 'views/dashboard/selectionView', + 'views/dashboard/filterView', + 'views/loginView', + 'views/settings/settingsView', + 'views/accounts/accountListView' +], function( + App, Backbone, HeaderView, NotificationView, DashboardView, SelectionView, FilterView, LoginView, SettingsView, AccountListView) { + 'use strict'; + // TODO some views does not need to be loaded instantly + + return { + + header: function() { + if (!App.header.currentView) { + App.header.show(new HeaderView()); + App.header.currentView.init(); + App.notification.attachView(new NotificationView()); + } + }, + + dashboard: function() { + this.header(); + + App.actionbar.show(new FilterView()); + // TODO: not completly visible after reattaching + App.selection.attachView(new SelectionView()); + App.content.show(new DashboardView()); + }, + + login: function() { + App.content.show(new LoginView()); + }, + + logout: function() { + alert('Not implemented'); + }, + + settings: function() { + this.header(); + + var view = new SettingsView(); + App.actionbar.show(new view.actionbar()); + App.content.show(view); + }, + + accounts: function() { + this.header(); + + var view = new AccountListView(); + App.actionbar.show(new view.actionbar()); + App.content.show(view); + }, + + admin: function() { + alert('Not implemented'); + } + }; + +}); diff --git a/pyload/web/app/scripts/default.js b/pyload/web/app/scripts/default.js new file mode 100644 index 000000000..a337cee21 --- /dev/null +++ b/pyload/web/app/scripts/default.js @@ -0,0 +1,30 @@ +define('default', ['backbone', 'jquery', 'app', 'router', 'models/userSession'], + function(Backbone, $, App, Router, UserSession) { + 'use strict'; + + // Global ajax options + var options = { + statusCode: { + 401: function() { + console.log('Not logged in.'); + App.navigate('login'); + } + }, + xhrFields: {withCredentials: true} + }; + + $.ajaxSetup(options); + + Backbone.ajax = function() { + Backbone.$.ajaxSetup.call(Backbone.$, options); + return Backbone.$.ajax.apply(Backbone.$, arguments); + }; + + $(function() { + App.session = new UserSession(); + App.router = new Router(); + App.start(); + }); + + return App; + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/helpers/fileHelper.js b/pyload/web/app/scripts/helpers/fileHelper.js new file mode 100644 index 000000000..156be58f0 --- /dev/null +++ b/pyload/web/app/scripts/helpers/fileHelper.js @@ -0,0 +1,55 @@ +// Helpers to render the file view +define('helpers/fileHelper', ['handlebars', 'utils/apitypes', 'helpers/formatTime'], + function(Handlebars, Api, formatTime) { + 'use strict'; + + function fileClass(file, options) { + if (file.finished) + return 'finished'; + else if (file.failed) + return 'failed'; + else if (file.offline) + return 'offline'; + else if (file.online) + return 'online'; + else if (file.waiting) + return 'waiting'; + else if (file.downloading) + return 'downloading'; + + return ''; + } + + // TODO + function fileIcon(media, options) { + return 'icon-music'; + } + + // TODO rest of the states + function fileStatus(file, options) { + var s; + var msg = file.download.statusmsg; + + if (file.failed) { + s = ' '; + if (file.download.error) + s += file.download.error; + else s += msg; + } else if (file.finished) + s = ' ' + msg; + else if (file.downloading) + s = '
          ' + + formatTime(file.eta) + '
        '; + else if (file.waiting) + s = ' ' + formatTime(file.eta); + else + s = msg; + + return new Handlebars.SafeString(s); + } + + Handlebars.registerHelper('fileClass', fileClass); + Handlebars.registerHelper('fileIcon', fileIcon); + Handlebars.registerHelper('fileStatus', fileStatus); + return fileClass; + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/helpers/formatSize.js b/pyload/web/app/scripts/helpers/formatSize.js new file mode 100644 index 000000000..3b62e74c7 --- /dev/null +++ b/pyload/web/app/scripts/helpers/formatSize.js @@ -0,0 +1,15 @@ +// Format bytes in human readable format +define('helpers/formatSize', ['handlebars'], function(Handlebars) { + 'use strict'; + + var sizes = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB']; + function formatSize(bytes, options) { + if (!bytes || bytes === 0) return '0 B'; + var i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)), 10); + // round to two digits + return (bytes / Math.pow(1024, i)).toFixed(2) + ' ' + sizes[i]; + } + + Handlebars.registerHelper('formatSize', formatSize); + return formatSize; +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/helpers/formatTime.js b/pyload/web/app/scripts/helpers/formatTime.js new file mode 100644 index 000000000..757ff73ad --- /dev/null +++ b/pyload/web/app/scripts/helpers/formatTime.js @@ -0,0 +1,17 @@ +// Format bytes in human readable format +define('helpers/formatTime', ['handlebars', 'vendor/remaining'], function(Handlebars, Remaining) { + 'use strict'; + + function formatTime(seconds, options) { + if (seconds === Infinity) + return '∞'; + else if (!seconds || seconds <= 0) + return '-'; + + // TODO: digital or written string + return Remaining.getStringDigital(seconds, window.dates); + } + + Handlebars.registerHelper('formatTime', formatTime); + return formatTime; +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/helpers/pluginIcon.js b/pyload/web/app/scripts/helpers/pluginIcon.js new file mode 100644 index 000000000..6b2fdc67f --- /dev/null +++ b/pyload/web/app/scripts/helpers/pluginIcon.js @@ -0,0 +1,14 @@ +// Resolves name of plugin to icon path +define('helpers/pluginIcon', ['handlebars', 'app'], function(Handlebars, App) { + 'use strict'; + + function pluginIcon(name) { + if (typeof name === 'object' && typeof name.get === 'function') + name = name.get('plugin'); + + return App.apiUrl('icons/' + name); + } + + Handlebars.registerHelper('pluginIcon', pluginIcon); + return pluginIcon; +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/models/Account.js b/pyload/web/app/scripts/models/Account.js new file mode 100644 index 000000000..a2e24b056 --- /dev/null +++ b/pyload/web/app/scripts/models/Account.js @@ -0,0 +1,51 @@ +define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes'], function($, Backbone, _, App, Api) { + 'use strict'; + + return Backbone.Model.extend({ + + // TODO + // generated, not submitted + idAttribute: 'user', + + defaults: { + plugin: null, + loginname: null, + owner: -1, + valid: false, + validuntil: -1, + trafficleft: -1, + maxtraffic: -1, + premium: false, + activated: false, + shared: false, + options: null + }, + + // Model Constructor + initialize: function() { + }, + + // Any time a model attribute is set, this method is called + validate: function(attrs) { + + }, + + save: function(options) { + options = App.apiRequest('updateAccountInfo', {account: this.toJSON()}, options); + return $.ajax(options); + }, + + destroy: function(options) { + options = App.apiRequest('removeAccount', {account: this.toJSON()}, options); + var self = this; + options.success = function() { + self.trigger('destroy', self, self.collection, options); + }; + + // TODO request is not dispatched +// return Backbone.Model.prototype.destroy.call(this, options); + return $.ajax(options); + } + }); + +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/models/ConfigHolder.js b/pyload/web/app/scripts/models/ConfigHolder.js new file mode 100644 index 000000000..40efbc7c0 --- /dev/null +++ b/pyload/web/app/scripts/models/ConfigHolder.js @@ -0,0 +1,68 @@ +define(['jquery', 'backbone', 'underscore', 'app', './ConfigItem'], + function($, Backbone, _, App, ConfigItem) { + 'use strict'; + + return Backbone.Model.extend({ + + defaults: { + name: '', + label: '', + description: '', + long_description: null, + // simple list but no collection + items: null, + info: null + }, + + // Model Constructor + initialize: function() { + + }, + + // Loads it from server by name + fetch: function(options) { + options = App.apiRequest('loadConfig/"' + this.get('name') + '"', null, options); + return Backbone.Model.prototype.fetch.call(this, options); + }, + + save: function(options) { + var config = this.toJSON(); + var items = []; + // Convert changed items to json + _.each(config.items, function(item) { + if (item.isChanged()) { + items.push(item.prepareSave()); + } + }); + config.items = items; + // TODO: only set new values on success + + options = App.apiRequest('saveConfig', {config: config}, options); + + return $.ajax(options); + }, + + parse: function(resp) { + // Create item models + resp.items = _.map(resp.items, function(item) { + return new ConfigItem(item); + }); + + return Backbone.Model.prototype.parse.call(this, resp); + }, + + isLoaded: function() { + return this.has('items') || this.has('long_description'); + }, + + // check if any of the items has changes + hasChanges: function() { + var items = this.get('items'); + if (!items) return false; + return _.reduce(items, function(a, b) { + return a || b.isChanged(); + }, false); + } + + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/models/ConfigItem.js b/pyload/web/app/scripts/models/ConfigItem.js new file mode 100644 index 000000000..2d325c2a2 --- /dev/null +++ b/pyload/web/app/scripts/models/ConfigItem.js @@ -0,0 +1,40 @@ +define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes'], + function($, Backbone, _, App, Api) { + 'use strict'; + + return Backbone.Model.extend({ + + defaults: { + name: '', + label: '', + description: '', + input: null, + default_value: null, + value: null, + // additional attributes + inputView: null + }, + + // Model Constructor + initialize: function() { + + }, + + isChanged: function() { + return this.get('inputView') && this.get('inputView').getVal() !== this.get('value'); + }, + + // set new value and return json + prepareSave: function() { + // set the new value + if (this.get('inputView')) + this.set('value', this.get('inputView').getVal()); + + var data = this.toJSON(); + delete data.inputView; + delete data.description; + + return data; + } + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/models/File.js b/pyload/web/app/scripts/models/File.js new file mode 100644 index 000000000..3beb7f270 --- /dev/null +++ b/pyload/web/app/scripts/models/File.js @@ -0,0 +1,92 @@ +define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes'], function($, Backbone, _, App, Api) { + 'use strict'; + + var Finished = [Api.DownloadStatus.Finished, Api.DownloadStatus.Skipped]; + var Failed = [Api.DownloadStatus.Failed, Api.DownloadStatus.Aborted, Api.DownloadStatus.TempOffline, Api.DownloadStatus.Offline]; + // Unfinished - Other + + return Backbone.Model.extend({ + + idAttribute: 'fid', + + defaults: { + fid: -1, + name: null, + package: -1, + owner: -1, + size: -1, + status: -1, + media: -1, + added: -1, + fileorder: -1, + download: null, + + // UI attributes + selected: false, + visible: true, + progress: 0, + eta: 0 + }, + + // Model Constructor + initialize: function() { + + }, + + fetch: function(options) { + options = App.apiRequest( + 'getFileInfo', + {fid: this.get('fid')}, + options); + + return Backbone.Model.prototype.fetch.call(this, options); + }, + + destroy: function(options) { + // also not working when using data + options = App.apiRequest( + 'deleteFiles/[' + this.get('fid') + ']', + null, options); + options.method = 'post'; + + return Backbone.Model.prototype.destroy.call(this, options); + }, + + // Does not send a request to the server + destroyLocal: function(options) { + this.trigger('destroy', this, this.collection, options); + }, + + restart: function(options) { + options = App.apiRequest( + 'restartFile', + {fid: this.get('fid')}, + options); + + return $.ajax(options); + }, + + // Any time a model attribute is set, this method is called + validate: function(attrs) { + + }, + + isDownload: function() { + return this.has('download'); + }, + + isFinished: function() { + return _.indexOf(Finished, this.get('download').status) > -1; + }, + + isUnfinished: function() { + return _.indexOf(Finished, this.get('download').status) === -1 && _.indexOf(Failed, this.get('download').status) === -1; + }, + + isFailed: function() { + return _.indexOf(Failed, this.get('download').status) > -1; + } + + }); + +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/models/InteractionTask.js b/pyload/web/app/scripts/models/InteractionTask.js new file mode 100644 index 000000000..54c739d4b --- /dev/null +++ b/pyload/web/app/scripts/models/InteractionTask.js @@ -0,0 +1,41 @@ +define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes'], + function($, Backbone, _, App, Api) { + 'use strict'; + + return Backbone.Model.extend({ + + idAttribute: 'iid', + + defaults: { + iid: -1, + type: null, + input: null, + default_value: null, + title: '', + description: '', + plugin: '', + // additional attributes + result: '' + }, + + // Model Constructor + initialize: function() { + + }, + + save: function(options) { + options = App.apiRequest('setInteractionResult/' + this.get('iid'), + {result: this.get('result')}, options); + + return $.ajax(options); + }, + + isNotification: function() { + return this.get('type') === Api.Interaction.Notification; + }, + + isCaptcha: function() { + return this.get('type') === Api.Interaction.Captcha; + } + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/models/Package.js b/pyload/web/app/scripts/models/Package.js new file mode 100644 index 000000000..a34ec1c69 --- /dev/null +++ b/pyload/web/app/scripts/models/Package.js @@ -0,0 +1,119 @@ +define(['jquery', 'backbone', 'underscore', 'app', 'collections/FileList', 'require'], + function($, Backbone, _, App, FileList, require) { + 'use strict'; + + return Backbone.Model.extend({ + + idAttribute: 'pid', + + defaults: { + pid: -1, + name: null, + folder: '', + root: -1, + owner: -1, + site: '', + comment: '', + password: '', + added: -1, + tags: null, + status: -1, + shared: false, + packageorder: -1, + stats: null, + fids: null, + pids: null, + files: null, // Collection + packs: null, // Collection + + selected: false // For Checkbox + }, + + // Model Constructor + initialize: function() { + }, + + toJSON: function(options) { + var obj = Backbone.Model.prototype.toJSON.call(this, options); + obj.percent = Math.round(obj.stats.linksdone * 100 / obj.stats.linkstotal); + + return obj; + }, + + // Changes url + method and delegates call to super class + fetch: function(options) { + options = App.apiRequest( + 'getFileTree/' + this.get('pid'), + {full: false}, + options); + + return Backbone.Model.prototype.fetch.call(this, options); + }, + + // Create a pseudo package und use search to populate data + search: function(qry, options) { + options = App.apiRequest( + 'findFiles', + {pattern: qry}, + options); + + return Backbone.Model.prototype.fetch.call(this, options); + }, + + save: function(options) { + // TODO + }, + + destroy: function(options) { + // TODO: Not working when using data?, array seems to break it + options = App.apiRequest( + 'deletePackages/[' + this.get('pid') + ']', + null, options); + options.method = 'post'; + + console.log(options); + + return Backbone.Model.prototype.destroy.call(this, options); + }, + + restart: function(options) { + options = App.apiRequest( + 'restartPackage', + {pid: this.get('pid')}, + options); + + var self = this; + options.success = function() { + self.fetch(); + }; + return $.ajax(options); + }, + + parse: function(resp) { + // Package is loaded from tree collection + if (_.has(resp, 'root')) { + if (!this.has('files')) + resp.root.files = new FileList(_.values(resp.files)); + else + this.get('files').set(_.values(resp.files)); + + // circular dependencies needs to be avoided + var PackageList = require('collections/PackageList'); + + if (!this.has('packs')) + resp.root.packs = new PackageList(_.values(resp.packages)); + else + this.get('packs').set(_.values(resp.packages)); + + return resp.root; + } + return Backbone.model.prototype.parse.call(this, resp); + }, + + // Any time a model attribute is set, this method is called + validate: function(attrs) { + + } + + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/models/Progress.js b/pyload/web/app/scripts/models/Progress.js new file mode 100644 index 000000000..b0bbb684d --- /dev/null +++ b/pyload/web/app/scripts/models/Progress.js @@ -0,0 +1,50 @@ +define(['jquery', 'backbone', 'underscore', 'utils/apitypes'], function($, Backbone, _, Api) { + 'use strict'; + + return Backbone.Model.extend({ + + // generated, not submitted + idAttribute: 'pid', + + defaults: { + pid: -1, + plugin: null, + name: null, + statusmsg: -1, + eta: -1, + done: -1, + total: -1, + download: null + }, + + getPercent: function() { + if (this.get('total') > 0) + return Math.round(this.get('done') * 100 / this.get('total')); + return 0; + }, + + // Model Constructor + initialize: function() { + + }, + + // Any time a model attribute is set, this method is called + validate: function(attrs) { + + }, + + toJSON: function(options) { + var obj = Backbone.Model.prototype.toJSON.call(this, options); + obj.percent = this.getPercent(); + obj.downloading = this.isDownload() && this.get('download').status === Api.DownloadStatus.Downloading; + + return obj; + }, + + isDownload : function() { + return this.has('download'); + } + + }); + +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/models/ServerStatus.js b/pyload/web/app/scripts/models/ServerStatus.js new file mode 100644 index 000000000..59739b41e --- /dev/null +++ b/pyload/web/app/scripts/models/ServerStatus.js @@ -0,0 +1,47 @@ +define(['jquery', 'backbone', 'underscore'], + function($, Backbone, _) { + 'use strict'; + + return Backbone.Model.extend({ + + defaults: { + speed: 0, + linkstotal: 0, + linksqueue: 0, + sizetotal: 0, + sizequeue: 0, + notifications: -1, + paused: false, + download: false, + reconnect: false + }, + + // Model Constructor + initialize: function() { + + }, + + fetch: function(options) { + options || (options = {}); + options.url = 'api/getServerStatus'; + + return Backbone.Model.prototype.fetch.call(this, options); + }, + + toJSON: function(options) { + var obj = Backbone.Model.prototype.toJSON.call(this, options); + + obj.linksdone = obj.linkstotal - obj.linksqueue; + obj.sizedone = obj.sizetotal - obj.sizequeue; + if (obj.speed && obj.speed > 0) + obj.eta = Math.round(obj.sizequeue / obj.speed); + else if (obj.sizequeue > 0) + obj.eta = Infinity; + else + obj.eta = 0; + + return obj; + } + + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/models/TreeCollection.js b/pyload/web/app/scripts/models/TreeCollection.js new file mode 100644 index 000000000..2f761e6cc --- /dev/null +++ b/pyload/web/app/scripts/models/TreeCollection.js @@ -0,0 +1,50 @@ +define(['jquery', 'backbone', 'underscore', 'app', 'models/Package', 'collections/FileList', 'collections/PackageList'], + function($, Backbone, _, App, Package, FileList, PackageList) { + 'use strict'; + + // TreeCollection + // A Model and not a collection, aggregates other collections + return Backbone.Model.extend({ + + defaults: { + root: null, + packages: null, + files: null + }, + + initialize: function() { + + }, + + fetch: function(options) { + options || (options = {}); + var pid = options.pid || -1; + + options = App.apiRequest( + 'getFileTree/' + pid, + {full: false}, + options); + + console.log('Fetching package tree ' + pid); + return Backbone.Model.prototype.fetch.call(this, options); + }, + + // Parse the response and updates the collections + parse: function(resp) { + var ret = {}; + if (!this.has('packages')) + ret.packages = new PackageList(_.values(resp.packages)); + else + this.get('packages').set(_.values(resp.packages)); + + if (!this.has('files')) + ret.files = new FileList(_.values(resp.files)); + else + this.get('files').set(_.values(resp.files)); + + ret.root = new Package(resp.root); + return ret; + } + + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/models/UserSession.js b/pyload/web/app/scripts/models/UserSession.js new file mode 100644 index 000000000..a7e9aa848 --- /dev/null +++ b/pyload/web/app/scripts/models/UserSession.js @@ -0,0 +1,20 @@ +define(['jquery', 'backbone', 'underscore', 'utils/apitypes', 'cookie'], + function($, Backbone, _, Api) { + 'use strict'; + + return Backbone.Model.extend({ + + idAttribute: 'username', + + defaults: { + username: null, + permissions: null, + session: null + }, + + // Model Constructor + initialize: function() { + this.set('session', $.cookie('beaker.session.id')); + } + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/router.js b/pyload/web/app/scripts/router.js new file mode 100644 index 000000000..68ea5575d --- /dev/null +++ b/pyload/web/app/scripts/router.js @@ -0,0 +1,29 @@ +/** + * Router defines routes that are handled by registered controller + */ +define([ + // Libraries + 'backbone', + 'marionette', + + // Modules + 'controller' +], + function(Backbone, Marionette, Controller) { + 'use strict'; + + return Backbone.Marionette.AppRouter.extend({ + + appRoutes: { + '': 'dashboard', + 'login': 'login', + 'logout': 'logout', + 'settings': 'settings', + 'accounts': 'accounts', + 'admin': 'admin' + }, + + // Our controller to handle the routes + controller: Controller + }); + }); diff --git a/pyload/web/app/scripts/routers/defaultRouter.js b/pyload/web/app/scripts/routers/defaultRouter.js new file mode 100644 index 000000000..4b00d160c --- /dev/null +++ b/pyload/web/app/scripts/routers/defaultRouter.js @@ -0,0 +1,30 @@ +define(['jquery', 'backbone', 'views/headerView'], function($, Backbone, HeaderView) { + 'use strict'; + + var Router = Backbone.Router.extend({ + + initialize: function() { + Backbone.history.start(); + }, + + // All of your Backbone Routes (add more) + routes: { + + // When there is no hash bang on the url, the home method is called + '': 'home' + + }, + + 'home': function() { + // Instantiating mainView and anotherView instances + var headerView = new HeaderView(); + + // Renders the mainView template + headerView.render(); + + } + }); + + // Returns the Router class + return Router; +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/routers/mobileRouter.js b/pyload/web/app/scripts/routers/mobileRouter.js new file mode 100644 index 000000000..e24cb7a34 --- /dev/null +++ b/pyload/web/app/scripts/routers/mobileRouter.js @@ -0,0 +1,56 @@ +define(['jquery', 'backbone', 'underscore'], function($, Backbone, _) { + 'use strict'; + + return Backbone.Router.extend({ + + initialize: function() { + _.bindAll(this, 'changePage'); + + this.$el = $('#content'); + + // Tells Backbone to start watching for hashchange events + Backbone.history.start(); + + }, + + // All of your Backbone Routes (add more) + routes: { + + // When there is no hash bang on the url, the home method is called + '': 'home' + + }, + + 'home': function() { + + var self = this; + + $('#p1').fastClick(function() { + self.changePage($('

        Page 1


        some content
        sdfdsf
        sdffg

        oiuzz

        ')); + }); + + $('#p2').bind('click', function() { + self.changePage($('

        Page 2


        some content
        sdfdsf

        sdfsdf

        sdffg
        ')); + }); + + }, + + changePage: function(content) { + + var oldpage = this.$el.find('.page'); + content.css({x: '100%'}); + this.$el.append(content); + content.transition({x: 0}, function() { + window.setTimeout(function() { + oldpage.remove(); + }, 400); + }); + +// $("#viewport").transition({x: "100%"}, function(){ +// $("#viewport").html(content); +// $("#viewport").transition({x: 0}); +// }); + } + + }); +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/utils/animations.js b/pyload/web/app/scripts/utils/animations.js new file mode 100644 index 000000000..7f89afef1 --- /dev/null +++ b/pyload/web/app/scripts/utils/animations.js @@ -0,0 +1,129 @@ +define(['jquery', 'underscore', 'transit'], function(jQuery, _) { + 'use strict'; + + // Adds an element and computes its height, which is saved as data attribute + // Important function to have slide animations + jQuery.fn.appendWithHeight = function(element, hide) { + var o = jQuery(this[0]); + element = jQuery(element); + + // TODO: additionally it could be placed out of viewport first + // The real height can only be retrieved when element is on DOM and display:true + element.css('visibility', 'hidden'); + o.append(element); + + var height = element.height(); + + // Hide the element + if (hide === true) { + element.hide(); + element.height(0); + } + + element.css('visibility', ''); + element.data('height', height); + + return this; + }; + + // Shortcut to have a animation when element is added + jQuery.fn.appendWithAnimation = function(element, animation) { + var o = jQuery(this[0]); + element = jQuery(element); + + if (animation === true) + element.hide(); + + o.append(element); + + if (animation === true) + element.fadeIn(); + +// element.calculateHeight(); + + return this; + }; + + // calculate the height and write it to data, should be used on invisible elements + jQuery.fn.calculateHeight = function(setHeight) { + var o = jQuery(this[0]); + var height = o.height(); + if (!height) { + var display = o.css('display'); + o.css('visibility', 'hidden'); + o.show(); + height = o.height(); + + o.css('display', display); + o.css('visibility', ''); + } + + if (setHeight) + o.css('height', height); + + o.data('height', height); + return this; + }; + + // TODO: carry arguments, optional height argument + + // reset arguments, sets overflow hidden + jQuery.fn.slideOut = function(reset) { + var o = jQuery(this[0]); + o.animate({height: o.data('height'), opacity: 'show'}, function() { + // reset css attributes; + if (reset) { + this.css('overflow', ''); + this.css('height', ''); + } + }); + return this; + }; + + jQuery.fn.slideIn = function(reset) { + var o = jQuery(this[0]); + if (reset) { + o.css('overflow', 'hidden'); + } + o.animate({height: 0, opacity: 'hide'}); + return this; + }; + + jQuery.fn.initTooltips = function(placement) { + placement || (placement = 'top'); + + var o = jQuery(this[0]); + o.find('[data-toggle="tooltip"]').tooltip( + { + delay: {show: 800, hide: 100}, + placement: placement + }); + + return this; + }; + + jQuery.fn._transit = jQuery.fn.transit; + + // Overriding transit plugin to support hide and show + jQuery.fn.transit = jQuery.fn.transition = function(props, duration, easing, callback) { + var self = this; + var cb = callback; + var newprops = _.extend({}, props); + + if (newprops && (newprops.opacity === 'hide')) { + newprops.opacity = 0; + + callback = function() { + self.css({display: 'none'}); + if (typeof cb === 'function') { + cb.apply(self); + } + }; + } else if (newprops && (newprops.opacity === 'show')) { + newprops.opacity = 1; + this.css({display: 'block'}); + } + + return this._transit(newprops, duration, easing, callback); + }; +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/utils/apitypes.js b/pyload/web/app/scripts/utils/apitypes.js new file mode 100644 index 000000000..cbbc9064f --- /dev/null +++ b/pyload/web/app/scripts/utils/apitypes.js @@ -0,0 +1,16 @@ +// Autogenerated, do not edit! +/*jslint -W070: false*/ +define([], function() { + 'use strict'; + return { + DownloadState: {'Failed': 3, 'All': 0, 'Unmanaged': 4, 'Finished': 1, 'Unfinished': 2}, + DownloadStatus: {'Downloading': 10, 'NA': 0, 'Processing': 14, 'Waiting': 9, 'Decrypting': 13, 'Paused': 4, 'Failed': 7, 'Finished': 5, 'Skipped': 6, 'Unknown': 16, 'Aborted': 12, 'Online': 2, 'TempOffline': 11, 'Offline': 1, 'Custom': 15, 'Starting': 8, 'Queued': 3}, + FileStatus: {'Remote': 2, 'Ok': 0, 'Missing': 1}, + InputType: {'Multiple': 10, 'Int': 2, 'NA': 0, 'List': 11, 'Bool': 7, 'File': 3, 'Text': 1, 'Table': 12, 'Folder': 4, 'Password': 6, 'Click': 8, 'Select': 9, 'Textbox': 5}, + Interaction: {'Captcha': 2, 'All': 0, 'Query': 4, 'Notification': 1}, + MediaType: {'All': 0, 'Audio': 2, 'Image': 4, 'Other': 1, 'Video': 8, 'Document': 16, 'Archive': 32}, + PackageStatus: {'Paused': 1, 'Remote': 3, 'Folder': 2, 'Ok': 0}, + Permission: {'All': 0, 'Interaction': 32, 'Modify': 4, 'Add': 1, 'Accounts': 16, 'Plugins': 64, 'Download': 8, 'Delete': 2}, + Role: {'Admin': 0, 'User': 1}, + }; +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/utils/dialogs.js b/pyload/web/app/scripts/utils/dialogs.js new file mode 100644 index 000000000..4933b7ed2 --- /dev/null +++ b/pyload/web/app/scripts/utils/dialogs.js @@ -0,0 +1,16 @@ +// Loads all helper and set own handlebars rules +define(['jquery', 'underscore', 'views/abstract/modalView'], function($, _, Modal) { + 'use strict'; + + // Shows the confirm dialog for given context + // on success executes func with context + _.confirm = function(template, func, context) { + template = 'text!tpl/' + template; + _.requireOnce([template], function(html) { + var template = _.compile(html); + var dialog = new Modal(template, _.bind(func, context)); + dialog.show(); + }); + + }; +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/utils/initHB.js b/pyload/web/app/scripts/utils/initHB.js new file mode 100644 index 000000000..d7f582521 --- /dev/null +++ b/pyload/web/app/scripts/utils/initHB.js @@ -0,0 +1,11 @@ +// Loads all helper and set own handlebars rules +define(['underscore', 'handlebars', + 'helpers/formatSize', 'helpers/fileHelper', 'helpers/formatTime'], + function(_, Handlebars) { + 'use strict'; + // Replace with own lexer rules compiled from handlebars.l + Handlebars.Parser.lexer.rules = [/^(?:[^\x00]*?(?=(<%)))/, /^(?:[^\x00]+)/, /^(?:[^\x00]{2,}?(?=(\{\{|$)))/, /^(?:\{\{>)/, /^(?:<%=)/, /^(?:<%\/)/, /^(?:\{\{\^)/, /^(?:<%\s*else\b)/, /^(?:\{<%%)/, /^(?:\{\{&)/, /^(?:<%![\s\S]*?%>)/, /^(?:<%)/, /^(?:=)/, /^(?:\.(?=[%} ]))/, /^(?:\.\.)/, /^(?:[\/.])/, /^(?:\s+)/, /^(?:%%>)/, /^(?:%>)/, /^(?:"(\\["]|[^"])*")/, /^(?:'(\\[']|[^'])*')/, /^(?:@[a-zA-Z]+)/, /^(?:true(?=[%}\s]))/, /^(?:false(?=[%}\s]))/, /^(?:[0-9]+(?=[%}\s]))/, /^(?:[a-zA-Z0-9_$-]+(?=[=%}\s\/.]))/, /^(?:\[[^\]]*\])/, /^(?:.)/, /^(?:$)/]; + _.compile = Handlebars.compile; + + return Handlebars; + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/utils/lazyRequire.js b/pyload/web/app/scripts/utils/lazyRequire.js new file mode 100644 index 000000000..96c07aa24 --- /dev/null +++ b/pyload/web/app/scripts/utils/lazyRequire.js @@ -0,0 +1,97 @@ +// Define the module. +define( + [ + 'require', 'underscore' + ], + function( require, _ ){ + 'use strict'; + + + // Define the states of loading for a given set of modules + // within a require() statement. + var states = { + unloaded: 'UNLOADED', + loading: 'LOADING', + loaded: 'LOADED' + }; + + + // Define the top-level module container. Mostly, we're making + // the top-level container a non-Function so that users won't + // try to invoke this without calling the once() method below. + var lazyRequire = {}; + + + // I will return a new, unique instance of the requrieOnce() + // method. Each instance will only call the require() method + // once internally. + lazyRequire.once = function(){ + + // The modules start in an unloaded state before + // requireOnce() is invoked by the calling code. + var state = states.unloaded; + var args; + + var requireOnce = function(dependencies, loadCallback ){ + + // Use the module state to determine which method to + // invoke (or just to ignore the invocation). + if (state === states.loaded){ + loadCallback.apply(null, args); + + // The modules have not yet been requested - let's + // lazy load them. + } else if (state !== states.loading){ + + // We're about to load the modules asynchronously; + // flag the interim state. + state = states.loading; + + // Load the modules. + require( + dependencies, + function(){ + + args = arguments; + loadCallback.apply( null, args ); + state = states.loaded; + + + } + ); + + // RequireJS is currently loading the modules + // asynchronously, but they have not finished + // loading yet. + } else { + + // Simply ignore this call. + return; + + } + + }; + + // Return the new lazy loader. + return( requireOnce ); + + }; + + + // -------------------------------------------------- // + // -------------------------------------------------- // + + // Set up holder for underscore + var instances = {}; + _.requireOnce = function(dependencies, loadCallback) { + if (!_.has(instances, dependencies)) + instances[dependencies] = lazyRequire.once(); + + return instances[dependencies](dependencies, loadCallback); + }; + + + // Return the module definition. + return( lazyRequire ); + } +); \ No newline at end of file diff --git a/pyload/web/app/scripts/vendor/Handlebars-1.0rc1.js b/pyload/web/app/scripts/vendor/Handlebars-1.0rc1.js new file mode 100644 index 000000000..991242461 --- /dev/null +++ b/pyload/web/app/scripts/vendor/Handlebars-1.0rc1.js @@ -0,0 +1,1927 @@ +// lib/handlebars/base.js +(function () { +/*jshint eqnull:true*/ +this.Handlebars = {}; + +(function(Handlebars) { + +Handlebars.VERSION = "1.0.rc.1"; + +Handlebars.helpers = {}; +Handlebars.partials = {}; + +Handlebars.registerHelper = function(name, fn, inverse) { + if(inverse) { fn.not = inverse; } + this.helpers[name] = fn; +}; + +Handlebars.registerPartial = function(name, str) { + this.partials[name] = str; +}; + +Handlebars.registerHelper('helperMissing', function(arg) { + if(arguments.length === 2) { + return undefined; + } else { + throw new Error("Could not find property '" + arg + "'"); + } +}); + +var toString = Object.prototype.toString, functionType = "[object Function]"; + +Handlebars.registerHelper('blockHelperMissing', function(context, options) { + var inverse = options.inverse || function() {}, fn = options.fn; + + + var ret = ""; + var type = toString.call(context); + + if(type === functionType) { context = context.call(this); } + + if(context === true) { + return fn(this); + } else if(context === false || context == null) { + return inverse(this); + } else if(type === "[object Array]") { + if(context.length > 0) { + return Handlebars.helpers.each(context, options); + } else { + return inverse(this); + } + } else { + return fn(context); + } +}); + +Handlebars.K = function() {}; + +Handlebars.createFrame = Object.create || function(object) { + Handlebars.K.prototype = object; + var obj = new Handlebars.K(); + Handlebars.K.prototype = null; + return obj; +}; + +Handlebars.registerHelper('each', function(context, options) { + var fn = options.fn, inverse = options.inverse; + var ret = "", data; + + if (options.data) { + data = Handlebars.createFrame(options.data); + } + + if(context && context.length > 0) { + for(var i=0, j=context.length; i 2) { + expected.push("'" + this.terminals_[p] + "'"); + } + if (this.lexer.showPosition) { + errStr = "Parse error on line " + (yylineno + 1) + ":\n" + this.lexer.showPosition() + "\nExpecting " + expected.join(", ") + ", got '" + (this.terminals_[symbol] || symbol) + "'"; + } else { + errStr = "Parse error on line " + (yylineno + 1) + ": Unexpected " + (symbol == 1?"end of input":"'" + (this.terminals_[symbol] || symbol) + "'"); + } + this.parseError(errStr, {text: this.lexer.match, token: this.terminals_[symbol] || symbol, line: this.lexer.yylineno, loc: yyloc, expected: expected}); + } + } + if (action[0] instanceof Array && action.length > 1) { + throw new Error("Parse Error: multiple actions possible at state: " + state + ", token: " + symbol); + } + switch (action[0]) { + case 1: + stack.push(symbol); + vstack.push(this.lexer.yytext); + lstack.push(this.lexer.yylloc); + stack.push(action[1]); + symbol = null; + if (!preErrorSymbol) { + yyleng = this.lexer.yyleng; + yytext = this.lexer.yytext; + yylineno = this.lexer.yylineno; + yyloc = this.lexer.yylloc; + if (recovering > 0) + recovering--; + } else { + symbol = preErrorSymbol; + preErrorSymbol = null; + } + break; + case 2: + len = this.productions_[action[1]][1]; + yyval.$ = vstack[vstack.length - len]; + yyval._$ = {first_line: lstack[lstack.length - (len || 1)].first_line, last_line: lstack[lstack.length - 1].last_line, first_column: lstack[lstack.length - (len || 1)].first_column, last_column: lstack[lstack.length - 1].last_column}; + if (ranges) { + yyval._$.range = [lstack[lstack.length - (len || 1)].range[0], lstack[lstack.length - 1].range[1]]; + } + r = this.performAction.call(yyval, yytext, yyleng, yylineno, this.yy, action[1], vstack, lstack); + if (typeof r !== "undefined") { + return r; + } + if (len) { + stack = stack.slice(0, -1 * len * 2); + vstack = vstack.slice(0, -1 * len); + lstack = lstack.slice(0, -1 * len); + } + stack.push(this.productions_[action[1]][0]); + vstack.push(yyval.$); + lstack.push(yyval._$); + newState = table[stack[stack.length - 2]][stack[stack.length - 1]]; + stack.push(newState); + break; + case 3: + return true; + } + } + return true; +} +}; +/* Jison generated lexer */ +var lexer = (function(){ +var lexer = ({EOF:1, +parseError:function parseError(str, hash) { + if (this.yy.parser) { + this.yy.parser.parseError(str, hash); + } else { + throw new Error(str); + } + }, +setInput:function (input) { + this._input = input; + this._more = this._less = this.done = false; + this.yylineno = this.yyleng = 0; + this.yytext = this.matched = this.match = ''; + this.conditionStack = ['INITIAL']; + this.yylloc = {first_line:1,first_column:0,last_line:1,last_column:0}; + if (this.options.ranges) this.yylloc.range = [0,0]; + this.offset = 0; + return this; + }, +input:function () { + var ch = this._input[0]; + this.yytext += ch; + this.yyleng++; + this.offset++; + this.match += ch; + this.matched += ch; + var lines = ch.match(/(?:\r\n?|\n).*/g); + if (lines) { + this.yylineno++; + this.yylloc.last_line++; + } else { + this.yylloc.last_column++; + } + if (this.options.ranges) this.yylloc.range[1]++; + + this._input = this._input.slice(1); + return ch; + }, +unput:function (ch) { + var len = ch.length; + var lines = ch.split(/(?:\r\n?|\n)/g); + + this._input = ch + this._input; + this.yytext = this.yytext.substr(0, this.yytext.length-len-1); + //this.yyleng -= len; + this.offset -= len; + var oldLines = this.match.split(/(?:\r\n?|\n)/g); + this.match = this.match.substr(0, this.match.length-1); + this.matched = this.matched.substr(0, this.matched.length-1); + + if (lines.length-1) this.yylineno -= lines.length-1; + var r = this.yylloc.range; + + this.yylloc = {first_line: this.yylloc.first_line, + last_line: this.yylineno+1, + first_column: this.yylloc.first_column, + last_column: lines ? + (lines.length === oldLines.length ? this.yylloc.first_column : 0) + oldLines[oldLines.length - lines.length].length - lines[0].length: + this.yylloc.first_column - len + }; + + if (this.options.ranges) { + this.yylloc.range = [r[0], r[0] + this.yyleng - len]; + } + return this; + }, +more:function () { + this._more = true; + return this; + }, +less:function (n) { + this.unput(this.match.slice(n)); + }, +pastInput:function () { + var past = this.matched.substr(0, this.matched.length - this.match.length); + return (past.length > 20 ? '...':'') + past.substr(-20).replace(/\n/g, ""); + }, +upcomingInput:function () { + var next = this.match; + if (next.length < 20) { + next += this._input.substr(0, 20-next.length); + } + return (next.substr(0,20)+(next.length > 20 ? '...':'')).replace(/\n/g, ""); + }, +showPosition:function () { + var pre = this.pastInput(); + var c = new Array(pre.length + 1).join("-"); + return pre + this.upcomingInput() + "\n" + c+"^"; + }, +next:function () { + if (this.done) { + return this.EOF; + } + if (!this._input) this.done = true; + + var token, + match, + tempMatch, + index, + col, + lines; + if (!this._more) { + this.yytext = ''; + this.match = ''; + } + var rules = this._currentRules(); + for (var i=0;i < rules.length; i++) { + tempMatch = this._input.match(this.rules[rules[i]]); + if (tempMatch && (!match || tempMatch[0].length > match[0].length)) { + match = tempMatch; + index = i; + if (!this.options.flex) break; + } + } + if (match) { + lines = match[0].match(/(?:\r\n?|\n).*/g); + if (lines) this.yylineno += lines.length; + this.yylloc = {first_line: this.yylloc.last_line, + last_line: this.yylineno+1, + first_column: this.yylloc.last_column, + last_column: lines ? lines[lines.length-1].length-lines[lines.length-1].match(/\r?\n?/)[0].length : this.yylloc.last_column + match[0].length}; + this.yytext += match[0]; + this.match += match[0]; + this.matches = match; + this.yyleng = this.yytext.length; + if (this.options.ranges) { + this.yylloc.range = [this.offset, this.offset += this.yyleng]; + } + this._more = false; + this._input = this._input.slice(match[0].length); + this.matched += match[0]; + token = this.performAction.call(this, this.yy, this, rules[index],this.conditionStack[this.conditionStack.length-1]); + if (this.done && this._input) this.done = false; + if (token) return token; + else return; + } + if (this._input === "") { + return this.EOF; + } else { + return this.parseError('Lexical error on line '+(this.yylineno+1)+'. Unrecognized text.\n'+this.showPosition(), + {text: "", token: null, line: this.yylineno}); + } + }, +lex:function lex() { + var r = this.next(); + if (typeof r !== 'undefined') { + return r; + } else { + return this.lex(); + } + }, +begin:function begin(condition) { + this.conditionStack.push(condition); + }, +popState:function popState() { + return this.conditionStack.pop(); + }, +_currentRules:function _currentRules() { + return this.conditions[this.conditionStack[this.conditionStack.length-1]].rules; + }, +topState:function () { + return this.conditionStack[this.conditionStack.length-2]; + }, +pushState:function begin(condition) { + this.begin(condition); + }}); +lexer.options = {}; +lexer.performAction = function anonymous(yy,yy_,$avoiding_name_collisions,YY_START) { + +var YYSTATE=YY_START +switch($avoiding_name_collisions) { +case 0: + if(yy_.yytext.slice(-1) !== "\\") this.begin("mu"); + if(yy_.yytext.slice(-1) === "\\") yy_.yytext = yy_.yytext.substr(0,yy_.yyleng-1), this.begin("emu"); + if(yy_.yytext) return 14; + +break; +case 1: return 14; +break; +case 2: + if(yy_.yytext.slice(-1) !== "\\") this.popState(); + if(yy_.yytext.slice(-1) === "\\") yy_.yytext = yy_.yytext.substr(0,yy_.yyleng-1); + return 14; + +break; +case 3: return 24; +break; +case 4: return 16; +break; +case 5: return 20; +break; +case 6: return 19; +break; +case 7: return 19; +break; +case 8: return 23; +break; +case 9: return 23; +break; +case 10: yy_.yytext = yy_.yytext.substr(3,yy_.yyleng-5); this.popState(); return 15; +break; +case 11: return 22; +break; +case 12: return 35; +break; +case 13: return 34; +break; +case 14: return 34; +break; +case 15: return 37; +break; +case 16: /*ignore whitespace*/ +break; +case 17: this.popState(); return 18; +break; +case 18: this.popState(); return 18; +break; +case 19: yy_.yytext = yy_.yytext.substr(1,yy_.yyleng-2).replace(/\\"/g,'"'); return 29; +break; +case 20: yy_.yytext = yy_.yytext.substr(1,yy_.yyleng-2).replace(/\\"/g,'"'); return 29; +break; +case 21: yy_.yytext = yy_.yytext.substr(1); return 27; +break; +case 22: return 31; +break; +case 23: return 31; +break; +case 24: return 30; +break; +case 25: return 34; +break; +case 26: yy_.yytext = yy_.yytext.substr(1, yy_.yyleng-2); return 34; +break; +case 27: return 'INVALID'; +break; +case 28: return 5; +break; +} +}; +lexer.rules = [/^(?:[^\x00]*?(?=(\{\{)))/,/^(?:[^\x00]+)/,/^(?:[^\x00]{2,}?(?=(\{\{|$)))/,/^(?:\{\{>)/,/^(?:\{\{#)/,/^(?:\{\{\/)/,/^(?:\{\{\^)/,/^(?:\{\{\s*else\b)/,/^(?:\{\{\{)/,/^(?:\{\{&)/,/^(?:\{\{![\s\S]*?\}\})/,/^(?:\{\{)/,/^(?:=)/,/^(?:\.(?=[} ]))/,/^(?:\.\.)/,/^(?:[\/.])/,/^(?:\s+)/,/^(?:\}\}\})/,/^(?:\}\})/,/^(?:"(\\["]|[^"])*")/,/^(?:'(\\[']|[^'])*')/,/^(?:@[a-zA-Z]+)/,/^(?:true(?=[}\s]))/,/^(?:false(?=[}\s]))/,/^(?:[0-9]+(?=[}\s]))/,/^(?:[a-zA-Z0-9_$-]+(?=[=}\s\/.]))/,/^(?:\[[^\]]*\])/,/^(?:.)/,/^(?:$)/]; +lexer.conditions = {"mu":{"rules":[3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28],"inclusive":false},"emu":{"rules":[2],"inclusive":false},"INITIAL":{"rules":[0,1,28],"inclusive":true}}; +return lexer;})() +parser.lexer = lexer; +function Parser () { this.yy = {}; }Parser.prototype = parser;parser.Parser = Parser; +return new Parser; +})(); +if (typeof require !== 'undefined' && typeof exports !== 'undefined') { +exports.parser = handlebars; +exports.Parser = handlebars.Parser; +exports.parse = function () { return handlebars.parse.apply(handlebars, arguments); } +exports.main = function commonjsMain(args) { + if (!args[1]) + throw new Error('Usage: '+args[0]+' FILE'); + var source, cwd; + if (typeof process !== 'undefined') { + source = require('fs').readFileSync(require('path').resolve(args[1]), "utf8"); + } else { + source = require("file").path(require("file").cwd()).join(args[1]).read({charset: "utf-8"}); + } + return exports.parser.parse(source); +} +if (typeof module !== 'undefined' && require.main === module) { + exports.main(typeof process !== 'undefined' ? process.argv.slice(1) : require("system").args); +} +}; +; +// lib/handlebars/compiler/base.js +Handlebars.Parser = handlebars; + +Handlebars.parse = function(string) { + Handlebars.Parser.yy = Handlebars.AST; + return Handlebars.Parser.parse(string); +}; + +Handlebars.print = function(ast) { + return new Handlebars.PrintVisitor().accept(ast); +}; + +Handlebars.logger = { + DEBUG: 0, INFO: 1, WARN: 2, ERROR: 3, level: 3, + + // override in the host environment + log: function(level, str) {} +}; + +Handlebars.log = function(level, str) { Handlebars.logger.log(level, str); }; +; +// lib/handlebars/compiler/ast.js +(function() { + + Handlebars.AST = {}; + + Handlebars.AST.ProgramNode = function(statements, inverse) { + this.type = "program"; + this.statements = statements; + if(inverse) { this.inverse = new Handlebars.AST.ProgramNode(inverse); } + }; + + Handlebars.AST.MustacheNode = function(rawParams, hash, unescaped) { + this.type = "mustache"; + this.escaped = !unescaped; + this.hash = hash; + + var id = this.id = rawParams[0]; + var params = this.params = rawParams.slice(1); + + // a mustache is an eligible helper if: + // * its id is simple (a single part, not `this` or `..`) + var eligibleHelper = this.eligibleHelper = id.isSimple; + + // a mustache is definitely a helper if: + // * it is an eligible helper, and + // * it has at least one parameter or hash segment + this.isHelper = eligibleHelper && (params.length || hash); + + // if a mustache is an eligible helper but not a definite + // helper, it is ambiguous, and will be resolved in a later + // pass or at runtime. + }; + + Handlebars.AST.PartialNode = function(id, context) { + this.type = "partial"; + + // TODO: disallow complex IDs + + this.id = id; + this.context = context; + }; + + var verifyMatch = function(open, close) { + if(open.original !== close.original) { + throw new Handlebars.Exception(open.original + " doesn't match " + close.original); + } + }; + + Handlebars.AST.BlockNode = function(mustache, program, inverse, close) { + verifyMatch(mustache.id, close); + this.type = "block"; + this.mustache = mustache; + this.program = program; + this.inverse = inverse; + + if (this.inverse && !this.program) { + this.isInverse = true; + } + }; + + Handlebars.AST.ContentNode = function(string) { + this.type = "content"; + this.string = string; + }; + + Handlebars.AST.HashNode = function(pairs) { + this.type = "hash"; + this.pairs = pairs; + }; + + Handlebars.AST.IdNode = function(parts) { + this.type = "ID"; + this.original = parts.join("."); + + var dig = [], depth = 0; + + for(var i=0,l=parts.length; i": ">", + '"': """, + "'": "'", + "`": "`" + }; + + var badChars = /[&<>"'`]/g; + var possible = /[&<>"'`]/; + + var escapeChar = function(chr) { + return escape[chr] || "&"; + }; + + Handlebars.Utils = { + escapeExpression: function(string) { + // don't escape SafeStrings, since they're already safe + if (string instanceof Handlebars.SafeString) { + return string.toString(); + } else if (string == null || string === false) { + return ""; + } + + if(!possible.test(string)) { return string; } + return string.replace(badChars, escapeChar); + }, + + isEmpty: function(value) { + if (typeof value === "undefined") { + return true; + } else if (value === null) { + return true; + } else if (value === false) { + return true; + } else if(Object.prototype.toString.call(value) === "[object Array]" && value.length === 0) { + return true; + } else { + return false; + } + } + }; +})();; +// lib/handlebars/compiler/compiler.js + +/*jshint eqnull:true*/ +Handlebars.Compiler = function() {}; +Handlebars.JavaScriptCompiler = function() {}; + +(function(Compiler, JavaScriptCompiler) { + // the foundHelper register will disambiguate helper lookup from finding a + // function in a context. This is necessary for mustache compatibility, which + // requires that context functions in blocks are evaluated by blockHelperMissing, + // and then proceed as if the resulting value was provided to blockHelperMissing. + + Compiler.prototype = { + compiler: Compiler, + + disassemble: function() { + var opcodes = this.opcodes, opcode, out = [], params, param; + + for (var i=0, l=opcodes.length; i 0) { + this.source[1] = this.source[1] + ", " + locals.join(", "); + } + + // Generate minimizer alias mappings + if (!this.isChild) { + var aliases = []; + for (var alias in this.context.aliases) { + this.source[1] = this.source[1] + ', ' + alias + '=' + this.context.aliases[alias]; + } + } + + if (this.source[1]) { + this.source[1] = "var " + this.source[1].substring(2) + ";"; + } + + // Merge children + if (!this.isChild) { + this.source[1] += '\n' + this.context.programs.join('\n') + '\n'; + } + + if (!this.environment.isSimple) { + this.source.push("return buffer;"); + } + + var params = this.isChild ? ["depth0", "data"] : ["Handlebars", "depth0", "helpers", "partials", "data"]; + + for(var i=0, l=this.environment.depths.list.length; i this.stackVars.length) { this.stackVars.push("stack" + this.stackSlot); } + return "stack" + this.stackSlot; + }, + + popStack: function() { + var item = this.compileStack.pop(); + + if (item instanceof Literal) { + return item.value; + } else { + this.stackSlot--; + return item; + } + }, + + topStack: function() { + var item = this.compileStack[this.compileStack.length - 1]; + + if (item instanceof Literal) { + return item.value; + } else { + return item; + } + }, + + quotedString: function(str) { + return '"' + str + .replace(/\\/g, '\\\\') + .replace(/"/g, '\\"') + .replace(/\n/g, '\\n') + .replace(/\r/g, '\\r') + '"'; + }, + + setupHelper: function(paramSize, name) { + var params = []; + this.setupParams(paramSize, params); + var foundHelper = this.nameLookup('helpers', name, 'helper'); + + return { + params: params, + name: foundHelper, + callParams: ["depth0"].concat(params).join(", "), + helperMissingParams: ["depth0", this.quotedString(name)].concat(params).join(", ") + }; + }, + + // the params and contexts arguments are passed in arrays + // to fill in + setupParams: function(paramSize, params) { + var options = [], contexts = [], param, inverse, program; + + options.push("hash:" + this.popStack()); + + inverse = this.popStack(); + program = this.popStack(); + + // Avoid setting fn and inverse if neither are set. This allows + // helpers to do a check for `if (options.fn)` + if (program || inverse) { + if (!program) { + this.context.aliases.self = "this"; + program = "self.noop"; + } + + if (!inverse) { + this.context.aliases.self = "this"; + inverse = "self.noop"; + } + + options.push("inverse:" + inverse); + options.push("fn:" + program); + } + + for(var i=0; i') + .appendTo(document.body) + + this.$backdrop.click( + this.options.backdrop == 'static' ? + $.proxy(this.$element[0].focus, this.$element[0]) + : $.proxy(this.hide, this) + ) + + if (doAnimate) this.$backdrop[0].offsetWidth // force reflow + + this.$backdrop.addClass('in') + + if (!callback) return + + doAnimate ? + this.$backdrop.one($.support.transition.end, callback) : + callback() + + } else if (!this.isShown && this.$backdrop) { + this.$backdrop.removeClass('in') + + $.support.transition && this.$element.hasClass('fade')? + this.$backdrop.one($.support.transition.end, callback) : + callback() + + } else if (callback) { + callback() + } + } + } + + + /* MODAL PLUGIN DEFINITION + * ======================= */ + + var old = $.fn.modal + + $.fn.modal = function (option) { + return this.each(function () { + var $this = $(this) + , data = $this.data('modal') + , options = $.extend({}, $.fn.modal.defaults, $this.data(), typeof option == 'object' && option) + if (!data) $this.data('modal', (data = new Modal(this, options))) + if (typeof option == 'string') data[option]() + else if (options.show) data.show() + }) + } + + $.fn.modal.defaults = { + backdrop: true + , keyboard: true + , show: true + } + + $.fn.modal.Constructor = Modal + + + /* MODAL NO CONFLICT + * ================= */ + + $.fn.modal.noConflict = function () { + $.fn.modal = old + return this + } + + + /* MODAL DATA-API + * ============== */ + + $(document).on('click.modal.data-api', '[data-toggle="modal"]', function (e) { + var $this = $(this) + , href = $this.attr('href') + , $target = $($this.attr('data-target') || (href && href.replace(/.*(?=#[^\s]+$)/, ''))) //strip for ie7 + , option = $target.data('modal') ? 'toggle' : $.extend({ remote:!/#/.test(href) && href }, $target.data(), $this.data()) + + e.preventDefault() + + $target + .modal(option) + .one('hide', function () { + $this.focus() + }) + }) + +}(window.jQuery); + +/* ============================================================ + * bootstrap-dropdown.js v2.3.2 + * http://twitter.github.com/bootstrap/javascript.html#dropdowns + * ============================================================ + * Copyright 2012 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================ */ + + +!function ($) { + + "use strict"; // jshint ;_; + + + /* DROPDOWN CLASS DEFINITION + * ========================= */ + + var toggle = '[data-toggle=dropdown]' + , Dropdown = function (element) { + var $el = $(element).on('click.dropdown.data-api', this.toggle) + $('html').on('click.dropdown.data-api', function () { + $el.parent().removeClass('open') + }) + } + + Dropdown.prototype = { + + constructor: Dropdown + + , toggle: function (e) { + var $this = $(this) + , $parent + , isActive + + if ($this.is('.disabled, :disabled')) return + + $parent = getParent($this) + + isActive = $parent.hasClass('open') + + clearMenus() + + if (!isActive) { + if ('ontouchstart' in document.documentElement) { + // if mobile we we use a backdrop because click events don't delegate + $('
        ').insertBefore($(this)).on('click', clearMenus) + } + $parent.toggleClass('open') + } + + $this.focus() + + return false + } + + , keydown: function (e) { + var $this + , $items + , $active + , $parent + , isActive + , index + + if (!/(38|40|27)/.test(e.keyCode)) return + + $this = $(this) + + e.preventDefault() + e.stopPropagation() + + if ($this.is('.disabled, :disabled')) return + + $parent = getParent($this) + + isActive = $parent.hasClass('open') + + if (!isActive || (isActive && e.keyCode == 27)) { + if (e.which == 27) $parent.find(toggle).focus() + return $this.click() + } + + $items = $('[role=menu] li:not(.divider):visible a', $parent) + + if (!$items.length) return + + index = $items.index($items.filter(':focus')) + + if (e.keyCode == 38 && index > 0) index-- // up + if (e.keyCode == 40 && index < $items.length - 1) index++ // down + if (!~index) index = 0 + + $items + .eq(index) + .focus() + } + + } + + function clearMenus() { + $('.dropdown-backdrop').remove() + $(toggle).each(function () { + getParent($(this)).removeClass('open') + }) + } + + function getParent($this) { + var selector = $this.attr('data-target') + , $parent + + if (!selector) { + selector = $this.attr('href') + selector = selector && /#/.test(selector) && selector.replace(/.*(?=#[^\s]*$)/, '') //strip for ie7 + } + + $parent = selector && $(selector) + + if (!$parent || !$parent.length) $parent = $this.parent() + + return $parent + } + + + /* DROPDOWN PLUGIN DEFINITION + * ========================== */ + + var old = $.fn.dropdown + + $.fn.dropdown = function (option) { + return this.each(function () { + var $this = $(this) + , data = $this.data('dropdown') + if (!data) $this.data('dropdown', (data = new Dropdown(this))) + if (typeof option == 'string') data[option].call($this) + }) + } + + $.fn.dropdown.Constructor = Dropdown + + + /* DROPDOWN NO CONFLICT + * ==================== */ + + $.fn.dropdown.noConflict = function () { + $.fn.dropdown = old + return this + } + + + /* APPLY TO STANDARD DROPDOWN ELEMENTS + * =================================== */ + + $(document) + .on('click.dropdown.data-api', clearMenus) + .on('click.dropdown.data-api', '.dropdown form', function (e) { e.stopPropagation() }) + .on('click.dropdown.data-api' , toggle, Dropdown.prototype.toggle) + .on('keydown.dropdown.data-api', toggle + ', [role=menu]' , Dropdown.prototype.keydown) + +}(window.jQuery); + +/* ============================================================= + * bootstrap-scrollspy.js v2.3.2 + * http://twitter.github.com/bootstrap/javascript.html#scrollspy + * ============================================================= + * Copyright 2012 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================== */ + + +!function ($) { + + "use strict"; // jshint ;_; + + + /* SCROLLSPY CLASS DEFINITION + * ========================== */ + + function ScrollSpy(element, options) { + var process = $.proxy(this.process, this) + , $element = $(element).is('body') ? $(window) : $(element) + , href + this.options = $.extend({}, $.fn.scrollspy.defaults, options) + this.$scrollElement = $element.on('scroll.scroll-spy.data-api', process) + this.selector = (this.options.target + || ((href = $(element).attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '')) //strip for ie7 + || '') + ' .nav li > a' + this.$body = $('body') + this.refresh() + this.process() + } + + ScrollSpy.prototype = { + + constructor: ScrollSpy + + , refresh: function () { + var self = this + , $targets + + this.offsets = $([]) + this.targets = $([]) + + $targets = this.$body + .find(this.selector) + .map(function () { + var $el = $(this) + , href = $el.data('target') || $el.attr('href') + , $href = /^#\w/.test(href) && $(href) + return ( $href + && $href.length + && [[ $href.position().top + (!$.isWindow(self.$scrollElement.get(0)) && self.$scrollElement.scrollTop()), href ]] ) || null + }) + .sort(function (a, b) { return a[0] - b[0] }) + .each(function () { + self.offsets.push(this[0]) + self.targets.push(this[1]) + }) + } + + , process: function () { + var scrollTop = this.$scrollElement.scrollTop() + this.options.offset + , scrollHeight = this.$scrollElement[0].scrollHeight || this.$body[0].scrollHeight + , maxScroll = scrollHeight - this.$scrollElement.height() + , offsets = this.offsets + , targets = this.targets + , activeTarget = this.activeTarget + , i + + if (scrollTop >= maxScroll) { + return activeTarget != (i = targets.last()[0]) + && this.activate ( i ) + } + + for (i = offsets.length; i--;) { + activeTarget != targets[i] + && scrollTop >= offsets[i] + && (!offsets[i + 1] || scrollTop <= offsets[i + 1]) + && this.activate( targets[i] ) + } + } + + , activate: function (target) { + var active + , selector + + this.activeTarget = target + + $(this.selector) + .parent('.active') + .removeClass('active') + + selector = this.selector + + '[data-target="' + target + '"],' + + this.selector + '[href="' + target + '"]' + + active = $(selector) + .parent('li') + .addClass('active') + + if (active.parent('.dropdown-menu').length) { + active = active.closest('li.dropdown').addClass('active') + } + + active.trigger('activate') + } + + } + + + /* SCROLLSPY PLUGIN DEFINITION + * =========================== */ + + var old = $.fn.scrollspy + + $.fn.scrollspy = function (option) { + return this.each(function () { + var $this = $(this) + , data = $this.data('scrollspy') + , options = typeof option == 'object' && option + if (!data) $this.data('scrollspy', (data = new ScrollSpy(this, options))) + if (typeof option == 'string') data[option]() + }) + } + + $.fn.scrollspy.Constructor = ScrollSpy + + $.fn.scrollspy.defaults = { + offset: 10 + } + + + /* SCROLLSPY NO CONFLICT + * ===================== */ + + $.fn.scrollspy.noConflict = function () { + $.fn.scrollspy = old + return this + } + + + /* SCROLLSPY DATA-API + * ================== */ + + $(window).on('load', function () { + $('[data-spy="scroll"]').each(function () { + var $spy = $(this) + $spy.scrollspy($spy.data()) + }) + }) + +}(window.jQuery); +/* ======================================================== + * bootstrap-tab.js v2.3.2 + * http://twitter.github.com/bootstrap/javascript.html#tabs + * ======================================================== + * Copyright 2012 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ======================================================== */ + + +!function ($) { + + "use strict"; // jshint ;_; + + + /* TAB CLASS DEFINITION + * ==================== */ + + var Tab = function (element) { + this.element = $(element) + } + + Tab.prototype = { + + constructor: Tab + + , show: function () { + var $this = this.element + , $ul = $this.closest('ul:not(.dropdown-menu)') + , selector = $this.attr('data-target') + , previous + , $target + , e + + if (!selector) { + selector = $this.attr('href') + selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') //strip for ie7 + } + + if ( $this.parent('li').hasClass('active') ) return + + previous = $ul.find('.active:last a')[0] + + e = $.Event('show', { + relatedTarget: previous + }) + + $this.trigger(e) + + if (e.isDefaultPrevented()) return + + $target = $(selector) + + this.activate($this.parent('li'), $ul) + this.activate($target, $target.parent(), function () { + $this.trigger({ + type: 'shown' + , relatedTarget: previous + }) + }) + } + + , activate: function ( element, container, callback) { + var $active = container.find('> .active') + , transition = callback + && $.support.transition + && $active.hasClass('fade') + + function next() { + $active + .removeClass('active') + .find('> .dropdown-menu > .active') + .removeClass('active') + + element.addClass('active') + + if (transition) { + element[0].offsetWidth // reflow for transition + element.addClass('in') + } else { + element.removeClass('fade') + } + + if ( element.parent('.dropdown-menu') ) { + element.closest('li.dropdown').addClass('active') + } + + callback && callback() + } + + transition ? + $active.one($.support.transition.end, next) : + next() + + $active.removeClass('in') + } + } + + + /* TAB PLUGIN DEFINITION + * ===================== */ + + var old = $.fn.tab + + $.fn.tab = function ( option ) { + return this.each(function () { + var $this = $(this) + , data = $this.data('tab') + if (!data) $this.data('tab', (data = new Tab(this))) + if (typeof option == 'string') data[option]() + }) + } + + $.fn.tab.Constructor = Tab + + + /* TAB NO CONFLICT + * =============== */ + + $.fn.tab.noConflict = function () { + $.fn.tab = old + return this + } + + + /* TAB DATA-API + * ============ */ + + $(document).on('click.tab.data-api', '[data-toggle="tab"], [data-toggle="pill"]', function (e) { + e.preventDefault() + $(this).tab('show') + }) + +}(window.jQuery); +/* =========================================================== + * bootstrap-tooltip.js v2.3.2 + * http://twitter.github.com/bootstrap/javascript.html#tooltips + * Inspired by the original jQuery.tipsy by Jason Frame + * =========================================================== + * Copyright 2012 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ========================================================== */ + + +!function ($) { + + "use strict"; // jshint ;_; + + + /* TOOLTIP PUBLIC CLASS DEFINITION + * =============================== */ + + var Tooltip = function (element, options) { + this.init('tooltip', element, options) + } + + Tooltip.prototype = { + + constructor: Tooltip + + , init: function (type, element, options) { + var eventIn + , eventOut + , triggers + , trigger + , i + + this.type = type + this.$element = $(element) + this.options = this.getOptions(options) + this.enabled = true + + triggers = this.options.trigger.split(' ') + + for (i = triggers.length; i--;) { + trigger = triggers[i] + if (trigger == 'click') { + this.$element.on('click.' + this.type, this.options.selector, $.proxy(this.toggle, this)) + } else if (trigger != 'manual') { + eventIn = trigger == 'hover' ? 'mouseenter' : 'focus' + eventOut = trigger == 'hover' ? 'mouseleave' : 'blur' + this.$element.on(eventIn + '.' + this.type, this.options.selector, $.proxy(this.enter, this)) + this.$element.on(eventOut + '.' + this.type, this.options.selector, $.proxy(this.leave, this)) + } + } + + this.options.selector ? + (this._options = $.extend({}, this.options, { trigger: 'manual', selector: '' })) : + this.fixTitle() + } + + , getOptions: function (options) { + options = $.extend({}, $.fn[this.type].defaults, this.$element.data(), options) + + if (options.delay && typeof options.delay == 'number') { + options.delay = { + show: options.delay + , hide: options.delay + } + } + + return options + } + + , enter: function (e) { + var defaults = $.fn[this.type].defaults + , options = {} + , self + + this._options && $.each(this._options, function (key, value) { + if (defaults[key] != value) options[key] = value + }, this) + + self = $(e.currentTarget)[this.type](options).data(this.type) + + if (!self.options.delay || !self.options.delay.show) return self.show() + + clearTimeout(this.timeout) + self.hoverState = 'in' + this.timeout = setTimeout(function() { + if (self.hoverState == 'in') self.show() + }, self.options.delay.show) + } + + , leave: function (e) { + var self = $(e.currentTarget)[this.type](this._options).data(this.type) + + if (this.timeout) clearTimeout(this.timeout) + if (!self.options.delay || !self.options.delay.hide) return self.hide() + + self.hoverState = 'out' + this.timeout = setTimeout(function() { + if (self.hoverState == 'out') self.hide() + }, self.options.delay.hide) + } + + , show: function () { + var $tip + , pos + , actualWidth + , actualHeight + , placement + , tp + , e = $.Event('show') + + if (this.hasContent() && this.enabled) { + this.$element.trigger(e) + if (e.isDefaultPrevented()) return + $tip = this.tip() + this.setContent() + + if (this.options.animation) { + $tip.addClass('fade') + } + + placement = typeof this.options.placement == 'function' ? + this.options.placement.call(this, $tip[0], this.$element[0]) : + this.options.placement + + $tip + .detach() + .css({ top: 0, left: 0, display: 'block' }) + + this.options.container ? $tip.appendTo(this.options.container) : $tip.insertAfter(this.$element) + + pos = this.getPosition() + + actualWidth = $tip[0].offsetWidth + actualHeight = $tip[0].offsetHeight + + switch (placement) { + case 'bottom': + tp = {top: pos.top + pos.height, left: pos.left + pos.width / 2 - actualWidth / 2} + break + case 'top': + tp = {top: pos.top - actualHeight, left: pos.left + pos.width / 2 - actualWidth / 2} + break + case 'left': + tp = {top: pos.top + pos.height / 2 - actualHeight / 2, left: pos.left - actualWidth} + break + case 'right': + tp = {top: pos.top + pos.height / 2 - actualHeight / 2, left: pos.left + pos.width} + break + } + + this.applyPlacement(tp, placement) + this.$element.trigger('shown') + } + } + + , applyPlacement: function(offset, placement){ + var $tip = this.tip() + , width = $tip[0].offsetWidth + , height = $tip[0].offsetHeight + , actualWidth + , actualHeight + , delta + , replace + + $tip + .offset(offset) + .addClass(placement) + .addClass('in') + + actualWidth = $tip[0].offsetWidth + actualHeight = $tip[0].offsetHeight + + if (placement == 'top' && actualHeight != height) { + offset.top = offset.top + height - actualHeight + replace = true + } + + if (placement == 'bottom' || placement == 'top') { + delta = 0 + + if (offset.left < 0){ + delta = offset.left * -2 + offset.left = 0 + $tip.offset(offset) + actualWidth = $tip[0].offsetWidth + actualHeight = $tip[0].offsetHeight + } + + this.replaceArrow(delta - width + actualWidth, actualWidth, 'left') + } else { + this.replaceArrow(actualHeight - height, actualHeight, 'top') + } + + if (replace) $tip.offset(offset) + } + + , replaceArrow: function(delta, dimension, position){ + this + .arrow() + .css(position, delta ? (50 * (1 - delta / dimension) + "%") : '') + } + + , setContent: function () { + var $tip = this.tip() + , title = this.getTitle() + + $tip.find('.tooltip-inner')[this.options.html ? 'html' : 'text'](title) + $tip.removeClass('fade in top bottom left right') + } + + , hide: function () { + var that = this + , $tip = this.tip() + , e = $.Event('hide') + + this.$element.trigger(e) + if (e.isDefaultPrevented()) return + + $tip.removeClass('in') + + function removeWithAnimation() { + var timeout = setTimeout(function () { + $tip.off($.support.transition.end).detach() + }, 500) + + $tip.one($.support.transition.end, function () { + clearTimeout(timeout) + $tip.detach() + }) + } + + $.support.transition && this.$tip.hasClass('fade') ? + removeWithAnimation() : + $tip.detach() + + this.$element.trigger('hidden') + + return this + } + + , fixTitle: function () { + var $e = this.$element + if ($e.attr('title') || typeof($e.attr('data-original-title')) != 'string') { + $e.attr('data-original-title', $e.attr('title') || '').attr('title', '') + } + } + + , hasContent: function () { + return this.getTitle() + } + + , getPosition: function () { + var el = this.$element[0] + return $.extend({}, (typeof el.getBoundingClientRect == 'function') ? el.getBoundingClientRect() : { + width: el.offsetWidth + , height: el.offsetHeight + }, this.$element.offset()) + } + + , getTitle: function () { + var title + , $e = this.$element + , o = this.options + + title = $e.attr('data-original-title') + || (typeof o.title == 'function' ? o.title.call($e[0]) : o.title) + + return title + } + + , tip: function () { + return this.$tip = this.$tip || $(this.options.template) + } + + , arrow: function(){ + return this.$arrow = this.$arrow || this.tip().find(".tooltip-arrow") + } + + , validate: function () { + if (!this.$element[0].parentNode) { + this.hide() + this.$element = null + this.options = null + } + } + + , enable: function () { + this.enabled = true + } + + , disable: function () { + this.enabled = false + } + + , toggleEnabled: function () { + this.enabled = !this.enabled + } + + , toggle: function (e) { + var self = e ? $(e.currentTarget)[this.type](this._options).data(this.type) : this + self.tip().hasClass('in') ? self.hide() : self.show() + } + + , destroy: function () { + this.hide().$element.off('.' + this.type).removeData(this.type) + } + + } + + + /* TOOLTIP PLUGIN DEFINITION + * ========================= */ + + var old = $.fn.tooltip + + $.fn.tooltip = function ( option ) { + return this.each(function () { + var $this = $(this) + , data = $this.data('tooltip') + , options = typeof option == 'object' && option + if (!data) $this.data('tooltip', (data = new Tooltip(this, options))) + if (typeof option == 'string') data[option]() + }) + } + + $.fn.tooltip.Constructor = Tooltip + + $.fn.tooltip.defaults = { + animation: true + , placement: 'top' + , selector: false + , template: '
        ' + , trigger: 'hover focus' + , title: '' + , delay: 0 + , html: false + , container: false + } + + + /* TOOLTIP NO CONFLICT + * =================== */ + + $.fn.tooltip.noConflict = function () { + $.fn.tooltip = old + return this + } + +}(window.jQuery); + +/* =========================================================== + * bootstrap-popover.js v2.3.2 + * http://twitter.github.com/bootstrap/javascript.html#popovers + * =========================================================== + * Copyright 2012 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * =========================================================== */ + + +!function ($) { + + "use strict"; // jshint ;_; + + + /* POPOVER PUBLIC CLASS DEFINITION + * =============================== */ + + var Popover = function (element, options) { + this.init('popover', element, options) + } + + + /* NOTE: POPOVER EXTENDS BOOTSTRAP-TOOLTIP.js + ========================================== */ + + Popover.prototype = $.extend({}, $.fn.tooltip.Constructor.prototype, { + + constructor: Popover + + , setContent: function () { + var $tip = this.tip() + , title = this.getTitle() + , content = this.getContent() + + $tip.find('.popover-title')[this.options.html ? 'html' : 'text'](title) + $tip.find('.popover-content')[this.options.html ? 'html' : 'text'](content) + + $tip.removeClass('fade top bottom left right in') + } + + , hasContent: function () { + return this.getTitle() || this.getContent() + } + + , getContent: function () { + var content + , $e = this.$element + , o = this.options + + content = (typeof o.content == 'function' ? o.content.call($e[0]) : o.content) + || $e.attr('data-content') + + return content + } + + , tip: function () { + if (!this.$tip) { + this.$tip = $(this.options.template) + } + return this.$tip + } + + , destroy: function () { + this.hide().$element.off('.' + this.type).removeData(this.type) + } + + }) + + + /* POPOVER PLUGIN DEFINITION + * ======================= */ + + var old = $.fn.popover + + $.fn.popover = function (option) { + return this.each(function () { + var $this = $(this) + , data = $this.data('popover') + , options = typeof option == 'object' && option + if (!data) $this.data('popover', (data = new Popover(this, options))) + if (typeof option == 'string') data[option]() + }) + } + + $.fn.popover.Constructor = Popover + + $.fn.popover.defaults = $.extend({} , $.fn.tooltip.defaults, { + placement: 'right' + , trigger: 'click' + , content: '' + , template: '

        ' + }) + + + /* POPOVER NO CONFLICT + * =================== */ + + $.fn.popover.noConflict = function () { + $.fn.popover = old + return this + } + +}(window.jQuery); + +/* ========================================================== + * bootstrap-affix.js v2.3.2 + * http://twitter.github.com/bootstrap/javascript.html#affix + * ========================================================== + * Copyright 2012 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ========================================================== */ + + +!function ($) { + + "use strict"; // jshint ;_; + + + /* AFFIX CLASS DEFINITION + * ====================== */ + + var Affix = function (element, options) { + this.options = $.extend({}, $.fn.affix.defaults, options) + this.$window = $(window) + .on('scroll.affix.data-api', $.proxy(this.checkPosition, this)) + .on('click.affix.data-api', $.proxy(function () { setTimeout($.proxy(this.checkPosition, this), 1) }, this)) + this.$element = $(element) + this.checkPosition() + } + + Affix.prototype.checkPosition = function () { + if (!this.$element.is(':visible')) return + + var scrollHeight = $(document).height() + , scrollTop = this.$window.scrollTop() + , position = this.$element.offset() + , offset = this.options.offset + , offsetBottom = offset.bottom + , offsetTop = offset.top + , reset = 'affix affix-top affix-bottom' + , affix + + if (typeof offset != 'object') offsetBottom = offsetTop = offset + if (typeof offsetTop == 'function') offsetTop = offset.top() + if (typeof offsetBottom == 'function') offsetBottom = offset.bottom() + + affix = this.unpin != null && (scrollTop + this.unpin <= position.top) ? + false : offsetBottom != null && (position.top + this.$element.height() >= scrollHeight - offsetBottom) ? + 'bottom' : offsetTop != null && scrollTop <= offsetTop ? + 'top' : false + + if (this.affixed === affix) return + + this.affixed = affix + this.unpin = affix == 'bottom' ? position.top - scrollTop : null + + this.$element.removeClass(reset).addClass('affix' + (affix ? '-' + affix : '')) + } + + + /* AFFIX PLUGIN DEFINITION + * ======================= */ + + var old = $.fn.affix + + $.fn.affix = function (option) { + return this.each(function () { + var $this = $(this) + , data = $this.data('affix') + , options = typeof option == 'object' && option + if (!data) $this.data('affix', (data = new Affix(this, options))) + if (typeof option == 'string') data[option]() + }) + } + + $.fn.affix.Constructor = Affix + + $.fn.affix.defaults = { + offset: 0 + } + + + /* AFFIX NO CONFLICT + * ================= */ + + $.fn.affix.noConflict = function () { + $.fn.affix = old + return this + } + + + /* AFFIX DATA-API + * ============== */ + + $(window).on('load', function () { + $('[data-spy="affix"]').each(function () { + var $spy = $(this) + , data = $spy.data() + + data.offset = data.offset || {} + + data.offsetBottom && (data.offset.bottom = data.offsetBottom) + data.offsetTop && (data.offset.top = data.offsetTop) + + $spy.affix(data) + }) + }) + + +}(window.jQuery); +/* ========================================================== + * bootstrap-alert.js v2.3.2 + * http://twitter.github.com/bootstrap/javascript.html#alerts + * ========================================================== + * Copyright 2012 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ========================================================== */ + + +!function ($) { + + "use strict"; // jshint ;_; + + + /* ALERT CLASS DEFINITION + * ====================== */ + + var dismiss = '[data-dismiss="alert"]' + , Alert = function (el) { + $(el).on('click', dismiss, this.close) + } + + Alert.prototype.close = function (e) { + var $this = $(this) + , selector = $this.attr('data-target') + , $parent + + if (!selector) { + selector = $this.attr('href') + selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') //strip for ie7 + } + + $parent = $(selector) + + e && e.preventDefault() + + $parent.length || ($parent = $this.hasClass('alert') ? $this : $this.parent()) + + $parent.trigger(e = $.Event('close')) + + if (e.isDefaultPrevented()) return + + $parent.removeClass('in') + + function removeElement() { + $parent + .trigger('closed') + .remove() + } + + $.support.transition && $parent.hasClass('fade') ? + $parent.on($.support.transition.end, removeElement) : + removeElement() + } + + + /* ALERT PLUGIN DEFINITION + * ======================= */ + + var old = $.fn.alert + + $.fn.alert = function (option) { + return this.each(function () { + var $this = $(this) + , data = $this.data('alert') + if (!data) $this.data('alert', (data = new Alert(this))) + if (typeof option == 'string') data[option].call($this) + }) + } + + $.fn.alert.Constructor = Alert + + + /* ALERT NO CONFLICT + * ================= */ + + $.fn.alert.noConflict = function () { + $.fn.alert = old + return this + } + + + /* ALERT DATA-API + * ============== */ + + $(document).on('click.alert.data-api', dismiss, Alert.prototype.close) + +}(window.jQuery); +/* ============================================================ + * bootstrap-button.js v2.3.2 + * http://twitter.github.com/bootstrap/javascript.html#buttons + * ============================================================ + * Copyright 2012 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================ */ + + +!function ($) { + + "use strict"; // jshint ;_; + + + /* BUTTON PUBLIC CLASS DEFINITION + * ============================== */ + + var Button = function (element, options) { + this.$element = $(element) + this.options = $.extend({}, $.fn.button.defaults, options) + } + + Button.prototype.setState = function (state) { + var d = 'disabled' + , $el = this.$element + , data = $el.data() + , val = $el.is('input') ? 'val' : 'html' + + state = state + 'Text' + data.resetText || $el.data('resetText', $el[val]()) + + $el[val](data[state] || this.options[state]) + + // push to event loop to allow forms to submit + setTimeout(function () { + state == 'loadingText' ? + $el.addClass(d).attr(d, d) : + $el.removeClass(d).removeAttr(d) + }, 0) + } + + Button.prototype.toggle = function () { + var $parent = this.$element.closest('[data-toggle="buttons-radio"]') + + $parent && $parent + .find('.active') + .removeClass('active') + + this.$element.toggleClass('active') + } + + + /* BUTTON PLUGIN DEFINITION + * ======================== */ + + var old = $.fn.button + + $.fn.button = function (option) { + return this.each(function () { + var $this = $(this) + , data = $this.data('button') + , options = typeof option == 'object' && option + if (!data) $this.data('button', (data = new Button(this, options))) + if (option == 'toggle') data.toggle() + else if (option) data.setState(option) + }) + } + + $.fn.button.defaults = { + loadingText: 'loading...' + } + + $.fn.button.Constructor = Button + + + /* BUTTON NO CONFLICT + * ================== */ + + $.fn.button.noConflict = function () { + $.fn.button = old + return this + } + + + /* BUTTON DATA-API + * =============== */ + + $(document).on('click.button.data-api', '[data-toggle^=button]', function (e) { + var $btn = $(e.target) + if (!$btn.hasClass('btn')) $btn = $btn.closest('.btn') + $btn.button('toggle') + }) + +}(window.jQuery); +/* ============================================================= + * bootstrap-collapse.js v2.3.2 + * http://twitter.github.com/bootstrap/javascript.html#collapse + * ============================================================= + * Copyright 2012 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================ */ + + +!function ($) { + + "use strict"; // jshint ;_; + + + /* COLLAPSE PUBLIC CLASS DEFINITION + * ================================ */ + + var Collapse = function (element, options) { + this.$element = $(element) + this.options = $.extend({}, $.fn.collapse.defaults, options) + + if (this.options.parent) { + this.$parent = $(this.options.parent) + } + + this.options.toggle && this.toggle() + } + + Collapse.prototype = { + + constructor: Collapse + + , dimension: function () { + var hasWidth = this.$element.hasClass('width') + return hasWidth ? 'width' : 'height' + } + + , show: function () { + var dimension + , scroll + , actives + , hasData + + if (this.transitioning || this.$element.hasClass('in')) return + + dimension = this.dimension() + scroll = $.camelCase(['scroll', dimension].join('-')) + actives = this.$parent && this.$parent.find('> .accordion-group > .in') + + if (actives && actives.length) { + hasData = actives.data('collapse') + if (hasData && hasData.transitioning) return + actives.collapse('hide') + hasData || actives.data('collapse', null) + } + + this.$element[dimension](0) + this.transition('addClass', $.Event('show'), 'shown') + $.support.transition && this.$element[dimension](this.$element[0][scroll]) + } + + , hide: function () { + var dimension + if (this.transitioning || !this.$element.hasClass('in')) return + dimension = this.dimension() + this.reset(this.$element[dimension]()) + this.transition('removeClass', $.Event('hide'), 'hidden') + this.$element[dimension](0) + } + + , reset: function (size) { + var dimension = this.dimension() + + this.$element + .removeClass('collapse') + [dimension](size || 'auto') + [0].offsetWidth + + this.$element[size !== null ? 'addClass' : 'removeClass']('collapse') + + return this + } + + , transition: function (method, startEvent, completeEvent) { + var that = this + , complete = function () { + if (startEvent.type == 'show') that.reset() + that.transitioning = 0 + that.$element.trigger(completeEvent) + } + + this.$element.trigger(startEvent) + + if (startEvent.isDefaultPrevented()) return + + this.transitioning = 1 + + this.$element[method]('in') + + $.support.transition && this.$element.hasClass('collapse') ? + this.$element.one($.support.transition.end, complete) : + complete() + } + + , toggle: function () { + this[this.$element.hasClass('in') ? 'hide' : 'show']() + } + + } + + + /* COLLAPSE PLUGIN DEFINITION + * ========================== */ + + var old = $.fn.collapse + + $.fn.collapse = function (option) { + return this.each(function () { + var $this = $(this) + , data = $this.data('collapse') + , options = $.extend({}, $.fn.collapse.defaults, $this.data(), typeof option == 'object' && option) + if (!data) $this.data('collapse', (data = new Collapse(this, options))) + if (typeof option == 'string') data[option]() + }) + } + + $.fn.collapse.defaults = { + toggle: true + } + + $.fn.collapse.Constructor = Collapse + + + /* COLLAPSE NO CONFLICT + * ==================== */ + + $.fn.collapse.noConflict = function () { + $.fn.collapse = old + return this + } + + + /* COLLAPSE DATA-API + * ================= */ + + $(document).on('click.collapse.data-api', '[data-toggle=collapse]', function (e) { + var $this = $(this), href + , target = $this.attr('data-target') + || e.preventDefault() + || (href = $this.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '') //strip for ie7 + , option = $(target).data('collapse') ? 'toggle' : $this.data() + $this[$(target).hasClass('in') ? 'addClass' : 'removeClass']('collapsed') + $(target).collapse(option) + }) + +}(window.jQuery); +/* ========================================================== + * bootstrap-carousel.js v2.3.2 + * http://twitter.github.com/bootstrap/javascript.html#carousel + * ========================================================== + * Copyright 2012 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ========================================================== */ + + +!function ($) { + + "use strict"; // jshint ;_; + + + /* CAROUSEL CLASS DEFINITION + * ========================= */ + + var Carousel = function (element, options) { + this.$element = $(element) + this.$indicators = this.$element.find('.carousel-indicators') + this.options = options + this.options.pause == 'hover' && this.$element + .on('mouseenter', $.proxy(this.pause, this)) + .on('mouseleave', $.proxy(this.cycle, this)) + } + + Carousel.prototype = { + + cycle: function (e) { + if (!e) this.paused = false + if (this.interval) clearInterval(this.interval); + this.options.interval + && !this.paused + && (this.interval = setInterval($.proxy(this.next, this), this.options.interval)) + return this + } + + , getActiveIndex: function () { + this.$active = this.$element.find('.item.active') + this.$items = this.$active.parent().children() + return this.$items.index(this.$active) + } + + , to: function (pos) { + var activeIndex = this.getActiveIndex() + , that = this + + if (pos > (this.$items.length - 1) || pos < 0) return + + if (this.sliding) { + return this.$element.one('slid', function () { + that.to(pos) + }) + } + + if (activeIndex == pos) { + return this.pause().cycle() + } + + return this.slide(pos > activeIndex ? 'next' : 'prev', $(this.$items[pos])) + } + + , pause: function (e) { + if (!e) this.paused = true + if (this.$element.find('.next, .prev').length && $.support.transition.end) { + this.$element.trigger($.support.transition.end) + this.cycle(true) + } + clearInterval(this.interval) + this.interval = null + return this + } + + , next: function () { + if (this.sliding) return + return this.slide('next') + } + + , prev: function () { + if (this.sliding) return + return this.slide('prev') + } + + , slide: function (type, next) { + var $active = this.$element.find('.item.active') + , $next = next || $active[type]() + , isCycling = this.interval + , direction = type == 'next' ? 'left' : 'right' + , fallback = type == 'next' ? 'first' : 'last' + , that = this + , e + + this.sliding = true + + isCycling && this.pause() + + $next = $next.length ? $next : this.$element.find('.item')[fallback]() + + e = $.Event('slide', { + relatedTarget: $next[0] + , direction: direction + }) + + if ($next.hasClass('active')) return + + if (this.$indicators.length) { + this.$indicators.find('.active').removeClass('active') + this.$element.one('slid', function () { + var $nextIndicator = $(that.$indicators.children()[that.getActiveIndex()]) + $nextIndicator && $nextIndicator.addClass('active') + }) + } + + if ($.support.transition && this.$element.hasClass('slide')) { + this.$element.trigger(e) + if (e.isDefaultPrevented()) return + $next.addClass(type) + $next[0].offsetWidth // force reflow + $active.addClass(direction) + $next.addClass(direction) + this.$element.one($.support.transition.end, function () { + $next.removeClass([type, direction].join(' ')).addClass('active') + $active.removeClass(['active', direction].join(' ')) + that.sliding = false + setTimeout(function () { that.$element.trigger('slid') }, 0) + }) + } else { + this.$element.trigger(e) + if (e.isDefaultPrevented()) return + $active.removeClass('active') + $next.addClass('active') + this.sliding = false + this.$element.trigger('slid') + } + + isCycling && this.cycle() + + return this + } + + } + + + /* CAROUSEL PLUGIN DEFINITION + * ========================== */ + + var old = $.fn.carousel + + $.fn.carousel = function (option) { + return this.each(function () { + var $this = $(this) + , data = $this.data('carousel') + , options = $.extend({}, $.fn.carousel.defaults, typeof option == 'object' && option) + , action = typeof option == 'string' ? option : options.slide + if (!data) $this.data('carousel', (data = new Carousel(this, options))) + if (typeof option == 'number') data.to(option) + else if (action) data[action]() + else if (options.interval) data.pause().cycle() + }) + } + + $.fn.carousel.defaults = { + interval: 5000 + , pause: 'hover' + } + + $.fn.carousel.Constructor = Carousel + + + /* CAROUSEL NO CONFLICT + * ==================== */ + + $.fn.carousel.noConflict = function () { + $.fn.carousel = old + return this + } + + /* CAROUSEL DATA-API + * ================= */ + + $(document).on('click.carousel.data-api', '[data-slide], [data-slide-to]', function (e) { + var $this = $(this), href + , $target = $($this.attr('data-target') || (href = $this.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '')) //strip for ie7 + , options = $.extend({}, $target.data(), $this.data()) + , slideIndex + + $target.carousel(options) + + if (slideIndex = $this.attr('data-slide-to')) { + $target.data('carousel').pause().to(slideIndex).cycle() + } + + e.preventDefault() + }) + +}(window.jQuery); +/* ============================================================= + * bootstrap-typeahead.js v2.3.2 + * http://twitter.github.com/bootstrap/javascript.html#typeahead + * ============================================================= + * Copyright 2012 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================ */ + + +!function($){ + + "use strict"; // jshint ;_; + + + /* TYPEAHEAD PUBLIC CLASS DEFINITION + * ================================= */ + + var Typeahead = function (element, options) { + this.$element = $(element) + this.options = $.extend({}, $.fn.typeahead.defaults, options) + this.matcher = this.options.matcher || this.matcher + this.sorter = this.options.sorter || this.sorter + this.highlighter = this.options.highlighter || this.highlighter + this.updater = this.options.updater || this.updater + this.source = this.options.source + this.$menu = $(this.options.menu) + this.shown = false + this.listen() + } + + Typeahead.prototype = { + + constructor: Typeahead + + , select: function () { + var val = this.$menu.find('.active').attr('data-value') + this.$element + .val(this.updater(val)) + .change() + return this.hide() + } + + , updater: function (item) { + return item + } + + , show: function () { + var pos = $.extend({}, this.$element.position(), { + height: this.$element[0].offsetHeight + }) + + this.$menu + .insertAfter(this.$element) + .css({ + top: pos.top + pos.height + , left: pos.left + }) + .show() + + this.shown = true + return this + } + + , hide: function () { + this.$menu.hide() + this.shown = false + return this + } + + , lookup: function (event) { + var items + + this.query = this.$element.val() + + if (!this.query || this.query.length < this.options.minLength) { + return this.shown ? this.hide() : this + } + + items = $.isFunction(this.source) ? this.source(this.query, $.proxy(this.process, this)) : this.source + + return items ? this.process(items) : this + } + + , process: function (items) { + var that = this + + items = $.grep(items, function (item) { + return that.matcher(item) + }) + + items = this.sorter(items) + + if (!items.length) { + return this.shown ? this.hide() : this + } + + return this.render(items.slice(0, this.options.items)).show() + } + + , matcher: function (item) { + return ~item.toLowerCase().indexOf(this.query.toLowerCase()) + } + + , sorter: function (items) { + var beginswith = [] + , caseSensitive = [] + , caseInsensitive = [] + , item + + while (item = items.shift()) { + if (!item.toLowerCase().indexOf(this.query.toLowerCase())) beginswith.push(item) + else if (~item.indexOf(this.query)) caseSensitive.push(item) + else caseInsensitive.push(item) + } + + return beginswith.concat(caseSensitive, caseInsensitive) + } + + , highlighter: function (item) { + var query = this.query.replace(/[\-\[\]{}()*+?.,\\\^$|#\s]/g, '\\$&') + return item.replace(new RegExp('(' + query + ')', 'ig'), function ($1, match) { + return '' + match + '' + }) + } + + , render: function (items) { + var that = this + + items = $(items).map(function (i, item) { + i = $(that.options.item).attr('data-value', item) + i.find('a').html(that.highlighter(item)) + return i[0] + }) + + items.first().addClass('active') + this.$menu.html(items) + return this + } + + , next: function (event) { + var active = this.$menu.find('.active').removeClass('active') + , next = active.next() + + if (!next.length) { + next = $(this.$menu.find('li')[0]) + } + + next.addClass('active') + } + + , prev: function (event) { + var active = this.$menu.find('.active').removeClass('active') + , prev = active.prev() + + if (!prev.length) { + prev = this.$menu.find('li').last() + } + + prev.addClass('active') + } + + , listen: function () { + this.$element + .on('focus', $.proxy(this.focus, this)) + .on('blur', $.proxy(this.blur, this)) + .on('keypress', $.proxy(this.keypress, this)) + .on('keyup', $.proxy(this.keyup, this)) + + if (this.eventSupported('keydown')) { + this.$element.on('keydown', $.proxy(this.keydown, this)) + } + + this.$menu + .on('click', $.proxy(this.click, this)) + .on('mouseenter', 'li', $.proxy(this.mouseenter, this)) + .on('mouseleave', 'li', $.proxy(this.mouseleave, this)) + } + + , eventSupported: function(eventName) { + var isSupported = eventName in this.$element + if (!isSupported) { + this.$element.setAttribute(eventName, 'return;') + isSupported = typeof this.$element[eventName] === 'function' + } + return isSupported + } + + , move: function (e) { + if (!this.shown) return + + switch(e.keyCode) { + case 9: // tab + case 13: // enter + case 27: // escape + e.preventDefault() + break + + case 38: // up arrow + e.preventDefault() + this.prev() + break + + case 40: // down arrow + e.preventDefault() + this.next() + break + } + + e.stopPropagation() + } + + , keydown: function (e) { + this.suppressKeyPressRepeat = ~$.inArray(e.keyCode, [40,38,9,13,27]) + this.move(e) + } + + , keypress: function (e) { + if (this.suppressKeyPressRepeat) return + this.move(e) + } + + , keyup: function (e) { + switch(e.keyCode) { + case 40: // down arrow + case 38: // up arrow + case 16: // shift + case 17: // ctrl + case 18: // alt + break + + case 9: // tab + case 13: // enter + if (!this.shown) return + this.select() + break + + case 27: // escape + if (!this.shown) return + this.hide() + break + + default: + this.lookup() + } + + e.stopPropagation() + e.preventDefault() + } + + , focus: function (e) { + this.focused = true + } + + , blur: function (e) { + this.focused = false + if (!this.mousedover && this.shown) this.hide() + } + + , click: function (e) { + e.stopPropagation() + e.preventDefault() + this.select() + this.$element.focus() + } + + , mouseenter: function (e) { + this.mousedover = true + this.$menu.find('.active').removeClass('active') + $(e.currentTarget).addClass('active') + } + + , mouseleave: function (e) { + this.mousedover = false + if (!this.focused && this.shown) this.hide() + } + + } + + + /* TYPEAHEAD PLUGIN DEFINITION + * =========================== */ + + var old = $.fn.typeahead + + $.fn.typeahead = function (option) { + return this.each(function () { + var $this = $(this) + , data = $this.data('typeahead') + , options = typeof option == 'object' && option + if (!data) $this.data('typeahead', (data = new Typeahead(this, options))) + if (typeof option == 'string') data[option]() + }) + } + + $.fn.typeahead.defaults = { + source: [] + , items: 8 + , menu: '
          ' + , item: '
          • ' + , minLength: 1 + } + + $.fn.typeahead.Constructor = Typeahead + + + /* TYPEAHEAD NO CONFLICT + * =================== */ + + $.fn.typeahead.noConflict = function () { + $.fn.typeahead = old + return this + } + + + /* TYPEAHEAD DATA-API + * ================== */ + + $(document).on('focus.typeahead.data-api', '[data-provide="typeahead"]', function (e) { + var $this = $(this) + if ($this.data('typeahead')) return + $this.typeahead($this.data()) + }) + +}(window.jQuery); diff --git a/pyload/web/app/scripts/vendor/jquery.omniwindow.js b/pyload/web/app/scripts/vendor/jquery.omniwindow.js new file mode 100644 index 000000000..e1f0b8f77 --- /dev/null +++ b/pyload/web/app/scripts/vendor/jquery.omniwindow.js @@ -0,0 +1,141 @@ +// jQuery OmniWindow plugin +// @version: 0.7.0 +// @author: Rudenka Alexander (mur.mailbox@gmail.com) +// @license: MIT + +;(function($) { + "use strict"; + $.fn.extend({ + omniWindow: function(options) { + + options = $.extend(true, { + animationsPriority: { + show: ['overlay', 'modal'], + hide: ['modal', 'overlay'] + }, + overlay: { + selector: '.ow-overlay', + hideClass: 'ow-closed', + animations: { + show: function(subjects, internalCallback) { return internalCallback(subjects); }, + hide: function(subjects, internalCallback) { return internalCallback(subjects); }, + internal: { + show: function(subjects){ subjects.overlay.removeClass(options.overlay.hideClass); }, + hide: function(subjects){ subjects.overlay.addClass(options.overlay.hideClass); } + } + } + }, + modal: { + hideClass: 'ow-closed', + animations: { + show: function(subjects, internalCallback) { return internalCallback(subjects); }, + hide: function(subjects, internalCallback) { return internalCallback(subjects); }, + internal: { + show: function(subjects){ subjects.modal.removeClass(options.modal.hideClass); }, + hide: function(subjects){ subjects.modal.addClass(options.modal.hideClass); } + } + }, + internal: { + stateAttribute: 'ow-active' + } + }, + eventsNames: { + show: 'show.ow', + hide: 'hide.ow', + internal: { + overlayClick: 'click.ow', + keyboardKeyUp: 'keyup.ow' + } + }, + callbacks: { // Callbacks execution chain + beforeShow: function(subjects, internalCallback) { return internalCallback(subjects); }, // 1 (stop if retruns false) + positioning: function(subjects, internalCallback) { return internalCallback(subjects); }, // 2 + afterShow: function(subjects, internalCallback) { return internalCallback(subjects); }, // 3 + beforeHide: function(subjects, internalCallback) { return internalCallback(subjects); }, // 4 (stop if retruns false) + afterHide: function(subjects, internalCallback) { return internalCallback(subjects); }, // 5 + internal: { + beforeShow: function(subjects) { + if (subjects.modal.data(options.modal.internal.stateAttribute)) { + return false; + } else { + subjects.modal.data(options.modal.internal.stateAttribute, true); + return true; + } + }, + afterShow: function(subjects) { + $(document).on(options.eventsNames.internal.keyboardKeyUp, function(e) { + if (e.keyCode === 27) { // if the key pressed is the ESC key + subjects.modal.trigger(options.eventsNames.hide); + } + }); + + subjects.overlay.on(options.eventsNames.internal.overlayClick, function(){ + subjects.modal.trigger(options.eventsNames.hide); + }); + }, + positioning: function(subjects) { + subjects.modal.css('margin-left', Math.round(subjects.modal.outerWidth() / -2)); + }, + beforeHide: function(subjects) { + if (subjects.modal.data(options.modal.internal.stateAttribute)) { + subjects.modal.data(options.modal.internal.stateAttribute, false); + return true; + } else { + return false; + } + }, + afterHide: function(subjects) { + subjects.overlay.off(options.eventsNames.internal.overlayClick); + $(document).off(options.eventsNames.internal.keyboardKeyUp); + + subjects.overlay.css('display', ''); // clear inline styles after jQ animations + subjects.modal.css('display', ''); + } + } + } + }, options); + + var animate = function(process, subjects, callbackName) { + var first = options.animationsPriority[process][0], + second = options.animationsPriority[process][1]; + + options[first].animations[process](subjects, function(subjs) { // call USER's FIRST animation (depends on priority) + options[first].animations.internal[process](subjs); // call internal FIRST animation + + options[second].animations[process](subjects, function(subjs) { // call USER's SECOND animation + options[second].animations.internal[process](subjs); // call internal SECOND animation + + // then we need to call USER's + // afterShow of afterHide callback + options.callbacks[callbackName](subjects, options.callbacks.internal[callbackName]); + }); + }); + }; + + var showModal = function(subjects) { + if (!options.callbacks.beforeShow(subjects, options.callbacks.internal.beforeShow)) { return; } // cancel showing if beforeShow callback return false + + options.callbacks.positioning(subjects, options.callbacks.internal.positioning); + + animate('show', subjects, 'afterShow'); + }; + + var hideModal = function(subjects) { + if (!options.callbacks.beforeHide(subjects, options.callbacks.internal.beforeHide)) { return; } // cancel hiding if beforeHide callback return false + + animate('hide', subjects, 'afterHide'); + }; + + + var $overlay = $(options.overlay.selector); + + return this.each(function() { + var $modal = $(this); + var subjects = {modal: $modal, overlay: $overlay}; + + $modal.bind(options.eventsNames.show, function(){ showModal(subjects); }) + .bind(options.eventsNames.hide, function(){ hideModal(subjects); }); + }); + } + }); +})(jQuery); \ No newline at end of file diff --git a/pyload/web/app/scripts/vendor/remaining.js b/pyload/web/app/scripts/vendor/remaining.js new file mode 100644 index 000000000..d66a2931a --- /dev/null +++ b/pyload/web/app/scripts/vendor/remaining.js @@ -0,0 +1,149 @@ +/** + * Javascript Countdown + * Copyright (c) 2009 Markus Hedlund + * Version 1.1 + * Licensed under MIT license + * http://www.opensource.org/licenses/mit-license.php + * http://labs.mimmin.com/countdown + */ +define([], function() { + var remaining = { + /** + * Get the difference of the passed date, and now. The different formats of the taget parameter are: + * January 12, 2009 15:14:00 (Month dd, yyyy hh:mm:ss) + * January 12, 2009 (Month dd, yyyy) + * 09,00,12,15,14,00 (yy,mm,dd,hh,mm,ss) Months range from 0-11, not 1-12. + * 09,00,12 (yy,mm,dd) Months range from 0-11, not 1-12. + * 500 (milliseconds) + * 2009-01-12 15:14:00 (yyyy-mm-dd hh-mm-ss) + * 2009-01-12 15:14 (yyyy-mm-dd hh-mm) + * @param target Target date. Can be either a date object or a string (formated like '24 December, 2010 15:00:00') + * @return Difference in seconds + */ + getSeconds: function(target) { + var today = new Date(); + + if (typeof(target) == 'object') { + var targetDate = target; + } else { + var matches = target.match(/(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2})(:(\d{2}))?/); // YYYY-MM-DD HH-MM-SS + if (matches != null) { + matches[7] = typeof(matches[7]) == 'undefined' ? '00' : matches[7]; + var targetDate = new Date(matches[1], matches[2] - 1, matches[3], matches[4], matches[5], matches[7]); + } else { + var targetDate = new Date(target); + } + } + + return Math.floor((targetDate.getTime() - today.getTime()) / 1000); + }, + + /** + * @param seconds Difference in seconds + * @param i18n A language object (see code) + * @param onlyLargestUnit Return only the largest unit (see documentation) + * @param hideEmpty Hide empty units (see documentation) + * @return String formated something like '1 week, 1 hours, 1 second' + */ + getString: function(seconds, i18n, onlyLargestUnit, hideEmpty) { + if (seconds < 1) { + return ''; + } + + if (typeof(hideEmpty) == 'undefined' || hideEmpty == null) { + hideEmpty = true; + } + if (typeof(onlyLargestUnit) == 'undefined' || onlyLargestUnit == null) { + onlyLargestUnit = false; + } + if (typeof(i18n) == 'undefined' || i18n == null) { + i18n = { + weeks: ['week', 'weeks'], + days: ['day', 'days'], + hours: ['hour', 'hours'], + minutes: ['minute', 'minutes'], + seconds: ['second', 'seconds'] + }; + } + + var units = { + weeks: 7 * 24 * 60 * 60, + days: 24 * 60 * 60, + hours: 60 * 60, + minutes: 60, + seconds: 1 + }; + + var returnArray = []; + var value; + for (unit in units) { + value = units[unit]; + if (seconds / value >= 1 || unit == 'seconds' || !hideEmpty) { + secondsConverted = Math.floor(seconds / value); + var i18nUnit = i18n[unit][secondsConverted == 1 ? 0 : 1]; + returnArray.push(secondsConverted + ' ' + i18nUnit); + seconds -= secondsConverted * value; + + if (onlyLargestUnit) { + break; + } + } + } + ; + + return returnArray.join(', '); + }, + + /** + * @param seconds Difference in seconds + * @return String formated something like '169:00:01' + */ + getStringDigital: function(seconds) { + if (seconds < 1) { + return ''; + } + + remainingTime = remaining.getArray(seconds); + + for (index in remainingTime) { + remainingTime[index] = remaining.padNumber(remainingTime[index]); + } + ; + + return remainingTime.join(':'); + }, + + /** + * @param seconds Difference in seconds + * @return Array with hours, minutes and seconds + */ + getArray: function(seconds) { + if (seconds < 1) { + return []; + } + + var units = [60 * 60, 60, 1]; + + var returnArray = []; + var value; + for (index in units) { + value = units[index]; + secondsConverted = Math.floor(seconds / value); + returnArray.push(secondsConverted); + seconds -= secondsConverted * value; + } + ; + + return returnArray; + }, + + /** + * @param number An integer + * @return Integer padded with a 0 if necessary + */ + padNumber: function(number) { + return (number >= 0 && number < 10) ? '0' + number : number; + } + }; + return remaining; +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/abstract/itemView.js b/pyload/web/app/scripts/views/abstract/itemView.js new file mode 100644 index 000000000..c37118a4c --- /dev/null +++ b/pyload/web/app/scripts/views/abstract/itemView.js @@ -0,0 +1,47 @@ +define(['jquery', 'backbone', 'underscore'], function($, Backbone, _) { + 'use strict'; + + // A view that is meant for temporary displaying + // All events must be unbound in onDestroy + return Backbone.View.extend({ + + tagName: 'li', + destroy: function() { + this.undelegateEvents(); + this.unbind(); + if (this.onDestroy) { + this.onDestroy(); + } + this.$el.removeData().unbind(); + this.remove(); + }, + + hide: function() { + this.$el.slideUp(); + }, + + show: function() { + this.$el.slideDown(); + }, + + unrender: function() { + var self = this; + this.$el.slideUp(function() { + self.destroy(); + }); + }, + + deleteItem: function(e) { + if (e) + e.stopPropagation(); + this.model.destroy(); + }, + + restart: function(e) { + if(e) + e.stopPropagation(); + this.model.restart(); + } + + }); +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/abstract/modalView.js b/pyload/web/app/scripts/views/abstract/modalView.js new file mode 100644 index 000000000..9d1d72869 --- /dev/null +++ b/pyload/web/app/scripts/views/abstract/modalView.js @@ -0,0 +1,125 @@ +define(['jquery', 'backbone', 'underscore', 'omniwindow'], function($, Backbone, _) { + 'use strict'; + + return Backbone.View.extend({ + + events: { + 'click .btn-confirm': 'confirm', + 'click .btn-close': 'hide', + 'click .close': 'hide' + }, + + template: null, + dialog: null, + + onHideDestroy: false, + confirmCallback: null, + + initialize: function(template, confirm) { + this.confirmCallback = confirm; + var self = this; + if (this.template === null) { + if (template) { + this.template = template; + // When template was provided this is a temporary dialog + this.onHideDestroy = true; + } + else + require(['text!tpl/default/modal.html'], function(template) { + self.template = template; + }); + } + + }, + + // TODO: whole modal stuff is not very elegant + render: function() { + this.$el.html(this.template(this.renderContent())); + this.onRender(); + + if (this.dialog === null) { + this.$el.addClass('modal hide'); + this.$el.css({opacity: 0, scale: 0.7}); + + var self = this; + $('body').append(this.el); + this.dialog = this.$el.omniWindow({ + overlay: { + selector: '#modal-overlay', + hideClass: 'hide', + animations: { + hide: function(subjects, internalCallback) { + subjects.overlay.transition({opacity: 'hide', delay: 100}, 300, function() { + internalCallback(subjects); + self.onHide(); + if (self.onHideDestroy) + self.destroy(); + }); + }, + show: function(subjects, internalCallback) { + subjects.overlay.fadeIn(300); + internalCallback(subjects); + }}}, + modal: { + hideClass: 'hide', + animations: { + hide: function(subjects, internalCallback) { + subjects.modal.transition({opacity: 'hide', scale: 0.7}, 300); + internalCallback(subjects); + }, + + show: function(subjects, internalCallback) { + subjects.modal.transition({opacity: 'show', scale: 1, delay: 100}, 300, function() { + internalCallback(subjects); + }); + }} + }}); + } + + return this; + }, + + onRender: function() { + + }, + + renderContent: function() { + return {}; + }, + + show: function() { + if (this.dialog === null) + this.render(); + + this.dialog.trigger('show'); + + this.onShow(); + }, + + onShow: function() { + + }, + + hide: function() { + this.dialog.trigger('hide'); + }, + + onHide: function() { + + }, + + confirm: function() { + if (this.confirmCallback) + this.confirmCallback.apply(); + + this.hide(); + }, + + destroy: function() { + this.$el.remove(); + this.dialog = null; + this.remove(); + } + + }); +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/accounts/accountListView.js b/pyload/web/app/scripts/views/accounts/accountListView.js new file mode 100644 index 000000000..4eb5bfe7d --- /dev/null +++ b/pyload/web/app/scripts/views/accounts/accountListView.js @@ -0,0 +1,52 @@ +define(['jquery', 'underscore', 'backbone', 'app', 'collections/AccountList', './accountView', + 'hbs!tpl/accounts/layout', 'hbs!tpl/accounts/actionbar'], + function($, _, Backbone, App, AccountList, accountView, template, templateBar) { + 'use strict'; + + // Renders settings over view page + return Backbone.Marionette.CollectionView.extend({ + + itemView: accountView, + template: template, + + collection: null, + modal: null, + + initialize: function() { + this.actionbar = Backbone.Marionette.ItemView.extend({ + template: templateBar, + events: { + 'click .btn': 'addAccount' + }, + addAccount: _.bind(this.addAccount, this) + }); + + this.collection = new AccountList(); + this.update(); + + this.listenTo(App.vent, 'accounts:updated', this.update); + }, + + update: function() { + this.collection.fetch(); + }, + + onBeforeRender: function() { + this.$el.html(template()); + }, + + appendHtml: function(collectionView, itemView, index) { + this.$('.account-list').append(itemView.el); + }, + + addAccount: function() { + var self = this; + _.requireOnce(['views/accounts/accountModal'], function(Modal) { + if (self.modal === null) + self.modal = new Modal(); + + self.modal.show(); + }); + } + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/accounts/accountModal.js b/pyload/web/app/scripts/views/accounts/accountModal.js new file mode 100644 index 000000000..6c2b226df --- /dev/null +++ b/pyload/web/app/scripts/views/accounts/accountModal.js @@ -0,0 +1,72 @@ +define(['jquery', 'underscore', 'app', 'views/abstract/modalView', 'hbs!tpl/dialogs/addAccount', 'helpers/pluginIcon', 'select2'], + function($, _, App, modalView, template, pluginIcon) { + 'use strict'; + return modalView.extend({ + + events: { + 'submit form': 'add', + 'click .btn-add': 'add' + }, + template: template, + plugins: null, + select: null, + + initialize: function() { + // Inherit parent events + this.events = _.extend({}, modalView.prototype.events, this.events); + var self = this; + $.ajax(App.apiRequest('getAccountTypes', null, {success: function(data) { + self.plugins = _.sortBy(data, function(item) { + return item; + }); + self.render(); + }})); + }, + + onRender: function() { + // TODO: could be a separate input type if needed on multiple pages + if (this.plugins) + this.select = this.$('#pluginSelect').select2({ + escapeMarkup: function(m) { + return m; + }, + formatResult: this.format, + formatSelection: this.format, + data: {results: this.plugins, text: function(item) { + return item; + }}, + id: function(item) { + return item; + } + }); + }, + + onShow: function() { + }, + + onHide: function() { + }, + + format: function(data) { + return ' ' + data; + }, + + add: function(e) { + e.stopPropagation(); + if (this.select) { + var plugin = this.select.val(), + login = this.$('#login').val(), + password = this.$('#password').val(), + self = this; + + $.ajax(App.apiRequest('updateAccount', { + plugin: plugin, login: login, password: password + }, { success: function() { + App.vent.trigger('accounts:updated'); + self.hide(); + }})); + } + return false; + } + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/accounts/accountView.js b/pyload/web/app/scripts/views/accounts/accountView.js new file mode 100644 index 000000000..89f69d7e7 --- /dev/null +++ b/pyload/web/app/scripts/views/accounts/accountView.js @@ -0,0 +1,18 @@ +define(['jquery', 'underscore', 'backbone', 'app', 'hbs!tpl/accounts/account'], + function($, _, Backbone, App, template) { + 'use strict'; + + return Backbone.Marionette.ItemView.extend({ + + tagName: 'tr', + template: template, + + events: { + 'click .btn-danger': 'deleteAccount' + }, + + deleteAccount: function() { + this.model.destroy(); + } + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/dashboard/dashboardView.js b/pyload/web/app/scripts/views/dashboard/dashboardView.js new file mode 100644 index 000000000..e7adba475 --- /dev/null +++ b/pyload/web/app/scripts/views/dashboard/dashboardView.js @@ -0,0 +1,168 @@ +define(['jquery', 'backbone', 'underscore', 'app', 'models/TreeCollection', + './packageView', './fileView', 'hbs!tpl/dashboard/layout', 'select2'], + function($, Backbone, _, App, TreeCollection, PackageView, FileView, template) { + 'use strict'; + // Renders whole dashboard + return Backbone.Marionette.ItemView.extend({ + + // TODO: refactor + active: $('.breadcrumb .active'), + + template: template, + + events: { + }, + + ui: { + 'packages': '.package-list', + 'files': '.file-list' + }, + + // Package tree + tree: null, + // Current open files + files: null, + // True when loading animation is running + isLoading: false, + + initialize: function() { + App.dashboard = this; + this.tree = new TreeCollection(); + + var self = this; + // When package is added we reload the data + App.vent.on('package:added', function() { + console.log('Package tree caught, package:added event'); + self.tree.fetch(); + }); + + App.vent.on('file:updated', _.bind(this.fileUpdated, this)); + + // TODO: merge? + this.init(); + // TODO: file:added + // TODO: package:deleted + // TODO: package:updated + }, + + init: function() { + var self = this; + // TODO: put in separated function + // TODO: order of elements? + // Init the tree and callback for package added + this.tree.fetch({success: function() { + self.update(); + self.tree.get('packages').on('add', function(pack) { + console.log('Package ' + pack.get('pid') + ' added to tree'); + self.appendPackage(pack, 0, true); + self.openPackage(pack); + }); + }}); + + this.$('.input').select2({tags: ['a', 'b', 'sdf']}); + }, + + update: function() { + console.log('Update package list'); + + // TODO: Both null + var packs = this.tree.get('packages'); + this.files = this.tree.get('files'); + + if (packs) + packs.each(_.bind(this.appendPackage, this)); + + if (!this.files || this.files.length === 0) { + // no files are displayed + this.files = null; + // Open the first package + if (packs && packs.length >= 1) + this.openPackage(packs.at(0)); + } + else + this.files.each(_.bind(this.appendFile, this)); + + return this; + }, + + // TODO sorting ?! + // Append a package to the list, index, animate it + appendPackage: function(pack, i, animation) { + var el = new PackageView({model: pack}).render().el; + $(this.ui.packages).appendWithAnimation(el, animation); + }, + + appendFile: function(file, i, animation) { + var el = new FileView({model: file}).render().el; + $(this.ui.files).appendWithAnimation(el, animation); + }, + + // Show content of the packages on main view + openPackage: function(pack) { + var self = this; + + // load animation only when something is shown and its different from current package + if (this.files && this.files !== pack.get('files')) + self.loading(); + + pack.fetch({silent: true, success: function() { + console.log('Package ' + pack.get('pid') + ' loaded'); + self.active.text(pack.get('name')); + self.contentReady(pack.get('files')); + }, failure: function() { + self.failure(); + }}); + + }, + + contentReady: function(files) { + var old_files = this.files; + this.files = files; + App.vent.trigger('dashboard:contentReady'); + + // show the files when no loading animation is running and not already open + if (!this.isLoading && old_files !== files) + this.show(); + }, + + // Do load animation, remove the old stuff + loading: function() { + this.isLoading = true; + this.files = null; + var self = this; + $(this.ui.files).fadeOut({complete: function() { + // All file views should vanish + App.vent.trigger('dashboard:destroyContent'); + + // Loading was faster than animation + if (self.files) + self.show(); + + self.isLoading = false; + }}); + }, + + failure: function() { + // TODO + }, + + show: function() { + // fileUL has to be resetted before + this.files.each(_.bind(this.appendFile, this)); + //TODO: show placeholder when nothing is displayed (filtered content empty) + $(this.ui.files).fadeIn(); + App.vent.trigger('dashboard:updated'); + }, + + // Refresh the file if it is currently shown + fileUpdated: function(data) { + // this works with ids and object + var file = this.files.get(data); + if (file) + if (_.isObject(data)) // update directly + file.set(data); + else // fetch from server + file.fetch(); + } + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/dashboard/fileView.js b/pyload/web/app/scripts/views/dashboard/fileView.js new file mode 100644 index 000000000..4e5884ed8 --- /dev/null +++ b/pyload/web/app/scripts/views/dashboard/fileView.js @@ -0,0 +1,102 @@ +define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes', 'views/abstract/itemView', 'helpers/formatTime', 'hbs!tpl/dashboard/file'], + function($, Backbone, _, App, Api, ItemView, formatTime, template) { + 'use strict'; + + // Renders single file item + return ItemView.extend({ + + tagName: 'li', + className: 'file-view row-fluid', + template: template, + events: { + 'click .checkbox': 'select', + 'click .btn-delete': 'deleteItem', + 'click .btn-restart': 'restart' + }, + + initialize: function() { + this.listenTo(this.model, 'change', this.render); + // This will be triggered manually and changed before with silent=true + this.listenTo(this.model, 'change:visible', this.visibility_changed); + this.listenTo(this.model, 'change:progress', this.progress_changed); + this.listenTo(this.model, 'remove', this.unrender); + this.listenTo(App.vent, 'dashboard:destroyContent', this.destroy); + }, + + onDestroy: function() { + }, + + render: function() { + var data = this.model.toJSON(); + if (data.download) { + var status = data.download.status; + if (status === Api.DownloadStatus.Offline || status === Api.DownloadStatus.TempOffline) + data.offline = true; + else if (status === Api.DownloadStatus.Online) + data.online = true; + else if (status === Api.DownloadStatus.Waiting) + data.waiting = true; + else if (status === Api.DownloadStatus.Downloading) + data.downloading = true; + else if (this.model.isFailed()) + data.failed = true; + else if (this.model.isFinished()) + data.finished = true; + } + + this.$el.html(this.template(data)); + if (this.model.get('selected')) + this.$el.addClass('ui-selected'); + else + this.$el.removeClass('ui-selected'); + + if (this.model.get('visible')) + this.$el.show(); + else + this.$el.hide(); + + return this; + }, + + select: function(e) { + e.preventDefault(); + var checked = this.$el.hasClass('ui-selected'); + // toggle class immediately, so no re-render needed + this.model.set('selected', !checked, {silent: true}); + this.$el.toggleClass('ui-selected'); + App.vent.trigger('file:selection'); + }, + + visibility_changed: function(visible) { + // TODO: improve animation, height is not available when element was not visible + if (visible) + this.$el.slideOut(true); + else { + this.$el.calculateHeight(true); + this.$el.slideIn(true); + } + }, + + progress_changed: function() { + if (!this.model.isDownload()) + return; + + if (this.model.get('download').status === Api.DownloadStatus.Downloading) { + var bar = this.$('.progress .bar'); + if (!bar) { // ensure that the dl bar is rendered + this.render(); + bar = this.$('.progress .bar'); + } + + bar.width(this.model.get('progress') + '%'); + bar.html('  ' + formatTime(this.model.get('eta'))); + } else if (this.model.get('download').status === Api.DownloadStatus.Waiting) { + this.$('.second').html( + ' ' + formatTime(this.model.get('eta'))); + + } else // Every else state can be renderred normally + this.render(); + + } + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/dashboard/filterView.js b/pyload/web/app/scripts/views/dashboard/filterView.js new file mode 100644 index 000000000..64bc56724 --- /dev/null +++ b/pyload/web/app/scripts/views/dashboard/filterView.js @@ -0,0 +1,133 @@ +define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes', 'models/Package', 'hbs!tpl/dashboard/actionbar'], + /*jslint -W040: false*/ + function($, Backbone, _, App, Api, Package, template) { + 'use strict'; + + // Modified version of type ahead show, nearly the same without absolute positioning + function show() { + this.$menu + .insertAfter(this.$element) + .show(); + + this.shown = true; + return this; + } + + // Renders the actionbar for the dashboard, handles everything related to filtering displayed files + return Backbone.Marionette.ItemView.extend({ + + events: { + 'click .filter-type': 'filter_type', + 'click .filter-state': 'switch_filter', + 'submit .form-search': 'search' + }, + + ui: { + 'search': '.search-query', + 'stateMenu': '.dropdown-toggle .state' + }, + + template: template, + state: null, + + initialize: function() { + this.state = Api.DownloadState.All; + + // Apply the filter before the content is shown + App.vent.on('dashboard:contentReady', _.bind(this.apply_filter, this)); + }, + + onRender: function() { + // use our modified method + $.fn.typeahead.Constructor.prototype.show = show; + this.ui.search.typeahead({ + minLength: 2, + source: this.getSuggestions + }); + + }, + + // TODO: app level api request + search: function(e) { + e.stopPropagation(); + var query = this.ui.search.val(); + this.ui.search.val(''); + + var pack = new Package(); + // Overwrite fetch method to use a search + // TODO: quite hackish, could be improved to filter packages + // or show performed search + pack.fetch = function(options) { + pack.search(query, options); + }; + + App.dashboard.openPackage(pack); + }, + + getSuggestions: function(query, callback) { + $.ajax(App.apiRequest('searchSuggestions', {pattern: query}, { + method: 'POST', + success: function(data) { + callback(data); + } + })); + }, + + switch_filter: function(e) { + e.stopPropagation(); + var element = $(e.target); + var state = parseInt(element.data('state'), 10); + var menu = this.ui.stateMenu.parent().parent(); + menu.removeClass('open'); + + if (state === Api.DownloadState.Finished) { + menu.removeClass().addClass('dropdown finished'); + } else if (state === Api.DownloadState.Unfinished) { + menu.removeClass().addClass('dropdown active'); + } else if (state === Api.DownloadState.Failed) { + menu.removeClass().addClass('dropdown failed'); + } else { + menu.removeClass().addClass('dropdown'); + } + + this.state = state; + this.ui.stateMenu.text(element.text()); + this.apply_filter(); + }, + + // Applies the filtering to current open files + apply_filter: function() { + if (!App.dashboard.files) + return; + + var self = this; + App.dashboard.files.map(function(file) { + var visible = file.get('visible'); + if (visible !== self.is_visible(file)) { + file.set('visible', !visible, {silent: true}); + file.trigger('change:visible', !visible); + } + }); + + App.vent.trigger('dashboard:filtered'); + }, + + // determine if a file should be visible + // TODO: non download files + is_visible: function(file) { + if (this.state === Api.DownloadState.Finished) + return file.isFinished(); + else if (this.state === Api.DownloadState.Unfinished) + return file.isUnfinished(); + else if (this.state === Api.DownloadState.Failed) + return file.isFailed(); + + return true; + }, + + filter_type: function(e) { + + } + + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/dashboard/packageView.js b/pyload/web/app/scripts/views/dashboard/packageView.js new file mode 100644 index 000000000..2738fcbea --- /dev/null +++ b/pyload/web/app/scripts/views/dashboard/packageView.js @@ -0,0 +1,75 @@ +define(['jquery', 'app', 'views/abstract/itemView', 'underscore', 'hbs!tpl/dashboard/package'], + function($, App, itemView, _, template) { + 'use strict'; + + // Renders a single package item + return itemView.extend({ + + tagName: 'li', + className: 'package-view', + template: template, + events: { + 'click .package-name, .btn-open': 'open', + 'click .icon-refresh': 'restart', + 'click .select': 'select', + 'click .btn-delete': 'deleteItem' + }, + + // Ul for child packages (unused) + ul: null, + // Currently unused + expanded: false, + + initialize: function() { + this.listenTo(this.model, 'filter:added', this.hide); + this.listenTo(this.model, 'filter:removed', this.show); + this.listenTo(this.model, 'change', this.render); + this.listenTo(this.model, 'remove', this.unrender); + + // Clear drop down menu + var self = this; + this.$el.on('mouseleave', function() { + self.$('.dropdown-menu').parent().removeClass('open'); + }); + }, + + onDestroy: function() { + }, + + // Render everything, optional only the fileViews + render: function() { + this.$el.html(this.template(this.model.toJSON())); + this.$el.initTooltips(); + + return this; + }, + + unrender: function() { + itemView.prototype.unrender.apply(this); + + // TODO: display other package + App.vent.trigger('dashboard:loading', null); + }, + + + // TODO + // Toggle expanding of packages + expand: function(e) { + e.preventDefault(); + }, + + open: function(e) { + e.preventDefault(); + App.dashboard.openPackage(this.model); + }, + + select: function(e) { + e.preventDefault(); + var checked = this.$('.select').hasClass('icon-check'); + // toggle class immediately, so no re-render needed + this.model.set('selected', !checked, {silent: true}); + this.$('.select').toggleClass('icon-check').toggleClass('icon-check-empty'); + App.vent.trigger('package:selection'); + } + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/dashboard/selectionView.js b/pyload/web/app/scripts/views/dashboard/selectionView.js new file mode 100644 index 000000000..8685fd849 --- /dev/null +++ b/pyload/web/app/scripts/views/dashboard/selectionView.js @@ -0,0 +1,155 @@ +define(['jquery', 'backbone', 'underscore', 'app', 'hbs!tpl/dashboard/select'], + function($, Backbone, _, App, template) { + 'use strict'; + + // Renders context actions for selection packages and files + return Backbone.Marionette.ItemView.extend({ + + el: '#selection-area', + template: template, + + events: { + 'click .icon-check': 'deselect', + 'click .icon-pause': 'pause', + 'click .icon-trash': 'trash', + 'click .icon-refresh': 'restart' + }, + + // Element of the action bar + actionBar: null, + // number of currently selected elements + current: 0, + + initialize: function() { + this.$el.calculateHeight().height(0); + var render = _.bind(this.render, this); + + App.vent.on('dashboard:updated', render); + App.vent.on('dashboard:filtered', render); + App.vent.on('package:selection', render); + App.vent.on('file:selection', render); + + this.actionBar = $('.actionbar .btn-check'); + this.actionBar.parent().click(_.bind(this.select_toggle, this)); + + // API events, maybe better to rely on internal ones? + App.vent.on('package:deleted', render); + App.vent.on('file:deleted', render); + }, + + get_files: function(all) { + var files = []; + if (App.dashboard.files) + if (all) + files = App.dashboard.files.where({visible: true}); + else + files = App.dashboard.files.where({selected: true, visible: true}); + + return files; + }, + + get_packs: function() { + if (!App.dashboard.tree.get('packages')) + return []; // TODO + + return App.dashboard.tree.get('packages').where({selected: true}); + }, + + render: function() { + var files = this.get_files().length; + var packs = this.get_packs().length; + + if (files + packs > 0) { + this.$el.html(this.template({files: files, packs: packs})); + this.$el.initTooltips('bottom'); + } + + if (files + packs > 0 && this.current === 0) + this.$el.slideOut(); + else if (files + packs === 0 && this.current > 0) + this.$el.slideIn(); + + // TODO: accessing ui directly, should be events + if (files > 0) { + this.actionBar.addClass('icon-check').removeClass('icon-check-empty'); + App.dashboard.ui.packages.addClass('ui-files-selected'); + } + else { + this.actionBar.addClass('icon-check-empty').removeClass('icon-check'); + App.dashboard.ui.packages.removeClass('ui-files-selected'); + } + + this.current = files + packs; + }, + + // Deselects all items + deselect: function() { + this.get_files().map(function(file) { + file.set('selected', false); + }); + + this.get_packs().map(function(pack) { + pack.set('selected', false); + }); + + this.render(); + }, + + pause: function() { + alert('Not implemented yet'); + this.deselect(); + }, + + trash: function() { + _.confirm('default/confirmDialog.html', function() { + + var pids = []; + // TODO: delete many at once + this.get_packs().map(function(pack) { + pids.push(pack.get('pid')); + pack.destroy(); + }); + + // get only the fids of non deleted packages + var fids = _.filter(this.get_files(),function(file) { + return !_.contains(pids, file.get('package')); + }).map(function(file) { + file.destroyLocal(); + return file.get('fid'); + }); + + if (fids.length > 0) + $.ajax(App.apiRequest('deleteFiles', {fids: fids})); + + this.deselect(); + }, this); + }, + + restart: function() { + this.get_files().map(function(file) { + file.restart(); + }); + this.get_packs().map(function(pack) { + pack.restart(); + }); + + this.deselect(); + }, + + // Select or deselect all visible files + select_toggle: function() { + var files = this.get_files(); + if (files.length === 0) { + this.get_files(true).map(function(file) { + file.set('selected', true); + }); + + } else + files.map(function(file) { + file.set('selected', false); + }); + + this.render(); + } + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/headerView.js b/pyload/web/app/scripts/views/headerView.js new file mode 100644 index 000000000..512c7259b --- /dev/null +++ b/pyload/web/app/scripts/views/headerView.js @@ -0,0 +1,240 @@ +define(['jquery', 'underscore', 'backbone', 'app', 'models/ServerStatus', 'collections/ProgressList', + 'views/progressView', 'views/notificationView', 'helpers/formatSize', 'hbs!tpl/header/layout', + 'hbs!tpl/header/status', 'hbs!tpl/header/progressbar' , 'flot'], + function($, _, Backbone, App, ServerStatus, ProgressList, ProgressView, NotificationView, formatSize, + template, templateStatus, templateHeader) { + 'use strict'; + // Renders the header with all information + return Backbone.Marionette.ItemView.extend({ + + events: { + 'click .icon-list': 'toggle_taskList', + 'click .popover .close': 'toggle_taskList', + 'click .btn-grabber': 'open_grabber' + }, + + ui: { + progress: '.progress-list', + speedgraph: '#speedgraph' + }, + + // todo: maybe combine these + template: template, + templateStatus: templateStatus, + templateHeader: templateHeader, + + // view + grabber: null, + speedgraph: null, + + // models and data + ws: null, + status: null, + progressList: null, + speeds: null, + + // sub view + notificationView: null, + + // save if last progress was empty + wasEmpty: false, + + initialize: function() { + var self = this; + this.notificationView = new NotificationView(); + + this.status = new ServerStatus(); + this.listenTo(this.status, 'change', this.update); + + this.progressList = new ProgressList(); + this.listenTo(this.progressList, 'add', function(model) { + self.ui.progress.appendWithAnimation(new ProgressView({model: model}).render().el); + }); + + // TODO: button to start stop refresh + var ws = App.openWebSocket('/async'); + ws.onopen = function() { + ws.send(JSON.stringify('start')); + }; + // TODO compare with polling + ws.onmessage = _.bind(this.onData, this); + ws.onerror = function(error) { + console.log(error); + alert('WebSocket error' + error); + }; + + this.ws = ws; + }, + + initGraph: function() { + var totalPoints = 120; + var data = []; + + // init with empty data + while (data.length < totalPoints) + data.push([data.length, 0]); + + this.speeds = data; + this.speedgraph = $.plot(this.ui.speedgraph, [this.speeds], { + series: { + lines: { show: true, lineWidth: 2 }, + shadowSize: 0, + color: '#fee247' + }, + xaxis: { ticks: [] }, + yaxis: { ticks: [], min: 1, autoscaleMargin: 0.1, tickFormatter: function(data) { + return formatSize(data * 1024); + }, position: 'right' }, + grid: { + show: true, +// borderColor: "#757575", + borderColor: 'white', + borderWidth: 1, + labelMargin: 0, + axisMargin: 0, + minBorderMargin: 0 + } + }); + + }, + + // Must be called after view was attached + init: function() { + this.initGraph(); + this.update(); + }, + + update: function() { + // TODO: what should be displayed in the header + // queue/processing size? + + var status = this.status.toJSON(); + status.maxspeed = _.max(this.speeds, function(speed) { + return speed[1]; + })[1] * 1024; + this.$('.status-block').html( + this.templateStatus(status) + ); + + var data = {tasks: 0, downloads: 0, speed: 0, single: false}; + this.progressList.each(function(progress) { + if (progress.isDownload()) { + data.downloads += 1; + data.speed += progress.get('download').speed; + } else + data.tasks++; + }); + + // Show progress of one task + if (data.tasks + data.downloads === 1) { + var progress = this.progressList.at(0); + data.single = true; + data.eta = progress.get('eta'); + data.percent = progress.getPercent(); + data.name = progress.get('name'); + data.statusmsg = progress.get('statusmsg'); + } + // TODO: better progressbar rendering + + data.etaqueue = status.eta; + data.linksqueue = status.linksqueue; + data.sizequeue = status.sizequeue; + + this.$('#progress-info').html( + this.templateHeader(data) + ); + return this; + }, + + toggle_taskList: function() { + this.$('.popover').animate({opacity: 'toggle'}); + }, + + open_grabber: function() { + var self = this; + _.requireOnce(['views/linkGrabberModal'], function(ModalView) { + if (self.grabber === null) + self.grabber = new ModalView(); + + self.grabber.show(); + }); + }, + + onData: function(evt) { + var data = JSON.parse(evt.data); + if (data === null) return; + + if (data['@class'] === 'ServerStatus') { + // TODO: load interaction when none available + this.status.set(data); + + // There tasks at the server, but not in queue: so fetch them + // or there are tasks in our queue but not on the server + if (this.status.get('notifications') && !this.notificationView.tasks.hasTaskWaiting() || + !this.status.get('notifications') && this.notificationView.tasks.hasTaskWaiting()) + this.notificationView.tasks.fetch(); + + this.speeds = this.speeds.slice(1); + this.speeds.push([this.speeds[this.speeds.length - 1][0] + 1, Math.floor(data.speed / 1024)]); + + // TODO: if everything is 0 re-render is not needed + this.speedgraph.setData([this.speeds]); + // adjust the axis + this.speedgraph.setupGrid(); + this.speedgraph.draw(); + + } + else if (_.isArray(data)) + this.onProgressUpdate(data); + else if (data['@class'] === 'EventInfo') + this.onEvent(data.eventname, data.event_args); + else + console.log('Unknown Async input', data); + + }, + + onProgressUpdate: function(progress) { + // generate a unique id + _.each(progress, function(prog) { + if (prog.download) + prog.pid = prog.download.fid; + else + prog.pid = prog.plugin + prog.name; + }); + + this.progressList.set(progress); + // update currently open files with progress + this.progressList.each(function(prog) { + if (prog.isDownload() && App.dashboard.files) { + var file = App.dashboard.files.get(prog.get('download').fid); + if (file) { + file.set({ + progress: prog.getPercent(), + eta: prog.get('eta') + }, {silent: true}); + + file.trigger('change:progress'); + } + } + }); + + if (progress.length === 0) { + // only render one time when last was not empty already + if (!this.wasEmpty) { + this.update(); + this.wasEmpty = true; + } + } else { + this.wasEmpty = false; + this.update(); + } + }, + + onEvent: function(event, args) { + args.unshift(event); + console.log('Core send event', args); + App.vent.trigger.apply(App.vent, args); + } + + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/input/inputLoader.js b/pyload/web/app/scripts/views/input/inputLoader.js new file mode 100644 index 000000000..11665abb4 --- /dev/null +++ b/pyload/web/app/scripts/views/input/inputLoader.js @@ -0,0 +1,8 @@ +define(['./textInput'], function(textInput) { + 'use strict'; + + // selects appropriate input element + return function(input, value, default_value, description) { + return textInput; + }; +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/input/inputView.js b/pyload/web/app/scripts/views/input/inputView.js new file mode 100644 index 000000000..1fbe5042d --- /dev/null +++ b/pyload/web/app/scripts/views/input/inputView.js @@ -0,0 +1,86 @@ +define(['jquery', 'backbone', 'underscore'], function($, Backbone, _) { + 'use strict'; + + // Renders input elements + return Backbone.View.extend({ + + tagName: 'input', + + input: null, + value: null, + default_value: null, + description: null, + + // enables tooltips + tooltip: true, + + initialize: function(options) { + this.input = options.input; + this.value = options.value; + this.default_value = options.default_value; + this.description = options.description; + }, + + render: function() { + this.renderInput(); + // data for tooltips + if (this.description && this.tooltip) { + this.$el.data('content', this.description); + // TODO: render default value in popup? +// this.$el.data('title', "TODO: title"); + this.$el.popover({ + placement: 'right', + trigger: 'hover' +// delay: { show: 500, hide: 100 } + }); + } + + return this; + }, + + renderInput: function() { + // Overwrite this + }, + + showTooltip: function() { + if (this.description && this.tooltip) + this.$el.popover('show'); + }, + + hideTooltip: function() { + if (this.description && this.tooltip) + this.$el.popover('hide'); + }, + + destroy: function() { + this.undelegateEvents(); + this.unbind(); + if (this.onDestroy) { + this.onDestroy(); + } + this.$el.removeData().unbind(); + this.remove(); + }, + + // focus the input element + focus: function() { + this.$el.focus(); + }, + + // Clear the input + clear: function() { + + }, + + // retrieve value of the input + getVal: function() { + return this.value; + }, + + // the child class must call this when the value changed + setVal: function(value) { + this.value = value; + this.trigger('change', value); + } + }); +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/input/textInput.js b/pyload/web/app/scripts/views/input/textInput.js new file mode 100644 index 000000000..0eebbf91e --- /dev/null +++ b/pyload/web/app/scripts/views/input/textInput.js @@ -0,0 +1,36 @@ +define(['jquery', 'backbone', 'underscore', './inputView'], function($, Backbone, _, inputView) { + 'use strict'; + + return inputView.extend({ + + // TODO + tagName: 'input', + events: { + 'keyup': 'onChange', + 'focus': 'showTooltip', + 'focusout': 'hideTooltip' + }, + + renderInput: function() { + this.$el.attr('type', 'text'); + this.$el.attr('name', 'textInput'); + + if (this.default_value) + this.$el.attr('placeholder', this.default_value); + + if (this.value) + this.$el.val(this.value); + + return this; + }, + + clear: function() { + this.$el.val(''); + }, + + onChange: function(e) { + this.setVal(this.$el.val()); + } + + }); +}); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/linkGrabberModal.js b/pyload/web/app/scripts/views/linkGrabberModal.js new file mode 100644 index 000000000..e6f59c134 --- /dev/null +++ b/pyload/web/app/scripts/views/linkGrabberModal.js @@ -0,0 +1,49 @@ +define(['jquery', 'underscore', 'app', 'views/abstract/modalView', 'hbs!tpl/dialogs/linkgrabber'], + function($, _, App, modalView, template) { + 'use strict'; + // Modal dialog for package adding - triggers package:added when package was added + return modalView.extend({ + + events: { + 'click .btn-success': 'addPackage', + 'keypress #inputPackageName': 'addOnEnter' + }, + + template: template, + + initialize: function() { + // Inherit parent events + this.events = _.extend({}, modalView.prototype.events, this.events); + }, + + addOnEnter: function(e) { + if (e.keyCode !== 13) return; + this.addPackage(e); + }, + + addPackage: function(e) { + var self = this; + var options = App.apiRequest('addPackage', + { + name: $('#inputPackageName').val(), + // TODO: better parsing / tokenization + links: $('#inputLinks').val().split('\n') + }, + { + success: function() { + App.vent.trigger('package:added'); + self.hide(); + } + }); + + $.ajax(options); + $('#inputPackageName').val(''); + $('#inputLinks').val(''); + }, + + onShow: function() { + this.$('#inputPackageName').focus(); + } + + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/loginView.js b/pyload/web/app/scripts/views/loginView.js new file mode 100644 index 000000000..891b3ec99 --- /dev/null +++ b/pyload/web/app/scripts/views/loginView.js @@ -0,0 +1,37 @@ +define(['jquery', 'backbone', 'underscore', 'app', 'hbs!tpl/login'], + function($, Backbone, _, App, template) { + 'use strict'; + + // Renders context actions for selection packages and files + return Backbone.Marionette.ItemView.extend({ + template: template, + + events: { + 'submit form': 'login' + }, + + ui: { + 'form': 'form' + }, + + login: function(e) { + e.stopPropagation(); + + var options = App.apiRequest('login', null, { + data: this.ui.form.serialize(), + type : 'post', + success: function(data) { + // TODO: go to last page, better error + if (data) + App.navigate(''); + else + alert('Wrong login'); + } + }); + + $.ajax(options); + return false; + } + + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/notificationView.js b/pyload/web/app/scripts/views/notificationView.js new file mode 100644 index 000000000..abfcd8079 --- /dev/null +++ b/pyload/web/app/scripts/views/notificationView.js @@ -0,0 +1,83 @@ +define(['jquery', 'backbone', 'underscore', 'app', 'collections/InteractionList', 'hbs!tpl/notification'], + function($, Backbone, _, App, InteractionList, queryModal, template) { + 'use strict'; + + // Renders context actions for selection packages and files + return Backbone.View.extend({ + + // Only view for this area so it's hardcoded + el: '#notification-area', + template: template, + + events: { + 'click .btn-query': 'openQuery', + 'click .btn-notification': 'openNotifications' + }, + + tasks: null, + // area is slided out + visible: false, + // the dialog + modal: null, + + initialize: function() { + this.tasks = new InteractionList(); + + this.$el.calculateHeight().height(0); + + App.vent.on('interaction:added', _.bind(this.onAdd, this)); + App.vent.on('interaction:deleted', _.bind(this.onDelete, this)); + + var render = _.bind(this.render, this); + this.listenTo(this.tasks, 'add', render); + this.listenTo(this.tasks, 'remove', render); + + }, + + onAdd: function(task) { + this.tasks.add(task); + }, + + onDelete: function(task) { + this.tasks.remove(task); + }, + + render: function() { + + // only render when it will be visible + if (this.tasks.length > 0) + this.$el.html(this.template(this.tasks.toJSON())); + + if (this.tasks.length > 0 && !this.visible) { + this.$el.slideOut(); + this.visible = true; + } + else if (this.tasks.length === 0 && this.visible) { + this.$el.slideIn(); + this.visible = false; + } + + return this; + }, + + openQuery: function() { + var self = this; + + _.requireOnce(['views/queryModal'], function(ModalView) { + if (self.modal === null) { + self.modal = new ModalView(); + self.modal.parent = self; + } + + self.modal.model = self.tasks.at(0); + self.modal.render(); + self.modal.show(); + }); + + }, + + openNotifications: function() { + + } + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/progressView.js b/pyload/web/app/scripts/views/progressView.js new file mode 100644 index 000000000..3a4bb2825 --- /dev/null +++ b/pyload/web/app/scripts/views/progressView.js @@ -0,0 +1,33 @@ +define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes', 'views/abstract/itemView', + 'hbs!tpl/header/progress', 'helpers/pluginIcon'], + function($, Backbone, _, App, Api, ItemView, template, pluginIcon) { + 'use strict'; + + // Renders single file item + return ItemView.extend({ + + idAttribute: 'pid', + tagName: 'li', + template: template, + events: { + }, + + initialize: function() { + this.listenTo(this.model, 'change', this.render); + this.listenTo(this.model, 'remove', this.unrender); + }, + + onDestroy: function() { + }, + + render: function() { + // TODO: icon + // TODO: other states + // TODO: non download progress + // TODO: better progressbar rendering + this.$el.css('background-image', 'url('+ pluginIcon('todo') +')'); + this.$el.html(this.template(this.model.toJSON())); + return this; + } + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/queryModal.js b/pyload/web/app/scripts/views/queryModal.js new file mode 100644 index 000000000..7c6439b49 --- /dev/null +++ b/pyload/web/app/scripts/views/queryModal.js @@ -0,0 +1,69 @@ +define(['jquery', 'underscore', 'app', 'views/abstract/modalView', './input/inputLoader', 'text!tpl/default/queryDialog.html'], + function($, _, App, modalView, load_input, template) { + 'use strict'; + return modalView.extend({ + + // TODO: submit on enter reloads the page sometimes + events: { + 'click .btn-success': 'submit', + 'submit form': 'submit' + }, + template: _.compile(template), + + // the notificationView + parent: null, + + model: null, + input: null, + + initialize: function() { + // Inherit parent events + this.events = _.extend({}, modalView.prototype.events, this.events); + }, + + renderContent: function() { + var data = { + title: this.model.get('title'), + plugin: this.model.get('plugin'), + description: this.model.get('description') + }; + + var input = this.model.get('input').data; + if (this.model.isCaptcha()) { + data.captcha = input[0]; + data.type = input[1]; + } + return data; + }, + + onRender: function() { + // instantiate the input + var input = this.model.get('input'); + var InputView = load_input(input); + this.input = new InputView(input); + // only renders after wards + this.$('#inputField').append(this.input.render().el); + }, + + submit: function(e) { + e.stopPropagation(); + // TODO: load next task + + this.model.set('result', this.input.getVal()); + var self = this; + this.model.save({success: function() { + self.hide(); + }}); + + this.input.clear(); + }, + + onShow: function() { + this.input.focus(); + }, + + onHide: function() { + this.input.destroy(); + } + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/settings/configSectionView.js b/pyload/web/app/scripts/views/settings/configSectionView.js new file mode 100644 index 000000000..e05701b2a --- /dev/null +++ b/pyload/web/app/scripts/views/settings/configSectionView.js @@ -0,0 +1,99 @@ +define(['jquery', 'underscore', 'backbone', 'app', '../abstract/itemView', '../input/inputLoader', + 'hbs!tpl/settings/config', 'hbs!tpl/settings/configItem'], + function($, _, Backbone, App, itemView, load_input, template, templateItem) { + 'use strict'; + + // Renders settings over view page + return itemView.extend({ + + tagName: 'div', + + template: template, + templateItem: templateItem, + + // Will only render one time with further attribute updates + rendered: false, + + events: { + 'click .btn-primary': 'submit', + 'click .btn-reset': 'reset' + }, + + initialize: function() { + this.listenTo(this.model, 'destroy', this.destroy); + }, + + render: function() { + if (!this.rendered) { + this.$el.html(this.template(this.model.toJSON())); + + // initialize the popover + this.$('.page-header a').popover({ + placement: 'left' +// trigger: 'hover' + }); + + var container = this.$('.control-content'); + var self = this; + _.each(this.model.get('items'), function(item) { + var json = item.toJSON(); + var el = $('
          ').html(self.templateItem(json)); + var InputView = load_input(item.get('input')); + var input = new InputView(json).render(); + item.set('inputView', input); + + self.listenTo(input, 'change', _.bind(self.render, self)); + el.find('.controls').append(input.el); + container.append(el); + }); + this.rendered = true; + } + // Enable button if something is changed + if (this.model.hasChanges()) + this.$('.btn-primary').removeClass('disabled'); + else + this.$('.btn-primary').addClass('disabled'); + + // Mark all inputs that are modified + _.each(this.model.get('items'), function(item) { + var input = item.get('inputView'); + var el = input.$el.parent().parent(); + if (item.isChanged()) + el.addClass('info'); + else + el.removeClass('info'); + }); + + return this; + }, + + onDestroy: function() { + // TODO: correct cleanup after building up so many views and models + }, + + submit: function(e) { + e.stopPropagation(); + // TODO: success / failure popups + var self = this; + this.model.save({success: function() { + self.render(); + App.settingsView.refresh(); + }}); + + }, + + reset: function(e) { + e.stopPropagation(); + // restore the original value + _.each(this.model.get('items'), function(item) { + if (item.has('inputView')) { + var input = item.get('inputView'); + input.setVal(item.get('value')); + input.render(); + } + }); + this.render(); + } + + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/settings/pluginChooserModal.js b/pyload/web/app/scripts/views/settings/pluginChooserModal.js new file mode 100644 index 000000000..91e9f11b3 --- /dev/null +++ b/pyload/web/app/scripts/views/settings/pluginChooserModal.js @@ -0,0 +1,69 @@ +define(['jquery', 'underscore', 'app', 'views/abstract/modalView', 'hbs!tpl/dialogs/addPluginConfig', + 'helpers/pluginIcon', 'select2'], + function($, _, App, modalView, template, pluginIcon) { + 'use strict'; + return modalView.extend({ + + events: { + 'click .btn-add': 'add' + }, + template: template, + plugins: null, + select: null, + + initialize: function() { + // Inherit parent events + this.events = _.extend({}, modalView.prototype.events, this.events); + var self = this; + $.ajax(App.apiRequest('getAvailablePlugins', null, {success: function(data) { + self.plugins = _.sortBy(data, function(item) { + return item.name; + }); + self.render(); + }})); + }, + + onRender: function() { + // TODO: could be a seperate input type if needed on multiple pages + if (this.plugins) + this.select = this.$('#pluginSelect').select2({ + escapeMarkup: function(m) { + return m; + }, + formatResult: this.format, + formatSelection: this.formatSelection, + data: {results: this.plugins, text: function(item) { + return item.label; + }}, + id: function(item) { + return item.name; + } + }); + }, + + onShow: function() { + }, + + onHide: function() { + }, + + format: function(data) { + var s = '
          ' + data.label; + s += '
          ' + data.description + '
          '; + return s; + }, + + formatSelection: function(data) { + return ' ' + data.label; + }, + + add: function(e) { + e.stopPropagation(); + if (this.select) { + var plugin = this.select.val(); + App.vent.trigger('config:open', plugin); + this.hide(); + } + } + }); + }); \ No newline at end of file diff --git a/pyload/web/app/scripts/views/settings/settingsView.js b/pyload/web/app/scripts/views/settings/settingsView.js new file mode 100644 index 000000000..cad5ab075 --- /dev/null +++ b/pyload/web/app/scripts/views/settings/settingsView.js @@ -0,0 +1,184 @@ +define(['jquery', 'underscore', 'backbone', 'app', 'models/ConfigHolder', './configSectionView', + 'hbs!tpl/settings/layout', 'hbs!tpl/settings/menu', 'hbs!tpl/settings/actionbar'], + function($, _, Backbone, App, ConfigHolder, ConfigSectionView, template, templateMenu, templateBar) { + 'use strict'; + + // Renders settings over view page + return Backbone.Marionette.ItemView.extend({ + + template: template, + templateMenu: templateMenu, + + events: { + 'click .settings-menu li > a': 'change_section', + 'click .btn-add': 'choosePlugin', // TODO not in scope + 'click .icon-remove': 'deleteConfig' + }, + + ui: { + 'menu': '.settings-menu', + 'content': '.setting-box > form' + }, + + selected: null, + modal: null, + + coreConfig: null, // It seems collections are not needed + pluginConfig: null, + + // currently open configHolder + config: null, + lastConfig: null, + isLoading: false, + + initialize: function() { + this.actionbar = Backbone.Marionette.ItemView.extend({ + template: templateBar, + events: { + 'click .btn': 'choosePlugin' + }, + choosePlugin: _.bind(this.choosePlugin, this) + + }); + this.listenTo(App.vent, 'config:open', this.openConfig); + + this.refresh(); + }, + + refresh: function() { + var self = this; + $.ajax(App.apiRequest('getCoreConfig', null, {success: function(data) { + self.coreConfig = data; + self.renderMenu(); + }})); + $.ajax(App.apiRequest('getPluginConfig', null, {success: function(data) { + self.pluginConfig = data; + self.renderMenu(); + }})); + }, + + onRender: function() { + // set a height with css so animations will work + this.ui.content.height(this.ui.content.height()); + }, + + renderMenu: function() { + var plugins = [], + addons = []; + + // separate addons and default plugins + // addons have an activated state + _.each(this.pluginConfig, function(item) { + if (item.activated === null) + plugins.push(item); + else + addons.push(item); + }); + + this.ui.menu.html(this.templateMenu({ + core: this.coreConfig, + plugin: plugins, + addon: addons + })); + + // mark the selected element + this.$('li[data-name="' + this.selected + '"]').addClass('active'); + }, + + openConfig: function(name) { + // Do nothing when this config is already open + if (this.config && this.config.get('name') === name) + return; + + this.lastConfig = this.config; + this.config = new ConfigHolder({name: name}); + this.loading(); + + var self = this; + this.config.fetch({success: function() { + if (!self.isLoading) + self.show(); + + }, failure: _.bind(this.failure, this)}); + + }, + + loading: function() { + this.isLoading = true; + var self = this; + this.ui.content.fadeOut({complete: function() { + if (self.config.isLoaded()) + self.show(); + + self.isLoading = false; + }}); + + }, + + show: function() { + // TODO animations are bit sloppy + this.ui.content.css('display', 'block'); + var oldHeight = this.ui.content.height(); + + // this will destroy the old view + if (this.lastConfig) + this.lastConfig.trigger('destroy'); + else + this.ui.content.empty(); + + // reset the height + this.ui.content.css('height', ''); + // append the new element + this.ui.content.append(new ConfigSectionView({model: this.config}).render().el); + // get the new height + var height = this.ui.content.height(); + // set the old height again + this.ui.content.height(oldHeight); + this.ui.content.animate({ + opacity: 'show', + height: height + }); + }, + + failure: function() { + // TODO + this.config = null; + }, + + change_section: function(e) { + // TODO check for changes + // TODO move this into render? + + var el = $(e.target).closest('li'); + + this.selected = el.data('name'); + this.openConfig(this.selected); + + this.ui.menu.find('li.active').removeClass('active'); + el.addClass('active'); + e.preventDefault(); + }, + + choosePlugin: function(e) { + var self = this; + _.requireOnce(['views/settings/pluginChooserModal'], function(Modal) { + if (self.modal === null) + self.modal = new Modal(); + + self.modal.show(); + }); + }, + + deleteConfig: function(e) { + e.stopPropagation(); + var el = $(e.target).parent().parent(); + var name = el.data('name'); + var self = this; + $.ajax(App.apiRequest('deleteConfig', {plugin: name}, { success: function() { + self.refresh(); + }})); + + } + + }); + }); \ No newline at end of file diff --git a/pyload/web/app/styles/default/accounts.less b/pyload/web/app/styles/default/accounts.less new file mode 100644 index 000000000..9b45b64b3 --- /dev/null +++ b/pyload/web/app/styles/default/accounts.less @@ -0,0 +1,6 @@ +@import "common"; + +.logo-select { + width: 20px; + height: 20px; +} \ No newline at end of file diff --git a/pyload/web/app/styles/default/admin.less b/pyload/web/app/styles/default/admin.less new file mode 100644 index 000000000..92524c153 --- /dev/null +++ b/pyload/web/app/styles/default/admin.less @@ -0,0 +1,17 @@ +@import "common"; + +/* + Admin +*/ + +#btn_newuser { + float: right; +} + +#user_permissions { + float: right; +} + +.userperm { + width: 115px; +} \ No newline at end of file diff --git a/pyload/web/app/styles/default/base.less b/pyload/web/app/styles/default/base.less new file mode 100644 index 000000000..bd318127e --- /dev/null +++ b/pyload/web/app/styles/default/base.less @@ -0,0 +1,163 @@ +@import "common"; + +/* + General +*/ + +* { + margin: 0; +} + +html, body { + height: 100%; +} + +body { + margin: 0; + padding: 0; + font-family: 'Abel', sans-serif; + font-size: 16px; + background: url("../../images/default/bgpattern.png") repeat scroll 0 0 transparent; +} + +h1, h2, h3 { + margin: 0; + padding: 0; + font-weight: normal; +} + +a { + text-decoration: none; + color: @blue; +} + +a:hover { + text-decoration: none; + color: @emph; +} + +#wrap { + min-height: 100%; +} + +#content { + padding-bottom: @footer-height; +} + +#content-container:before { + display: block; + content: " "; + height: @header-height; +} + +/* + Additional Responsive Class for larger desktop +*/ + +.visible-large-screen { + display: inherit !important; +} + +@media (max-width: @large-screen) { + .visible-large-screen { + display: none !important; + } +} + +.btn-blue { + background-color: hsl(206, 49%, 35%) !important; + background-repeat: repeat-x; + background-image: -khtml-gradient(linear, left top, left bottom, from(#5493c4), to(#2d5f84)); + background-image: -moz-linear-gradient(top, #5493c4, #2d5f84); + background-image: -ms-linear-gradient(top, #5493c4, #2d5f84); + background-image: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #5493c4), color-stop(100%, #2d5f84)); + background-image: -webkit-linear-gradient(top, #5493c4, #2d5f84); + background-image: -o-linear-gradient(top, #5493c4, #2d5f84); + background-image: linear-gradient(#5493c4, #2d5f84); + border-color: #2d5f84 #2d5f84 hsl(206, 49%, 30%); + color: #fff !important; + text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.33); + -webkit-font-smoothing: antialiased; + + .caret { + border-bottom-color: #FFFFFF; + border-top-color: #FFFFFF; + } + +} + +.btn-yellow { + background-color: hsl(46, 100%, 57%) !important; + background-repeat: repeat-x; + background-image: -khtml-gradient(linear, left top, left bottom, from(#ffe389), to(#fecb23)); + background-image: -moz-linear-gradient(top, #ffe389, #fecb23); + background-image: -ms-linear-gradient(top, #ffe389, #fecb23); + background-image: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #ffe389), color-stop(100%, #fecb23)); + background-image: -webkit-linear-gradient(top, #ffe389, #fecb23); + background-image: -o-linear-gradient(top, #ffe389, #fecb23); + background-image: linear-gradient(#ffe389, #fecb23); + border-color: #fecb23 #fecb23 hsl(46, 100%, 52%); + color: #333 !important; + text-shadow: 0 1px 1px rgba(255, 255, 255, 0.33); + -webkit-font-smoothing: antialiased; + + .caret { + border-bottom-color: #FFFFFF; + border-top-color: #FFFFFF; + } +} + +.icon-8x { + font-size: 8em; +} + +// Will not work well in buttons/navs, would require more rules +.icon-larger { + vertical-align: -10%; + font-size: 1.5em; +} + +/* + Modal Overlay +*/ +#modal-overlay { + content: " "; + height: 100%; + width: 100%; + position: absolute; + left: 0; + top: 0; + background: -moz-radial-gradient(center, ellipse cover, rgba(236, 208, 66, 0) 0%, rgba(40, 119, 171, 0.9) 100%); + background: -webkit-gradient(radial, center center, 0px, center center, 100%, color-stop(0%, rgba(236, 208, 66, 0)), color-stop(100%, rgba(40, 119, 171, 0.9))); + background: -webkit-radial-gradient(center, ellipse cover, rgba(236, 208, 66, 0) 0%, rgba(40, 119, 171, 0.9) 100%); + background: -o-radial-gradient(center, ellipse cover, rgba(236, 208, 66, 0) 0%, rgba(40, 119, 171, 0.9) 100%); + background: -ms-radial-gradient(center, ellipse cover, rgba(236, 208, 66, 0) 0%, rgba(40, 119, 171, 0.9) 100%); + background: radial-gradient(center, ellipse cover, rgba(236, 208, 66, 0) 0%, rgba(40, 119, 171, 0.9) 100%); + z-index: 100; + opacity: 0; +} + +div.modal-header { + border-top-left-radius: 4px; + border-top-right-radius: 4px; + background-color: @blueDark; + color: #FFFFFF; + + .close { + color: @light; + } + +} + +.select2-container { + min-width: 220px; // same as other input fields +} + +div.modal-body { + max-height: 100%; +} + +div.modal { + width: 600px; + +} \ No newline at end of file diff --git a/pyload/web/app/styles/default/common.less b/pyload/web/app/styles/default/common.less new file mode 100644 index 000000000..338ed38e7 --- /dev/null +++ b/pyload/web/app/styles/default/common.less @@ -0,0 +1,90 @@ +/* + Definitions +*/ + + +/* + Threshold for slightly larger screen +*/ +@large-screen: 1150px; + +@header-height: 70px; +@actionbar-height: 40px; +@footer-height: 66px; + +@light: #ffffff; +@dark: #333333; + +@grey: #757575; +@greyLight: #E5E5E5; +@greyLighter: #F5F5F5; +@greyDark: #444444; +@greyDarker: #111113; + +@yellow: #ffd856; +@yellowLighter: lighten(spin(@yellow, 10), 20%); +@yellowLightest: lighten(spin(@yellow, 15), 30%); +@yellowDark: darken(@yellow, 10%); + +@blue: #3571a2; +@blueLight: lighten(spin(@blue, 5), 10%); +@blueLighter: lighten(spin(@blue, 10), 20%); +@blueLightest: lighten(spin(@blue, 20), 40%); +@blueDark: darken(spin(@blue, -5), 10%); +@blueDarker: darken(spin(@blue, -10), 20%); + +@green: #468847; +@greenLight: lighten(spin(@green, 5), 10%); +@greenDark: darken(spin(@green, -5), 10%); + +@red: #b94a48; +@redLight: lighten(spin(@red, 5), 10%); +@redDark: darken(spin(@red, -5), 10%); + +@emph: #FF7637; + + +/* + Mixins +*/ + +.gradient(@origin: left, @start: #ffffff, @stop: #000000) { + background-color: @start; + background-image: -webkit-linear-gradient(@origin, @start, @stop); + background-image: -moz-linear-gradient(@origin, @start, @stop); + background-image: -o-linear-gradient(@origin, @start, @stop); + background-image: -ms-linear-gradient(@origin, @start, @stop); + background-image: linear-gradient(@origin, @start, @stop); +} + +.transition(@prop: all, @time: 0.25s, @ease: linear) { + -webkit-transition: @prop @time @ease; + -moz-transition: @prop @time @ease; + -o-transition: @prop @time @ease; + -ms-transition: @prop @time @ease; + transition: @prop @time @ease; +} + +.stripes(@color, @color2: rgba(255, 255, 255, 0.15)) { + background-color: @color; + background-image: -webkit-linear-gradient(45deg, @color2 25%, transparent 25%, transparent 50%, @color2 50%, @color2 75%, transparent 75%, transparent); + background-image: -moz-linear-gradient(45deg, @color2 25%, transparent 25%, transparent 50%, @color2 50%, @color2 75%, transparent 75%, transparent); + background-image: -o-linear-gradient(45deg, @color2 25%, transparent 25%, transparent 50%, @color2 50%, @color2 75%, transparent 75%, transparent); + background-image: linear-gradient(45deg, @color2 25%, transparent 25%, transparent 50%, @color2 50%, @color2 75%, transparent 75%, transparent); + -webkit-background-size: 40px 40px; + -moz-background-size: 40px 40px; + -o-background-size: 40px 40px; + background-size: 40px 40px; +} +.stripes-animated { + -webkit-animation: progress-bar-stripes 2s linear infinite; + -moz-animation: progress-bar-stripes 2s linear infinite; + -ms-animation: progress-bar-stripes 2s linear infinite; + -o-animation: progress-bar-stripes 2s linear infinite; + animation: progress-bar-stripes 2s linear infinite; +} + + +.default-shadow { + box-shadow: 0 0 5px @dark; +} \ No newline at end of file diff --git a/pyload/web/app/styles/default/dashboard.less b/pyload/web/app/styles/default/dashboard.less new file mode 100644 index 000000000..ab61ba29b --- /dev/null +++ b/pyload/web/app/styles/default/dashboard.less @@ -0,0 +1,331 @@ +@import "common"; + +/* + Dashboard +*/ + +#dashboard ul { + margin: 0; + list-style: none; +} + +.sidebar-header { + font-size: 25px; + line-height: 25px; + margin: 4px 0; + border-bottom: 1px dashed @grey; +} + +/* + Packages +*/ + +.package-list { + list-style: none; + margin-left: 0; +} + +@frame-top: 20px; +@frame-bottom: 18px; + +.package-frame { + position: absolute; + top: -@frame-top; + left: -@frame-top / 2; + right: -@frame-top / 2; + bottom: -@frame-bottom + 2px; // + size of visible bar + z-index: -1; // lies under package + border: 1px solid @grey; + border-radius: 5px; + box-shadow: 3px 3px 6px rgba(0, 0, 0, 0.75); +} + +.package-view { + padding-bottom: 4px; + margin: 8px 0; + position: relative; + overflow: hidden; + + -webkit-hyphens: auto; + -moz-hyphens: auto; + hyphens: auto; + + i { + cursor: pointer; + } + + & > i { + vertical-align: middle; + } + + .progress { + position: absolute; + height: @frame-bottom; + line-height: @frame-bottom; + font-size: 12px; + text-align: center; + border-radius: 0; + border-bottom-left-radius: 5px; + border-bottom-right-radius: 5px; + bottom: 0; + left: 0; + right: 0; + margin-bottom: 0; + background-image: none; + color: @light; + background-color: @yellow; + } + + .bar-info { + background-image: none; + background-color: @blue; + } + + &:hover { + overflow: visible; + z-index: 10; + + .package-frame { + background-color: @light; + } + } + + &.ui-selected:hover { + color: @light; + + .package-frame { + background-color: @dark; + } + + } +} + +.package-name { + cursor: pointer; +} + +.package-indicator { + position: absolute; + top: 0; + right: 0; + float: right; + color: @blue; + text-shadow: @yellowDark 1px 1px; + height: @frame-top; + line-height: @frame-top; + + & > i:hover { + color: @green; + } + + .dropdown-menu { + text-shadow: none; + } + + .tooltip { + text-shadow: none; + width: 100%; + } + + .btn-move { + color: @green; + display: none; + } + +} + +.ui-files-selected .btn-move { + display: inline; +} + +// Tag area with different effect on hover +.tag-area { + position: absolute; + top: -2px; + left: 0; + + .badge { + font-size: 11px; + line-height: 11px; + } + + .badge i { + cursor: pointer; + &:hover:before { + content: "\f024"; // show Remove icon + } + } + + .badge-ghost { + visibility: hidden; + cursor: pointer; + opacity: 0.5; + } + + &:hover .badge-ghost { + visibility: visible; + } + +} + +/* + File View +*/ + +.file-list { + list-style: none; + margin: 0; +} + +@file-height: 22px; + +.file-view { + position: relative; + padding: 0 4px; + border-top: 1px solid #dddddd; + line-height: @file-height; + + &:first-child { + border-top: none; + } + + &:hover, &.ui-selected:hover { + border-radius: 5px; + .gradient(top, @blue, @blueLight); + color: @light; + } + + &.ui-selected { + .gradient(top, @yellow, @yellowDark); + color: @dark; + border-color: @greenDark; + + .file-row.downloading .bar { + .gradient(top, @green, @greenLight); + } + + } + + img { // plugin logo + margin-top: -2px; + padding: 0 2px; + height: @file-height; + width: @file-height; + } + + .icon-chevron-down:hover { + cursor: pointer; + color: @yellow; + } + +} + +.file-row { + min-height: 0 !important; +// padding-left: 5px; + padding-top: 4px; + padding-bottom: 4px; + + // TODO: better styling for filestatus + &.second { +// border-radius: 4px; +// background: @light; + font-size: small; + font-weight: bold; +// box-shadow: 3px 3px 6px rgba(0, 0, 0, 0.75); +// .default-shadow; + } + + &.third { + margin-left: 0; + position: relative; + font-size: small; + } + + .dropdown-menu { + font-size: medium; + } +} + +/* + TODO: more colorful states + better fileView design +*/ + +.file-row.finished { +// .gradient(top, @green, @greenLight); +// color: @light; + color: @green; +} + +.file-row.failed { +// .gradient(top, @red, @redLight); +// color: @light; + color: @red; +} + +.file-row.downloading { + + .progress { + height: @file-height; + background: @light; + margin: 0; + } + + .bar { + text-align: left; + .gradient(top, @yellow, @yellowDark); + color: @dark; + } + +} + +/* +FANCY CHECKBOXES +*/ +.file-view .checkbox { + width: 20px; + height: 21px; + background: url(../../images/default/checks_sheet.png) left top no-repeat; + cursor: pointer; +} + +.file-view.ui-selected .checkbox { + background: url(../../images/default/checks_sheet.png) -21px top no-repeat; +} + +/* + Actionbar +*/ + +.form-search { + position: relative; + + .dropdown-menu { + min-width: 100%; + position: absolute; + right: 0; + left: auto; + } + +} + +li.finished > a, li.finished:hover > a { + background-color: @green; + color: @light; + + .caret, .caret:hover { + border-bottom-color: @light !important; + border-top-color: @light !important; + } +} + +li.failed > a, li.failed:hover > a { + background-color: @red; + color: @light; + + .caret, .caret:hover { + border-bottom-color: @light !important; + border-top-color: @light !important; + } +} \ No newline at end of file diff --git a/pyload/web/app/styles/default/main.less b/pyload/web/app/styles/default/main.less new file mode 100644 index 000000000..15d9943f8 --- /dev/null +++ b/pyload/web/app/styles/default/main.less @@ -0,0 +1,11 @@ +@import "bootstrap/less/bootstrap"; +@import "bootstrap/less/responsive"; +@import "font-awesome/build/assets/font-awesome/less/font-awesome"; + +@FontAwesomePath: "../../fonts"; + +@import "style"; +@import "dashboard"; +@import "settings"; +@import "accounts"; +@import "admin"; diff --git a/pyload/web/app/styles/default/settings.less b/pyload/web/app/styles/default/settings.less new file mode 100644 index 000000000..34bfcb92a --- /dev/null +++ b/pyload/web/app/styles/default/settings.less @@ -0,0 +1,121 @@ +@import "common"; + +/* + Settings +*/ +.settings-menu { + background-color: #FFF; + box-shadow: 0 0 5px #000; // border: 10px solid #EEE; + + .nav-header { + background: @blueDark; + color: @light; + } + + li > a, .nav-header { + margin-left: -16px; + margin-right: -16px; + text-shadow: none; + } + + i { + margin-top: 0; + } + + .plugin, .addon { + a { + padding-left: 28px; + background-position: 4px 2px; + background-repeat: no-repeat; + background-size: 20px 20px; + } + + .icon-remove { + display: none; + } + + &:hover { + i { + display: block; + } + } + + } + + .addon { + div { + font-size: small; + } + .addon-on { + color: @green; + } + + .addon-off { + color: @red; + } + + } + + border-top-left-radius: 0; + border-top-right-radius: 0; + + .nav > li > a:hover { + color: @blueDark; + } +} + +.setting-box { + border: 10px solid @blueDark; + box-shadow: 0 0 5px @dark; // .gradient(bottom, @yellowLightest, @light); + overflow: hidden; + + .page-header { + margin: 0; + + .btn { + float: right; + margin-top: 5px; + } + + .popover { + font-size: medium; + } + + } + + // Bit wider control labels + .control-label { + width: 180px; + } + .controls { + margin-left: 200px; + } + .form-actions { + padding-left: 200px; + } + +} + +/* + Plugin select +*/ + +.plugin-select { + background-position: left 2px; + background-repeat: no-repeat; + background-size: 20px 20px; + padding-left: 24px; + + font-weight: bold; + span { + line-height: 14px; + font-size: small; + font-weight: normal; + } + +} + +.logo-select { + width: 20px; + height: 20px; +} \ No newline at end of file diff --git a/pyload/web/app/styles/default/style.less b/pyload/web/app/styles/default/style.less new file mode 100644 index 000000000..6fb5a4857 --- /dev/null +++ b/pyload/web/app/styles/default/style.less @@ -0,0 +1,366 @@ +@import "default/base"; + + +/* + Header +*/ + +header { // background-color: @greyDark; + .gradient(to bottom, #222222, #111111); + height: @header-height; + position: fixed; + top: 0; + vertical-align: top; + width: 100%; + z-index: 10; + color: #ffffff; + + a { + color: #ffffff; + } + .container-fluid, .row-fluid { + height: @header-height; + } + +} + +@header-inner-height: @header-height - 16px; + +// centered header element +.centered { + height: @header-inner-height; + margin: 8px 0; +} + +header:before { + position: absolute; + content: ' '; + top: 0; + right: 0; + bottom: 0; + left: 0; + background-color: transparent; + z-index: -1; + .default-shadow; +} + +header span.title { + color: white; + float: left; + font-family: SansationRegular, sans-serif; + font-size: 40px; + line-height: @header-height; + cursor: default; +} + +header .logo { + float: left; + margin-right: 10px; + margin-top: 10px; + width: 105px; + height: 107px; + background: url("../../images/default/logo.png") no-repeat; +} + +.header-block { + .centered; + float: left; + line-height: @header-inner-height / 3; // 3 rows + font-size: small; +} + +.status-block { + min-width: 15%; +} + +.header-btn { + float: right; + position: relative; + .centered; + + .lower { + position: absolute; + bottom: 0; + left: 0; + right: 0; + margin-left: 0; + + button { + width: 100% / 3; // 3 buttons + } + + } +} + +#progress-area { + .centered; + position: relative; + margin-top: 8px; + line-height: 16px; + + #progress-info { + padding-left: 2px; + } + + .sub { + font-size: small; + padding: 0 2px; + } + + .popover { // display: block; + max-width: none; + width: 120%; + left: -60%; // Half of width + margin-left: 50%; + top: 100%; + } + + .popover-title, .popover-content { + color: @greyDark; + } + + .icon-list { + cursor: pointer; + margin-right: 2px; // same as globalprogress margin + + &:hover { + color: @yellow; + } + } + .close { + line-height: 14px; + } +} + +.progress-list { + list-style: none; + margin: 0; + font-size: small; + + li { + background-repeat: no-repeat; + background-size: 32px 32px; + background-position: 0px 8px; + padding-left: 40px; + + &:not(:last-child) { + margin-bottom: 5px; + padding-bottom: 5px; + border-bottom: 1px dashed @greyLight; + } + + .progress { + height: 8px; + margin-bottom: 0; + + .bar { + .gradient(bottom, @blue, @blueLight); + } + } + } +} + +#globalprogress { + background-color: @greyDark; + background-image: none; + height: 8px; + margin: 4px 0; + border-radius: 8px; + border: 2px solid @grey; + + .bar { + color: @dark; + background-image: none; + background-color: @yellow; + + &.running { + width: 100%; + .stripes(@yellowLighter, @yellowDark); + } + } +} + +.speedgraph-container { + // Allows speedgraph to take up remaining space + display: block; + overflow: hidden; + padding: 0 8px; + + #speedgraph { + float: right; + width: 100%; + .centered; +// height: @header-height - 16px; +// margin: 8px 0; + font-family: sans-serif; + } +} + +.header-area { + display: none; // hidden by default + position: absolute; + bottom: -28px; + line-height: 18px; + top: @header-height; + padding: 4px 10px 6px 10px; + text-align: center; + border-radius: 0 0 6px 6px; + color: @light; + background-color: @greyDark; + .default-shadow; +} + +#notification-area { + .header-area; + left: 140px; + + .badge { + vertical-align: top; + } + + .btn-query, .btn-notification { + cursor: pointer; + } +} + +#selection-area { + .header-area; + left: 50%; + min-width: 15%; + + i { + cursor: pointer; + + &:hover { + color: @yellow; + } + } + +} + +/* + Actionbar +*/ + +.nav > li > a:hover { + color: @blue; +} + +.actionbar { + padding-bottom: 3px; + margin-bottom: 0; + border-bottom: 1px dashed @grey; + + height: @actionbar-height; + + padding-top: 2px; + margin-bottom: 5px; + +} + +.actionbar > li > a { + margin-top: 4px; +} + +.actionbar .breadcrumb { + margin: 0; + padding-top: 10px; + padding-bottom: 0; + + .active { + color: @grey; + } + +} + +.actionbar form { + margin-top: 6px; + margin-bottom: 0; +} + +.actionbar input, .actionbar button { + padding-top: 2px; + padding-bottom: 2px; +} + +.actionbar .dropdown-menu i { + margin-top: 4px; + padding-right: 5px; +} + +/* + Login +*/ +.login { + vertical-align: middle; + border: 2px solid @dark; + padding: 15px 50px; + font-size: 17px; + border-radius: 15px; + -moz-border-radius: 15px; + -webkit-border-radius: 15px; +} + + +/* + Footer +*/ +footer { // Same gradient as navbar + .gradient(top, #222222, #111111); + color: @grey; + min-height: @footer-height; + margin-top: -@footer-height; + position: relative; + width: 100%; + line-height: 16px; + z-index: 10; +} + +footer:before { + position: absolute; + content: ' '; + top: 0; + right: 0; + bottom: 0; + left: 0; + background-color: transparent; + box-shadow: 0 0 5px black; + z-index: -1; +} + +footer hr { + margin: 4px 0; + border-top-color: @greyDarker; + border-bottom-color: @greyDark; +} + +footer .span2 { + font-size: 12px; + padding-top: 12px; + padding-bottom: 12px; +} + +// This is the copyright span +footer .offset1 { + padding-top: 8px; + padding-bottom: 0; +} + +footer .copyright { + background: url(../../images/default/logo_grey.png) no-repeat; + background-size: 40px 40px; + background-position: 12px center; + height: 40px; + padding-left: 40px; + padding-top: 10px; + text-align: center; +} + +footer h2 { + color: @light; + font-family: SansationLight, sans-serif; + font-size: 16px; + font-weight: normal; + line-height: 16px; + margin: 0; +} \ No newline at end of file diff --git a/pyload/web/app/styles/font.css b/pyload/web/app/styles/font.css new file mode 100644 index 000000000..ee117d43b --- /dev/null +++ b/pyload/web/app/styles/font.css @@ -0,0 +1,37 @@ +/** + * @file + * Font styling + */ + +@font-face { + font-family: 'SansationRegular'; + src: url('../fonts/Sansation_Regular-webfont.eot'); + src: url('../fonts/Sansation_Regular-webfont.eot?#iefix') format('embedded-opentype'), + url('../fonts/Sansation_Regular-webfont.woff') format('woff'), + url('../fonts/Sansation_Regular-webfont.ttf') format('truetype'), + url('../fonts/Sansation_Regular-webfont.svg#SansationRegular') format('svg'); + font-weight: normal; + font-style: normal; +} + +@font-face { + font-family: 'SansationLight'; + src: url('../fonts/Sansation_Light-webfont.eot'); + src: url('../fonts/Sansation_Light-webfont.eot?#iefix') format('embedded-opentype'), + url('../fonts/Sansation_Light-webfont.woff') format('woff'), + url('../fonts/Sansation_Light-webfont.ttf') format('truetype'), + url('../fonts/Sansation_Light-webfont.svg#SansationLight') format('svg'); + font-weight: normal; + font-style: normal; +} + +@font-face { + font-family: 'SansationBold'; + src: url('../fonts/Sansation_Bold-webfont.eot'); + src: url('../fonts/Sansation_Bold-webfont.eot?#iefix') format('embedded-opentype'), + url('../fonts/Sansation_Bold-webfont.woff') format('woff'), + url('../fonts/Sansation_Bold-webfont.ttf') format('truetype'), + url('../fonts/Sansation_Bold-webfont.svg#SansationBold') format('svg'); + font-weight: normal; + font-style: normal; +} \ No newline at end of file diff --git a/pyload/web/app/templates/default/accounts/account.html b/pyload/web/app/templates/default/accounts/account.html new file mode 100644 index 000000000..c2ded16f6 --- /dev/null +++ b/pyload/web/app/templates/default/accounts/account.html @@ -0,0 +1,10 @@ +<% plugin %> +<% loginname %> +<% valid %> +<% premium %> +<% trafficleft %> +<% shared %> +<% activated %> + + + \ No newline at end of file diff --git a/pyload/web/app/templates/default/accounts/actionbar.html b/pyload/web/app/templates/default/accounts/actionbar.html new file mode 100644 index 000000000..f4652ec42 --- /dev/null +++ b/pyload/web/app/templates/default/accounts/actionbar.html @@ -0,0 +1,5 @@ +
          +
          + + + \ No newline at end of file diff --git a/pyload/web/app/templates/default/accounts/layout.html b/pyload/web/app/templates/default/accounts/layout.html new file mode 100644 index 000000000..e6627500d --- /dev/null +++ b/pyload/web/app/templates/default/accounts/layout.html @@ -0,0 +1,19 @@ + +
          + + + + + + + + + + + + + + + +
          PluginNameValidPremiumTrafficSharedActivatedDelete
          +
          \ No newline at end of file diff --git a/pyload/web/app/templates/default/admin.html b/pyload/web/app/templates/default/admin.html new file mode 100644 index 000000000..2eb90d7e0 --- /dev/null +++ b/pyload/web/app/templates/default/admin.html @@ -0,0 +1,223 @@ +{% extends 'default/base.html' %} + +{% block title %}{{ _("Admin") }} - {{ super() }} {% endblock %} +{% block subtitle %}{{ _("Admin") }} +{% endblock %} + +{% block css %} + + +{% endblock %} + +{% block require %} +{% endblock %} + +{% block content %} +
          +
          +
          +
          +

          Admin Bereich + Userverwaltung, Systeminfos + +

          + + + +
          + +
          + User + + +
          + + + + + +
          + +
          +
          +

          Support

          + + + + + + + + + + + + + + + + + +
          + + + Wiki | + Forum | + Chat +
          + + + Documentation +
          + + + Development +
          + + + Issue Tracker +
          +
          + System + +
          +
          +
          + + + {##} + + +{% endblock %} \ No newline at end of file diff --git a/pyload/web/app/templates/default/dashboard/actionbar.html b/pyload/web/app/templates/default/dashboard/actionbar.html new file mode 100644 index 000000000..815d4593c --- /dev/null +++ b/pyload/web/app/templates/default/dashboard/actionbar.html @@ -0,0 +1,54 @@ +
          +
          + \ No newline at end of file diff --git a/pyload/web/app/templates/default/dashboard/file.html b/pyload/web/app/templates/default/dashboard/file.html new file mode 100644 index 000000000..6256254da --- /dev/null +++ b/pyload/web/app/templates/default/dashboard/file.html @@ -0,0 +1,34 @@ +
          +   + + <% name %> + +
          +
          + <% fileStatus this %> +
          + +
          +   + <% formatSize size %> + + + <% download.plugin %>  + + + +
          \ No newline at end of file diff --git a/pyload/web/app/templates/default/dashboard/layout.html b/pyload/web/app/templates/default/dashboard/layout.html new file mode 100644 index 000000000..945d11762 --- /dev/null +++ b/pyload/web/app/templates/default/dashboard/layout.html @@ -0,0 +1,35 @@ +
          +
          + Local +
          + +
          +
          +
          +
            + +
          +
          + Shared +
          +
            +
          • Content from
            • +
          • Other user
            • +
          • which they shared
            • +
          +
          + Remote +
          +
            +
          • Content from
            • +
          • remote sites
            • +
          • mega
            • +
          • dropbox
            • +
          • other pyloads
            • +
          +
          +
          +
            + +
          +
          \ No newline at end of file diff --git a/pyload/web/app/templates/default/dashboard/package.html b/pyload/web/app/templates/default/dashboard/package.html new file mode 100644 index 000000000..c0690a9bf --- /dev/null +++ b/pyload/web/app/templates/default/dashboard/package.html @@ -0,0 +1,50 @@ +<%= if selected %> + + <% else %> + + <%/if%> + + <% name %> + + +
          +
          + video + Add Tag +
          +
          + + + + <%= if shared %> + + <% else %> + + <%/if%> + + + +
          +
          + + <% stats.linksdone %> / <% stats.linkstotal %> + +
          + + <% formatSize stats.sizedone %> / <% formatSize stats.sizetotal %> + +
          +
          \ No newline at end of file diff --git a/pyload/web/app/templates/default/dashboard/select.html b/pyload/web/app/templates/default/dashboard/select.html new file mode 100644 index 000000000..73ea391cd --- /dev/null +++ b/pyload/web/app/templates/default/dashboard/select.html @@ -0,0 +1,11 @@ +  +<%= if packs %><% packs %> package(s)<%/if %> +<%= if files %> +<%= if packs %>, <%/if %> +<% files %> file(s) +<%/if %> +selected + |  +  +  + \ No newline at end of file diff --git a/pyload/web/app/templates/default/dialogs/addAccount.html b/pyload/web/app/templates/default/dialogs/addAccount.html new file mode 100755 index 000000000..bdc8a609a --- /dev/null +++ b/pyload/web/app/templates/default/dialogs/addAccount.html @@ -0,0 +1,42 @@ +
          + +

          Add an account

          +
          +
          +
          + + Please enter your account data + +
          + + +
          + +
          +
          +
          + + +
          + +
          +
          +
          + + +
          + +
          +
          +
          +
          +
          + Add + Close +
          \ No newline at end of file diff --git a/pyload/web/app/templates/default/dialogs/addPluginConfig.html b/pyload/web/app/templates/default/dialogs/addPluginConfig.html new file mode 100755 index 000000000..e7a42a208 --- /dev/null +++ b/pyload/web/app/templates/default/dialogs/addPluginConfig.html @@ -0,0 +1,26 @@ +
          + +

          + Choose a plugin +

          +
          +
          +
          + + Please choose a plugin, which you want to configure + +
          + + +
          + +
          +
          +
          +
          +
          + Add + Close +
          \ No newline at end of file diff --git a/pyload/web/app/templates/default/dialogs/confirmDelete.html b/pyload/web/app/templates/default/dialogs/confirmDelete.html new file mode 100644 index 000000000..65ae1cb21 --- /dev/null +++ b/pyload/web/app/templates/default/dialogs/confirmDelete.html @@ -0,0 +1,11 @@ +
          + +

          Please confirm

          +
          +
          + Do you want to delete the selected items? +
          +
          + Delete + Cancel +
          \ No newline at end of file diff --git a/pyload/web/app/templates/default/dialogs/interactionTask.html b/pyload/web/app/templates/default/dialogs/interactionTask.html new file mode 100755 index 000000000..ae325e83d --- /dev/null +++ b/pyload/web/app/templates/default/dialogs/interactionTask.html @@ -0,0 +1,37 @@ +
          + +

          + <% title %> + + <% plugin %> + +

          +
          +
          +
          + <% description %> + <%= if captcha %> +
          + + +
          + +
          +
          +
          + + +
          +
          +
          + <% else %> + <% content %> + <%/if%> +
          +
          +
          + Submit + Close +
          \ No newline at end of file diff --git a/pyload/web/app/templates/default/dialogs/linkgrabber.html b/pyload/web/app/templates/default/dialogs/linkgrabber.html new file mode 100755 index 000000000..08418cf03 --- /dev/null +++ b/pyload/web/app/templates/default/dialogs/linkgrabber.html @@ -0,0 +1,49 @@ +
          + +

          + AddPackage + paste&add links to pyLoad +

          +
          + +
          + +
          +
          + + +
          + +
          +
          +
          + + +
          + +
          +
          +
          + + +
          + +
          +
          +
          + + +
          + + +
          +
          +
          +
          + +
          + Add + Close +
          \ No newline at end of file diff --git a/pyload/web/app/templates/default/dialogs/modal.html b/pyload/web/app/templates/default/dialogs/modal.html new file mode 100755 index 000000000..1e44cc99c --- /dev/null +++ b/pyload/web/app/templates/default/dialogs/modal.html @@ -0,0 +1,10 @@ +
          + +

          Dialog

          +
          +
          +
          +
          + Close + Save +
          \ No newline at end of file diff --git a/pyload/web/app/templates/default/header/layout.html b/pyload/web/app/templates/default/header/layout.html new file mode 100644 index 000000000..890a5b018 --- /dev/null +++ b/pyload/web/app/templates/default/header/layout.html @@ -0,0 +1,62 @@ +
          + + pyLoad +
          +
          +
          + + + +
          +
          +
          +

          + Running... + +

          +
          +
            +
            +
            +
            +
            +
            +
            +
            + Max. Speed:
            + Running:
            + Reconnect:
            +
            + +
            + +
            +
            + User + + +
            +
            + + + +
            +
            + +
            +             +
            \ No newline at end of file diff --git a/pyload/web/app/templates/default/header/progress.html b/pyload/web/app/templates/default/header/progress.html new file mode 100644 index 000000000..65ae9a880 --- /dev/null +++ b/pyload/web/app/templates/default/header/progress.html @@ -0,0 +1,14 @@ +<% name %> +<% plugin %> + +
            +
            +
            +<%= if downloading %> +<% formatSize done %> of <% formatSize total %> (<% formatSize download.speed %>/s) +<% else %> +<% statusmsg %> +<%/if%> + + <% formatTime eta %> + \ No newline at end of file diff --git a/pyload/web/app/templates/default/header/progressbar.html b/pyload/web/app/templates/default/header/progressbar.html new file mode 100644 index 000000000..41645c92f --- /dev/null +++ b/pyload/web/app/templates/default/header/progressbar.html @@ -0,0 +1,27 @@ + <%= if single %> + <% name %> (<% statusmsg %>) + <% else %> + <%= if downloads %> + <% downloads %> downloads running <%= if speed %>(<% formatSize speed %>/s)<%/if%> + <% else %> + No running tasks + <%/if%> + <%/if%> + + +
            + <%= if single %> +
            + <% else %> +
            + <%/if%> +
            +
            +
            + <%= if linksqueue %> + <% linksqueue %> downloads left (<% formatSize sizequeue %>) + <%/if%> + + <% formatTime etaqueue %> + +
            \ No newline at end of file diff --git a/pyload/web/app/templates/default/header/status.html b/pyload/web/app/templates/default/header/status.html new file mode 100644 index 000000000..3a22bb75b --- /dev/null +++ b/pyload/web/app/templates/default/header/status.html @@ -0,0 +1,3 @@ +<% formatSize maxspeed %>/s
            +<% paused %>
            +<%= if reconnect %>true<% else %>false<%/if%> \ No newline at end of file diff --git a/pyload/web/app/templates/default/login.html b/pyload/web/app/templates/default/login.html new file mode 100644 index 000000000..9e8d9eeb6 --- /dev/null +++ b/pyload/web/app/templates/default/login.html @@ -0,0 +1,28 @@ +
            + +
            + diff --git a/pyload/web/app/templates/default/notification.html b/pyload/web/app/templates/default/notification.html new file mode 100644 index 000000000..0f9b2c9d2 --- /dev/null +++ b/pyload/web/app/templates/default/notification.html @@ -0,0 +1,11 @@ +<%= if queries %> + +Queries <% queries %> + +<%/if%> +<%= if notifications %> + +Notifications <% notifications %> + +<%/if%> + \ No newline at end of file diff --git a/pyload/web/app/templates/default/settings/actionbar.html b/pyload/web/app/templates/default/settings/actionbar.html new file mode 100644 index 000000000..25b10d463 --- /dev/null +++ b/pyload/web/app/templates/default/settings/actionbar.html @@ -0,0 +1,5 @@ +
            +
            + + + \ No newline at end of file diff --git a/pyload/web/app/templates/default/settings/config.html b/pyload/web/app/templates/default/settings/config.html new file mode 100644 index 000000000..a9ca6214c --- /dev/null +++ b/pyload/web/app/templates/default/settings/config.html @@ -0,0 +1,17 @@ + +
            +

            <% label %> + <% description %> + <%= if long_description %> + + <%/if%> +

            +
            +             +
            +
            +
            + + +
            \ No newline at end of file diff --git a/pyload/web/app/templates/default/settings/configItem.html b/pyload/web/app/templates/default/settings/configItem.html new file mode 100644 index 000000000..3ddf16c84 --- /dev/null +++ b/pyload/web/app/templates/default/settings/configItem.html @@ -0,0 +1,7 @@ +
            + + +
            + +
            +
            \ No newline at end of file diff --git a/pyload/web/app/templates/default/settings/layout.html b/pyload/web/app/templates/default/settings/layout.html new file mode 100644 index 000000000..39f1a2ec9 --- /dev/null +++ b/pyload/web/app/templates/default/settings/layout.html @@ -0,0 +1,11 @@ +
            +
              +
            +
            +
            +
            +
            +

            Please choose a config section

            +
            +
            +
            \ No newline at end of file diff --git a/pyload/web/app/templates/default/settings/menu.html b/pyload/web/app/templates/default/settings/menu.html new file mode 100644 index 000000000..ef814414a --- /dev/null +++ b/pyload/web/app/templates/default/settings/menu.html @@ -0,0 +1,40 @@ +<%=if core%> +
          • General
            • +<%= each core%> +
          • <% label %>
            • +<%/each%> +<%/if%> +
          • +
          • Addons
            • +<%= each addon %> +
          • + + <% label %> + + <%= if activated %> +
            + active + <%else%> +
            + inactive + <%/if%> + <%= if user_context %> + + + <%else%> + + <%/if%> +
            +             +
            • +<%/each%> +
          • +
          • Plugin Configs
            • +<%= each plugin %> +
          • + + <% label %> + + +
            • +<%/each%> \ No newline at end of file diff --git a/pyload/web/app/templates/default/setup.html b/pyload/web/app/templates/default/setup.html new file mode 100644 index 000000000..e5c9f4b8c --- /dev/null +++ b/pyload/web/app/templates/default/setup.html @@ -0,0 +1,16 @@ +{% extends 'default/base.html' %} +{% block title %} + {{_("Setup")}} - {{ super()}} +{% endblock %} + +{% block content %} +
            +

            You did it!

            +

            pyLoad is running and ready for configuration.

            +

            + + Go on + +

            +
            +{% endblock %} \ No newline at end of file diff --git a/pyload/web/app/unavailable.html b/pyload/web/app/unavailable.html new file mode 100644 index 000000000..6706a693c --- /dev/null +++ b/pyload/web/app/unavailable.html @@ -0,0 +1,18 @@ + + + + WebUI not available + + + +

            WebUI not available

            +You are running a pyLoad version without prebuilt webUI. You can download a build from our website or deactivate the dev mode. +If desired you can build it yourself by running: +
              +
            • npm install
              • +
            • bower install
              • +
            • grunt build
              • +
            + + + \ No newline at end of file diff --git a/pyload/web/bower.json b/pyload/web/bower.json new file mode 100644 index 000000000..6d8adb8a7 --- /dev/null +++ b/pyload/web/bower.json @@ -0,0 +1,23 @@ +{ + "name": "pyload", + "version": "0.1.0", + "dependencies": { + "requirejs": "~2.1.6", + "requirejs-text": "*", + "require-handlebars-plugin": "*", + "jquery": "~1.9.1", + "jquery.transit": "~0.9.9", + "jquery.cookie": "~1.3.1", + "jquery.animate-enhanced": "*", + "flot": "~0.8.1", + "underscore": "~1.4.4", + "backbone": "~1.0.0", + "backbone.marionette": "~1.0.3", + "handlebars.js": "~1.0.0", + "jed": "~0.5.4", + "select2": "~3.4.0", + "bootstrap": "~2.3.2", + "font-awesome": "~3.1.1" + }, + "devDependencies": {} +} diff --git a/pyload/web/cnl_app.py b/pyload/web/cnl_app.py new file mode 100644 index 000000000..90aa76d72 --- /dev/null +++ b/pyload/web/cnl_app.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from os.path import join +import re +from urllib import unquote +from base64 import standard_b64decode +from binascii import unhexlify + +from pyload.utils.fs import save_filename + +from bottle import route, request, HTTPError +from webinterface import PYLOAD, DL_ROOT, JS + +try: + from Crypto.Cipher import AES +except: + pass + + +def local_check(function): + def _view(*args, **kwargs): + if request.environ.get('REMOTE_ADDR', "0") in ('127.0.0.1', 'localhost') \ + or request.environ.get('HTTP_HOST','0') in ('127.0.0.1:9666', 'localhost:9666'): + return function(*args, **kwargs) + else: + return HTTPError(403, "Forbidden") + + return _view + + +@route("/flash") +@route("/flash/:id") +@route("/flash", method="POST") +@local_check +def flash(id="0"): + return "JDownloader\r\n" + +@route("/flash/add", method="POST") +@local_check +def add(request): + package = request.POST.get('referer', None) + urls = filter(lambda x: x != "", request.POST['urls'].split("\n")) + + if package: + PYLOAD.addPackage(package, urls, 0) + else: + PYLOAD.generateAndAddPackages(urls, 0) + + return "" + +@route("/flash/addcrypted", method="POST") +@local_check +def addcrypted(): + + package = request.forms.get('referer', 'ClickAndLoad Package') + dlc = request.forms['crypted'].replace(" ", "+") + + dlc_path = join(DL_ROOT, save_filename(package) + ".dlc") + dlc_file = open(dlc_path, "wb") + dlc_file.write(dlc) + dlc_file.close() + + try: + PYLOAD.addPackage(package, [dlc_path], 0) + except: + return HTTPError() + else: + return "success\r\n" + +@route("/flash/addcrypted2", method="POST") +@local_check +def addcrypted2(): + + package = request.forms.get("source", None) + crypted = request.forms["crypted"] + jk = request.forms["jk"] + + crypted = standard_b64decode(unquote(crypted.replace(" ", "+"))) + if JS: + jk = "%s f()" % jk + jk = JS.eval(jk) + + else: + try: + jk = re.findall(r"return ('|\")(.+)('|\")", jk)[0][1] + except: + ## Test for some known js functions to decode + if jk.find("dec") > -1 and jk.find("org") > -1: + org = re.findall(r"var org = ('|\")([^\"']+)", jk)[0][1] + jk = list(org) + jk.reverse() + jk = "".join(jk) + else: + print "Could not decrypt key, please install py-spidermonkey or ossp-js" + + try: + Key = unhexlify(jk) + except: + print "Could not decrypt key, please install py-spidermonkey or ossp-js" + return "failed" + + IV = Key + + obj = AES.new(Key, AES.MODE_CBC, IV) + result = obj.decrypt(crypted).replace("\x00", "").replace("\r","").split("\n") + + result = filter(lambda x: x != "", result) + + try: + if package: + PYLOAD.addPackage(package, result, 0) + else: + PYLOAD.generateAndAddPackages(result, 0) + except: + return "failed can't add" + else: + return "success\r\n" + +@route("/flashgot_pyload") +@route("/flashgot_pyload", method="POST") +@route("/flashgot") +@route("/flashgot", method="POST") +@local_check +def flashgot(): + if request.environ['HTTP_REFERER'] != "http://localhost:9666/flashgot" and request.environ['HTTP_REFERER'] != "http://127.0.0.1:9666/flashgot": + return HTTPError() + + autostart = int(request.forms.get('autostart', 0)) + package = request.forms.get('package', None) + urls = filter(lambda x: x != "", request.forms['urls'].split("\n")) + folder = request.forms.get('dir', None) + + if package: + PYLOAD.addPackage(package, urls, autostart) + else: + PYLOAD.generateAndAddPackages(urls, autostart) + + return "" + +@route("/crossdomain.xml") +@local_check +def crossdomain(): + rep = "\n" + rep += "\n" + rep += "\n" + rep += "\n" + rep += "" + return rep + + +@route("/flash/checkSupportForUrl") +@local_check +def checksupport(): + + url = request.GET.get("url") + res = PYLOAD.checkURLs([url]) + supported = (not res[0][1] is None) + + return str(supported).lower() + +@route("/jdcheck.js") +@local_check +def jdcheck(): + rep = "jdownloader=true;\n" + rep += "var version='9.581;'" + return rep diff --git a/pyload/web/middlewares.py b/pyload/web/middlewares.py new file mode 100644 index 000000000..ae0911cc3 --- /dev/null +++ b/pyload/web/middlewares.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# gzip is optional on some platform +try: + import gzip +except ImportError: + gzip = None + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + +class StripPathMiddleware(object): + def __init__(self, app): + self.app = app + + def __call__(self, e, h): + e['PATH_INFO'] = e['PATH_INFO'].rstrip('/') + return self.app(e, h) + + +class PrefixMiddleware(object): + def __init__(self, app, prefix="/pyload"): + self.app = app + self.prefix = prefix + + def __call__(self, e, h): + path = e["PATH_INFO"] + if path.startswith(self.prefix): + e['PATH_INFO'] = path.replace(self.prefix, "", 1) + return self.app(e, h) + +# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org) +# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php + +# WSGI middleware +# Gzip-encodes the response. + +class GZipMiddleWare(object): + + def __init__(self, application, compress_level=6): + self.application = application + self.compress_level = int(compress_level) + + def __call__(self, environ, start_response): + if 'gzip' not in environ.get('HTTP_ACCEPT_ENCODING', ''): + # nothing for us to do, so this middleware will + # be a no-op: + return self.application(environ, start_response) + response = GzipResponse(start_response, self.compress_level) + app_iter = self.application(environ, + response.gzip_start_response) + if app_iter is not None: + response.finish_response(app_iter) + + return response.write() + +def header_value(headers, key): + for header, value in headers: + if key.lower() == header.lower(): + return value + +def update_header(headers, key, value): + remove_header(headers, key) + headers.append((key, value)) + +def remove_header(headers, key): + for header, value in headers: + if key.lower() == header.lower(): + headers.remove((header, value)) + break + +class GzipResponse(object): + + def __init__(self, start_response, compress_level): + self.start_response = start_response + self.compress_level = compress_level + self.buffer = StringIO() + self.compressible = False + self.content_length = None + self.headers = () + + def gzip_start_response(self, status, headers, exc_info=None): + self.headers = headers + ct = header_value(headers,'content-type') + ce = header_value(headers,'content-encoding') + cl = header_value(headers, 'content-length') + + # don't compress on unknown size, it may be too huge + cl = int(cl) if cl else 0 + + if ce: + self.compressible = False + elif gzip is not None and ct and (ct.startswith('text/') or ct.startswith('application/')) \ + and 'zip' not in ct and 200 < cl < 1024*1024: + self.compressible = True + headers.append(('content-encoding', 'gzip')) + headers.append(('vary', 'Accept-Encoding')) + + remove_header(headers, 'content-length') + self.headers = headers + self.status = status + return self.buffer.write + + def write(self): + out = self.buffer + out.seek(0) + s = out.getvalue() + out.close() + return [s] + + def finish_response(self, app_iter): + if self.compressible: + output = gzip.GzipFile(mode='wb', compresslevel=self.compress_level, + fileobj=self.buffer) + else: + output = self.buffer + try: + for s in app_iter: + output.write(s) + if self.compressible: + output.close() + finally: + if hasattr(app_iter, 'close'): + try: + app_iter.close() + except : + pass + + content_length = self.buffer.tell() + update_header(self.headers, "Content-Length" , str(content_length)) + self.start_response(self.status, self.headers) \ No newline at end of file diff --git a/pyload/web/package.json b/pyload/web/package.json new file mode 100644 index 000000000..aaf1c9a8d --- /dev/null +++ b/pyload/web/package.json @@ -0,0 +1,32 @@ +{ + "name": "pyload", + "version": "0.1.0", + "dependencies": {}, + "devDependencies": { + "grunt": "~0.4.1", + "grunt-contrib-copy": "~0.4.1", + "grunt-contrib-concat": "~0.1.3", + "grunt-contrib-uglify": "~0.2.0", + "grunt-contrib-jshint": "~0.4.1", + "grunt-contrib-less": "~0.5.2", + "grunt-contrib-cssmin": "~0.6.0", + "grunt-contrib-connect": "~0.2.0", + "grunt-contrib-clean": "~0.4.0", + "grunt-contrib-htmlmin": "~0.1.3", + "grunt-bower-requirejs": "~0.4.3", + "grunt-contrib-requirejs": "~0.4.0", + "grunt-contrib-imagemin": "~0.1.3", + "grunt-contrib-watch": "~0.4.0", + "grunt-rev": "~0.1.0", + "grunt-usemin": "~0.1.10", + "grunt-mocha": "~0.3.0", + "grunt-open": "~0.2.0", + "grunt-svgmin": "~0.1.0", + "grunt-concurrent": "~0.1.0", + "matchdep": "~0.1.1", + "connect-livereload": "~0.1.1" + }, + "engines": { + "node": ">=0.8.0" + } +} diff --git a/pyload/web/pyload_app.py b/pyload/web/pyload_app.py new file mode 100644 index 000000000..724fddec2 --- /dev/null +++ b/pyload/web/pyload_app.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" +import time +from os.path import join, exists + +from bottle import route, static_file, response, redirect + +from webinterface import PROJECT_DIR, SETUP, DEVELOP + +from utils import login_required + +########## +# Helper +########## + +app_path = "app" +UNAVAILALBE = False + +# webUI build is available +if exists(join(PROJECT_DIR, "dist", "index.html")) and not DEVELOP: + app_path = "dist" +elif not exists(join(PROJECT_DIR, "app", "components")) or not exists(join(PROJECT_DIR, ".tmp")): + UNAVAILALBE = True + + +@route('/icons/') +def serve_icon(path): + # TODO + return redirect('/images/icon.png') + # return static_file(path, root=join("tmp", "icons")) + +@route("/download/:fid") +@login_required('Download') +def download(fid, api): + path, name = api.getFilePath(fid) + return static_file(name, path, download=True) + + +@route('/') +def index(): + if UNAVAILALBE: + return server_static("unavailable.html") + + if SETUP: + # TODO show different page + pass + + # TODO: render it as simple template with configuration + return server_static('index.html') + +# Very last route that is registered, could match all uris +@route('/') +def server_static(path): + response.headers['Expires'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", + time.gmtime(time.time() + 60 * 60 * 24 * 7)) + response.headers['Cache-control'] = "public" + resp = static_file(path, root=join(PROJECT_DIR, app_path)) + # Also serve from .tmp folder in dev mode + if resp.status_code == 404 and app_path == "app": + return static_file(path, root=join(PROJECT_DIR, '.tmp')) + + return resp \ No newline at end of file diff --git a/pyload/web/servers.py b/pyload/web/servers.py new file mode 100644 index 000000000..a3c51e36b --- /dev/null +++ b/pyload/web/servers.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from bottle import ServerAdapter as BaseAdapter + +class ServerAdapter(BaseAdapter): + SSL = False + NAME = "" + + def __init__(self, host, port, key, cert, connections, debug, **kwargs): + BaseAdapter.__init__(self, host, port, **kwargs) + self.key = key + self.cert = cert + self.connection = connections + self.debug = debug + + @classmethod + def find(cls): + """ Check if server is available by trying to import it + + :raises Exception: importing C dependant library could also fail with other reasons + :return: True on success + """ + try: + __import__(cls.NAME) + return True + except ImportError: + return False + + def run(self, handler): + raise NotImplementedError + + +class CherryPyWSGI(ServerAdapter): + SSL = True + NAME = "threaded" + + @classmethod + def find(cls): + return True + + def run(self, handler): + from wsgiserver import CherryPyWSGIServer + + if self.cert and self.key: + CherryPyWSGIServer.ssl_certificate = self.cert + CherryPyWSGIServer.ssl_private_key = self.key + + server = CherryPyWSGIServer((self.host, self.port), handler, numthreads=self.connection) + server.start() + + +class FapwsServer(ServerAdapter): + """ Does not work very good currently """ + + NAME = "fapws" + + def run(self, handler): # pragma: no cover + import fapws._evwsgi as evwsgi + from fapws import base, config + + port = self.port + if float(config.SERVER_IDENT[-2:]) > 0.4: + # fapws3 silently changed its API in 0.5 + port = str(port) + evwsgi.start(self.host, port) + evwsgi.set_base_module(base) + + def app(environ, start_response): + environ['wsgi.multiprocess'] = False + return handler(environ, start_response) + + evwsgi.wsgi_cb(('', app)) + evwsgi.run() + + +# TODO: ssl +class MeinheldServer(ServerAdapter): + SSL = True + NAME = "meinheld" + + def run(self, handler): + from meinheld import server + + if self.quiet: + server.set_access_logger(None) + server.set_error_logger(None) + + server.listen((self.host, self.port)) + server.run(handler) + +# todo:ssl +class TornadoServer(ServerAdapter): + """ The super hyped asynchronous server by facebook. Untested. """ + + SSL = True + NAME = "tornado" + + def run(self, handler): # pragma: no cover + import tornado.wsgi, tornado.httpserver, tornado.ioloop + + container = tornado.wsgi.WSGIContainer(handler) + server = tornado.httpserver.HTTPServer(container) + server.listen(port=self.port) + tornado.ioloop.IOLoop.instance().start() + + +class BjoernServer(ServerAdapter): + """ Fast server written in C: https://github.com/jonashaag/bjoern """ + + NAME = "bjoern" + + def run(self, handler): + from bjoern import run + + run(handler, self.host, self.port) + + +# todo: ssl +class EventletServer(ServerAdapter): + + SSL = True + NAME = "eventlet" + + def run(self, handler): + from eventlet import wsgi, listen + + try: + wsgi.server(listen((self.host, self.port)), handler, + log_output=(not self.quiet)) + except TypeError: + # Needed to ignore the log + class NoopLog: + def write(self, *args): + pass + + # Fallback, if we have old version of eventlet + wsgi.server(listen((self.host, self.port)), handler, log=NoopLog()) + + +class FlupFCGIServer(ServerAdapter): + + SSL = False + NAME = "flup" + + def run(self, handler): # pragma: no cover + import flup.server.fcgi + from flup.server.threadedserver import ThreadedServer + + def noop(*args, **kwargs): + pass + + # Monkey patch signal handler, it does not work from threads + ThreadedServer._installSignalHandlers = noop + + self.options.setdefault('bindAddress', (self.host, self.port)) + flup.server.fcgi.WSGIServer(handler, **self.options).run() + +# Order is important and gives every server precedence over others! +all_server = [BjoernServer, TornadoServer, EventletServer, CherryPyWSGI] +# Some are deactivated because they have some flaws +##all_server = [FapwsServer, MeinheldServer, BjoernServer, TornadoServer, EventletServer, CherryPyWSGI] \ No newline at end of file diff --git a/pyload/web/setup_app.py b/pyload/web/setup_app.py new file mode 100644 index 000000000..cd44ad08e --- /dev/null +++ b/pyload/web/setup_app.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from bottle import route, request, response, HTTPError, redirect + +from webinterface import PROJECT_DIR, SETUP + +def setup_required(func): + def _view(*args, **kwargs): + # setup needs to be running + if SETUP is None: + redirect("/nopermission") + + return func(*args, **kwargs) + return _view + + +@route("/setup") +@setup_required +def setup(): + pass # TODO diff --git a/pyload/web/utils.py b/pyload/web/utils.py new file mode 100644 index 000000000..b5a933b26 --- /dev/null +++ b/pyload/web/utils.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +from bottle import request, HTTPError, redirect + +from webinterface import PYLOAD, SETUP + + +def set_session(request, user): + s = request.environ.get('beaker.session') + s["uid"] = user.uid + s.save() + return s + +def get_user_api(s): + if s: + uid = s.get("uid", None) + if (uid is not None) and (PYLOAD is not None): + return PYLOAD.withUserContext(uid) + return None + +def is_mobile(): + if request.get_cookie("mobile"): + if request.get_cookie("mobile") == "True": + return True + else: + return False + mobile_ua = request.headers.get('User-Agent', '').lower() + if mobile_ua.find('opera mini') > 0: + return True + if mobile_ua.find('windows') > 0: + return False + if request.headers.get('Accept', '').lower().find('application/vnd.wap.xhtml+xml') > 0: + return True + if re.search('(up.browser|up.link|mmp|symbian|smartphone|midp|wap|phone|android)', mobile_ua) is not None: + return True + mobile_ua = mobile_ua[:4] + mobile_agents = ['w3c ','acs-','alav','alca','amoi','audi','avan','benq','bird','blac','blaz','brew','cell','cldc','cmd-', + 'dang','doco','eric','hipt','inno','ipaq','java','jigs','kddi','keji','leno','lg-c','lg-d','lg-g','lge-', + 'maui','maxo','midp','mits','mmef','mobi','mot-','moto','mwbp','nec-','newt','noki','palm','pana','pant', + 'phil','play','port','prox','qwap','sage','sams','sany','sch-','sec-','send','seri','sgh-','shar','sie-', + 'siem','smal','smar','sony','sph-','symb','t-mo','teli','tim-','tosh','tsm-','upg1','upsi','vk-v','voda', + 'wap-','wapa','wapi','wapp','wapr','webc','winw','winw','xda ','xda-'] + if mobile_ua in mobile_agents: + return True + return False + + +def login_required(perm=None): + def _dec(func): + def _view(*args, **kwargs): + + # In case of setup, no login methods can be accessed + if SETUP is not None: + redirect("/setup") + + s = request.environ.get('beaker.session') + api = get_user_api(s) + if api is not None: + if perm: + if api.user.hasPermission(perm): + if request.headers.get('X-Requested-With') == 'XMLHttpRequest': + return HTTPError(403, "Forbidden") + else: + return redirect("/nopermission") + + kwargs["api"] = api + return func(*args, **kwargs) + else: + if request.headers.get('X-Requested-With') == 'XMLHttpRequest': + return HTTPError(403, "Forbidden") + else: + return redirect("/login") + + return _view + + return _dec diff --git a/pyload/web/webinterface.py b/pyload/web/webinterface.py new file mode 100644 index 000000000..37d06bd68 --- /dev/null +++ b/pyload/web/webinterface.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +import sys + +from os.path import join, abspath, dirname + +PROJECT_DIR = abspath(dirname(__file__)) +PYLOAD_DIR = abspath(join(PROJECT_DIR, "..", "..")) + +sys.path.append(PYLOAD_DIR) + +from pyload import InitHomeDir + +import bottle +from bottle import run, app + +from middlewares import StripPathMiddleware, GZipMiddleWare, PrefixMiddleware + +SETUP = None +PYLOAD = None + +import ServerThread + +if not ServerThread.core: + if ServerThread.setup: + SETUP = ServerThread.setup + config = SETUP.config + else: + raise Exception("Could not access pyLoad Core") +else: + PYLOAD = ServerThread.core.api + config = ServerThread.core.config + +from pyload.utils.JsEngine import JsEngine + +JS = JsEngine() + +TEMPLATE = config.get('webinterface', 'template') +DL_ROOT = config.get('general', 'download_folder') +PREFIX = config.get('webinterface', 'prefix') +DEVELOP = config.get('webinterface', 'develop') + +if PREFIX: + PREFIX = PREFIX.rstrip("/") + if PREFIX and not PREFIX.startswith("/"): + PREFIX = "/" + PREFIX + +DEBUG = config.get("general", "debug_mode") or "-d" in sys.argv or "--debug" in sys.argv +bottle.debug(DEBUG) + + +# Middlewares +from beaker.middleware import SessionMiddleware + +session_opts = { + 'session.type': 'file', + 'session.cookie_expires': False, + 'session.data_dir': './tmp', + 'session.auto': False +} + +session = SessionMiddleware(app(), session_opts) +web = StripPathMiddleware(session) +web = GZipMiddleWare(web) + +if PREFIX: + web = PrefixMiddleware(web, prefix=PREFIX) + +import api_app +import cnl_app +import setup_app +# Last routes to register, +import pyload_app + +# Server Adapter +def run_server(host, port, server): + run(app=web, host=host, port=port, quiet=True, server=server) + + +if __name__ == "__main__": + run(app=web, port=8001) -- cgit v1.2.3