From 7fc24c10cd3dad19a69392398671a78abecd71d1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 3 Oct 2014 21:28:28 +0200 Subject: Remove lib --- pyload/lib/BeautifulSoup.py | 2017 ---------- pyload/lib/Getch.py | 76 - pyload/lib/MultipartPostHandler.py | 139 - pyload/lib/SafeEval.py | 47 - pyload/lib/__init__.py | 0 pyload/lib/beaker/__init__.py | 1 - pyload/lib/beaker/cache.py | 589 --- pyload/lib/beaker/container.py | 750 ---- pyload/lib/beaker/converters.py | 29 - pyload/lib/beaker/crypto/__init__.py | 44 - pyload/lib/beaker/crypto/jcecrypto.py | 32 - pyload/lib/beaker/crypto/nsscrypto.py | 45 - pyload/lib/beaker/crypto/pbkdf2.py | 347 -- pyload/lib/beaker/crypto/pycrypto.py | 34 - pyload/lib/beaker/crypto/util.py | 30 - pyload/lib/beaker/exceptions.py | 29 - pyload/lib/beaker/ext/__init__.py | 0 pyload/lib/beaker/ext/database.py | 174 - pyload/lib/beaker/ext/google.py | 121 - pyload/lib/beaker/ext/memcached.py | 203 - pyload/lib/beaker/ext/sqla.py | 136 - pyload/lib/beaker/middleware.py | 168 - pyload/lib/beaker/session.py | 726 ---- pyload/lib/beaker/synchronization.py | 386 -- pyload/lib/beaker/util.py | 462 --- pyload/lib/bottle.py | 3732 ------------------ pyload/lib/colorama/__init__.py | 7 - pyload/lib/colorama/ansi.py | 50 - pyload/lib/colorama/ansitowin32.py | 191 - pyload/lib/colorama/initialise.py | 66 - pyload/lib/colorama/win32.py | 136 - pyload/lib/colorama/winterm.py | 120 - pyload/lib/feedparser.py | 4013 -------------------- pyload/lib/jinja2/__init__.py | 69 - pyload/lib/jinja2/_compat.py | 150 - pyload/lib/jinja2/_stringdefs.py | 132 - pyload/lib/jinja2/bccache.py | 344 -- pyload/lib/jinja2/compiler.py | 1640 -------- pyload/lib/jinja2/constants.py | 32 - pyload/lib/jinja2/debug.py | 337 -- pyload/lib/jinja2/defaults.py | 43 - pyload/lib/jinja2/environment.py | 1191 ------ pyload/lib/jinja2/exceptions.py | 146 - pyload/lib/jinja2/ext.py | 636 ---- pyload/lib/jinja2/filters.py | 987 ----- pyload/lib/jinja2/lexer.py | 733 ---- pyload/lib/jinja2/loaders.py | 471 --- pyload/lib/jinja2/meta.py | 103 - pyload/lib/jinja2/nodes.py | 914 ----- pyload/lib/jinja2/optimizer.py | 68 - pyload/lib/jinja2/parser.py | 895 ----- pyload/lib/jinja2/runtime.py | 581 --- pyload/lib/jinja2/sandbox.py | 368 -- pyload/lib/jinja2/tests.py | 149 - pyload/lib/jinja2/testsuite/__init__.py | 156 - pyload/lib/jinja2/testsuite/api.py | 261 -- pyload/lib/jinja2/testsuite/bytecode_cache.py | 37 - pyload/lib/jinja2/testsuite/core_tags.py | 305 -- pyload/lib/jinja2/testsuite/debug.py | 58 - pyload/lib/jinja2/testsuite/doctests.py | 29 - pyload/lib/jinja2/testsuite/ext.py | 459 --- pyload/lib/jinja2/testsuite/filters.py | 515 --- pyload/lib/jinja2/testsuite/imports.py | 141 - pyload/lib/jinja2/testsuite/inheritance.py | 250 -- pyload/lib/jinja2/testsuite/lexnparse.py | 593 --- pyload/lib/jinja2/testsuite/loader.py | 226 -- pyload/lib/jinja2/testsuite/regression.py | 279 -- pyload/lib/jinja2/testsuite/res/__init__.py | 0 .../lib/jinja2/testsuite/res/templates/broken.html | 3 - .../jinja2/testsuite/res/templates/foo/test.html | 1 - .../testsuite/res/templates/syntaxerror.html | 4 - .../lib/jinja2/testsuite/res/templates/test.html | 1 - pyload/lib/jinja2/testsuite/security.py | 166 - pyload/lib/jinja2/testsuite/tests.py | 93 - pyload/lib/jinja2/testsuite/utils.py | 82 - pyload/lib/jinja2/utils.py | 520 --- pyload/lib/jinja2/visitor.py | 87 - pyload/lib/markupsafe/__init__.py | 298 -- pyload/lib/markupsafe/_compat.py | 26 - pyload/lib/markupsafe/_constants.py | 267 -- pyload/lib/markupsafe/_native.py | 46 - pyload/lib/markupsafe/_speedups.c | 239 -- pyload/lib/markupsafe/tests.py | 179 - pyload/lib/rename_process.py | 14 - pyload/lib/simplejson/__init__.py | 560 --- pyload/lib/simplejson/_speedups.c | 3339 ---------------- pyload/lib/simplejson/compat.py | 46 - pyload/lib/simplejson/decoder.py | 400 -- pyload/lib/simplejson/encoder.py | 648 ---- pyload/lib/simplejson/ordered_dict.py | 119 - pyload/lib/simplejson/scanner.py | 133 - pyload/lib/simplejson/tests/__init__.py | 88 - .../lib/simplejson/tests/test_bigint_as_string.py | 67 - .../simplejson/tests/test_bitsize_int_as_string.py | 73 - pyload/lib/simplejson/tests/test_check_circular.py | 30 - pyload/lib/simplejson/tests/test_decimal.py | 71 - pyload/lib/simplejson/tests/test_decode.py | 99 - pyload/lib/simplejson/tests/test_default.py | 9 - pyload/lib/simplejson/tests/test_dump.py | 121 - .../tests/test_encode_basestring_ascii.py | 47 - .../lib/simplejson/tests/test_encode_for_html.py | 30 - pyload/lib/simplejson/tests/test_errors.py | 51 - pyload/lib/simplejson/tests/test_fail.py | 176 - pyload/lib/simplejson/tests/test_float.py | 35 - pyload/lib/simplejson/tests/test_for_json.py | 97 - pyload/lib/simplejson/tests/test_indent.py | 86 - pyload/lib/simplejson/tests/test_item_sort_key.py | 20 - pyload/lib/simplejson/tests/test_namedtuple.py | 122 - pyload/lib/simplejson/tests/test_pass1.py | 71 - pyload/lib/simplejson/tests/test_pass2.py | 14 - pyload/lib/simplejson/tests/test_pass3.py | 20 - pyload/lib/simplejson/tests/test_recursion.py | 67 - pyload/lib/simplejson/tests/test_scanstring.py | 194 - pyload/lib/simplejson/tests/test_separators.py | 42 - pyload/lib/simplejson/tests/test_speedups.py | 39 - pyload/lib/simplejson/tests/test_tool.py | 97 - pyload/lib/simplejson/tests/test_tuple.py | 51 - pyload/lib/simplejson/tests/test_unicode.py | 153 - pyload/lib/simplejson/tool.py | 42 - pyload/lib/thrift/TSCons.py | 35 - pyload/lib/thrift/TSerialization.py | 38 - pyload/lib/thrift/TTornado.py | 153 - pyload/lib/thrift/Thrift.py | 170 - pyload/lib/thrift/__init__.py | 20 - pyload/lib/thrift/protocol/TBase.py | 81 - pyload/lib/thrift/protocol/TBinaryProtocol.py | 260 -- pyload/lib/thrift/protocol/TCompactProtocol.py | 403 -- pyload/lib/thrift/protocol/TJSONProtocol.py | 550 --- pyload/lib/thrift/protocol/TProtocol.py | 406 -- pyload/lib/thrift/protocol/__init__.py | 20 - pyload/lib/thrift/protocol/fastbinary.c | 1219 ------ pyload/lib/thrift/server/THttpServer.py | 87 - pyload/lib/thrift/server/TNonblockingServer.py | 346 -- pyload/lib/thrift/server/TProcessPoolServer.py | 118 - pyload/lib/thrift/server/TServer.py | 269 -- pyload/lib/thrift/server/__init__.py | 20 - pyload/lib/thrift/transport/THttpClient.py | 149 - pyload/lib/thrift/transport/TSSLSocket.py | 214 -- pyload/lib/thrift/transport/TSocket.py | 176 - pyload/lib/thrift/transport/TTransport.py | 330 -- pyload/lib/thrift/transport/TTwisted.py | 221 -- pyload/lib/thrift/transport/TZlibTransport.py | 248 -- pyload/lib/thrift/transport/__init__.py | 20 - pyload/lib/wsgiserver.py | 2299 ----------- 144 files changed, 45333 deletions(-) delete mode 100644 pyload/lib/BeautifulSoup.py delete mode 100644 pyload/lib/Getch.py delete mode 100644 pyload/lib/MultipartPostHandler.py delete mode 100644 pyload/lib/SafeEval.py delete mode 100644 pyload/lib/__init__.py delete mode 100644 pyload/lib/beaker/__init__.py delete mode 100644 pyload/lib/beaker/cache.py delete mode 100644 pyload/lib/beaker/container.py delete mode 100644 pyload/lib/beaker/converters.py delete mode 100644 pyload/lib/beaker/crypto/__init__.py delete mode 100644 pyload/lib/beaker/crypto/jcecrypto.py delete mode 100644 pyload/lib/beaker/crypto/nsscrypto.py delete mode 100644 pyload/lib/beaker/crypto/pbkdf2.py delete mode 100644 pyload/lib/beaker/crypto/pycrypto.py delete mode 100644 pyload/lib/beaker/crypto/util.py delete mode 100644 pyload/lib/beaker/exceptions.py delete mode 100644 pyload/lib/beaker/ext/__init__.py delete mode 100644 pyload/lib/beaker/ext/database.py delete mode 100644 pyload/lib/beaker/ext/google.py delete mode 100644 pyload/lib/beaker/ext/memcached.py delete mode 100644 pyload/lib/beaker/ext/sqla.py delete mode 100644 pyload/lib/beaker/middleware.py delete mode 100644 pyload/lib/beaker/session.py delete mode 100644 pyload/lib/beaker/synchronization.py delete mode 100644 pyload/lib/beaker/util.py delete mode 100644 pyload/lib/bottle.py delete mode 100644 pyload/lib/colorama/__init__.py delete mode 100644 pyload/lib/colorama/ansi.py delete mode 100644 pyload/lib/colorama/ansitowin32.py delete mode 100644 pyload/lib/colorama/initialise.py delete mode 100644 pyload/lib/colorama/win32.py delete mode 100644 pyload/lib/colorama/winterm.py delete mode 100644 pyload/lib/feedparser.py delete mode 100644 pyload/lib/jinja2/__init__.py delete mode 100644 pyload/lib/jinja2/_compat.py delete mode 100644 pyload/lib/jinja2/_stringdefs.py delete mode 100644 pyload/lib/jinja2/bccache.py delete mode 100644 pyload/lib/jinja2/compiler.py delete mode 100644 pyload/lib/jinja2/constants.py delete mode 100644 pyload/lib/jinja2/debug.py delete mode 100644 pyload/lib/jinja2/defaults.py delete mode 100644 pyload/lib/jinja2/environment.py delete mode 100644 pyload/lib/jinja2/exceptions.py delete mode 100644 pyload/lib/jinja2/ext.py delete mode 100644 pyload/lib/jinja2/filters.py delete mode 100644 pyload/lib/jinja2/lexer.py delete mode 100644 pyload/lib/jinja2/loaders.py delete mode 100644 pyload/lib/jinja2/meta.py delete mode 100644 pyload/lib/jinja2/nodes.py delete mode 100644 pyload/lib/jinja2/optimizer.py delete mode 100644 pyload/lib/jinja2/parser.py delete mode 100644 pyload/lib/jinja2/runtime.py delete mode 100644 pyload/lib/jinja2/sandbox.py delete mode 100644 pyload/lib/jinja2/tests.py delete mode 100644 pyload/lib/jinja2/testsuite/__init__.py delete mode 100644 pyload/lib/jinja2/testsuite/api.py delete mode 100644 pyload/lib/jinja2/testsuite/bytecode_cache.py delete mode 100644 pyload/lib/jinja2/testsuite/core_tags.py delete mode 100644 pyload/lib/jinja2/testsuite/debug.py delete mode 100644 pyload/lib/jinja2/testsuite/doctests.py delete mode 100644 pyload/lib/jinja2/testsuite/ext.py delete mode 100644 pyload/lib/jinja2/testsuite/filters.py delete mode 100644 pyload/lib/jinja2/testsuite/imports.py delete mode 100644 pyload/lib/jinja2/testsuite/inheritance.py delete mode 100644 pyload/lib/jinja2/testsuite/lexnparse.py delete mode 100644 pyload/lib/jinja2/testsuite/loader.py delete mode 100644 pyload/lib/jinja2/testsuite/regression.py delete mode 100644 pyload/lib/jinja2/testsuite/res/__init__.py delete mode 100644 pyload/lib/jinja2/testsuite/res/templates/broken.html delete mode 100644 pyload/lib/jinja2/testsuite/res/templates/foo/test.html delete mode 100644 pyload/lib/jinja2/testsuite/res/templates/syntaxerror.html delete mode 100644 pyload/lib/jinja2/testsuite/res/templates/test.html delete mode 100644 pyload/lib/jinja2/testsuite/security.py delete mode 100644 pyload/lib/jinja2/testsuite/tests.py delete mode 100644 pyload/lib/jinja2/testsuite/utils.py delete mode 100644 pyload/lib/jinja2/utils.py delete mode 100644 pyload/lib/jinja2/visitor.py delete mode 100644 pyload/lib/markupsafe/__init__.py delete mode 100644 pyload/lib/markupsafe/_compat.py delete mode 100644 pyload/lib/markupsafe/_constants.py delete mode 100644 pyload/lib/markupsafe/_native.py delete mode 100644 pyload/lib/markupsafe/_speedups.c delete mode 100644 pyload/lib/markupsafe/tests.py delete mode 100644 pyload/lib/rename_process.py delete mode 100644 pyload/lib/simplejson/__init__.py delete mode 100644 pyload/lib/simplejson/_speedups.c delete mode 100644 pyload/lib/simplejson/compat.py delete mode 100644 pyload/lib/simplejson/decoder.py delete mode 100644 pyload/lib/simplejson/encoder.py delete mode 100644 pyload/lib/simplejson/ordered_dict.py delete mode 100644 pyload/lib/simplejson/scanner.py delete mode 100644 pyload/lib/simplejson/tests/__init__.py delete mode 100644 pyload/lib/simplejson/tests/test_bigint_as_string.py delete mode 100644 pyload/lib/simplejson/tests/test_bitsize_int_as_string.py delete mode 100644 pyload/lib/simplejson/tests/test_check_circular.py delete mode 100644 pyload/lib/simplejson/tests/test_decimal.py delete mode 100644 pyload/lib/simplejson/tests/test_decode.py delete mode 100644 pyload/lib/simplejson/tests/test_default.py delete mode 100644 pyload/lib/simplejson/tests/test_dump.py delete mode 100644 pyload/lib/simplejson/tests/test_encode_basestring_ascii.py delete mode 100644 pyload/lib/simplejson/tests/test_encode_for_html.py delete mode 100644 pyload/lib/simplejson/tests/test_errors.py delete mode 100644 pyload/lib/simplejson/tests/test_fail.py delete mode 100644 pyload/lib/simplejson/tests/test_float.py delete mode 100644 pyload/lib/simplejson/tests/test_for_json.py delete mode 100644 pyload/lib/simplejson/tests/test_indent.py delete mode 100644 pyload/lib/simplejson/tests/test_item_sort_key.py delete mode 100644 pyload/lib/simplejson/tests/test_namedtuple.py delete mode 100644 pyload/lib/simplejson/tests/test_pass1.py delete mode 100644 pyload/lib/simplejson/tests/test_pass2.py delete mode 100644 pyload/lib/simplejson/tests/test_pass3.py delete mode 100644 pyload/lib/simplejson/tests/test_recursion.py delete mode 100644 pyload/lib/simplejson/tests/test_scanstring.py delete mode 100644 pyload/lib/simplejson/tests/test_separators.py delete mode 100644 pyload/lib/simplejson/tests/test_speedups.py delete mode 100644 pyload/lib/simplejson/tests/test_tool.py delete mode 100644 pyload/lib/simplejson/tests/test_tuple.py delete mode 100644 pyload/lib/simplejson/tests/test_unicode.py delete mode 100644 pyload/lib/simplejson/tool.py delete mode 100644 pyload/lib/thrift/TSCons.py delete mode 100644 pyload/lib/thrift/TSerialization.py delete mode 100644 pyload/lib/thrift/TTornado.py delete mode 100644 pyload/lib/thrift/Thrift.py delete mode 100644 pyload/lib/thrift/__init__.py delete mode 100644 pyload/lib/thrift/protocol/TBase.py delete mode 100644 pyload/lib/thrift/protocol/TBinaryProtocol.py delete mode 100644 pyload/lib/thrift/protocol/TCompactProtocol.py delete mode 100644 pyload/lib/thrift/protocol/TJSONProtocol.py delete mode 100644 pyload/lib/thrift/protocol/TProtocol.py delete mode 100644 pyload/lib/thrift/protocol/__init__.py delete mode 100644 pyload/lib/thrift/protocol/fastbinary.c delete mode 100644 pyload/lib/thrift/server/THttpServer.py delete mode 100644 pyload/lib/thrift/server/TNonblockingServer.py delete mode 100644 pyload/lib/thrift/server/TProcessPoolServer.py delete mode 100644 pyload/lib/thrift/server/TServer.py delete mode 100644 pyload/lib/thrift/server/__init__.py delete mode 100644 pyload/lib/thrift/transport/THttpClient.py delete mode 100644 pyload/lib/thrift/transport/TSSLSocket.py delete mode 100644 pyload/lib/thrift/transport/TSocket.py delete mode 100644 pyload/lib/thrift/transport/TTransport.py delete mode 100644 pyload/lib/thrift/transport/TTwisted.py delete mode 100644 pyload/lib/thrift/transport/TZlibTransport.py delete mode 100644 pyload/lib/thrift/transport/__init__.py delete mode 100644 pyload/lib/wsgiserver.py (limited to 'pyload') diff --git a/pyload/lib/BeautifulSoup.py b/pyload/lib/BeautifulSoup.py deleted file mode 100644 index 7278215ca..000000000 --- a/pyload/lib/BeautifulSoup.py +++ /dev/null @@ -1,2017 +0,0 @@ -"""Beautiful Soup -Elixir and Tonic -"The Screen-Scraper's Friend" -http://www.crummy.com/software/BeautifulSoup/ - -Beautiful Soup parses a (possibly invalid) XML or HTML document into a -tree representation. It provides methods and Pythonic idioms that make -it easy to navigate, search, and modify the tree. - -A well-formed XML/HTML document yields a well-formed data -structure. An ill-formed XML/HTML document yields a correspondingly -ill-formed data structure. If your document is only locally -well-formed, you can use this library to find and process the -well-formed part of it. - -Beautiful Soup works with Python 2.2 and up. It has no external -dependencies, but you'll have more success at converting data to UTF-8 -if you also install these three packages: - -* chardet, for auto-detecting character encodings - http://chardet.feedparser.org/ -* cjkcodecs and iconv_codec, which add more encodings to the ones supported - by stock Python. - http://cjkpython.i18n.org/ - -Beautiful Soup defines classes for two main parsing strategies: - - * BeautifulStoneSoup, for parsing XML, SGML, or your domain-specific - language that kind of looks like XML. - - * BeautifulSoup, for parsing run-of-the-mill HTML code, be it valid - or invalid. This class has web browser-like heuristics for - obtaining a sensible parse tree in the face of common HTML errors. - -Beautiful Soup also defines a class (UnicodeDammit) for autodetecting -the encoding of an HTML or XML document, and converting it to -Unicode. Much of this code is taken from Mark Pilgrim's Universal Feed Parser. - -For more than you ever wanted to know about Beautiful Soup, see the -documentation: -http://www.crummy.com/software/BeautifulSoup/documentation.html - -Here, have some legalese: - -Copyright (c) 2004-2010, Leonard Richardson - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - * Neither the name of the the Beautiful Soup Consortium and All - Night Kosher Bakery nor the names of its contributors may be - used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE, DAMMIT. - -""" -from __future__ import generators - -__author__ = "Leonard Richardson (leonardr@segfault.org)" -__version__ = "3.2.1" -__copyright__ = "Copyright (c) 2004-2012 Leonard Richardson" -__license__ = "New-style BSD" - -from sgmllib import SGMLParser, SGMLParseError -import codecs -import markupbase -import types -import re -import sgmllib -try: - from htmlentitydefs import name2codepoint -except ImportError: - name2codepoint = {} -try: - set -except NameError: - from sets import Set as set - -#These hacks make Beautiful Soup able to parse XML with namespaces -sgmllib.tagfind = re.compile('[a-zA-Z][-_.:a-zA-Z0-9]*') -markupbase._declname_match = re.compile(r'[a-zA-Z][-_.:a-zA-Z0-9]*\s*').match - -DEFAULT_OUTPUT_ENCODING = "utf-8" - -def _match_css_class(str): - """Build a RE to match the given CSS class.""" - return re.compile(r"(^|.*\s)%s($|\s)" % str) - -# First, the classes that represent markup elements. - -class PageElement(object): - """Contains the navigational information for some part of the page - (either a tag or a piece of text)""" - - def _invert(h): - "Cheap function to invert a hash." - i = {} - for k,v in h.items(): - i[v] = k - return i - - XML_ENTITIES_TO_SPECIAL_CHARS = { "apos" : "'", - "quot" : '"', - "amp" : "&", - "lt" : "<", - "gt" : ">" } - - XML_SPECIAL_CHARS_TO_ENTITIES = _invert(XML_ENTITIES_TO_SPECIAL_CHARS) - - def setup(self, parent=None, previous=None): - """Sets up the initial relations between this element and - other elements.""" - self.parent = parent - self.previous = previous - self.next = None - self.previousSibling = None - self.nextSibling = None - if self.parent and self.parent.contents: - self.previousSibling = self.parent.contents[-1] - self.previousSibling.nextSibling = self - - def replaceWith(self, replaceWith): - oldParent = self.parent - myIndex = self.parent.index(self) - if hasattr(replaceWith, "parent")\ - and replaceWith.parent is self.parent: - # We're replacing this element with one of its siblings. - index = replaceWith.parent.index(replaceWith) - if index and index < myIndex: - # Furthermore, it comes before this element. That - # means that when we extract it, the index of this - # element will change. - myIndex = myIndex - 1 - self.extract() - oldParent.insert(myIndex, replaceWith) - - def replaceWithChildren(self): - myParent = self.parent - myIndex = self.parent.index(self) - self.extract() - reversedChildren = list(self.contents) - reversedChildren.reverse() - for child in reversedChildren: - myParent.insert(myIndex, child) - - def extract(self): - """Destructively rips this element out of the tree.""" - if self.parent: - try: - del self.parent.contents[self.parent.index(self)] - except ValueError: - pass - - #Find the two elements that would be next to each other if - #this element (and any children) hadn't been parsed. Connect - #the two. - lastChild = self._lastRecursiveChild() - nextElement = lastChild.next - - if self.previous: - self.previous.next = nextElement - if nextElement: - nextElement.previous = self.previous - self.previous = None - lastChild.next = None - - self.parent = None - if self.previousSibling: - self.previousSibling.nextSibling = self.nextSibling - if self.nextSibling: - self.nextSibling.previousSibling = self.previousSibling - self.previousSibling = self.nextSibling = None - return self - - def _lastRecursiveChild(self): - "Finds the last element beneath this object to be parsed." - lastChild = self - while hasattr(lastChild, 'contents') and lastChild.contents: - lastChild = lastChild.contents[-1] - return lastChild - - def insert(self, position, newChild): - if isinstance(newChild, basestring) \ - and not isinstance(newChild, NavigableString): - newChild = NavigableString(newChild) - - position = min(position, len(self.contents)) - if hasattr(newChild, 'parent') and newChild.parent is not None: - # We're 'inserting' an element that's already one - # of this object's children. - if newChild.parent is self: - index = self.index(newChild) - if index > position: - # Furthermore we're moving it further down the - # list of this object's children. That means that - # when we extract this element, our target index - # will jump down one. - position = position - 1 - newChild.extract() - - newChild.parent = self - previousChild = None - if position == 0: - newChild.previousSibling = None - newChild.previous = self - else: - previousChild = self.contents[position-1] - newChild.previousSibling = previousChild - newChild.previousSibling.nextSibling = newChild - newChild.previous = previousChild._lastRecursiveChild() - if newChild.previous: - newChild.previous.next = newChild - - newChildsLastElement = newChild._lastRecursiveChild() - - if position >= len(self.contents): - newChild.nextSibling = None - - parent = self - parentsNextSibling = None - while not parentsNextSibling: - parentsNextSibling = parent.nextSibling - parent = parent.parent - if not parent: # This is the last element in the document. - break - if parentsNextSibling: - newChildsLastElement.next = parentsNextSibling - else: - newChildsLastElement.next = None - else: - nextChild = self.contents[position] - newChild.nextSibling = nextChild - if newChild.nextSibling: - newChild.nextSibling.previousSibling = newChild - newChildsLastElement.next = nextChild - - if newChildsLastElement.next: - newChildsLastElement.next.previous = newChildsLastElement - self.contents.insert(position, newChild) - - def append(self, tag): - """Appends the given tag to the contents of this tag.""" - self.insert(len(self.contents), tag) - - def findNext(self, name=None, attrs={}, text=None, **kwargs): - """Returns the first item that matches the given criteria and - appears after this Tag in the document.""" - return self._findOne(self.findAllNext, name, attrs, text, **kwargs) - - def findAllNext(self, name=None, attrs={}, text=None, limit=None, - **kwargs): - """Returns all items that match the given criteria and appear - after this Tag in the document.""" - return self._findAll(name, attrs, text, limit, self.nextGenerator, - **kwargs) - - def findNextSibling(self, name=None, attrs={}, text=None, **kwargs): - """Returns the closest sibling to this Tag that matches the - given criteria and appears after this Tag in the document.""" - return self._findOne(self.findNextSiblings, name, attrs, text, - **kwargs) - - def findNextSiblings(self, name=None, attrs={}, text=None, limit=None, - **kwargs): - """Returns the siblings of this Tag that match the given - criteria and appear after this Tag in the document.""" - return self._findAll(name, attrs, text, limit, - self.nextSiblingGenerator, **kwargs) - fetchNextSiblings = findNextSiblings # Compatibility with pre-3.x - - def findPrevious(self, name=None, attrs={}, text=None, **kwargs): - """Returns the first item that matches the given criteria and - appears before this Tag in the document.""" - return self._findOne(self.findAllPrevious, name, attrs, text, **kwargs) - - def findAllPrevious(self, name=None, attrs={}, text=None, limit=None, - **kwargs): - """Returns all items that match the given criteria and appear - before this Tag in the document.""" - return self._findAll(name, attrs, text, limit, self.previousGenerator, - **kwargs) - fetchPrevious = findAllPrevious # Compatibility with pre-3.x - - def findPreviousSibling(self, name=None, attrs={}, text=None, **kwargs): - """Returns the closest sibling to this Tag that matches the - given criteria and appears before this Tag in the document.""" - return self._findOne(self.findPreviousSiblings, name, attrs, text, - **kwargs) - - def findPreviousSiblings(self, name=None, attrs={}, text=None, - limit=None, **kwargs): - """Returns the siblings of this Tag that match the given - criteria and appear before this Tag in the document.""" - return self._findAll(name, attrs, text, limit, - self.previousSiblingGenerator, **kwargs) - fetchPreviousSiblings = findPreviousSiblings # Compatibility with pre-3.x - - def findParent(self, name=None, attrs={}, **kwargs): - """Returns the closest parent of this Tag that matches the given - criteria.""" - # NOTE: We can't use _findOne because findParents takes a different - # set of arguments. - r = None - l = self.findParents(name, attrs, 1) - if l: - r = l[0] - return r - - def findParents(self, name=None, attrs={}, limit=None, **kwargs): - """Returns the parents of this Tag that match the given - criteria.""" - - return self._findAll(name, attrs, None, limit, self.parentGenerator, - **kwargs) - fetchParents = findParents # Compatibility with pre-3.x - - #These methods do the real heavy lifting. - - def _findOne(self, method, name, attrs, text, **kwargs): - r = None - l = method(name, attrs, text, 1, **kwargs) - if l: - r = l[0] - return r - - def _findAll(self, name, attrs, text, limit, generator, **kwargs): - "Iterates over a generator looking for things that match." - - if isinstance(name, SoupStrainer): - strainer = name - # (Possibly) special case some findAll*(...) searches - elif text is None and not limit and not attrs and not kwargs: - # findAll*(True) - if name is True: - return [element for element in generator() - if isinstance(element, Tag)] - # findAll*('tag-name') - elif isinstance(name, basestring): - return [element for element in generator() - if isinstance(element, Tag) and - element.name == name] - else: - strainer = SoupStrainer(name, attrs, text, **kwargs) - # Build a SoupStrainer - else: - strainer = SoupStrainer(name, attrs, text, **kwargs) - results = ResultSet(strainer) - g = generator() - while True: - try: - i = g.next() - except StopIteration: - break - if i: - found = strainer.search(i) - if found: - results.append(found) - if limit and len(results) >= limit: - break - return results - - #These Generators can be used to navigate starting from both - #NavigableStrings and Tags. - def nextGenerator(self): - i = self - while i is not None: - i = i.next - yield i - - def nextSiblingGenerator(self): - i = self - while i is not None: - i = i.nextSibling - yield i - - def previousGenerator(self): - i = self - while i is not None: - i = i.previous - yield i - - def previousSiblingGenerator(self): - i = self - while i is not None: - i = i.previousSibling - yield i - - def parentGenerator(self): - i = self - while i is not None: - i = i.parent - yield i - - # Utility methods - def substituteEncoding(self, str, encoding=None): - encoding = encoding or "utf-8" - return str.replace("%SOUP-ENCODING%", encoding) - - def toEncoding(self, s, encoding=None): - """Encodes an object to a string in some encoding, or to Unicode. - .""" - if isinstance(s, unicode): - if encoding: - s = s.encode(encoding) - elif isinstance(s, str): - if encoding: - s = s.encode(encoding) - else: - s = unicode(s) - else: - if encoding: - s = self.toEncoding(str(s), encoding) - else: - s = unicode(s) - return s - - BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|" - + "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)" - + ")") - - def _sub_entity(self, x): - """Used with a regular expression to substitute the - appropriate XML entity for an XML special character.""" - return "&" + self.XML_SPECIAL_CHARS_TO_ENTITIES[x.group(0)[0]] + ";" - - -class NavigableString(unicode, PageElement): - - def __new__(cls, value): - """Create a new NavigableString. - - When unpickling a NavigableString, this method is called with - the string in DEFAULT_OUTPUT_ENCODING. That encoding needs to be - passed in to the superclass's __new__ or the superclass won't know - how to handle non-ASCII characters. - """ - if isinstance(value, unicode): - return unicode.__new__(cls, value) - return unicode.__new__(cls, value, DEFAULT_OUTPUT_ENCODING) - - def __getnewargs__(self): - return (NavigableString.__str__(self),) - - def __getattr__(self, attr): - """text.string gives you text. This is for backwards - compatibility for Navigable*String, but for CData* it lets you - get the string without the CData wrapper.""" - if attr == 'string': - return self - else: - raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__.__name__, attr) - - def __unicode__(self): - return str(self).decode(DEFAULT_OUTPUT_ENCODING) - - def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): - # Substitute outgoing XML entities. - data = self.BARE_AMPERSAND_OR_BRACKET.sub(self._sub_entity, self) - if encoding: - return data.encode(encoding) - else: - return data - -class CData(NavigableString): - - def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): - return "" % NavigableString.__str__(self, encoding) - -class ProcessingInstruction(NavigableString): - def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): - output = self - if "%SOUP-ENCODING%" in output: - output = self.substituteEncoding(output, encoding) - return "" % self.toEncoding(output, encoding) - -class Comment(NavigableString): - def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): - return "" % NavigableString.__str__(self, encoding) - -class Declaration(NavigableString): - def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): - return "" % NavigableString.__str__(self, encoding) - -class Tag(PageElement): - - """Represents a found HTML tag with its attributes and contents.""" - - def _convertEntities(self, match): - """Used in a call to re.sub to replace HTML, XML, and numeric - entities with the appropriate Unicode characters. If HTML - entities are being converted, any unrecognized entities are - escaped.""" - x = match.group(1) - if self.convertHTMLEntities and x in name2codepoint: - return unichr(name2codepoint[x]) - elif x in self.XML_ENTITIES_TO_SPECIAL_CHARS: - if self.convertXMLEntities: - return self.XML_ENTITIES_TO_SPECIAL_CHARS[x] - else: - return u'&%s;' % x - elif len(x) > 0 and x[0] == '#': - # Handle numeric entities - if len(x) > 1 and x[1] == 'x': - return unichr(int(x[2:], 16)) - else: - return unichr(int(x[1:])) - - elif self.escapeUnrecognizedEntities: - return u'&%s;' % x - else: - return u'&%s;' % x - - def __init__(self, parser, name, attrs=None, parent=None, - previous=None): - "Basic constructor." - - # We don't actually store the parser object: that lets extracted - # chunks be garbage-collected - self.parserClass = parser.__class__ - self.isSelfClosing = parser.isSelfClosingTag(name) - self.name = name - if attrs is None: - attrs = [] - elif isinstance(attrs, dict): - attrs = attrs.items() - self.attrs = attrs - self.contents = [] - self.setup(parent, previous) - self.hidden = False - self.containsSubstitutions = False - self.convertHTMLEntities = parser.convertHTMLEntities - self.convertXMLEntities = parser.convertXMLEntities - self.escapeUnrecognizedEntities = parser.escapeUnrecognizedEntities - - # Convert any HTML, XML, or numeric entities in the attribute values. - convert = lambda(k, val): (k, - re.sub("&(#\d+|#x[0-9a-fA-F]+|\w+);", - self._convertEntities, - val)) - self.attrs = map(convert, self.attrs) - - def getString(self): - if (len(self.contents) == 1 - and isinstance(self.contents[0], NavigableString)): - return self.contents[0] - - def setString(self, string): - """Replace the contents of the tag with a string""" - self.clear() - self.append(string) - - string = property(getString, setString) - - def getText(self, separator=u""): - if not len(self.contents): - return u"" - stopNode = self._lastRecursiveChild().next - strings = [] - current = self.contents[0] - while current is not stopNode: - if isinstance(current, NavigableString): - strings.append(current.strip()) - current = current.next - return separator.join(strings) - - text = property(getText) - - def get(self, key, default=None): - """Returns the value of the 'key' attribute for the tag, or - the value given for 'default' if it doesn't have that - attribute.""" - return self._getAttrMap().get(key, default) - - def clear(self): - """Extract all children.""" - for child in self.contents[:]: - child.extract() - - def index(self, element): - for i, child in enumerate(self.contents): - if child is element: - return i - raise ValueError("Tag.index: element not in tag") - - def has_key(self, key): - return self._getAttrMap().has_key(key) - - def __getitem__(self, key): - """tag[key] returns the value of the 'key' attribute for the tag, - and throws an exception if it's not there.""" - return self._getAttrMap()[key] - - def __iter__(self): - "Iterating over a tag iterates over its contents." - return iter(self.contents) - - def __len__(self): - "The length of a tag is the length of its list of contents." - return len(self.contents) - - def __contains__(self, x): - return x in self.contents - - def __nonzero__(self): - "A tag is non-None even if it has no contents." - return True - - def __setitem__(self, key, value): - """Setting tag[key] sets the value of the 'key' attribute for the - tag.""" - self._getAttrMap() - self.attrMap[key] = value - found = False - for i in range(0, len(self.attrs)): - if self.attrs[i][0] == key: - self.attrs[i] = (key, value) - found = True - if not found: - self.attrs.append((key, value)) - self._getAttrMap()[key] = value - - def __delitem__(self, key): - "Deleting tag[key] deletes all 'key' attributes for the tag." - for item in self.attrs: - if item[0] == key: - self.attrs.remove(item) - #We don't break because bad HTML can define the same - #attribute multiple times. - self._getAttrMap() - if self.attrMap.has_key(key): - del self.attrMap[key] - - def __call__(self, *args, **kwargs): - """Calling a tag like a function is the same as calling its - findAll() method. Eg. tag('a') returns a list of all the A tags - found within this tag.""" - return apply(self.findAll, args, kwargs) - - def __getattr__(self, tag): - #print "Getattr %s.%s" % (self.__class__, tag) - if len(tag) > 3 and tag.rfind('Tag') == len(tag)-3: - return self.find(tag[:-3]) - elif tag.find('__') != 0: - return self.find(tag) - raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__, tag) - - def __eq__(self, other): - """Returns true iff this tag has the same name, the same attributes, - and the same contents (recursively) as the given tag. - - NOTE: right now this will return false if two tags have the - same attributes in a different order. Should this be fixed?""" - if other is self: - return True - if not hasattr(other, 'name') or not hasattr(other, 'attrs') or not hasattr(other, 'contents') or self.name != other.name or self.attrs != other.attrs or len(self) != len(other): - return False - for i in range(0, len(self.contents)): - if self.contents[i] != other.contents[i]: - return False - return True - - def __ne__(self, other): - """Returns true iff this tag is not identical to the other tag, - as defined in __eq__.""" - return not self == other - - def __repr__(self, encoding=DEFAULT_OUTPUT_ENCODING): - """Renders this tag as a string.""" - return self.__str__(encoding) - - def __unicode__(self): - return self.__str__(None) - - def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING, - prettyPrint=False, indentLevel=0): - """Returns a string or Unicode representation of this tag and - its contents. To get Unicode, pass None for encoding. - - NOTE: since Python's HTML parser consumes whitespace, this - method is not certain to reproduce the whitespace present in - the original string.""" - - encodedName = self.toEncoding(self.name, encoding) - - attrs = [] - if self.attrs: - for key, val in self.attrs: - fmt = '%s="%s"' - if isinstance(val, basestring): - if self.containsSubstitutions and '%SOUP-ENCODING%' in val: - val = self.substituteEncoding(val, encoding) - - # The attribute value either: - # - # * Contains no embedded double quotes or single quotes. - # No problem: we enclose it in double quotes. - # * Contains embedded single quotes. No problem: - # double quotes work here too. - # * Contains embedded double quotes. No problem: - # we enclose it in single quotes. - # * Embeds both single _and_ double quotes. This - # can't happen naturally, but it can happen if - # you modify an attribute value after parsing - # the document. Now we have a bit of a - # problem. We solve it by enclosing the - # attribute in single quotes, and escaping any - # embedded single quotes to XML entities. - if '"' in val: - fmt = "%s='%s'" - if "'" in val: - # TODO: replace with apos when - # appropriate. - val = val.replace("'", "&squot;") - - # Now we're okay w/r/t quotes. But the attribute - # value might also contain angle brackets, or - # ampersands that aren't part of entities. We need - # to escape those to XML entities too. - val = self.BARE_AMPERSAND_OR_BRACKET.sub(self._sub_entity, val) - - attrs.append(fmt % (self.toEncoding(key, encoding), - self.toEncoding(val, encoding))) - close = '' - closeTag = '' - if self.isSelfClosing: - close = ' /' - else: - closeTag = '' % encodedName - - indentTag, indentContents = 0, 0 - if prettyPrint: - indentTag = indentLevel - space = (' ' * (indentTag-1)) - indentContents = indentTag + 1 - contents = self.renderContents(encoding, prettyPrint, indentContents) - if self.hidden: - s = contents - else: - s = [] - attributeString = '' - if attrs: - attributeString = ' ' + ' '.join(attrs) - if prettyPrint: - s.append(space) - s.append('<%s%s%s>' % (encodedName, attributeString, close)) - if prettyPrint: - s.append("\n") - s.append(contents) - if prettyPrint and contents and contents[-1] != "\n": - s.append("\n") - if prettyPrint and closeTag: - s.append(space) - s.append(closeTag) - if prettyPrint and closeTag and self.nextSibling: - s.append("\n") - s = ''.join(s) - return s - - def decompose(self): - """Recursively destroys the contents of this tree.""" - self.extract() - if len(self.contents) == 0: - return - current = self.contents[0] - while current is not None: - next = current.next - if isinstance(current, Tag): - del current.contents[:] - current.parent = None - current.previous = None - current.previousSibling = None - current.next = None - current.nextSibling = None - current = next - - def prettify(self, encoding=DEFAULT_OUTPUT_ENCODING): - return self.__str__(encoding, True) - - def renderContents(self, encoding=DEFAULT_OUTPUT_ENCODING, - prettyPrint=False, indentLevel=0): - """Renders the contents of this tag as a string in the given - encoding. If encoding is None, returns a Unicode string..""" - s=[] - for c in self: - text = None - if isinstance(c, NavigableString): - text = c.__str__(encoding) - elif isinstance(c, Tag): - s.append(c.__str__(encoding, prettyPrint, indentLevel)) - if text and prettyPrint: - text = text.strip() - if text: - if prettyPrint: - s.append(" " * (indentLevel-1)) - s.append(text) - if prettyPrint: - s.append("\n") - return ''.join(s) - - #Soup methods - - def find(self, name=None, attrs={}, recursive=True, text=None, - **kwargs): - """Return only the first child of this Tag matching the given - criteria.""" - r = None - l = self.findAll(name, attrs, recursive, text, 1, **kwargs) - if l: - r = l[0] - return r - findChild = find - - def findAll(self, name=None, attrs={}, recursive=True, text=None, - limit=None, **kwargs): - """Extracts a list of Tag objects that match the given - criteria. You can specify the name of the Tag and any - attributes you want the Tag to have. - - The value of a key-value pair in the 'attrs' map can be a - string, a list of strings, a regular expression object, or a - callable that takes a string and returns whether or not the - string matches for some custom definition of 'matches'. The - same is true of the tag name.""" - generator = self.recursiveChildGenerator - if not recursive: - generator = self.childGenerator - return self._findAll(name, attrs, text, limit, generator, **kwargs) - findChildren = findAll - - # Pre-3.x compatibility methods - first = find - fetch = findAll - - def fetchText(self, text=None, recursive=True, limit=None): - return self.findAll(text=text, recursive=recursive, limit=limit) - - def firstText(self, text=None, recursive=True): - return self.find(text=text, recursive=recursive) - - #Private methods - - def _getAttrMap(self): - """Initializes a map representation of this tag's attributes, - if not already initialized.""" - if not getattr(self, 'attrMap'): - self.attrMap = {} - for (key, value) in self.attrs: - self.attrMap[key] = value - return self.attrMap - - #Generator methods - def childGenerator(self): - # Just use the iterator from the contents - return iter(self.contents) - - def recursiveChildGenerator(self): - if not len(self.contents): - raise StopIteration - stopNode = self._lastRecursiveChild().next - current = self.contents[0] - while current is not stopNode: - yield current - current = current.next - - -# Next, a couple classes to represent queries and their results. -class SoupStrainer: - """Encapsulates a number of ways of matching a markup element (tag or - text).""" - - def __init__(self, name=None, attrs={}, text=None, **kwargs): - self.name = name - if isinstance(attrs, basestring): - kwargs['class'] = _match_css_class(attrs) - attrs = None - if kwargs: - if attrs: - attrs = attrs.copy() - attrs.update(kwargs) - else: - attrs = kwargs - self.attrs = attrs - self.text = text - - def __str__(self): - if self.text: - return self.text - else: - return "%s|%s" % (self.name, self.attrs) - - def searchTag(self, markupName=None, markupAttrs={}): - found = None - markup = None - if isinstance(markupName, Tag): - markup = markupName - markupAttrs = markup - callFunctionWithTagData = callable(self.name) \ - and not isinstance(markupName, Tag) - - if (not self.name) \ - or callFunctionWithTagData \ - or (markup and self._matches(markup, self.name)) \ - or (not markup and self._matches(markupName, self.name)): - if callFunctionWithTagData: - match = self.name(markupName, markupAttrs) - else: - match = True - markupAttrMap = None - for attr, matchAgainst in self.attrs.items(): - if not markupAttrMap: - if hasattr(markupAttrs, 'get'): - markupAttrMap = markupAttrs - else: - markupAttrMap = {} - for k,v in markupAttrs: - markupAttrMap[k] = v - attrValue = markupAttrMap.get(attr) - if not self._matches(attrValue, matchAgainst): - match = False - break - if match: - if markup: - found = markup - else: - found = markupName - return found - - def search(self, markup): - #print 'looking for %s in %s' % (self, markup) - found = None - # If given a list of items, scan it for a text element that - # matches. - if hasattr(markup, "__iter__") \ - and not isinstance(markup, Tag): - for element in markup: - if isinstance(element, NavigableString) \ - and self.search(element): - found = element - break - # If it's a Tag, make sure its name or attributes match. - # Don't bother with Tags if we're searching for text. - elif isinstance(markup, Tag): - if not self.text: - found = self.searchTag(markup) - # If it's text, make sure the text matches. - elif isinstance(markup, NavigableString) or \ - isinstance(markup, basestring): - if self._matches(markup, self.text): - found = markup - else: - raise Exception, "I don't know how to match against a %s" \ - % markup.__class__ - return found - - def _matches(self, markup, matchAgainst): - #print "Matching %s against %s" % (markup, matchAgainst) - result = False - if matchAgainst is True: - result = markup is not None - elif callable(matchAgainst): - result = matchAgainst(markup) - else: - #Custom match methods take the tag as an argument, but all - #other ways of matching match the tag name as a string. - if isinstance(markup, Tag): - markup = markup.name - if markup and not isinstance(markup, basestring): - markup = unicode(markup) - #Now we know that chunk is either a string, or None. - if hasattr(matchAgainst, 'match'): - # It's a regexp object. - result = markup and matchAgainst.search(markup) - elif hasattr(matchAgainst, '__iter__'): # list-like - result = markup in matchAgainst - elif hasattr(matchAgainst, 'items'): - result = markup.has_key(matchAgainst) - elif matchAgainst and isinstance(markup, basestring): - if isinstance(markup, unicode): - matchAgainst = unicode(matchAgainst) - else: - matchAgainst = str(matchAgainst) - - if not result: - result = matchAgainst == markup - return result - -class ResultSet(list): - """A ResultSet is just a list that keeps track of the SoupStrainer - that created it.""" - def __init__(self, source): - list.__init__([]) - self.source = source - -# Now, some helper functions. - -def buildTagMap(default, *args): - """Turns a list of maps, lists, or scalars into a single map. - Used to build the SELF_CLOSING_TAGS, NESTABLE_TAGS, and - NESTING_RESET_TAGS maps out of lists and partial maps.""" - built = {} - for portion in args: - if hasattr(portion, 'items'): - #It's a map. Merge it. - for k,v in portion.items(): - built[k] = v - elif hasattr(portion, '__iter__'): # is a list - #It's a list. Map each item to the default. - for k in portion: - built[k] = default - else: - #It's a scalar. Map it to the default. - built[portion] = default - return built - -# Now, the parser classes. - -class BeautifulStoneSoup(Tag, SGMLParser): - - """This class contains the basic parser and search code. It defines - a parser that knows nothing about tag behavior except for the - following: - - You can't close a tag without closing all the tags it encloses. - That is, "" actually means - "". - - [Another possible explanation is "", but since - this class defines no SELF_CLOSING_TAGS, it will never use that - explanation.] - - This class is useful for parsing XML or made-up markup languages, - or when BeautifulSoup makes an assumption counter to what you were - expecting.""" - - SELF_CLOSING_TAGS = {} - NESTABLE_TAGS = {} - RESET_NESTING_TAGS = {} - QUOTE_TAGS = {} - PRESERVE_WHITESPACE_TAGS = [] - - MARKUP_MASSAGE = [(re.compile('(<[^<>]*)/>'), - lambda x: x.group(1) + ' />'), - (re.compile(']*)>'), - lambda x: '') - ] - - ROOT_TAG_NAME = u'[document]' - - HTML_ENTITIES = "html" - XML_ENTITIES = "xml" - XHTML_ENTITIES = "xhtml" - # TODO: This only exists for backwards-compatibility - ALL_ENTITIES = XHTML_ENTITIES - - # Used when determining whether a text node is all whitespace and - # can be replaced with a single space. A text node that contains - # fancy Unicode spaces (usually non-breaking) should be left - # alone. - STRIP_ASCII_SPACES = { 9: None, 10: None, 12: None, 13: None, 32: None, } - - def __init__(self, markup="", parseOnlyThese=None, fromEncoding=None, - markupMassage=True, smartQuotesTo=XML_ENTITIES, - convertEntities=None, selfClosingTags=None, isHTML=False): - """The Soup object is initialized as the 'root tag', and the - provided markup (which can be a string or a file-like object) - is fed into the underlying parser. - - sgmllib will process most bad HTML, and the BeautifulSoup - class has some tricks for dealing with some HTML that kills - sgmllib, but Beautiful Soup can nonetheless choke or lose data - if your data uses self-closing tags or declarations - incorrectly. - - By default, Beautiful Soup uses regexes to sanitize input, - avoiding the vast majority of these problems. If the problems - don't apply to you, pass in False for markupMassage, and - you'll get better performance. - - The default parser massage techniques fix the two most common - instances of invalid HTML that choke sgmllib: - -
(No space between name of closing tag and tag close) - (Extraneous whitespace in declaration) - - You can pass in a custom list of (RE object, replace method) - tuples to get Beautiful Soup to scrub your input the way you - want.""" - - self.parseOnlyThese = parseOnlyThese - self.fromEncoding = fromEncoding - self.smartQuotesTo = smartQuotesTo - self.convertEntities = convertEntities - # Set the rules for how we'll deal with the entities we - # encounter - if self.convertEntities: - # It doesn't make sense to convert encoded characters to - # entities even while you're converting entities to Unicode. - # Just convert it all to Unicode. - self.smartQuotesTo = None - if convertEntities == self.HTML_ENTITIES: - self.convertXMLEntities = False - self.convertHTMLEntities = True - self.escapeUnrecognizedEntities = True - elif convertEntities == self.XHTML_ENTITIES: - self.convertXMLEntities = True - self.convertHTMLEntities = True - self.escapeUnrecognizedEntities = False - elif convertEntities == self.XML_ENTITIES: - self.convertXMLEntities = True - self.convertHTMLEntities = False - self.escapeUnrecognizedEntities = False - else: - self.convertXMLEntities = False - self.convertHTMLEntities = False - self.escapeUnrecognizedEntities = False - - self.instanceSelfClosingTags = buildTagMap(None, selfClosingTags) - SGMLParser.__init__(self) - - if hasattr(markup, 'read'): # It's a file-type object. - markup = markup.read() - self.markup = markup - self.markupMassage = markupMassage - try: - self._feed(isHTML=isHTML) - except StopParsing: - pass - self.markup = None # The markup can now be GCed - - def convert_charref(self, name): - """This method fixes a bug in Python's SGMLParser.""" - try: - n = int(name) - except ValueError: - return - if not 0 <= n <= 127 : # ASCII ends at 127, not 255 - return - return self.convert_codepoint(n) - - def _feed(self, inDocumentEncoding=None, isHTML=False): - # Convert the document to Unicode. - markup = self.markup - if isinstance(markup, unicode): - if not hasattr(self, 'originalEncoding'): - self.originalEncoding = None - else: - dammit = UnicodeDammit\ - (markup, [self.fromEncoding, inDocumentEncoding], - smartQuotesTo=self.smartQuotesTo, isHTML=isHTML) - markup = dammit.unicode - self.originalEncoding = dammit.originalEncoding - self.declaredHTMLEncoding = dammit.declaredHTMLEncoding - if markup: - if self.markupMassage: - if not hasattr(self.markupMassage, "__iter__"): - self.markupMassage = self.MARKUP_MASSAGE - for fix, m in self.markupMassage: - markup = fix.sub(m, markup) - # TODO: We get rid of markupMassage so that the - # soup object can be deepcopied later on. Some - # Python installations can't copy regexes. If anyone - # was relying on the existence of markupMassage, this - # might cause problems. - del(self.markupMassage) - self.reset() - - SGMLParser.feed(self, markup) - # Close out any unfinished strings and close all the open tags. - self.endData() - while self.currentTag.name != self.ROOT_TAG_NAME: - self.popTag() - - def __getattr__(self, methodName): - """This method routes method call requests to either the SGMLParser - superclass or the Tag superclass, depending on the method name.""" - #print "__getattr__ called on %s.%s" % (self.__class__, methodName) - - if methodName.startswith('start_') or methodName.startswith('end_') \ - or methodName.startswith('do_'): - return SGMLParser.__getattr__(self, methodName) - elif not methodName.startswith('__'): - return Tag.__getattr__(self, methodName) - else: - raise AttributeError - - def isSelfClosingTag(self, name): - """Returns true iff the given string is the name of a - self-closing tag according to this parser.""" - return self.SELF_CLOSING_TAGS.has_key(name) \ - or self.instanceSelfClosingTags.has_key(name) - - def reset(self): - Tag.__init__(self, self, self.ROOT_TAG_NAME) - self.hidden = 1 - SGMLParser.reset(self) - self.currentData = [] - self.currentTag = None - self.tagStack = [] - self.quoteStack = [] - self.pushTag(self) - - def popTag(self): - tag = self.tagStack.pop() - - #print "Pop", tag.name - if self.tagStack: - self.currentTag = self.tagStack[-1] - return self.currentTag - - def pushTag(self, tag): - #print "Push", tag.name - if self.currentTag: - self.currentTag.contents.append(tag) - self.tagStack.append(tag) - self.currentTag = self.tagStack[-1] - - def endData(self, containerClass=NavigableString): - if self.currentData: - currentData = u''.join(self.currentData) - if (currentData.translate(self.STRIP_ASCII_SPACES) == '' and - not set([tag.name for tag in self.tagStack]).intersection( - self.PRESERVE_WHITESPACE_TAGS)): - if '\n' in currentData: - currentData = '\n' - else: - currentData = ' ' - self.currentData = [] - if self.parseOnlyThese and len(self.tagStack) <= 1 and \ - (not self.parseOnlyThese.text or \ - not self.parseOnlyThese.search(currentData)): - return - o = containerClass(currentData) - o.setup(self.currentTag, self.previous) - if self.previous: - self.previous.next = o - self.previous = o - self.currentTag.contents.append(o) - - - def _popToTag(self, name, inclusivePop=True): - """Pops the tag stack up to and including the most recent - instance of the given tag. If inclusivePop is false, pops the tag - stack up to but *not* including the most recent instqance of - the given tag.""" - #print "Popping to %s" % name - if name == self.ROOT_TAG_NAME: - return - - numPops = 0 - mostRecentTag = None - for i in range(len(self.tagStack)-1, 0, -1): - if name == self.tagStack[i].name: - numPops = len(self.tagStack)-i - break - if not inclusivePop: - numPops = numPops - 1 - - for i in range(0, numPops): - mostRecentTag = self.popTag() - return mostRecentTag - - def _smartPop(self, name): - - """We need to pop up to the previous tag of this type, unless - one of this tag's nesting reset triggers comes between this - tag and the previous tag of this type, OR unless this tag is a - generic nesting trigger and another generic nesting trigger - comes between this tag and the previous tag of this type. - - Examples: -

FooBar *

* should pop to 'p', not 'b'. -

FooBar *

* should pop to 'table', not 'p'. -

Foo

Bar *

* should pop to 'tr', not 'p'. - -

    • *
    • * should pop to 'ul', not the first 'li'. -
    • ** should pop to 'table', not the first 'tr' - tag should - implicitly close the previous tag within the same
    ** should pop to 'tr', not the first 'td' - """ - - nestingResetTriggers = self.NESTABLE_TAGS.get(name) - isNestable = nestingResetTriggers != None - isResetNesting = self.RESET_NESTING_TAGS.has_key(name) - popTo = None - inclusive = True - for i in range(len(self.tagStack)-1, 0, -1): - p = self.tagStack[i] - if (not p or p.name == name) and not isNestable: - #Non-nestable tags get popped to the top or to their - #last occurance. - popTo = name - break - if (nestingResetTriggers is not None - and p.name in nestingResetTriggers) \ - or (nestingResetTriggers is None and isResetNesting - and self.RESET_NESTING_TAGS.has_key(p.name)): - - #If we encounter one of the nesting reset triggers - #peculiar to this tag, or we encounter another tag - #that causes nesting to reset, pop up to but not - #including that tag. - popTo = p.name - inclusive = False - break - p = p.parent - if popTo: - self._popToTag(popTo, inclusive) - - def unknown_starttag(self, name, attrs, selfClosing=0): - #print "Start tag %s: %s" % (name, attrs) - if self.quoteStack: - #This is not a real tag. - #print "<%s> is not real!" % name - attrs = ''.join([' %s="%s"' % (x, y) for x, y in attrs]) - self.handle_data('<%s%s>' % (name, attrs)) - return - self.endData() - - if not self.isSelfClosingTag(name) and not selfClosing: - self._smartPop(name) - - if self.parseOnlyThese and len(self.tagStack) <= 1 \ - and (self.parseOnlyThese.text or not self.parseOnlyThese.searchTag(name, attrs)): - return - - tag = Tag(self, name, attrs, self.currentTag, self.previous) - if self.previous: - self.previous.next = tag - self.previous = tag - self.pushTag(tag) - if selfClosing or self.isSelfClosingTag(name): - self.popTag() - if name in self.QUOTE_TAGS: - #print "Beginning quote (%s)" % name - self.quoteStack.append(name) - self.literal = 1 - return tag - - def unknown_endtag(self, name): - #print "End tag %s" % name - if self.quoteStack and self.quoteStack[-1] != name: - #This is not a real end tag. - #print " is not real!" % name - self.handle_data('' % name) - return - self.endData() - self._popToTag(name) - if self.quoteStack and self.quoteStack[-1] == name: - self.quoteStack.pop() - self.literal = (len(self.quoteStack) > 0) - - def handle_data(self, data): - self.currentData.append(data) - - def _toStringSubclass(self, text, subclass): - """Adds a certain piece of text to the tree as a NavigableString - subclass.""" - self.endData() - self.handle_data(text) - self.endData(subclass) - - def handle_pi(self, text): - """Handle a processing instruction as a ProcessingInstruction - object, possibly one with a %SOUP-ENCODING% slot into which an - encoding will be plugged later.""" - if text[:3] == "xml": - text = u"xml version='1.0' encoding='%SOUP-ENCODING%'" - self._toStringSubclass(text, ProcessingInstruction) - - def handle_comment(self, text): - "Handle comments as Comment objects." - self._toStringSubclass(text, Comment) - - def handle_charref(self, ref): - "Handle character references as data." - if self.convertEntities: - data = unichr(int(ref)) - else: - data = '&#%s;' % ref - self.handle_data(data) - - def handle_entityref(self, ref): - """Handle entity references as data, possibly converting known - HTML and/or XML entity references to the corresponding Unicode - characters.""" - data = None - if self.convertHTMLEntities: - try: - data = unichr(name2codepoint[ref]) - except KeyError: - pass - - if not data and self.convertXMLEntities: - data = self.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref) - - if not data and self.convertHTMLEntities and \ - not self.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref): - # TODO: We've got a problem here. We're told this is - # an entity reference, but it's not an XML entity - # reference or an HTML entity reference. Nonetheless, - # the logical thing to do is to pass it through as an - # unrecognized entity reference. - # - # Except: when the input is "&carol;" this function - # will be called with input "carol". When the input is - # "AT&T", this function will be called with input - # "T". We have no way of knowing whether a semicolon - # was present originally, so we don't know whether - # this is an unknown entity or just a misplaced - # ampersand. - # - # The more common case is a misplaced ampersand, so I - # escape the ampersand and omit the trailing semicolon. - data = "&%s" % ref - if not data: - # This case is different from the one above, because we - # haven't already gone through a supposedly comprehensive - # mapping of entities to Unicode characters. We might not - # have gone through any mapping at all. So the chances are - # very high that this is a real entity, and not a - # misplaced ampersand. - data = "&%s;" % ref - self.handle_data(data) - - def handle_decl(self, data): - "Handle DOCTYPEs and the like as Declaration objects." - self._toStringSubclass(data, Declaration) - - def parse_declaration(self, i): - """Treat a bogus SGML declaration as raw data. Treat a CDATA - declaration as a CData object.""" - j = None - if self.rawdata[i:i+9] == '', i) - if k == -1: - k = len(self.rawdata) - data = self.rawdata[i+9:k] - j = k+3 - self._toStringSubclass(data, CData) - else: - try: - j = SGMLParser.parse_declaration(self, i) - except SGMLParseError: - toHandle = self.rawdata[i:] - self.handle_data(toHandle) - j = i + len(toHandle) - return j - -class BeautifulSoup(BeautifulStoneSoup): - - """This parser knows the following facts about HTML: - - * Some tags have no closing tag and should be interpreted as being - closed as soon as they are encountered. - - * The text inside some tags (ie. 'script') may contain tags which - are not really part of the document and which should be parsed - as text, not tags. If you want to parse the text as tags, you can - always fetch it and parse it explicitly. - - * Tag nesting rules: - - Most tags can't be nested at all. For instance, the occurance of - a

    tag should implicitly close the previous

    tag. - -

    Para1

    Para2 - should be transformed into: -

    Para1

    Para2 - - Some tags can be nested arbitrarily. For instance, the occurance - of a

    tag should _not_ implicitly close the previous -
    tag. - - Alice said:
    Bob said:
    Blah - should NOT be transformed into: - Alice said:
    Bob said:
    Blah - - Some tags can be nested, but the nesting is reset by the - interposition of other tags. For instance, a
    , - but not close a tag in another table. - -
    BlahBlah - should be transformed into: -
    BlahBlah - but, - Blah
    Blah - should NOT be transformed into - Blah
    Blah - - Differing assumptions about tag nesting rules are a major source - of problems with the BeautifulSoup class. If BeautifulSoup is not - treating as nestable a tag your page author treats as nestable, - try ICantBelieveItsBeautifulSoup, MinimalSoup, or - BeautifulStoneSoup before writing your own subclass.""" - - def __init__(self, *args, **kwargs): - if not kwargs.has_key('smartQuotesTo'): - kwargs['smartQuotesTo'] = self.HTML_ENTITIES - kwargs['isHTML'] = True - BeautifulStoneSoup.__init__(self, *args, **kwargs) - - SELF_CLOSING_TAGS = buildTagMap(None, - ('br' , 'hr', 'input', 'img', 'meta', - 'spacer', 'link', 'frame', 'base', 'col')) - - PRESERVE_WHITESPACE_TAGS = set(['pre', 'textarea']) - - QUOTE_TAGS = {'script' : None, 'textarea' : None} - - #According to the HTML standard, each of these inline tags can - #contain another tag of the same type. Furthermore, it's common - #to actually use these tags this way. - NESTABLE_INLINE_TAGS = ('span', 'font', 'q', 'object', 'bdo', 'sub', 'sup', - 'center') - - #According to the HTML standard, these block tags can contain - #another tag of the same type. Furthermore, it's common - #to actually use these tags this way. - NESTABLE_BLOCK_TAGS = ('blockquote', 'div', 'fieldset', 'ins', 'del') - - #Lists can contain other lists, but there are restrictions. - NESTABLE_LIST_TAGS = { 'ol' : [], - 'ul' : [], - 'li' : ['ul', 'ol'], - 'dl' : [], - 'dd' : ['dl'], - 'dt' : ['dl'] } - - #Tables can contain other tables, but there are restrictions. - NESTABLE_TABLE_TAGS = {'table' : [], - 'tr' : ['table', 'tbody', 'tfoot', 'thead'], - 'td' : ['tr'], - 'th' : ['tr'], - 'thead' : ['table'], - 'tbody' : ['table'], - 'tfoot' : ['table'], - } - - NON_NESTABLE_BLOCK_TAGS = ('address', 'form', 'p', 'pre') - - #If one of these tags is encountered, all tags up to the next tag of - #this type are popped. - RESET_NESTING_TAGS = buildTagMap(None, NESTABLE_BLOCK_TAGS, 'noscript', - NON_NESTABLE_BLOCK_TAGS, - NESTABLE_LIST_TAGS, - NESTABLE_TABLE_TAGS) - - NESTABLE_TAGS = buildTagMap([], NESTABLE_INLINE_TAGS, NESTABLE_BLOCK_TAGS, - NESTABLE_LIST_TAGS, NESTABLE_TABLE_TAGS) - - # Used to detect the charset in a META tag; see start_meta - CHARSET_RE = re.compile("((^|;)\s*charset=)([^;]*)", re.M) - - def start_meta(self, attrs): - """Beautiful Soup can detect a charset included in a META tag, - try to convert the document to that charset, and re-parse the - document from the beginning.""" - httpEquiv = None - contentType = None - contentTypeIndex = None - tagNeedsEncodingSubstitution = False - - for i in range(0, len(attrs)): - key, value = attrs[i] - key = key.lower() - if key == 'http-equiv': - httpEquiv = value - elif key == 'content': - contentType = value - contentTypeIndex = i - - if httpEquiv and contentType: # It's an interesting meta tag. - match = self.CHARSET_RE.search(contentType) - if match: - if (self.declaredHTMLEncoding is not None or - self.originalEncoding == self.fromEncoding): - # An HTML encoding was sniffed while converting - # the document to Unicode, or an HTML encoding was - # sniffed during a previous pass through the - # document, or an encoding was specified - # explicitly and it worked. Rewrite the meta tag. - def rewrite(match): - return match.group(1) + "%SOUP-ENCODING%" - newAttr = self.CHARSET_RE.sub(rewrite, contentType) - attrs[contentTypeIndex] = (attrs[contentTypeIndex][0], - newAttr) - tagNeedsEncodingSubstitution = True - else: - # This is our first pass through the document. - # Go through it again with the encoding information. - newCharset = match.group(3) - if newCharset and newCharset != self.originalEncoding: - self.declaredHTMLEncoding = newCharset - self._feed(self.declaredHTMLEncoding) - raise StopParsing - pass - tag = self.unknown_starttag("meta", attrs) - if tag and tagNeedsEncodingSubstitution: - tag.containsSubstitutions = True - -class StopParsing(Exception): - pass - -class ICantBelieveItsBeautifulSoup(BeautifulSoup): - - """The BeautifulSoup class is oriented towards skipping over - common HTML errors like unclosed tags. However, sometimes it makes - errors of its own. For instance, consider this fragment: - - FooBar - - This is perfectly valid (if bizarre) HTML. However, the - BeautifulSoup class will implicitly close the first b tag when it - encounters the second 'b'. It will think the author wrote - "FooBar", and didn't close the first 'b' tag, because - there's no real-world reason to bold something that's already - bold. When it encounters '' it will close two more 'b' - tags, for a grand total of three tags closed instead of two. This - can throw off the rest of your document structure. The same is - true of a number of other tags, listed below. - - It's much more common for someone to forget to close a 'b' tag - than to actually use nested 'b' tags, and the BeautifulSoup class - handles the common case. This class handles the not-co-common - case: where you can't believe someone wrote what they did, but - it's valid HTML and BeautifulSoup screwed up by assuming it - wouldn't be.""" - - I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS = \ - ('em', 'big', 'i', 'small', 'tt', 'abbr', 'acronym', 'strong', - 'cite', 'code', 'dfn', 'kbd', 'samp', 'strong', 'var', 'b', - 'big') - - I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS = ('noscript',) - - NESTABLE_TAGS = buildTagMap([], BeautifulSoup.NESTABLE_TAGS, - I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS, - I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS) - -class MinimalSoup(BeautifulSoup): - """The MinimalSoup class is for parsing HTML that contains - pathologically bad markup. It makes no assumptions about tag - nesting, but it does know which tags are self-closing, that - ' - safe = Markup('username') - assert unsafe + safe == text_type(escape(unsafe)) + text_type(safe) - - # string interpolations are safe to use too - assert Markup('%s') % '' == \ - '<bad user>' - assert Markup('%(username)s') % { - 'username': '' - } == '<bad user>' - - # an escaped object is markup too - assert type(Markup('foo') + 'bar') is Markup - - # and it implements __html__ by returning itself - x = Markup("foo") - assert x.__html__() is x - - # it also knows how to treat __html__ objects - class Foo(object): - def __html__(self): - return 'awesome' - def __unicode__(self): - return 'awesome' - assert Markup(Foo()) == 'awesome' - assert Markup('%s') % Foo() == \ - 'awesome' - - # escaping and unescaping - assert escape('"<>&\'') == '"<>&'' - assert Markup("Foo & Bar").striptags() == "Foo & Bar" - assert Markup("<test>").unescape() == "" - - def test_template_data(self): - env = Environment(autoescape=True) - t = env.from_string('{% macro say_hello(name) %}' - '

    Hello {{ name }}!

    {% endmacro %}' - '{{ say_hello("foo") }}') - escaped_out = '

    Hello <blink>foo</blink>!

    ' - assert t.render() == escaped_out - assert text_type(t.module) == escaped_out - assert escape(t.module) == escaped_out - assert t.module.say_hello('foo') == escaped_out - assert escape(t.module.say_hello('foo')) == escaped_out - - def test_attr_filter(self): - env = SandboxedEnvironment() - tmpl = env.from_string('{{ cls|attr("__subclasses__")() }}') - self.assert_raises(SecurityError, tmpl.render, cls=int) - - def test_binary_operator_intercepting(self): - def disable_op(left, right): - raise TemplateRuntimeError('that operator so does not work') - for expr, ctx, rv in ('1 + 2', {}, '3'), ('a + 2', {'a': 2}, '4'): - env = SandboxedEnvironment() - env.binop_table['+'] = disable_op - t = env.from_string('{{ %s }}' % expr) - assert t.render(ctx) == rv - env.intercepted_binops = frozenset(['+']) - t = env.from_string('{{ %s }}' % expr) - try: - t.render(ctx) - except TemplateRuntimeError as e: - pass - else: - self.fail('expected runtime error') - - def test_unary_operator_intercepting(self): - def disable_op(arg): - raise TemplateRuntimeError('that operator so does not work') - for expr, ctx, rv in ('-1', {}, '-1'), ('-a', {'a': 2}, '-2'): - env = SandboxedEnvironment() - env.unop_table['-'] = disable_op - t = env.from_string('{{ %s }}' % expr) - assert t.render(ctx) == rv - env.intercepted_unops = frozenset(['-']) - t = env.from_string('{{ %s }}' % expr) - try: - t.render(ctx) - except TemplateRuntimeError as e: - pass - else: - self.fail('expected runtime error') - - -def suite(): - suite = unittest.TestSuite() - suite.addTest(unittest.makeSuite(SandboxTestCase)) - return suite diff --git a/pyload/lib/jinja2/testsuite/tests.py b/pyload/lib/jinja2/testsuite/tests.py deleted file mode 100644 index 3ece7a8ff..000000000 --- a/pyload/lib/jinja2/testsuite/tests.py +++ /dev/null @@ -1,93 +0,0 @@ -# -*- coding: utf-8 -*- -""" - jinja2.testsuite.tests - ~~~~~~~~~~~~~~~~~~~~~~ - - Who tests the tests? - - :copyright: (c) 2010 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" -import unittest -from jinja2.testsuite import JinjaTestCase - -from jinja2 import Markup, Environment - -env = Environment() - - -class TestsTestCase(JinjaTestCase): - - def test_defined(self): - tmpl = env.from_string('{{ missing is defined }}|{{ true is defined }}') - assert tmpl.render() == 'False|True' - - def test_even(self): - tmpl = env.from_string('''{{ 1 is even }}|{{ 2 is even }}''') - assert tmpl.render() == 'False|True' - - def test_odd(self): - tmpl = env.from_string('''{{ 1 is odd }}|{{ 2 is odd }}''') - assert tmpl.render() == 'True|False' - - def test_lower(self): - tmpl = env.from_string('''{{ "foo" is lower }}|{{ "FOO" is lower }}''') - assert tmpl.render() == 'True|False' - - def test_typechecks(self): - tmpl = env.from_string(''' - {{ 42 is undefined }} - {{ 42 is defined }} - {{ 42 is none }} - {{ none is none }} - {{ 42 is number }} - {{ 42 is string }} - {{ "foo" is string }} - {{ "foo" is sequence }} - {{ [1] is sequence }} - {{ range is callable }} - {{ 42 is callable }} - {{ range(5) is iterable }} - {{ {} is mapping }} - {{ mydict is mapping }} - {{ [] is mapping }} - ''') - class MyDict(dict): - pass - assert tmpl.render(mydict=MyDict()).split() == [ - 'False', 'True', 'False', 'True', 'True', 'False', - 'True', 'True', 'True', 'True', 'False', 'True', - 'True', 'True', 'False' - ] - - def test_sequence(self): - tmpl = env.from_string( - '{{ [1, 2, 3] is sequence }}|' - '{{ "foo" is sequence }}|' - '{{ 42 is sequence }}' - ) - assert tmpl.render() == 'True|True|False' - - def test_upper(self): - tmpl = env.from_string('{{ "FOO" is upper }}|{{ "foo" is upper }}') - assert tmpl.render() == 'True|False' - - def test_sameas(self): - tmpl = env.from_string('{{ foo is sameas false }}|' - '{{ 0 is sameas false }}') - assert tmpl.render(foo=False) == 'True|False' - - def test_no_paren_for_arg1(self): - tmpl = env.from_string('{{ foo is sameas none }}') - assert tmpl.render(foo=None) == 'True' - - def test_escaped(self): - env = Environment(autoescape=True) - tmpl = env.from_string('{{ x is escaped }}|{{ y is escaped }}') - assert tmpl.render(x='foo', y=Markup('foo')) == 'False|True' - - -def suite(): - suite = unittest.TestSuite() - suite.addTest(unittest.makeSuite(TestsTestCase)) - return suite diff --git a/pyload/lib/jinja2/testsuite/utils.py b/pyload/lib/jinja2/testsuite/utils.py deleted file mode 100644 index cab9b09a9..000000000 --- a/pyload/lib/jinja2/testsuite/utils.py +++ /dev/null @@ -1,82 +0,0 @@ -# -*- coding: utf-8 -*- -""" - jinja2.testsuite.utils - ~~~~~~~~~~~~~~~~~~~~~~ - - Tests utilities jinja uses. - - :copyright: (c) 2010 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" -import gc -import unittest - -import pickle - -from jinja2.testsuite import JinjaTestCase - -from jinja2.utils import LRUCache, escape, object_type_repr - - -class LRUCacheTestCase(JinjaTestCase): - - def test_simple(self): - d = LRUCache(3) - d["a"] = 1 - d["b"] = 2 - d["c"] = 3 - d["a"] - d["d"] = 4 - assert len(d) == 3 - assert 'a' in d and 'c' in d and 'd' in d and 'b' not in d - - def test_pickleable(self): - cache = LRUCache(2) - cache["foo"] = 42 - cache["bar"] = 23 - cache["foo"] - - for protocol in range(3): - copy = pickle.loads(pickle.dumps(cache, protocol)) - assert copy.capacity == cache.capacity - assert copy._mapping == cache._mapping - assert copy._queue == cache._queue - - -class HelpersTestCase(JinjaTestCase): - - def test_object_type_repr(self): - class X(object): - pass - self.assert_equal(object_type_repr(42), 'int object') - self.assert_equal(object_type_repr([]), 'list object') - self.assert_equal(object_type_repr(X()), - 'jinja2.testsuite.utils.X object') - self.assert_equal(object_type_repr(None), 'None') - self.assert_equal(object_type_repr(Ellipsis), 'Ellipsis') - - -class MarkupLeakTestCase(JinjaTestCase): - - def test_markup_leaks(self): - counts = set() - for count in range(20): - for item in range(1000): - escape("foo") - escape("") - escape(u"foo") - escape(u"") - counts.add(len(gc.get_objects())) - assert len(counts) == 1, 'ouch, c extension seems to leak objects' - - -def suite(): - suite = unittest.TestSuite() - suite.addTest(unittest.makeSuite(LRUCacheTestCase)) - suite.addTest(unittest.makeSuite(HelpersTestCase)) - - # this test only tests the c extension - if not hasattr(escape, 'func_code'): - suite.addTest(unittest.makeSuite(MarkupLeakTestCase)) - - return suite diff --git a/pyload/lib/jinja2/utils.py b/pyload/lib/jinja2/utils.py deleted file mode 100644 index ddc47da0a..000000000 --- a/pyload/lib/jinja2/utils.py +++ /dev/null @@ -1,520 +0,0 @@ -# -*- coding: utf-8 -*- -""" - jinja2.utils - ~~~~~~~~~~~~ - - Utility functions. - - :copyright: (c) 2010 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" -import re -import errno -from collections import deque -from jinja2._compat import text_type, string_types, implements_iterator, \ - allocate_lock, url_quote - - -_word_split_re = re.compile(r'(\s+)') -_punctuation_re = re.compile( - '^(?P(?:%s)*)(?P.*?)(?P(?:%s)*)$' % ( - '|'.join(map(re.escape, ('(', '<', '<'))), - '|'.join(map(re.escape, ('.', ',', ')', '>', '\n', '>'))) - ) -) -_simple_email_re = re.compile(r'^\S+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+$') -_striptags_re = re.compile(r'(|<[^>]*>)') -_entity_re = re.compile(r'&([^;]+);') -_letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' -_digits = '0123456789' - -# special singleton representing missing values for the runtime -missing = type('MissingType', (), {'__repr__': lambda x: 'missing'})() - -# internal code -internal_code = set() - -concat = u''.join - - -def contextfunction(f): - """This decorator can be used to mark a function or method context callable. - A context callable is passed the active :class:`Context` as first argument when - called from the template. This is useful if a function wants to get access - to the context or functions provided on the context object. For example - a function that returns a sorted list of template variables the current - template exports could look like this:: - - @contextfunction - def get_exported_names(context): - return sorted(context.exported_vars) - """ - f.contextfunction = True - return f - - -def evalcontextfunction(f): - """This decorator can be used to mark a function or method as an eval - context callable. This is similar to the :func:`contextfunction` - but instead of passing the context, an evaluation context object is - passed. For more information about the eval context, see - :ref:`eval-context`. - - .. versionadded:: 2.4 - """ - f.evalcontextfunction = True - return f - - -def environmentfunction(f): - """This decorator can be used to mark a function or method as environment - callable. This decorator works exactly like the :func:`contextfunction` - decorator just that the first argument is the active :class:`Environment` - and not context. - """ - f.environmentfunction = True - return f - - -def internalcode(f): - """Marks the function as internally used""" - internal_code.add(f.__code__) - return f - - -def is_undefined(obj): - """Check if the object passed is undefined. This does nothing more than - performing an instance check against :class:`Undefined` but looks nicer. - This can be used for custom filters or tests that want to react to - undefined variables. For example a custom default filter can look like - this:: - - def default(var, default=''): - if is_undefined(var): - return default - return var - """ - from jinja2.runtime import Undefined - return isinstance(obj, Undefined) - - -def consume(iterable): - """Consumes an iterable without doing anything with it.""" - for event in iterable: - pass - - -def clear_caches(): - """Jinja2 keeps internal caches for environments and lexers. These are - used so that Jinja2 doesn't have to recreate environments and lexers all - the time. Normally you don't have to care about that but if you are - messuring memory consumption you may want to clean the caches. - """ - from jinja2.environment import _spontaneous_environments - from jinja2.lexer import _lexer_cache - _spontaneous_environments.clear() - _lexer_cache.clear() - - -def import_string(import_name, silent=False): - """Imports an object based on a string. This is useful if you want to - use import paths as endpoints or something similar. An import path can - be specified either in dotted notation (``xml.sax.saxutils.escape``) - or with a colon as object delimiter (``xml.sax.saxutils:escape``). - - If the `silent` is True the return value will be `None` if the import - fails. - - :return: imported object - """ - try: - if ':' in import_name: - module, obj = import_name.split(':', 1) - elif '.' in import_name: - items = import_name.split('.') - module = '.'.join(items[:-1]) - obj = items[-1] - else: - return __import__(import_name) - return getattr(__import__(module, None, None, [obj]), obj) - except (ImportError, AttributeError): - if not silent: - raise - - -def open_if_exists(filename, mode='rb'): - """Returns a file descriptor for the filename if that file exists, - otherwise `None`. - """ - try: - return open(filename, mode) - except IOError as e: - if e.errno not in (errno.ENOENT, errno.EISDIR): - raise - - -def object_type_repr(obj): - """Returns the name of the object's type. For some recognized - singletons the name of the object is returned instead. (For - example for `None` and `Ellipsis`). - """ - if obj is None: - return 'None' - elif obj is Ellipsis: - return 'Ellipsis' - # __builtin__ in 2.x, builtins in 3.x - if obj.__class__.__module__ in ('__builtin__', 'builtins'): - name = obj.__class__.__name__ - else: - name = obj.__class__.__module__ + '.' + obj.__class__.__name__ - return '%s object' % name - - -def pformat(obj, verbose=False): - """Prettyprint an object. Either use the `pretty` library or the - builtin `pprint`. - """ - try: - from pretty import pretty - return pretty(obj, verbose=verbose) - except ImportError: - from pprint import pformat - return pformat(obj) - - -def urlize(text, trim_url_limit=None, nofollow=False): - """Converts any URLs in text into clickable links. Works on http://, - https:// and www. links. Links can have trailing punctuation (periods, - commas, close-parens) and leading punctuation (opening parens) and - it'll still do the right thing. - - If trim_url_limit is not None, the URLs in link text will be limited - to trim_url_limit characters. - - If nofollow is True, the URLs in link text will get a rel="nofollow" - attribute. - """ - trim_url = lambda x, limit=trim_url_limit: limit is not None \ - and (x[:limit] + (len(x) >=limit and '...' - or '')) or x - words = _word_split_re.split(text_type(escape(text))) - nofollow_attr = nofollow and ' rel="nofollow"' or '' - for i, word in enumerate(words): - match = _punctuation_re.match(word) - if match: - lead, middle, trail = match.groups() - if middle.startswith('www.') or ( - '@' not in middle and - not middle.startswith('http://') and - not middle.startswith('https://') and - len(middle) > 0 and - middle[0] in _letters + _digits and ( - middle.endswith('.org') or - middle.endswith('.net') or - middle.endswith('.com') - )): - middle = '%s' % (middle, - nofollow_attr, trim_url(middle)) - if middle.startswith('http://') or \ - middle.startswith('https://'): - middle = '%s' % (middle, - nofollow_attr, trim_url(middle)) - if '@' in middle and not middle.startswith('www.') and \ - not ':' in middle and _simple_email_re.match(middle): - middle = '%s' % (middle, middle) - if lead + middle + trail != word: - words[i] = lead + middle + trail - return u''.join(words) - - -def generate_lorem_ipsum(n=5, html=True, min=20, max=100): - """Generate some lorem impsum for the template.""" - from jinja2.constants import LOREM_IPSUM_WORDS - from random import choice, randrange - words = LOREM_IPSUM_WORDS.split() - result = [] - - for _ in range(n): - next_capitalized = True - last_comma = last_fullstop = 0 - word = None - last = None - p = [] - - # each paragraph contains out of 20 to 100 words. - for idx, _ in enumerate(range(randrange(min, max))): - while True: - word = choice(words) - if word != last: - last = word - break - if next_capitalized: - word = word.capitalize() - next_capitalized = False - # add commas - if idx - randrange(3, 8) > last_comma: - last_comma = idx - last_fullstop += 2 - word += ',' - # add end of sentences - if idx - randrange(10, 20) > last_fullstop: - last_comma = last_fullstop = idx - word += '.' - next_capitalized = True - p.append(word) - - # ensure that the paragraph ends with a dot. - p = u' '.join(p) - if p.endswith(','): - p = p[:-1] + '.' - elif not p.endswith('.'): - p += '.' - result.append(p) - - if not html: - return u'\n\n'.join(result) - return Markup(u'\n'.join(u'

    %s

    ' % escape(x) for x in result)) - - -def unicode_urlencode(obj, charset='utf-8'): - """URL escapes a single bytestring or unicode string with the - given charset if applicable to URL safe quoting under all rules - that need to be considered under all supported Python versions. - - If non strings are provided they are converted to their unicode - representation first. - """ - if not isinstance(obj, string_types): - obj = text_type(obj) - if isinstance(obj, text_type): - obj = obj.encode(charset) - return text_type(url_quote(obj)) - - -class LRUCache(object): - """A simple LRU Cache implementation.""" - - # this is fast for small capacities (something below 1000) but doesn't - # scale. But as long as it's only used as storage for templates this - # won't do any harm. - - def __init__(self, capacity): - self.capacity = capacity - self._mapping = {} - self._queue = deque() - self._postinit() - - def _postinit(self): - # alias all queue methods for faster lookup - self._popleft = self._queue.popleft - self._pop = self._queue.pop - self._remove = self._queue.remove - self._wlock = allocate_lock() - self._append = self._queue.append - - def __getstate__(self): - return { - 'capacity': self.capacity, - '_mapping': self._mapping, - '_queue': self._queue - } - - def __setstate__(self, d): - self.__dict__.update(d) - self._postinit() - - def __getnewargs__(self): - return (self.capacity,) - - def copy(self): - """Return a shallow copy of the instance.""" - rv = self.__class__(self.capacity) - rv._mapping.update(self._mapping) - rv._queue = deque(self._queue) - return rv - - def get(self, key, default=None): - """Return an item from the cache dict or `default`""" - try: - return self[key] - except KeyError: - return default - - def setdefault(self, key, default=None): - """Set `default` if the key is not in the cache otherwise - leave unchanged. Return the value of this key. - """ - self._wlock.acquire() - try: - try: - return self[key] - except KeyError: - self[key] = default - return default - finally: - self._wlock.release() - - def clear(self): - """Clear the cache.""" - self._wlock.acquire() - try: - self._mapping.clear() - self._queue.clear() - finally: - self._wlock.release() - - def __contains__(self, key): - """Check if a key exists in this cache.""" - return key in self._mapping - - def __len__(self): - """Return the current size of the cache.""" - return len(self._mapping) - - def __repr__(self): - return '<%s %r>' % ( - self.__class__.__name__, - self._mapping - ) - - def __getitem__(self, key): - """Get an item from the cache. Moves the item up so that it has the - highest priority then. - - Raise a `KeyError` if it does not exist. - """ - self._wlock.acquire() - try: - rv = self._mapping[key] - if self._queue[-1] != key: - try: - self._remove(key) - except ValueError: - # if something removed the key from the container - # when we read, ignore the ValueError that we would - # get otherwise. - pass - self._append(key) - return rv - finally: - self._wlock.release() - - def __setitem__(self, key, value): - """Sets the value for an item. Moves the item up so that it - has the highest priority then. - """ - self._wlock.acquire() - try: - if key in self._mapping: - self._remove(key) - elif len(self._mapping) == self.capacity: - del self._mapping[self._popleft()] - self._append(key) - self._mapping[key] = value - finally: - self._wlock.release() - - def __delitem__(self, key): - """Remove an item from the cache dict. - Raise a `KeyError` if it does not exist. - """ - self._wlock.acquire() - try: - del self._mapping[key] - try: - self._remove(key) - except ValueError: - # __getitem__ is not locked, it might happen - pass - finally: - self._wlock.release() - - def items(self): - """Return a list of items.""" - result = [(key, self._mapping[key]) for key in list(self._queue)] - result.reverse() - return result - - def iteritems(self): - """Iterate over all items.""" - return iter(self.items()) - - def values(self): - """Return a list of all values.""" - return [x[1] for x in self.items()] - - def itervalue(self): - """Iterate over all values.""" - return iter(self.values()) - - def keys(self): - """Return a list of all keys ordered by most recent usage.""" - return list(self) - - def iterkeys(self): - """Iterate over all keys in the cache dict, ordered by - the most recent usage. - """ - return reversed(tuple(self._queue)) - - __iter__ = iterkeys - - def __reversed__(self): - """Iterate over the values in the cache dict, oldest items - coming first. - """ - return iter(tuple(self._queue)) - - __copy__ = copy - - -# register the LRU cache as mutable mapping if possible -try: - from collections import MutableMapping - MutableMapping.register(LRUCache) -except ImportError: - pass - - -@implements_iterator -class Cycler(object): - """A cycle helper for templates.""" - - def __init__(self, *items): - if not items: - raise RuntimeError('at least one item has to be provided') - self.items = items - self.reset() - - def reset(self): - """Resets the cycle.""" - self.pos = 0 - - @property - def current(self): - """Returns the current item.""" - return self.items[self.pos] - - def __next__(self): - """Goes one item ahead and returns it.""" - rv = self.current - self.pos = (self.pos + 1) % len(self.items) - return rv - - -class Joiner(object): - """A joining helper for templates.""" - - def __init__(self, sep=u', '): - self.sep = sep - self.used = False - - def __call__(self): - if not self.used: - self.used = True - return u'' - return self.sep - - -# Imported here because that's where it was in the past -from markupsafe import Markup, escape, soft_unicode diff --git a/pyload/lib/jinja2/visitor.py b/pyload/lib/jinja2/visitor.py deleted file mode 100644 index 413e7c309..000000000 --- a/pyload/lib/jinja2/visitor.py +++ /dev/null @@ -1,87 +0,0 @@ -# -*- coding: utf-8 -*- -""" - jinja2.visitor - ~~~~~~~~~~~~~~ - - This module implements a visitor for the nodes. - - :copyright: (c) 2010 by the Jinja Team. - :license: BSD. -""" -from jinja2.nodes import Node - - -class NodeVisitor(object): - """Walks the abstract syntax tree and call visitor functions for every - node found. The visitor functions may return values which will be - forwarded by the `visit` method. - - Per default the visitor functions for the nodes are ``'visit_'`` + - class name of the node. So a `TryFinally` node visit function would - be `visit_TryFinally`. This behavior can be changed by overriding - the `get_visitor` function. If no visitor function exists for a node - (return value `None`) the `generic_visit` visitor is used instead. - """ - - def get_visitor(self, node): - """Return the visitor function for this node or `None` if no visitor - exists for this node. In that case the generic visit function is - used instead. - """ - method = 'visit_' + node.__class__.__name__ - return getattr(self, method, None) - - def visit(self, node, *args, **kwargs): - """Visit a node.""" - f = self.get_visitor(node) - if f is not None: - return f(node, *args, **kwargs) - return self.generic_visit(node, *args, **kwargs) - - def generic_visit(self, node, *args, **kwargs): - """Called if no explicit visitor function exists for a node.""" - for node in node.iter_child_nodes(): - self.visit(node, *args, **kwargs) - - -class NodeTransformer(NodeVisitor): - """Walks the abstract syntax tree and allows modifications of nodes. - - The `NodeTransformer` will walk the AST and use the return value of the - visitor functions to replace or remove the old node. If the return - value of the visitor function is `None` the node will be removed - from the previous location otherwise it's replaced with the return - value. The return value may be the original node in which case no - replacement takes place. - """ - - def generic_visit(self, node, *args, **kwargs): - for field, old_value in node.iter_fields(): - if isinstance(old_value, list): - new_values = [] - for value in old_value: - if isinstance(value, Node): - value = self.visit(value, *args, **kwargs) - if value is None: - continue - elif not isinstance(value, Node): - new_values.extend(value) - continue - new_values.append(value) - old_value[:] = new_values - elif isinstance(old_value, Node): - new_node = self.visit(old_value, *args, **kwargs) - if new_node is None: - delattr(node, field) - else: - setattr(node, field, new_node) - return node - - def visit_list(self, node, *args, **kwargs): - """As transformers may return lists in some places this method - can be used to enforce a list as return value. - """ - rv = self.visit(node, *args, **kwargs) - if not isinstance(rv, list): - rv = [rv] - return rv diff --git a/pyload/lib/markupsafe/__init__.py b/pyload/lib/markupsafe/__init__.py deleted file mode 100644 index 275540154..000000000 --- a/pyload/lib/markupsafe/__init__.py +++ /dev/null @@ -1,298 +0,0 @@ -# -*- coding: utf-8 -*- -""" - markupsafe - ~~~~~~~~~~ - - Implements a Markup string. - - :copyright: (c) 2010 by Armin Ronacher. - :license: BSD, see LICENSE for more details. -""" -import re -import string -from collections import Mapping -from markupsafe._compat import text_type, string_types, int_types, \ - unichr, iteritems, PY2 - - -__all__ = ['Markup', 'soft_unicode', 'escape', 'escape_silent'] - - -_striptags_re = re.compile(r'(|<[^>]*>)') -_entity_re = re.compile(r'&([^;]+);') - - -class Markup(text_type): - r"""Marks a string as being safe for inclusion in HTML/XML output without - needing to be escaped. This implements the `__html__` interface a couple - of frameworks and web applications use. :class:`Markup` is a direct - subclass of `unicode` and provides all the methods of `unicode` just that - it escapes arguments passed and always returns `Markup`. - - The `escape` function returns markup objects so that double escaping can't - happen. - - The constructor of the :class:`Markup` class can be used for three - different things: When passed an unicode object it's assumed to be safe, - when passed an object with an HTML representation (has an `__html__` - method) that representation is used, otherwise the object passed is - converted into a unicode string and then assumed to be safe: - - >>> Markup("Hello World!") - Markup(u'Hello World!') - >>> class Foo(object): - ... def __html__(self): - ... return 'foo' - ... - >>> Markup(Foo()) - Markup(u'foo') - - If you want object passed being always treated as unsafe you can use the - :meth:`escape` classmethod to create a :class:`Markup` object: - - >>> Markup.escape("Hello World!") - Markup(u'Hello <em>World</em>!') - - Operations on a markup string are markup aware which means that all - arguments are passed through the :func:`escape` function: - - >>> em = Markup("%s") - >>> em % "foo & bar" - Markup(u'foo & bar') - >>> strong = Markup("%(text)s") - >>> strong % {'text': 'hacker here'} - Markup(u'<blink>hacker here</blink>') - >>> Markup("Hello ") + "" - Markup(u'Hello <foo>') - """ - __slots__ = () - - def __new__(cls, base=u'', encoding=None, errors='strict'): - if hasattr(base, '__html__'): - base = base.__html__() - if encoding is None: - return text_type.__new__(cls, base) - return text_type.__new__(cls, base, encoding, errors) - - def __html__(self): - return self - - def __add__(self, other): - if isinstance(other, string_types) or hasattr(other, '__html__'): - return self.__class__(super(Markup, self).__add__(self.escape(other))) - return NotImplemented - - def __radd__(self, other): - if hasattr(other, '__html__') or isinstance(other, string_types): - return self.escape(other).__add__(self) - return NotImplemented - - def __mul__(self, num): - if isinstance(num, int_types): - return self.__class__(text_type.__mul__(self, num)) - return NotImplemented - __rmul__ = __mul__ - - def __mod__(self, arg): - if isinstance(arg, tuple): - arg = tuple(_MarkupEscapeHelper(x, self.escape) for x in arg) - else: - arg = _MarkupEscapeHelper(arg, self.escape) - return self.__class__(text_type.__mod__(self, arg)) - - def __repr__(self): - return '%s(%s)' % ( - self.__class__.__name__, - text_type.__repr__(self) - ) - - def join(self, seq): - return self.__class__(text_type.join(self, map(self.escape, seq))) - join.__doc__ = text_type.join.__doc__ - - def split(self, *args, **kwargs): - return list(map(self.__class__, text_type.split(self, *args, **kwargs))) - split.__doc__ = text_type.split.__doc__ - - def rsplit(self, *args, **kwargs): - return list(map(self.__class__, text_type.rsplit(self, *args, **kwargs))) - rsplit.__doc__ = text_type.rsplit.__doc__ - - def splitlines(self, *args, **kwargs): - return list(map(self.__class__, text_type.splitlines( - self, *args, **kwargs))) - splitlines.__doc__ = text_type.splitlines.__doc__ - - def unescape(self): - r"""Unescape markup again into an text_type string. This also resolves - known HTML4 and XHTML entities: - - >>> Markup("Main » About").unescape() - u'Main \xbb About' - """ - from markupsafe._constants import HTML_ENTITIES - def handle_match(m): - name = m.group(1) - if name in HTML_ENTITIES: - return unichr(HTML_ENTITIES[name]) - try: - if name[:2] in ('#x', '#X'): - return unichr(int(name[2:], 16)) - elif name.startswith('#'): - return unichr(int(name[1:])) - except ValueError: - pass - return u'' - return _entity_re.sub(handle_match, text_type(self)) - - def striptags(self): - r"""Unescape markup into an text_type string and strip all tags. This - also resolves known HTML4 and XHTML entities. Whitespace is - normalized to one: - - >>> Markup("Main » About").striptags() - u'Main \xbb About' - """ - stripped = u' '.join(_striptags_re.sub('', self).split()) - return Markup(stripped).unescape() - - @classmethod - def escape(cls, s): - """Escape the string. Works like :func:`escape` with the difference - that for subclasses of :class:`Markup` this function would return the - correct subclass. - """ - rv = escape(s) - if rv.__class__ is not cls: - return cls(rv) - return rv - - def make_simple_escaping_wrapper(name): - orig = getattr(text_type, name) - def func(self, *args, **kwargs): - args = _escape_argspec(list(args), enumerate(args), self.escape) - _escape_argspec(kwargs, iteritems(kwargs), self.escape) - return self.__class__(orig(self, *args, **kwargs)) - func.__name__ = orig.__name__ - func.__doc__ = orig.__doc__ - return func - - for method in '__getitem__', 'capitalize', \ - 'title', 'lower', 'upper', 'replace', 'ljust', \ - 'rjust', 'lstrip', 'rstrip', 'center', 'strip', \ - 'translate', 'expandtabs', 'swapcase', 'zfill': - locals()[method] = make_simple_escaping_wrapper(method) - - # new in python 2.5 - if hasattr(text_type, 'partition'): - def partition(self, sep): - return tuple(map(self.__class__, - text_type.partition(self, self.escape(sep)))) - def rpartition(self, sep): - return tuple(map(self.__class__, - text_type.rpartition(self, self.escape(sep)))) - - # new in python 2.6 - if hasattr(text_type, 'format'): - def format(*args, **kwargs): - self, args = args[0], args[1:] - formatter = EscapeFormatter(self.escape) - kwargs = _MagicFormatMapping(args, kwargs) - return self.__class__(formatter.vformat(self, args, kwargs)) - - def __html_format__(self, format_spec): - if format_spec: - raise ValueError('Unsupported format specification ' - 'for Markup.') - return self - - # not in python 3 - if hasattr(text_type, '__getslice__'): - __getslice__ = make_simple_escaping_wrapper('__getslice__') - - del method, make_simple_escaping_wrapper - - -class _MagicFormatMapping(Mapping): - """This class implements a dummy wrapper to fix a bug in the Python - standard library for string formatting. - - See http://bugs.python.org/issue13598 for information about why - this is necessary. - """ - - def __init__(self, args, kwargs): - self._args = args - self._kwargs = kwargs - self._last_index = 0 - - def __getitem__(self, key): - if key == '': - idx = self._last_index - self._last_index += 1 - try: - return self._args[idx] - except LookupError: - pass - key = str(idx) - return self._kwargs[key] - - def __iter__(self): - return iter(self._kwargs) - - def __len__(self): - return len(self._kwargs) - - -if hasattr(text_type, 'format'): - class EscapeFormatter(string.Formatter): - - def __init__(self, escape): - self.escape = escape - - def format_field(self, value, format_spec): - if hasattr(value, '__html_format__'): - rv = value.__html_format__(format_spec) - elif hasattr(value, '__html__'): - if format_spec: - raise ValueError('No format specification allowed ' - 'when formatting an object with ' - 'its __html__ method.') - rv = value.__html__() - else: - rv = string.Formatter.format_field(self, value, format_spec) - return text_type(self.escape(rv)) - - -def _escape_argspec(obj, iterable, escape): - """Helper for various string-wrapped functions.""" - for key, value in iterable: - if hasattr(value, '__html__') or isinstance(value, string_types): - obj[key] = escape(value) - return obj - - -class _MarkupEscapeHelper(object): - """Helper for Markup.__mod__""" - - def __init__(self, obj, escape): - self.obj = obj - self.escape = escape - - __getitem__ = lambda s, x: _MarkupEscapeHelper(s.obj[x], s.escape) - __unicode__ = __str__ = lambda s: text_type(s.escape(s.obj)) - __repr__ = lambda s: str(s.escape(repr(s.obj))) - __int__ = lambda s: int(s.obj) - __float__ = lambda s: float(s.obj) - - -# we have to import it down here as the speedups and native -# modules imports the markup type which is define above. -try: - from markupsafe._speedups import escape, escape_silent, soft_unicode -except ImportError: - from markupsafe._native import escape, escape_silent, soft_unicode - -if not PY2: - soft_str = soft_unicode - __all__.append('soft_str') diff --git a/pyload/lib/markupsafe/_compat.py b/pyload/lib/markupsafe/_compat.py deleted file mode 100644 index 62e5632ad..000000000 --- a/pyload/lib/markupsafe/_compat.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- -""" - markupsafe._compat - ~~~~~~~~~~~~~~~~~~ - - Compatibility module for different Python versions. - - :copyright: (c) 2013 by Armin Ronacher. - :license: BSD, see LICENSE for more details. -""" -import sys - -PY2 = sys.version_info[0] == 2 - -if not PY2: - text_type = str - string_types = (str,) - unichr = chr - int_types = (int,) - iteritems = lambda x: iter(x.items()) -else: - text_type = unicode - string_types = (str, unicode) - unichr = unichr - int_types = (int, long) - iteritems = lambda x: x.iteritems() diff --git a/pyload/lib/markupsafe/_constants.py b/pyload/lib/markupsafe/_constants.py deleted file mode 100644 index 919bf03c5..000000000 --- a/pyload/lib/markupsafe/_constants.py +++ /dev/null @@ -1,267 +0,0 @@ -# -*- coding: utf-8 -*- -""" - markupsafe._constants - ~~~~~~~~~~~~~~~~~~~~~ - - Highlevel implementation of the Markup string. - - :copyright: (c) 2010 by Armin Ronacher. - :license: BSD, see LICENSE for more details. -""" - - -HTML_ENTITIES = { - 'AElig': 198, - 'Aacute': 193, - 'Acirc': 194, - 'Agrave': 192, - 'Alpha': 913, - 'Aring': 197, - 'Atilde': 195, - 'Auml': 196, - 'Beta': 914, - 'Ccedil': 199, - 'Chi': 935, - 'Dagger': 8225, - 'Delta': 916, - 'ETH': 208, - 'Eacute': 201, - 'Ecirc': 202, - 'Egrave': 200, - 'Epsilon': 917, - 'Eta': 919, - 'Euml': 203, - 'Gamma': 915, - 'Iacute': 205, - 'Icirc': 206, - 'Igrave': 204, - 'Iota': 921, - 'Iuml': 207, - 'Kappa': 922, - 'Lambda': 923, - 'Mu': 924, - 'Ntilde': 209, - 'Nu': 925, - 'OElig': 338, - 'Oacute': 211, - 'Ocirc': 212, - 'Ograve': 210, - 'Omega': 937, - 'Omicron': 927, - 'Oslash': 216, - 'Otilde': 213, - 'Ouml': 214, - 'Phi': 934, - 'Pi': 928, - 'Prime': 8243, - 'Psi': 936, - 'Rho': 929, - 'Scaron': 352, - 'Sigma': 931, - 'THORN': 222, - 'Tau': 932, - 'Theta': 920, - 'Uacute': 218, - 'Ucirc': 219, - 'Ugrave': 217, - 'Upsilon': 933, - 'Uuml': 220, - 'Xi': 926, - 'Yacute': 221, - 'Yuml': 376, - 'Zeta': 918, - 'aacute': 225, - 'acirc': 226, - 'acute': 180, - 'aelig': 230, - 'agrave': 224, - 'alefsym': 8501, - 'alpha': 945, - 'amp': 38, - 'and': 8743, - 'ang': 8736, - 'apos': 39, - 'aring': 229, - 'asymp': 8776, - 'atilde': 227, - 'auml': 228, - 'bdquo': 8222, - 'beta': 946, - 'brvbar': 166, - 'bull': 8226, - 'cap': 8745, - 'ccedil': 231, - 'cedil': 184, - 'cent': 162, - 'chi': 967, - 'circ': 710, - 'clubs': 9827, - 'cong': 8773, - 'copy': 169, - 'crarr': 8629, - 'cup': 8746, - 'curren': 164, - 'dArr': 8659, - 'dagger': 8224, - 'darr': 8595, - 'deg': 176, - 'delta': 948, - 'diams': 9830, - 'divide': 247, - 'eacute': 233, - 'ecirc': 234, - 'egrave': 232, - 'empty': 8709, - 'emsp': 8195, - 'ensp': 8194, - 'epsilon': 949, - 'equiv': 8801, - 'eta': 951, - 'eth': 240, - 'euml': 235, - 'euro': 8364, - 'exist': 8707, - 'fnof': 402, - 'forall': 8704, - 'frac12': 189, - 'frac14': 188, - 'frac34': 190, - 'frasl': 8260, - 'gamma': 947, - 'ge': 8805, - 'gt': 62, - 'hArr': 8660, - 'harr': 8596, - 'hearts': 9829, - 'hellip': 8230, - 'iacute': 237, - 'icirc': 238, - 'iexcl': 161, - 'igrave': 236, - 'image': 8465, - 'infin': 8734, - 'int': 8747, - 'iota': 953, - 'iquest': 191, - 'isin': 8712, - 'iuml': 239, - 'kappa': 954, - 'lArr': 8656, - 'lambda': 955, - 'lang': 9001, - 'laquo': 171, - 'larr': 8592, - 'lceil': 8968, - 'ldquo': 8220, - 'le': 8804, - 'lfloor': 8970, - 'lowast': 8727, - 'loz': 9674, - 'lrm': 8206, - 'lsaquo': 8249, - 'lsquo': 8216, - 'lt': 60, - 'macr': 175, - 'mdash': 8212, - 'micro': 181, - 'middot': 183, - 'minus': 8722, - 'mu': 956, - 'nabla': 8711, - 'nbsp': 160, - 'ndash': 8211, - 'ne': 8800, - 'ni': 8715, - 'not': 172, - 'notin': 8713, - 'nsub': 8836, - 'ntilde': 241, - 'nu': 957, - 'oacute': 243, - 'ocirc': 244, - 'oelig': 339, - 'ograve': 242, - 'oline': 8254, - 'omega': 969, - 'omicron': 959, - 'oplus': 8853, - 'or': 8744, - 'ordf': 170, - 'ordm': 186, - 'oslash': 248, - 'otilde': 245, - 'otimes': 8855, - 'ouml': 246, - 'para': 182, - 'part': 8706, - 'permil': 8240, - 'perp': 8869, - 'phi': 966, - 'pi': 960, - 'piv': 982, - 'plusmn': 177, - 'pound': 163, - 'prime': 8242, - 'prod': 8719, - 'prop': 8733, - 'psi': 968, - 'quot': 34, - 'rArr': 8658, - 'radic': 8730, - 'rang': 9002, - 'raquo': 187, - 'rarr': 8594, - 'rceil': 8969, - 'rdquo': 8221, - 'real': 8476, - 'reg': 174, - 'rfloor': 8971, - 'rho': 961, - 'rlm': 8207, - 'rsaquo': 8250, - 'rsquo': 8217, - 'sbquo': 8218, - 'scaron': 353, - 'sdot': 8901, - 'sect': 167, - 'shy': 173, - 'sigma': 963, - 'sigmaf': 962, - 'sim': 8764, - 'spades': 9824, - 'sub': 8834, - 'sube': 8838, - 'sum': 8721, - 'sup': 8835, - 'sup1': 185, - 'sup2': 178, - 'sup3': 179, - 'supe': 8839, - 'szlig': 223, - 'tau': 964, - 'there4': 8756, - 'theta': 952, - 'thetasym': 977, - 'thinsp': 8201, - 'thorn': 254, - 'tilde': 732, - 'times': 215, - 'trade': 8482, - 'uArr': 8657, - 'uacute': 250, - 'uarr': 8593, - 'ucirc': 251, - 'ugrave': 249, - 'uml': 168, - 'upsih': 978, - 'upsilon': 965, - 'uuml': 252, - 'weierp': 8472, - 'xi': 958, - 'yacute': 253, - 'yen': 165, - 'yuml': 255, - 'zeta': 950, - 'zwj': 8205, - 'zwnj': 8204 -} diff --git a/pyload/lib/markupsafe/_native.py b/pyload/lib/markupsafe/_native.py deleted file mode 100644 index 5e83f10a1..000000000 --- a/pyload/lib/markupsafe/_native.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- -""" - markupsafe._native - ~~~~~~~~~~~~~~~~~~ - - Native Python implementation the C module is not compiled. - - :copyright: (c) 2010 by Armin Ronacher. - :license: BSD, see LICENSE for more details. -""" -from markupsafe import Markup -from markupsafe._compat import text_type - - -def escape(s): - """Convert the characters &, <, >, ' and " in string s to HTML-safe - sequences. Use this if you need to display text that might contain - such characters in HTML. Marks return value as markup string. - """ - if hasattr(s, '__html__'): - return s.__html__() - return Markup(text_type(s) - .replace('&', '&') - .replace('>', '>') - .replace('<', '<') - .replace("'", ''') - .replace('"', '"') - ) - - -def escape_silent(s): - """Like :func:`escape` but converts `None` into an empty - markup string. - """ - if s is None: - return Markup() - return escape(s) - - -def soft_unicode(s): - """Make a string unicode if it isn't already. That way a markup - string is not converted back to unicode. - """ - if not isinstance(s, text_type): - s = text_type(s) - return s diff --git a/pyload/lib/markupsafe/_speedups.c b/pyload/lib/markupsafe/_speedups.c deleted file mode 100644 index f349febf2..000000000 --- a/pyload/lib/markupsafe/_speedups.c +++ /dev/null @@ -1,239 +0,0 @@ -/** - * markupsafe._speedups - * ~~~~~~~~~~~~~~~~~~~~ - * - * This module implements functions for automatic escaping in C for better - * performance. - * - * :copyright: (c) 2010 by Armin Ronacher. - * :license: BSD. - */ - -#include - -#define ESCAPED_CHARS_TABLE_SIZE 63 -#define UNICHR(x) (PyUnicode_AS_UNICODE((PyUnicodeObject*)PyUnicode_DecodeASCII(x, strlen(x), NULL))); - -#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) -typedef int Py_ssize_t; -#define PY_SSIZE_T_MAX INT_MAX -#define PY_SSIZE_T_MIN INT_MIN -#endif - - -static PyObject* markup; -static Py_ssize_t escaped_chars_delta_len[ESCAPED_CHARS_TABLE_SIZE]; -static Py_UNICODE *escaped_chars_repl[ESCAPED_CHARS_TABLE_SIZE]; - -static int -init_constants(void) -{ - PyObject *module; - /* happing of characters to replace */ - escaped_chars_repl['"'] = UNICHR("""); - escaped_chars_repl['\''] = UNICHR("'"); - escaped_chars_repl['&'] = UNICHR("&"); - escaped_chars_repl['<'] = UNICHR("<"); - escaped_chars_repl['>'] = UNICHR(">"); - - /* lengths of those characters when replaced - 1 */ - memset(escaped_chars_delta_len, 0, sizeof (escaped_chars_delta_len)); - escaped_chars_delta_len['"'] = escaped_chars_delta_len['\''] = \ - escaped_chars_delta_len['&'] = 4; - escaped_chars_delta_len['<'] = escaped_chars_delta_len['>'] = 3; - - /* import markup type so that we can mark the return value */ - module = PyImport_ImportModule("markupsafe"); - if (!module) - return 0; - markup = PyObject_GetAttrString(module, "Markup"); - Py_DECREF(module); - - return 1; -} - -static PyObject* -escape_unicode(PyUnicodeObject *in) -{ - PyUnicodeObject *out; - Py_UNICODE *inp = PyUnicode_AS_UNICODE(in); - const Py_UNICODE *inp_end = PyUnicode_AS_UNICODE(in) + PyUnicode_GET_SIZE(in); - Py_UNICODE *next_escp; - Py_UNICODE *outp; - Py_ssize_t delta=0, erepl=0, delta_len=0; - - /* First we need to figure out how long the escaped string will be */ - while (*(inp) || inp < inp_end) { - if (*inp < ESCAPED_CHARS_TABLE_SIZE) { - delta += escaped_chars_delta_len[*inp]; - erepl += !!escaped_chars_delta_len[*inp]; - } - ++inp; - } - - /* Do we need to escape anything at all? */ - if (!erepl) { - Py_INCREF(in); - return (PyObject*)in; - } - - out = (PyUnicodeObject*)PyUnicode_FromUnicode(NULL, PyUnicode_GET_SIZE(in) + delta); - if (!out) - return NULL; - - outp = PyUnicode_AS_UNICODE(out); - inp = PyUnicode_AS_UNICODE(in); - while (erepl-- > 0) { - /* look for the next substitution */ - next_escp = inp; - while (next_escp < inp_end) { - if (*next_escp < ESCAPED_CHARS_TABLE_SIZE && - (delta_len = escaped_chars_delta_len[*next_escp])) { - ++delta_len; - break; - } - ++next_escp; - } - - if (next_escp > inp) { - /* copy unescaped chars between inp and next_escp */ - Py_UNICODE_COPY(outp, inp, next_escp-inp); - outp += next_escp - inp; - } - - /* escape 'next_escp' */ - Py_UNICODE_COPY(outp, escaped_chars_repl[*next_escp], delta_len); - outp += delta_len; - - inp = next_escp + 1; - } - if (inp < inp_end) - Py_UNICODE_COPY(outp, inp, PyUnicode_GET_SIZE(in) - (inp - PyUnicode_AS_UNICODE(in))); - - return (PyObject*)out; -} - - -static PyObject* -escape(PyObject *self, PyObject *text) -{ - PyObject *s = NULL, *rv = NULL, *html; - - /* we don't have to escape integers, bools or floats */ - if (PyLong_CheckExact(text) || -#if PY_MAJOR_VERSION < 3 - PyInt_CheckExact(text) || -#endif - PyFloat_CheckExact(text) || PyBool_Check(text) || - text == Py_None) - return PyObject_CallFunctionObjArgs(markup, text, NULL); - - /* if the object has an __html__ method that performs the escaping */ - html = PyObject_GetAttrString(text, "__html__"); - if (html) { - rv = PyObject_CallObject(html, NULL); - Py_DECREF(html); - return rv; - } - - /* otherwise make the object unicode if it isn't, then escape */ - PyErr_Clear(); - if (!PyUnicode_Check(text)) { -#if PY_MAJOR_VERSION < 3 - PyObject *unicode = PyObject_Unicode(text); -#else - PyObject *unicode = PyObject_Str(text); -#endif - if (!unicode) - return NULL; - s = escape_unicode((PyUnicodeObject*)unicode); - Py_DECREF(unicode); - } - else - s = escape_unicode((PyUnicodeObject*)text); - - /* convert the unicode string into a markup object. */ - rv = PyObject_CallFunctionObjArgs(markup, (PyObject*)s, NULL); - Py_DECREF(s); - return rv; -} - - -static PyObject* -escape_silent(PyObject *self, PyObject *text) -{ - if (text != Py_None) - return escape(self, text); - return PyObject_CallFunctionObjArgs(markup, NULL); -} - - -static PyObject* -soft_unicode(PyObject *self, PyObject *s) -{ - if (!PyUnicode_Check(s)) -#if PY_MAJOR_VERSION < 3 - return PyObject_Unicode(s); -#else - return PyObject_Str(s); -#endif - Py_INCREF(s); - return s; -} - - -static PyMethodDef module_methods[] = { - {"escape", (PyCFunction)escape, METH_O, - "escape(s) -> markup\n\n" - "Convert the characters &, <, >, ', and \" in string s to HTML-safe\n" - "sequences. Use this if you need to display text that might contain\n" - "such characters in HTML. Marks return value as markup string."}, - {"escape_silent", (PyCFunction)escape_silent, METH_O, - "escape_silent(s) -> markup\n\n" - "Like escape but converts None to an empty string."}, - {"soft_unicode", (PyCFunction)soft_unicode, METH_O, - "soft_unicode(object) -> string\n\n" - "Make a string unicode if it isn't already. That way a markup\n" - "string is not converted back to unicode."}, - {NULL, NULL, 0, NULL} /* Sentinel */ -}; - - -#if PY_MAJOR_VERSION < 3 - -#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ -#define PyMODINIT_FUNC void -#endif -PyMODINIT_FUNC -init_speedups(void) -{ - if (!init_constants()) - return; - - Py_InitModule3("markupsafe._speedups", module_methods, ""); -} - -#else /* Python 3.x module initialization */ - -static struct PyModuleDef module_definition = { - PyModuleDef_HEAD_INIT, - "markupsafe._speedups", - NULL, - -1, - module_methods, - NULL, - NULL, - NULL, - NULL -}; - -PyMODINIT_FUNC -PyInit__speedups(void) -{ - if (!init_constants()) - return NULL; - - return PyModule_Create(&module_definition); -} - -#endif diff --git a/pyload/lib/markupsafe/tests.py b/pyload/lib/markupsafe/tests.py deleted file mode 100644 index 636993629..000000000 --- a/pyload/lib/markupsafe/tests.py +++ /dev/null @@ -1,179 +0,0 @@ -# -*- coding: utf-8 -*- -import gc -import sys -import unittest -from markupsafe import Markup, escape, escape_silent -from markupsafe._compat import text_type - - -class MarkupTestCase(unittest.TestCase): - - def test_adding(self): - # adding two strings should escape the unsafe one - unsafe = '' - safe = Markup('username') - assert unsafe + safe == text_type(escape(unsafe)) + text_type(safe) - - def test_string_interpolation(self): - # string interpolations are safe to use too - assert Markup('%s') % '' == \ - '<bad user>' - assert Markup('%(username)s') % { - 'username': '' - } == '<bad user>' - - assert Markup('%i') % 3.14 == '3' - assert Markup('%.2f') % 3.14 == '3.14' - - def test_type_behavior(self): - # an escaped object is markup too - assert type(Markup('foo') + 'bar') is Markup - - # and it implements __html__ by returning itself - x = Markup("foo") - assert x.__html__() is x - - def test_html_interop(self): - # it also knows how to treat __html__ objects - class Foo(object): - def __html__(self): - return 'awesome' - def __unicode__(self): - return 'awesome' - __str__ = __unicode__ - assert Markup(Foo()) == 'awesome' - assert Markup('%s') % Foo() == \ - 'awesome' - - def test_tuple_interpol(self): - self.assertEqual(Markup('%s:%s') % ( - '', - '', - ), Markup(u'<foo>:<bar>')) - - def test_dict_interpol(self): - self.assertEqual(Markup('%(foo)s') % { - 'foo': '', - }, Markup(u'<foo>')) - self.assertEqual(Markup('%(foo)s:%(bar)s') % { - 'foo': '', - 'bar': '', - }, Markup(u'<foo>:<bar>')) - - def test_escaping(self): - # escaping and unescaping - assert escape('"<>&\'') == '"<>&'' - assert Markup("Foo & Bar").striptags() == "Foo & Bar" - assert Markup("<test>").unescape() == "" - - def test_formatting(self): - for actual, expected in ( - (Markup('%i') % 3.14, '3'), - (Markup('%.2f') % 3.14159, '3.14'), - (Markup('%s %s %s') % ('<', 123, '>'), '< 123 >'), - (Markup('{awesome}').format(awesome=''), - '<awesome>'), - (Markup('{0[1][bar]}').format([0, {'bar': ''}]), - '<bar/>'), - (Markup('{0[1][bar]}').format([0, {'bar': Markup('')}]), - '')): - assert actual == expected, "%r should be %r!" % (actual, expected) - - # This is new in 2.7 - if sys.version_info >= (2, 7): - def test_formatting_empty(self): - formatted = Markup('{}').format(0) - assert formatted == Markup('0') - - def test_custom_formatting(self): - class HasHTMLOnly(object): - def __html__(self): - return Markup('') - - class HasHTMLAndFormat(object): - def __html__(self): - return Markup('') - def __html_format__(self, spec): - return Markup('') - - assert Markup('{0}').format(HasHTMLOnly()) == Markup('') - assert Markup('{0}').format(HasHTMLAndFormat()) == Markup('') - - def test_complex_custom_formatting(self): - class User(object): - def __init__(self, id, username): - self.id = id - self.username = username - def __html_format__(self, format_spec): - if format_spec == 'link': - return Markup('{1}').format( - self.id, - self.__html__(), - ) - elif format_spec: - raise ValueError('Invalid format spec') - return self.__html__() - def __html__(self): - return Markup('{0}').format(self.username) - - user = User(1, 'foo') - assert Markup('

    User: {0:link}').format(user) == \ - Markup('

    User: foo') - - def test_all_set(self): - import markupsafe as markup - for item in markup.__all__: - getattr(markup, item) - - def test_escape_silent(self): - assert escape_silent(None) == Markup() - assert escape(None) == Markup(None) - assert escape_silent('') == Markup(u'<foo>') - - def test_splitting(self): - self.assertEqual(Markup('a b').split(), [ - Markup('a'), - Markup('b') - ]) - self.assertEqual(Markup('a b').rsplit(), [ - Markup('a'), - Markup('b') - ]) - self.assertEqual(Markup('a\nb').splitlines(), [ - Markup('a'), - Markup('b') - ]) - - def test_mul(self): - self.assertEqual(Markup('a') * 3, Markup('aaa')) - - -class MarkupLeakTestCase(unittest.TestCase): - - def test_markup_leaks(self): - counts = set() - for count in range(20): - for item in range(1000): - escape("foo") - escape("") - escape(u"foo") - escape(u"") - counts.add(len(gc.get_objects())) - assert len(counts) == 1, 'ouch, c extension seems to leak objects' - - -def suite(): - suite = unittest.TestSuite() - suite.addTest(unittest.makeSuite(MarkupTestCase)) - - # this test only tests the c extension - if not hasattr(escape, 'func_code'): - suite.addTest(unittest.makeSuite(MarkupLeakTestCase)) - - return suite - - -if __name__ == '__main__': - unittest.main(defaultTest='suite') - -# vim:sts=4:sw=4:et: diff --git a/pyload/lib/rename_process.py b/pyload/lib/rename_process.py deleted file mode 100644 index 2527cef39..000000000 --- a/pyload/lib/rename_process.py +++ /dev/null @@ -1,14 +0,0 @@ -import sys - -def renameProcess(new_name): - """ Renames the process calling the function to the given name. """ - if sys.platform != 'linux2': - return False - try: - from ctypes import CDLL - libc = CDLL('libc.so.6') - libc.prctl(15, new_name, 0, 0, 0) - return True - except Exception, e: - #print "Rename process failed", e - return False diff --git a/pyload/lib/simplejson/__init__.py b/pyload/lib/simplejson/__init__.py deleted file mode 100644 index a02c4deab..000000000 --- a/pyload/lib/simplejson/__init__.py +++ /dev/null @@ -1,560 +0,0 @@ -r"""JSON (JavaScript Object Notation) is a subset of -JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data -interchange format. - -:mod:`simplejson` exposes an API familiar to users of the standard library -:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained -version of the :mod:`json` library contained in Python 2.6, but maintains -compatibility with Python 2.4 and Python 2.5 and (currently) has -significant performance advantages, even without using the optional C -extension for speedups. - -Encoding basic Python object hierarchies:: - - >>> import simplejson as json - >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) - '["foo", {"bar": ["baz", null, 1.0, 2]}]' - >>> print(json.dumps("\"foo\bar")) - "\"foo\bar" - >>> print(json.dumps(u'\u1234')) - "\u1234" - >>> print(json.dumps('\\')) - "\\" - >>> print(json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True)) - {"a": 0, "b": 0, "c": 0} - >>> from simplejson.compat import StringIO - >>> io = StringIO() - >>> json.dump(['streaming API'], io) - >>> io.getvalue() - '["streaming API"]' - -Compact encoding:: - - >>> import simplejson as json - >>> obj = [1,2,3,{'4': 5, '6': 7}] - >>> json.dumps(obj, separators=(',',':'), sort_keys=True) - '[1,2,3,{"4":5,"6":7}]' - -Pretty printing:: - - >>> import simplejson as json - >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=' ')) - { - "4": 5, - "6": 7 - } - -Decoding JSON:: - - >>> import simplejson as json - >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] - >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj - True - >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' - True - >>> from simplejson.compat import StringIO - >>> io = StringIO('["streaming API"]') - >>> json.load(io)[0] == 'streaming API' - True - -Specializing JSON object decoding:: - - >>> import simplejson as json - >>> def as_complex(dct): - ... if '__complex__' in dct: - ... return complex(dct['real'], dct['imag']) - ... return dct - ... - >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', - ... object_hook=as_complex) - (1+2j) - >>> from decimal import Decimal - >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1') - True - -Specializing JSON object encoding:: - - >>> import simplejson as json - >>> def encode_complex(obj): - ... if isinstance(obj, complex): - ... return [obj.real, obj.imag] - ... raise TypeError(repr(o) + " is not JSON serializable") - ... - >>> json.dumps(2 + 1j, default=encode_complex) - '[2.0, 1.0]' - >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) - '[2.0, 1.0]' - >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) - '[2.0, 1.0]' - - -Using simplejson.tool from the shell to validate and pretty-print:: - - $ echo '{"json":"obj"}' | python -m simplejson.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m simplejson.tool - Expecting property name: line 1 column 3 (char 2) -""" -from __future__ import absolute_import -__version__ = '3.6.3' -__all__ = [ - 'dump', 'dumps', 'load', 'loads', - 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', - 'OrderedDict', 'simple_first', -] - -__author__ = 'Bob Ippolito ' - -from decimal import Decimal - -from .scanner import JSONDecodeError -from .decoder import JSONDecoder -from .encoder import JSONEncoder, JSONEncoderForHTML -def _import_OrderedDict(): - import collections - try: - return collections.OrderedDict - except AttributeError: - from . import ordered_dict - return ordered_dict.OrderedDict -OrderedDict = _import_OrderedDict() - -def _import_c_make_encoder(): - try: - from ._speedups import make_encoder - return make_encoder - except ImportError: - return None - -_default_encoder = JSONEncoder( - skipkeys=False, - ensure_ascii=True, - check_circular=True, - allow_nan=True, - indent=None, - separators=None, - encoding='utf-8', - default=None, - use_decimal=True, - namedtuple_as_object=True, - tuple_as_array=True, - bigint_as_string=False, - item_sort_key=None, - for_json=False, - ignore_nan=False, - int_as_string_bitcount=None, -) - -def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=True, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, use_decimal=True, - namedtuple_as_object=True, tuple_as_array=True, - bigint_as_string=False, sort_keys=False, item_sort_key=None, - for_json=False, ignore_nan=False, int_as_string_bitcount=None, **kw): - """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a - ``.write()``-supporting file-like object). - - If *skipkeys* is true then ``dict`` keys that are not basic types - (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If *ensure_ascii* is false, then the some chunks written to ``fp`` - may be ``unicode`` instances, subject to normal Python ``str`` to - ``unicode`` coercion rules. Unless ``fp.write()`` explicitly - understands ``unicode`` (as in ``codecs.getwriter()``) this is likely - to cause an error. - - If *check_circular* is false, then the circular reference check - for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). - - If *allow_nan* is false, then it will be a ``ValueError`` to - serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) - in strict compliance of the original JSON specification, instead of using - the JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). See - *ignore_nan* for ECMA-262 compliant behavior. - - If *indent* is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If specified, *separators* should be an - ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` - if *indent* is ``None`` and ``(',', ': ')`` otherwise. To get the most - compact JSON representation, you should specify ``(',', ':')`` to eliminate - whitespace. - - *encoding* is the character encoding for str instances, default is UTF-8. - - *default(obj)* is a function that should return a serializable version - of obj or raise ``TypeError``. The default simply raises ``TypeError``. - - If *use_decimal* is true (default: ``True``) then decimal.Decimal - will be natively serialized to JSON with full precision. - - If *namedtuple_as_object* is true (default: ``True``), - :class:`tuple` subclasses with ``_asdict()`` methods will be encoded - as JSON objects. - - If *tuple_as_array* is true (default: ``True``), - :class:`tuple` (and subclasses) will be encoded as JSON arrays. - - If *bigint_as_string* is true (default: ``False``), ints 2**53 and higher - or lower than -2**53 will be encoded as strings. This is to avoid the - rounding that happens in Javascript otherwise. Note that this is still a - lossy operation that will not round-trip correctly and should be used - sparingly. - - If *int_as_string_bitcount* is a positive number (n), then int of size - greater than or equal to 2**n or lower than or equal to -2**n will be - encoded as strings. - - If specified, *item_sort_key* is a callable used to sort the items in - each dictionary. This is useful if you want to sort items other than - in alphabetical order by key. This option takes precedence over - *sort_keys*. - - If *sort_keys* is true (default: ``False``), the output of dictionaries - will be sorted by item. - - If *for_json* is true (default: ``False``), objects with a ``for_json()`` - method will use the return value of that method for encoding as JSON - instead of the object. - - If *ignore_nan* is true (default: ``False``), then out of range - :class:`float` values (``nan``, ``inf``, ``-inf``) will be serialized as - ``null`` in compliance with the ECMA-262 specification. If true, this will - override *allow_nan*. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. NOTE: You should use *default* or *for_json* instead - of subclassing whenever possible. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and use_decimal - and namedtuple_as_object and tuple_as_array - and not bigint_as_string and int_as_string_bitcount is None - and not item_sort_key and not for_json and not ignore_nan and not kw): - iterable = _default_encoder.iterencode(obj) - else: - if cls is None: - cls = JSONEncoder - iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, - default=default, use_decimal=use_decimal, - namedtuple_as_object=namedtuple_as_object, - tuple_as_array=tuple_as_array, - bigint_as_string=bigint_as_string, - sort_keys=sort_keys, - item_sort_key=item_sort_key, - for_json=for_json, - ignore_nan=ignore_nan, - int_as_string_bitcount=int_as_string_bitcount, - **kw).iterencode(obj) - # could accelerate with writelines in some versions of Python, at - # a debuggability cost - for chunk in iterable: - fp.write(chunk) - - -def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=True, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, use_decimal=True, - namedtuple_as_object=True, tuple_as_array=True, - bigint_as_string=False, sort_keys=False, item_sort_key=None, - for_json=False, ignore_nan=False, int_as_string_bitcount=None, **kw): - """Serialize ``obj`` to a JSON formatted ``str``. - - If ``skipkeys`` is false then ``dict`` keys that are not basic types - (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If ``ensure_ascii`` is false, then the return value will be a - ``unicode`` instance subject to normal Python ``str`` to ``unicode`` - coercion rules instead of being escaped to an ASCII ``str``. - - If ``check_circular`` is false, then the circular reference check - for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). - - If ``allow_nan`` is false, then it will be a ``ValueError`` to - serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in - strict compliance of the JSON specification, instead of using the - JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). - - If ``indent`` is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If specified, ``separators`` should be an - ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` - if *indent* is ``None`` and ``(',', ': ')`` otherwise. To get the most - compact JSON representation, you should specify ``(',', ':')`` to eliminate - whitespace. - - ``encoding`` is the character encoding for str instances, default is UTF-8. - - ``default(obj)`` is a function that should return a serializable version - of obj or raise TypeError. The default simply raises TypeError. - - If *use_decimal* is true (default: ``True``) then decimal.Decimal - will be natively serialized to JSON with full precision. - - If *namedtuple_as_object* is true (default: ``True``), - :class:`tuple` subclasses with ``_asdict()`` methods will be encoded - as JSON objects. - - If *tuple_as_array* is true (default: ``True``), - :class:`tuple` (and subclasses) will be encoded as JSON arrays. - - If *bigint_as_string* is true (not the default), ints 2**53 and higher - or lower than -2**53 will be encoded as strings. This is to avoid the - rounding that happens in Javascript otherwise. - - If *int_as_string_bitcount* is a positive number (n), then int of size - greater than or equal to 2**n or lower than or equal to -2**n will be - encoded as strings. - - If specified, *item_sort_key* is a callable used to sort the items in - each dictionary. This is useful if you want to sort items other than - in alphabetical order by key. This option takes precendence over - *sort_keys*. - - If *sort_keys* is true (default: ``False``), the output of dictionaries - will be sorted by item. - - If *for_json* is true (default: ``False``), objects with a ``for_json()`` - method will use the return value of that method for encoding as JSON - instead of the object. - - If *ignore_nan* is true (default: ``False``), then out of range - :class:`float` values (``nan``, ``inf``, ``-inf``) will be serialized as - ``null`` in compliance with the ECMA-262 specification. If true, this will - override *allow_nan*. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. NOTE: You should use *default* instead of subclassing - whenever possible. - - """ - # cached encoder - if ( - not skipkeys and ensure_ascii and - check_circular and allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and use_decimal - and namedtuple_as_object and tuple_as_array - and not bigint_as_string and int_as_string_bitcount is None - and not sort_keys and not item_sort_key and not for_json - and not ignore_nan and not kw - ): - return _default_encoder.encode(obj) - if cls is None: - cls = JSONEncoder - return cls( - skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, default=default, - use_decimal=use_decimal, - namedtuple_as_object=namedtuple_as_object, - tuple_as_array=tuple_as_array, - bigint_as_string=bigint_as_string, - sort_keys=sort_keys, - item_sort_key=item_sort_key, - for_json=for_json, - ignore_nan=ignore_nan, - int_as_string_bitcount=int_as_string_bitcount, - **kw).encode(obj) - - -_default_decoder = JSONDecoder(encoding=None, object_hook=None, - object_pairs_hook=None) - - -def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, - use_decimal=False, namedtuple_as_object=True, tuple_as_array=True, - **kw): - """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing - a JSON document) to a Python object. - - *encoding* determines the encoding used to interpret any - :class:`str` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as :class:`unicode`. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *parse_constant*, if specified, will be called with one of the - following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This - can be used to raise an exception if invalid JSON numbers are - encountered. - - If *use_decimal* is true (default: ``False``) then it implies - parse_float=decimal.Decimal for parity with ``dump``. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. NOTE: You should use *object_hook* or *object_pairs_hook* instead - of subclassing whenever possible. - - """ - return loads(fp.read(), - encoding=encoding, cls=cls, object_hook=object_hook, - parse_float=parse_float, parse_int=parse_int, - parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, - use_decimal=use_decimal, **kw) - - -def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, - use_decimal=False, **kw): - """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON - document) to a Python object. - - *encoding* determines the encoding used to interpret any - :class:`str` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as :class:`unicode`. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *parse_constant*, if specified, will be called with one of the - following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This - can be used to raise an exception if invalid JSON numbers are - encountered. - - If *use_decimal* is true (default: ``False``) then it implies - parse_float=decimal.Decimal for parity with ``dump``. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. NOTE: You should use *object_hook* or *object_pairs_hook* instead - of subclassing whenever possible. - - """ - if (cls is None and encoding is None and object_hook is None and - parse_int is None and parse_float is None and - parse_constant is None and object_pairs_hook is None - and not use_decimal and not kw): - return _default_decoder.decode(s) - if cls is None: - cls = JSONDecoder - if object_hook is not None: - kw['object_hook'] = object_hook - if object_pairs_hook is not None: - kw['object_pairs_hook'] = object_pairs_hook - if parse_float is not None: - kw['parse_float'] = parse_float - if parse_int is not None: - kw['parse_int'] = parse_int - if parse_constant is not None: - kw['parse_constant'] = parse_constant - if use_decimal: - if parse_float is not None: - raise TypeError("use_decimal=True implies parse_float=Decimal") - kw['parse_float'] = Decimal - return cls(encoding=encoding, **kw).decode(s) - - -def _toggle_speedups(enabled): - from . import decoder as dec - from . import encoder as enc - from . import scanner as scan - c_make_encoder = _import_c_make_encoder() - if enabled: - dec.scanstring = dec.c_scanstring or dec.py_scanstring - enc.c_make_encoder = c_make_encoder - enc.encode_basestring_ascii = (enc.c_encode_basestring_ascii or - enc.py_encode_basestring_ascii) - scan.make_scanner = scan.c_make_scanner or scan.py_make_scanner - else: - dec.scanstring = dec.py_scanstring - enc.c_make_encoder = None - enc.encode_basestring_ascii = enc.py_encode_basestring_ascii - scan.make_scanner = scan.py_make_scanner - dec.make_scanner = scan.make_scanner - global _default_decoder - _default_decoder = JSONDecoder( - encoding=None, - object_hook=None, - object_pairs_hook=None, - ) - global _default_encoder - _default_encoder = JSONEncoder( - skipkeys=False, - ensure_ascii=True, - check_circular=True, - allow_nan=True, - indent=None, - separators=None, - encoding='utf-8', - default=None, - ) - -def simple_first(kv): - """Helper function to pass to item_sort_key to sort simple - elements to the top, then container elements. - """ - return (isinstance(kv[1], (list, dict, tuple)), kv[0]) diff --git a/pyload/lib/simplejson/_speedups.c b/pyload/lib/simplejson/_speedups.c deleted file mode 100644 index 01614c49a..000000000 --- a/pyload/lib/simplejson/_speedups.c +++ /dev/null @@ -1,3339 +0,0 @@ -/* -*- mode: C; c-file-style: "python"; c-basic-offset: 4 -*- */ -#include "Python.h" -#include "structmember.h" - -#if PY_MAJOR_VERSION >= 3 -#define PyInt_FromSsize_t PyLong_FromSsize_t -#define PyInt_AsSsize_t PyLong_AsSsize_t -#define PyString_Check PyBytes_Check -#define PyString_GET_SIZE PyBytes_GET_SIZE -#define PyString_AS_STRING PyBytes_AS_STRING -#define PyString_FromStringAndSize PyBytes_FromStringAndSize -#define PyInt_Check(obj) 0 -#define JSON_UNICHR Py_UCS4 -#define JSON_InternFromString PyUnicode_InternFromString -#define JSON_Intern_GET_SIZE PyUnicode_GET_SIZE -#define JSON_ASCII_Check PyUnicode_Check -#define JSON_ASCII_AS_STRING PyUnicode_AsUTF8 -#define PyInt_Type PyLong_Type -#define PyInt_FromString PyLong_FromString -#define PY2_UNUSED -#define PY3_UNUSED UNUSED -#define JSON_NewEmptyUnicode() PyUnicode_New(0, 127) -#else /* PY_MAJOR_VERSION >= 3 */ -#define PY2_UNUSED UNUSED -#define PY3_UNUSED -#define PyUnicode_READY(obj) 0 -#define PyUnicode_KIND(obj) (sizeof(Py_UNICODE)) -#define PyUnicode_DATA(obj) ((void *)(PyUnicode_AS_UNICODE(obj))) -#define PyUnicode_READ(kind, data, index) ((JSON_UNICHR)((const Py_UNICODE *)(data))[(index)]) -#define PyUnicode_GetLength PyUnicode_GET_SIZE -#define JSON_UNICHR Py_UNICODE -#define JSON_ASCII_Check PyString_Check -#define JSON_ASCII_AS_STRING PyString_AS_STRING -#define JSON_InternFromString PyString_InternFromString -#define JSON_Intern_GET_SIZE PyString_GET_SIZE -#define JSON_NewEmptyUnicode() PyUnicode_FromUnicode(NULL, 0) -#endif /* PY_MAJOR_VERSION < 3 */ - -#if PY_VERSION_HEX < 0x02070000 -#if !defined(PyOS_string_to_double) -#define PyOS_string_to_double json_PyOS_string_to_double -static double -json_PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception); -static double -json_PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception) -{ - double x; - assert(endptr == NULL); - assert(overflow_exception == NULL); - PyFPE_START_PROTECT("json_PyOS_string_to_double", return -1.0;) - x = PyOS_ascii_atof(s); - PyFPE_END_PROTECT(x) - return x; -} -#endif -#endif /* PY_VERSION_HEX < 0x02070000 */ - -#if PY_VERSION_HEX < 0x02060000 -#if !defined(Py_TYPE) -#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) -#endif -#if !defined(Py_SIZE) -#define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) -#endif -#if !defined(PyVarObject_HEAD_INIT) -#define PyVarObject_HEAD_INIT(type, size) PyObject_HEAD_INIT(type) size, -#endif -#endif /* PY_VERSION_HEX < 0x02060000 */ - -#if PY_VERSION_HEX < 0x02050000 -#if !defined(PY_SSIZE_T_MIN) -typedef int Py_ssize_t; -#define PY_SSIZE_T_MAX INT_MAX -#define PY_SSIZE_T_MIN INT_MIN -#define PyInt_FromSsize_t PyInt_FromLong -#define PyInt_AsSsize_t PyInt_AsLong -#endif -#if !defined(Py_IS_FINITE) -#define Py_IS_FINITE(X) (!Py_IS_INFINITY(X) && !Py_IS_NAN(X)) -#endif -#endif /* PY_VERSION_HEX < 0x02050000 */ - -#ifdef __GNUC__ -#define UNUSED __attribute__((__unused__)) -#else -#define UNUSED -#endif - -#define DEFAULT_ENCODING "utf-8" - -#define PyScanner_Check(op) PyObject_TypeCheck(op, &PyScannerType) -#define PyScanner_CheckExact(op) (Py_TYPE(op) == &PyScannerType) -#define PyEncoder_Check(op) PyObject_TypeCheck(op, &PyEncoderType) -#define PyEncoder_CheckExact(op) (Py_TYPE(op) == &PyEncoderType) - -#define JSON_ALLOW_NAN 1 -#define JSON_IGNORE_NAN 2 - -static PyTypeObject PyScannerType; -static PyTypeObject PyEncoderType; - -typedef struct { - PyObject *large_strings; /* A list of previously accumulated large strings */ - PyObject *small_strings; /* Pending small strings */ -} JSON_Accu; - -static int -JSON_Accu_Init(JSON_Accu *acc); -static int -JSON_Accu_Accumulate(JSON_Accu *acc, PyObject *unicode); -static PyObject * -JSON_Accu_FinishAsList(JSON_Accu *acc); -static void -JSON_Accu_Destroy(JSON_Accu *acc); - -#define ERR_EXPECTING_VALUE "Expecting value" -#define ERR_ARRAY_DELIMITER "Expecting ',' delimiter or ']'" -#define ERR_ARRAY_VALUE_FIRST "Expecting value or ']'" -#define ERR_OBJECT_DELIMITER "Expecting ',' delimiter or '}'" -#define ERR_OBJECT_PROPERTY "Expecting property name enclosed in double quotes" -#define ERR_OBJECT_PROPERTY_FIRST "Expecting property name enclosed in double quotes or '}'" -#define ERR_OBJECT_PROPERTY_DELIMITER "Expecting ':' delimiter" -#define ERR_STRING_UNTERMINATED "Unterminated string starting at" -#define ERR_STRING_CONTROL "Invalid control character %r at" -#define ERR_STRING_ESC1 "Invalid \\X escape sequence %r" -#define ERR_STRING_ESC4 "Invalid \\uXXXX escape sequence" - -typedef struct _PyScannerObject { - PyObject_HEAD - PyObject *encoding; - PyObject *strict; - PyObject *object_hook; - PyObject *pairs_hook; - PyObject *parse_float; - PyObject *parse_int; - PyObject *parse_constant; - PyObject *memo; -} PyScannerObject; - -static PyMemberDef scanner_members[] = { - {"encoding", T_OBJECT, offsetof(PyScannerObject, encoding), READONLY, "encoding"}, - {"strict", T_OBJECT, offsetof(PyScannerObject, strict), READONLY, "strict"}, - {"object_hook", T_OBJECT, offsetof(PyScannerObject, object_hook), READONLY, "object_hook"}, - {"object_pairs_hook", T_OBJECT, offsetof(PyScannerObject, pairs_hook), READONLY, "object_pairs_hook"}, - {"parse_float", T_OBJECT, offsetof(PyScannerObject, parse_float), READONLY, "parse_float"}, - {"parse_int", T_OBJECT, offsetof(PyScannerObject, parse_int), READONLY, "parse_int"}, - {"parse_constant", T_OBJECT, offsetof(PyScannerObject, parse_constant), READONLY, "parse_constant"}, - {NULL} -}; - -typedef struct _PyEncoderObject { - PyObject_HEAD - PyObject *markers; - PyObject *defaultfn; - PyObject *encoder; - PyObject *indent; - PyObject *key_separator; - PyObject *item_separator; - PyObject *sort_keys; - PyObject *key_memo; - PyObject *encoding; - PyObject *Decimal; - PyObject *skipkeys_bool; - int skipkeys; - int fast_encode; - /* 0, JSON_ALLOW_NAN, JSON_IGNORE_NAN */ - int allow_or_ignore_nan; - int use_decimal; - int namedtuple_as_object; - int tuple_as_array; - PyObject *max_long_size; - PyObject *min_long_size; - PyObject *item_sort_key; - PyObject *item_sort_kw; - int for_json; -} PyEncoderObject; - -static PyMemberDef encoder_members[] = { - {"markers", T_OBJECT, offsetof(PyEncoderObject, markers), READONLY, "markers"}, - {"default", T_OBJECT, offsetof(PyEncoderObject, defaultfn), READONLY, "default"}, - {"encoder", T_OBJECT, offsetof(PyEncoderObject, encoder), READONLY, "encoder"}, - {"encoding", T_OBJECT, offsetof(PyEncoderObject, encoder), READONLY, "encoding"}, - {"indent", T_OBJECT, offsetof(PyEncoderObject, indent), READONLY, "indent"}, - {"key_separator", T_OBJECT, offsetof(PyEncoderObject, key_separator), READONLY, "key_separator"}, - {"item_separator", T_OBJECT, offsetof(PyEncoderObject, item_separator), READONLY, "item_separator"}, - {"sort_keys", T_OBJECT, offsetof(PyEncoderObject, sort_keys), READONLY, "sort_keys"}, - /* Python 2.5 does not support T_BOOl */ - {"skipkeys", T_OBJECT, offsetof(PyEncoderObject, skipkeys_bool), READONLY, "skipkeys"}, - {"key_memo", T_OBJECT, offsetof(PyEncoderObject, key_memo), READONLY, "key_memo"}, - {"item_sort_key", T_OBJECT, offsetof(PyEncoderObject, item_sort_key), READONLY, "item_sort_key"}, - {"max_long_size", T_OBJECT, offsetof(PyEncoderObject, max_long_size), READONLY, "max_long_size"}, - {"min_long_size", T_OBJECT, offsetof(PyEncoderObject, min_long_size), READONLY, "min_long_size"}, - {NULL} -}; - -static PyObject * -join_list_unicode(PyObject *lst); -static PyObject * -JSON_ParseEncoding(PyObject *encoding); -static PyObject * -JSON_UnicodeFromChar(JSON_UNICHR c); -static PyObject * -maybe_quote_bigint(PyEncoderObject* s, PyObject *encoded, PyObject *obj); -static Py_ssize_t -ascii_char_size(JSON_UNICHR c); -static Py_ssize_t -ascii_escape_char(JSON_UNICHR c, char *output, Py_ssize_t chars); -static PyObject * -ascii_escape_unicode(PyObject *pystr); -static PyObject * -ascii_escape_str(PyObject *pystr); -static PyObject * -py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr); -#if PY_MAJOR_VERSION < 3 -static PyObject * -join_list_string(PyObject *lst); -static PyObject * -scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); -static PyObject * -scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_ssize_t *next_end_ptr); -static PyObject * -_parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); -#endif -static PyObject * -scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr); -static PyObject * -scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); -static PyObject * -_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx); -static PyObject * -scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static int -scanner_init(PyObject *self, PyObject *args, PyObject *kwds); -static void -scanner_dealloc(PyObject *self); -static int -scanner_clear(PyObject *self); -static PyObject * -encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static int -encoder_init(PyObject *self, PyObject *args, PyObject *kwds); -static void -encoder_dealloc(PyObject *self); -static int -encoder_clear(PyObject *self); -static PyObject * -encoder_stringify_key(PyEncoderObject *s, PyObject *key); -static int -encoder_listencode_list(PyEncoderObject *s, JSON_Accu *rval, PyObject *seq, Py_ssize_t indent_level); -static int -encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ssize_t indent_level); -static int -encoder_listencode_dict(PyEncoderObject *s, JSON_Accu *rval, PyObject *dct, Py_ssize_t indent_level); -static PyObject * -_encoded_const(PyObject *obj); -static void -raise_errmsg(char *msg, PyObject *s, Py_ssize_t end); -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj); -static int -_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr); -static PyObject * -_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr); -static PyObject * -encoder_encode_float(PyEncoderObject *s, PyObject *obj); -static int -_is_namedtuple(PyObject *obj); -static int -_has_for_json_hook(PyObject *obj); -static PyObject * -moduleinit(void); - -#define S_CHAR(c) (c >= ' ' && c <= '~' && c != '\\' && c != '"') -#define IS_WHITESPACE(c) (((c) == ' ') || ((c) == '\t') || ((c) == '\n') || ((c) == '\r')) - -#define MIN_EXPANSION 6 - -static int -JSON_Accu_Init(JSON_Accu *acc) -{ - /* Lazily allocated */ - acc->large_strings = NULL; - acc->small_strings = PyList_New(0); - if (acc->small_strings == NULL) - return -1; - return 0; -} - -static int -flush_accumulator(JSON_Accu *acc) -{ - Py_ssize_t nsmall = PyList_GET_SIZE(acc->small_strings); - if (nsmall) { - int ret; - PyObject *joined; - if (acc->large_strings == NULL) { - acc->large_strings = PyList_New(0); - if (acc->large_strings == NULL) - return -1; - } -#if PY_MAJOR_VERSION >= 3 - joined = join_list_unicode(acc->small_strings); -#else /* PY_MAJOR_VERSION >= 3 */ - joined = join_list_string(acc->small_strings); -#endif /* PY_MAJOR_VERSION < 3 */ - if (joined == NULL) - return -1; - if (PyList_SetSlice(acc->small_strings, 0, nsmall, NULL)) { - Py_DECREF(joined); - return -1; - } - ret = PyList_Append(acc->large_strings, joined); - Py_DECREF(joined); - return ret; - } - return 0; -} - -static int -JSON_Accu_Accumulate(JSON_Accu *acc, PyObject *unicode) -{ - Py_ssize_t nsmall; -#if PY_MAJOR_VERSION >= 3 - assert(PyUnicode_Check(unicode)); -#else /* PY_MAJOR_VERSION >= 3 */ - assert(JSON_ASCII_Check(unicode) || PyUnicode_Check(unicode)); -#endif /* PY_MAJOR_VERSION < 3 */ - - if (PyList_Append(acc->small_strings, unicode)) - return -1; - nsmall = PyList_GET_SIZE(acc->small_strings); - /* Each item in a list of unicode objects has an overhead (in 64-bit - * builds) of: - * - 8 bytes for the list slot - * - 56 bytes for the header of the unicode object - * that is, 64 bytes. 100000 such objects waste more than 6MB - * compared to a single concatenated string. - */ - if (nsmall < 100000) - return 0; - return flush_accumulator(acc); -} - -static PyObject * -JSON_Accu_FinishAsList(JSON_Accu *acc) -{ - int ret; - PyObject *res; - - ret = flush_accumulator(acc); - Py_CLEAR(acc->small_strings); - if (ret) { - Py_CLEAR(acc->large_strings); - return NULL; - } - res = acc->large_strings; - acc->large_strings = NULL; - if (res == NULL) - return PyList_New(0); - return res; -} - -static void -JSON_Accu_Destroy(JSON_Accu *acc) -{ - Py_CLEAR(acc->small_strings); - Py_CLEAR(acc->large_strings); -} - -static int -IS_DIGIT(JSON_UNICHR c) -{ - return c >= '0' && c <= '9'; -} - -static PyObject * -JSON_UnicodeFromChar(JSON_UNICHR c) -{ -#if PY_MAJOR_VERSION >= 3 - PyObject *rval = PyUnicode_New(1, c); - if (rval) - PyUnicode_WRITE(PyUnicode_KIND(rval), PyUnicode_DATA(rval), 0, c); - return rval; -#else /* PY_MAJOR_VERSION >= 3 */ - return PyUnicode_FromUnicode(&c, 1); -#endif /* PY_MAJOR_VERSION < 3 */ -} - -static PyObject * -maybe_quote_bigint(PyEncoderObject* s, PyObject *encoded, PyObject *obj) -{ - if (s->max_long_size != Py_None && s->min_long_size != Py_None) { - if (PyObject_RichCompareBool(obj, s->max_long_size, Py_GE) || - PyObject_RichCompareBool(obj, s->min_long_size, Py_LE)) { -#if PY_MAJOR_VERSION >= 3 - PyObject* quoted = PyUnicode_FromFormat("\"%U\"", encoded); -#else - PyObject* quoted = PyString_FromFormat("\"%s\"", - PyString_AsString(encoded)); -#endif - Py_DECREF(encoded); - encoded = quoted; - } - } - - return encoded; -} - -static int -_is_namedtuple(PyObject *obj) -{ - int rval = 0; - PyObject *_asdict = PyObject_GetAttrString(obj, "_asdict"); - if (_asdict == NULL) { - PyErr_Clear(); - return 0; - } - rval = PyCallable_Check(_asdict); - Py_DECREF(_asdict); - return rval; -} - -static int -_has_for_json_hook(PyObject *obj) -{ - int rval = 0; - PyObject *for_json = PyObject_GetAttrString(obj, "for_json"); - if (for_json == NULL) { - PyErr_Clear(); - return 0; - } - rval = PyCallable_Check(for_json); - Py_DECREF(for_json); - return rval; -} - -static int -_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr) -{ - /* PyObject to Py_ssize_t converter */ - *size_ptr = PyInt_AsSsize_t(o); - if (*size_ptr == -1 && PyErr_Occurred()) - return 0; - return 1; -} - -static PyObject * -_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr) -{ - /* Py_ssize_t to PyObject converter */ - return PyInt_FromSsize_t(*size_ptr); -} - -static Py_ssize_t -ascii_escape_char(JSON_UNICHR c, char *output, Py_ssize_t chars) -{ - /* Escape unicode code point c to ASCII escape sequences - in char *output. output must have at least 12 bytes unused to - accommodate an escaped surrogate pair "\uXXXX\uXXXX" */ - if (S_CHAR(c)) { - output[chars++] = (char)c; - } - else { - output[chars++] = '\\'; - switch (c) { - case '\\': output[chars++] = (char)c; break; - case '"': output[chars++] = (char)c; break; - case '\b': output[chars++] = 'b'; break; - case '\f': output[chars++] = 'f'; break; - case '\n': output[chars++] = 'n'; break; - case '\r': output[chars++] = 'r'; break; - case '\t': output[chars++] = 't'; break; - default: -#if defined(Py_UNICODE_WIDE) || PY_MAJOR_VERSION >= 3 - if (c >= 0x10000) { - /* UTF-16 surrogate pair */ - JSON_UNICHR v = c - 0x10000; - c = 0xd800 | ((v >> 10) & 0x3ff); - output[chars++] = 'u'; - output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; - output[chars++] = "0123456789abcdef"[(c ) & 0xf]; - c = 0xdc00 | (v & 0x3ff); - output[chars++] = '\\'; - } -#endif - output[chars++] = 'u'; - output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; - output[chars++] = "0123456789abcdef"[(c ) & 0xf]; - } - } - return chars; -} - -static Py_ssize_t -ascii_char_size(JSON_UNICHR c) -{ - if (S_CHAR(c)) { - return 1; - } - else if (c == '\\' || - c == '"' || - c == '\b' || - c == '\f' || - c == '\n' || - c == '\r' || - c == '\t') { - return 2; - } -#if defined(Py_UNICODE_WIDE) || PY_MAJOR_VERSION >= 3 - else if (c >= 0x10000U) { - return 2 * MIN_EXPANSION; - } -#endif - else { - return MIN_EXPANSION; - } -} - -static PyObject * -ascii_escape_unicode(PyObject *pystr) -{ - /* Take a PyUnicode pystr and return a new ASCII-only escaped PyString */ - Py_ssize_t i; - Py_ssize_t input_chars; - Py_ssize_t output_size; - Py_ssize_t chars; - PY2_UNUSED int kind; - void *data; - PyObject *rval; - char *output; - - if (PyUnicode_READY(pystr)) - return NULL; - - kind = PyUnicode_KIND(pystr); - data = PyUnicode_DATA(pystr); - input_chars = PyUnicode_GetLength(pystr); - output_size = 2; - for (i = 0; i < input_chars; i++) { - output_size += ascii_char_size(PyUnicode_READ(kind, data, i)); - } -#if PY_MAJOR_VERSION >= 3 - rval = PyUnicode_New(output_size, 127); - if (rval == NULL) { - return NULL; - } - assert(PyUnicode_KIND(rval) == PyUnicode_1BYTE_KIND); - output = (char *)PyUnicode_DATA(rval); -#else - rval = PyString_FromStringAndSize(NULL, output_size); - if (rval == NULL) { - return NULL; - } - output = PyString_AS_STRING(rval); -#endif - chars = 0; - output[chars++] = '"'; - for (i = 0; i < input_chars; i++) { - chars = ascii_escape_char(PyUnicode_READ(kind, data, i), output, chars); - } - output[chars++] = '"'; - assert(chars == output_size); - return rval; -} - -#if PY_MAJOR_VERSION >= 3 - -static PyObject * -ascii_escape_str(PyObject *pystr) -{ - PyObject *rval; - PyObject *input = PyUnicode_DecodeUTF8(PyString_AS_STRING(pystr), PyString_GET_SIZE(pystr), NULL); - if (input == NULL) - return NULL; - rval = ascii_escape_unicode(input); - Py_DECREF(input); - return rval; -} - -#else /* PY_MAJOR_VERSION >= 3 */ - -static PyObject * -ascii_escape_str(PyObject *pystr) -{ - /* Take a PyString pystr and return a new ASCII-only escaped PyString */ - Py_ssize_t i; - Py_ssize_t input_chars; - Py_ssize_t output_size; - Py_ssize_t chars; - PyObject *rval; - char *output; - char *input_str; - - input_chars = PyString_GET_SIZE(pystr); - input_str = PyString_AS_STRING(pystr); - output_size = 2; - - /* Fast path for a string that's already ASCII */ - for (i = 0; i < input_chars; i++) { - JSON_UNICHR c = (JSON_UNICHR)input_str[i]; - if (c > 0x7f) { - /* We hit a non-ASCII character, bail to unicode mode */ - PyObject *uni; - uni = PyUnicode_DecodeUTF8(input_str, input_chars, "strict"); - if (uni == NULL) { - return NULL; - } - rval = ascii_escape_unicode(uni); - Py_DECREF(uni); - return rval; - } - output_size += ascii_char_size(c); - } - - rval = PyString_FromStringAndSize(NULL, output_size); - if (rval == NULL) { - return NULL; - } - chars = 0; - output = PyString_AS_STRING(rval); - output[chars++] = '"'; - for (i = 0; i < input_chars; i++) { - chars = ascii_escape_char((JSON_UNICHR)input_str[i], output, chars); - } - output[chars++] = '"'; - assert(chars == output_size); - return rval; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -encoder_stringify_key(PyEncoderObject *s, PyObject *key) -{ - if (PyUnicode_Check(key)) { - Py_INCREF(key); - return key; - } - else if (PyString_Check(key)) { -#if PY_MAJOR_VERSION >= 3 - return PyUnicode_Decode( - PyString_AS_STRING(key), - PyString_GET_SIZE(key), - JSON_ASCII_AS_STRING(s->encoding), - NULL); -#else /* PY_MAJOR_VERSION >= 3 */ - Py_INCREF(key); - return key; -#endif /* PY_MAJOR_VERSION < 3 */ - } - else if (PyFloat_Check(key)) { - return encoder_encode_float(s, key); - } - else if (key == Py_True || key == Py_False || key == Py_None) { - /* This must come before the PyInt_Check because - True and False are also 1 and 0.*/ - return _encoded_const(key); - } - else if (PyInt_Check(key) || PyLong_Check(key)) { - return PyObject_Str(key); - } - else if (s->use_decimal && PyObject_TypeCheck(key, (PyTypeObject *)s->Decimal)) { - return PyObject_Str(key); - } - else if (s->skipkeys) { - Py_INCREF(Py_None); - return Py_None; - } - PyErr_SetString(PyExc_TypeError, "keys must be a string"); - return NULL; -} - -static PyObject * -encoder_dict_iteritems(PyEncoderObject *s, PyObject *dct) -{ - PyObject *items; - PyObject *iter = NULL; - PyObject *lst = NULL; - PyObject *item = NULL; - PyObject *kstr = NULL; - static PyObject *sortfun = NULL; - static PyObject *sortargs = NULL; - - if (sortargs == NULL) { - sortargs = PyTuple_New(0); - if (sortargs == NULL) - return NULL; - } - - if (PyDict_CheckExact(dct)) - items = PyDict_Items(dct); - else - items = PyMapping_Items(dct); - if (items == NULL) - return NULL; - iter = PyObject_GetIter(items); - Py_DECREF(items); - if (iter == NULL) - return NULL; - if (s->item_sort_kw == Py_None) - return iter; - lst = PyList_New(0); - if (lst == NULL) - goto bail; - while ((item = PyIter_Next(iter))) { - PyObject *key, *value; - if (!PyTuple_Check(item) || Py_SIZE(item) != 2) { - PyErr_SetString(PyExc_ValueError, "items must return 2-tuples"); - goto bail; - } - key = PyTuple_GET_ITEM(item, 0); - if (key == NULL) - goto bail; -#if PY_MAJOR_VERSION < 3 - else if (PyString_Check(key)) { - /* item can be added as-is */ - } -#endif /* PY_MAJOR_VERSION < 3 */ - else if (PyUnicode_Check(key)) { - /* item can be added as-is */ - } - else { - PyObject *tpl; - kstr = encoder_stringify_key(s, key); - if (kstr == NULL) - goto bail; - else if (kstr == Py_None) { - /* skipkeys */ - Py_DECREF(kstr); - continue; - } - value = PyTuple_GET_ITEM(item, 1); - if (value == NULL) - goto bail; - tpl = PyTuple_Pack(2, kstr, value); - if (tpl == NULL) - goto bail; - Py_CLEAR(kstr); - Py_DECREF(item); - item = tpl; - } - if (PyList_Append(lst, item)) - goto bail; - Py_DECREF(item); - } - Py_CLEAR(iter); - if (PyErr_Occurred()) - goto bail; - sortfun = PyObject_GetAttrString(lst, "sort"); - if (sortfun == NULL) - goto bail; - if (!PyObject_Call(sortfun, sortargs, s->item_sort_kw)) - goto bail; - Py_CLEAR(sortfun); - iter = PyObject_GetIter(lst); - Py_CLEAR(lst); - return iter; -bail: - Py_XDECREF(sortfun); - Py_XDECREF(kstr); - Py_XDECREF(item); - Py_XDECREF(lst); - Py_XDECREF(iter); - return NULL; -} - -static void -raise_errmsg(char *msg, PyObject *s, Py_ssize_t end) -{ - /* Use JSONDecodeError exception to raise a nice looking ValueError subclass */ - static PyObject *JSONDecodeError = NULL; - PyObject *exc; - if (JSONDecodeError == NULL) { - PyObject *scanner = PyImport_ImportModule("simplejson.scanner"); - if (scanner == NULL) - return; - JSONDecodeError = PyObject_GetAttrString(scanner, "JSONDecodeError"); - Py_DECREF(scanner); - if (JSONDecodeError == NULL) - return; - } - exc = PyObject_CallFunction(JSONDecodeError, "(zOO&)", msg, s, _convertPyInt_FromSsize_t, &end); - if (exc) { - PyErr_SetObject(JSONDecodeError, exc); - Py_DECREF(exc); - } -} - -static PyObject * -join_list_unicode(PyObject *lst) -{ - /* return u''.join(lst) */ - static PyObject *joinfn = NULL; - if (joinfn == NULL) { - PyObject *ustr = JSON_NewEmptyUnicode(); - if (ustr == NULL) - return NULL; - - joinfn = PyObject_GetAttrString(ustr, "join"); - Py_DECREF(ustr); - if (joinfn == NULL) - return NULL; - } - return PyObject_CallFunctionObjArgs(joinfn, lst, NULL); -} - -#if PY_MAJOR_VERSION >= 3 -#define join_list_string join_list_unicode -#else /* PY_MAJOR_VERSION >= 3 */ -static PyObject * -join_list_string(PyObject *lst) -{ - /* return ''.join(lst) */ - static PyObject *joinfn = NULL; - if (joinfn == NULL) { - PyObject *ustr = PyString_FromStringAndSize(NULL, 0); - if (ustr == NULL) - return NULL; - - joinfn = PyObject_GetAttrString(ustr, "join"); - Py_DECREF(ustr); - if (joinfn == NULL) - return NULL; - } - return PyObject_CallFunctionObjArgs(joinfn, lst, NULL); -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) -{ - /* return (rval, idx) tuple, stealing reference to rval */ - PyObject *tpl; - PyObject *pyidx; - /* - steal a reference to rval, returns (rval, idx) - */ - if (rval == NULL) { - assert(PyErr_Occurred()); - return NULL; - } - pyidx = PyInt_FromSsize_t(idx); - if (pyidx == NULL) { - Py_DECREF(rval); - return NULL; - } - tpl = PyTuple_New(2); - if (tpl == NULL) { - Py_DECREF(pyidx); - Py_DECREF(rval); - return NULL; - } - PyTuple_SET_ITEM(tpl, 0, rval); - PyTuple_SET_ITEM(tpl, 1, pyidx); - return tpl; -} - -#define APPEND_OLD_CHUNK \ - if (chunk != NULL) { \ - if (chunks == NULL) { \ - chunks = PyList_New(0); \ - if (chunks == NULL) { \ - goto bail; \ - } \ - } \ - if (PyList_Append(chunks, chunk)) { \ - goto bail; \ - } \ - Py_CLEAR(chunk); \ - } - -#if PY_MAJOR_VERSION < 3 -static PyObject * -scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_ssize_t *next_end_ptr) -{ - /* Read the JSON string from PyString pystr. - end is the index of the first character after the quote. - encoding is the encoding of pystr (must be an ASCII superset) - if strict is zero then literal control characters are allowed - *next_end_ptr is a return-by-reference index of the character - after the end quote - - Return value is a new PyString (if ASCII-only) or PyUnicode - */ - PyObject *rval; - Py_ssize_t len = PyString_GET_SIZE(pystr); - Py_ssize_t begin = end - 1; - Py_ssize_t next = begin; - int has_unicode = 0; - char *buf = PyString_AS_STRING(pystr); - PyObject *chunks = NULL; - PyObject *chunk = NULL; - PyObject *strchunk = NULL; - - if (len == end) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - else if (end < 0 || len < end) { - PyErr_SetString(PyExc_ValueError, "end is out of bounds"); - goto bail; - } - while (1) { - /* Find the end of the string or the next escape */ - Py_UNICODE c = 0; - for (next = end; next < len; next++) { - c = (unsigned char)buf[next]; - if (c == '"' || c == '\\') { - break; - } - else if (strict && c <= 0x1f) { - raise_errmsg(ERR_STRING_CONTROL, pystr, next); - goto bail; - } - else if (c > 0x7f) { - has_unicode = 1; - } - } - if (!(c == '"' || c == '\\')) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - /* Pick up this chunk if it's not zero length */ - if (next != end) { - APPEND_OLD_CHUNK -#if PY_MAJOR_VERSION >= 3 - if (!has_unicode) { - chunk = PyUnicode_DecodeASCII(&buf[end], next - end, NULL); - } - else { - chunk = PyUnicode_Decode(&buf[end], next - end, encoding, NULL); - } - if (chunk == NULL) { - goto bail; - } -#else /* PY_MAJOR_VERSION >= 3 */ - strchunk = PyString_FromStringAndSize(&buf[end], next - end); - if (strchunk == NULL) { - goto bail; - } - if (has_unicode) { - chunk = PyUnicode_FromEncodedObject(strchunk, encoding, NULL); - Py_DECREF(strchunk); - if (chunk == NULL) { - goto bail; - } - } - else { - chunk = strchunk; - } -#endif /* PY_MAJOR_VERSION < 3 */ - } - next++; - if (c == '"') { - end = next; - break; - } - if (next == len) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - c = buf[next]; - if (c != 'u') { - /* Non-unicode backslash escapes */ - end = next + 1; - switch (c) { - case '"': break; - case '\\': break; - case '/': break; - case 'b': c = '\b'; break; - case 'f': c = '\f'; break; - case 'n': c = '\n'; break; - case 'r': c = '\r'; break; - case 't': c = '\t'; break; - default: c = 0; - } - if (c == 0) { - raise_errmsg(ERR_STRING_ESC1, pystr, end - 2); - goto bail; - } - } - else { - c = 0; - next++; - end = next + 4; - if (end >= len) { - raise_errmsg(ERR_STRING_ESC4, pystr, next - 1); - goto bail; - } - /* Decode 4 hex digits */ - for (; next < end; next++) { - JSON_UNICHR digit = (JSON_UNICHR)buf[next]; - c <<= 4; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } -#if (PY_MAJOR_VERSION >= 3 || defined(Py_UNICODE_WIDE)) - /* Surrogate pair */ - if ((c & 0xfc00) == 0xd800) { - if (end + 6 < len && buf[next] == '\\' && buf[next+1] == 'u') { - JSON_UNICHR c2 = 0; - end += 6; - /* Decode 4 hex digits */ - for (next += 2; next < end; next++) { - c2 <<= 4; - JSON_UNICHR digit = buf[next]; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c2 |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c2 |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c2 |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } - if ((c2 & 0xfc00) != 0xdc00) { - /* not a low surrogate, rewind */ - end -= 6; - next = end; - } - else { - c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00)); - } - } - } -#endif /* PY_MAJOR_VERSION >= 3 || Py_UNICODE_WIDE */ - } - if (c > 0x7f) { - has_unicode = 1; - } - APPEND_OLD_CHUNK -#if PY_MAJOR_VERSION >= 3 - chunk = JSON_UnicodeFromChar(c); - if (chunk == NULL) { - goto bail; - } -#else /* PY_MAJOR_VERSION >= 3 */ - if (has_unicode) { - chunk = JSON_UnicodeFromChar(c); - if (chunk == NULL) { - goto bail; - } - } - else { - char c_char = Py_CHARMASK(c); - chunk = PyString_FromStringAndSize(&c_char, 1); - if (chunk == NULL) { - goto bail; - } - } -#endif - } - - if (chunks == NULL) { - if (chunk != NULL) - rval = chunk; - else - rval = JSON_NewEmptyUnicode(); - } - else { - APPEND_OLD_CHUNK - rval = join_list_string(chunks); - if (rval == NULL) { - goto bail; - } - Py_CLEAR(chunks); - } - - *next_end_ptr = end; - return rval; -bail: - *next_end_ptr = -1; - Py_XDECREF(chunk); - Py_XDECREF(chunks); - return NULL; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr) -{ - /* Read the JSON string from PyUnicode pystr. - end is the index of the first character after the quote. - if strict is zero then literal control characters are allowed - *next_end_ptr is a return-by-reference index of the character - after the end quote - - Return value is a new PyUnicode - */ - PyObject *rval; - Py_ssize_t begin = end - 1; - Py_ssize_t next = begin; - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - Py_ssize_t len = PyUnicode_GetLength(pystr); - void *buf = PyUnicode_DATA(pystr); - PyObject *chunks = NULL; - PyObject *chunk = NULL; - - if (len == end) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - else if (end < 0 || len < end) { - PyErr_SetString(PyExc_ValueError, "end is out of bounds"); - goto bail; - } - while (1) { - /* Find the end of the string or the next escape */ - JSON_UNICHR c = 0; - for (next = end; next < len; next++) { - c = PyUnicode_READ(kind, buf, next); - if (c == '"' || c == '\\') { - break; - } - else if (strict && c <= 0x1f) { - raise_errmsg(ERR_STRING_CONTROL, pystr, next); - goto bail; - } - } - if (!(c == '"' || c == '\\')) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - /* Pick up this chunk if it's not zero length */ - if (next != end) { - APPEND_OLD_CHUNK -#if PY_MAJOR_VERSION < 3 - chunk = PyUnicode_FromUnicode(&((const Py_UNICODE *)buf)[end], next - end); -#else - chunk = PyUnicode_Substring(pystr, end, next); -#endif - if (chunk == NULL) { - goto bail; - } - } - next++; - if (c == '"') { - end = next; - break; - } - if (next == len) { - raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin); - goto bail; - } - c = PyUnicode_READ(kind, buf, next); - if (c != 'u') { - /* Non-unicode backslash escapes */ - end = next + 1; - switch (c) { - case '"': break; - case '\\': break; - case '/': break; - case 'b': c = '\b'; break; - case 'f': c = '\f'; break; - case 'n': c = '\n'; break; - case 'r': c = '\r'; break; - case 't': c = '\t'; break; - default: c = 0; - } - if (c == 0) { - raise_errmsg(ERR_STRING_ESC1, pystr, end - 2); - goto bail; - } - } - else { - c = 0; - next++; - end = next + 4; - if (end >= len) { - raise_errmsg(ERR_STRING_ESC4, pystr, next - 1); - goto bail; - } - /* Decode 4 hex digits */ - for (; next < end; next++) { - JSON_UNICHR digit = PyUnicode_READ(kind, buf, next); - c <<= 4; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } -#if PY_MAJOR_VERSION >= 3 || defined(Py_UNICODE_WIDE) - /* Surrogate pair */ - if ((c & 0xfc00) == 0xd800) { - JSON_UNICHR c2 = 0; - if (end + 6 < len && - PyUnicode_READ(kind, buf, next) == '\\' && - PyUnicode_READ(kind, buf, next + 1) == 'u') { - end += 6; - /* Decode 4 hex digits */ - for (next += 2; next < end; next++) { - JSON_UNICHR digit = PyUnicode_READ(kind, buf, next); - c2 <<= 4; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c2 |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c2 |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c2 |= (digit - 'A' + 10); break; - default: - raise_errmsg(ERR_STRING_ESC4, pystr, end - 5); - goto bail; - } - } - if ((c2 & 0xfc00) != 0xdc00) { - /* not a low surrogate, rewind */ - end -= 6; - next = end; - } - else { - c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00)); - } - } - } -#endif - } - APPEND_OLD_CHUNK - chunk = JSON_UnicodeFromChar(c); - if (chunk == NULL) { - goto bail; - } - } - - if (chunks == NULL) { - if (chunk != NULL) - rval = chunk; - else - rval = JSON_NewEmptyUnicode(); - } - else { - APPEND_OLD_CHUNK - rval = join_list_unicode(chunks); - if (rval == NULL) { - goto bail; - } - Py_CLEAR(chunks); - } - *next_end_ptr = end; - return rval; -bail: - *next_end_ptr = -1; - Py_XDECREF(chunk); - Py_XDECREF(chunks); - return NULL; -} - -PyDoc_STRVAR(pydoc_scanstring, - "scanstring(basestring, end, encoding, strict=True) -> (str, end)\n" - "\n" - "Scan the string s for a JSON string. End is the index of the\n" - "character in s after the quote that started the JSON string.\n" - "Unescapes all valid JSON string escape sequences and raises ValueError\n" - "on attempt to decode an invalid string. If strict is False then literal\n" - "control characters are allowed in the string.\n" - "\n" - "Returns a tuple of the decoded string and the index of the character in s\n" - "after the end quote." -); - -static PyObject * -py_scanstring(PyObject* self UNUSED, PyObject *args) -{ - PyObject *pystr; - PyObject *rval; - Py_ssize_t end; - Py_ssize_t next_end = -1; - char *encoding = NULL; - int strict = 1; - if (!PyArg_ParseTuple(args, "OO&|zi:scanstring", &pystr, _convertPyInt_AsSsize_t, &end, &encoding, &strict)) { - return NULL; - } - if (encoding == NULL) { - encoding = DEFAULT_ENCODING; - } - if (PyUnicode_Check(pystr)) { - rval = scanstring_unicode(pystr, end, strict, &next_end); - } -#if PY_MAJOR_VERSION < 3 - /* Using a bytes input is unsupported for scanning in Python 3. - It is coerced to str in the decoder before it gets here. */ - else if (PyString_Check(pystr)) { - rval = scanstring_str(pystr, end, encoding, strict, &next_end); - } -#endif - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } - return _build_rval_index_tuple(rval, next_end); -} - -PyDoc_STRVAR(pydoc_encode_basestring_ascii, - "encode_basestring_ascii(basestring) -> str\n" - "\n" - "Return an ASCII-only JSON representation of a Python string" -); - -static PyObject * -py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr) -{ - /* Return an ASCII-only JSON representation of a Python string */ - /* METH_O */ - if (PyString_Check(pystr)) { - return ascii_escape_str(pystr); - } - else if (PyUnicode_Check(pystr)) { - return ascii_escape_unicode(pystr); - } - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } -} - -static void -scanner_dealloc(PyObject *self) -{ - /* Deallocate scanner object */ - scanner_clear(self); - Py_TYPE(self)->tp_free(self); -} - -static int -scanner_traverse(PyObject *self, visitproc visit, void *arg) -{ - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - Py_VISIT(s->encoding); - Py_VISIT(s->strict); - Py_VISIT(s->object_hook); - Py_VISIT(s->pairs_hook); - Py_VISIT(s->parse_float); - Py_VISIT(s->parse_int); - Py_VISIT(s->parse_constant); - Py_VISIT(s->memo); - return 0; -} - -static int -scanner_clear(PyObject *self) -{ - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - Py_CLEAR(s->encoding); - Py_CLEAR(s->strict); - Py_CLEAR(s->object_hook); - Py_CLEAR(s->pairs_hook); - Py_CLEAR(s->parse_float); - Py_CLEAR(s->parse_int); - Py_CLEAR(s->parse_constant); - Py_CLEAR(s->memo); - return 0; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -_parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON object from PyString pystr. - idx is the index of the first character after the opening curly brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing curly brace. - - Returns a new PyObject (usually a dict, but object_hook or - object_pairs_hook can change that) - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - PyObject *rval = NULL; - PyObject *pairs = NULL; - PyObject *item; - PyObject *key = NULL; - PyObject *val = NULL; - char *encoding = JSON_ASCII_AS_STRING(s->encoding); - int strict = PyObject_IsTrue(s->strict); - int has_pairs_hook = (s->pairs_hook != Py_None); - int did_parse = 0; - Py_ssize_t next_idx; - if (has_pairs_hook) { - pairs = PyList_New(0); - if (pairs == NULL) - return NULL; - } - else { - rval = PyDict_New(); - if (rval == NULL) - return NULL; - } - - /* skip whitespace after { */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* only loop if the object is non-empty */ - if (idx <= end_idx && str[idx] != '}') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - PyObject *memokey; - trailing_delimiter = 0; - - /* read key */ - if (str[idx] != '"') { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - key = scanstring_str(pystr, idx + 1, encoding, strict, &next_idx); - if (key == NULL) - goto bail; - memokey = PyDict_GetItem(s->memo, key); - if (memokey != NULL) { - Py_INCREF(memokey); - Py_DECREF(key); - key = memokey; - } - else { - if (PyDict_SetItem(s->memo, key, key) < 0) - goto bail; - } - idx = next_idx; - - /* skip whitespace between key and : delimiter, read :, skip whitespace */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - if (idx > end_idx || str[idx] != ':') { - raise_errmsg(ERR_OBJECT_PROPERTY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* read any JSON data type */ - val = scan_once_str(s, pystr, idx, &next_idx); - if (val == NULL) - goto bail; - - if (has_pairs_hook) { - item = PyTuple_Pack(2, key, val); - if (item == NULL) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - if (PyList_Append(pairs, item) == -1) { - Py_DECREF(item); - goto bail; - } - Py_DECREF(item); - } - else { - if (PyDict_SetItem(rval, key, val) < 0) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - } - idx = next_idx; - - /* skip whitespace before } or , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* bail if the object is closed or we didn't get the , delimiter */ - did_parse = 1; - if (idx > end_idx) break; - if (str[idx] == '}') { - break; - } - else if (str[idx] != ',') { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , delimiter */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - } - /* verify that idx < end_idx, str[idx] should be '}' */ - if (idx > end_idx || str[idx] != '}') { - if (did_parse) { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_OBJECT_PROPERTY_FIRST, pystr, idx); - } - goto bail; - } - - /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */ - if (s->pairs_hook != Py_None) { - val = PyObject_CallFunctionObjArgs(s->pairs_hook, pairs, NULL); - if (val == NULL) - goto bail; - Py_DECREF(pairs); - *next_idx_ptr = idx + 1; - return val; - } - - /* if object_hook is not None: rval = object_hook(rval) */ - if (s->object_hook != Py_None) { - val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL); - if (val == NULL) - goto bail; - Py_DECREF(rval); - rval = val; - val = NULL; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(rval); - Py_XDECREF(key); - Py_XDECREF(val); - Py_XDECREF(pairs); - return NULL; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON object from PyUnicode pystr. - idx is the index of the first character after the opening curly brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing curly brace. - - Returns a new PyObject (usually a dict, but object_hook can change that) - */ - void *str = PyUnicode_DATA(pystr); - Py_ssize_t end_idx = PyUnicode_GetLength(pystr) - 1; - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - PyObject *rval = NULL; - PyObject *pairs = NULL; - PyObject *item; - PyObject *key = NULL; - PyObject *val = NULL; - int strict = PyObject_IsTrue(s->strict); - int has_pairs_hook = (s->pairs_hook != Py_None); - int did_parse = 0; - Py_ssize_t next_idx; - - if (has_pairs_hook) { - pairs = PyList_New(0); - if (pairs == NULL) - return NULL; - } - else { - rval = PyDict_New(); - if (rval == NULL) - return NULL; - } - - /* skip whitespace after { */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* only loop if the object is non-empty */ - if (idx <= end_idx && PyUnicode_READ(kind, str, idx) != '}') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - PyObject *memokey; - trailing_delimiter = 0; - - /* read key */ - if (PyUnicode_READ(kind, str, idx) != '"') { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - key = scanstring_unicode(pystr, idx + 1, strict, &next_idx); - if (key == NULL) - goto bail; - memokey = PyDict_GetItem(s->memo, key); - if (memokey != NULL) { - Py_INCREF(memokey); - Py_DECREF(key); - key = memokey; - } - else { - if (PyDict_SetItem(s->memo, key, key) < 0) - goto bail; - } - idx = next_idx; - - /* skip whitespace between key and : delimiter, read :, skip - whitespace */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - if (idx > end_idx || PyUnicode_READ(kind, str, idx) != ':') { - raise_errmsg(ERR_OBJECT_PROPERTY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* read any JSON term */ - val = scan_once_unicode(s, pystr, idx, &next_idx); - if (val == NULL) - goto bail; - - if (has_pairs_hook) { - item = PyTuple_Pack(2, key, val); - if (item == NULL) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - if (PyList_Append(pairs, item) == -1) { - Py_DECREF(item); - goto bail; - } - Py_DECREF(item); - } - else { - if (PyDict_SetItem(rval, key, val) < 0) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - } - idx = next_idx; - - /* skip whitespace before } or , */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* bail if the object is closed or we didn't get the , - delimiter */ - did_parse = 1; - if (idx > end_idx) break; - if (PyUnicode_READ(kind, str, idx) == '}') { - break; - } - else if (PyUnicode_READ(kind, str, idx) != ',') { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , delimiter */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx); - goto bail; - } - } - - /* verify that idx < end_idx, str[idx] should be '}' */ - if (idx > end_idx || PyUnicode_READ(kind, str, idx) != '}') { - if (did_parse) { - raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_OBJECT_PROPERTY_FIRST, pystr, idx); - } - goto bail; - } - - /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */ - if (s->pairs_hook != Py_None) { - val = PyObject_CallFunctionObjArgs(s->pairs_hook, pairs, NULL); - if (val == NULL) - goto bail; - Py_DECREF(pairs); - *next_idx_ptr = idx + 1; - return val; - } - - /* if object_hook is not None: rval = object_hook(rval) */ - if (s->object_hook != Py_None) { - val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL); - if (val == NULL) - goto bail; - Py_DECREF(rval); - rval = val; - val = NULL; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(rval); - Py_XDECREF(key); - Py_XDECREF(val); - Py_XDECREF(pairs); - return NULL; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -_parse_array_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON array from PyString pystr. - idx is the index of the first character after the opening brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing brace. - - Returns a new PyList - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - PyObject *val = NULL; - PyObject *rval = PyList_New(0); - Py_ssize_t next_idx; - if (rval == NULL) - return NULL; - - /* skip whitespace after [ */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* only loop if the array is non-empty */ - if (idx <= end_idx && str[idx] != ']') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - trailing_delimiter = 0; - /* read any JSON term and de-tuplefy the (rval, idx) */ - val = scan_once_str(s, pystr, idx, &next_idx); - if (val == NULL) { - goto bail; - } - - if (PyList_Append(rval, val) == -1) - goto bail; - - Py_CLEAR(val); - idx = next_idx; - - /* skip whitespace between term and , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* bail if the array is closed or we didn't get the , delimiter */ - if (idx > end_idx) break; - if (str[idx] == ']') { - break; - } - else if (str[idx] != ',') { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - goto bail; - } - } - - /* verify that idx < end_idx, str[idx] should be ']' */ - if (idx > end_idx || str[idx] != ']') { - if (PyList_GET_SIZE(rval)) { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_ARRAY_VALUE_FIRST, pystr, idx); - } - goto bail; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(val); - Py_DECREF(rval); - return NULL; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_parse_array_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON array from PyString pystr. - idx is the index of the first character after the opening brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing brace. - - Returns a new PyList - */ - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - void *str = PyUnicode_DATA(pystr); - Py_ssize_t end_idx = PyUnicode_GetLength(pystr) - 1; - PyObject *val = NULL; - PyObject *rval = PyList_New(0); - Py_ssize_t next_idx; - if (rval == NULL) - return NULL; - - /* skip whitespace after [ */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* only loop if the array is non-empty */ - if (idx <= end_idx && PyUnicode_READ(kind, str, idx) != ']') { - int trailing_delimiter = 0; - while (idx <= end_idx) { - trailing_delimiter = 0; - /* read any JSON term */ - val = scan_once_unicode(s, pystr, idx, &next_idx); - if (val == NULL) { - goto bail; - } - - if (PyList_Append(rval, val) == -1) - goto bail; - - Py_CLEAR(val); - idx = next_idx; - - /* skip whitespace between term and , */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - - /* bail if the array is closed or we didn't get the , delimiter */ - if (idx > end_idx) break; - if (PyUnicode_READ(kind, str, idx) == ']') { - break; - } - else if (PyUnicode_READ(kind, str, idx) != ',') { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , */ - while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; - trailing_delimiter = 1; - } - if (trailing_delimiter) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - goto bail; - } - } - - /* verify that idx < end_idx, str[idx] should be ']' */ - if (idx > end_idx || PyUnicode_READ(kind, str, idx) != ']') { - if (PyList_GET_SIZE(rval)) { - raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx); - } else { - raise_errmsg(ERR_ARRAY_VALUE_FIRST, pystr, idx); - } - goto bail; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(val); - Py_DECREF(rval); - return NULL; -} - -static PyObject * -_parse_constant(PyScannerObject *s, char *constant, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON constant from PyString pystr. - constant is the constant string that was found - ("NaN", "Infinity", "-Infinity"). - idx is the index of the first character of the constant - *next_idx_ptr is a return-by-reference index to the first character after - the constant. - - Returns the result of parse_constant - */ - PyObject *cstr; - PyObject *rval; - /* constant is "NaN", "Infinity", or "-Infinity" */ - cstr = JSON_InternFromString(constant); - if (cstr == NULL) - return NULL; - - /* rval = parse_constant(constant) */ - rval = PyObject_CallFunctionObjArgs(s->parse_constant, cstr, NULL); - idx += JSON_Intern_GET_SIZE(cstr); - Py_DECREF(cstr); - *next_idx_ptr = idx; - return rval; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -_match_number_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON number from PyString pystr. - idx is the index of the first character of the number - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of that number: - PyInt, PyLong, or PyFloat. - May return other types if parse_int or parse_float are set - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - Py_ssize_t idx = start; - int is_float = 0; - PyObject *rval; - PyObject *numstr; - - /* read a sign if it's there, make sure it's not the end of the string */ - if (str[idx] == '-') { - if (idx >= end_idx) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - return NULL; - } - idx++; - } - - /* read as many integer digits as we find as long as it doesn't start with 0 */ - if (str[idx] >= '1' && str[idx] <= '9') { - idx++; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - } - /* if it starts with 0 we only expect one integer digit */ - else if (str[idx] == '0') { - idx++; - } - /* no integer digits, error */ - else { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - return NULL; - } - - /* if the next char is '.' followed by a digit then read all float digits */ - if (idx < end_idx && str[idx] == '.' && str[idx + 1] >= '0' && str[idx + 1] <= '9') { - is_float = 1; - idx += 2; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - } - - /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ - if (idx < end_idx && (str[idx] == 'e' || str[idx] == 'E')) { - - /* save the index of the 'e' or 'E' just in case we need to backtrack */ - Py_ssize_t e_start = idx; - idx++; - - /* read an exponent sign if present */ - if (idx < end_idx && (str[idx] == '-' || str[idx] == '+')) idx++; - - /* read all digits */ - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - - /* if we got a digit, then parse as float. if not, backtrack */ - if (str[idx - 1] >= '0' && str[idx - 1] <= '9') { - is_float = 1; - } - else { - idx = e_start; - } - } - - /* copy the section we determined to be a number */ - numstr = PyString_FromStringAndSize(&str[start], idx - start); - if (numstr == NULL) - return NULL; - if (is_float) { - /* parse as a float using a fast path if available, otherwise call user defined method */ - if (s->parse_float != (PyObject *)&PyFloat_Type) { - rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL); - } - else { - /* rval = PyFloat_FromDouble(PyOS_ascii_atof(PyString_AS_STRING(numstr))); */ - double d = PyOS_string_to_double(PyString_AS_STRING(numstr), - NULL, NULL); - if (d == -1.0 && PyErr_Occurred()) - return NULL; - rval = PyFloat_FromDouble(d); - } - } - else { - /* parse as an int using a fast path if available, otherwise call user defined method */ - if (s->parse_int != (PyObject *)&PyInt_Type) { - rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL); - } - else { - rval = PyInt_FromString(PyString_AS_STRING(numstr), NULL, 10); - } - } - Py_DECREF(numstr); - *next_idx_ptr = idx; - return rval; -} -#endif /* PY_MAJOR_VERSION < 3 */ - -static PyObject * -_match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) -{ - /* Read a JSON number from PyUnicode pystr. - idx is the index of the first character of the number - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of that number: - PyInt, PyLong, or PyFloat. - May return other types if parse_int or parse_float are set - */ - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - void *str = PyUnicode_DATA(pystr); - Py_ssize_t end_idx = PyUnicode_GetLength(pystr) - 1; - Py_ssize_t idx = start; - int is_float = 0; - JSON_UNICHR c; - PyObject *rval; - PyObject *numstr; - - /* read a sign if it's there, make sure it's not the end of the string */ - if (PyUnicode_READ(kind, str, idx) == '-') { - if (idx >= end_idx) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - return NULL; - } - idx++; - } - - /* read as many integer digits as we find as long as it doesn't start with 0 */ - c = PyUnicode_READ(kind, str, idx); - if (c == '0') { - /* if it starts with 0 we only expect one integer digit */ - idx++; - } - else if (IS_DIGIT(c)) { - idx++; - while (idx <= end_idx && IS_DIGIT(PyUnicode_READ(kind, str, idx))) { - idx++; - } - } - else { - /* no integer digits, error */ - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - return NULL; - } - - /* if the next char is '.' followed by a digit then read all float digits */ - if (idx < end_idx && - PyUnicode_READ(kind, str, idx) == '.' && - IS_DIGIT(PyUnicode_READ(kind, str, idx + 1))) { - is_float = 1; - idx += 2; - while (idx <= end_idx && IS_DIGIT(PyUnicode_READ(kind, str, idx))) idx++; - } - - /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ - if (idx < end_idx && - (PyUnicode_READ(kind, str, idx) == 'e' || - PyUnicode_READ(kind, str, idx) == 'E')) { - Py_ssize_t e_start = idx; - idx++; - - /* read an exponent sign if present */ - if (idx < end_idx && - (PyUnicode_READ(kind, str, idx) == '-' || - PyUnicode_READ(kind, str, idx) == '+')) idx++; - - /* read all digits */ - while (idx <= end_idx && IS_DIGIT(PyUnicode_READ(kind, str, idx))) idx++; - - /* if we got a digit, then parse as float. if not, backtrack */ - if (IS_DIGIT(PyUnicode_READ(kind, str, idx - 1))) { - is_float = 1; - } - else { - idx = e_start; - } - } - - /* copy the section we determined to be a number */ -#if PY_MAJOR_VERSION >= 3 - numstr = PyUnicode_Substring(pystr, start, idx); -#else - numstr = PyUnicode_FromUnicode(&((Py_UNICODE *)str)[start], idx - start); -#endif - if (numstr == NULL) - return NULL; - if (is_float) { - /* parse as a float using a fast path if available, otherwise call user defined method */ - if (s->parse_float != (PyObject *)&PyFloat_Type) { - rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL); - } - else { -#if PY_MAJOR_VERSION >= 3 - rval = PyFloat_FromString(numstr); -#else - rval = PyFloat_FromString(numstr, NULL); -#endif - } - } - else { - /* no fast path for unicode -> int, just call */ - rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL); - } - Py_DECREF(numstr); - *next_idx_ptr = idx; - return rval; -} - -#if PY_MAJOR_VERSION < 3 -static PyObject * -scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read one JSON term (of any kind) from PyString pystr. - idx is the index of the first character of the term - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of the term. - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t length = PyString_GET_SIZE(pystr); - PyObject *rval = NULL; - int fallthrough = 0; - if (idx < 0 || idx >= length) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - return NULL; - } - switch (str[idx]) { - case '"': - /* string */ - rval = scanstring_str(pystr, idx + 1, - JSON_ASCII_AS_STRING(s->encoding), - PyObject_IsTrue(s->strict), - next_idx_ptr); - break; - case '{': - /* object */ - if (Py_EnterRecursiveCall(" while decoding a JSON object " - "from a string")) - return NULL; - rval = _parse_object_str(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case '[': - /* array */ - if (Py_EnterRecursiveCall(" while decoding a JSON array " - "from a string")) - return NULL; - rval = _parse_array_str(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case 'n': - /* null */ - if ((idx + 3 < length) && str[idx + 1] == 'u' && str[idx + 2] == 'l' && str[idx + 3] == 'l') { - Py_INCREF(Py_None); - *next_idx_ptr = idx + 4; - rval = Py_None; - } - else - fallthrough = 1; - break; - case 't': - /* true */ - if ((idx + 3 < length) && str[idx + 1] == 'r' && str[idx + 2] == 'u' && str[idx + 3] == 'e') { - Py_INCREF(Py_True); - *next_idx_ptr = idx + 4; - rval = Py_True; - } - else - fallthrough = 1; - break; - case 'f': - /* false */ - if ((idx + 4 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'l' && str[idx + 3] == 's' && str[idx + 4] == 'e') { - Py_INCREF(Py_False); - *next_idx_ptr = idx + 5; - rval = Py_False; - } - else - fallthrough = 1; - break; - case 'N': - /* NaN */ - if ((idx + 2 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'N') { - rval = _parse_constant(s, "NaN", idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case 'I': - /* Infinity */ - if ((idx + 7 < length) && str[idx + 1] == 'n' && str[idx + 2] == 'f' && str[idx + 3] == 'i' && str[idx + 4] == 'n' && str[idx + 5] == 'i' && str[idx + 6] == 't' && str[idx + 7] == 'y') { - rval = _parse_constant(s, "Infinity", idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case '-': - /* -Infinity */ - if ((idx + 8 < length) && str[idx + 1] == 'I' && str[idx + 2] == 'n' && str[idx + 3] == 'f' && str[idx + 4] == 'i' && str[idx + 5] == 'n' && str[idx + 6] == 'i' && str[idx + 7] == 't' && str[idx + 8] == 'y') { - rval = _parse_constant(s, "-Infinity", idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - default: - fallthrough = 1; - } - /* Didn't find a string, object, array, or named constant. Look for a number. */ - if (fallthrough) - rval = _match_number_str(s, pystr, idx, next_idx_ptr); - return rval; -} -#endif /* PY_MAJOR_VERSION < 3 */ - - -static PyObject * -scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read one JSON term (of any kind) from PyUnicode pystr. - idx is the index of the first character of the term - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of the term. - */ - PY2_UNUSED int kind = PyUnicode_KIND(pystr); - void *str = PyUnicode_DATA(pystr); - Py_ssize_t length = PyUnicode_GetLength(pystr); - PyObject *rval = NULL; - int fallthrough = 0; - if (idx < 0 || idx >= length) { - raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx); - return NULL; - } - switch (PyUnicode_READ(kind, str, idx)) { - case '"': - /* string */ - rval = scanstring_unicode(pystr, idx + 1, - PyObject_IsTrue(s->strict), - next_idx_ptr); - break; - case '{': - /* object */ - if (Py_EnterRecursiveCall(" while decoding a JSON object " - "from a unicode string")) - return NULL; - rval = _parse_object_unicode(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case '[': - /* array */ - if (Py_EnterRecursiveCall(" while decoding a JSON array " - "from a unicode string")) - return NULL; - rval = _parse_array_unicode(s, pystr, idx + 1, next_idx_ptr); - Py_LeaveRecursiveCall(); - break; - case 'n': - /* null */ - if ((idx + 3 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'u' && - PyUnicode_READ(kind, str, idx + 2) == 'l' && - PyUnicode_READ(kind, str, idx + 3) == 'l') { - Py_INCREF(Py_None); - *next_idx_ptr = idx + 4; - rval = Py_None; - } - else - fallthrough = 1; - break; - case 't': - /* true */ - if ((idx + 3 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'r' && - PyUnicode_READ(kind, str, idx + 2) == 'u' && - PyUnicode_READ(kind, str, idx + 3) == 'e') { - Py_INCREF(Py_True); - *next_idx_ptr = idx + 4; - rval = Py_True; - } - else - fallthrough = 1; - break; - case 'f': - /* false */ - if ((idx + 4 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'a' && - PyUnicode_READ(kind, str, idx + 2) == 'l' && - PyUnicode_READ(kind, str, idx + 3) == 's' && - PyUnicode_READ(kind, str, idx + 4) == 'e') { - Py_INCREF(Py_False); - *next_idx_ptr = idx + 5; - rval = Py_False; - } - else - fallthrough = 1; - break; - case 'N': - /* NaN */ - if ((idx + 2 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'a' && - PyUnicode_READ(kind, str, idx + 2) == 'N') { - rval = _parse_constant(s, "NaN", idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case 'I': - /* Infinity */ - if ((idx + 7 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'n' && - PyUnicode_READ(kind, str, idx + 2) == 'f' && - PyUnicode_READ(kind, str, idx + 3) == 'i' && - PyUnicode_READ(kind, str, idx + 4) == 'n' && - PyUnicode_READ(kind, str, idx + 5) == 'i' && - PyUnicode_READ(kind, str, idx + 6) == 't' && - PyUnicode_READ(kind, str, idx + 7) == 'y') { - rval = _parse_constant(s, "Infinity", idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - case '-': - /* -Infinity */ - if ((idx + 8 < length) && - PyUnicode_READ(kind, str, idx + 1) == 'I' && - PyUnicode_READ(kind, str, idx + 2) == 'n' && - PyUnicode_READ(kind, str, idx + 3) == 'f' && - PyUnicode_READ(kind, str, idx + 4) == 'i' && - PyUnicode_READ(kind, str, idx + 5) == 'n' && - PyUnicode_READ(kind, str, idx + 6) == 'i' && - PyUnicode_READ(kind, str, idx + 7) == 't' && - PyUnicode_READ(kind, str, idx + 8) == 'y') { - rval = _parse_constant(s, "-Infinity", idx, next_idx_ptr); - } - else - fallthrough = 1; - break; - default: - fallthrough = 1; - } - /* Didn't find a string, object, array, or named constant. Look for a number. */ - if (fallthrough) - rval = _match_number_unicode(s, pystr, idx, next_idx_ptr); - return rval; -} - -static PyObject * -scanner_call(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Python callable interface to scan_once_{str,unicode} */ - PyObject *pystr; - PyObject *rval; - Py_ssize_t idx; - Py_ssize_t next_idx = -1; - static char *kwlist[] = {"string", "idx", NULL}; - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:scan_once", kwlist, &pystr, _convertPyInt_AsSsize_t, &idx)) - return NULL; - - if (PyUnicode_Check(pystr)) { - rval = scan_once_unicode(s, pystr, idx, &next_idx); - } -#if PY_MAJOR_VERSION < 3 - else if (PyString_Check(pystr)) { - rval = scan_once_str(s, pystr, idx, &next_idx); - } -#endif /* PY_MAJOR_VERSION < 3 */ - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } - PyDict_Clear(s->memo); - return _build_rval_index_tuple(rval, next_idx); -} - -static PyObject * -scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - PyScannerObject *s; - s = (PyScannerObject *)type->tp_alloc(type, 0); - if (s != NULL) { - s->encoding = NULL; - s->strict = NULL; - s->object_hook = NULL; - s->pairs_hook = NULL; - s->parse_float = NULL; - s->parse_int = NULL; - s->parse_constant = NULL; - } - return (PyObject *)s; -} - -static PyObject * -JSON_ParseEncoding(PyObject *encoding) -{ - if (encoding == NULL) - return NULL; - if (encoding == Py_None) - return JSON_InternFromString(DEFAULT_ENCODING); -#if PY_MAJOR_VERSION < 3 - if (PyUnicode_Check(encoding)) - return PyUnicode_AsEncodedString(encoding, NULL, NULL); -#endif - if (JSON_ASCII_Check(encoding)) { - Py_INCREF(encoding); - return encoding; - } - PyErr_SetString(PyExc_TypeError, "encoding must be a string"); - return NULL; -} - -static int -scanner_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Initialize Scanner object */ - PyObject *ctx; - static char *kwlist[] = {"context", NULL}; - PyScannerObject *s; - PyObject *encoding; - - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:make_scanner", kwlist, &ctx)) - return -1; - - if (s->memo == NULL) { - s->memo = PyDict_New(); - if (s->memo == NULL) - goto bail; - } - - /* JSON_ASCII_AS_STRING is used on encoding */ - encoding = PyObject_GetAttrString(ctx, "encoding"); - s->encoding = JSON_ParseEncoding(encoding); - Py_XDECREF(encoding); - if (s->encoding == NULL) - goto bail; - - /* All of these will fail "gracefully" so we don't need to verify them */ - s->strict = PyObject_GetAttrString(ctx, "strict"); - if (s->strict == NULL) - goto bail; - s->object_hook = PyObject_GetAttrString(ctx, "object_hook"); - if (s->object_hook == NULL) - goto bail; - s->pairs_hook = PyObject_GetAttrString(ctx, "object_pairs_hook"); - if (s->pairs_hook == NULL) - goto bail; - s->parse_float = PyObject_GetAttrString(ctx, "parse_float"); - if (s->parse_float == NULL) - goto bail; - s->parse_int = PyObject_GetAttrString(ctx, "parse_int"); - if (s->parse_int == NULL) - goto bail; - s->parse_constant = PyObject_GetAttrString(ctx, "parse_constant"); - if (s->parse_constant == NULL) - goto bail; - - return 0; - -bail: - Py_CLEAR(s->encoding); - Py_CLEAR(s->strict); - Py_CLEAR(s->object_hook); - Py_CLEAR(s->pairs_hook); - Py_CLEAR(s->parse_float); - Py_CLEAR(s->parse_int); - Py_CLEAR(s->parse_constant); - return -1; -} - -PyDoc_STRVAR(scanner_doc, "JSON scanner object"); - -static -PyTypeObject PyScannerType = { - PyVarObject_HEAD_INIT(NULL, 0) - "simplejson._speedups.Scanner", /* tp_name */ - sizeof(PyScannerObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - scanner_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - scanner_call, /* tp_call */ - 0, /* tp_str */ - 0,/* PyObject_GenericGetAttr, */ /* tp_getattro */ - 0,/* PyObject_GenericSetAttr, */ /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - scanner_doc, /* tp_doc */ - scanner_traverse, /* tp_traverse */ - scanner_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - scanner_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - scanner_init, /* tp_init */ - 0,/* PyType_GenericAlloc, */ /* tp_alloc */ - scanner_new, /* tp_new */ - 0,/* PyObject_GC_Del, */ /* tp_free */ -}; - -static PyObject * -encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - PyEncoderObject *s; - s = (PyEncoderObject *)type->tp_alloc(type, 0); - if (s != NULL) { - s->markers = NULL; - s->defaultfn = NULL; - s->encoder = NULL; - s->encoding = NULL; - s->indent = NULL; - s->key_separator = NULL; - s->item_separator = NULL; - s->key_memo = NULL; - s->sort_keys = NULL; - s->item_sort_key = NULL; - s->item_sort_kw = NULL; - s->Decimal = NULL; - s->max_long_size = NULL; - s->min_long_size = NULL; - } - return (PyObject *)s; -} - -static int -encoder_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* initialize Encoder object */ - static char *kwlist[] = { - "markers", - "default", - "encoder", - "indent", - "key_separator", - "item_separator", - "sort_keys", - "skipkeys", - "allow_nan", - "key_memo", - "use_decimal", - "namedtuple_as_object", - "tuple_as_array", - "int_as_string_bitcount", - "item_sort_key", - "encoding", - "for_json", - "ignore_nan", - "Decimal", - NULL}; - - PyEncoderObject *s; - PyObject *markers, *defaultfn, *encoder, *indent, *key_separator; - PyObject *item_separator, *sort_keys, *skipkeys, *allow_nan, *key_memo; - PyObject *use_decimal, *namedtuple_as_object, *tuple_as_array; - PyObject *int_as_string_bitcount, *item_sort_key, *encoding, *for_json; - PyObject *ignore_nan, *Decimal; - - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOOOOOOOOOOOOOOOO:make_encoder", kwlist, - &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator, - &sort_keys, &skipkeys, &allow_nan, &key_memo, &use_decimal, - &namedtuple_as_object, &tuple_as_array, - &int_as_string_bitcount, &item_sort_key, &encoding, &for_json, - &ignore_nan, &Decimal)) - return -1; - - Py_INCREF(markers); - s->markers = markers; - Py_INCREF(defaultfn); - s->defaultfn = defaultfn; - Py_INCREF(encoder); - s->encoder = encoder; - s->encoding = JSON_ParseEncoding(encoding); - if (s->encoding == NULL) - return -1; - Py_INCREF(indent); - s->indent = indent; - Py_INCREF(key_separator); - s->key_separator = key_separator; - Py_INCREF(item_separator); - s->item_separator = item_separator; - Py_INCREF(skipkeys); - s->skipkeys_bool = skipkeys; - s->skipkeys = PyObject_IsTrue(skipkeys); - Py_INCREF(key_memo); - s->key_memo = key_memo; - s->fast_encode = (PyCFunction_Check(s->encoder) && PyCFunction_GetFunction(s->encoder) == (PyCFunction)py_encode_basestring_ascii); - s->allow_or_ignore_nan = ( - (PyObject_IsTrue(ignore_nan) ? JSON_IGNORE_NAN : 0) | - (PyObject_IsTrue(allow_nan) ? JSON_ALLOW_NAN : 0)); - s->use_decimal = PyObject_IsTrue(use_decimal); - s->namedtuple_as_object = PyObject_IsTrue(namedtuple_as_object); - s->tuple_as_array = PyObject_IsTrue(tuple_as_array); - if (PyInt_Check(int_as_string_bitcount) || PyLong_Check(int_as_string_bitcount)) { - static const unsigned int long_long_bitsize = SIZEOF_LONG_LONG * 8; - int int_as_string_bitcount_val = PyLong_AsLong(int_as_string_bitcount); - if (int_as_string_bitcount_val > 0 && int_as_string_bitcount_val < long_long_bitsize) { - s->max_long_size = PyLong_FromUnsignedLongLong(1ULL << int_as_string_bitcount_val); - s->min_long_size = PyLong_FromLongLong(-1LL << int_as_string_bitcount_val); - if (s->min_long_size == NULL || s->max_long_size == NULL) { - return -1; - } - } - else { - PyErr_Format(PyExc_TypeError, - "int_as_string_bitcount (%d) must be greater than 0 and less than the number of bits of a `long long` type (%u bits)", - int_as_string_bitcount_val, long_long_bitsize); - return -1; - } - } - else if (int_as_string_bitcount == Py_None) { - Py_INCREF(Py_None); - s->max_long_size = Py_None; - Py_INCREF(Py_None); - s->min_long_size = Py_None; - } - else { - PyErr_SetString(PyExc_TypeError, "int_as_string_bitcount must be None or an integer"); - return -1; - } - if (item_sort_key != Py_None) { - if (!PyCallable_Check(item_sort_key)) { - PyErr_SetString(PyExc_TypeError, "item_sort_key must be None or callable"); - return -1; - } - } - else if (PyObject_IsTrue(sort_keys)) { - static PyObject *itemgetter0 = NULL; - if (!itemgetter0) { - PyObject *operator = PyImport_ImportModule("operator"); - if (!operator) - return -1; - itemgetter0 = PyObject_CallMethod(operator, "itemgetter", "i", 0); - Py_DECREF(operator); - } - item_sort_key = itemgetter0; - if (!item_sort_key) - return -1; - } - if (item_sort_key == Py_None) { - Py_INCREF(Py_None); - s->item_sort_kw = Py_None; - } - else { - s->item_sort_kw = PyDict_New(); - if (s->item_sort_kw == NULL) - return -1; - if (PyDict_SetItemString(s->item_sort_kw, "key", item_sort_key)) - return -1; - } - Py_INCREF(sort_keys); - s->sort_keys = sort_keys; - Py_INCREF(item_sort_key); - s->item_sort_key = item_sort_key; - Py_INCREF(Decimal); - s->Decimal = Decimal; - s->for_json = PyObject_IsTrue(for_json); - - return 0; -} - -static PyObject * -encoder_call(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Python callable interface to encode_listencode_obj */ - static char *kwlist[] = {"obj", "_current_indent_level", NULL}; - PyObject *obj; - Py_ssize_t indent_level; - PyEncoderObject *s; - JSON_Accu rval; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:_iterencode", kwlist, - &obj, _convertPyInt_AsSsize_t, &indent_level)) - return NULL; - if (JSON_Accu_Init(&rval)) - return NULL; - if (encoder_listencode_obj(s, &rval, obj, indent_level)) { - JSON_Accu_Destroy(&rval); - return NULL; - } - return JSON_Accu_FinishAsList(&rval); -} - -static PyObject * -_encoded_const(PyObject *obj) -{ - /* Return the JSON string representation of None, True, False */ - if (obj == Py_None) { - static PyObject *s_null = NULL; - if (s_null == NULL) { - s_null = JSON_InternFromString("null"); - } - Py_INCREF(s_null); - return s_null; - } - else if (obj == Py_True) { - static PyObject *s_true = NULL; - if (s_true == NULL) { - s_true = JSON_InternFromString("true"); - } - Py_INCREF(s_true); - return s_true; - } - else if (obj == Py_False) { - static PyObject *s_false = NULL; - if (s_false == NULL) { - s_false = JSON_InternFromString("false"); - } - Py_INCREF(s_false); - return s_false; - } - else { - PyErr_SetString(PyExc_ValueError, "not a const"); - return NULL; - } -} - -static PyObject * -encoder_encode_float(PyEncoderObject *s, PyObject *obj) -{ - /* Return the JSON representation of a PyFloat */ - double i = PyFloat_AS_DOUBLE(obj); - if (!Py_IS_FINITE(i)) { - if (!s->allow_or_ignore_nan) { - PyErr_SetString(PyExc_ValueError, "Out of range float values are not JSON compliant"); - return NULL; - } - if (s->allow_or_ignore_nan & JSON_IGNORE_NAN) { - return _encoded_const(Py_None); - } - /* JSON_ALLOW_NAN is set */ - else if (i > 0) { - static PyObject *sInfinity = NULL; - if (sInfinity == NULL) - sInfinity = JSON_InternFromString("Infinity"); - if (sInfinity) - Py_INCREF(sInfinity); - return sInfinity; - } - else if (i < 0) { - static PyObject *sNegInfinity = NULL; - if (sNegInfinity == NULL) - sNegInfinity = JSON_InternFromString("-Infinity"); - if (sNegInfinity) - Py_INCREF(sNegInfinity); - return sNegInfinity; - } - else { - static PyObject *sNaN = NULL; - if (sNaN == NULL) - sNaN = JSON_InternFromString("NaN"); - if (sNaN) - Py_INCREF(sNaN); - return sNaN; - } - } - /* Use a better float format here? */ - return PyObject_Repr(obj); -} - -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj) -{ - /* Return the JSON representation of a string */ - if (s->fast_encode) - return py_encode_basestring_ascii(NULL, obj); - else - return PyObject_CallFunctionObjArgs(s->encoder, obj, NULL); -} - -static int -_steal_accumulate(JSON_Accu *accu, PyObject *stolen) -{ - /* Append stolen and then decrement its reference count */ - int rval = JSON_Accu_Accumulate(accu, stolen); - Py_DECREF(stolen); - return rval; -} - -static int -encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ssize_t indent_level) -{ - /* Encode Python object obj to a JSON term, rval is a PyList */ - int rv = -1; - do { - if (obj == Py_None || obj == Py_True || obj == Py_False) { - PyObject *cstr = _encoded_const(obj); - if (cstr != NULL) - rv = _steal_accumulate(rval, cstr); - } - else if (PyString_Check(obj) || PyUnicode_Check(obj)) - { - PyObject *encoded = encoder_encode_string(s, obj); - if (encoded != NULL) - rv = _steal_accumulate(rval, encoded); - } - else if (PyInt_Check(obj) || PyLong_Check(obj)) { - PyObject *encoded = PyObject_Str(obj); - if (encoded != NULL) { - encoded = maybe_quote_bigint(s, encoded, obj); - if (encoded == NULL) - break; - rv = _steal_accumulate(rval, encoded); - } - } - else if (PyFloat_Check(obj)) { - PyObject *encoded = encoder_encode_float(s, obj); - if (encoded != NULL) - rv = _steal_accumulate(rval, encoded); - } - else if (s->for_json && _has_for_json_hook(obj)) { - PyObject *newobj; - if (Py_EnterRecursiveCall(" while encoding a JSON object")) - return rv; - newobj = PyObject_CallMethod(obj, "for_json", NULL); - if (newobj != NULL) { - rv = encoder_listencode_obj(s, rval, newobj, indent_level); - Py_DECREF(newobj); - } - Py_LeaveRecursiveCall(); - } - else if (s->namedtuple_as_object && _is_namedtuple(obj)) { - PyObject *newobj; - if (Py_EnterRecursiveCall(" while encoding a JSON object")) - return rv; - newobj = PyObject_CallMethod(obj, "_asdict", NULL); - if (newobj != NULL) { - rv = encoder_listencode_dict(s, rval, newobj, indent_level); - Py_DECREF(newobj); - } - Py_LeaveRecursiveCall(); - } - else if (PyList_Check(obj) || (s->tuple_as_array && PyTuple_Check(obj))) { - if (Py_EnterRecursiveCall(" while encoding a JSON object")) - return rv; - rv = encoder_listencode_list(s, rval, obj, indent_level); - Py_LeaveRecursiveCall(); - } - else if (PyDict_Check(obj)) { - if (Py_EnterRecursiveCall(" while encoding a JSON object")) - return rv; - rv = encoder_listencode_dict(s, rval, obj, indent_level); - Py_LeaveRecursiveCall(); - } - else if (s->use_decimal && PyObject_TypeCheck(obj, (PyTypeObject *)s->Decimal)) { - PyObject *encoded = PyObject_Str(obj); - if (encoded != NULL) - rv = _steal_accumulate(rval, encoded); - } - else { - PyObject *ident = NULL; - PyObject *newobj; - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(obj); - if (ident == NULL) - break; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - Py_DECREF(ident); - break; - } - if (PyDict_SetItem(s->markers, ident, obj)) { - Py_DECREF(ident); - break; - } - } - if (Py_EnterRecursiveCall(" while encoding a JSON object")) - return rv; - newobj = PyObject_CallFunctionObjArgs(s->defaultfn, obj, NULL); - if (newobj == NULL) { - Py_XDECREF(ident); - Py_LeaveRecursiveCall(); - break; - } - rv = encoder_listencode_obj(s, rval, newobj, indent_level); - Py_LeaveRecursiveCall(); - Py_DECREF(newobj); - if (rv) { - Py_XDECREF(ident); - rv = -1; - } - else if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) { - Py_XDECREF(ident); - rv = -1; - } - Py_XDECREF(ident); - } - } - } while (0); - return rv; -} - -static int -encoder_listencode_dict(PyEncoderObject *s, JSON_Accu *rval, PyObject *dct, Py_ssize_t indent_level) -{ - /* Encode Python dict dct a JSON term */ - static PyObject *open_dict = NULL; - static PyObject *close_dict = NULL; - static PyObject *empty_dict = NULL; - PyObject *kstr = NULL; - PyObject *ident = NULL; - PyObject *iter = NULL; - PyObject *item = NULL; - PyObject *items = NULL; - PyObject *encoded = NULL; - Py_ssize_t idx; - - if (open_dict == NULL || close_dict == NULL || empty_dict == NULL) { - open_dict = JSON_InternFromString("{"); - close_dict = JSON_InternFromString("}"); - empty_dict = JSON_InternFromString("{}"); - if (open_dict == NULL || close_dict == NULL || empty_dict == NULL) - return -1; - } - if (PyDict_Size(dct) == 0) - return JSON_Accu_Accumulate(rval, empty_dict); - - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(dct); - if (ident == NULL) - goto bail; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - goto bail; - } - if (PyDict_SetItem(s->markers, ident, dct)) { - goto bail; - } - } - - if (JSON_Accu_Accumulate(rval, open_dict)) - goto bail; - - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - */ - } - - iter = encoder_dict_iteritems(s, dct); - if (iter == NULL) - goto bail; - - idx = 0; - while ((item = PyIter_Next(iter))) { - PyObject *encoded, *key, *value; - if (!PyTuple_Check(item) || Py_SIZE(item) != 2) { - PyErr_SetString(PyExc_ValueError, "items must return 2-tuples"); - goto bail; - } - key = PyTuple_GET_ITEM(item, 0); - if (key == NULL) - goto bail; - value = PyTuple_GET_ITEM(item, 1); - if (value == NULL) - goto bail; - - encoded = PyDict_GetItem(s->key_memo, key); - if (encoded != NULL) { - Py_INCREF(encoded); - } else { - kstr = encoder_stringify_key(s, key); - if (kstr == NULL) - goto bail; - else if (kstr == Py_None) { - /* skipkeys */ - Py_DECREF(item); - Py_DECREF(kstr); - continue; - } - } - if (idx) { - if (JSON_Accu_Accumulate(rval, s->item_separator)) - goto bail; - } - if (encoded == NULL) { - encoded = encoder_encode_string(s, kstr); - Py_CLEAR(kstr); - if (encoded == NULL) - goto bail; - if (PyDict_SetItem(s->key_memo, key, encoded)) - goto bail; - } - if (JSON_Accu_Accumulate(rval, encoded)) { - goto bail; - } - Py_CLEAR(encoded); - if (JSON_Accu_Accumulate(rval, s->key_separator)) - goto bail; - if (encoder_listencode_obj(s, rval, value, indent_level)) - goto bail; - Py_CLEAR(item); - idx += 1; - } - Py_CLEAR(iter); - if (PyErr_Occurred()) - goto bail; - if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) - goto bail; - Py_CLEAR(ident); - } - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level -= 1; - /* - yield '\n' + (_indent * _current_indent_level) - */ - } - if (JSON_Accu_Accumulate(rval, close_dict)) - goto bail; - return 0; - -bail: - Py_XDECREF(encoded); - Py_XDECREF(items); - Py_XDECREF(iter); - Py_XDECREF(kstr); - Py_XDECREF(ident); - return -1; -} - - -static int -encoder_listencode_list(PyEncoderObject *s, JSON_Accu *rval, PyObject *seq, Py_ssize_t indent_level) -{ - /* Encode Python list seq to a JSON term */ - static PyObject *open_array = NULL; - static PyObject *close_array = NULL; - static PyObject *empty_array = NULL; - PyObject *ident = NULL; - PyObject *iter = NULL; - PyObject *obj = NULL; - int is_true; - int i = 0; - - if (open_array == NULL || close_array == NULL || empty_array == NULL) { - open_array = JSON_InternFromString("["); - close_array = JSON_InternFromString("]"); - empty_array = JSON_InternFromString("[]"); - if (open_array == NULL || close_array == NULL || empty_array == NULL) - return -1; - } - ident = NULL; - is_true = PyObject_IsTrue(seq); - if (is_true == -1) - return -1; - else if (is_true == 0) - return JSON_Accu_Accumulate(rval, empty_array); - - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(seq); - if (ident == NULL) - goto bail; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - goto bail; - } - if (PyDict_SetItem(s->markers, ident, seq)) { - goto bail; - } - } - - iter = PyObject_GetIter(seq); - if (iter == NULL) - goto bail; - - if (JSON_Accu_Accumulate(rval, open_array)) - goto bail; - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - */ - } - while ((obj = PyIter_Next(iter))) { - if (i) { - if (JSON_Accu_Accumulate(rval, s->item_separator)) - goto bail; - } - if (encoder_listencode_obj(s, rval, obj, indent_level)) - goto bail; - i++; - Py_CLEAR(obj); - } - Py_CLEAR(iter); - if (PyErr_Occurred()) - goto bail; - if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) - goto bail; - Py_CLEAR(ident); - } - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level -= 1; - /* - yield '\n' + (_indent * _current_indent_level) - */ - } - if (JSON_Accu_Accumulate(rval, close_array)) - goto bail; - return 0; - -bail: - Py_XDECREF(obj); - Py_XDECREF(iter); - Py_XDECREF(ident); - return -1; -} - -static void -encoder_dealloc(PyObject *self) -{ - /* Deallocate Encoder */ - encoder_clear(self); - Py_TYPE(self)->tp_free(self); -} - -static int -encoder_traverse(PyObject *self, visitproc visit, void *arg) -{ - PyEncoderObject *s; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - Py_VISIT(s->markers); - Py_VISIT(s->defaultfn); - Py_VISIT(s->encoder); - Py_VISIT(s->encoding); - Py_VISIT(s->indent); - Py_VISIT(s->key_separator); - Py_VISIT(s->item_separator); - Py_VISIT(s->key_memo); - Py_VISIT(s->sort_keys); - Py_VISIT(s->item_sort_kw); - Py_VISIT(s->item_sort_key); - Py_VISIT(s->max_long_size); - Py_VISIT(s->min_long_size); - Py_VISIT(s->Decimal); - return 0; -} - -static int -encoder_clear(PyObject *self) -{ - /* Deallocate Encoder */ - PyEncoderObject *s; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - Py_CLEAR(s->markers); - Py_CLEAR(s->defaultfn); - Py_CLEAR(s->encoder); - Py_CLEAR(s->encoding); - Py_CLEAR(s->indent); - Py_CLEAR(s->key_separator); - Py_CLEAR(s->item_separator); - Py_CLEAR(s->key_memo); - Py_CLEAR(s->skipkeys_bool); - Py_CLEAR(s->sort_keys); - Py_CLEAR(s->item_sort_kw); - Py_CLEAR(s->item_sort_key); - Py_CLEAR(s->max_long_size); - Py_CLEAR(s->min_long_size); - Py_CLEAR(s->Decimal); - return 0; -} - -PyDoc_STRVAR(encoder_doc, "_iterencode(obj, _current_indent_level) -> iterable"); - -static -PyTypeObject PyEncoderType = { - PyVarObject_HEAD_INIT(NULL, 0) - "simplejson._speedups.Encoder", /* tp_name */ - sizeof(PyEncoderObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - encoder_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - encoder_call, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - encoder_doc, /* tp_doc */ - encoder_traverse, /* tp_traverse */ - encoder_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - encoder_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - encoder_init, /* tp_init */ - 0, /* tp_alloc */ - encoder_new, /* tp_new */ - 0, /* tp_free */ -}; - -static PyMethodDef speedups_methods[] = { - {"encode_basestring_ascii", - (PyCFunction)py_encode_basestring_ascii, - METH_O, - pydoc_encode_basestring_ascii}, - {"scanstring", - (PyCFunction)py_scanstring, - METH_VARARGS, - pydoc_scanstring}, - {NULL, NULL, 0, NULL} -}; - -PyDoc_STRVAR(module_doc, -"simplejson speedups\n"); - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "_speedups", /* m_name */ - module_doc, /* m_doc */ - -1, /* m_size */ - speedups_methods, /* m_methods */ - NULL, /* m_reload */ - NULL, /* m_traverse */ - NULL, /* m_clear*/ - NULL, /* m_free */ -}; -#endif - -static PyObject * -moduleinit(void) -{ - PyObject *m; - PyScannerType.tp_new = PyType_GenericNew; - if (PyType_Ready(&PyScannerType) < 0) - return NULL; - PyEncoderType.tp_new = PyType_GenericNew; - if (PyType_Ready(&PyEncoderType) < 0) - return NULL; - -#if PY_MAJOR_VERSION >= 3 - m = PyModule_Create(&moduledef); -#else - m = Py_InitModule3("_speedups", speedups_methods, module_doc); -#endif - Py_INCREF((PyObject*)&PyScannerType); - PyModule_AddObject(m, "make_scanner", (PyObject*)&PyScannerType); - Py_INCREF((PyObject*)&PyEncoderType); - PyModule_AddObject(m, "make_encoder", (PyObject*)&PyEncoderType); - return m; -} - -#if PY_MAJOR_VERSION >= 3 -PyMODINIT_FUNC -PyInit__speedups(void) -{ - return moduleinit(); -} -#else -void -init_speedups(void) -{ - moduleinit(); -} -#endif diff --git a/pyload/lib/simplejson/compat.py b/pyload/lib/simplejson/compat.py deleted file mode 100644 index a0af4a1cb..000000000 --- a/pyload/lib/simplejson/compat.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Python 3 compatibility shims -""" -import sys -if sys.version_info[0] < 3: - PY3 = False - def b(s): - return s - def u(s): - return unicode(s, 'unicode_escape') - import cStringIO as StringIO - StringIO = BytesIO = StringIO.StringIO - text_type = unicode - binary_type = str - string_types = (basestring,) - integer_types = (int, long) - unichr = unichr - reload_module = reload - def fromhex(s): - return s.decode('hex') - -else: - PY3 = True - if sys.version_info[:2] >= (3, 4): - from importlib import reload as reload_module - else: - from imp import reload as reload_module - import codecs - def b(s): - return codecs.latin_1_encode(s)[0] - def u(s): - return s - import io - StringIO = io.StringIO - BytesIO = io.BytesIO - text_type = str - binary_type = bytes - string_types = (str,) - integer_types = (int,) - - def unichr(s): - return u(chr(s)) - - def fromhex(s): - return bytes.fromhex(s) - -long_type = integer_types[-1] diff --git a/pyload/lib/simplejson/decoder.py b/pyload/lib/simplejson/decoder.py deleted file mode 100644 index 545e65877..000000000 --- a/pyload/lib/simplejson/decoder.py +++ /dev/null @@ -1,400 +0,0 @@ -"""Implementation of JSONDecoder -""" -from __future__ import absolute_import -import re -import sys -import struct -from .compat import fromhex, b, u, text_type, binary_type, PY3, unichr -from .scanner import make_scanner, JSONDecodeError - -def _import_c_scanstring(): - try: - from ._speedups import scanstring - return scanstring - except ImportError: - return None -c_scanstring = _import_c_scanstring() - -# NOTE (3.1.0): JSONDecodeError may still be imported from this module for -# compatibility, but it was never in the __all__ -__all__ = ['JSONDecoder'] - -FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL - -def _floatconstants(): - _BYTES = fromhex('7FF80000000000007FF0000000000000') - # The struct module in Python 2.4 would get frexp() out of range here - # when an endian is specified in the format string. Fixed in Python 2.5+ - if sys.byteorder != 'big': - _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1] - nan, inf = struct.unpack('dd', _BYTES) - return nan, inf, -inf - -NaN, PosInf, NegInf = _floatconstants() - -_CONSTANTS = { - '-Infinity': NegInf, - 'Infinity': PosInf, - 'NaN': NaN, -} - -STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) -BACKSLASH = { - '"': u('"'), '\\': u('\u005c'), '/': u('/'), - 'b': u('\b'), 'f': u('\f'), 'n': u('\n'), 'r': u('\r'), 't': u('\t'), -} - -DEFAULT_ENCODING = "utf-8" - -def py_scanstring(s, end, encoding=None, strict=True, - _b=BACKSLASH, _m=STRINGCHUNK.match, _join=u('').join, - _PY3=PY3, _maxunicode=sys.maxunicode): - """Scan the string s for a JSON string. End is the index of the - character in s after the quote that started the JSON string. - Unescapes all valid JSON string escape sequences and raises ValueError - on attempt to decode an invalid string. If strict is False then literal - control characters are allowed in the string. - - Returns a tuple of the decoded string and the index of the character in s - after the end quote.""" - if encoding is None: - encoding = DEFAULT_ENCODING - chunks = [] - _append = chunks.append - begin = end - 1 - while 1: - chunk = _m(s, end) - if chunk is None: - raise JSONDecodeError( - "Unterminated string starting at", s, begin) - end = chunk.end() - content, terminator = chunk.groups() - # Content is contains zero or more unescaped string characters - if content: - if not _PY3 and not isinstance(content, text_type): - content = text_type(content, encoding) - _append(content) - # Terminator is the end of string, a literal control character, - # or a backslash denoting that an escape sequence follows - if terminator == '"': - break - elif terminator != '\\': - if strict: - msg = "Invalid control character %r at" - raise JSONDecodeError(msg, s, end) - else: - _append(terminator) - continue - try: - esc = s[end] - except IndexError: - raise JSONDecodeError( - "Unterminated string starting at", s, begin) - # If not a unicode escape sequence, must be in the lookup table - if esc != 'u': - try: - char = _b[esc] - except KeyError: - msg = "Invalid \\X escape sequence %r" - raise JSONDecodeError(msg, s, end) - end += 1 - else: - # Unicode escape sequence - msg = "Invalid \\uXXXX escape sequence" - esc = s[end + 1:end + 5] - escX = esc[1:2] - if len(esc) != 4 or escX == 'x' or escX == 'X': - raise JSONDecodeError(msg, s, end - 1) - try: - uni = int(esc, 16) - except ValueError: - raise JSONDecodeError(msg, s, end - 1) - end += 5 - # Check for surrogate pair on UCS-4 systems - # Note that this will join high/low surrogate pairs - # but will also pass unpaired surrogates through - if (_maxunicode > 65535 and - uni & 0xfc00 == 0xd800 and - s[end:end + 2] == '\\u'): - esc2 = s[end + 2:end + 6] - escX = esc2[1:2] - if len(esc2) == 4 and not (escX == 'x' or escX == 'X'): - try: - uni2 = int(esc2, 16) - except ValueError: - raise JSONDecodeError(msg, s, end) - if uni2 & 0xfc00 == 0xdc00: - uni = 0x10000 + (((uni - 0xd800) << 10) | - (uni2 - 0xdc00)) - end += 6 - char = unichr(uni) - # Append the unescaped character - _append(char) - return _join(chunks), end - - -# Use speedup if available -scanstring = c_scanstring or py_scanstring - -WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) -WHITESPACE_STR = ' \t\n\r' - -def JSONObject(state, encoding, strict, scan_once, object_hook, - object_pairs_hook, memo=None, - _w=WHITESPACE.match, _ws=WHITESPACE_STR): - (s, end) = state - # Backwards compatibility - if memo is None: - memo = {} - memo_get = memo.setdefault - pairs = [] - # Use a slice to prevent IndexError from being raised, the following - # check will raise a more specific ValueError if the string is empty - nextchar = s[end:end + 1] - # Normally we expect nextchar == '"' - if nextchar != '"': - if nextchar in _ws: - end = _w(s, end).end() - nextchar = s[end:end + 1] - # Trivial empty object - if nextchar == '}': - if object_pairs_hook is not None: - result = object_pairs_hook(pairs) - return result, end + 1 - pairs = {} - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end + 1 - elif nextchar != '"': - raise JSONDecodeError( - "Expecting property name enclosed in double quotes", - s, end) - end += 1 - while True: - key, end = scanstring(s, end, encoding, strict) - key = memo_get(key, key) - - # To skip some function call overhead we optimize the fast paths where - # the JSON key separator is ": " or just ":". - if s[end:end + 1] != ':': - end = _w(s, end).end() - if s[end:end + 1] != ':': - raise JSONDecodeError("Expecting ':' delimiter", s, end) - - end += 1 - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - value, end = scan_once(s, end) - pairs.append((key, value)) - - try: - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - end += 1 - - if nextchar == '}': - break - elif nextchar != ',': - raise JSONDecodeError("Expecting ',' delimiter or '}'", s, end - 1) - - try: - nextchar = s[end] - if nextchar in _ws: - end += 1 - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - - end += 1 - if nextchar != '"': - raise JSONDecodeError( - "Expecting property name enclosed in double quotes", - s, end - 1) - - if object_pairs_hook is not None: - result = object_pairs_hook(pairs) - return result, end - pairs = dict(pairs) - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end - -def JSONArray(state, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): - (s, end) = state - values = [] - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - # Look-ahead for trivial empty array - if nextchar == ']': - return values, end + 1 - elif nextchar == '': - raise JSONDecodeError("Expecting value or ']'", s, end) - _append = values.append - while True: - value, end = scan_once(s, end) - _append(value) - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - end += 1 - if nextchar == ']': - break - elif nextchar != ',': - raise JSONDecodeError("Expecting ',' delimiter or ']'", s, end - 1) - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - return values, end - -class JSONDecoder(object): - """Simple JSON decoder - - Performs the following translations in decoding by default: - - +---------------+-------------------+ - | JSON | Python | - +===============+===================+ - | object | dict | - +---------------+-------------------+ - | array | list | - +---------------+-------------------+ - | string | str, unicode | - +---------------+-------------------+ - | number (int) | int, long | - +---------------+-------------------+ - | number (real) | float | - +---------------+-------------------+ - | true | True | - +---------------+-------------------+ - | false | False | - +---------------+-------------------+ - | null | None | - +---------------+-------------------+ - - It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as - their corresponding ``float`` values, which is outside the JSON spec. - - """ - - def __init__(self, encoding=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, strict=True, - object_pairs_hook=None): - """ - *encoding* determines the encoding used to interpret any - :class:`str` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as :class:`unicode`. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *parse_constant*, if specified, will be called with one of the - following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This - can be used to raise an exception if invalid JSON numbers are - encountered. - - *strict* controls the parser's behavior when it encounters an - invalid control character in a string. The default setting of - ``True`` means that unescaped control characters are parse errors, if - ``False`` then control characters will be allowed in strings. - - """ - if encoding is None: - encoding = DEFAULT_ENCODING - self.encoding = encoding - self.object_hook = object_hook - self.object_pairs_hook = object_pairs_hook - self.parse_float = parse_float or float - self.parse_int = parse_int or int - self.parse_constant = parse_constant or _CONSTANTS.__getitem__ - self.strict = strict - self.parse_object = JSONObject - self.parse_array = JSONArray - self.parse_string = scanstring - self.memo = {} - self.scan_once = make_scanner(self) - - def decode(self, s, _w=WHITESPACE.match, _PY3=PY3): - """Return the Python representation of ``s`` (a ``str`` or ``unicode`` - instance containing a JSON document) - - """ - if _PY3 and isinstance(s, binary_type): - s = s.decode(self.encoding) - obj, end = self.raw_decode(s) - end = _w(s, end).end() - if end != len(s): - raise JSONDecodeError("Extra data", s, end, len(s)) - return obj - - def raw_decode(self, s, idx=0, _w=WHITESPACE.match, _PY3=PY3): - """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` - beginning with a JSON document) and return a 2-tuple of the Python - representation and the index in ``s`` where the document ended. - Optionally, ``idx`` can be used to specify an offset in ``s`` where - the JSON document begins. - - This can be used to decode a JSON document from a string that may - have extraneous data at the end. - - """ - if idx < 0: - # Ensure that raw_decode bails on negative indexes, the regex - # would otherwise mask this behavior. #98 - raise JSONDecodeError('Expecting value', s, idx) - if _PY3 and not isinstance(s, text_type): - raise TypeError("Input string must be text, not bytes") - # strip UTF-8 bom - if len(s) > idx: - ord0 = ord(s[idx]) - if ord0 == 0xfeff: - idx += 1 - elif ord0 == 0xef and s[idx:idx + 3] == '\xef\xbb\xbf': - idx += 3 - return self.scan_once(s, idx=_w(s, idx).end()) diff --git a/pyload/lib/simplejson/encoder.py b/pyload/lib/simplejson/encoder.py deleted file mode 100644 index db18244ec..000000000 --- a/pyload/lib/simplejson/encoder.py +++ /dev/null @@ -1,648 +0,0 @@ -"""Implementation of JSONEncoder -""" -from __future__ import absolute_import -import re -from operator import itemgetter -from decimal import Decimal -from .compat import u, unichr, binary_type, string_types, integer_types, PY3 -def _import_speedups(): - try: - from . import _speedups - return _speedups.encode_basestring_ascii, _speedups.make_encoder - except ImportError: - return None, None -c_encode_basestring_ascii, c_make_encoder = _import_speedups() - -from simplejson.decoder import PosInf - -#ESCAPE = re.compile(ur'[\x00-\x1f\\"\b\f\n\r\t\u2028\u2029]') -# This is required because u() will mangle the string and ur'' isn't valid -# python3 syntax -ESCAPE = re.compile(u'[\\x00-\\x1f\\\\"\\b\\f\\n\\r\\t\u2028\u2029]') -ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') -HAS_UTF8 = re.compile(r'[\x80-\xff]') -ESCAPE_DCT = { - '\\': '\\\\', - '"': '\\"', - '\b': '\\b', - '\f': '\\f', - '\n': '\\n', - '\r': '\\r', - '\t': '\\t', -} -for i in range(0x20): - #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) - ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) -for i in [0x2028, 0x2029]: - ESCAPE_DCT.setdefault(unichr(i), '\\u%04x' % (i,)) - -FLOAT_REPR = repr - -def encode_basestring(s, _PY3=PY3, _q=u('"')): - """Return a JSON representation of a Python string - - """ - if _PY3: - if isinstance(s, binary_type): - s = s.decode('utf-8') - else: - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = s.decode('utf-8') - def replace(match): - return ESCAPE_DCT[match.group(0)] - return _q + ESCAPE.sub(replace, s) + _q - - -def py_encode_basestring_ascii(s, _PY3=PY3): - """Return an ASCII-only JSON representation of a Python string - - """ - if _PY3: - if isinstance(s, binary_type): - s = s.decode('utf-8') - else: - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = s.decode('utf-8') - def replace(match): - s = match.group(0) - try: - return ESCAPE_DCT[s] - except KeyError: - n = ord(s) - if n < 0x10000: - #return '\\u{0:04x}'.format(n) - return '\\u%04x' % (n,) - else: - # surrogate pair - n -= 0x10000 - s1 = 0xd800 | ((n >> 10) & 0x3ff) - s2 = 0xdc00 | (n & 0x3ff) - #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) - return '\\u%04x\\u%04x' % (s1, s2) - return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' - - -encode_basestring_ascii = ( - c_encode_basestring_ascii or py_encode_basestring_ascii) - -class JSONEncoder(object): - """Extensible JSON encoder for Python data structures. - - Supports the following objects and types by default: - - +-------------------+---------------+ - | Python | JSON | - +===================+===============+ - | dict, namedtuple | object | - +-------------------+---------------+ - | list, tuple | array | - +-------------------+---------------+ - | str, unicode | string | - +-------------------+---------------+ - | int, long, float | number | - +-------------------+---------------+ - | True | true | - +-------------------+---------------+ - | False | false | - +-------------------+---------------+ - | None | null | - +-------------------+---------------+ - - To extend this to recognize other objects, subclass and implement a - ``.default()`` method with another method that returns a serializable - object for ``o`` if possible, otherwise it should call the superclass - implementation (to raise ``TypeError``). - - """ - item_separator = ', ' - key_separator = ': ' - - def __init__(self, skipkeys=False, ensure_ascii=True, - check_circular=True, allow_nan=True, sort_keys=False, - indent=None, separators=None, encoding='utf-8', default=None, - use_decimal=True, namedtuple_as_object=True, - tuple_as_array=True, bigint_as_string=False, - item_sort_key=None, for_json=False, ignore_nan=False, - int_as_string_bitcount=None): - """Constructor for JSONEncoder, with sensible defaults. - - If skipkeys is false, then it is a TypeError to attempt - encoding of keys that are not str, int, long, float or None. If - skipkeys is True, such items are simply skipped. - - If ensure_ascii is true, the output is guaranteed to be str - objects with all incoming unicode characters escaped. If - ensure_ascii is false, the output will be unicode object. - - If check_circular is true, then lists, dicts, and custom encoded - objects will be checked for circular references during encoding to - prevent an infinite recursion (which would cause an OverflowError). - Otherwise, no such check takes place. - - If allow_nan is true, then NaN, Infinity, and -Infinity will be - encoded as such. This behavior is not JSON specification compliant, - but is consistent with most JavaScript based encoders and decoders. - Otherwise, it will be a ValueError to encode such floats. - - If sort_keys is true, then the output of dictionaries will be - sorted by key; this is useful for regression tests to ensure - that JSON serializations can be compared on a day-to-day basis. - - If indent is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If specified, separators should be an (item_separator, key_separator) - tuple. The default is (', ', ': ') if *indent* is ``None`` and - (',', ': ') otherwise. To get the most compact JSON representation, - you should specify (',', ':') to eliminate whitespace. - - If specified, default is a function that gets called for objects - that can't otherwise be serialized. It should return a JSON encodable - version of the object or raise a ``TypeError``. - - If encoding is not None, then all input strings will be - transformed into unicode using that encoding prior to JSON-encoding. - The default is UTF-8. - - If use_decimal is true (not the default), ``decimal.Decimal`` will - be supported directly by the encoder. For the inverse, decode JSON - with ``parse_float=decimal.Decimal``. - - If namedtuple_as_object is true (the default), objects with - ``_asdict()`` methods will be encoded as JSON objects. - - If tuple_as_array is true (the default), tuple (and subclasses) will - be encoded as JSON arrays. - - If bigint_as_string is true (not the default), ints 2**53 and higher - or lower than -2**53 will be encoded as strings. This is to avoid the - rounding that happens in Javascript otherwise. - - If int_as_string_bitcount is a positive number (n), then int of size - greater than or equal to 2**n or lower than or equal to -2**n will be - encoded as strings. - - If specified, item_sort_key is a callable used to sort the items in - each dictionary. This is useful if you want to sort items other than - in alphabetical order by key. - - If for_json is true (not the default), objects with a ``for_json()`` - method will use the return value of that method for encoding as JSON - instead of the object. - - If *ignore_nan* is true (default: ``False``), then out of range - :class:`float` values (``nan``, ``inf``, ``-inf``) will be serialized - as ``null`` in compliance with the ECMA-262 specification. If true, - this will override *allow_nan*. - - """ - - self.skipkeys = skipkeys - self.ensure_ascii = ensure_ascii - self.check_circular = check_circular - self.allow_nan = allow_nan - self.sort_keys = sort_keys - self.use_decimal = use_decimal - self.namedtuple_as_object = namedtuple_as_object - self.tuple_as_array = tuple_as_array - self.bigint_as_string = bigint_as_string - self.item_sort_key = item_sort_key - self.for_json = for_json - self.ignore_nan = ignore_nan - self.int_as_string_bitcount = int_as_string_bitcount - if indent is not None and not isinstance(indent, string_types): - indent = indent * ' ' - self.indent = indent - if separators is not None: - self.item_separator, self.key_separator = separators - elif indent is not None: - self.item_separator = ',' - if default is not None: - self.default = default - self.encoding = encoding - - def default(self, o): - """Implement this method in a subclass such that it returns - a serializable object for ``o``, or calls the base implementation - (to raise a ``TypeError``). - - For example, to support arbitrary iterators, you could - implement default like this:: - - def default(self, o): - try: - iterable = iter(o) - except TypeError: - pass - else: - return list(iterable) - return JSONEncoder.default(self, o) - - """ - raise TypeError(repr(o) + " is not JSON serializable") - - def encode(self, o): - """Return a JSON string representation of a Python data structure. - - >>> from simplejson import JSONEncoder - >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) - '{"foo": ["bar", "baz"]}' - - """ - # This is for extremely simple cases and benchmarks. - if isinstance(o, binary_type): - _encoding = self.encoding - if (_encoding is not None and not (_encoding == 'utf-8')): - o = o.decode(_encoding) - if isinstance(o, string_types): - if self.ensure_ascii: - return encode_basestring_ascii(o) - else: - return encode_basestring(o) - # This doesn't pass the iterator directly to ''.join() because the - # exceptions aren't as detailed. The list call should be roughly - # equivalent to the PySequence_Fast that ''.join() would do. - chunks = self.iterencode(o, _one_shot=True) - if not isinstance(chunks, (list, tuple)): - chunks = list(chunks) - if self.ensure_ascii: - return ''.join(chunks) - else: - return u''.join(chunks) - - def iterencode(self, o, _one_shot=False): - """Encode the given object and yield each string - representation as available. - - For example:: - - for chunk in JSONEncoder().iterencode(bigobject): - mysocket.write(chunk) - - """ - if self.check_circular: - markers = {} - else: - markers = None - if self.ensure_ascii: - _encoder = encode_basestring_ascii - else: - _encoder = encode_basestring - if self.encoding != 'utf-8': - def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): - if isinstance(o, binary_type): - o = o.decode(_encoding) - return _orig_encoder(o) - - def floatstr(o, allow_nan=self.allow_nan, ignore_nan=self.ignore_nan, - _repr=FLOAT_REPR, _inf=PosInf, _neginf=-PosInf): - # Check for specials. Note that this type of test is processor - # and/or platform-specific, so do tests which don't depend on - # the internals. - - if o != o: - text = 'NaN' - elif o == _inf: - text = 'Infinity' - elif o == _neginf: - text = '-Infinity' - else: - return _repr(o) - - if ignore_nan: - text = 'null' - elif not allow_nan: - raise ValueError( - "Out of range float values are not JSON compliant: " + - repr(o)) - - return text - - key_memo = {} - int_as_string_bitcount = ( - 53 if self.bigint_as_string else self.int_as_string_bitcount) - if (_one_shot and c_make_encoder is not None - and self.indent is None): - _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, self.allow_nan, key_memo, self.use_decimal, - self.namedtuple_as_object, self.tuple_as_array, - int_as_string_bitcount, - self.item_sort_key, self.encoding, self.for_json, - self.ignore_nan, Decimal) - else: - _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, _one_shot, self.use_decimal, - self.namedtuple_as_object, self.tuple_as_array, - int_as_string_bitcount, - self.item_sort_key, self.encoding, self.for_json, - Decimal=Decimal) - try: - return _iterencode(o, 0) - finally: - key_memo.clear() - - -class JSONEncoderForHTML(JSONEncoder): - """An encoder that produces JSON safe to embed in HTML. - - To embed JSON content in, say, a script tag on a web page, the - characters &, < and > should be escaped. They cannot be escaped - with the usual entities (e.g. &) because they are not expanded - within ' - self.assertEqual( - r'"\u003c/script\u003e\u003cscript\u003e' - r'alert(\"gotcha\")\u003c/script\u003e"', - self.encoder.encode(bad_string)) - self.assertEqual( - bad_string, self.decoder.decode( - self.encoder.encode(bad_string))) diff --git a/pyload/lib/simplejson/tests/test_errors.py b/pyload/lib/simplejson/tests/test_errors.py deleted file mode 100644 index 8dede38fe..000000000 --- a/pyload/lib/simplejson/tests/test_errors.py +++ /dev/null @@ -1,51 +0,0 @@ -import sys, pickle -from unittest import TestCase - -import simplejson as json -from simplejson.compat import u, b - -class TestErrors(TestCase): - def test_string_keys_error(self): - data = [{'a': 'A', 'b': (2, 4), 'c': 3.0, ('d',): 'D tuple'}] - self.assertRaises(TypeError, json.dumps, data) - - def test_decode_error(self): - err = None - try: - json.loads('{}\na\nb') - except json.JSONDecodeError: - err = sys.exc_info()[1] - else: - self.fail('Expected JSONDecodeError') - self.assertEqual(err.lineno, 2) - self.assertEqual(err.colno, 1) - self.assertEqual(err.endlineno, 3) - self.assertEqual(err.endcolno, 2) - - def test_scan_error(self): - err = None - for t in (u, b): - try: - json.loads(t('{"asdf": "')) - except json.JSONDecodeError: - err = sys.exc_info()[1] - else: - self.fail('Expected JSONDecodeError') - self.assertEqual(err.lineno, 1) - self.assertEqual(err.colno, 10) - - def test_error_is_pickable(self): - err = None - try: - json.loads('{}\na\nb') - except json.JSONDecodeError: - err = sys.exc_info()[1] - else: - self.fail('Expected JSONDecodeError') - s = pickle.dumps(err) - e = pickle.loads(s) - - self.assertEqual(err.msg, e.msg) - self.assertEqual(err.doc, e.doc) - self.assertEqual(err.pos, e.pos) - self.assertEqual(err.end, e.end) diff --git a/pyload/lib/simplejson/tests/test_fail.py b/pyload/lib/simplejson/tests/test_fail.py deleted file mode 100644 index 788f3a525..000000000 --- a/pyload/lib/simplejson/tests/test_fail.py +++ /dev/null @@ -1,176 +0,0 @@ -import sys -from unittest import TestCase - -import simplejson as json - -# 2007-10-05 -JSONDOCS = [ - # http://json.org/JSON_checker/test/fail1.json - '"A JSON payload should be an object or array, not a string."', - # http://json.org/JSON_checker/test/fail2.json - '["Unclosed array"', - # http://json.org/JSON_checker/test/fail3.json - '{unquoted_key: "keys must be quoted"}', - # http://json.org/JSON_checker/test/fail4.json - '["extra comma",]', - # http://json.org/JSON_checker/test/fail5.json - '["double extra comma",,]', - # http://json.org/JSON_checker/test/fail6.json - '[ , "<-- missing value"]', - # http://json.org/JSON_checker/test/fail7.json - '["Comma after the close"],', - # http://json.org/JSON_checker/test/fail8.json - '["Extra close"]]', - # http://json.org/JSON_checker/test/fail9.json - '{"Extra comma": true,}', - # http://json.org/JSON_checker/test/fail10.json - '{"Extra value after close": true} "misplaced quoted value"', - # http://json.org/JSON_checker/test/fail11.json - '{"Illegal expression": 1 + 2}', - # http://json.org/JSON_checker/test/fail12.json - '{"Illegal invocation": alert()}', - # http://json.org/JSON_checker/test/fail13.json - '{"Numbers cannot have leading zeroes": 013}', - # http://json.org/JSON_checker/test/fail14.json - '{"Numbers cannot be hex": 0x14}', - # http://json.org/JSON_checker/test/fail15.json - '["Illegal backslash escape: \\x15"]', - # http://json.org/JSON_checker/test/fail16.json - '[\\naked]', - # http://json.org/JSON_checker/test/fail17.json - '["Illegal backslash escape: \\017"]', - # http://json.org/JSON_checker/test/fail18.json - '[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', - # http://json.org/JSON_checker/test/fail19.json - '{"Missing colon" null}', - # http://json.org/JSON_checker/test/fail20.json - '{"Double colon":: null}', - # http://json.org/JSON_checker/test/fail21.json - '{"Comma instead of colon", null}', - # http://json.org/JSON_checker/test/fail22.json - '["Colon instead of comma": false]', - # http://json.org/JSON_checker/test/fail23.json - '["Bad value", truth]', - # http://json.org/JSON_checker/test/fail24.json - "['single quote']", - # http://json.org/JSON_checker/test/fail25.json - '["\ttab\tcharacter\tin\tstring\t"]', - # http://json.org/JSON_checker/test/fail26.json - '["tab\\ character\\ in\\ string\\ "]', - # http://json.org/JSON_checker/test/fail27.json - '["line\nbreak"]', - # http://json.org/JSON_checker/test/fail28.json - '["line\\\nbreak"]', - # http://json.org/JSON_checker/test/fail29.json - '[0e]', - # http://json.org/JSON_checker/test/fail30.json - '[0e+]', - # http://json.org/JSON_checker/test/fail31.json - '[0e+-1]', - # http://json.org/JSON_checker/test/fail32.json - '{"Comma instead if closing brace": true,', - # http://json.org/JSON_checker/test/fail33.json - '["mismatch"}', - # http://code.google.com/p/simplejson/issues/detail?id=3 - u'["A\u001FZ control characters in string"]', - # misc based on coverage - '{', - '{]', - '{"foo": "bar"]', - '{"foo": "bar"', - 'nul', - 'nulx', - '-', - '-x', - '-e', - '-e0', - '-Infinite', - '-Inf', - 'Infinit', - 'Infinite', - 'NaM', - 'NuN', - 'falsy', - 'fal', - 'trug', - 'tru', - '1e', - '1ex', - '1e-', - '1e-x', -] - -SKIPS = { - 1: "why not have a string payload?", - 18: "spec doesn't specify any nesting limitations", -} - -class TestFail(TestCase): - def test_failures(self): - for idx, doc in enumerate(JSONDOCS): - idx = idx + 1 - if idx in SKIPS: - json.loads(doc) - continue - try: - json.loads(doc) - except json.JSONDecodeError: - pass - else: - self.fail("Expected failure for fail%d.json: %r" % (idx, doc)) - - def test_array_decoder_issue46(self): - # http://code.google.com/p/simplejson/issues/detail?id=46 - for doc in [u'[,]', '[,]']: - try: - json.loads(doc) - except json.JSONDecodeError: - e = sys.exc_info()[1] - self.assertEqual(e.pos, 1) - self.assertEqual(e.lineno, 1) - self.assertEqual(e.colno, 2) - except Exception: - e = sys.exc_info()[1] - self.fail("Unexpected exception raised %r %s" % (e, e)) - else: - self.fail("Unexpected success parsing '[,]'") - - def test_truncated_input(self): - test_cases = [ - ('', 'Expecting value', 0), - ('[', "Expecting value or ']'", 1), - ('[42', "Expecting ',' delimiter", 3), - ('[42,', 'Expecting value', 4), - ('["', 'Unterminated string starting at', 1), - ('["spam', 'Unterminated string starting at', 1), - ('["spam"', "Expecting ',' delimiter", 7), - ('["spam",', 'Expecting value', 8), - ('{', 'Expecting property name enclosed in double quotes', 1), - ('{"', 'Unterminated string starting at', 1), - ('{"spam', 'Unterminated string starting at', 1), - ('{"spam"', "Expecting ':' delimiter", 7), - ('{"spam":', 'Expecting value', 8), - ('{"spam":42', "Expecting ',' delimiter", 10), - ('{"spam":42,', 'Expecting property name enclosed in double quotes', - 11), - ('"', 'Unterminated string starting at', 0), - ('"spam', 'Unterminated string starting at', 0), - ('[,', "Expecting value", 1), - ] - for data, msg, idx in test_cases: - try: - json.loads(data) - except json.JSONDecodeError: - e = sys.exc_info()[1] - self.assertEqual( - e.msg[:len(msg)], - msg, - "%r doesn't start with %r for %r" % (e.msg, msg, data)) - self.assertEqual( - e.pos, idx, - "pos %r != %r for %r" % (e.pos, idx, data)) - except Exception: - e = sys.exc_info()[1] - self.fail("Unexpected exception raised %r %s" % (e, e)) - else: - self.fail("Unexpected success parsing '%r'" % (data,)) diff --git a/pyload/lib/simplejson/tests/test_float.py b/pyload/lib/simplejson/tests/test_float.py deleted file mode 100644 index e382ec21a..000000000 --- a/pyload/lib/simplejson/tests/test_float.py +++ /dev/null @@ -1,35 +0,0 @@ -import math -from unittest import TestCase -from simplejson.compat import long_type, text_type -import simplejson as json -from simplejson.decoder import NaN, PosInf, NegInf - -class TestFloat(TestCase): - def test_degenerates_allow(self): - for inf in (PosInf, NegInf): - self.assertEqual(json.loads(json.dumps(inf)), inf) - # Python 2.5 doesn't have math.isnan - nan = json.loads(json.dumps(NaN)) - self.assertTrue((0 + nan) != nan) - - def test_degenerates_ignore(self): - for f in (PosInf, NegInf, NaN): - self.assertEqual(json.loads(json.dumps(f, ignore_nan=True)), None) - - def test_degenerates_deny(self): - for f in (PosInf, NegInf, NaN): - self.assertRaises(ValueError, json.dumps, f, allow_nan=False) - - def test_floats(self): - for num in [1617161771.7650001, math.pi, math.pi**100, - math.pi**-100, 3.1]: - self.assertEqual(float(json.dumps(num)), num) - self.assertEqual(json.loads(json.dumps(num)), num) - self.assertEqual(json.loads(text_type(json.dumps(num))), num) - - def test_ints(self): - for num in [1, long_type(1), 1<<32, 1<<64]: - self.assertEqual(json.dumps(num), str(num)) - self.assertEqual(int(json.dumps(num)), num) - self.assertEqual(json.loads(json.dumps(num)), num) - self.assertEqual(json.loads(text_type(json.dumps(num))), num) diff --git a/pyload/lib/simplejson/tests/test_for_json.py b/pyload/lib/simplejson/tests/test_for_json.py deleted file mode 100644 index b791b883b..000000000 --- a/pyload/lib/simplejson/tests/test_for_json.py +++ /dev/null @@ -1,97 +0,0 @@ -import unittest -import simplejson as json - - -class ForJson(object): - def for_json(self): - return {'for_json': 1} - - -class NestedForJson(object): - def for_json(self): - return {'nested': ForJson()} - - -class ForJsonList(object): - def for_json(self): - return ['list'] - - -class DictForJson(dict): - def for_json(self): - return {'alpha': 1} - - -class ListForJson(list): - def for_json(self): - return ['list'] - - -class TestForJson(unittest.TestCase): - def assertRoundTrip(self, obj, other, for_json=True): - if for_json is None: - # None will use the default - s = json.dumps(obj) - else: - s = json.dumps(obj, for_json=for_json) - self.assertEqual( - json.loads(s), - other) - - def test_for_json_encodes_stand_alone_object(self): - self.assertRoundTrip( - ForJson(), - ForJson().for_json()) - - def test_for_json_encodes_object_nested_in_dict(self): - self.assertRoundTrip( - {'hooray': ForJson()}, - {'hooray': ForJson().for_json()}) - - def test_for_json_encodes_object_nested_in_list_within_dict(self): - self.assertRoundTrip( - {'list': [0, ForJson(), 2, 3]}, - {'list': [0, ForJson().for_json(), 2, 3]}) - - def test_for_json_encodes_object_nested_within_object(self): - self.assertRoundTrip( - NestedForJson(), - {'nested': {'for_json': 1}}) - - def test_for_json_encodes_list(self): - self.assertRoundTrip( - ForJsonList(), - ForJsonList().for_json()) - - def test_for_json_encodes_list_within_object(self): - self.assertRoundTrip( - {'nested': ForJsonList()}, - {'nested': ForJsonList().for_json()}) - - def test_for_json_encodes_dict_subclass(self): - self.assertRoundTrip( - DictForJson(a=1), - DictForJson(a=1).for_json()) - - def test_for_json_encodes_list_subclass(self): - self.assertRoundTrip( - ListForJson(['l']), - ListForJson(['l']).for_json()) - - def test_for_json_ignored_if_not_true_with_dict_subclass(self): - for for_json in (None, False): - self.assertRoundTrip( - DictForJson(a=1), - {'a': 1}, - for_json=for_json) - - def test_for_json_ignored_if_not_true_with_list_subclass(self): - for for_json in (None, False): - self.assertRoundTrip( - ListForJson(['l']), - ['l'], - for_json=for_json) - - def test_raises_typeerror_if_for_json_not_true_with_object(self): - self.assertRaises(TypeError, json.dumps, ForJson()) - self.assertRaises(TypeError, json.dumps, ForJson(), for_json=False) diff --git a/pyload/lib/simplejson/tests/test_indent.py b/pyload/lib/simplejson/tests/test_indent.py deleted file mode 100644 index cea25a575..000000000 --- a/pyload/lib/simplejson/tests/test_indent.py +++ /dev/null @@ -1,86 +0,0 @@ -from unittest import TestCase -import textwrap - -import simplejson as json -from simplejson.compat import StringIO - -class TestIndent(TestCase): - def test_indent(self): - h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', - 'i-vhbjkhnth', - {'nifty': 87}, {'field': 'yes', 'morefield': False} ] - - expect = textwrap.dedent("""\ - [ - \t[ - \t\t"blorpie" - \t], - \t[ - \t\t"whoops" - \t], - \t[], - \t"d-shtaeou", - \t"d-nthiouh", - \t"i-vhbjkhnth", - \t{ - \t\t"nifty": 87 - \t}, - \t{ - \t\t"field": "yes", - \t\t"morefield": false - \t} - ]""") - - - d1 = json.dumps(h) - d2 = json.dumps(h, indent='\t', sort_keys=True, separators=(',', ': ')) - d3 = json.dumps(h, indent=' ', sort_keys=True, separators=(',', ': ')) - d4 = json.dumps(h, indent=2, sort_keys=True, separators=(',', ': ')) - - h1 = json.loads(d1) - h2 = json.loads(d2) - h3 = json.loads(d3) - h4 = json.loads(d4) - - self.assertEqual(h1, h) - self.assertEqual(h2, h) - self.assertEqual(h3, h) - self.assertEqual(h4, h) - self.assertEqual(d3, expect.replace('\t', ' ')) - self.assertEqual(d4, expect.replace('\t', ' ')) - # NOTE: Python 2.4 textwrap.dedent converts tabs to spaces, - # so the following is expected to fail. Python 2.4 is not a - # supported platform in simplejson 2.1.0+. - self.assertEqual(d2, expect) - - def test_indent0(self): - h = {3: 1} - def check(indent, expected): - d1 = json.dumps(h, indent=indent) - self.assertEqual(d1, expected) - - sio = StringIO() - json.dump(h, sio, indent=indent) - self.assertEqual(sio.getvalue(), expected) - - # indent=0 should emit newlines - check(0, '{\n"3": 1\n}') - # indent=None is more compact - check(None, '{"3": 1}') - - def test_separators(self): - lst = [1,2,3,4] - expect = '[\n1,\n2,\n3,\n4\n]' - expect_spaces = '[\n1, \n2, \n3, \n4\n]' - # Ensure that separators still works - self.assertEqual( - expect_spaces, - json.dumps(lst, indent=0, separators=(', ', ': '))) - # Force the new defaults - self.assertEqual( - expect, - json.dumps(lst, indent=0, separators=(',', ': '))) - # Added in 2.1.4 - self.assertEqual( - expect, - json.dumps(lst, indent=0)) diff --git a/pyload/lib/simplejson/tests/test_item_sort_key.py b/pyload/lib/simplejson/tests/test_item_sort_key.py deleted file mode 100644 index b05bfc814..000000000 --- a/pyload/lib/simplejson/tests/test_item_sort_key.py +++ /dev/null @@ -1,20 +0,0 @@ -from unittest import TestCase - -import simplejson as json -from operator import itemgetter - -class TestItemSortKey(TestCase): - def test_simple_first(self): - a = {'a': 1, 'c': 5, 'jack': 'jill', 'pick': 'axe', 'array': [1, 5, 6, 9], 'tuple': (83, 12, 3), 'crate': 'dog', 'zeak': 'oh'} - self.assertEqual( - '{"a": 1, "c": 5, "crate": "dog", "jack": "jill", "pick": "axe", "zeak": "oh", "array": [1, 5, 6, 9], "tuple": [83, 12, 3]}', - json.dumps(a, item_sort_key=json.simple_first)) - - def test_case(self): - a = {'a': 1, 'c': 5, 'Jack': 'jill', 'pick': 'axe', 'Array': [1, 5, 6, 9], 'tuple': (83, 12, 3), 'crate': 'dog', 'zeak': 'oh'} - self.assertEqual( - '{"Array": [1, 5, 6, 9], "Jack": "jill", "a": 1, "c": 5, "crate": "dog", "pick": "axe", "tuple": [83, 12, 3], "zeak": "oh"}', - json.dumps(a, item_sort_key=itemgetter(0))) - self.assertEqual( - '{"a": 1, "Array": [1, 5, 6, 9], "c": 5, "crate": "dog", "Jack": "jill", "pick": "axe", "tuple": [83, 12, 3], "zeak": "oh"}', - json.dumps(a, item_sort_key=lambda kv: kv[0].lower())) diff --git a/pyload/lib/simplejson/tests/test_namedtuple.py b/pyload/lib/simplejson/tests/test_namedtuple.py deleted file mode 100644 index 438789405..000000000 --- a/pyload/lib/simplejson/tests/test_namedtuple.py +++ /dev/null @@ -1,122 +0,0 @@ -from __future__ import absolute_import -import unittest -import simplejson as json -from simplejson.compat import StringIO - -try: - from collections import namedtuple -except ImportError: - class Value(tuple): - def __new__(cls, *args): - return tuple.__new__(cls, args) - - def _asdict(self): - return {'value': self[0]} - class Point(tuple): - def __new__(cls, *args): - return tuple.__new__(cls, args) - - def _asdict(self): - return {'x': self[0], 'y': self[1]} -else: - Value = namedtuple('Value', ['value']) - Point = namedtuple('Point', ['x', 'y']) - -class DuckValue(object): - def __init__(self, *args): - self.value = Value(*args) - - def _asdict(self): - return self.value._asdict() - -class DuckPoint(object): - def __init__(self, *args): - self.point = Point(*args) - - def _asdict(self): - return self.point._asdict() - -class DeadDuck(object): - _asdict = None - -class DeadDict(dict): - _asdict = None - -CONSTRUCTORS = [ - lambda v: v, - lambda v: [v], - lambda v: [{'key': v}], -] - -class TestNamedTuple(unittest.TestCase): - def test_namedtuple_dumps(self): - for v in [Value(1), Point(1, 2), DuckValue(1), DuckPoint(1, 2)]: - d = v._asdict() - self.assertEqual(d, json.loads(json.dumps(v))) - self.assertEqual( - d, - json.loads(json.dumps(v, namedtuple_as_object=True))) - self.assertEqual(d, json.loads(json.dumps(v, tuple_as_array=False))) - self.assertEqual( - d, - json.loads(json.dumps(v, namedtuple_as_object=True, - tuple_as_array=False))) - - def test_namedtuple_dumps_false(self): - for v in [Value(1), Point(1, 2)]: - l = list(v) - self.assertEqual( - l, - json.loads(json.dumps(v, namedtuple_as_object=False))) - self.assertRaises(TypeError, json.dumps, v, - tuple_as_array=False, namedtuple_as_object=False) - - def test_namedtuple_dump(self): - for v in [Value(1), Point(1, 2), DuckValue(1), DuckPoint(1, 2)]: - d = v._asdict() - sio = StringIO() - json.dump(v, sio) - self.assertEqual(d, json.loads(sio.getvalue())) - sio = StringIO() - json.dump(v, sio, namedtuple_as_object=True) - self.assertEqual( - d, - json.loads(sio.getvalue())) - sio = StringIO() - json.dump(v, sio, tuple_as_array=False) - self.assertEqual(d, json.loads(sio.getvalue())) - sio = StringIO() - json.dump(v, sio, namedtuple_as_object=True, - tuple_as_array=False) - self.assertEqual( - d, - json.loads(sio.getvalue())) - - def test_namedtuple_dump_false(self): - for v in [Value(1), Point(1, 2)]: - l = list(v) - sio = StringIO() - json.dump(v, sio, namedtuple_as_object=False) - self.assertEqual( - l, - json.loads(sio.getvalue())) - self.assertRaises(TypeError, json.dump, v, StringIO(), - tuple_as_array=False, namedtuple_as_object=False) - - def test_asdict_not_callable_dump(self): - for f in CONSTRUCTORS: - self.assertRaises(TypeError, - json.dump, f(DeadDuck()), StringIO(), namedtuple_as_object=True) - sio = StringIO() - json.dump(f(DeadDict()), sio, namedtuple_as_object=True) - self.assertEqual( - json.dumps(f({})), - sio.getvalue()) - - def test_asdict_not_callable_dumps(self): - for f in CONSTRUCTORS: - self.assertRaises(TypeError, - json.dumps, f(DeadDuck()), namedtuple_as_object=True) - self.assertEqual( - json.dumps(f({})), - json.dumps(f(DeadDict()), namedtuple_as_object=True)) diff --git a/pyload/lib/simplejson/tests/test_pass1.py b/pyload/lib/simplejson/tests/test_pass1.py deleted file mode 100644 index f0b5b10e7..000000000 --- a/pyload/lib/simplejson/tests/test_pass1.py +++ /dev/null @@ -1,71 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -# from http://json.org/JSON_checker/test/pass1.json -JSON = r''' -[ - "JSON Test Pattern pass1", - {"object with 1 member":["array with 1 element"]}, - {}, - [], - -42, - true, - false, - null, - { - "integer": 1234567890, - "real": -9876.543210, - "e": 0.123456789e-12, - "E": 1.234567890E+34, - "": 23456789012E66, - "zero": 0, - "one": 1, - "space": " ", - "quote": "\"", - "backslash": "\\", - "controls": "\b\f\n\r\t", - "slash": "/ & \/", - "alpha": "abcdefghijklmnopqrstuvwyz", - "ALPHA": "ABCDEFGHIJKLMNOPQRSTUVWYZ", - "digit": "0123456789", - "special": "`1~!@#$%^&*()_+-={':[,]}|;.?", - "hex": "\u0123\u4567\u89AB\uCDEF\uabcd\uef4A", - "true": true, - "false": false, - "null": null, - "array":[ ], - "object":{ }, - "address": "50 St. James Street", - "url": "http://www.JSON.org/", - "comment": "// /* */": " ", - " s p a c e d " :[1,2 , 3 - -, - -4 , 5 , 6 ,7 ],"compact": [1,2,3,4,5,6,7], - "jsontext": "{\"object with 1 member\":[\"array with 1 element\"]}", - "quotes": "" \u0022 %22 0x22 034 "", - "\/\\\"\uCAFE\uBABE\uAB98\uFCDE\ubcda\uef4A\b\f\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?" -: "A key can be any string" - }, - 0.5 ,98.6 -, -99.44 -, - -1066, -1e1, -0.1e1, -1e-1, -1e00,2e+00,2e-00 -,"rosebud"] -''' - -class TestPass1(TestCase): - def test_parse(self): - # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) diff --git a/pyload/lib/simplejson/tests/test_pass2.py b/pyload/lib/simplejson/tests/test_pass2.py deleted file mode 100644 index 5d812b3bb..000000000 --- a/pyload/lib/simplejson/tests/test_pass2.py +++ /dev/null @@ -1,14 +0,0 @@ -from unittest import TestCase -import simplejson as json - -# from http://json.org/JSON_checker/test/pass2.json -JSON = r''' -[[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]] -''' - -class TestPass2(TestCase): - def test_parse(self): - # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) diff --git a/pyload/lib/simplejson/tests/test_pass3.py b/pyload/lib/simplejson/tests/test_pass3.py deleted file mode 100644 index 821d60b22..000000000 --- a/pyload/lib/simplejson/tests/test_pass3.py +++ /dev/null @@ -1,20 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -# from http://json.org/JSON_checker/test/pass3.json -JSON = r''' -{ - "JSON Test Pattern pass3": { - "The outermost value": "must be an object or array.", - "In this test": "It is an object." - } -} -''' - -class TestPass3(TestCase): - def test_parse(self): - # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) diff --git a/pyload/lib/simplejson/tests/test_recursion.py b/pyload/lib/simplejson/tests/test_recursion.py deleted file mode 100644 index 662eb667e..000000000 --- a/pyload/lib/simplejson/tests/test_recursion.py +++ /dev/null @@ -1,67 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -class JSONTestObject: - pass - - -class RecursiveJSONEncoder(json.JSONEncoder): - recurse = False - def default(self, o): - if o is JSONTestObject: - if self.recurse: - return [JSONTestObject] - else: - return 'JSONTestObject' - return json.JSONEncoder.default(o) - - -class TestRecursion(TestCase): - def test_listrecursion(self): - x = [] - x.append(x) - try: - json.dumps(x) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on list recursion") - x = [] - y = [x] - x.append(y) - try: - json.dumps(x) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on alternating list recursion") - y = [] - x = [y, y] - # ensure that the marker is cleared - json.dumps(x) - - def test_dictrecursion(self): - x = {} - x["test"] = x - try: - json.dumps(x) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on dict recursion") - x = {} - y = {"a": x, "b": x} - # ensure that the marker is cleared - json.dumps(y) - - def test_defaultrecursion(self): - enc = RecursiveJSONEncoder() - self.assertEqual(enc.encode(JSONTestObject), '"JSONTestObject"') - enc.recurse = True - try: - enc.encode(JSONTestObject) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on default recursion") diff --git a/pyload/lib/simplejson/tests/test_scanstring.py b/pyload/lib/simplejson/tests/test_scanstring.py deleted file mode 100644 index 3d98f0d82..000000000 --- a/pyload/lib/simplejson/tests/test_scanstring.py +++ /dev/null @@ -1,194 +0,0 @@ -import sys -from unittest import TestCase - -import simplejson as json -import simplejson.decoder -from simplejson.compat import b, PY3 - -class TestScanString(TestCase): - # The bytes type is intentionally not used in most of these tests - # under Python 3 because the decoder immediately coerces to str before - # calling scanstring. In Python 2 we are testing the code paths - # for both unicode and str. - # - # The reason this is done is because Python 3 would require - # entirely different code paths for parsing bytes and str. - # - def test_py_scanstring(self): - self._test_scanstring(simplejson.decoder.py_scanstring) - - def test_c_scanstring(self): - if not simplejson.decoder.c_scanstring: - return - self._test_scanstring(simplejson.decoder.c_scanstring) - - def _test_scanstring(self, scanstring): - if sys.maxunicode == 65535: - self.assertEqual( - scanstring(u'"z\U0001d120x"', 1, None, True), - (u'z\U0001d120x', 6)) - else: - self.assertEqual( - scanstring(u'"z\U0001d120x"', 1, None, True), - (u'z\U0001d120x', 5)) - - self.assertEqual( - scanstring('"\\u007b"', 1, None, True), - (u'{', 8)) - - self.assertEqual( - scanstring('"A JSON payload should be an object or array, not a string."', 1, None, True), - (u'A JSON payload should be an object or array, not a string.', 60)) - - self.assertEqual( - scanstring('["Unclosed array"', 2, None, True), - (u'Unclosed array', 17)) - - self.assertEqual( - scanstring('["extra comma",]', 2, None, True), - (u'extra comma', 14)) - - self.assertEqual( - scanstring('["double extra comma",,]', 2, None, True), - (u'double extra comma', 21)) - - self.assertEqual( - scanstring('["Comma after the close"],', 2, None, True), - (u'Comma after the close', 24)) - - self.assertEqual( - scanstring('["Extra close"]]', 2, None, True), - (u'Extra close', 14)) - - self.assertEqual( - scanstring('{"Extra comma": true,}', 2, None, True), - (u'Extra comma', 14)) - - self.assertEqual( - scanstring('{"Extra value after close": true} "misplaced quoted value"', 2, None, True), - (u'Extra value after close', 26)) - - self.assertEqual( - scanstring('{"Illegal expression": 1 + 2}', 2, None, True), - (u'Illegal expression', 21)) - - self.assertEqual( - scanstring('{"Illegal invocation": alert()}', 2, None, True), - (u'Illegal invocation', 21)) - - self.assertEqual( - scanstring('{"Numbers cannot have leading zeroes": 013}', 2, None, True), - (u'Numbers cannot have leading zeroes', 37)) - - self.assertEqual( - scanstring('{"Numbers cannot be hex": 0x14}', 2, None, True), - (u'Numbers cannot be hex', 24)) - - self.assertEqual( - scanstring('[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', 21, None, True), - (u'Too deep', 30)) - - self.assertEqual( - scanstring('{"Missing colon" null}', 2, None, True), - (u'Missing colon', 16)) - - self.assertEqual( - scanstring('{"Double colon":: null}', 2, None, True), - (u'Double colon', 15)) - - self.assertEqual( - scanstring('{"Comma instead of colon", null}', 2, None, True), - (u'Comma instead of colon', 25)) - - self.assertEqual( - scanstring('["Colon instead of comma": false]', 2, None, True), - (u'Colon instead of comma', 25)) - - self.assertEqual( - scanstring('["Bad value", truth]', 2, None, True), - (u'Bad value', 12)) - - for c in map(chr, range(0x00, 0x1f)): - self.assertEqual( - scanstring(c + '"', 0, None, False), - (c, 2)) - self.assertRaises( - ValueError, - scanstring, c + '"', 0, None, True) - - self.assertRaises(ValueError, scanstring, '', 0, None, True) - self.assertRaises(ValueError, scanstring, 'a', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u0', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u01', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u012', 0, None, True) - self.assertRaises(ValueError, scanstring, '\\u0123', 0, None, True) - if sys.maxunicode > 65535: - self.assertRaises(ValueError, - scanstring, '\\ud834\\u"', 0, None, True) - self.assertRaises(ValueError, - scanstring, '\\ud834\\x0123"', 0, None, True) - - def test_issue3623(self): - self.assertRaises(ValueError, json.decoder.scanstring, "xxx", 1, - "xxx") - self.assertRaises(UnicodeDecodeError, - json.encoder.encode_basestring_ascii, b("xx\xff")) - - def test_overflow(self): - # Python 2.5 does not have maxsize, Python 3 does not have maxint - maxsize = getattr(sys, 'maxsize', getattr(sys, 'maxint', None)) - assert maxsize is not None - self.assertRaises(OverflowError, json.decoder.scanstring, "xxx", - maxsize + 1) - - def test_surrogates(self): - scanstring = json.decoder.scanstring - - def assertScan(given, expect, test_utf8=True): - givens = [given] - if not PY3 and test_utf8: - givens.append(given.encode('utf8')) - for given in givens: - (res, count) = scanstring(given, 1, None, True) - self.assertEqual(len(given), count) - self.assertEqual(res, expect) - - assertScan( - u'"z\\ud834\\u0079x"', - u'z\ud834yx') - assertScan( - u'"z\\ud834\\udd20x"', - u'z\U0001d120x') - assertScan( - u'"z\\ud834\\ud834\\udd20x"', - u'z\ud834\U0001d120x') - assertScan( - u'"z\\ud834x"', - u'z\ud834x') - assertScan( - u'"z\\udd20x"', - u'z\udd20x') - assertScan( - u'"z\ud834x"', - u'z\ud834x') - # It may look strange to join strings together, but Python is drunk. - # https://gist.github.com/etrepum/5538443 - assertScan( - u'"z\\ud834\udd20x12345"', - u''.join([u'z\ud834', u'\udd20x12345'])) - assertScan( - u'"z\ud834\\udd20x"', - u''.join([u'z\ud834', u'\udd20x'])) - # these have different behavior given UTF8 input, because the surrogate - # pair may be joined (in maxunicode > 65535 builds) - assertScan( - u''.join([u'"z\ud834', u'\udd20x"']), - u''.join([u'z\ud834', u'\udd20x']), - test_utf8=False) - - self.assertRaises(ValueError, - scanstring, u'"z\\ud83x"', 1, None, True) - self.assertRaises(ValueError, - scanstring, u'"z\\ud834\\udd2x"', 1, None, True) diff --git a/pyload/lib/simplejson/tests/test_separators.py b/pyload/lib/simplejson/tests/test_separators.py deleted file mode 100644 index 91b4d4fb6..000000000 --- a/pyload/lib/simplejson/tests/test_separators.py +++ /dev/null @@ -1,42 +0,0 @@ -import textwrap -from unittest import TestCase - -import simplejson as json - - -class TestSeparators(TestCase): - def test_separators(self): - h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth', - {'nifty': 87}, {'field': 'yes', 'morefield': False} ] - - expect = textwrap.dedent("""\ - [ - [ - "blorpie" - ] , - [ - "whoops" - ] , - [] , - "d-shtaeou" , - "d-nthiouh" , - "i-vhbjkhnth" , - { - "nifty" : 87 - } , - { - "field" : "yes" , - "morefield" : false - } - ]""") - - - d1 = json.dumps(h) - d2 = json.dumps(h, indent=' ', sort_keys=True, separators=(' ,', ' : ')) - - h1 = json.loads(d1) - h2 = json.loads(d2) - - self.assertEqual(h1, h) - self.assertEqual(h2, h) - self.assertEqual(d2, expect) diff --git a/pyload/lib/simplejson/tests/test_speedups.py b/pyload/lib/simplejson/tests/test_speedups.py deleted file mode 100644 index 0a2b63bff..000000000 --- a/pyload/lib/simplejson/tests/test_speedups.py +++ /dev/null @@ -1,39 +0,0 @@ -import sys -import unittest -from unittest import TestCase - -from simplejson import encoder, scanner - - -def has_speedups(): - return encoder.c_make_encoder is not None - - -def skip_if_speedups_missing(func): - def wrapper(*args, **kwargs): - if not has_speedups(): - if hasattr(unittest, 'SkipTest'): - raise unittest.SkipTest("C Extension not available") - else: - sys.stdout.write("C Extension not available") - return - return func(*args, **kwargs) - - return wrapper - - -class TestDecode(TestCase): - @skip_if_speedups_missing - def test_make_scanner(self): - self.assertRaises(AttributeError, scanner.c_make_scanner, 1) - - @skip_if_speedups_missing - def test_make_encoder(self): - self.assertRaises( - TypeError, - encoder.c_make_encoder, - None, - ("\xCD\x7D\x3D\x4E\x12\x4C\xF9\x79\xD7" - "\x52\xBA\x82\xF2\x27\x4A\x7D\xA0\xCA\x75"), - None - ) diff --git a/pyload/lib/simplejson/tests/test_tool.py b/pyload/lib/simplejson/tests/test_tool.py deleted file mode 100644 index ac2a14c90..000000000 --- a/pyload/lib/simplejson/tests/test_tool.py +++ /dev/null @@ -1,97 +0,0 @@ -from __future__ import with_statement -import os -import sys -import textwrap -import unittest -import subprocess -import tempfile -try: - # Python 3.x - from test.support import strip_python_stderr -except ImportError: - # Python 2.6+ - try: - from test.test_support import strip_python_stderr - except ImportError: - # Python 2.5 - import re - def strip_python_stderr(stderr): - return re.sub( - r"\[\d+ refs\]\r?\n?$".encode(), - "".encode(), - stderr).strip() - -class TestTool(unittest.TestCase): - data = """ - - [["blorpie"],[ "whoops" ] , [ - ],\t"d-shtaeou",\r"d-nthiouh", - "i-vhbjkhnth", {"nifty":87}, {"morefield" :\tfalse,"field" - :"yes"} ] - """ - - expect = textwrap.dedent("""\ - [ - [ - "blorpie" - ], - [ - "whoops" - ], - [], - "d-shtaeou", - "d-nthiouh", - "i-vhbjkhnth", - { - "nifty": 87 - }, - { - "field": "yes", - "morefield": false - } - ] - """) - - def runTool(self, args=None, data=None): - argv = [sys.executable, '-m', 'simplejson.tool'] - if args: - argv.extend(args) - proc = subprocess.Popen(argv, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE) - out, err = proc.communicate(data) - self.assertEqual(strip_python_stderr(err), ''.encode()) - self.assertEqual(proc.returncode, 0) - return out - - def test_stdin_stdout(self): - self.assertEqual( - self.runTool(data=self.data.encode()), - self.expect.encode()) - - def test_infile_stdout(self): - with tempfile.NamedTemporaryFile() as infile: - infile.write(self.data.encode()) - infile.flush() - self.assertEqual( - self.runTool(args=[infile.name]), - self.expect.encode()) - - def test_infile_outfile(self): - with tempfile.NamedTemporaryFile() as infile: - infile.write(self.data.encode()) - infile.flush() - # outfile will get overwritten by tool, so the delete - # may not work on some platforms. Do it manually. - outfile = tempfile.NamedTemporaryFile() - try: - self.assertEqual( - self.runTool(args=[infile.name, outfile.name]), - ''.encode()) - with open(outfile.name, 'rb') as f: - self.assertEqual(f.read(), self.expect.encode()) - finally: - outfile.close() - if os.path.exists(outfile.name): - os.unlink(outfile.name) diff --git a/pyload/lib/simplejson/tests/test_tuple.py b/pyload/lib/simplejson/tests/test_tuple.py deleted file mode 100644 index a6a991005..000000000 --- a/pyload/lib/simplejson/tests/test_tuple.py +++ /dev/null @@ -1,51 +0,0 @@ -import unittest - -from simplejson.compat import StringIO -import simplejson as json - -class TestTuples(unittest.TestCase): - def test_tuple_array_dumps(self): - t = (1, 2, 3) - expect = json.dumps(list(t)) - # Default is True - self.assertEqual(expect, json.dumps(t)) - self.assertEqual(expect, json.dumps(t, tuple_as_array=True)) - self.assertRaises(TypeError, json.dumps, t, tuple_as_array=False) - # Ensure that the "default" does not get called - self.assertEqual(expect, json.dumps(t, default=repr)) - self.assertEqual(expect, json.dumps(t, tuple_as_array=True, - default=repr)) - # Ensure that the "default" gets called - self.assertEqual( - json.dumps(repr(t)), - json.dumps(t, tuple_as_array=False, default=repr)) - - def test_tuple_array_dump(self): - t = (1, 2, 3) - expect = json.dumps(list(t)) - # Default is True - sio = StringIO() - json.dump(t, sio) - self.assertEqual(expect, sio.getvalue()) - sio = StringIO() - json.dump(t, sio, tuple_as_array=True) - self.assertEqual(expect, sio.getvalue()) - self.assertRaises(TypeError, json.dump, t, StringIO(), - tuple_as_array=False) - # Ensure that the "default" does not get called - sio = StringIO() - json.dump(t, sio, default=repr) - self.assertEqual(expect, sio.getvalue()) - sio = StringIO() - json.dump(t, sio, tuple_as_array=True, default=repr) - self.assertEqual(expect, sio.getvalue()) - # Ensure that the "default" gets called - sio = StringIO() - json.dump(t, sio, tuple_as_array=False, default=repr) - self.assertEqual( - json.dumps(repr(t)), - sio.getvalue()) - -class TestNamedTuple(unittest.TestCase): - def test_namedtuple_dump(self): - pass diff --git a/pyload/lib/simplejson/tests/test_unicode.py b/pyload/lib/simplejson/tests/test_unicode.py deleted file mode 100644 index 3b37f6599..000000000 --- a/pyload/lib/simplejson/tests/test_unicode.py +++ /dev/null @@ -1,153 +0,0 @@ -import sys -import codecs -from unittest import TestCase - -import simplejson as json -from simplejson.compat import unichr, text_type, b, u, BytesIO - -class TestUnicode(TestCase): - def test_encoding1(self): - encoder = json.JSONEncoder(encoding='utf-8') - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - s = u.encode('utf-8') - ju = encoder.encode(u) - js = encoder.encode(s) - self.assertEqual(ju, js) - - def test_encoding2(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - s = u.encode('utf-8') - ju = json.dumps(u, encoding='utf-8') - js = json.dumps(s, encoding='utf-8') - self.assertEqual(ju, js) - - def test_encoding3(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps(u) - self.assertEqual(j, '"\\u03b1\\u03a9"') - - def test_encoding4(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps([u]) - self.assertEqual(j, '["\\u03b1\\u03a9"]') - - def test_encoding5(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps(u, ensure_ascii=False) - self.assertEqual(j, u'"' + u + u'"') - - def test_encoding6(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps([u], ensure_ascii=False) - self.assertEqual(j, u'["' + u + u'"]') - - def test_big_unicode_encode(self): - u = u'\U0001d120' - self.assertEqual(json.dumps(u), '"\\ud834\\udd20"') - self.assertEqual(json.dumps(u, ensure_ascii=False), u'"\U0001d120"') - - def test_big_unicode_decode(self): - u = u'z\U0001d120x' - self.assertEqual(json.loads('"' + u + '"'), u) - self.assertEqual(json.loads('"z\\ud834\\udd20x"'), u) - - def test_unicode_decode(self): - for i in range(0, 0xd7ff): - u = unichr(i) - #s = '"\\u{0:04x}"'.format(i) - s = '"\\u%04x"' % (i,) - self.assertEqual(json.loads(s), u) - - def test_object_pairs_hook_with_unicode(self): - s = u'{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' - p = [(u"xkd", 1), (u"kcw", 2), (u"art", 3), (u"hxm", 4), - (u"qrt", 5), (u"pad", 6), (u"hoy", 7)] - self.assertEqual(json.loads(s), eval(s)) - self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p) - od = json.loads(s, object_pairs_hook=json.OrderedDict) - self.assertEqual(od, json.OrderedDict(p)) - self.assertEqual(type(od), json.OrderedDict) - # the object_pairs_hook takes priority over the object_hook - self.assertEqual(json.loads(s, - object_pairs_hook=json.OrderedDict, - object_hook=lambda x: None), - json.OrderedDict(p)) - - - def test_default_encoding(self): - self.assertEqual(json.loads(u'{"a": "\xe9"}'.encode('utf-8')), - {'a': u'\xe9'}) - - def test_unicode_preservation(self): - self.assertEqual(type(json.loads(u'""')), text_type) - self.assertEqual(type(json.loads(u'"a"')), text_type) - self.assertEqual(type(json.loads(u'["a"]')[0]), text_type) - - def test_ensure_ascii_false_returns_unicode(self): - # http://code.google.com/p/simplejson/issues/detail?id=48 - self.assertEqual(type(json.dumps([], ensure_ascii=False)), text_type) - self.assertEqual(type(json.dumps(0, ensure_ascii=False)), text_type) - self.assertEqual(type(json.dumps({}, ensure_ascii=False)), text_type) - self.assertEqual(type(json.dumps("", ensure_ascii=False)), text_type) - - def test_ensure_ascii_false_bytestring_encoding(self): - # http://code.google.com/p/simplejson/issues/detail?id=48 - doc1 = {u'quux': b('Arr\xc3\xaat sur images')} - doc2 = {u'quux': u('Arr\xeat sur images')} - doc_ascii = '{"quux": "Arr\\u00eat sur images"}' - doc_unicode = u'{"quux": "Arr\xeat sur images"}' - self.assertEqual(json.dumps(doc1), doc_ascii) - self.assertEqual(json.dumps(doc2), doc_ascii) - self.assertEqual(json.dumps(doc1, ensure_ascii=False), doc_unicode) - self.assertEqual(json.dumps(doc2, ensure_ascii=False), doc_unicode) - - def test_ensure_ascii_linebreak_encoding(self): - # http://timelessrepo.com/json-isnt-a-javascript-subset - s1 = u'\u2029\u2028' - s2 = s1.encode('utf8') - expect = '"\\u2029\\u2028"' - self.assertEqual(json.dumps(s1), expect) - self.assertEqual(json.dumps(s2), expect) - self.assertEqual(json.dumps(s1, ensure_ascii=False), expect) - self.assertEqual(json.dumps(s2, ensure_ascii=False), expect) - - def test_invalid_escape_sequences(self): - # incomplete escape sequence - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u1') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u12') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u123') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u1234') - # invalid escape sequence - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u123x"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u12x4"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\u1x34"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ux234"') - if sys.maxunicode > 65535: - # invalid escape sequence for low surrogate - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u0"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u00"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u000"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u000x"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u00x0"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u0x00"') - self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\ux000"') - - def test_ensure_ascii_still_works(self): - # in the ascii range, ensure that everything is the same - for c in map(unichr, range(0, 127)): - self.assertEqual( - json.dumps(c, ensure_ascii=False), - json.dumps(c)) - snowman = u'\N{SNOWMAN}' - self.assertEqual( - json.dumps(c, ensure_ascii=False), - '"' + c + '"') - - def test_strip_bom(self): - content = u"\u3053\u3093\u306b\u3061\u308f" - json_doc = codecs.BOM_UTF8 + b(json.dumps(content)) - self.assertEqual(json.load(BytesIO(json_doc)), content) - for doc in json_doc, json_doc.decode('utf8'): - self.assertEqual(json.loads(doc), content) diff --git a/pyload/lib/simplejson/tool.py b/pyload/lib/simplejson/tool.py deleted file mode 100644 index 062e8e2c1..000000000 --- a/pyload/lib/simplejson/tool.py +++ /dev/null @@ -1,42 +0,0 @@ -r"""Command-line tool to validate and pretty-print JSON - -Usage:: - - $ echo '{"json":"obj"}' | python -m simplejson.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m simplejson.tool - Expecting property name: line 1 column 2 (char 2) - -""" -from __future__ import with_statement -import sys -import simplejson as json - -def main(): - if len(sys.argv) == 1: - infile = sys.stdin - outfile = sys.stdout - elif len(sys.argv) == 2: - infile = open(sys.argv[1], 'r') - outfile = sys.stdout - elif len(sys.argv) == 3: - infile = open(sys.argv[1], 'r') - outfile = open(sys.argv[2], 'w') - else: - raise SystemExit(sys.argv[0] + " [infile [outfile]]") - with infile: - try: - obj = json.load(infile, - object_pairs_hook=json.OrderedDict, - use_decimal=True) - except ValueError: - raise SystemExit(sys.exc_info()[1]) - with outfile: - json.dump(obj, outfile, sort_keys=True, indent=' ', use_decimal=True) - outfile.write('\n') - - -if __name__ == '__main__': - main() diff --git a/pyload/lib/thrift/TSCons.py b/pyload/lib/thrift/TSCons.py deleted file mode 100644 index da8d2833b..000000000 --- a/pyload/lib/thrift/TSCons.py +++ /dev/null @@ -1,35 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -from os import path -from SCons.Builder import Builder - - -def scons_env(env, add=''): - opath = path.dirname(path.abspath('$TARGET')) - lstr = 'thrift --gen cpp -o ' + opath + ' ' + add + ' $SOURCE' - cppbuild = Builder(action=lstr) - env.Append(BUILDERS={'ThriftCpp': cppbuild}) - - -def gen_cpp(env, dir, file): - scons_env(env) - suffixes = ['_types.h', '_types.cpp'] - targets = map(lambda s: 'gen-cpp/' + file + s, suffixes) - return env.ThriftCpp(targets, dir + file + '.thrift') diff --git a/pyload/lib/thrift/TSerialization.py b/pyload/lib/thrift/TSerialization.py deleted file mode 100644 index 8a58d89df..000000000 --- a/pyload/lib/thrift/TSerialization.py +++ /dev/null @@ -1,38 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -from protocol import TBinaryProtocol -from transport import TTransport - - -def serialize(thrift_object, - protocol_factory=TBinaryProtocol.TBinaryProtocolFactory()): - transport = TTransport.TMemoryBuffer() - protocol = protocol_factory.getProtocol(transport) - thrift_object.write(protocol) - return transport.getvalue() - - -def deserialize(base, - buf, - protocol_factory=TBinaryProtocol.TBinaryProtocolFactory()): - transport = TTransport.TMemoryBuffer(buf) - protocol = protocol_factory.getProtocol(transport) - base.read(protocol) - return base diff --git a/pyload/lib/thrift/TTornado.py b/pyload/lib/thrift/TTornado.py deleted file mode 100644 index af309c3d9..000000000 --- a/pyload/lib/thrift/TTornado.py +++ /dev/null @@ -1,153 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -from cStringIO import StringIO -import logging -import socket -import struct - -from thrift.transport import TTransport -from thrift.transport.TTransport import TTransportException - -from tornado import gen -from tornado import iostream -from tornado import netutil - - -class TTornadoStreamTransport(TTransport.TTransportBase): - """a framed, buffered transport over a Tornado stream""" - def __init__(self, host, port, stream=None): - self.host = host - self.port = port - self.is_queuing_reads = False - self.read_queue = [] - self.__wbuf = StringIO() - - # servers provide a ready-to-go stream - self.stream = stream - if self.stream is not None: - self._set_close_callback() - - # not the same number of parameters as TTransportBase.open - def open(self, callback): - logging.debug('socket connecting') - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) - self.stream = iostream.IOStream(sock) - - def on_close_in_connect(*_): - message = 'could not connect to {}:{}'.format(self.host, self.port) - raise TTransportException( - type=TTransportException.NOT_OPEN, - message=message) - self.stream.set_close_callback(on_close_in_connect) - - def finish(*_): - self._set_close_callback() - callback() - - self.stream.connect((self.host, self.port), callback=finish) - - def _set_close_callback(self): - def on_close(): - raise TTransportException( - type=TTransportException.END_OF_FILE, - message='socket closed') - self.stream.set_close_callback(self.close) - - def close(self): - # don't raise if we intend to close - self.stream.set_close_callback(None) - self.stream.close() - - def read(self, _): - # The generated code for Tornado shouldn't do individual reads -- only - # frames at a time - assert "you're doing it wrong" is True - - @gen.engine - def readFrame(self, callback): - self.read_queue.append(callback) - logging.debug('read queue: %s', self.read_queue) - - if self.is_queuing_reads: - # If a read is already in flight, then the while loop below should - # pull it from self.read_queue - return - - self.is_queuing_reads = True - while self.read_queue: - next_callback = self.read_queue.pop() - result = yield gen.Task(self._readFrameFromStream) - next_callback(result) - self.is_queuing_reads = False - - @gen.engine - def _readFrameFromStream(self, callback): - logging.debug('_readFrameFromStream') - frame_header = yield gen.Task(self.stream.read_bytes, 4) - frame_length, = struct.unpack('!i', frame_header) - logging.debug('received frame header, frame length = %i', frame_length) - frame = yield gen.Task(self.stream.read_bytes, frame_length) - logging.debug('received frame payload') - callback(frame) - - def write(self, buf): - self.__wbuf.write(buf) - - def flush(self, callback=None): - wout = self.__wbuf.getvalue() - wsz = len(wout) - # reset wbuf before write/flush to preserve state on underlying failure - self.__wbuf = StringIO() - # N.B.: Doing this string concatenation is WAY cheaper than making - # two separate calls to the underlying socket object. Socket writes in - # Python turn out to be REALLY expensive, but it seems to do a pretty - # good job of managing string buffer operations without excessive copies - buf = struct.pack("!i", wsz) + wout - - logging.debug('writing frame length = %i', wsz) - self.stream.write(buf, callback) - - -class TTornadoServer(netutil.TCPServer): - def __init__(self, processor, iprot_factory, oprot_factory=None, - *args, **kwargs): - super(TTornadoServer, self).__init__(*args, **kwargs) - - self._processor = processor - self._iprot_factory = iprot_factory - self._oprot_factory = (oprot_factory if oprot_factory is not None - else iprot_factory) - - def handle_stream(self, stream, address): - try: - host, port = address - trans = TTornadoStreamTransport(host=host, port=port, stream=stream) - oprot = self._oprot_factory.getProtocol(trans) - - def next_pass(): - if not trans.stream.closed(): - self._processor.process(trans, self._iprot_factory, oprot, - callback=next_pass) - - next_pass() - - except Exception: - logging.exception('thrift exception in handle_stream') - trans.close() diff --git a/pyload/lib/thrift/Thrift.py b/pyload/lib/thrift/Thrift.py deleted file mode 100644 index 9890af7e1..000000000 --- a/pyload/lib/thrift/Thrift.py +++ /dev/null @@ -1,170 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -import sys - - -class TType: - STOP = 0 - VOID = 1 - BOOL = 2 - BYTE = 3 - I08 = 3 - DOUBLE = 4 - I16 = 6 - I32 = 8 - I64 = 10 - STRING = 11 - UTF7 = 11 - STRUCT = 12 - MAP = 13 - SET = 14 - LIST = 15 - UTF8 = 16 - UTF16 = 17 - - _VALUES_TO_NAMES = ('STOP', - 'VOID', - 'BOOL', - 'BYTE', - 'DOUBLE', - None, - 'I16', - None, - 'I32', - None, - 'I64', - 'STRING', - 'STRUCT', - 'MAP', - 'SET', - 'LIST', - 'UTF8', - 'UTF16') - - -class TMessageType: - CALL = 1 - REPLY = 2 - EXCEPTION = 3 - ONEWAY = 4 - - -class TProcessor: - """Base class for procsessor, which works on two streams.""" - - def process(iprot, oprot): - pass - - -class TException(Exception): - """Base class for all thrift exceptions.""" - - # BaseException.message is deprecated in Python v[2.6,3.0) - if (2, 6, 0) <= sys.version_info < (3, 0): - def _get_message(self): - return self._message - - def _set_message(self, message): - self._message = message - message = property(_get_message, _set_message) - - def __init__(self, message=None): - Exception.__init__(self, message) - self.message = message - - -class TApplicationException(TException): - """Application level thrift exceptions.""" - - UNKNOWN = 0 - UNKNOWN_METHOD = 1 - INVALID_MESSAGE_TYPE = 2 - WRONG_METHOD_NAME = 3 - BAD_SEQUENCE_ID = 4 - MISSING_RESULT = 5 - INTERNAL_ERROR = 6 - PROTOCOL_ERROR = 7 - INVALID_TRANSFORM = 8 - INVALID_PROTOCOL = 9 - UNSUPPORTED_CLIENT_TYPE = 10 - - def __init__(self, type=UNKNOWN, message=None): - TException.__init__(self, message) - self.type = type - - def __str__(self): - if self.message: - return self.message - elif self.type == self.UNKNOWN_METHOD: - return 'Unknown method' - elif self.type == self.INVALID_MESSAGE_TYPE: - return 'Invalid message type' - elif self.type == self.WRONG_METHOD_NAME: - return 'Wrong method name' - elif self.type == self.BAD_SEQUENCE_ID: - return 'Bad sequence ID' - elif self.type == self.MISSING_RESULT: - return 'Missing result' - elif self.type == self.INTERNAL_ERROR: - return 'Internal error' - elif self.type == self.PROTOCOL_ERROR: - return 'Protocol error' - elif self.type == self.INVALID_TRANSFORM: - return 'Invalid transform' - elif self.type == self.INVALID_PROTOCOL: - return 'Invalid protocol' - elif self.type == self.UNSUPPORTED_CLIENT_TYPE: - return 'Unsupported client type' - else: - return 'Default (unknown) TApplicationException' - - def read(self, iprot): - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRING: - self.message = iprot.readString() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.I32: - self.type = iprot.readI32() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - oprot.writeStructBegin('TApplicationException') - if self.message is not None: - oprot.writeFieldBegin('message', TType.STRING, 1) - oprot.writeString(self.message) - oprot.writeFieldEnd() - if self.type is not None: - oprot.writeFieldBegin('type', TType.I32, 2) - oprot.writeI32(self.type) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() diff --git a/pyload/lib/thrift/__init__.py b/pyload/lib/thrift/__init__.py deleted file mode 100644 index 48d659c40..000000000 --- a/pyload/lib/thrift/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -__all__ = ['Thrift', 'TSCons'] diff --git a/pyload/lib/thrift/protocol/TBase.py b/pyload/lib/thrift/protocol/TBase.py deleted file mode 100644 index 6cbd5f39a..000000000 --- a/pyload/lib/thrift/protocol/TBase.py +++ /dev/null @@ -1,81 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -from thrift.Thrift import * -from thrift.protocol import TBinaryProtocol -from thrift.transport import TTransport - -try: - from thrift.protocol import fastbinary -except: - fastbinary = None - - -class TBase(object): - __slots__ = [] - - def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - for attr in self.__slots__: - my_val = getattr(self, attr) - other_val = getattr(other, attr) - if my_val != other_val: - return False - return True - - def __ne__(self, other): - return not (self == other) - - def read(self, iprot): - if (iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and - isinstance(iprot.trans, TTransport.CReadableTransport) and - self.thrift_spec is not None and - fastbinary is not None): - fastbinary.decode_binary(self, - iprot.trans, - (self.__class__, self.thrift_spec)) - return - iprot.readStruct(self, self.thrift_spec) - - def write(self, oprot): - if (oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and - self.thrift_spec is not None and - fastbinary is not None): - oprot.trans.write( - fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) - return - oprot.writeStruct(self, self.thrift_spec) - - -class TExceptionBase(Exception): - # old style class so python2.4 can raise exceptions derived from this - # This can't inherit from TBase because of that limitation. - __slots__ = [] - - __repr__ = TBase.__repr__.im_func - __eq__ = TBase.__eq__.im_func - __ne__ = TBase.__ne__.im_func - read = TBase.read.im_func - write = TBase.write.im_func diff --git a/pyload/lib/thrift/protocol/TBinaryProtocol.py b/pyload/lib/thrift/protocol/TBinaryProtocol.py deleted file mode 100644 index 6fdd08c26..000000000 --- a/pyload/lib/thrift/protocol/TBinaryProtocol.py +++ /dev/null @@ -1,260 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -from TProtocol import * -from struct import pack, unpack - - -class TBinaryProtocol(TProtocolBase): - """Binary implementation of the Thrift protocol driver.""" - - # NastyHaxx. Python 2.4+ on 32-bit machines forces hex constants to be - # positive, converting this into a long. If we hardcode the int value - # instead it'll stay in 32 bit-land. - - # VERSION_MASK = 0xffff0000 - VERSION_MASK = -65536 - - # VERSION_1 = 0x80010000 - VERSION_1 = -2147418112 - - TYPE_MASK = 0x000000ff - - def __init__(self, trans, strictRead=False, strictWrite=True): - TProtocolBase.__init__(self, trans) - self.strictRead = strictRead - self.strictWrite = strictWrite - - def writeMessageBegin(self, name, type, seqid): - if self.strictWrite: - self.writeI32(TBinaryProtocol.VERSION_1 | type) - self.writeString(name) - self.writeI32(seqid) - else: - self.writeString(name) - self.writeByte(type) - self.writeI32(seqid) - - def writeMessageEnd(self): - pass - - def writeStructBegin(self, name): - pass - - def writeStructEnd(self): - pass - - def writeFieldBegin(self, name, type, id): - self.writeByte(type) - self.writeI16(id) - - def writeFieldEnd(self): - pass - - def writeFieldStop(self): - self.writeByte(TType.STOP) - - def writeMapBegin(self, ktype, vtype, size): - self.writeByte(ktype) - self.writeByte(vtype) - self.writeI32(size) - - def writeMapEnd(self): - pass - - def writeListBegin(self, etype, size): - self.writeByte(etype) - self.writeI32(size) - - def writeListEnd(self): - pass - - def writeSetBegin(self, etype, size): - self.writeByte(etype) - self.writeI32(size) - - def writeSetEnd(self): - pass - - def writeBool(self, bool): - if bool: - self.writeByte(1) - else: - self.writeByte(0) - - def writeByte(self, byte): - buff = pack("!b", byte) - self.trans.write(buff) - - def writeI16(self, i16): - buff = pack("!h", i16) - self.trans.write(buff) - - def writeI32(self, i32): - buff = pack("!i", i32) - self.trans.write(buff) - - def writeI64(self, i64): - buff = pack("!q", i64) - self.trans.write(buff) - - def writeDouble(self, dub): - buff = pack("!d", dub) - self.trans.write(buff) - - def writeString(self, str): - self.writeI32(len(str)) - self.trans.write(str) - - def readMessageBegin(self): - sz = self.readI32() - if sz < 0: - version = sz & TBinaryProtocol.VERSION_MASK - if version != TBinaryProtocol.VERSION_1: - raise TProtocolException( - type=TProtocolException.BAD_VERSION, - message='Bad version in readMessageBegin: %d' % (sz)) - type = sz & TBinaryProtocol.TYPE_MASK - name = self.readString() - seqid = self.readI32() - else: - if self.strictRead: - raise TProtocolException(type=TProtocolException.BAD_VERSION, - message='No protocol version header') - name = self.trans.readAll(sz) - type = self.readByte() - seqid = self.readI32() - return (name, type, seqid) - - def readMessageEnd(self): - pass - - def readStructBegin(self): - pass - - def readStructEnd(self): - pass - - def readFieldBegin(self): - type = self.readByte() - if type == TType.STOP: - return (None, type, 0) - id = self.readI16() - return (None, type, id) - - def readFieldEnd(self): - pass - - def readMapBegin(self): - ktype = self.readByte() - vtype = self.readByte() - size = self.readI32() - return (ktype, vtype, size) - - def readMapEnd(self): - pass - - def readListBegin(self): - etype = self.readByte() - size = self.readI32() - return (etype, size) - - def readListEnd(self): - pass - - def readSetBegin(self): - etype = self.readByte() - size = self.readI32() - return (etype, size) - - def readSetEnd(self): - pass - - def readBool(self): - byte = self.readByte() - if byte == 0: - return False - return True - - def readByte(self): - buff = self.trans.readAll(1) - val, = unpack('!b', buff) - return val - - def readI16(self): - buff = self.trans.readAll(2) - val, = unpack('!h', buff) - return val - - def readI32(self): - buff = self.trans.readAll(4) - val, = unpack('!i', buff) - return val - - def readI64(self): - buff = self.trans.readAll(8) - val, = unpack('!q', buff) - return val - - def readDouble(self): - buff = self.trans.readAll(8) - val, = unpack('!d', buff) - return val - - def readString(self): - len = self.readI32() - str = self.trans.readAll(len) - return str - - -class TBinaryProtocolFactory: - def __init__(self, strictRead=False, strictWrite=True): - self.strictRead = strictRead - self.strictWrite = strictWrite - - def getProtocol(self, trans): - prot = TBinaryProtocol(trans, self.strictRead, self.strictWrite) - return prot - - -class TBinaryProtocolAccelerated(TBinaryProtocol): - """C-Accelerated version of TBinaryProtocol. - - This class does not override any of TBinaryProtocol's methods, - but the generated code recognizes it directly and will call into - our C module to do the encoding, bypassing this object entirely. - We inherit from TBinaryProtocol so that the normal TBinaryProtocol - encoding can happen if the fastbinary module doesn't work for some - reason. (TODO(dreiss): Make this happen sanely in more cases.) - - In order to take advantage of the C module, just use - TBinaryProtocolAccelerated instead of TBinaryProtocol. - - NOTE: This code was contributed by an external developer. - The internal Thrift team has reviewed and tested it, - but we cannot guarantee that it is production-ready. - Please feel free to report bugs and/or success stories - to the public mailing list. - """ - pass - - -class TBinaryProtocolAcceleratedFactory: - def getProtocol(self, trans): - return TBinaryProtocolAccelerated(trans) diff --git a/pyload/lib/thrift/protocol/TCompactProtocol.py b/pyload/lib/thrift/protocol/TCompactProtocol.py deleted file mode 100644 index cdec60773..000000000 --- a/pyload/lib/thrift/protocol/TCompactProtocol.py +++ /dev/null @@ -1,403 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -from TProtocol import * -from struct import pack, unpack - -__all__ = ['TCompactProtocol', 'TCompactProtocolFactory'] - -CLEAR = 0 -FIELD_WRITE = 1 -VALUE_WRITE = 2 -CONTAINER_WRITE = 3 -BOOL_WRITE = 4 -FIELD_READ = 5 -CONTAINER_READ = 6 -VALUE_READ = 7 -BOOL_READ = 8 - - -def make_helper(v_from, container): - def helper(func): - def nested(self, *args, **kwargs): - assert self.state in (v_from, container), (self.state, v_from, container) - return func(self, *args, **kwargs) - return nested - return helper -writer = make_helper(VALUE_WRITE, CONTAINER_WRITE) -reader = make_helper(VALUE_READ, CONTAINER_READ) - - -def makeZigZag(n, bits): - return (n << 1) ^ (n >> (bits - 1)) - - -def fromZigZag(n): - return (n >> 1) ^ -(n & 1) - - -def writeVarint(trans, n): - out = [] - while True: - if n & ~0x7f == 0: - out.append(n) - break - else: - out.append((n & 0xff) | 0x80) - n = n >> 7 - trans.write(''.join(map(chr, out))) - - -def readVarint(trans): - result = 0 - shift = 0 - while True: - x = trans.readAll(1) - byte = ord(x) - result |= (byte & 0x7f) << shift - if byte >> 7 == 0: - return result - shift += 7 - - -class CompactType: - STOP = 0x00 - TRUE = 0x01 - FALSE = 0x02 - BYTE = 0x03 - I16 = 0x04 - I32 = 0x05 - I64 = 0x06 - DOUBLE = 0x07 - BINARY = 0x08 - LIST = 0x09 - SET = 0x0A - MAP = 0x0B - STRUCT = 0x0C - -CTYPES = {TType.STOP: CompactType.STOP, - TType.BOOL: CompactType.TRUE, # used for collection - TType.BYTE: CompactType.BYTE, - TType.I16: CompactType.I16, - TType.I32: CompactType.I32, - TType.I64: CompactType.I64, - TType.DOUBLE: CompactType.DOUBLE, - TType.STRING: CompactType.BINARY, - TType.STRUCT: CompactType.STRUCT, - TType.LIST: CompactType.LIST, - TType.SET: CompactType.SET, - TType.MAP: CompactType.MAP - } - -TTYPES = {} -for k, v in CTYPES.items(): - TTYPES[v] = k -TTYPES[CompactType.FALSE] = TType.BOOL -del k -del v - - -class TCompactProtocol(TProtocolBase): - """Compact implementation of the Thrift protocol driver.""" - - PROTOCOL_ID = 0x82 - VERSION = 1 - VERSION_MASK = 0x1f - TYPE_MASK = 0xe0 - TYPE_SHIFT_AMOUNT = 5 - - def __init__(self, trans): - TProtocolBase.__init__(self, trans) - self.state = CLEAR - self.__last_fid = 0 - self.__bool_fid = None - self.__bool_value = None - self.__structs = [] - self.__containers = [] - - def __writeVarint(self, n): - writeVarint(self.trans, n) - - def writeMessageBegin(self, name, type, seqid): - assert self.state == CLEAR - self.__writeUByte(self.PROTOCOL_ID) - self.__writeUByte(self.VERSION | (type << self.TYPE_SHIFT_AMOUNT)) - self.__writeVarint(seqid) - self.__writeString(name) - self.state = VALUE_WRITE - - def writeMessageEnd(self): - assert self.state == VALUE_WRITE - self.state = CLEAR - - def writeStructBegin(self, name): - assert self.state in (CLEAR, CONTAINER_WRITE, VALUE_WRITE), self.state - self.__structs.append((self.state, self.__last_fid)) - self.state = FIELD_WRITE - self.__last_fid = 0 - - def writeStructEnd(self): - assert self.state == FIELD_WRITE - self.state, self.__last_fid = self.__structs.pop() - - def writeFieldStop(self): - self.__writeByte(0) - - def __writeFieldHeader(self, type, fid): - delta = fid - self.__last_fid - if 0 < delta <= 15: - self.__writeUByte(delta << 4 | type) - else: - self.__writeByte(type) - self.__writeI16(fid) - self.__last_fid = fid - - def writeFieldBegin(self, name, type, fid): - assert self.state == FIELD_WRITE, self.state - if type == TType.BOOL: - self.state = BOOL_WRITE - self.__bool_fid = fid - else: - self.state = VALUE_WRITE - self.__writeFieldHeader(CTYPES[type], fid) - - def writeFieldEnd(self): - assert self.state in (VALUE_WRITE, BOOL_WRITE), self.state - self.state = FIELD_WRITE - - def __writeUByte(self, byte): - self.trans.write(pack('!B', byte)) - - def __writeByte(self, byte): - self.trans.write(pack('!b', byte)) - - def __writeI16(self, i16): - self.__writeVarint(makeZigZag(i16, 16)) - - def __writeSize(self, i32): - self.__writeVarint(i32) - - def writeCollectionBegin(self, etype, size): - assert self.state in (VALUE_WRITE, CONTAINER_WRITE), self.state - if size <= 14: - self.__writeUByte(size << 4 | CTYPES[etype]) - else: - self.__writeUByte(0xf0 | CTYPES[etype]) - self.__writeSize(size) - self.__containers.append(self.state) - self.state = CONTAINER_WRITE - writeSetBegin = writeCollectionBegin - writeListBegin = writeCollectionBegin - - def writeMapBegin(self, ktype, vtype, size): - assert self.state in (VALUE_WRITE, CONTAINER_WRITE), self.state - if size == 0: - self.__writeByte(0) - else: - self.__writeSize(size) - self.__writeUByte(CTYPES[ktype] << 4 | CTYPES[vtype]) - self.__containers.append(self.state) - self.state = CONTAINER_WRITE - - def writeCollectionEnd(self): - assert self.state == CONTAINER_WRITE, self.state - self.state = self.__containers.pop() - writeMapEnd = writeCollectionEnd - writeSetEnd = writeCollectionEnd - writeListEnd = writeCollectionEnd - - def writeBool(self, bool): - if self.state == BOOL_WRITE: - if bool: - ctype = CompactType.TRUE - else: - ctype = CompactType.FALSE - self.__writeFieldHeader(ctype, self.__bool_fid) - elif self.state == CONTAINER_WRITE: - if bool: - self.__writeByte(CompactType.TRUE) - else: - self.__writeByte(CompactType.FALSE) - else: - raise AssertionError("Invalid state in compact protocol") - - writeByte = writer(__writeByte) - writeI16 = writer(__writeI16) - - @writer - def writeI32(self, i32): - self.__writeVarint(makeZigZag(i32, 32)) - - @writer - def writeI64(self, i64): - self.__writeVarint(makeZigZag(i64, 64)) - - @writer - def writeDouble(self, dub): - self.trans.write(pack('!d', dub)) - - def __writeString(self, s): - self.__writeSize(len(s)) - self.trans.write(s) - writeString = writer(__writeString) - - def readFieldBegin(self): - assert self.state == FIELD_READ, self.state - type = self.__readUByte() - if type & 0x0f == TType.STOP: - return (None, 0, 0) - delta = type >> 4 - if delta == 0: - fid = self.__readI16() - else: - fid = self.__last_fid + delta - self.__last_fid = fid - type = type & 0x0f - if type == CompactType.TRUE: - self.state = BOOL_READ - self.__bool_value = True - elif type == CompactType.FALSE: - self.state = BOOL_READ - self.__bool_value = False - else: - self.state = VALUE_READ - return (None, self.__getTType(type), fid) - - def readFieldEnd(self): - assert self.state in (VALUE_READ, BOOL_READ), self.state - self.state = FIELD_READ - - def __readUByte(self): - result, = unpack('!B', self.trans.readAll(1)) - return result - - def __readByte(self): - result, = unpack('!b', self.trans.readAll(1)) - return result - - def __readVarint(self): - return readVarint(self.trans) - - def __readZigZag(self): - return fromZigZag(self.__readVarint()) - - def __readSize(self): - result = self.__readVarint() - if result < 0: - raise TException("Length < 0") - return result - - def readMessageBegin(self): - assert self.state == CLEAR - proto_id = self.__readUByte() - if proto_id != self.PROTOCOL_ID: - raise TProtocolException(TProtocolException.BAD_VERSION, - 'Bad protocol id in the message: %d' % proto_id) - ver_type = self.__readUByte() - type = (ver_type & self.TYPE_MASK) >> self.TYPE_SHIFT_AMOUNT - version = ver_type & self.VERSION_MASK - if version != self.VERSION: - raise TProtocolException(TProtocolException.BAD_VERSION, - 'Bad version: %d (expect %d)' % (version, self.VERSION)) - seqid = self.__readVarint() - name = self.__readString() - return (name, type, seqid) - - def readMessageEnd(self): - assert self.state == CLEAR - assert len(self.__structs) == 0 - - def readStructBegin(self): - assert self.state in (CLEAR, CONTAINER_READ, VALUE_READ), self.state - self.__structs.append((self.state, self.__last_fid)) - self.state = FIELD_READ - self.__last_fid = 0 - - def readStructEnd(self): - assert self.state == FIELD_READ - self.state, self.__last_fid = self.__structs.pop() - - def readCollectionBegin(self): - assert self.state in (VALUE_READ, CONTAINER_READ), self.state - size_type = self.__readUByte() - size = size_type >> 4 - type = self.__getTType(size_type) - if size == 15: - size = self.__readSize() - self.__containers.append(self.state) - self.state = CONTAINER_READ - return type, size - readSetBegin = readCollectionBegin - readListBegin = readCollectionBegin - - def readMapBegin(self): - assert self.state in (VALUE_READ, CONTAINER_READ), self.state - size = self.__readSize() - types = 0 - if size > 0: - types = self.__readUByte() - vtype = self.__getTType(types) - ktype = self.__getTType(types >> 4) - self.__containers.append(self.state) - self.state = CONTAINER_READ - return (ktype, vtype, size) - - def readCollectionEnd(self): - assert self.state == CONTAINER_READ, self.state - self.state = self.__containers.pop() - readSetEnd = readCollectionEnd - readListEnd = readCollectionEnd - readMapEnd = readCollectionEnd - - def readBool(self): - if self.state == BOOL_READ: - return self.__bool_value == CompactType.TRUE - elif self.state == CONTAINER_READ: - return self.__readByte() == CompactType.TRUE - else: - raise AssertionError("Invalid state in compact protocol: %d" % - self.state) - - readByte = reader(__readByte) - __readI16 = __readZigZag - readI16 = reader(__readZigZag) - readI32 = reader(__readZigZag) - readI64 = reader(__readZigZag) - - @reader - def readDouble(self): - buff = self.trans.readAll(8) - val, = unpack('!d', buff) - return val - - def __readString(self): - len = self.__readSize() - return self.trans.readAll(len) - readString = reader(__readString) - - def __getTType(self, byte): - return TTYPES[byte & 0x0f] - - -class TCompactProtocolFactory: - def __init__(self): - pass - - def getProtocol(self, trans): - return TCompactProtocol(trans) diff --git a/pyload/lib/thrift/protocol/TJSONProtocol.py b/pyload/lib/thrift/protocol/TJSONProtocol.py deleted file mode 100644 index 3048197d4..000000000 --- a/pyload/lib/thrift/protocol/TJSONProtocol.py +++ /dev/null @@ -1,550 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -from TProtocol import TType, TProtocolBase, TProtocolException -import base64 -import json -import math - -__all__ = ['TJSONProtocol', - 'TJSONProtocolFactory', - 'TSimpleJSONProtocol', - 'TSimpleJSONProtocolFactory'] - -VERSION = 1 - -COMMA = ',' -COLON = ':' -LBRACE = '{' -RBRACE = '}' -LBRACKET = '[' -RBRACKET = ']' -QUOTE = '"' -BACKSLASH = '\\' -ZERO = '0' - -ESCSEQ = '\\u00' -ESCAPE_CHAR = '"\\bfnrt' -ESCAPE_CHAR_VALS = ['"', '\\', '\b', '\f', '\n', '\r', '\t'] -NUMERIC_CHAR = '+-.0123456789Ee' - -CTYPES = {TType.BOOL: 'tf', - TType.BYTE: 'i8', - TType.I16: 'i16', - TType.I32: 'i32', - TType.I64: 'i64', - TType.DOUBLE: 'dbl', - TType.STRING: 'str', - TType.STRUCT: 'rec', - TType.LIST: 'lst', - TType.SET: 'set', - TType.MAP: 'map'} - -JTYPES = {} -for key in CTYPES.keys(): - JTYPES[CTYPES[key]] = key - - -class JSONBaseContext(object): - - def __init__(self, protocol): - self.protocol = protocol - self.first = True - - def doIO(self, function): - pass - - def write(self): - pass - - def read(self): - pass - - def escapeNum(self): - return False - - def __str__(self): - return self.__class__.__name__ - - -class JSONListContext(JSONBaseContext): - - def doIO(self, function): - if self.first is True: - self.first = False - else: - function(COMMA) - - def write(self): - self.doIO(self.protocol.trans.write) - - def read(self): - self.doIO(self.protocol.readJSONSyntaxChar) - - -class JSONPairContext(JSONBaseContext): - - def __init__(self, protocol): - super(JSONPairContext, self).__init__(protocol) - self.colon = True - - def doIO(self, function): - if self.first: - self.first = False - self.colon = True - else: - function(COLON if self.colon else COMMA) - self.colon = not self.colon - - def write(self): - self.doIO(self.protocol.trans.write) - - def read(self): - self.doIO(self.protocol.readJSONSyntaxChar) - - def escapeNum(self): - return self.colon - - def __str__(self): - return '%s, colon=%s' % (self.__class__.__name__, self.colon) - - -class LookaheadReader(): - hasData = False - data = '' - - def __init__(self, protocol): - self.protocol = protocol - - def read(self): - if self.hasData is True: - self.hasData = False - else: - self.data = self.protocol.trans.read(1) - return self.data - - def peek(self): - if self.hasData is False: - self.data = self.protocol.trans.read(1) - self.hasData = True - return self.data - -class TJSONProtocolBase(TProtocolBase): - - def __init__(self, trans): - TProtocolBase.__init__(self, trans) - self.resetWriteContext() - self.resetReadContext() - - def resetWriteContext(self): - self.context = JSONBaseContext(self) - self.contextStack = [self.context] - - def resetReadContext(self): - self.resetWriteContext() - self.reader = LookaheadReader(self) - - def pushContext(self, ctx): - self.contextStack.append(ctx) - self.context = ctx - - def popContext(self): - self.contextStack.pop() - if self.contextStack: - self.context = self.contextStack[-1] - else: - self.context = JSONBaseContext(self) - - def writeJSONString(self, string): - self.context.write() - self.trans.write(json.dumps(string)) - - def writeJSONNumber(self, number): - self.context.write() - jsNumber = str(number) - if self.context.escapeNum(): - jsNumber = "%s%s%s" % (QUOTE, jsNumber, QUOTE) - self.trans.write(jsNumber) - - def writeJSONBase64(self, binary): - self.context.write() - self.trans.write(QUOTE) - self.trans.write(base64.b64encode(binary)) - self.trans.write(QUOTE) - - def writeJSONObjectStart(self): - self.context.write() - self.trans.write(LBRACE) - self.pushContext(JSONPairContext(self)) - - def writeJSONObjectEnd(self): - self.popContext() - self.trans.write(RBRACE) - - def writeJSONArrayStart(self): - self.context.write() - self.trans.write(LBRACKET) - self.pushContext(JSONListContext(self)) - - def writeJSONArrayEnd(self): - self.popContext() - self.trans.write(RBRACKET) - - def readJSONSyntaxChar(self, character): - current = self.reader.read() - if character != current: - raise TProtocolException(TProtocolException.INVALID_DATA, - "Unexpected character: %s" % current) - - def readJSONString(self, skipContext): - string = [] - if skipContext is False: - self.context.read() - self.readJSONSyntaxChar(QUOTE) - while True: - character = self.reader.read() - if character == QUOTE: - break - if character == ESCSEQ[0]: - character = self.reader.read() - if character == ESCSEQ[1]: - self.readJSONSyntaxChar(ZERO) - self.readJSONSyntaxChar(ZERO) - character = json.JSONDecoder().decode('"\u00%s"' % self.trans.read(2)) - else: - off = ESCAPE_CHAR.find(character) - if off == -1: - raise TProtocolException(TProtocolException.INVALID_DATA, - "Expected control char") - character = ESCAPE_CHAR_VALS[off] - string.append(character) - return ''.join(string) - - def isJSONNumeric(self, character): - return (True if NUMERIC_CHAR.find(character) != - 1 else False) - - def readJSONQuotes(self): - if (self.context.escapeNum()): - self.readJSONSyntaxChar(QUOTE) - - def readJSONNumericChars(self): - numeric = [] - while True: - character = self.reader.peek() - if self.isJSONNumeric(character) is False: - break - numeric.append(self.reader.read()) - return ''.join(numeric) - - def readJSONInteger(self): - self.context.read() - self.readJSONQuotes() - numeric = self.readJSONNumericChars() - self.readJSONQuotes() - try: - return int(numeric) - except ValueError: - raise TProtocolException(TProtocolException.INVALID_DATA, - "Bad data encounted in numeric data") - - def readJSONDouble(self): - self.context.read() - if self.reader.peek() == QUOTE: - string = self.readJSONString(True) - try: - double = float(string) - if (self.context.escapeNum is False and - not math.isinf(double) and - not math.isnan(double)): - raise TProtocolException(TProtocolException.INVALID_DATA, - "Numeric data unexpectedly quoted") - return double - except ValueError: - raise TProtocolException(TProtocolException.INVALID_DATA, - "Bad data encounted in numeric data") - else: - if self.context.escapeNum() is True: - self.readJSONSyntaxChar(QUOTE) - try: - return float(self.readJSONNumericChars()) - except ValueError: - raise TProtocolException(TProtocolException.INVALID_DATA, - "Bad data encounted in numeric data") - - def readJSONBase64(self): - string = self.readJSONString(False) - return base64.b64decode(string) - - def readJSONObjectStart(self): - self.context.read() - self.readJSONSyntaxChar(LBRACE) - self.pushContext(JSONPairContext(self)) - - def readJSONObjectEnd(self): - self.readJSONSyntaxChar(RBRACE) - self.popContext() - - def readJSONArrayStart(self): - self.context.read() - self.readJSONSyntaxChar(LBRACKET) - self.pushContext(JSONListContext(self)) - - def readJSONArrayEnd(self): - self.readJSONSyntaxChar(RBRACKET) - self.popContext() - - -class TJSONProtocol(TJSONProtocolBase): - - def readMessageBegin(self): - self.resetReadContext() - self.readJSONArrayStart() - if self.readJSONInteger() != VERSION: - raise TProtocolException(TProtocolException.BAD_VERSION, - "Message contained bad version.") - name = self.readJSONString(False) - typen = self.readJSONInteger() - seqid = self.readJSONInteger() - return (name, typen, seqid) - - def readMessageEnd(self): - self.readJSONArrayEnd() - - def readStructBegin(self): - self.readJSONObjectStart() - - def readStructEnd(self): - self.readJSONObjectEnd() - - def readFieldBegin(self): - character = self.reader.peek() - ttype = 0 - id = 0 - if character == RBRACE: - ttype = TType.STOP - else: - id = self.readJSONInteger() - self.readJSONObjectStart() - ttype = JTYPES[self.readJSONString(False)] - return (None, ttype, id) - - def readFieldEnd(self): - self.readJSONObjectEnd() - - def readMapBegin(self): - self.readJSONArrayStart() - keyType = JTYPES[self.readJSONString(False)] - valueType = JTYPES[self.readJSONString(False)] - size = self.readJSONInteger() - self.readJSONObjectStart() - return (keyType, valueType, size) - - def readMapEnd(self): - self.readJSONObjectEnd() - self.readJSONArrayEnd() - - def readCollectionBegin(self): - self.readJSONArrayStart() - elemType = JTYPES[self.readJSONString(False)] - size = self.readJSONInteger() - return (elemType, size) - readListBegin = readCollectionBegin - readSetBegin = readCollectionBegin - - def readCollectionEnd(self): - self.readJSONArrayEnd() - readSetEnd = readCollectionEnd - readListEnd = readCollectionEnd - - def readBool(self): - return (False if self.readJSONInteger() == 0 else True) - - def readNumber(self): - return self.readJSONInteger() - readByte = readNumber - readI16 = readNumber - readI32 = readNumber - readI64 = readNumber - - def readDouble(self): - return self.readJSONDouble() - - def readString(self): - return self.readJSONString(False) - - def readBinary(self): - return self.readJSONBase64() - - def writeMessageBegin(self, name, request_type, seqid): - self.resetWriteContext() - self.writeJSONArrayStart() - self.writeJSONNumber(VERSION) - self.writeJSONString(name) - self.writeJSONNumber(request_type) - self.writeJSONNumber(seqid) - - def writeMessageEnd(self): - self.writeJSONArrayEnd() - - def writeStructBegin(self, name): - self.writeJSONObjectStart() - - def writeStructEnd(self): - self.writeJSONObjectEnd() - - def writeFieldBegin(self, name, ttype, id): - self.writeJSONNumber(id) - self.writeJSONObjectStart() - self.writeJSONString(CTYPES[ttype]) - - def writeFieldEnd(self): - self.writeJSONObjectEnd() - - def writeFieldStop(self): - pass - - def writeMapBegin(self, ktype, vtype, size): - self.writeJSONArrayStart() - self.writeJSONString(CTYPES[ktype]) - self.writeJSONString(CTYPES[vtype]) - self.writeJSONNumber(size) - self.writeJSONObjectStart() - - def writeMapEnd(self): - self.writeJSONObjectEnd() - self.writeJSONArrayEnd() - - def writeListBegin(self, etype, size): - self.writeJSONArrayStart() - self.writeJSONString(CTYPES[etype]) - self.writeJSONNumber(size) - - def writeListEnd(self): - self.writeJSONArrayEnd() - - def writeSetBegin(self, etype, size): - self.writeJSONArrayStart() - self.writeJSONString(CTYPES[etype]) - self.writeJSONNumber(size) - - def writeSetEnd(self): - self.writeJSONArrayEnd() - - def writeBool(self, boolean): - self.writeJSONNumber(1 if boolean is True else 0) - - def writeInteger(self, integer): - self.writeJSONNumber(integer) - writeByte = writeInteger - writeI16 = writeInteger - writeI32 = writeInteger - writeI64 = writeInteger - - def writeDouble(self, dbl): - self.writeJSONNumber(dbl) - - def writeString(self, string): - self.writeJSONString(string) - - def writeBinary(self, binary): - self.writeJSONBase64(binary) - - -class TJSONProtocolFactory: - - def getProtocol(self, trans): - return TJSONProtocol(trans) - - -class TSimpleJSONProtocol(TJSONProtocolBase): - """Simple, readable, write-only JSON protocol. - - Useful for interacting with scripting languages. - """ - - def readMessageBegin(self): - raise NotImplementedError() - - def readMessageEnd(self): - raise NotImplementedError() - - def readStructBegin(self): - raise NotImplementedError() - - def readStructEnd(self): - raise NotImplementedError() - - def writeMessageBegin(self, name, request_type, seqid): - self.resetWriteContext() - - def writeMessageEnd(self): - pass - - def writeStructBegin(self, name): - self.writeJSONObjectStart() - - def writeStructEnd(self): - self.writeJSONObjectEnd() - - def writeFieldBegin(self, name, ttype, fid): - self.writeJSONString(name) - - def writeFieldEnd(self): - pass - - def writeMapBegin(self, ktype, vtype, size): - self.writeJSONObjectStart() - - def writeMapEnd(self): - self.writeJSONObjectEnd() - - def _writeCollectionBegin(self, etype, size): - self.writeJSONArrayStart() - - def _writeCollectionEnd(self): - self.writeJSONArrayEnd() - writeListBegin = _writeCollectionBegin - writeListEnd = _writeCollectionEnd - writeSetBegin = _writeCollectionBegin - writeSetEnd = _writeCollectionEnd - - def writeInteger(self, integer): - self.writeJSONNumber(integer) - writeByte = writeInteger - writeI16 = writeInteger - writeI32 = writeInteger - writeI64 = writeInteger - - def writeBool(self, boolean): - self.writeJSONNumber(1 if boolean is True else 0) - - def writeDouble(self, dbl): - self.writeJSONNumber(dbl) - - def writeString(self, string): - self.writeJSONString(string) - - def writeBinary(self, binary): - self.writeJSONBase64(binary) - - -class TSimpleJSONProtocolFactory(object): - - def getProtocol(self, trans): - return TSimpleJSONProtocol(trans) diff --git a/pyload/lib/thrift/protocol/TProtocol.py b/pyload/lib/thrift/protocol/TProtocol.py deleted file mode 100644 index dc2b095de..000000000 --- a/pyload/lib/thrift/protocol/TProtocol.py +++ /dev/null @@ -1,406 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -from thrift.Thrift import * - - -class TProtocolException(TException): - """Custom Protocol Exception class""" - - UNKNOWN = 0 - INVALID_DATA = 1 - NEGATIVE_SIZE = 2 - SIZE_LIMIT = 3 - BAD_VERSION = 4 - - def __init__(self, type=UNKNOWN, message=None): - TException.__init__(self, message) - self.type = type - - -class TProtocolBase: - """Base class for Thrift protocol driver.""" - - def __init__(self, trans): - self.trans = trans - - def writeMessageBegin(self, name, ttype, seqid): - pass - - def writeMessageEnd(self): - pass - - def writeStructBegin(self, name): - pass - - def writeStructEnd(self): - pass - - def writeFieldBegin(self, name, ttype, fid): - pass - - def writeFieldEnd(self): - pass - - def writeFieldStop(self): - pass - - def writeMapBegin(self, ktype, vtype, size): - pass - - def writeMapEnd(self): - pass - - def writeListBegin(self, etype, size): - pass - - def writeListEnd(self): - pass - - def writeSetBegin(self, etype, size): - pass - - def writeSetEnd(self): - pass - - def writeBool(self, bool_val): - pass - - def writeByte(self, byte): - pass - - def writeI16(self, i16): - pass - - def writeI32(self, i32): - pass - - def writeI64(self, i64): - pass - - def writeDouble(self, dub): - pass - - def writeString(self, str_val): - pass - - def readMessageBegin(self): - pass - - def readMessageEnd(self): - pass - - def readStructBegin(self): - pass - - def readStructEnd(self): - pass - - def readFieldBegin(self): - pass - - def readFieldEnd(self): - pass - - def readMapBegin(self): - pass - - def readMapEnd(self): - pass - - def readListBegin(self): - pass - - def readListEnd(self): - pass - - def readSetBegin(self): - pass - - def readSetEnd(self): - pass - - def readBool(self): - pass - - def readByte(self): - pass - - def readI16(self): - pass - - def readI32(self): - pass - - def readI64(self): - pass - - def readDouble(self): - pass - - def readString(self): - pass - - def skip(self, ttype): - if ttype == TType.STOP: - return - elif ttype == TType.BOOL: - self.readBool() - elif ttype == TType.BYTE: - self.readByte() - elif ttype == TType.I16: - self.readI16() - elif ttype == TType.I32: - self.readI32() - elif ttype == TType.I64: - self.readI64() - elif ttype == TType.DOUBLE: - self.readDouble() - elif ttype == TType.STRING: - self.readString() - elif ttype == TType.STRUCT: - name = self.readStructBegin() - while True: - (name, ttype, id) = self.readFieldBegin() - if ttype == TType.STOP: - break - self.skip(ttype) - self.readFieldEnd() - self.readStructEnd() - elif ttype == TType.MAP: - (ktype, vtype, size) = self.readMapBegin() - for i in xrange(size): - self.skip(ktype) - self.skip(vtype) - self.readMapEnd() - elif ttype == TType.SET: - (etype, size) = self.readSetBegin() - for i in xrange(size): - self.skip(etype) - self.readSetEnd() - elif ttype == TType.LIST: - (etype, size) = self.readListBegin() - for i in xrange(size): - self.skip(etype) - self.readListEnd() - - # tuple of: ( 'reader method' name, is_container bool, 'writer_method' name ) - _TTYPE_HANDLERS = ( - (None, None, False), # 0 TType.STOP - (None, None, False), # 1 TType.VOID # TODO: handle void? - ('readBool', 'writeBool', False), # 2 TType.BOOL - ('readByte', 'writeByte', False), # 3 TType.BYTE and I08 - ('readDouble', 'writeDouble', False), # 4 TType.DOUBLE - (None, None, False), # 5 undefined - ('readI16', 'writeI16', False), # 6 TType.I16 - (None, None, False), # 7 undefined - ('readI32', 'writeI32', False), # 8 TType.I32 - (None, None, False), # 9 undefined - ('readI64', 'writeI64', False), # 10 TType.I64 - ('readString', 'writeString', False), # 11 TType.STRING and UTF7 - ('readContainerStruct', 'writeContainerStruct', True), # 12 *.STRUCT - ('readContainerMap', 'writeContainerMap', True), # 13 TType.MAP - ('readContainerSet', 'writeContainerSet', True), # 14 TType.SET - ('readContainerList', 'writeContainerList', True), # 15 TType.LIST - (None, None, False), # 16 TType.UTF8 # TODO: handle utf8 types? - (None, None, False) # 17 TType.UTF16 # TODO: handle utf16 types? - ) - - def readFieldByTType(self, ttype, spec): - try: - (r_handler, w_handler, is_container) = self._TTYPE_HANDLERS[ttype] - except IndexError: - raise TProtocolException(type=TProtocolException.INVALID_DATA, - message='Invalid field type %d' % (ttype)) - if r_handler is None: - raise TProtocolException(type=TProtocolException.INVALID_DATA, - message='Invalid field type %d' % (ttype)) - reader = getattr(self, r_handler) - if not is_container: - return reader() - return reader(spec) - - def readContainerList(self, spec): - results = [] - ttype, tspec = spec[0], spec[1] - r_handler = self._TTYPE_HANDLERS[ttype][0] - reader = getattr(self, r_handler) - (list_type, list_len) = self.readListBegin() - if tspec is None: - # list values are simple types - for idx in xrange(list_len): - results.append(reader()) - else: - # this is like an inlined readFieldByTType - container_reader = self._TTYPE_HANDLERS[list_type][0] - val_reader = getattr(self, container_reader) - for idx in xrange(list_len): - val = val_reader(tspec) - results.append(val) - self.readListEnd() - return results - - def readContainerSet(self, spec): - results = set() - ttype, tspec = spec[0], spec[1] - r_handler = self._TTYPE_HANDLERS[ttype][0] - reader = getattr(self, r_handler) - (set_type, set_len) = self.readSetBegin() - if tspec is None: - # set members are simple types - for idx in xrange(set_len): - results.add(reader()) - else: - container_reader = self._TTYPE_HANDLERS[set_type][0] - val_reader = getattr(self, container_reader) - for idx in xrange(set_len): - results.add(val_reader(tspec)) - self.readSetEnd() - return results - - def readContainerStruct(self, spec): - (obj_class, obj_spec) = spec - obj = obj_class() - obj.read(self) - return obj - - def readContainerMap(self, spec): - results = dict() - key_ttype, key_spec = spec[0], spec[1] - val_ttype, val_spec = spec[2], spec[3] - (map_ktype, map_vtype, map_len) = self.readMapBegin() - # TODO: compare types we just decoded with thrift_spec and - # abort/skip if types disagree - key_reader = getattr(self, self._TTYPE_HANDLERS[key_ttype][0]) - val_reader = getattr(self, self._TTYPE_HANDLERS[val_ttype][0]) - # list values are simple types - for idx in xrange(map_len): - if key_spec is None: - k_val = key_reader() - else: - k_val = self.readFieldByTType(key_ttype, key_spec) - if val_spec is None: - v_val = val_reader() - else: - v_val = self.readFieldByTType(val_ttype, val_spec) - # this raises a TypeError with unhashable keys types - # i.e. this fails: d=dict(); d[[0,1]] = 2 - results[k_val] = v_val - self.readMapEnd() - return results - - def readStruct(self, obj, thrift_spec): - self.readStructBegin() - while True: - (fname, ftype, fid) = self.readFieldBegin() - if ftype == TType.STOP: - break - try: - field = thrift_spec[fid] - except IndexError: - self.skip(ftype) - else: - if field is not None and ftype == field[1]: - fname = field[2] - fspec = field[3] - val = self.readFieldByTType(ftype, fspec) - setattr(obj, fname, val) - else: - self.skip(ftype) - self.readFieldEnd() - self.readStructEnd() - - def writeContainerStruct(self, val, spec): - val.write(self) - - def writeContainerList(self, val, spec): - self.writeListBegin(spec[0], len(val)) - r_handler, w_handler, is_container = self._TTYPE_HANDLERS[spec[0]] - e_writer = getattr(self, w_handler) - if not is_container: - for elem in val: - e_writer(elem) - else: - for elem in val: - e_writer(elem, spec[1]) - self.writeListEnd() - - def writeContainerSet(self, val, spec): - self.writeSetBegin(spec[0], len(val)) - r_handler, w_handler, is_container = self._TTYPE_HANDLERS[spec[0]] - e_writer = getattr(self, w_handler) - if not is_container: - for elem in val: - e_writer(elem) - else: - for elem in val: - e_writer(elem, spec[1]) - self.writeSetEnd() - - def writeContainerMap(self, val, spec): - k_type = spec[0] - v_type = spec[2] - ignore, ktype_name, k_is_container = self._TTYPE_HANDLERS[k_type] - ignore, vtype_name, v_is_container = self._TTYPE_HANDLERS[v_type] - k_writer = getattr(self, ktype_name) - v_writer = getattr(self, vtype_name) - self.writeMapBegin(k_type, v_type, len(val)) - for m_key, m_val in val.iteritems(): - if not k_is_container: - k_writer(m_key) - else: - k_writer(m_key, spec[1]) - if not v_is_container: - v_writer(m_val) - else: - v_writer(m_val, spec[3]) - self.writeMapEnd() - - def writeStruct(self, obj, thrift_spec): - self.writeStructBegin(obj.__class__.__name__) - for field in thrift_spec: - if field is None: - continue - fname = field[2] - val = getattr(obj, fname) - if val is None: - # skip writing out unset fields - continue - fid = field[0] - ftype = field[1] - fspec = field[3] - # get the writer method for this value - self.writeFieldBegin(fname, ftype, fid) - self.writeFieldByTType(ftype, val, fspec) - self.writeFieldEnd() - self.writeFieldStop() - self.writeStructEnd() - - def writeFieldByTType(self, ttype, val, spec): - r_handler, w_handler, is_container = self._TTYPE_HANDLERS[ttype] - writer = getattr(self, w_handler) - if is_container: - writer(val, spec) - else: - writer(val) - - -class TProtocolFactory: - def getProtocol(self, trans): - pass diff --git a/pyload/lib/thrift/protocol/__init__.py b/pyload/lib/thrift/protocol/__init__.py deleted file mode 100644 index 7eefb458a..000000000 --- a/pyload/lib/thrift/protocol/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -__all__ = ['fastbinary', 'TBase', 'TBinaryProtocol', 'TCompactProtocol', 'TJSONProtocol', 'TProtocol'] diff --git a/pyload/lib/thrift/protocol/fastbinary.c b/pyload/lib/thrift/protocol/fastbinary.c deleted file mode 100644 index 2ce56603c..000000000 --- a/pyload/lib/thrift/protocol/fastbinary.c +++ /dev/null @@ -1,1219 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include -#include "cStringIO.h" -#include -#ifndef _WIN32 -# include -# include -#else -# include -# pragma comment (lib, "ws2_32.lib") -# define BIG_ENDIAN (4321) -# define LITTLE_ENDIAN (1234) -# define BYTE_ORDER LITTLE_ENDIAN -# if defined(_MSC_VER) && _MSC_VER < 1600 - typedef int _Bool; -# define bool _Bool -# define false 0 -# define true 1 -# endif -# define inline __inline -#endif - -/* Fix endianness issues on Solaris */ -#if defined (__SVR4) && defined (__sun) - #if defined(__i386) && !defined(__i386__) - #define __i386__ - #endif - - #ifndef BIG_ENDIAN - #define BIG_ENDIAN (4321) - #endif - #ifndef LITTLE_ENDIAN - #define LITTLE_ENDIAN (1234) - #endif - - /* I386 is LE, even on Solaris */ - #if !defined(BYTE_ORDER) && defined(__i386__) - #define BYTE_ORDER LITTLE_ENDIAN - #endif -#endif - -// TODO(dreiss): defval appears to be unused. Look into removing it. -// TODO(dreiss): Make parse_spec_args recursive, and cache the output -// permanently in the object. (Malloc and orphan.) -// TODO(dreiss): Why do we need cStringIO for reading, why not just char*? -// Can cStringIO let us work with a BufferedTransport? -// TODO(dreiss): Don't ignore the rv from cwrite (maybe). - -/* ====== BEGIN UTILITIES ====== */ - -#define INIT_OUTBUF_SIZE 128 - -// Stolen out of TProtocol.h. -// It would be a huge pain to have both get this from one place. -typedef enum TType { - T_STOP = 0, - T_VOID = 1, - T_BOOL = 2, - T_BYTE = 3, - T_I08 = 3, - T_I16 = 6, - T_I32 = 8, - T_U64 = 9, - T_I64 = 10, - T_DOUBLE = 4, - T_STRING = 11, - T_UTF7 = 11, - T_STRUCT = 12, - T_MAP = 13, - T_SET = 14, - T_LIST = 15, - T_UTF8 = 16, - T_UTF16 = 17 -} TType; - -#ifndef __BYTE_ORDER -# if defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && defined(BIG_ENDIAN) -# define __BYTE_ORDER BYTE_ORDER -# define __LITTLE_ENDIAN LITTLE_ENDIAN -# define __BIG_ENDIAN BIG_ENDIAN -# else -# error "Cannot determine endianness" -# endif -#endif - -// Same comment as the enum. Sorry. -#if __BYTE_ORDER == __BIG_ENDIAN -# define ntohll(n) (n) -# define htonll(n) (n) -#elif __BYTE_ORDER == __LITTLE_ENDIAN -# if defined(__GNUC__) && defined(__GLIBC__) -# include -# define ntohll(n) bswap_64(n) -# define htonll(n) bswap_64(n) -# else /* GNUC & GLIBC */ -# define ntohll(n) ( (((unsigned long long)ntohl(n)) << 32) + ntohl(n >> 32) ) -# define htonll(n) ( (((unsigned long long)htonl(n)) << 32) + htonl(n >> 32) ) -# endif /* GNUC & GLIBC */ -#else /* __BYTE_ORDER */ -# error "Can't define htonll or ntohll!" -#endif - -// Doing a benchmark shows that interning actually makes a difference, amazingly. -#define INTERN_STRING(value) _intern_ ## value - -#define INT_CONV_ERROR_OCCURRED(v) ( ((v) == -1) && PyErr_Occurred() ) -#define CHECK_RANGE(v, min, max) ( ((v) <= (max)) && ((v) >= (min)) ) - -// Py_ssize_t was not defined before Python 2.5 -#if (PY_VERSION_HEX < 0x02050000) -typedef int Py_ssize_t; -#endif - -/** - * A cache of the spec_args for a set or list, - * so we don't have to keep calling PyTuple_GET_ITEM. - */ -typedef struct { - TType element_type; - PyObject* typeargs; -} SetListTypeArgs; - -/** - * A cache of the spec_args for a map, - * so we don't have to keep calling PyTuple_GET_ITEM. - */ -typedef struct { - TType ktag; - TType vtag; - PyObject* ktypeargs; - PyObject* vtypeargs; -} MapTypeArgs; - -/** - * A cache of the spec_args for a struct, - * so we don't have to keep calling PyTuple_GET_ITEM. - */ -typedef struct { - PyObject* klass; - PyObject* spec; -} StructTypeArgs; - -/** - * A cache of the item spec from a struct specification, - * so we don't have to keep calling PyTuple_GET_ITEM. - */ -typedef struct { - int tag; - TType type; - PyObject* attrname; - PyObject* typeargs; - PyObject* defval; -} StructItemSpec; - -/** - * A cache of the two key attributes of a CReadableTransport, - * so we don't have to keep calling PyObject_GetAttr. - */ -typedef struct { - PyObject* stringiobuf; - PyObject* refill_callable; -} DecodeBuffer; - -/** Pointer to interned string to speed up attribute lookup. */ -static PyObject* INTERN_STRING(cstringio_buf); -/** Pointer to interned string to speed up attribute lookup. */ -static PyObject* INTERN_STRING(cstringio_refill); - -static inline bool -check_ssize_t_32(Py_ssize_t len) { - // error from getting the int - if (INT_CONV_ERROR_OCCURRED(len)) { - return false; - } - if (!CHECK_RANGE(len, 0, INT32_MAX)) { - PyErr_SetString(PyExc_OverflowError, "string size out of range"); - return false; - } - return true; -} - -static inline bool -parse_pyint(PyObject* o, int32_t* ret, int32_t min, int32_t max) { - long val = PyInt_AsLong(o); - - if (INT_CONV_ERROR_OCCURRED(val)) { - return false; - } - if (!CHECK_RANGE(val, min, max)) { - PyErr_SetString(PyExc_OverflowError, "int out of range"); - return false; - } - - *ret = (int32_t) val; - return true; -} - - -/* --- FUNCTIONS TO PARSE STRUCT SPECIFICATOINS --- */ - -static bool -parse_set_list_args(SetListTypeArgs* dest, PyObject* typeargs) { - if (PyTuple_Size(typeargs) != 2) { - PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for list/set type args"); - return false; - } - - dest->element_type = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0)); - if (INT_CONV_ERROR_OCCURRED(dest->element_type)) { - return false; - } - - dest->typeargs = PyTuple_GET_ITEM(typeargs, 1); - - return true; -} - -static bool -parse_map_args(MapTypeArgs* dest, PyObject* typeargs) { - if (PyTuple_Size(typeargs) != 4) { - PyErr_SetString(PyExc_TypeError, "expecting 4 arguments for typeargs to map"); - return false; - } - - dest->ktag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0)); - if (INT_CONV_ERROR_OCCURRED(dest->ktag)) { - return false; - } - - dest->vtag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 2)); - if (INT_CONV_ERROR_OCCURRED(dest->vtag)) { - return false; - } - - dest->ktypeargs = PyTuple_GET_ITEM(typeargs, 1); - dest->vtypeargs = PyTuple_GET_ITEM(typeargs, 3); - - return true; -} - -static bool -parse_struct_args(StructTypeArgs* dest, PyObject* typeargs) { - if (PyTuple_Size(typeargs) != 2) { - PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for struct args"); - return false; - } - - dest->klass = PyTuple_GET_ITEM(typeargs, 0); - dest->spec = PyTuple_GET_ITEM(typeargs, 1); - - return true; -} - -static int -parse_struct_item_spec(StructItemSpec* dest, PyObject* spec_tuple) { - - // i'd like to use ParseArgs here, but it seems to be a bottleneck. - if (PyTuple_Size(spec_tuple) != 5) { - PyErr_SetString(PyExc_TypeError, "expecting 5 arguments for spec tuple"); - return false; - } - - dest->tag = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 0)); - if (INT_CONV_ERROR_OCCURRED(dest->tag)) { - return false; - } - - dest->type = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 1)); - if (INT_CONV_ERROR_OCCURRED(dest->type)) { - return false; - } - - dest->attrname = PyTuple_GET_ITEM(spec_tuple, 2); - dest->typeargs = PyTuple_GET_ITEM(spec_tuple, 3); - dest->defval = PyTuple_GET_ITEM(spec_tuple, 4); - return true; -} - -/* ====== END UTILITIES ====== */ - - -/* ====== BEGIN WRITING FUNCTIONS ====== */ - -/* --- LOW-LEVEL WRITING FUNCTIONS --- */ - -static void writeByte(PyObject* outbuf, int8_t val) { - int8_t net = val; - PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int8_t)); -} - -static void writeI16(PyObject* outbuf, int16_t val) { - int16_t net = (int16_t)htons(val); - PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int16_t)); -} - -static void writeI32(PyObject* outbuf, int32_t val) { - int32_t net = (int32_t)htonl(val); - PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int32_t)); -} - -static void writeI64(PyObject* outbuf, int64_t val) { - int64_t net = (int64_t)htonll(val); - PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int64_t)); -} - -static void writeDouble(PyObject* outbuf, double dub) { - // Unfortunately, bitwise_cast doesn't work in C. Bad C! - union { - double f; - int64_t t; - } transfer; - transfer.f = dub; - writeI64(outbuf, transfer.t); -} - - -/* --- MAIN RECURSIVE OUTPUT FUCNTION -- */ - -static int -output_val(PyObject* output, PyObject* value, TType type, PyObject* typeargs) { - /* - * Refcounting Strategy: - * - * We assume that elements of the thrift_spec tuple are not going to be - * mutated, so we don't ref count those at all. Other than that, we try to - * keep a reference to all the user-created objects while we work with them. - * output_val assumes that a reference is already held. The *caller* is - * responsible for handling references - */ - - switch (type) { - - case T_BOOL: { - int v = PyObject_IsTrue(value); - if (v == -1) { - return false; - } - - writeByte(output, (int8_t) v); - break; - } - case T_I08: { - int32_t val; - - if (!parse_pyint(value, &val, INT8_MIN, INT8_MAX)) { - return false; - } - - writeByte(output, (int8_t) val); - break; - } - case T_I16: { - int32_t val; - - if (!parse_pyint(value, &val, INT16_MIN, INT16_MAX)) { - return false; - } - - writeI16(output, (int16_t) val); - break; - } - case T_I32: { - int32_t val; - - if (!parse_pyint(value, &val, INT32_MIN, INT32_MAX)) { - return false; - } - - writeI32(output, val); - break; - } - case T_I64: { - int64_t nval = PyLong_AsLongLong(value); - - if (INT_CONV_ERROR_OCCURRED(nval)) { - return false; - } - - if (!CHECK_RANGE(nval, INT64_MIN, INT64_MAX)) { - PyErr_SetString(PyExc_OverflowError, "int out of range"); - return false; - } - - writeI64(output, nval); - break; - } - - case T_DOUBLE: { - double nval = PyFloat_AsDouble(value); - if (nval == -1.0 && PyErr_Occurred()) { - return false; - } - - writeDouble(output, nval); - break; - } - - case T_STRING: { - Py_ssize_t len = PyString_Size(value); - - if (!check_ssize_t_32(len)) { - return false; - } - - writeI32(output, (int32_t) len); - PycStringIO->cwrite(output, PyString_AsString(value), (int32_t) len); - break; - } - - case T_LIST: - case T_SET: { - Py_ssize_t len; - SetListTypeArgs parsedargs; - PyObject *item; - PyObject *iterator; - - if (!parse_set_list_args(&parsedargs, typeargs)) { - return false; - } - - len = PyObject_Length(value); - - if (!check_ssize_t_32(len)) { - return false; - } - - writeByte(output, parsedargs.element_type); - writeI32(output, (int32_t) len); - - iterator = PyObject_GetIter(value); - if (iterator == NULL) { - return false; - } - - while ((item = PyIter_Next(iterator))) { - if (!output_val(output, item, parsedargs.element_type, parsedargs.typeargs)) { - Py_DECREF(item); - Py_DECREF(iterator); - return false; - } - Py_DECREF(item); - } - - Py_DECREF(iterator); - - if (PyErr_Occurred()) { - return false; - } - - break; - } - - case T_MAP: { - PyObject *k, *v; - Py_ssize_t pos = 0; - Py_ssize_t len; - - MapTypeArgs parsedargs; - - len = PyDict_Size(value); - if (!check_ssize_t_32(len)) { - return false; - } - - if (!parse_map_args(&parsedargs, typeargs)) { - return false; - } - - writeByte(output, parsedargs.ktag); - writeByte(output, parsedargs.vtag); - writeI32(output, len); - - // TODO(bmaurer): should support any mapping, not just dicts - while (PyDict_Next(value, &pos, &k, &v)) { - // TODO(dreiss): Think hard about whether these INCREFs actually - // turn any unsafe scenarios into safe scenarios. - Py_INCREF(k); - Py_INCREF(v); - - if (!output_val(output, k, parsedargs.ktag, parsedargs.ktypeargs) - || !output_val(output, v, parsedargs.vtag, parsedargs.vtypeargs)) { - Py_DECREF(k); - Py_DECREF(v); - return false; - } - Py_DECREF(k); - Py_DECREF(v); - } - break; - } - - // TODO(dreiss): Consider breaking this out as a function - // the way we did for decode_struct. - case T_STRUCT: { - StructTypeArgs parsedargs; - Py_ssize_t nspec; - Py_ssize_t i; - - if (!parse_struct_args(&parsedargs, typeargs)) { - return false; - } - - nspec = PyTuple_Size(parsedargs.spec); - - if (nspec == -1) { - return false; - } - - for (i = 0; i < nspec; i++) { - StructItemSpec parsedspec; - PyObject* spec_tuple; - PyObject* instval = NULL; - - spec_tuple = PyTuple_GET_ITEM(parsedargs.spec, i); - if (spec_tuple == Py_None) { - continue; - } - - if (!parse_struct_item_spec (&parsedspec, spec_tuple)) { - return false; - } - - instval = PyObject_GetAttr(value, parsedspec.attrname); - - if (!instval) { - return false; - } - - if (instval == Py_None) { - Py_DECREF(instval); - continue; - } - - writeByte(output, (int8_t) parsedspec.type); - writeI16(output, parsedspec.tag); - - if (!output_val(output, instval, parsedspec.type, parsedspec.typeargs)) { - Py_DECREF(instval); - return false; - } - - Py_DECREF(instval); - } - - writeByte(output, (int8_t)T_STOP); - break; - } - - case T_STOP: - case T_VOID: - case T_UTF16: - case T_UTF8: - case T_U64: - default: - PyErr_SetString(PyExc_TypeError, "Unexpected TType"); - return false; - - } - - return true; -} - - -/* --- TOP-LEVEL WRAPPER FOR OUTPUT -- */ - -static PyObject * -encode_binary(PyObject *self, PyObject *args) { - PyObject* enc_obj; - PyObject* type_args; - PyObject* buf; - PyObject* ret = NULL; - - if (!PyArg_ParseTuple(args, "OO", &enc_obj, &type_args)) { - return NULL; - } - - buf = PycStringIO->NewOutput(INIT_OUTBUF_SIZE); - if (output_val(buf, enc_obj, T_STRUCT, type_args)) { - ret = PycStringIO->cgetvalue(buf); - } - - Py_DECREF(buf); - return ret; -} - -/* ====== END WRITING FUNCTIONS ====== */ - - -/* ====== BEGIN READING FUNCTIONS ====== */ - -/* --- LOW-LEVEL READING FUNCTIONS --- */ - -static void -free_decodebuf(DecodeBuffer* d) { - Py_XDECREF(d->stringiobuf); - Py_XDECREF(d->refill_callable); -} - -static bool -decode_buffer_from_obj(DecodeBuffer* dest, PyObject* obj) { - dest->stringiobuf = PyObject_GetAttr(obj, INTERN_STRING(cstringio_buf)); - if (!dest->stringiobuf) { - return false; - } - - if (!PycStringIO_InputCheck(dest->stringiobuf)) { - free_decodebuf(dest); - PyErr_SetString(PyExc_TypeError, "expecting stringio input"); - return false; - } - - dest->refill_callable = PyObject_GetAttr(obj, INTERN_STRING(cstringio_refill)); - - if(!dest->refill_callable) { - free_decodebuf(dest); - return false; - } - - if (!PyCallable_Check(dest->refill_callable)) { - free_decodebuf(dest); - PyErr_SetString(PyExc_TypeError, "expecting callable"); - return false; - } - - return true; -} - -static bool readBytes(DecodeBuffer* input, char** output, int len) { - int read; - - // TODO(dreiss): Don't fear the malloc. Think about taking a copy of - // the partial read instead of forcing the transport - // to prepend it to its buffer. - - read = PycStringIO->cread(input->stringiobuf, output, len); - - if (read == len) { - return true; - } else if (read == -1) { - return false; - } else { - PyObject* newiobuf; - - // using building functions as this is a rare codepath - newiobuf = PyObject_CallFunction( - input->refill_callable, "s#i", *output, read, len, NULL); - if (newiobuf == NULL) { - return false; - } - - // must do this *AFTER* the call so that we don't deref the io buffer - Py_CLEAR(input->stringiobuf); - input->stringiobuf = newiobuf; - - read = PycStringIO->cread(input->stringiobuf, output, len); - - if (read == len) { - return true; - } else if (read == -1) { - return false; - } else { - // TODO(dreiss): This could be a valid code path for big binary blobs. - PyErr_SetString(PyExc_TypeError, - "refill claimed to have refilled the buffer, but didn't!!"); - return false; - } - } -} - -static int8_t readByte(DecodeBuffer* input) { - char* buf; - if (!readBytes(input, &buf, sizeof(int8_t))) { - return -1; - } - - return *(int8_t*) buf; -} - -static int16_t readI16(DecodeBuffer* input) { - char* buf; - if (!readBytes(input, &buf, sizeof(int16_t))) { - return -1; - } - - return (int16_t) ntohs(*(int16_t*) buf); -} - -static int32_t readI32(DecodeBuffer* input) { - char* buf; - if (!readBytes(input, &buf, sizeof(int32_t))) { - return -1; - } - return (int32_t) ntohl(*(int32_t*) buf); -} - - -static int64_t readI64(DecodeBuffer* input) { - char* buf; - if (!readBytes(input, &buf, sizeof(int64_t))) { - return -1; - } - - return (int64_t) ntohll(*(int64_t*) buf); -} - -static double readDouble(DecodeBuffer* input) { - union { - int64_t f; - double t; - } transfer; - - transfer.f = readI64(input); - if (transfer.f == -1) { - return -1; - } - return transfer.t; -} - -static bool -checkTypeByte(DecodeBuffer* input, TType expected) { - TType got = readByte(input); - if (INT_CONV_ERROR_OCCURRED(got)) { - return false; - } - - if (expected != got) { - PyErr_SetString(PyExc_TypeError, "got wrong ttype while reading field"); - return false; - } - return true; -} - -static bool -skip(DecodeBuffer* input, TType type) { -#define SKIPBYTES(n) \ - do { \ - if (!readBytes(input, &dummy_buf, (n))) { \ - return false; \ - } \ - } while(0) - - char* dummy_buf; - - switch (type) { - - case T_BOOL: - case T_I08: SKIPBYTES(1); break; - case T_I16: SKIPBYTES(2); break; - case T_I32: SKIPBYTES(4); break; - case T_I64: - case T_DOUBLE: SKIPBYTES(8); break; - - case T_STRING: { - // TODO(dreiss): Find out if these check_ssize_t32s are really necessary. - int len = readI32(input); - if (!check_ssize_t_32(len)) { - return false; - } - SKIPBYTES(len); - break; - } - - case T_LIST: - case T_SET: { - TType etype; - int len, i; - - etype = readByte(input); - if (etype == -1) { - return false; - } - - len = readI32(input); - if (!check_ssize_t_32(len)) { - return false; - } - - for (i = 0; i < len; i++) { - if (!skip(input, etype)) { - return false; - } - } - break; - } - - case T_MAP: { - TType ktype, vtype; - int len, i; - - ktype = readByte(input); - if (ktype == -1) { - return false; - } - - vtype = readByte(input); - if (vtype == -1) { - return false; - } - - len = readI32(input); - if (!check_ssize_t_32(len)) { - return false; - } - - for (i = 0; i < len; i++) { - if (!(skip(input, ktype) && skip(input, vtype))) { - return false; - } - } - break; - } - - case T_STRUCT: { - while (true) { - TType type; - - type = readByte(input); - if (type == -1) { - return false; - } - - if (type == T_STOP) - break; - - SKIPBYTES(2); // tag - if (!skip(input, type)) { - return false; - } - } - break; - } - - case T_STOP: - case T_VOID: - case T_UTF16: - case T_UTF8: - case T_U64: - default: - PyErr_SetString(PyExc_TypeError, "Unexpected TType"); - return false; - - } - - return true; - -#undef SKIPBYTES -} - - -/* --- HELPER FUNCTION FOR DECODE_VAL --- */ - -static PyObject* -decode_val(DecodeBuffer* input, TType type, PyObject* typeargs); - -static bool -decode_struct(DecodeBuffer* input, PyObject* output, PyObject* spec_seq) { - int spec_seq_len = PyTuple_Size(spec_seq); - if (spec_seq_len == -1) { - return false; - } - - while (true) { - TType type; - int16_t tag; - PyObject* item_spec; - PyObject* fieldval = NULL; - StructItemSpec parsedspec; - - type = readByte(input); - if (type == -1) { - return false; - } - if (type == T_STOP) { - break; - } - tag = readI16(input); - if (INT_CONV_ERROR_OCCURRED(tag)) { - return false; - } - if (tag >= 0 && tag < spec_seq_len) { - item_spec = PyTuple_GET_ITEM(spec_seq, tag); - } else { - item_spec = Py_None; - } - - if (item_spec == Py_None) { - if (!skip(input, type)) { - return false; - } else { - continue; - } - } - - if (!parse_struct_item_spec(&parsedspec, item_spec)) { - return false; - } - if (parsedspec.type != type) { - if (!skip(input, type)) { - PyErr_SetString(PyExc_TypeError, "struct field had wrong type while reading and can't be skipped"); - return false; - } else { - continue; - } - } - - fieldval = decode_val(input, parsedspec.type, parsedspec.typeargs); - if (fieldval == NULL) { - return false; - } - - if (PyObject_SetAttr(output, parsedspec.attrname, fieldval) == -1) { - Py_DECREF(fieldval); - return false; - } - Py_DECREF(fieldval); - } - return true; -} - - -/* --- MAIN RECURSIVE INPUT FUCNTION --- */ - -// Returns a new reference. -static PyObject* -decode_val(DecodeBuffer* input, TType type, PyObject* typeargs) { - switch (type) { - - case T_BOOL: { - int8_t v = readByte(input); - if (INT_CONV_ERROR_OCCURRED(v)) { - return NULL; - } - - switch (v) { - case 0: Py_RETURN_FALSE; - case 1: Py_RETURN_TRUE; - // Don't laugh. This is a potentially serious issue. - default: PyErr_SetString(PyExc_TypeError, "boolean out of range"); return NULL; - } - break; - } - case T_I08: { - int8_t v = readByte(input); - if (INT_CONV_ERROR_OCCURRED(v)) { - return NULL; - } - - return PyInt_FromLong(v); - } - case T_I16: { - int16_t v = readI16(input); - if (INT_CONV_ERROR_OCCURRED(v)) { - return NULL; - } - return PyInt_FromLong(v); - } - case T_I32: { - int32_t v = readI32(input); - if (INT_CONV_ERROR_OCCURRED(v)) { - return NULL; - } - return PyInt_FromLong(v); - } - - case T_I64: { - int64_t v = readI64(input); - if (INT_CONV_ERROR_OCCURRED(v)) { - return NULL; - } - // TODO(dreiss): Find out if we can take this fastpath always when - // sizeof(long) == sizeof(long long). - if (CHECK_RANGE(v, LONG_MIN, LONG_MAX)) { - return PyInt_FromLong((long) v); - } - - return PyLong_FromLongLong(v); - } - - case T_DOUBLE: { - double v = readDouble(input); - if (v == -1.0 && PyErr_Occurred()) { - return false; - } - return PyFloat_FromDouble(v); - } - - case T_STRING: { - Py_ssize_t len = readI32(input); - char* buf; - if (!readBytes(input, &buf, len)) { - return NULL; - } - - return PyString_FromStringAndSize(buf, len); - } - - case T_LIST: - case T_SET: { - SetListTypeArgs parsedargs; - int32_t len; - PyObject* ret = NULL; - int i; - - if (!parse_set_list_args(&parsedargs, typeargs)) { - return NULL; - } - - if (!checkTypeByte(input, parsedargs.element_type)) { - return NULL; - } - - len = readI32(input); - if (!check_ssize_t_32(len)) { - return NULL; - } - - ret = PyList_New(len); - if (!ret) { - return NULL; - } - - for (i = 0; i < len; i++) { - PyObject* item = decode_val(input, parsedargs.element_type, parsedargs.typeargs); - if (!item) { - Py_DECREF(ret); - return NULL; - } - PyList_SET_ITEM(ret, i, item); - } - - // TODO(dreiss): Consider biting the bullet and making two separate cases - // for list and set, avoiding this post facto conversion. - if (type == T_SET) { - PyObject* setret; -#if (PY_VERSION_HEX < 0x02050000) - // hack needed for older versions - setret = PyObject_CallFunctionObjArgs((PyObject*)&PySet_Type, ret, NULL); -#else - // official version - setret = PySet_New(ret); -#endif - Py_DECREF(ret); - return setret; - } - return ret; - } - - case T_MAP: { - int32_t len; - int i; - MapTypeArgs parsedargs; - PyObject* ret = NULL; - - if (!parse_map_args(&parsedargs, typeargs)) { - return NULL; - } - - if (!checkTypeByte(input, parsedargs.ktag)) { - return NULL; - } - if (!checkTypeByte(input, parsedargs.vtag)) { - return NULL; - } - - len = readI32(input); - if (!check_ssize_t_32(len)) { - return false; - } - - ret = PyDict_New(); - if (!ret) { - goto error; - } - - for (i = 0; i < len; i++) { - PyObject* k = NULL; - PyObject* v = NULL; - k = decode_val(input, parsedargs.ktag, parsedargs.ktypeargs); - if (k == NULL) { - goto loop_error; - } - v = decode_val(input, parsedargs.vtag, parsedargs.vtypeargs); - if (v == NULL) { - goto loop_error; - } - if (PyDict_SetItem(ret, k, v) == -1) { - goto loop_error; - } - - Py_DECREF(k); - Py_DECREF(v); - continue; - - // Yuck! Destructors, anyone? - loop_error: - Py_XDECREF(k); - Py_XDECREF(v); - goto error; - } - - return ret; - - error: - Py_XDECREF(ret); - return NULL; - } - - case T_STRUCT: { - StructTypeArgs parsedargs; - PyObject* ret; - if (!parse_struct_args(&parsedargs, typeargs)) { - return NULL; - } - - ret = PyObject_CallObject(parsedargs.klass, NULL); - if (!ret) { - return NULL; - } - - if (!decode_struct(input, ret, parsedargs.spec)) { - Py_DECREF(ret); - return NULL; - } - - return ret; - } - - case T_STOP: - case T_VOID: - case T_UTF16: - case T_UTF8: - case T_U64: - default: - PyErr_SetString(PyExc_TypeError, "Unexpected TType"); - return NULL; - } -} - - -/* --- TOP-LEVEL WRAPPER FOR INPUT -- */ - -static PyObject* -decode_binary(PyObject *self, PyObject *args) { - PyObject* output_obj = NULL; - PyObject* transport = NULL; - PyObject* typeargs = NULL; - StructTypeArgs parsedargs; - DecodeBuffer input = {0, 0}; - - if (!PyArg_ParseTuple(args, "OOO", &output_obj, &transport, &typeargs)) { - return NULL; - } - - if (!parse_struct_args(&parsedargs, typeargs)) { - return NULL; - } - - if (!decode_buffer_from_obj(&input, transport)) { - return NULL; - } - - if (!decode_struct(&input, output_obj, parsedargs.spec)) { - free_decodebuf(&input); - return NULL; - } - - free_decodebuf(&input); - - Py_RETURN_NONE; -} - -/* ====== END READING FUNCTIONS ====== */ - - -/* -- PYTHON MODULE SETUP STUFF --- */ - -static PyMethodDef ThriftFastBinaryMethods[] = { - - {"encode_binary", encode_binary, METH_VARARGS, ""}, - {"decode_binary", decode_binary, METH_VARARGS, ""}, - - {NULL, NULL, 0, NULL} /* Sentinel */ -}; - -PyMODINIT_FUNC -initfastbinary(void) { -#define INIT_INTERN_STRING(value) \ - do { \ - INTERN_STRING(value) = PyString_InternFromString(#value); \ - if(!INTERN_STRING(value)) return; \ - } while(0) - - INIT_INTERN_STRING(cstringio_buf); - INIT_INTERN_STRING(cstringio_refill); -#undef INIT_INTERN_STRING - - PycString_IMPORT; - if (PycStringIO == NULL) return; - - (void) Py_InitModule("thrift.protocol.fastbinary", ThriftFastBinaryMethods); -} diff --git a/pyload/lib/thrift/server/THttpServer.py b/pyload/lib/thrift/server/THttpServer.py deleted file mode 100644 index be54bab94..000000000 --- a/pyload/lib/thrift/server/THttpServer.py +++ /dev/null @@ -1,87 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -import BaseHTTPServer - -from thrift.server import TServer -from thrift.transport import TTransport - - -class ResponseException(Exception): - """Allows handlers to override the HTTP response - - Normally, THttpServer always sends a 200 response. If a handler wants - to override this behavior (e.g., to simulate a misconfigured or - overloaded web server during testing), it can raise a ResponseException. - The function passed to the constructor will be called with the - RequestHandler as its only argument. - """ - def __init__(self, handler): - self.handler = handler - - -class THttpServer(TServer.TServer): - """A simple HTTP-based Thrift server - - This class is not very performant, but it is useful (for example) for - acting as a mock version of an Apache-based PHP Thrift endpoint. - """ - def __init__(self, - processor, - server_address, - inputProtocolFactory, - outputProtocolFactory=None, - server_class=BaseHTTPServer.HTTPServer): - """Set up protocol factories and HTTP server. - - See BaseHTTPServer for server_address. - See TServer for protocol factories. - """ - if outputProtocolFactory is None: - outputProtocolFactory = inputProtocolFactory - - TServer.TServer.__init__(self, processor, None, None, None, - inputProtocolFactory, outputProtocolFactory) - - thttpserver = self - - class RequestHander(BaseHTTPServer.BaseHTTPRequestHandler): - def do_POST(self): - # Don't care about the request path. - itrans = TTransport.TFileObjectTransport(self.rfile) - otrans = TTransport.TFileObjectTransport(self.wfile) - itrans = TTransport.TBufferedTransport( - itrans, int(self.headers['Content-Length'])) - otrans = TTransport.TMemoryBuffer() - iprot = thttpserver.inputProtocolFactory.getProtocol(itrans) - oprot = thttpserver.outputProtocolFactory.getProtocol(otrans) - try: - thttpserver.processor.process(iprot, oprot) - except ResponseException, exn: - exn.handler(self) - else: - self.send_response(200) - self.send_header("content-type", "application/x-thrift") - self.end_headers() - self.wfile.write(otrans.getvalue()) - - self.httpd = server_class(server_address, RequestHander) - - def serve(self): - self.httpd.serve_forever() diff --git a/pyload/lib/thrift/server/TNonblockingServer.py b/pyload/lib/thrift/server/TNonblockingServer.py deleted file mode 100644 index fa478d01f..000000000 --- a/pyload/lib/thrift/server/TNonblockingServer.py +++ /dev/null @@ -1,346 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -"""Implementation of non-blocking server. - -The main idea of the server is to receive and send requests -only from the main thread. - -The thread poool should be sized for concurrent tasks, not -maximum connections -""" -import threading -import socket -import Queue -import select -import struct -import logging - -from thrift.transport import TTransport -from thrift.protocol.TBinaryProtocol import TBinaryProtocolFactory - -__all__ = ['TNonblockingServer'] - - -class Worker(threading.Thread): - """Worker is a small helper to process incoming connection.""" - - def __init__(self, queue): - threading.Thread.__init__(self) - self.queue = queue - - def run(self): - """Process queries from task queue, stop if processor is None.""" - while True: - try: - processor, iprot, oprot, otrans, callback = self.queue.get() - if processor is None: - break - processor.process(iprot, oprot) - callback(True, otrans.getvalue()) - except Exception: - logging.exception("Exception while processing request") - callback(False, '') - -WAIT_LEN = 0 -WAIT_MESSAGE = 1 -WAIT_PROCESS = 2 -SEND_ANSWER = 3 -CLOSED = 4 - - -def locked(func): - """Decorator which locks self.lock.""" - def nested(self, *args, **kwargs): - self.lock.acquire() - try: - return func(self, *args, **kwargs) - finally: - self.lock.release() - return nested - - -def socket_exception(func): - """Decorator close object on socket.error.""" - def read(self, *args, **kwargs): - try: - return func(self, *args, **kwargs) - except socket.error: - self.close() - return read - - -class Connection: - """Basic class is represented connection. - - It can be in state: - WAIT_LEN --- connection is reading request len. - WAIT_MESSAGE --- connection is reading request. - WAIT_PROCESS --- connection has just read whole request and - waits for call ready routine. - SEND_ANSWER --- connection is sending answer string (including length - of answer). - CLOSED --- socket was closed and connection should be deleted. - """ - def __init__(self, new_socket, wake_up): - self.socket = new_socket - self.socket.setblocking(False) - self.status = WAIT_LEN - self.len = 0 - self.message = '' - self.lock = threading.Lock() - self.wake_up = wake_up - - def _read_len(self): - """Reads length of request. - - It's a safer alternative to self.socket.recv(4) - """ - read = self.socket.recv(4 - len(self.message)) - if len(read) == 0: - # if we read 0 bytes and self.message is empty, then - # the client closed the connection - if len(self.message) != 0: - logging.error("can't read frame size from socket") - self.close() - return - self.message += read - if len(self.message) == 4: - self.len, = struct.unpack('!i', self.message) - if self.len < 0: - logging.error("negative frame size, it seems client " - "doesn't use FramedTransport") - self.close() - elif self.len == 0: - logging.error("empty frame, it's really strange") - self.close() - else: - self.message = '' - self.status = WAIT_MESSAGE - - @socket_exception - def read(self): - """Reads data from stream and switch state.""" - assert self.status in (WAIT_LEN, WAIT_MESSAGE) - if self.status == WAIT_LEN: - self._read_len() - # go back to the main loop here for simplicity instead of - # falling through, even though there is a good chance that - # the message is already available - elif self.status == WAIT_MESSAGE: - read = self.socket.recv(self.len - len(self.message)) - if len(read) == 0: - logging.error("can't read frame from socket (get %d of " - "%d bytes)" % (len(self.message), self.len)) - self.close() - return - self.message += read - if len(self.message) == self.len: - self.status = WAIT_PROCESS - - @socket_exception - def write(self): - """Writes data from socket and switch state.""" - assert self.status == SEND_ANSWER - sent = self.socket.send(self.message) - if sent == len(self.message): - self.status = WAIT_LEN - self.message = '' - self.len = 0 - else: - self.message = self.message[sent:] - - @locked - def ready(self, all_ok, message): - """Callback function for switching state and waking up main thread. - - This function is the only function witch can be called asynchronous. - - The ready can switch Connection to three states: - WAIT_LEN if request was oneway. - SEND_ANSWER if request was processed in normal way. - CLOSED if request throws unexpected exception. - - The one wakes up main thread. - """ - assert self.status == WAIT_PROCESS - if not all_ok: - self.close() - self.wake_up() - return - self.len = '' - if len(message) == 0: - # it was a oneway request, do not write answer - self.message = '' - self.status = WAIT_LEN - else: - self.message = struct.pack('!i', len(message)) + message - self.status = SEND_ANSWER - self.wake_up() - - @locked - def is_writeable(self): - """Return True if connection should be added to write list of select""" - return self.status == SEND_ANSWER - - # it's not necessary, but... - @locked - def is_readable(self): - """Return True if connection should be added to read list of select""" - return self.status in (WAIT_LEN, WAIT_MESSAGE) - - @locked - def is_closed(self): - """Returns True if connection is closed.""" - return self.status == CLOSED - - def fileno(self): - """Returns the file descriptor of the associated socket.""" - return self.socket.fileno() - - def close(self): - """Closes connection""" - self.status = CLOSED - self.socket.close() - - -class TNonblockingServer: - """Non-blocking server.""" - - def __init__(self, - processor, - lsocket, - inputProtocolFactory=None, - outputProtocolFactory=None, - threads=10): - self.processor = processor - self.socket = lsocket - self.in_protocol = inputProtocolFactory or TBinaryProtocolFactory() - self.out_protocol = outputProtocolFactory or self.in_protocol - self.threads = int(threads) - self.clients = {} - self.tasks = Queue.Queue() - self._read, self._write = socket.socketpair() - self.prepared = False - self._stop = False - - def setNumThreads(self, num): - """Set the number of worker threads that should be created.""" - # implement ThreadPool interface - assert not self.prepared, "Can't change number of threads after start" - self.threads = num - - def prepare(self): - """Prepares server for serve requests.""" - if self.prepared: - return - self.socket.listen() - for _ in xrange(self.threads): - thread = Worker(self.tasks) - thread.setDaemon(True) - thread.start() - self.prepared = True - - def wake_up(self): - """Wake up main thread. - - The server usualy waits in select call in we should terminate one. - The simplest way is using socketpair. - - Select always wait to read from the first socket of socketpair. - - In this case, we can just write anything to the second socket from - socketpair. - """ - self._write.send('1') - - def stop(self): - """Stop the server. - - This method causes the serve() method to return. stop() may be invoked - from within your handler, or from another thread. - - After stop() is called, serve() will return but the server will still - be listening on the socket. serve() may then be called again to resume - processing requests. Alternatively, close() may be called after - serve() returns to close the server socket and shutdown all worker - threads. - """ - self._stop = True - self.wake_up() - - def _select(self): - """Does select on open connections.""" - readable = [self.socket.handle.fileno(), self._read.fileno()] - writable = [] - for i, connection in self.clients.items(): - if connection.is_readable(): - readable.append(connection.fileno()) - if connection.is_writeable(): - writable.append(connection.fileno()) - if connection.is_closed(): - del self.clients[i] - return select.select(readable, writable, readable) - - def handle(self): - """Handle requests. - - WARNING! You must call prepare() BEFORE calling handle() - """ - assert self.prepared, "You have to call prepare before handle" - rset, wset, xset = self._select() - for readable in rset: - if readable == self._read.fileno(): - # don't care i just need to clean readable flag - self._read.recv(1024) - elif readable == self.socket.handle.fileno(): - client = self.socket.accept().handle - self.clients[client.fileno()] = Connection(client, - self.wake_up) - else: - connection = self.clients[readable] - connection.read() - if connection.status == WAIT_PROCESS: - itransport = TTransport.TMemoryBuffer(connection.message) - otransport = TTransport.TMemoryBuffer() - iprot = self.in_protocol.getProtocol(itransport) - oprot = self.out_protocol.getProtocol(otransport) - self.tasks.put([self.processor, iprot, oprot, - otransport, connection.ready]) - for writeable in wset: - self.clients[writeable].write() - for oob in xset: - self.clients[oob].close() - del self.clients[oob] - - def close(self): - """Closes the server.""" - for _ in xrange(self.threads): - self.tasks.put([None, None, None, None, None]) - self.socket.close() - self.prepared = False - - def serve(self): - """Serve requests. - - Serve requests forever, or until stop() is called. - """ - self._stop = False - self.prepare() - while not self._stop: - self.handle() diff --git a/pyload/lib/thrift/server/TProcessPoolServer.py b/pyload/lib/thrift/server/TProcessPoolServer.py deleted file mode 100644 index 7a695a883..000000000 --- a/pyload/lib/thrift/server/TProcessPoolServer.py +++ /dev/null @@ -1,118 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - - -import logging -from multiprocessing import Process, Value, Condition, reduction - -from TServer import TServer -from thrift.transport.TTransport import TTransportException - - -class TProcessPoolServer(TServer): - """Server with a fixed size pool of worker subprocesses to service requests - - Note that if you need shared state between the handlers - it's up to you! - Written by Dvir Volk, doat.com - """ - def __init__(self, *args): - TServer.__init__(self, *args) - self.numWorkers = 10 - self.workers = [] - self.isRunning = Value('b', False) - self.stopCondition = Condition() - self.postForkCallback = None - - def setPostForkCallback(self, callback): - if not callable(callback): - raise TypeError("This is not a callback!") - self.postForkCallback = callback - - def setNumWorkers(self, num): - """Set the number of worker threads that should be created""" - self.numWorkers = num - - def workerProcess(self): - """Loop getting clients from the shared queue and process them""" - if self.postForkCallback: - self.postForkCallback() - - while self.isRunning.value: - try: - client = self.serverTransport.accept() - self.serveClient(client) - except (KeyboardInterrupt, SystemExit): - return 0 - except Exception, x: - logging.exception(x) - - def serveClient(self, client): - """Process input/output from a client for as long as possible""" - itrans = self.inputTransportFactory.getTransport(client) - otrans = self.outputTransportFactory.getTransport(client) - iprot = self.inputProtocolFactory.getProtocol(itrans) - oprot = self.outputProtocolFactory.getProtocol(otrans) - - try: - while True: - self.processor.process(iprot, oprot) - except TTransportException, tx: - pass - except Exception, x: - logging.exception(x) - - itrans.close() - otrans.close() - - def serve(self): - """Start workers and put into queue""" - # this is a shared state that can tell the workers to exit when False - self.isRunning.value = True - - # first bind and listen to the port - self.serverTransport.listen() - - # fork the children - for i in range(self.numWorkers): - try: - w = Process(target=self.workerProcess) - w.daemon = True - w.start() - self.workers.append(w) - except Exception, x: - logging.exception(x) - - # wait until the condition is set by stop() - while True: - self.stopCondition.acquire() - try: - self.stopCondition.wait() - break - except (SystemExit, KeyboardInterrupt): - break - except Exception, x: - logging.exception(x) - - self.isRunning.value = False - - def stop(self): - self.isRunning.value = False - self.stopCondition.acquire() - self.stopCondition.notify() - self.stopCondition.release() diff --git a/pyload/lib/thrift/server/TServer.py b/pyload/lib/thrift/server/TServer.py deleted file mode 100644 index 2f24842c4..000000000 --- a/pyload/lib/thrift/server/TServer.py +++ /dev/null @@ -1,269 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -import Queue -import logging -import os -import sys -import threading -import traceback - -from thrift.Thrift import TProcessor -from thrift.protocol import TBinaryProtocol -from thrift.transport import TTransport - - -class TServer: - """Base interface for a server, which must have a serve() method. - - Three constructors for all servers: - 1) (processor, serverTransport) - 2) (processor, serverTransport, transportFactory, protocolFactory) - 3) (processor, serverTransport, - inputTransportFactory, outputTransportFactory, - inputProtocolFactory, outputProtocolFactory) - """ - def __init__(self, *args): - if (len(args) == 2): - self.__initArgs__(args[0], args[1], - TTransport.TTransportFactoryBase(), - TTransport.TTransportFactoryBase(), - TBinaryProtocol.TBinaryProtocolFactory(), - TBinaryProtocol.TBinaryProtocolFactory()) - elif (len(args) == 4): - self.__initArgs__(args[0], args[1], args[2], args[2], args[3], args[3]) - elif (len(args) == 6): - self.__initArgs__(args[0], args[1], args[2], args[3], args[4], args[5]) - - def __initArgs__(self, processor, serverTransport, - inputTransportFactory, outputTransportFactory, - inputProtocolFactory, outputProtocolFactory): - self.processor = processor - self.serverTransport = serverTransport - self.inputTransportFactory = inputTransportFactory - self.outputTransportFactory = outputTransportFactory - self.inputProtocolFactory = inputProtocolFactory - self.outputProtocolFactory = outputProtocolFactory - - def serve(self): - pass - - -class TSimpleServer(TServer): - """Simple single-threaded server that just pumps around one transport.""" - - def __init__(self, *args): - TServer.__init__(self, *args) - - def serve(self): - self.serverTransport.listen() - while True: - client = self.serverTransport.accept() - itrans = self.inputTransportFactory.getTransport(client) - otrans = self.outputTransportFactory.getTransport(client) - iprot = self.inputProtocolFactory.getProtocol(itrans) - oprot = self.outputProtocolFactory.getProtocol(otrans) - try: - while True: - self.processor.process(iprot, oprot) - except TTransport.TTransportException, tx: - pass - except Exception, x: - logging.exception(x) - - itrans.close() - otrans.close() - - -class TThreadedServer(TServer): - """Threaded server that spawns a new thread per each connection.""" - - def __init__(self, *args, **kwargs): - TServer.__init__(self, *args) - self.daemon = kwargs.get("daemon", False) - - def serve(self): - self.serverTransport.listen() - while True: - try: - client = self.serverTransport.accept() - t = threading.Thread(target=self.handle, args=(client,)) - t.setDaemon(self.daemon) - t.start() - except KeyboardInterrupt: - raise - except Exception, x: - logging.exception(x) - - def handle(self, client): - itrans = self.inputTransportFactory.getTransport(client) - otrans = self.outputTransportFactory.getTransport(client) - iprot = self.inputProtocolFactory.getProtocol(itrans) - oprot = self.outputProtocolFactory.getProtocol(otrans) - try: - while True: - self.processor.process(iprot, oprot) - except TTransport.TTransportException, tx: - pass - except Exception, x: - logging.exception(x) - - itrans.close() - otrans.close() - - -class TThreadPoolServer(TServer): - """Server with a fixed size pool of threads which service requests.""" - - def __init__(self, *args, **kwargs): - TServer.__init__(self, *args) - self.clients = Queue.Queue() - self.threads = 10 - self.daemon = kwargs.get("daemon", False) - - def setNumThreads(self, num): - """Set the number of worker threads that should be created""" - self.threads = num - - def serveThread(self): - """Loop around getting clients from the shared queue and process them.""" - while True: - try: - client = self.clients.get() - self.serveClient(client) - except Exception, x: - logging.exception(x) - - def serveClient(self, client): - """Process input/output from a client for as long as possible""" - itrans = self.inputTransportFactory.getTransport(client) - otrans = self.outputTransportFactory.getTransport(client) - iprot = self.inputProtocolFactory.getProtocol(itrans) - oprot = self.outputProtocolFactory.getProtocol(otrans) - try: - while True: - self.processor.process(iprot, oprot) - except TTransport.TTransportException, tx: - pass - except Exception, x: - logging.exception(x) - - itrans.close() - otrans.close() - - def serve(self): - """Start a fixed number of worker threads and put client into a queue""" - for i in range(self.threads): - try: - t = threading.Thread(target=self.serveThread) - t.setDaemon(self.daemon) - t.start() - except Exception, x: - logging.exception(x) - - # Pump the socket for clients - self.serverTransport.listen() - while True: - try: - client = self.serverTransport.accept() - self.clients.put(client) - except Exception, x: - logging.exception(x) - - -class TForkingServer(TServer): - """A Thrift server that forks a new process for each request - - This is more scalable than the threaded server as it does not cause - GIL contention. - - Note that this has different semantics from the threading server. - Specifically, updates to shared variables will no longer be shared. - It will also not work on windows. - - This code is heavily inspired by SocketServer.ForkingMixIn in the - Python stdlib. - """ - def __init__(self, *args): - TServer.__init__(self, *args) - self.children = [] - - def serve(self): - def try_close(file): - try: - file.close() - except IOError, e: - logging.warning(e, exc_info=True) - - self.serverTransport.listen() - while True: - client = self.serverTransport.accept() - try: - pid = os.fork() - - if pid: # parent - # add before collect, otherwise you race w/ waitpid - self.children.append(pid) - self.collect_children() - - # Parent must close socket or the connection may not get - # closed promptly - itrans = self.inputTransportFactory.getTransport(client) - otrans = self.outputTransportFactory.getTransport(client) - try_close(itrans) - try_close(otrans) - else: - itrans = self.inputTransportFactory.getTransport(client) - otrans = self.outputTransportFactory.getTransport(client) - - iprot = self.inputProtocolFactory.getProtocol(itrans) - oprot = self.outputProtocolFactory.getProtocol(otrans) - - ecode = 0 - try: - try: - while True: - self.processor.process(iprot, oprot) - except TTransport.TTransportException, tx: - pass - except Exception, e: - logging.exception(e) - ecode = 1 - finally: - try_close(itrans) - try_close(otrans) - - os._exit(ecode) - - except TTransport.TTransportException, tx: - pass - except Exception, x: - logging.exception(x) - - def collect_children(self): - while self.children: - try: - pid, status = os.waitpid(0, os.WNOHANG) - except os.error: - pid = None - - if pid: - self.children.remove(pid) - else: - break diff --git a/pyload/lib/thrift/server/__init__.py b/pyload/lib/thrift/server/__init__.py deleted file mode 100644 index 1bf6e254e..000000000 --- a/pyload/lib/thrift/server/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -__all__ = ['TServer', 'TNonblockingServer'] diff --git a/pyload/lib/thrift/transport/THttpClient.py b/pyload/lib/thrift/transport/THttpClient.py deleted file mode 100644 index ea80a1ae8..000000000 --- a/pyload/lib/thrift/transport/THttpClient.py +++ /dev/null @@ -1,149 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -import httplib -import os -import socket -import sys -import urllib -import urlparse -import warnings - -from cStringIO import StringIO - -from TTransport import * - - -class THttpClient(TTransportBase): - """Http implementation of TTransport base.""" - - def __init__(self, uri_or_host, port=None, path=None): - """THttpClient supports two different types constructor parameters. - - THttpClient(host, port, path) - deprecated - THttpClient(uri) - - Only the second supports https. - """ - if port is not None: - warnings.warn( - "Please use the THttpClient('http://host:port/path') syntax", - DeprecationWarning, - stacklevel=2) - self.host = uri_or_host - self.port = port - assert path - self.path = path - self.scheme = 'http' - else: - parsed = urlparse.urlparse(uri_or_host) - self.scheme = parsed.scheme - assert self.scheme in ('http', 'https') - if self.scheme == 'http': - self.port = parsed.port or httplib.HTTP_PORT - elif self.scheme == 'https': - self.port = parsed.port or httplib.HTTPS_PORT - self.host = parsed.hostname - self.path = parsed.path - if parsed.query: - self.path += '?%s' % parsed.query - self.__wbuf = StringIO() - self.__http = None - self.__timeout = None - self.__custom_headers = None - - def open(self): - if self.scheme == 'http': - self.__http = httplib.HTTP(self.host, self.port) - else: - self.__http = httplib.HTTPS(self.host, self.port) - - def close(self): - self.__http.close() - self.__http = None - - def isOpen(self): - return self.__http is not None - - def setTimeout(self, ms): - if not hasattr(socket, 'getdefaulttimeout'): - raise NotImplementedError - - if ms is None: - self.__timeout = None - else: - self.__timeout = ms / 1000.0 - - def setCustomHeaders(self, headers): - self.__custom_headers = headers - - def read(self, sz): - return self.__http.file.read(sz) - - def write(self, buf): - self.__wbuf.write(buf) - - def __withTimeout(f): - def _f(*args, **kwargs): - orig_timeout = socket.getdefaulttimeout() - socket.setdefaulttimeout(args[0].__timeout) - result = f(*args, **kwargs) - socket.setdefaulttimeout(orig_timeout) - return result - return _f - - def flush(self): - if self.isOpen(): - self.close() - self.open() - - # Pull data out of buffer - data = self.__wbuf.getvalue() - self.__wbuf = StringIO() - - # HTTP request - self.__http.putrequest('POST', self.path) - - # Write headers - self.__http.putheader('Host', self.host) - self.__http.putheader('Content-Type', 'application/x-thrift') - self.__http.putheader('Content-Length', str(len(data))) - - if not self.__custom_headers or 'User-Agent' not in self.__custom_headers: - user_agent = 'Python/THttpClient' - script = os.path.basename(sys.argv[0]) - if script: - user_agent = '%s (%s)' % (user_agent, urllib.quote(script)) - self.__http.putheader('User-Agent', user_agent) - - if self.__custom_headers: - for key, val in self.__custom_headers.iteritems(): - self.__http.putheader(key, val) - - self.__http.endheaders() - - # Write payload - self.__http.send(data) - - # Get reply to flush the request - self.code, self.message, self.headers = self.__http.getreply() - - # Decorate if we know how to timeout - if hasattr(socket, 'getdefaulttimeout'): - flush = __withTimeout(flush) diff --git a/pyload/lib/thrift/transport/TSSLSocket.py b/pyload/lib/thrift/transport/TSSLSocket.py deleted file mode 100644 index 81e098426..000000000 --- a/pyload/lib/thrift/transport/TSSLSocket.py +++ /dev/null @@ -1,214 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -import os -import socket -import ssl - -from thrift.transport import TSocket -from thrift.transport.TTransport import TTransportException - - -class TSSLSocket(TSocket.TSocket): - """ - SSL implementation of client-side TSocket - - This class creates outbound sockets wrapped using the - python standard ssl module for encrypted connections. - - The protocol used is set using the class variable - SSL_VERSION, which must be one of ssl.PROTOCOL_* and - defaults to ssl.PROTOCOL_TLSv1 for greatest security. - """ - SSL_VERSION = ssl.PROTOCOL_TLSv1 - - def __init__(self, - host='localhost', - port=9090, - validate=True, - ca_certs=None, - keyfile=None, - certfile=None, - unix_socket=None): - """Create SSL TSocket - - @param validate: Set to False to disable SSL certificate validation - @type validate: bool - @param ca_certs: Filename to the Certificate Authority pem file, possibly a - file downloaded from: http://curl.haxx.se/ca/cacert.pem This is passed to - the ssl_wrap function as the 'ca_certs' parameter. - @type ca_certs: str - @param keyfile: The private key - @type keyfile: str - @param certfile: The cert file - @type certfile: str - - Raises an IOError exception if validate is True and the ca_certs file is - None, not present or unreadable. - """ - self.validate = validate - self.is_valid = False - self.peercert = None - if not validate: - self.cert_reqs = ssl.CERT_NONE - else: - self.cert_reqs = ssl.CERT_REQUIRED - self.ca_certs = ca_certs - self.keyfile = keyfile - self.certfile = certfile - if validate: - if ca_certs is None or not os.access(ca_certs, os.R_OK): - raise IOError('Certificate Authority ca_certs file "%s" ' - 'is not readable, cannot validate SSL ' - 'certificates.' % (ca_certs)) - TSocket.TSocket.__init__(self, host, port, unix_socket) - - def open(self): - try: - res0 = self._resolveAddr() - for res in res0: - sock_family, sock_type = res[0:2] - ip_port = res[4] - plain_sock = socket.socket(sock_family, sock_type) - self.handle = ssl.wrap_socket(plain_sock, - ssl_version=self.SSL_VERSION, - do_handshake_on_connect=True, - ca_certs=self.ca_certs, - keyfile=self.keyfile, - certfile=self.certfile, - cert_reqs=self.cert_reqs) - self.handle.settimeout(self._timeout) - try: - self.handle.connect(ip_port) - except socket.error, e: - if res is not res0[-1]: - continue - else: - raise e - break - except socket.error, e: - if self._unix_socket: - message = 'Could not connect to secure socket %s: %s' \ - % (self._unix_socket, e) - else: - message = 'Could not connect to %s:%d: %s' % (self.host, self.port, e) - raise TTransportException(type=TTransportException.NOT_OPEN, - message=message) - if self.validate: - self._validate_cert() - - def _validate_cert(self): - """internal method to validate the peer's SSL certificate, and to check the - commonName of the certificate to ensure it matches the hostname we - used to make this connection. Does not support subjectAltName records - in certificates. - - raises TTransportException if the certificate fails validation. - """ - cert = self.handle.getpeercert() - self.peercert = cert - if 'subject' not in cert: - raise TTransportException( - type=TTransportException.NOT_OPEN, - message='No SSL certificate found from %s:%s' % (self.host, self.port)) - fields = cert['subject'] - for field in fields: - # ensure structure we get back is what we expect - if not isinstance(field, tuple): - continue - cert_pair = field[0] - if len(cert_pair) < 2: - continue - cert_key, cert_value = cert_pair[0:2] - if cert_key != 'commonName': - continue - certhost = cert_value - # this check should be performed by some sort of Access Manager - if certhost == self.host: - # success, cert commonName matches desired hostname - self.is_valid = True - return - else: - raise TTransportException( - type=TTransportException.UNKNOWN, - message='Hostname we connected to "%s" doesn\'t match certificate ' - 'provided commonName "%s"' % (self.host, certhost)) - raise TTransportException( - type=TTransportException.UNKNOWN, - message='Could not validate SSL certificate from ' - 'host "%s". Cert=%s' % (self.host, cert)) - - -class TSSLServerSocket(TSocket.TServerSocket): - """SSL implementation of TServerSocket - - This uses the ssl module's wrap_socket() method to provide SSL - negotiated encryption. - """ - SSL_VERSION = ssl.PROTOCOL_TLSv1 - - def __init__(self, - host=None, - port=9090, - certfile='cert.pem', - unix_socket=None): - """Initialize a TSSLServerSocket - - @param certfile: filename of the server certificate, defaults to cert.pem - @type certfile: str - @param host: The hostname or IP to bind the listen socket to, - i.e. 'localhost' for only allowing local network connections. - Pass None to bind to all interfaces. - @type host: str - @param port: The port to listen on for inbound connections. - @type port: int - """ - self.setCertfile(certfile) - TSocket.TServerSocket.__init__(self, host, port) - - def setCertfile(self, certfile): - """Set or change the server certificate file used to wrap new connections. - - @param certfile: The filename of the server certificate, - i.e. '/etc/certs/server.pem' - @type certfile: str - - Raises an IOError exception if the certfile is not present or unreadable. - """ - if not os.access(certfile, os.R_OK): - raise IOError('No such certfile found: %s' % (certfile)) - self.certfile = certfile - - def accept(self): - plain_client, addr = self.handle.accept() - try: - client = ssl.wrap_socket(plain_client, certfile=self.certfile, - server_side=True, ssl_version=self.SSL_VERSION) - except ssl.SSLError, ssl_exc: - # failed handshake/ssl wrap, close socket to client - plain_client.close() - # raise ssl_exc - # We can't raise the exception, because it kills most TServer derived - # serve() methods. - # Instead, return None, and let the TServer instance deal with it in - # other exception handling. (but TSimpleServer dies anyway) - return None - result = TSocket.TSocket() - result.setHandle(client) - return result diff --git a/pyload/lib/thrift/transport/TSocket.py b/pyload/lib/thrift/transport/TSocket.py deleted file mode 100644 index 9e2b3849b..000000000 --- a/pyload/lib/thrift/transport/TSocket.py +++ /dev/null @@ -1,176 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -import errno -import os -import socket -import sys - -from TTransport import * - - -class TSocketBase(TTransportBase): - def _resolveAddr(self): - if self._unix_socket is not None: - return [(socket.AF_UNIX, socket.SOCK_STREAM, None, None, - self._unix_socket)] - else: - return socket.getaddrinfo(self.host, - self.port, - socket.AF_UNSPEC, - socket.SOCK_STREAM, - 0, - socket.AI_PASSIVE | socket.AI_ADDRCONFIG) - - def close(self): - if self.handle: - self.handle.close() - self.handle = None - - -class TSocket(TSocketBase): - """Socket implementation of TTransport base.""" - - def __init__(self, host='localhost', port=9090, unix_socket=None): - """Initialize a TSocket - - @param host(str) The host to connect to. - @param port(int) The (TCP) port to connect to. - @param unix_socket(str) The filename of a unix socket to connect to. - (host and port will be ignored.) - """ - self.host = host - self.port = port - self.handle = None - self._unix_socket = unix_socket - self._timeout = None - - def setHandle(self, h): - self.handle = h - - def isOpen(self): - return self.handle is not None - - def setTimeout(self, ms): - if ms is None: - self._timeout = None - else: - self._timeout = ms / 1000.0 - - if self.handle is not None: - self.handle.settimeout(self._timeout) - - def open(self): - try: - res0 = self._resolveAddr() - for res in res0: - self.handle = socket.socket(res[0], res[1]) - self.handle.settimeout(self._timeout) - try: - self.handle.connect(res[4]) - except socket.error, e: - if res is not res0[-1]: - continue - else: - raise e - break - except socket.error, e: - if self._unix_socket: - message = 'Could not connect to socket %s' % self._unix_socket - else: - message = 'Could not connect to %s:%d' % (self.host, self.port) - raise TTransportException(type=TTransportException.NOT_OPEN, - message=message) - - def read(self, sz): - try: - buff = self.handle.recv(sz) - except socket.error, e: - if (e.args[0] == errno.ECONNRESET and - (sys.platform == 'darwin' or sys.platform.startswith('freebsd'))): - # freebsd and Mach don't follow POSIX semantic of recv - # and fail with ECONNRESET if peer performed shutdown. - # See corresponding comment and code in TSocket::read() - # in lib/cpp/src/transport/TSocket.cpp. - self.close() - # Trigger the check to raise the END_OF_FILE exception below. - buff = '' - else: - raise - if len(buff) == 0: - raise TTransportException(type=TTransportException.END_OF_FILE, - message='TSocket read 0 bytes') - return buff - - def write(self, buff): - if not self.handle: - raise TTransportException(type=TTransportException.NOT_OPEN, - message='Transport not open') - sent = 0 - have = len(buff) - while sent < have: - plus = self.handle.send(buff) - if plus == 0: - raise TTransportException(type=TTransportException.END_OF_FILE, - message='TSocket sent 0 bytes') - sent += plus - buff = buff[plus:] - - def flush(self): - pass - - -class TServerSocket(TSocketBase, TServerTransportBase): - """Socket implementation of TServerTransport base.""" - - def __init__(self, host=None, port=9090, unix_socket=None): - self.host = host - self.port = port - self._unix_socket = unix_socket - self.handle = None - - def listen(self): - res0 = self._resolveAddr() - for res in res0: - if res[0] is socket.AF_INET6 or res is res0[-1]: - break - - # We need remove the old unix socket if the file exists and - # nobody is listening on it. - if self._unix_socket: - tmp = socket.socket(res[0], res[1]) - try: - tmp.connect(res[4]) - except socket.error, err: - eno, message = err.args - if eno == errno.ECONNREFUSED: - os.unlink(res[4]) - - self.handle = socket.socket(res[0], res[1]) - self.handle.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if hasattr(self.handle, 'settimeout'): - self.handle.settimeout(None) - self.handle.bind(res[4]) - self.handle.listen(128) - - def accept(self): - client, addr = self.handle.accept() - result = TSocket() - result.setHandle(client) - return result diff --git a/pyload/lib/thrift/transport/TTransport.py b/pyload/lib/thrift/transport/TTransport.py deleted file mode 100644 index 4481371a6..000000000 --- a/pyload/lib/thrift/transport/TTransport.py +++ /dev/null @@ -1,330 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -from cStringIO import StringIO -from struct import pack, unpack -from thrift.Thrift import TException - - -class TTransportException(TException): - """Custom Transport Exception class""" - - UNKNOWN = 0 - NOT_OPEN = 1 - ALREADY_OPEN = 2 - TIMED_OUT = 3 - END_OF_FILE = 4 - - def __init__(self, type=UNKNOWN, message=None): - TException.__init__(self, message) - self.type = type - - -class TTransportBase: - """Base class for Thrift transport layer.""" - - def isOpen(self): - pass - - def open(self): - pass - - def close(self): - pass - - def read(self, sz): - pass - - def readAll(self, sz): - buff = '' - have = 0 - while (have < sz): - chunk = self.read(sz - have) - have += len(chunk) - buff += chunk - - if len(chunk) == 0: - raise EOFError() - - return buff - - def write(self, buf): - pass - - def flush(self): - pass - - -# This class should be thought of as an interface. -class CReadableTransport: - """base class for transports that are readable from C""" - - # TODO(dreiss): Think about changing this interface to allow us to use - # a (Python, not c) StringIO instead, because it allows - # you to write after reading. - - # NOTE: This is a classic class, so properties will NOT work - # correctly for setting. - @property - def cstringio_buf(self): - """A cStringIO buffer that contains the current chunk we are reading.""" - pass - - def cstringio_refill(self, partialread, reqlen): - """Refills cstringio_buf. - - Returns the currently used buffer (which can but need not be the same as - the old cstringio_buf). partialread is what the C code has read from the - buffer, and should be inserted into the buffer before any more reads. The - return value must be a new, not borrowed reference. Something along the - lines of self._buf should be fine. - - If reqlen bytes can't be read, throw EOFError. - """ - pass - - -class TServerTransportBase: - """Base class for Thrift server transports.""" - - def listen(self): - pass - - def accept(self): - pass - - def close(self): - pass - - -class TTransportFactoryBase: - """Base class for a Transport Factory""" - - def getTransport(self, trans): - return trans - - -class TBufferedTransportFactory: - """Factory transport that builds buffered transports""" - - def getTransport(self, trans): - buffered = TBufferedTransport(trans) - return buffered - - -class TBufferedTransport(TTransportBase, CReadableTransport): - """Class that wraps another transport and buffers its I/O. - - The implementation uses a (configurable) fixed-size read buffer - but buffers all writes until a flush is performed. - """ - DEFAULT_BUFFER = 4096 - - def __init__(self, trans, rbuf_size=DEFAULT_BUFFER): - self.__trans = trans - self.__wbuf = StringIO() - self.__rbuf = StringIO("") - self.__rbuf_size = rbuf_size - - def isOpen(self): - return self.__trans.isOpen() - - def open(self): - return self.__trans.open() - - def close(self): - return self.__trans.close() - - def read(self, sz): - ret = self.__rbuf.read(sz) - if len(ret) != 0: - return ret - - self.__rbuf = StringIO(self.__trans.read(max(sz, self.__rbuf_size))) - return self.__rbuf.read(sz) - - def write(self, buf): - self.__wbuf.write(buf) - - def flush(self): - out = self.__wbuf.getvalue() - # reset wbuf before write/flush to preserve state on underlying failure - self.__wbuf = StringIO() - self.__trans.write(out) - self.__trans.flush() - - # Implement the CReadableTransport interface. - @property - def cstringio_buf(self): - return self.__rbuf - - def cstringio_refill(self, partialread, reqlen): - retstring = partialread - if reqlen < self.__rbuf_size: - # try to make a read of as much as we can. - retstring += self.__trans.read(self.__rbuf_size) - - # but make sure we do read reqlen bytes. - if len(retstring) < reqlen: - retstring += self.__trans.readAll(reqlen - len(retstring)) - - self.__rbuf = StringIO(retstring) - return self.__rbuf - - -class TMemoryBuffer(TTransportBase, CReadableTransport): - """Wraps a cStringIO object as a TTransport. - - NOTE: Unlike the C++ version of this class, you cannot write to it - then immediately read from it. If you want to read from a - TMemoryBuffer, you must either pass a string to the constructor. - TODO(dreiss): Make this work like the C++ version. - """ - - def __init__(self, value=None): - """value -- a value to read from for stringio - - If value is set, this will be a transport for reading, - otherwise, it is for writing""" - if value is not None: - self._buffer = StringIO(value) - else: - self._buffer = StringIO() - - def isOpen(self): - return not self._buffer.closed - - def open(self): - pass - - def close(self): - self._buffer.close() - - def read(self, sz): - return self._buffer.read(sz) - - def write(self, buf): - self._buffer.write(buf) - - def flush(self): - pass - - def getvalue(self): - return self._buffer.getvalue() - - # Implement the CReadableTransport interface. - @property - def cstringio_buf(self): - return self._buffer - - def cstringio_refill(self, partialread, reqlen): - # only one shot at reading... - raise EOFError() - - -class TFramedTransportFactory: - """Factory transport that builds framed transports""" - - def getTransport(self, trans): - framed = TFramedTransport(trans) - return framed - - -class TFramedTransport(TTransportBase, CReadableTransport): - """Class that wraps another transport and frames its I/O when writing.""" - - def __init__(self, trans,): - self.__trans = trans - self.__rbuf = StringIO() - self.__wbuf = StringIO() - - def isOpen(self): - return self.__trans.isOpen() - - def open(self): - return self.__trans.open() - - def close(self): - return self.__trans.close() - - def read(self, sz): - ret = self.__rbuf.read(sz) - if len(ret) != 0: - return ret - - self.readFrame() - return self.__rbuf.read(sz) - - def readFrame(self): - buff = self.__trans.readAll(4) - sz, = unpack('!i', buff) - self.__rbuf = StringIO(self.__trans.readAll(sz)) - - def write(self, buf): - self.__wbuf.write(buf) - - def flush(self): - wout = self.__wbuf.getvalue() - wsz = len(wout) - # reset wbuf before write/flush to preserve state on underlying failure - self.__wbuf = StringIO() - # N.B.: Doing this string concatenation is WAY cheaper than making - # two separate calls to the underlying socket object. Socket writes in - # Python turn out to be REALLY expensive, but it seems to do a pretty - # good job of managing string buffer operations without excessive copies - buf = pack("!i", wsz) + wout - self.__trans.write(buf) - self.__trans.flush() - - # Implement the CReadableTransport interface. - @property - def cstringio_buf(self): - return self.__rbuf - - def cstringio_refill(self, prefix, reqlen): - # self.__rbuf will already be empty here because fastbinary doesn't - # ask for a refill until the previous buffer is empty. Therefore, - # we can start reading new frames immediately. - while len(prefix) < reqlen: - self.readFrame() - prefix += self.__rbuf.getvalue() - self.__rbuf = StringIO(prefix) - return self.__rbuf - - -class TFileObjectTransport(TTransportBase): - """Wraps a file-like object to make it work as a Thrift transport.""" - - def __init__(self, fileobj): - self.fileobj = fileobj - - def isOpen(self): - return True - - def close(self): - self.fileobj.close() - - def read(self, sz): - return self.fileobj.read(sz) - - def write(self, buf): - self.fileobj.write(buf) - - def flush(self): - self.fileobj.flush() diff --git a/pyload/lib/thrift/transport/TTwisted.py b/pyload/lib/thrift/transport/TTwisted.py deleted file mode 100644 index 3ce3eb220..000000000 --- a/pyload/lib/thrift/transport/TTwisted.py +++ /dev/null @@ -1,221 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -from cStringIO import StringIO - -from zope.interface import implements, Interface, Attribute -from twisted.internet.protocol import Protocol, ServerFactory, ClientFactory, \ - connectionDone -from twisted.internet import defer -from twisted.protocols import basic -from twisted.python import log -from twisted.web import server, resource, http - -from thrift.transport import TTransport - - -class TMessageSenderTransport(TTransport.TTransportBase): - - def __init__(self): - self.__wbuf = StringIO() - - def write(self, buf): - self.__wbuf.write(buf) - - def flush(self): - msg = self.__wbuf.getvalue() - self.__wbuf = StringIO() - self.sendMessage(msg) - - def sendMessage(self, message): - raise NotImplementedError - - -class TCallbackTransport(TMessageSenderTransport): - - def __init__(self, func): - TMessageSenderTransport.__init__(self) - self.func = func - - def sendMessage(self, message): - self.func(message) - - -class ThriftClientProtocol(basic.Int32StringReceiver): - - MAX_LENGTH = 2 ** 31 - 1 - - def __init__(self, client_class, iprot_factory, oprot_factory=None): - self._client_class = client_class - self._iprot_factory = iprot_factory - if oprot_factory is None: - self._oprot_factory = iprot_factory - else: - self._oprot_factory = oprot_factory - - self.recv_map = {} - self.started = defer.Deferred() - - def dispatch(self, msg): - self.sendString(msg) - - def connectionMade(self): - tmo = TCallbackTransport(self.dispatch) - self.client = self._client_class(tmo, self._oprot_factory) - self.started.callback(self.client) - - def connectionLost(self, reason=connectionDone): - for k, v in self.client._reqs.iteritems(): - tex = TTransport.TTransportException( - type=TTransport.TTransportException.END_OF_FILE, - message='Connection closed') - v.errback(tex) - - def stringReceived(self, frame): - tr = TTransport.TMemoryBuffer(frame) - iprot = self._iprot_factory.getProtocol(tr) - (fname, mtype, rseqid) = iprot.readMessageBegin() - - try: - method = self.recv_map[fname] - except KeyError: - method = getattr(self.client, 'recv_' + fname) - self.recv_map[fname] = method - - method(iprot, mtype, rseqid) - - -class ThriftServerProtocol(basic.Int32StringReceiver): - - MAX_LENGTH = 2 ** 31 - 1 - - def dispatch(self, msg): - self.sendString(msg) - - def processError(self, error): - self.transport.loseConnection() - - def processOk(self, _, tmo): - msg = tmo.getvalue() - - if len(msg) > 0: - self.dispatch(msg) - - def stringReceived(self, frame): - tmi = TTransport.TMemoryBuffer(frame) - tmo = TTransport.TMemoryBuffer() - - iprot = self.factory.iprot_factory.getProtocol(tmi) - oprot = self.factory.oprot_factory.getProtocol(tmo) - - d = self.factory.processor.process(iprot, oprot) - d.addCallbacks(self.processOk, self.processError, - callbackArgs=(tmo,)) - - -class IThriftServerFactory(Interface): - - processor = Attribute("Thrift processor") - - iprot_factory = Attribute("Input protocol factory") - - oprot_factory = Attribute("Output protocol factory") - - -class IThriftClientFactory(Interface): - - client_class = Attribute("Thrift client class") - - iprot_factory = Attribute("Input protocol factory") - - oprot_factory = Attribute("Output protocol factory") - - -class ThriftServerFactory(ServerFactory): - - implements(IThriftServerFactory) - - protocol = ThriftServerProtocol - - def __init__(self, processor, iprot_factory, oprot_factory=None): - self.processor = processor - self.iprot_factory = iprot_factory - if oprot_factory is None: - self.oprot_factory = iprot_factory - else: - self.oprot_factory = oprot_factory - - -class ThriftClientFactory(ClientFactory): - - implements(IThriftClientFactory) - - protocol = ThriftClientProtocol - - def __init__(self, client_class, iprot_factory, oprot_factory=None): - self.client_class = client_class - self.iprot_factory = iprot_factory - if oprot_factory is None: - self.oprot_factory = iprot_factory - else: - self.oprot_factory = oprot_factory - - def buildProtocol(self, addr): - p = self.protocol(self.client_class, self.iprot_factory, - self.oprot_factory) - p.factory = self - return p - - -class ThriftResource(resource.Resource): - - allowedMethods = ('POST',) - - def __init__(self, processor, inputProtocolFactory, - outputProtocolFactory=None): - resource.Resource.__init__(self) - self.inputProtocolFactory = inputProtocolFactory - if outputProtocolFactory is None: - self.outputProtocolFactory = inputProtocolFactory - else: - self.outputProtocolFactory = outputProtocolFactory - self.processor = processor - - def getChild(self, path, request): - return self - - def _cbProcess(self, _, request, tmo): - msg = tmo.getvalue() - request.setResponseCode(http.OK) - request.setHeader("content-type", "application/x-thrift") - request.write(msg) - request.finish() - - def render_POST(self, request): - request.content.seek(0, 0) - data = request.content.read() - tmi = TTransport.TMemoryBuffer(data) - tmo = TTransport.TMemoryBuffer() - - iprot = self.inputProtocolFactory.getProtocol(tmi) - oprot = self.outputProtocolFactory.getProtocol(tmo) - - d = self.processor.process(iprot, oprot) - d.addCallback(self._cbProcess, request, tmo) - return server.NOT_DONE_YET diff --git a/pyload/lib/thrift/transport/TZlibTransport.py b/pyload/lib/thrift/transport/TZlibTransport.py deleted file mode 100644 index a2f42a5d2..000000000 --- a/pyload/lib/thrift/transport/TZlibTransport.py +++ /dev/null @@ -1,248 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -"""TZlibTransport provides a compressed transport and transport factory -class, using the python standard library zlib module to implement -data compression. -""" - -from __future__ import division -import zlib -from cStringIO import StringIO -from TTransport import TTransportBase, CReadableTransport - - -class TZlibTransportFactory(object): - """Factory transport that builds zlib compressed transports. - - This factory caches the last single client/transport that it was passed - and returns the same TZlibTransport object that was created. - - This caching means the TServer class will get the _same_ transport - object for both input and output transports from this factory. - (For non-threaded scenarios only, since the cache only holds one object) - - The purpose of this caching is to allocate only one TZlibTransport where - only one is really needed (since it must have separate read/write buffers), - and makes the statistics from getCompSavings() and getCompRatio() - easier to understand. - """ - # class scoped cache of last transport given and zlibtransport returned - _last_trans = None - _last_z = None - - def getTransport(self, trans, compresslevel=9): - """Wrap a transport, trans, with the TZlibTransport - compressed transport class, returning a new - transport to the caller. - - @param compresslevel: The zlib compression level, ranging - from 0 (no compression) to 9 (best compression). Defaults to 9. - @type compresslevel: int - - This method returns a TZlibTransport which wraps the - passed C{trans} TTransport derived instance. - """ - if trans == self._last_trans: - return self._last_z - ztrans = TZlibTransport(trans, compresslevel) - self._last_trans = trans - self._last_z = ztrans - return ztrans - - -class TZlibTransport(TTransportBase, CReadableTransport): - """Class that wraps a transport with zlib, compressing writes - and decompresses reads, using the python standard - library zlib module. - """ - # Read buffer size for the python fastbinary C extension, - # the TBinaryProtocolAccelerated class. - DEFAULT_BUFFSIZE = 4096 - - def __init__(self, trans, compresslevel=9): - """Create a new TZlibTransport, wrapping C{trans}, another - TTransport derived object. - - @param trans: A thrift transport object, i.e. a TSocket() object. - @type trans: TTransport - @param compresslevel: The zlib compression level, ranging - from 0 (no compression) to 9 (best compression). Default is 9. - @type compresslevel: int - """ - self.__trans = trans - self.compresslevel = compresslevel - self.__rbuf = StringIO() - self.__wbuf = StringIO() - self._init_zlib() - self._init_stats() - - def _reinit_buffers(self): - """Internal method to initialize/reset the internal StringIO objects - for read and write buffers. - """ - self.__rbuf = StringIO() - self.__wbuf = StringIO() - - def _init_stats(self): - """Internal method to reset the internal statistics counters - for compression ratios and bandwidth savings. - """ - self.bytes_in = 0 - self.bytes_out = 0 - self.bytes_in_comp = 0 - self.bytes_out_comp = 0 - - def _init_zlib(self): - """Internal method for setting up the zlib compression and - decompression objects. - """ - self._zcomp_read = zlib.decompressobj() - self._zcomp_write = zlib.compressobj(self.compresslevel) - - def getCompRatio(self): - """Get the current measured compression ratios (in,out) from - this transport. - - Returns a tuple of: - (inbound_compression_ratio, outbound_compression_ratio) - - The compression ratios are computed as: - compressed / uncompressed - - E.g., data that compresses by 10x will have a ratio of: 0.10 - and data that compresses to half of ts original size will - have a ratio of 0.5 - - None is returned if no bytes have yet been processed in - a particular direction. - """ - r_percent, w_percent = (None, None) - if self.bytes_in > 0: - r_percent = self.bytes_in_comp / self.bytes_in - if self.bytes_out > 0: - w_percent = self.bytes_out_comp / self.bytes_out - return (r_percent, w_percent) - - def getCompSavings(self): - """Get the current count of saved bytes due to data - compression. - - Returns a tuple of: - (inbound_saved_bytes, outbound_saved_bytes) - - Note: if compression is actually expanding your - data (only likely with very tiny thrift objects), then - the values returned will be negative. - """ - r_saved = self.bytes_in - self.bytes_in_comp - w_saved = self.bytes_out - self.bytes_out_comp - return (r_saved, w_saved) - - def isOpen(self): - """Return the underlying transport's open status""" - return self.__trans.isOpen() - - def open(self): - """Open the underlying transport""" - self._init_stats() - return self.__trans.open() - - def listen(self): - """Invoke the underlying transport's listen() method""" - self.__trans.listen() - - def accept(self): - """Accept connections on the underlying transport""" - return self.__trans.accept() - - def close(self): - """Close the underlying transport,""" - self._reinit_buffers() - self._init_zlib() - return self.__trans.close() - - def read(self, sz): - """Read up to sz bytes from the decompressed bytes buffer, and - read from the underlying transport if the decompression - buffer is empty. - """ - ret = self.__rbuf.read(sz) - if len(ret) > 0: - return ret - # keep reading from transport until something comes back - while True: - if self.readComp(sz): - break - ret = self.__rbuf.read(sz) - return ret - - def readComp(self, sz): - """Read compressed data from the underlying transport, then - decompress it and append it to the internal StringIO read buffer - """ - zbuf = self.__trans.read(sz) - zbuf = self._zcomp_read.unconsumed_tail + zbuf - buf = self._zcomp_read.decompress(zbuf) - self.bytes_in += len(zbuf) - self.bytes_in_comp += len(buf) - old = self.__rbuf.read() - self.__rbuf = StringIO(old + buf) - if len(old) + len(buf) == 0: - return False - return True - - def write(self, buf): - """Write some bytes, putting them into the internal write - buffer for eventual compression. - """ - self.__wbuf.write(buf) - - def flush(self): - """Flush any queued up data in the write buffer and ensure the - compression buffer is flushed out to the underlying transport - """ - wout = self.__wbuf.getvalue() - if len(wout) > 0: - zbuf = self._zcomp_write.compress(wout) - self.bytes_out += len(wout) - self.bytes_out_comp += len(zbuf) - else: - zbuf = '' - ztail = self._zcomp_write.flush(zlib.Z_SYNC_FLUSH) - self.bytes_out_comp += len(ztail) - if (len(zbuf) + len(ztail)) > 0: - self.__wbuf = StringIO() - self.__trans.write(zbuf + ztail) - self.__trans.flush() - - @property - def cstringio_buf(self): - """Implement the CReadableTransport interface""" - return self.__rbuf - - def cstringio_refill(self, partialread, reqlen): - """Implement the CReadableTransport interface for refill""" - retstring = partialread - if reqlen < self.DEFAULT_BUFFSIZE: - retstring += self.read(self.DEFAULT_BUFFSIZE) - while len(retstring) < reqlen: - retstring += self.read(reqlen - len(retstring)) - self.__rbuf = StringIO(retstring) - return self.__rbuf diff --git a/pyload/lib/thrift/transport/__init__.py b/pyload/lib/thrift/transport/__init__.py deleted file mode 100644 index c9596d9a6..000000000 --- a/pyload/lib/thrift/transport/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -__all__ = ['TTransport', 'TSocket', 'THttpClient', 'TZlibTransport'] diff --git a/pyload/lib/wsgiserver.py b/pyload/lib/wsgiserver.py deleted file mode 100644 index 1058b19ff..000000000 --- a/pyload/lib/wsgiserver.py +++ /dev/null @@ -1,2299 +0,0 @@ -"""A high-speed, production ready, thread pooled, generic HTTP server. - -Simplest example on how to use this module directly -(without using CherryPy's application machinery):: - - from cherrypy import wsgiserver - - def my_crazy_app(environ, start_response): - status = '200 OK' - response_headers = [('Content-type','text/plain')] - start_response(status, response_headers) - return ['Hello world!'] - - server = wsgiserver.CherryPyWSGIServer( - ('0.0.0.0', 8070), my_crazy_app, - server_name='www.cherrypy.example') - server.start() - -The CherryPy WSGI server can serve as many WSGI applications -as you want in one instance by using a WSGIPathInfoDispatcher:: - - d = WSGIPathInfoDispatcher({'/': my_crazy_app, '/blog': my_blog_app}) - server = wsgiserver.CherryPyWSGIServer(('0.0.0.0', 80), d) - -Want SSL support? Just set server.ssl_adapter to an SSLAdapter instance. - -This won't call the CherryPy engine (application side) at all, only the -HTTP server, which is independent from the rest of CherryPy. Don't -let the name "CherryPyWSGIServer" throw you; the name merely reflects -its origin, not its coupling. - -For those of you wanting to understand internals of this module, here's the -basic call flow. The server's listening thread runs a very tight loop, -sticking incoming connections onto a Queue:: - - server = CherryPyWSGIServer(...) - server.start() - while True: - tick() - # This blocks until a request comes in: - child = socket.accept() - conn = HTTPConnection(child, ...) - server.requests.put(conn) - -Worker threads are kept in a pool and poll the Queue, popping off and then -handling each connection in turn. Each connection can consist of an arbitrary -number of requests and their responses, so we run a nested loop:: - - while True: - conn = server.requests.get() - conn.communicate() - -> while True: - req = HTTPRequest(...) - req.parse_request() - -> # Read the Request-Line, e.g. "GET /page HTTP/1.1" - req.rfile.readline() - read_headers(req.rfile, req.inheaders) - req.respond() - -> response = app(...) - try: - for chunk in response: - if chunk: - req.write(chunk) - finally: - if hasattr(response, "close"): - response.close() - if req.close_connection: - return -""" - -__all__ = ['HTTPRequest', 'HTTPConnection', 'HTTPServer', - 'SizeCheckWrapper', 'KnownLengthRFile', 'ChunkedRFile', - 'CP_fileobject', - 'MaxSizeExceeded', 'NoSSLError', 'FatalSSLAlert', - 'WorkerThread', 'ThreadPool', 'SSLAdapter', - 'CherryPyWSGIServer', - 'Gateway', 'WSGIGateway', 'WSGIGateway_10', 'WSGIGateway_u0', - 'WSGIPathInfoDispatcher'] - -import os -try: - import queue -except: - import Queue as queue -import re -import rfc822 -import socket -import sys -if 'win' in sys.platform and not hasattr(socket, 'IPPROTO_IPV6'): - socket.IPPROTO_IPV6 = 41 -try: - import cStringIO as StringIO -except ImportError: - import StringIO -DEFAULT_BUFFER_SIZE = -1 - -_fileobject_uses_str_type = isinstance(socket._fileobject(None)._rbuf, basestring) - -import threading -import time -import traceback -def format_exc(limit=None): - """Like print_exc() but return a string. Backport for Python 2.3.""" - try: - etype, value, tb = sys.exc_info() - return ''.join(traceback.format_exception(etype, value, tb, limit)) - finally: - etype = value = tb = None - - -from urllib import unquote -from urlparse import urlparse -import warnings - -if sys.version_info >= (3, 0): - bytestr = bytes - unicodestr = str - basestring = (bytes, str) - def ntob(n, encoding='ISO-8859-1'): - """Return the given native string as a byte string in the given encoding.""" - # In Python 3, the native string type is unicode - return n.encode(encoding) -else: - bytestr = str - unicodestr = unicode - basestring = basestring - def ntob(n, encoding='ISO-8859-1'): - """Return the given native string as a byte string in the given encoding.""" - # In Python 2, the native string type is bytes. Assume it's already - # in the given encoding, which for ISO-8859-1 is almost always what - # was intended. - return n - -LF = ntob('\n') -CRLF = ntob('\r\n') -TAB = ntob('\t') -SPACE = ntob(' ') -COLON = ntob(':') -SEMICOLON = ntob(';') -EMPTY = ntob('') -NUMBER_SIGN = ntob('#') -QUESTION_MARK = ntob('?') -ASTERISK = ntob('*') -FORWARD_SLASH = ntob('/') -quoted_slash = re.compile(ntob("(?i)%2F")) - -import errno - -def plat_specific_errors(*errnames): - """Return error numbers for all errors in errnames on this platform. - - The 'errno' module contains different global constants depending on - the specific platform (OS). This function will return the list of - numeric values for a given list of potential names. - """ - errno_names = dir(errno) - nums = [getattr(errno, k) for k in errnames if k in errno_names] - # de-dupe the list - return list(dict.fromkeys(nums).keys()) - -socket_error_eintr = plat_specific_errors("EINTR", "WSAEINTR") - -socket_errors_to_ignore = plat_specific_errors( - "EPIPE", - "EBADF", "WSAEBADF", - "ENOTSOCK", "WSAENOTSOCK", - "ETIMEDOUT", "WSAETIMEDOUT", - "ECONNREFUSED", "WSAECONNREFUSED", - "ECONNRESET", "WSAECONNRESET", - "ECONNABORTED", "WSAECONNABORTED", - "ENETRESET", "WSAENETRESET", - "EHOSTDOWN", "EHOSTUNREACH", - ) -socket_errors_to_ignore.append("timed out") -socket_errors_to_ignore.append("The read operation timed out") - -socket_errors_nonblocking = plat_specific_errors( - 'EAGAIN', 'EWOULDBLOCK', 'WSAEWOULDBLOCK') - -comma_separated_headers = [ntob(h) for h in - ['Accept', 'Accept-Charset', 'Accept-Encoding', - 'Accept-Language', 'Accept-Ranges', 'Allow', 'Cache-Control', - 'Connection', 'Content-Encoding', 'Content-Language', 'Expect', - 'If-Match', 'If-None-Match', 'Pragma', 'Proxy-Authenticate', 'TE', - 'Trailer', 'Transfer-Encoding', 'Upgrade', 'Vary', 'Via', 'Warning', - 'WWW-Authenticate']] - - -import logging -if not hasattr(logging, 'statistics'): logging.statistics = {} - - -def read_headers(rfile, hdict=None): - """Read headers from the given stream into the given header dict. - - If hdict is None, a new header dict is created. Returns the populated - header dict. - - Headers which are repeated are folded together using a comma if their - specification so dictates. - - This function raises ValueError when the read bytes violate the HTTP spec. - You should probably return "400 Bad Request" if this happens. - """ - if hdict is None: - hdict = {} - - while True: - line = rfile.readline() - if not line: - # No more data--illegal end of headers - raise ValueError("Illegal end of headers.") - - if line == CRLF: - # Normal end of headers - break - if not line.endswith(CRLF): - raise ValueError("HTTP requires CRLF terminators") - - if line[0] in (SPACE, TAB): - # It's a continuation line. - v = line.strip() - else: - try: - k, v = line.split(COLON, 1) - except ValueError: - raise ValueError("Illegal header line.") - # TODO: what about TE and WWW-Authenticate? - k = k.strip().title() - v = v.strip() - hname = k - - if k in comma_separated_headers: - existing = hdict.get(hname) - if existing: - v = ", ".join((existing, v)) - hdict[hname] = v - - return hdict - - -class MaxSizeExceeded(Exception): - pass - -class SizeCheckWrapper(object): - """Wraps a file-like object, raising MaxSizeExceeded if too large.""" - - def __init__(self, rfile, maxlen): - self.rfile = rfile - self.maxlen = maxlen - self.bytes_read = 0 - - def _check_length(self): - if self.maxlen and self.bytes_read > self.maxlen: - raise MaxSizeExceeded() - - def read(self, size=None): - data = self.rfile.read(size) - self.bytes_read += len(data) - self._check_length() - return data - - def readline(self, size=None): - if size is not None: - data = self.rfile.readline(size) - self.bytes_read += len(data) - self._check_length() - return data - - # User didn't specify a size ... - # We read the line in chunks to make sure it's not a 100MB line ! - res = [] - while True: - data = self.rfile.readline(256) - self.bytes_read += len(data) - self._check_length() - res.append(data) - # See http://www.cherrypy.org/ticket/421 - if len(data) < 256 or data[-1:] == "\n": - return EMPTY.join(res) - - def readlines(self, sizehint=0): - # Shamelessly stolen from StringIO - total = 0 - lines = [] - line = self.readline() - while line: - lines.append(line) - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline() - return lines - - def close(self): - self.rfile.close() - - def __iter__(self): - return self - - def __next__(self): - data = next(self.rfile) - self.bytes_read += len(data) - self._check_length() - return data - - def next(self): - data = self.rfile.next() - self.bytes_read += len(data) - self._check_length() - return data - - -class KnownLengthRFile(object): - """Wraps a file-like object, returning an empty string when exhausted.""" - - def __init__(self, rfile, content_length): - self.rfile = rfile - self.remaining = content_length - - def read(self, size=None): - if self.remaining == 0: - return '' - if size is None: - size = self.remaining - else: - size = min(size, self.remaining) - - data = self.rfile.read(size) - self.remaining -= len(data) - return data - - def readline(self, size=None): - if self.remaining == 0: - return '' - if size is None: - size = self.remaining - else: - size = min(size, self.remaining) - - data = self.rfile.readline(size) - self.remaining -= len(data) - return data - - def readlines(self, sizehint=0): - # Shamelessly stolen from StringIO - total = 0 - lines = [] - line = self.readline(sizehint) - while line: - lines.append(line) - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline(sizehint) - return lines - - def close(self): - self.rfile.close() - - def __iter__(self): - return self - - def __next__(self): - data = next(self.rfile) - self.remaining -= len(data) - return data - - -class ChunkedRFile(object): - """Wraps a file-like object, returning an empty string when exhausted. - - This class is intended to provide a conforming wsgi.input value for - request entities that have been encoded with the 'chunked' transfer - encoding. - """ - - def __init__(self, rfile, maxlen, bufsize=8192): - self.rfile = rfile - self.maxlen = maxlen - self.bytes_read = 0 - self.buffer = EMPTY - self.bufsize = bufsize - self.closed = False - - def _fetch(self): - if self.closed: - return - - line = self.rfile.readline() - self.bytes_read += len(line) - - if self.maxlen and self.bytes_read > self.maxlen: - raise MaxSizeExceeded("Request Entity Too Large", self.maxlen) - - line = line.strip().split(SEMICOLON, 1) - - try: - chunk_size = line.pop(0) - chunk_size = int(chunk_size, 16) - except ValueError: - raise ValueError("Bad chunked transfer size: " + repr(chunk_size)) - - if chunk_size <= 0: - self.closed = True - return - -## if line: chunk_extension = line[0] - - if self.maxlen and self.bytes_read + chunk_size > self.maxlen: - raise IOError("Request Entity Too Large") - - chunk = self.rfile.read(chunk_size) - self.bytes_read += len(chunk) - self.buffer += chunk - - crlf = self.rfile.read(2) - if crlf != CRLF: - raise ValueError( - "Bad chunked transfer coding (expected '\\r\\n', " - "got " + repr(crlf) + ")") - - def read(self, size=None): - data = EMPTY - while True: - if size and len(data) >= size: - return data - - if not self.buffer: - self._fetch() - if not self.buffer: - # EOF - return data - - if size: - remaining = size - len(data) - data += self.buffer[:remaining] - self.buffer = self.buffer[remaining:] - else: - data += self.buffer - - def readline(self, size=None): - data = EMPTY - while True: - if size and len(data) >= size: - return data - - if not self.buffer: - self._fetch() - if not self.buffer: - # EOF - return data - - newline_pos = self.buffer.find(LF) - if size: - if newline_pos == -1: - remaining = size - len(data) - data += self.buffer[:remaining] - self.buffer = self.buffer[remaining:] - else: - remaining = min(size - len(data), newline_pos) - data += self.buffer[:remaining] - self.buffer = self.buffer[remaining:] - else: - if newline_pos == -1: - data += self.buffer - else: - data += self.buffer[:newline_pos] - self.buffer = self.buffer[newline_pos:] - - def readlines(self, sizehint=0): - # Shamelessly stolen from StringIO - total = 0 - lines = [] - line = self.readline(sizehint) - while line: - lines.append(line) - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline(sizehint) - return lines - - def read_trailer_lines(self): - if not self.closed: - raise ValueError( - "Cannot read trailers until the request body has been read.") - - while True: - line = self.rfile.readline() - if not line: - # No more data--illegal end of headers - raise ValueError("Illegal end of headers.") - - self.bytes_read += len(line) - if self.maxlen and self.bytes_read > self.maxlen: - raise IOError("Request Entity Too Large") - - if line == CRLF: - # Normal end of headers - break - if not line.endswith(CRLF): - raise ValueError("HTTP requires CRLF terminators") - - yield line - - def close(self): - self.rfile.close() - - def __iter__(self): - # Shamelessly stolen from StringIO - total = 0 - line = self.readline(sizehint) - while line: - yield line - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline(sizehint) - - -class HTTPRequest(object): - """An HTTP Request (and response). - - A single HTTP connection may consist of multiple request/response pairs. - """ - - server = None - """The HTTPServer object which is receiving this request.""" - - conn = None - """The HTTPConnection object on which this request connected.""" - - inheaders = {} - """A dict of request headers.""" - - outheaders = [] - """A list of header tuples to write in the response.""" - - ready = False - """When True, the request has been parsed and is ready to begin generating - the response. When False, signals the calling Connection that the response - should not be generated and the connection should close.""" - - close_connection = False - """Signals the calling Connection that the request should close. This does - not imply an error! The client and/or server may each request that the - connection be closed.""" - - chunked_write = False - """If True, output will be encoded with the "chunked" transfer-coding. - - This value is set automatically inside send_headers.""" - - def __init__(self, server, conn): - self.server= server - self.conn = conn - - self.ready = False - self.started_request = False - self.scheme = ntob("http") - if self.server.ssl_adapter is not None: - self.scheme = ntob("https") - # Use the lowest-common protocol in case read_request_line errors. - self.response_protocol = 'HTTP/1.0' - self.inheaders = {} - - self.status = "" - self.outheaders = [] - self.sent_headers = False - self.close_connection = self.__class__.close_connection - self.chunked_read = False - self.chunked_write = self.__class__.chunked_write - - def parse_request(self): - """Parse the next HTTP request start-line and message-headers.""" - self.rfile = SizeCheckWrapper(self.conn.rfile, - self.server.max_request_header_size) - try: - self.read_request_line() - except MaxSizeExceeded: - self.simple_response("414 Request-URI Too Long", - "The Request-URI sent with the request exceeds the maximum " - "allowed bytes.") - return - - try: - success = self.read_request_headers() - except MaxSizeExceeded: - self.simple_response("413 Request Entity Too Large", - "The headers sent with the request exceed the maximum " - "allowed bytes.") - return - else: - if not success: - return - - self.ready = True - - def read_request_line(self): - # HTTP/1.1 connections are persistent by default. If a client - # requests a page, then idles (leaves the connection open), - # then rfile.readline() will raise socket.error("timed out"). - # Note that it does this based on the value given to settimeout(), - # and doesn't need the client to request or acknowledge the close - # (although your TCP stack might suffer for it: cf Apache's history - # with FIN_WAIT_2). - request_line = self.rfile.readline() - - # Set started_request to True so communicate() knows to send 408 - # from here on out. - self.started_request = True - if not request_line: - # Force self.ready = False so the connection will close. - self.ready = False - return - - if request_line == CRLF: - # RFC 2616 sec 4.1: "...if the server is reading the protocol - # stream at the beginning of a message and receives a CRLF - # first, it should ignore the CRLF." - # But only ignore one leading line! else we enable a DoS. - request_line = self.rfile.readline() - if not request_line: - self.ready = False - return - - if not request_line.endswith(CRLF): - self.simple_response("400 Bad Request", "HTTP requires CRLF terminators") - return - - try: - method, uri, req_protocol = request_line.strip().split(SPACE, 2) - rp = int(req_protocol[5]), int(req_protocol[7]) - except (ValueError, IndexError): - self.simple_response("400 Bad Request", "Malformed Request-Line") - return - - self.uri = uri - self.method = method - - # uri may be an abs_path (including "http://host.domain.tld"); - scheme, authority, path = self.parse_request_uri(uri) - if NUMBER_SIGN in path: - self.simple_response("400 Bad Request", - "Illegal #fragment in Request-URI.") - return - - if scheme: - self.scheme = scheme - - qs = EMPTY - if QUESTION_MARK in path: - path, qs = path.split(QUESTION_MARK, 1) - - # Unquote the path+params (e.g. "/this%20path" -> "/this path"). - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2 - # - # But note that "...a URI must be separated into its components - # before the escaped characters within those components can be - # safely decoded." http://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2 - # Therefore, "/this%2Fpath" becomes "/this%2Fpath", not "/this/path". - try: - atoms = [unquote(x) for x in quoted_slash.split(path)] - except ValueError: - ex = sys.exc_info()[1] - self.simple_response("400 Bad Request", ex.args[0]) - return - path = "%2F".join(atoms) - self.path = path - - # Note that, like wsgiref and most other HTTP servers, - # we "% HEX HEX"-unquote the path but not the query string. - self.qs = qs - - # Compare request and server HTTP protocol versions, in case our - # server does not support the requested protocol. Limit our output - # to min(req, server). We want the following output: - # request server actual written supported response - # protocol protocol response protocol feature set - # a 1.0 1.0 1.0 1.0 - # b 1.0 1.1 1.1 1.0 - # c 1.1 1.0 1.0 1.0 - # d 1.1 1.1 1.1 1.1 - # Notice that, in (b), the response will be "HTTP/1.1" even though - # the client only understands 1.0. RFC 2616 10.5.6 says we should - # only return 505 if the _major_ version is different. - sp = int(self.server.protocol[5]), int(self.server.protocol[7]) - - if sp[0] != rp[0]: - self.simple_response("505 HTTP Version Not Supported") - return - self.request_protocol = req_protocol - self.response_protocol = "HTTP/%s.%s" % min(rp, sp) - - def read_request_headers(self): - """Read self.rfile into self.inheaders. Return success.""" - - # then all the http headers - try: - read_headers(self.rfile, self.inheaders) - except ValueError: - ex = sys.exc_info()[1] - self.simple_response("400 Bad Request", ex.args[0]) - return False - - mrbs = self.server.max_request_body_size - if mrbs and int(self.inheaders.get("Content-Length", 0)) > mrbs: - self.simple_response("413 Request Entity Too Large", - "The entity sent with the request exceeds the maximum " - "allowed bytes.") - return False - - # Persistent connection support - if self.response_protocol == "HTTP/1.1": - # Both server and client are HTTP/1.1 - if self.inheaders.get("Connection", "") == "close": - self.close_connection = True - else: - # Either the server or client (or both) are HTTP/1.0 - if self.inheaders.get("Connection", "") != "Keep-Alive": - self.close_connection = True - - # Transfer-Encoding support - te = None - if self.response_protocol == "HTTP/1.1": - te = self.inheaders.get("Transfer-Encoding") - if te: - te = [x.strip().lower() for x in te.split(",") if x.strip()] - - self.chunked_read = False - - if te: - for enc in te: - if enc == "chunked": - self.chunked_read = True - else: - # Note that, even if we see "chunked", we must reject - # if there is an extension we don't recognize. - self.simple_response("501 Unimplemented") - self.close_connection = True - return False - - # From PEP 333: - # "Servers and gateways that implement HTTP 1.1 must provide - # transparent support for HTTP 1.1's "expect/continue" mechanism. - # This may be done in any of several ways: - # 1. Respond to requests containing an Expect: 100-continue request - # with an immediate "100 Continue" response, and proceed normally. - # 2. Proceed with the request normally, but provide the application - # with a wsgi.input stream that will send the "100 Continue" - # response if/when the application first attempts to read from - # the input stream. The read request must then remain blocked - # until the client responds. - # 3. Wait until the client decides that the server does not support - # expect/continue, and sends the request body on its own. - # (This is suboptimal, and is not recommended.) - # - # We used to do 3, but are now doing 1. Maybe we'll do 2 someday, - # but it seems like it would be a big slowdown for such a rare case. - if self.inheaders.get("Expect", "") == "100-continue": - # Don't use simple_response here, because it emits headers - # we don't want. See http://www.cherrypy.org/ticket/951 - msg = self.server.protocol + " 100 Continue\r\n\r\n" - try: - self.conn.wfile.sendall(msg) - except socket.error: - x = sys.exc_info()[1] - if x.args[0] not in socket_errors_to_ignore: - raise - return True - - def parse_request_uri(self, uri): - """Parse a Request-URI into (scheme, authority, path). - - Note that Request-URI's must be one of:: - - Request-URI = "*" | absoluteURI | abs_path | authority - - Therefore, a Request-URI which starts with a double forward-slash - cannot be a "net_path":: - - net_path = "//" authority [ abs_path ] - - Instead, it must be interpreted as an "abs_path" with an empty first - path segment:: - - abs_path = "/" path_segments - path_segments = segment *( "/" segment ) - segment = *pchar *( ";" param ) - param = *pchar - """ - if uri == ASTERISK: - return None, None, uri - - i = uri.find('://') - if i > 0 and QUESTION_MARK not in uri[:i]: - # An absoluteURI. - # If there's a scheme (and it must be http or https), then: - # http_URL = "http:" "//" host [ ":" port ] [ abs_path [ "?" query ]] - scheme, remainder = uri[:i].lower(), uri[i + 3:] - authority, path = remainder.split(FORWARD_SLASH, 1) - path = FORWARD_SLASH + path - return scheme, authority, path - - if uri.startswith(FORWARD_SLASH): - # An abs_path. - return None, None, uri - else: - # An authority. - return None, uri, None - - def respond(self): - """Call the gateway and write its iterable output.""" - mrbs = self.server.max_request_body_size - if self.chunked_read: - self.rfile = ChunkedRFile(self.conn.rfile, mrbs) - else: - cl = int(self.inheaders.get("Content-Length", 0)) - if mrbs and mrbs < cl: - if not self.sent_headers: - self.simple_response("413 Request Entity Too Large", - "The entity sent with the request exceeds the maximum " - "allowed bytes.") - return - self.rfile = KnownLengthRFile(self.conn.rfile, cl) - - self.server.gateway(self).respond() - - if (self.ready and not self.sent_headers): - self.sent_headers = True - self.send_headers() - if self.chunked_write: - self.conn.wfile.sendall("0\r\n\r\n") - - def simple_response(self, status, msg=""): - """Write a simple response back to the client.""" - status = str(status) - buf = [self.server.protocol + SPACE + - status + CRLF, - "Content-Length: %s\r\n" % len(msg), - "Content-Type: text/plain\r\n"] - - if status[:3] in ("413", "414"): - # Request Entity Too Large / Request-URI Too Long - self.close_connection = True - if self.response_protocol == 'HTTP/1.1': - # This will not be true for 414, since read_request_line - # usually raises 414 before reading the whole line, and we - # therefore cannot know the proper response_protocol. - buf.append("Connection: close\r\n") - else: - # HTTP/1.0 had no 413/414 status nor Connection header. - # Emit 400 instead and trust the message body is enough. - status = "400 Bad Request" - - buf.append(CRLF) - if msg: - if isinstance(msg, unicodestr): - msg = msg.encode("ISO-8859-1") - buf.append(msg) - - try: - self.conn.wfile.sendall("".join(buf)) - except socket.error: - x = sys.exc_info()[1] - if x.args[0] not in socket_errors_to_ignore: - raise - - def write(self, chunk): - """Write unbuffered data to the client.""" - if self.chunked_write and chunk: - buf = [hex(len(chunk))[2:], CRLF, chunk, CRLF] - self.conn.wfile.sendall(EMPTY.join(buf)) - else: - self.conn.wfile.sendall(chunk) - - def send_headers(self): - """Assert, process, and send the HTTP response message-headers. - - You must set self.status, and self.outheaders before calling this. - """ - hkeys = [key.lower() for key, value in self.outheaders] - status = int(self.status[:3]) - - if status == 413: - # Request Entity Too Large. Close conn to avoid garbage. - self.close_connection = True - elif "content-length" not in hkeys: - # "All 1xx (informational), 204 (no content), - # and 304 (not modified) responses MUST NOT - # include a message-body." So no point chunking. - if status < 200 or status in (204, 205, 304): - pass - else: - if (self.response_protocol == 'HTTP/1.1' - and self.method != 'HEAD'): - # Use the chunked transfer-coding - self.chunked_write = True - self.outheaders.append(("Transfer-Encoding", "chunked")) - else: - # Closing the conn is the only way to determine len. - self.close_connection = True - - if "connection" not in hkeys: - if self.response_protocol == 'HTTP/1.1': - # Both server and client are HTTP/1.1 or better - if self.close_connection: - self.outheaders.append(("Connection", "close")) - else: - # Server and/or client are HTTP/1.0 - if not self.close_connection: - self.outheaders.append(("Connection", "Keep-Alive")) - - if (not self.close_connection) and (not self.chunked_read): - # Read any remaining request body data on the socket. - # "If an origin server receives a request that does not include an - # Expect request-header field with the "100-continue" expectation, - # the request includes a request body, and the server responds - # with a final status code before reading the entire request body - # from the transport connection, then the server SHOULD NOT close - # the transport connection until it has read the entire request, - # or until the client closes the connection. Otherwise, the client - # might not reliably receive the response message. However, this - # requirement is not be construed as preventing a server from - # defending itself against denial-of-service attacks, or from - # badly broken client implementations." - remaining = getattr(self.rfile, 'remaining', 0) - if remaining > 0: - self.rfile.read(remaining) - - if "date" not in hkeys: - self.outheaders.append(("Date", rfc822.formatdate())) - - if "server" not in hkeys: - self.outheaders.append(("Server", self.server.server_name)) - - buf = [self.server.protocol + SPACE + self.status + CRLF] - for k, v in self.outheaders: - buf.append(k + COLON + SPACE + v + CRLF) - buf.append(CRLF) - self.conn.wfile.sendall(EMPTY.join(buf)) - - -class NoSSLError(Exception): - """Exception raised when a client speaks HTTP to an HTTPS socket.""" - pass - - -class FatalSSLAlert(Exception): - """Exception raised when the SSL implementation signals a fatal alert.""" - pass - - -class CP_fileobject(socket._fileobject): - """Faux file object attached to a socket object.""" - - def __init__(self, *args, **kwargs): - self.bytes_read = 0 - self.bytes_written = 0 - socket._fileobject.__init__(self, *args, **kwargs) - - def sendall(self, data): - """Sendall for non-blocking sockets.""" - while data: - try: - bytes_sent = self.send(data) - data = data[bytes_sent:] - except socket.error, e: - if e.args[0] not in socket_errors_nonblocking: - raise - - def send(self, data): - bytes_sent = self._sock.send(data) - self.bytes_written += bytes_sent - return bytes_sent - - def flush(self): - if self._wbuf: - buffer = "".join(self._wbuf) - self._wbuf = [] - self.sendall(buffer) - - def recv(self, size): - while True: - try: - data = self._sock.recv(size) - self.bytes_read += len(data) - return data - except socket.error, e: - if (e.args[0] not in socket_errors_nonblocking - and e.args[0] not in socket_error_eintr): - raise - - if not _fileobject_uses_str_type: - def read(self, size=-1): - # Use max, disallow tiny reads in a loop as they are very inefficient. - # We never leave read() with any leftover data from a new recv() call - # in our internal buffer. - rbufsize = max(self._rbufsize, self.default_bufsize) - # Our use of StringIO rather than lists of string objects returned by - # recv() minimizes memory usage and fragmentation that occurs when - # rbufsize is large compared to the typical return value of recv(). - buf = self._rbuf - buf.seek(0, 2) # seek end - if size < 0: - # Read until EOF - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. - while True: - data = self.recv(rbufsize) - if not data: - break - buf.write(data) - return buf.getvalue() - else: - # Read until size bytes or EOF seen, whichever comes first - buf_len = buf.tell() - if buf_len >= size: - # Already have size bytes in our buffer? Extract and return. - buf.seek(0) - rv = buf.read(size) - self._rbuf = StringIO.StringIO() - self._rbuf.write(buf.read()) - return rv - - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. - while True: - left = size - buf_len - # recv() will malloc the amount of memory given as its - # parameter even though it often returns much less data - # than that. The returned data string is short lived - # as we copy it into a StringIO and free it. This avoids - # fragmentation issues on many platforms. - data = self.recv(left) - if not data: - break - n = len(data) - if n == size and not buf_len: - # Shortcut. Avoid buffer data copies when: - # - We have no data in our buffer. - # AND - # - Our call to recv returned exactly the - # number of bytes we were asked to read. - return data - if n == left: - buf.write(data) - del data # explicit free - break - assert n <= left, "recv(%d) returned %d bytes" % (left, n) - buf.write(data) - buf_len += n - del data # explicit free - #assert buf_len == buf.tell() - return buf.getvalue() - - def readline(self, size=-1): - buf = self._rbuf - buf.seek(0, 2) # seek end - if buf.tell() > 0: - # check if we already have it in our buffer - buf.seek(0) - bline = buf.readline(size) - if bline.endswith('\n') or len(bline) == size: - self._rbuf = StringIO.StringIO() - self._rbuf.write(buf.read()) - return bline - del bline - if size < 0: - # Read until \n or EOF, whichever comes first - if self._rbufsize <= 1: - # Speed up unbuffered case - buf.seek(0) - buffers = [buf.read()] - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. - data = None - recv = self.recv - while data != "\n": - data = recv(1) - if not data: - break - buffers.append(data) - return "".join(buffers) - - buf.seek(0, 2) # seek end - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. - while True: - data = self.recv(self._rbufsize) - if not data: - break - nl = data.find('\n') - if nl >= 0: - nl += 1 - buf.write(data[:nl]) - self._rbuf.write(data[nl:]) - del data - break - buf.write(data) - return buf.getvalue() - else: - # Read until size bytes or \n or EOF seen, whichever comes first - buf.seek(0, 2) # seek end - buf_len = buf.tell() - if buf_len >= size: - buf.seek(0) - rv = buf.read(size) - self._rbuf = StringIO.StringIO() - self._rbuf.write(buf.read()) - return rv - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. - while True: - data = self.recv(self._rbufsize) - if not data: - break - left = size - buf_len - # did we just receive a newline? - nl = data.find('\n', 0, left) - if nl >= 0: - nl += 1 - # save the excess data to _rbuf - self._rbuf.write(data[nl:]) - if buf_len: - buf.write(data[:nl]) - break - else: - # Shortcut. Avoid data copy through buf when returning - # a substring of our first recv(). - return data[:nl] - n = len(data) - if n == size and not buf_len: - # Shortcut. Avoid data copy through buf when - # returning exactly all of our first recv(). - return data - if n >= left: - buf.write(data[:left]) - self._rbuf.write(data[left:]) - break - buf.write(data) - buf_len += n - #assert buf_len == buf.tell() - return buf.getvalue() - else: - def read(self, size=-1): - if size < 0: - # Read until EOF - buffers = [self._rbuf] - self._rbuf = "" - if self._rbufsize <= 1: - recv_size = self.default_bufsize - else: - recv_size = self._rbufsize - - while True: - data = self.recv(recv_size) - if not data: - break - buffers.append(data) - return "".join(buffers) - else: - # Read until size bytes or EOF seen, whichever comes first - data = self._rbuf - buf_len = len(data) - if buf_len >= size: - self._rbuf = data[size:] - return data[:size] - buffers = [] - if data: - buffers.append(data) - self._rbuf = "" - while True: - left = size - buf_len - recv_size = max(self._rbufsize, left) - data = self.recv(recv_size) - if not data: - break - buffers.append(data) - n = len(data) - if n >= left: - self._rbuf = data[left:] - buffers[-1] = data[:left] - break - buf_len += n - return "".join(buffers) - - def readline(self, size=-1): - data = self._rbuf - if size < 0: - # Read until \n or EOF, whichever comes first - if self._rbufsize <= 1: - # Speed up unbuffered case - assert data == "" - buffers = [] - while data != "\n": - data = self.recv(1) - if not data: - break - buffers.append(data) - return "".join(buffers) - nl = data.find('\n') - if nl >= 0: - nl += 1 - self._rbuf = data[nl:] - return data[:nl] - buffers = [] - if data: - buffers.append(data) - self._rbuf = "" - while True: - data = self.recv(self._rbufsize) - if not data: - break - buffers.append(data) - nl = data.find('\n') - if nl >= 0: - nl += 1 - self._rbuf = data[nl:] - buffers[-1] = data[:nl] - break - return "".join(buffers) - else: - # Read until size bytes or \n or EOF seen, whichever comes first - nl = data.find('\n', 0, size) - if nl >= 0: - nl += 1 - self._rbuf = data[nl:] - return data[:nl] - buf_len = len(data) - if buf_len >= size: - self._rbuf = data[size:] - return data[:size] - buffers = [] - if data: - buffers.append(data) - self._rbuf = "" - while True: - data = self.recv(self._rbufsize) - if not data: - break - buffers.append(data) - left = size - buf_len - nl = data.find('\n', 0, left) - if nl >= 0: - nl += 1 - self._rbuf = data[nl:] - buffers[-1] = data[:nl] - break - n = len(data) - if n >= left: - self._rbuf = data[left:] - buffers[-1] = data[:left] - break - buf_len += n - return "".join(buffers) - - -class HTTPConnection(object): - """An HTTP connection (active socket). - - server: the Server object which received this connection. - socket: the raw socket object (usually TCP) for this connection. - makefile: a fileobject class for reading from the socket. - """ - - remote_addr = None - remote_port = None - ssl_env = None - rbufsize = DEFAULT_BUFFER_SIZE - wbufsize = DEFAULT_BUFFER_SIZE - RequestHandlerClass = HTTPRequest - - def __init__(self, server, sock, makefile=CP_fileobject): - self.server = server - self.socket = sock - self.rfile = makefile(sock, "rb", self.rbufsize) - self.wfile = makefile(sock, "wb", self.wbufsize) - self.requests_seen = 0 - - def communicate(self): - """Read each request and respond appropriately.""" - request_seen = False - try: - while True: - # (re)set req to None so that if something goes wrong in - # the RequestHandlerClass constructor, the error doesn't - # get written to the previous request. - req = None - req = self.RequestHandlerClass(self.server, self) - - # This order of operations should guarantee correct pipelining. - req.parse_request() - if self.server.stats['Enabled']: - self.requests_seen += 1 - if not req.ready: - # Something went wrong in the parsing (and the server has - # probably already made a simple_response). Return and - # let the conn close. - return - - request_seen = True - req.respond() - if req.close_connection: - return - except socket.error: - e = sys.exc_info()[1] - errnum = e.args[0] - # sadly SSL sockets return a different (longer) time out string - if errnum == 'timed out' or errnum == 'The read operation timed out': - # Don't error if we're between requests; only error - # if 1) no request has been started at all, or 2) we're - # in the middle of a request. - # See http://www.cherrypy.org/ticket/853 - if (not request_seen) or (req and req.started_request): - # Don't bother writing the 408 if the response - # has already started being written. - if req and not req.sent_headers: - try: - req.simple_response("408 Request Timeout") - except FatalSSLAlert: - # Close the connection. - return - elif errnum not in socket_errors_to_ignore: - if req and not req.sent_headers: - try: - req.simple_response("500 Internal Server Error", - format_exc()) - except FatalSSLAlert: - # Close the connection. - return - return - except (KeyboardInterrupt, SystemExit): - raise - except FatalSSLAlert: - # Close the connection. - return - except NoSSLError: - if req and not req.sent_headers: - # Unwrap our wfile - self.wfile = CP_fileobject(self.socket._sock, "wb", self.wbufsize) - req.simple_response("400 Bad Request", - "The client sent a plain HTTP request, but " - "this server only speaks HTTPS on this port.") - self.linger = True - except Exception: - if req and not req.sent_headers: - try: - req.simple_response("500 Internal Server Error", format_exc()) - except FatalSSLAlert: - # Close the connection. - return - - linger = False - - def close(self): - """Close the socket underlying this connection.""" - self.rfile.close() - - if not self.linger: - # Python's socket module does NOT call close on the kernel socket - # when you call socket.close(). We do so manually here because we - # want this server to send a FIN TCP segment immediately. Note this - # must be called *before* calling socket.close(), because the latter - # drops its reference to the kernel socket. - if hasattr(self.socket, '_sock'): - self.socket._sock.close() - self.socket.close() - else: - # On the other hand, sometimes we want to hang around for a bit - # to make sure the client has a chance to read our entire - # response. Skipping the close() calls here delays the FIN - # packet until the socket object is garbage-collected later. - # Someday, perhaps, we'll do the full lingering_close that - # Apache does, but not today. - pass - - -class TrueyZero(object): - """An object which equals and does math like the integer '0' but evals True.""" - def __add__(self, other): - return other - def __radd__(self, other): - return other -trueyzero = TrueyZero() - - -_SHUTDOWNREQUEST = None - -class WorkerThread(threading.Thread): - """Thread which continuously polls a Queue for Connection objects. - - Due to the timing issues of polling a Queue, a WorkerThread does not - check its own 'ready' flag after it has started. To stop the thread, - it is necessary to stick a _SHUTDOWNREQUEST object onto the Queue - (one for each running WorkerThread). - """ - - conn = None - """The current connection pulled off the Queue, or None.""" - - server = None - """The HTTP Server which spawned this thread, and which owns the - Queue and is placing active connections into it.""" - - ready = False - """A simple flag for the calling server to know when this thread - has begun polling the Queue.""" - - - def __init__(self, server): - self.ready = False - self.server = server - - self.requests_seen = 0 - self.bytes_read = 0 - self.bytes_written = 0 - self.start_time = None - self.work_time = 0 - self.stats = { - 'Requests': lambda s: self.requests_seen + ((self.start_time is None) and trueyzero or self.conn.requests_seen), - 'Bytes Read': lambda s: self.bytes_read + ((self.start_time is None) and trueyzero or self.conn.rfile.bytes_read), - 'Bytes Written': lambda s: self.bytes_written + ((self.start_time is None) and trueyzero or self.conn.wfile.bytes_written), - 'Work Time': lambda s: self.work_time + ((self.start_time is None) and trueyzero or time.time() - self.start_time), - 'Read Throughput': lambda s: s['Bytes Read'](s) / (s['Work Time'](s) or 1e-6), - 'Write Throughput': lambda s: s['Bytes Written'](s) / (s['Work Time'](s) or 1e-6), - } - threading.Thread.__init__(self) - - def run(self): - self.server.stats['Worker Threads'][self.getName()] = self.stats - try: - self.ready = True - while True: - conn = self.server.requests.get() - if conn is _SHUTDOWNREQUEST: - return - - self.conn = conn - if self.server.stats['Enabled']: - self.start_time = time.time() - try: - conn.communicate() - finally: - conn.close() - if self.server.stats['Enabled']: - self.requests_seen += self.conn.requests_seen - self.bytes_read += self.conn.rfile.bytes_read - self.bytes_written += self.conn.wfile.bytes_written - self.work_time += time.time() - self.start_time - self.start_time = None - self.conn = None - except (KeyboardInterrupt, SystemExit): - exc = sys.exc_info()[1] - self.server.interrupt = exc - - -class ThreadPool(object): - """A Request Queue for an HTTPServer which pools threads. - - ThreadPool objects must provide min, get(), put(obj), start() - and stop(timeout) attributes. - """ - - def __init__(self, server, min=10, max=-1): - self.server = server - self.min = min - self.max = max - self._threads = [] - self._queue = queue.Queue() - self.get = self._queue.get - - def start(self): - """Start the pool of threads.""" - for i in range(self.min): - self._threads.append(WorkerThread(self.server)) - for worker in self._threads: - worker.setName("CP Server " + worker.getName()) - worker.start() - for worker in self._threads: - while not worker.ready: - time.sleep(.1) - - def _get_idle(self): - """Number of worker threads which are idle. Read-only.""" - return len([t for t in self._threads if t.conn is None]) - idle = property(_get_idle, doc=_get_idle.__doc__) - - def put(self, obj): - self._queue.put(obj) - if obj is _SHUTDOWNREQUEST: - return - - def grow(self, amount): - """Spawn new worker threads (not above self.max).""" - for i in range(amount): - if self.max > 0 and len(self._threads) >= self.max: - break - worker = WorkerThread(self.server) - worker.setName("CP Server " + worker.getName()) - self._threads.append(worker) - worker.start() - - def shrink(self, amount): - """Kill off worker threads (not below self.min).""" - # Grow/shrink the pool if necessary. - # Remove any dead threads from our list - for t in self._threads: - if not t.isAlive(): - self._threads.remove(t) - amount -= 1 - - if amount > 0: - for i in range(min(amount, len(self._threads) - self.min)): - # Put a number of shutdown requests on the queue equal - # to 'amount'. Once each of those is processed by a worker, - # that worker will terminate and be culled from our list - # in self.put. - self._queue.put(_SHUTDOWNREQUEST) - - def stop(self, timeout=5): - # Must shut down threads here so the code that calls - # this method can know when all threads are stopped. - for worker in self._threads: - self._queue.put(_SHUTDOWNREQUEST) - - # Don't join currentThread (when stop is called inside a request). - current = threading.currentThread() - if timeout and timeout >= 0: - endtime = time.time() + timeout - while self._threads: - worker = self._threads.pop() - if worker is not current and worker.isAlive(): - try: - if timeout is None or timeout < 0: - worker.join() - else: - remaining_time = endtime - time.time() - if remaining_time > 0: - worker.join(remaining_time) - if worker.isAlive(): - # We exhausted the timeout. - # Forcibly shut down the socket. - c = worker.conn - if c and not c.rfile.closed: - try: - c.socket.shutdown(socket.SHUT_RD) - except TypeError: - # pyOpenSSL sockets don't take an arg - c.socket.shutdown() - worker.join() - except (AssertionError, - # Ignore repeated Ctrl-C. - # See http://www.cherrypy.org/ticket/691. - KeyboardInterrupt): - pass - - def _get_qsize(self): - return self._queue.qsize() - qsize = property(_get_qsize) - - - -try: - import fcntl -except ImportError: - try: - from ctypes import windll, WinError - except ImportError: - def prevent_socket_inheritance(sock): - """Dummy function, since neither fcntl nor ctypes are available.""" - pass - else: - def prevent_socket_inheritance(sock): - """Mark the given socket fd as non-inheritable (Windows).""" - if not windll.kernel32.SetHandleInformation(sock.fileno(), 1, 0): - raise WinError() -else: - def prevent_socket_inheritance(sock): - """Mark the given socket fd as non-inheritable (POSIX).""" - fd = sock.fileno() - old_flags = fcntl.fcntl(fd, fcntl.F_GETFD) - fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC) - - -class SSLAdapter(object): - """Base class for SSL driver library adapters. - - Required methods: - - * ``wrap(sock) -> (wrapped socket, ssl environ dict)`` - * ``makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE) -> socket file object`` - """ - - def __init__(self, certificate, private_key, certificate_chain=None): - self.certificate = certificate - self.private_key = private_key - self.certificate_chain = certificate_chain - - def wrap(self, sock): - raise NotImplemented - - def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): - raise NotImplemented - - -class HTTPServer(object): - """An HTTP server.""" - - _bind_addr = "127.0.0.1" - _interrupt = None - - gateway = None - """A Gateway instance.""" - - minthreads = None - """The minimum number of worker threads to create (default 10).""" - - maxthreads = None - """The maximum number of worker threads to create (default -1 = no limit).""" - - server_name = None - """The name of the server; defaults to socket.gethostname().""" - - protocol = "HTTP/1.1" - """The version string to write in the Status-Line of all HTTP responses. - - For example, "HTTP/1.1" is the default. This also limits the supported - features used in the response.""" - - request_queue_size = 5 - """The 'backlog' arg to socket.listen(); max queued connections (default 5).""" - - shutdown_timeout = 5 - """The total time, in seconds, to wait for worker threads to cleanly exit.""" - - timeout = 10 - """The timeout in seconds for accepted connections (default 10).""" - - version = "CherryPy/3.2.1" - """A version string for the HTTPServer.""" - - software = None - """The value to set for the SERVER_SOFTWARE entry in the WSGI environ. - - If None, this defaults to ``'%s Server' % self.version``.""" - - ready = False - """An internal flag which marks whether the socket is accepting connections.""" - - max_request_header_size = 0 - """The maximum size, in bytes, for request headers, or 0 for no limit.""" - - max_request_body_size = 0 - """The maximum size, in bytes, for request bodies, or 0 for no limit.""" - - nodelay = True - """If True (the default since 3.1), sets the TCP_NODELAY socket option.""" - - ConnectionClass = HTTPConnection - """The class to use for handling HTTP connections.""" - - ssl_adapter = None - """An instance of SSLAdapter (or a subclass). - - You must have the corresponding SSL driver library installed.""" - - def __init__(self, bind_addr, gateway, minthreads=10, maxthreads=-1, - server_name=None): - self.bind_addr = bind_addr - self.gateway = gateway - - self.requests = ThreadPool(self, min=minthreads or 1, max=maxthreads) - - if not server_name: - server_name = socket.gethostname() - self.server_name = server_name - self.clear_stats() - - def clear_stats(self): - self._start_time = None - self._run_time = 0 - self.stats = { - 'Enabled': False, - 'Bind Address': lambda s: repr(self.bind_addr), - 'Run time': lambda s: (not s['Enabled']) and -1 or self.runtime(), - 'Accepts': 0, - 'Accepts/sec': lambda s: s['Accepts'] / self.runtime(), - 'Queue': lambda s: getattr(self.requests, "qsize", None), - 'Threads': lambda s: len(getattr(self.requests, "_threads", [])), - 'Threads Idle': lambda s: getattr(self.requests, "idle", None), - 'Socket Errors': 0, - 'Requests': lambda s: (not s['Enabled']) and -1 or sum([w['Requests'](w) for w - in s['Worker Threads'].values()], 0), - 'Bytes Read': lambda s: (not s['Enabled']) and -1 or sum([w['Bytes Read'](w) for w - in s['Worker Threads'].values()], 0), - 'Bytes Written': lambda s: (not s['Enabled']) and -1 or sum([w['Bytes Written'](w) for w - in s['Worker Threads'].values()], 0), - 'Work Time': lambda s: (not s['Enabled']) and -1 or sum([w['Work Time'](w) for w - in s['Worker Threads'].values()], 0), - 'Read Throughput': lambda s: (not s['Enabled']) and -1 or sum( - [w['Bytes Read'](w) / (w['Work Time'](w) or 1e-6) - for w in s['Worker Threads'].values()], 0), - 'Write Throughput': lambda s: (not s['Enabled']) and -1 or sum( - [w['Bytes Written'](w) / (w['Work Time'](w) or 1e-6) - for w in s['Worker Threads'].values()], 0), - 'Worker Threads': {}, - } - logging.statistics["CherryPy HTTPServer %d" % id(self)] = self.stats - - def runtime(self): - if self._start_time is None: - return self._run_time - else: - return self._run_time + (time.time() - self._start_time) - - def __str__(self): - return "%s.%s(%r)" % (self.__module__, self.__class__.__name__, - self.bind_addr) - - def _get_bind_addr(self): - return self._bind_addr - def _set_bind_addr(self, value): - if isinstance(value, tuple) and value[0] in ('', None): - # Despite the socket module docs, using '' does not - # allow AI_PASSIVE to work. Passing None instead - # returns '0.0.0.0' like we want. In other words: - # host AI_PASSIVE result - # '' Y 192.168.x.y - # '' N 192.168.x.y - # None Y 0.0.0.0 - # None N 127.0.0.1 - # But since you can get the same effect with an explicit - # '0.0.0.0', we deny both the empty string and None as values. - raise ValueError("Host values of '' or None are not allowed. " - "Use '0.0.0.0' (IPv4) or '::' (IPv6) instead " - "to listen on all active interfaces.") - self._bind_addr = value - bind_addr = property(_get_bind_addr, _set_bind_addr, - doc="""The interface on which to listen for connections. - - For TCP sockets, a (host, port) tuple. Host values may be any IPv4 - or IPv6 address, or any valid hostname. The string 'localhost' is a - synonym for '127.0.0.1' (or '::1', if your hosts file prefers IPv6). - The string '0.0.0.0' is a special IPv4 entry meaning "any active - interface" (INADDR_ANY), and '::' is the similar IN6ADDR_ANY for - IPv6. The empty string or None are not allowed. - - For UNIX sockets, supply the filename as a string.""") - - def start(self): - """Run the server forever.""" - # We don't have to trap KeyboardInterrupt or SystemExit here, - # because cherrpy.server already does so, calling self.stop() for us. - # If you're using this server with another framework, you should - # trap those exceptions in whatever code block calls start(). - self._interrupt = None - - if self.software is None: - self.software = "%s Server" % self.version - - # SSL backward compatibility - if (self.ssl_adapter is None and - getattr(self, 'ssl_certificate', None) and - getattr(self, 'ssl_private_key', None)): - warnings.warn( - "SSL attributes are deprecated in CherryPy 3.2, and will " - "be removed in CherryPy 3.3. Use an ssl_adapter attribute " - "instead.", - DeprecationWarning - ) - try: - from cherrypy.wsgiserver.ssl_pyopenssl import pyOpenSSLAdapter - except ImportError: - pass - else: - self.ssl_adapter = pyOpenSSLAdapter( - self.ssl_certificate, self.ssl_private_key, - getattr(self, 'ssl_certificate_chain', None)) - - # Select the appropriate socket - if isinstance(self.bind_addr, basestring): - # AF_UNIX socket - - # So we can reuse the socket... - try: os.unlink(self.bind_addr) - except: pass - - # So everyone can access the socket... - try: os.chmod(self.bind_addr, 511) # 0777 - except: pass - - info = [(socket.AF_UNIX, socket.SOCK_STREAM, 0, "", self.bind_addr)] - else: - # AF_INET or AF_INET6 socket - # Get the correct address family for our host (allows IPv6 addresses) - host, port = self.bind_addr - try: - info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM, 0, socket.AI_PASSIVE) - except socket.gaierror: - if ':' in self.bind_addr[0]: - info = [(socket.AF_INET6, socket.SOCK_STREAM, - 0, "", self.bind_addr + (0, 0))] - else: - info = [(socket.AF_INET, socket.SOCK_STREAM, - 0, "", self.bind_addr)] - - self.socket = None - msg = "No socket could be created" - for res in info: - af, socktype, proto, canonname, sa = res - try: - self.bind(af, socktype, proto) - except socket.error: - if self.socket: - self.socket.close() - self.socket = None - continue - break - if not self.socket: - raise socket.error(msg) - - # Timeout so KeyboardInterrupt can be caught on Win32 - self.socket.settimeout(1) - self.socket.listen(self.request_queue_size) - - # Create worker threads - self.requests.start() - - self.ready = True - self._start_time = time.time() - while self.ready: - self.tick() - if self.interrupt: - while self.interrupt is True: - # Wait for self.stop() to complete. See _set_interrupt. - time.sleep(0.1) - if self.interrupt: - raise self.interrupt - - def bind(self, family, type, proto=0): - """Create (or recreate) the actual socket object.""" - self.socket = socket.socket(family, type, proto) - prevent_socket_inheritance(self.socket) - self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if self.nodelay and not isinstance(self.bind_addr, str): - self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - - if self.ssl_adapter is not None: - self.socket = self.ssl_adapter.bind(self.socket) - - # If listening on the IPV6 any address ('::' = IN6ADDR_ANY), - # activate dual-stack. See http://www.cherrypy.org/ticket/871. - if (hasattr(socket, 'AF_INET6') and family == socket.AF_INET6 - and self.bind_addr[0] in ('::', '::0', '::0.0.0.0')): - try: - self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) - except (AttributeError, socket.error): - # Apparently, the socket option is not available in - # this machine's TCP stack - pass - - self.socket.bind(self.bind_addr) - - def tick(self): - """Accept a new connection and put it on the Queue.""" - try: - s, addr = self.socket.accept() - if self.stats['Enabled']: - self.stats['Accepts'] += 1 - if not self.ready: - return - - prevent_socket_inheritance(s) - if hasattr(s, 'settimeout'): - s.settimeout(self.timeout) - - makefile = CP_fileobject - ssl_env = {} - # if ssl cert and key are set, we try to be a secure HTTP server - if self.ssl_adapter is not None: - try: - s, ssl_env = self.ssl_adapter.wrap(s) - except NoSSLError: - msg = ("The client sent a plain HTTP request, but " - "this server only speaks HTTPS on this port.") - buf = ["%s 400 Bad Request\r\n" % self.protocol, - "Content-Length: %s\r\n" % len(msg), - "Content-Type: text/plain\r\n\r\n", - msg] - - wfile = makefile(s, "wb", DEFAULT_BUFFER_SIZE) - try: - wfile.sendall("".join(buf)) - except socket.error: - x = sys.exc_info()[1] - if x.args[0] not in socket_errors_to_ignore: - raise - return - if not s: - return - makefile = self.ssl_adapter.makefile - # Re-apply our timeout since we may have a new socket object - if hasattr(s, 'settimeout'): - s.settimeout(self.timeout) - - conn = self.ConnectionClass(self, s, makefile) - - if not isinstance(self.bind_addr, basestring): - # optional values - # Until we do DNS lookups, omit REMOTE_HOST - if addr is None: # sometimes this can happen - # figure out if AF_INET or AF_INET6. - if len(s.getsockname()) == 2: - # AF_INET - addr = ('0.0.0.0', 0) - else: - # AF_INET6 - addr = ('::', 0) - conn.remote_addr = addr[0] - conn.remote_port = addr[1] - - conn.ssl_env = ssl_env - - self.requests.put(conn) - except socket.timeout: - # The only reason for the timeout in start() is so we can - # notice keyboard interrupts on Win32, which don't interrupt - # accept() by default - return - except socket.error: - x = sys.exc_info()[1] - if self.stats['Enabled']: - self.stats['Socket Errors'] += 1 - if x.args[0] in socket_error_eintr: - # I *think* this is right. EINTR should occur when a signal - # is received during the accept() call; all docs say retry - # the call, and I *think* I'm reading it right that Python - # will then go ahead and poll for and handle the signal - # elsewhere. See http://www.cherrypy.org/ticket/707. - return - if x.args[0] in socket_errors_nonblocking: - # Just try again. See http://www.cherrypy.org/ticket/479. - return - if x.args[0] in socket_errors_to_ignore: - # Our socket was closed. - # See http://www.cherrypy.org/ticket/686. - return - raise - - def _get_interrupt(self): - return self._interrupt - def _set_interrupt(self, interrupt): - self._interrupt = True - self.stop() - self._interrupt = interrupt - interrupt = property(_get_interrupt, _set_interrupt, - doc="Set this to an Exception instance to " - "interrupt the server.") - - def stop(self): - """Gracefully shutdown a server that is serving forever.""" - self.ready = False - if self._start_time is not None: - self._run_time += (time.time() - self._start_time) - self._start_time = None - - sock = getattr(self, "socket", None) - if sock: - if not isinstance(self.bind_addr, basestring): - # Touch our own socket to make accept() return immediately. - try: - host, port = sock.getsockname()[:2] - except socket.error: - x = sys.exc_info()[1] - if x.args[0] not in socket_errors_to_ignore: - # Changed to use error code and not message - # See http://www.cherrypy.org/ticket/860. - raise - else: - # Note that we're explicitly NOT using AI_PASSIVE, - # here, because we want an actual IP to touch. - # localhost won't work if we've bound to a public IP, - # but it will if we bound to '0.0.0.0' (INADDR_ANY). - for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM): - af, socktype, proto, canonname, sa = res - s = None - try: - s = socket.socket(af, socktype, proto) - # See http://groups.google.com/group/cherrypy-users/ - # browse_frm/thread/bbfe5eb39c904fe0 - s.settimeout(1.0) - s.connect((host, port)) - s.close() - except socket.error: - if s: - s.close() - if hasattr(sock, "close"): - sock.close() - self.socket = None - - self.requests.stop(self.shutdown_timeout) - - -class Gateway(object): - """A base class to interface HTTPServer with other systems, such as WSGI.""" - - def __init__(self, req): - self.req = req - - def respond(self): - """Process the current request. Must be overridden in a subclass.""" - raise NotImplemented - - -# These may either be wsgiserver.SSLAdapter subclasses or the string names -# of such classes (in which case they will be lazily loaded). -ssl_adapters = { - 'builtin': 'cherrypy.wsgiserver.ssl_builtin.BuiltinSSLAdapter', - 'pyopenssl': 'cherrypy.wsgiserver.ssl_pyopenssl.pyOpenSSLAdapter', - } - -def get_ssl_adapter_class(name='pyopenssl'): - """Return an SSL adapter class for the given name.""" - adapter = ssl_adapters[name.lower()] - if isinstance(adapter, basestring): - last_dot = adapter.rfind(".") - attr_name = adapter[last_dot + 1:] - mod_path = adapter[:last_dot] - - try: - mod = sys.modules[mod_path] - if mod is None: - raise KeyError() - except KeyError: - # The last [''] is important. - mod = __import__(mod_path, globals(), locals(), ['']) - - # Let an AttributeError propagate outward. - try: - adapter = getattr(mod, attr_name) - except AttributeError: - raise AttributeError("'%s' object has no attribute '%s'" - % (mod_path, attr_name)) - - return adapter - -# -------------------------------- WSGI Stuff -------------------------------- # - - -class CherryPyWSGIServer(HTTPServer): - """A subclass of HTTPServer which calls a WSGI application.""" - - wsgi_version = (1, 0) - """The version of WSGI to produce.""" - - def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None, - max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5): - self.requests = ThreadPool(self, min=numthreads or 1, max=max) - self.wsgi_app = wsgi_app - self.gateway = wsgi_gateways[self.wsgi_version] - - self.bind_addr = bind_addr - if not server_name: - server_name = socket.gethostname() - self.server_name = server_name - self.request_queue_size = request_queue_size - - self.timeout = timeout - self.shutdown_timeout = shutdown_timeout - self.clear_stats() - - def _get_numthreads(self): - return self.requests.min - def _set_numthreads(self, value): - self.requests.min = value - numthreads = property(_get_numthreads, _set_numthreads) - - -class WSGIGateway(Gateway): - """A base class to interface HTTPServer with WSGI.""" - - def __init__(self, req): - self.req = req - self.started_response = False - self.env = self.get_environ() - self.remaining_bytes_out = None - - def get_environ(self): - """Return a new environ dict targeting the given wsgi.version""" - raise NotImplemented - - def respond(self): - """Process the current request.""" - response = self.req.server.wsgi_app(self.env, self.start_response) - try: - for chunk in response: - # "The start_response callable must not actually transmit - # the response headers. Instead, it must store them for the - # server or gateway to transmit only after the first - # iteration of the application return value that yields - # a NON-EMPTY string, or upon the application's first - # invocation of the write() callable." (PEP 333) - if chunk: - if isinstance(chunk, unicodestr): - chunk = chunk.encode('ISO-8859-1') - self.write(chunk) - finally: - if hasattr(response, "close"): - response.close() - - def start_response(self, status, headers, exc_info = None): - """WSGI callable to begin the HTTP response.""" - # "The application may call start_response more than once, - # if and only if the exc_info argument is provided." - if self.started_response and not exc_info: - raise AssertionError("WSGI start_response called a second " - "time with no exc_info.") - self.started_response = True - - # "if exc_info is provided, and the HTTP headers have already been - # sent, start_response must raise an error, and should raise the - # exc_info tuple." - if self.req.sent_headers: - try: - raise exc_info[0], exc_info[1], exc_info[2] - finally: - exc_info = None - - self.req.status = status - for k, v in headers: - if not isinstance(k, bytestr): - raise TypeError("WSGI response header key %r is not a byte string." % k) - if not isinstance(v, bytestr): - raise TypeError("WSGI response header value %r is not a byte string." % v) - if k.lower() == 'content-length': - self.remaining_bytes_out = int(v) - self.req.outheaders.extend(headers) - - return self.write - - def write(self, chunk): - """WSGI callable to write unbuffered data to the client. - - This method is also used internally by start_response (to write - data from the iterable returned by the WSGI application). - """ - if not self.started_response: - raise AssertionError("WSGI write called before start_response.") - - chunklen = len(chunk) - rbo = self.remaining_bytes_out - if rbo is not None and chunklen > rbo: - if not self.req.sent_headers: - # Whew. We can send a 500 to the client. - self.req.simple_response("500 Internal Server Error", - "The requested resource returned more bytes than the " - "declared Content-Length.") - else: - # Dang. We have probably already sent data. Truncate the chunk - # to fit (so the client doesn't hang) and raise an error later. - chunk = chunk[:rbo] - - if not self.req.sent_headers: - self.req.sent_headers = True - self.req.send_headers() - - self.req.write(chunk) - - if rbo is not None: - rbo -= chunklen - if rbo < 0: - raise ValueError( - "Response body exceeds the declared Content-Length.") - - -class WSGIGateway_10(WSGIGateway): - """A Gateway class to interface HTTPServer with WSGI 1.0.x.""" - - def get_environ(self): - """Return a new environ dict targeting the given wsgi.version""" - req = self.req - env = { - # set a non-standard environ entry so the WSGI app can know what - # the *real* server protocol is (and what features to support). - # See http://www.faqs.org/rfcs/rfc2145.html. - 'ACTUAL_SERVER_PROTOCOL': req.server.protocol, - 'PATH_INFO': req.path, - 'QUERY_STRING': req.qs, - 'REMOTE_ADDR': req.conn.remote_addr or '', - 'REMOTE_PORT': str(req.conn.remote_port or ''), - 'REQUEST_METHOD': req.method, - 'REQUEST_URI': req.uri, - 'SCRIPT_NAME': '', - 'SERVER_NAME': req.server.server_name, - # Bah. "SERVER_PROTOCOL" is actually the REQUEST protocol. - 'SERVER_PROTOCOL': req.request_protocol, - 'SERVER_SOFTWARE': req.server.software, - 'wsgi.errors': sys.stderr, - 'wsgi.input': req.rfile, - 'wsgi.multiprocess': False, - 'wsgi.multithread': True, - 'wsgi.run_once': False, - 'wsgi.url_scheme': req.scheme, - 'wsgi.version': (1, 0), - } - - if isinstance(req.server.bind_addr, basestring): - # AF_UNIX. This isn't really allowed by WSGI, which doesn't - # address unix domain sockets. But it's better than nothing. - env["SERVER_PORT"] = "" - else: - env["SERVER_PORT"] = str(req.server.bind_addr[1]) - - # Request headers - for k, v in req.inheaders.iteritems(): - env["HTTP_" + k.upper().replace("-", "_")] = v - - # CONTENT_TYPE/CONTENT_LENGTH - ct = env.pop("HTTP_CONTENT_TYPE", None) - if ct is not None: - env["CONTENT_TYPE"] = ct - cl = env.pop("HTTP_CONTENT_LENGTH", None) - if cl is not None: - env["CONTENT_LENGTH"] = cl - - if req.conn.ssl_env: - env.update(req.conn.ssl_env) - - return env - - -class WSGIGateway_u0(WSGIGateway_10): - """A Gateway class to interface HTTPServer with WSGI u.0. - - WSGI u.0 is an experimental protocol, which uses unicode for keys and values - in both Python 2 and Python 3. - """ - - def get_environ(self): - """Return a new environ dict targeting the given wsgi.version""" - req = self.req - env_10 = WSGIGateway_10.get_environ(self) - env = dict([(k.decode('ISO-8859-1'), v) for k, v in env_10.iteritems()]) - env[u'wsgi.version'] = ('u', 0) - - # Request-URI - env.setdefault(u'wsgi.url_encoding', u'utf-8') - try: - for key in [u"PATH_INFO", u"SCRIPT_NAME", u"QUERY_STRING"]: - env[key] = env_10[str(key)].decode(env[u'wsgi.url_encoding']) - except UnicodeDecodeError: - # Fall back to latin 1 so apps can transcode if needed. - env[u'wsgi.url_encoding'] = u'ISO-8859-1' - for key in [u"PATH_INFO", u"SCRIPT_NAME", u"QUERY_STRING"]: - env[key] = env_10[str(key)].decode(env[u'wsgi.url_encoding']) - - for k, v in sorted(env.items()): - if isinstance(v, str) and k not in ('REQUEST_URI', 'wsgi.input'): - env[k] = v.decode('ISO-8859-1') - - return env - -wsgi_gateways = { - (1, 0): WSGIGateway_10, - ('u', 0): WSGIGateway_u0, -} - -class WSGIPathInfoDispatcher(object): - """A WSGI dispatcher for dispatch based on the PATH_INFO. - - apps: a dict or list of (path_prefix, app) pairs. - """ - - def __init__(self, apps): - try: - apps = list(apps.items()) - except AttributeError: - pass - - # Sort the apps by len(path), descending - apps.sort(cmp=lambda x,y: cmp(len(x[0]), len(y[0]))) - apps.reverse() - - # The path_prefix strings must start, but not end, with a slash. - # Use "" instead of "/". - self.apps = [(p.rstrip("/"), a) for p, a in apps] - - def __call__(self, environ, start_response): - path = environ["PATH_INFO"] or "/" - for p, app in self.apps: - # The apps list should be sorted by length, descending. - if path.startswith(p + "/") or path == p: - environ = environ.copy() - environ["SCRIPT_NAME"] = environ["SCRIPT_NAME"] + p - environ["PATH_INFO"] = path[len(p):] - return app(environ, start_response) - - start_response('404 Not Found', [('Content-Type', 'text/plain'), - ('Content-Length', '0')]) - return [''] -- cgit v1.2.3