From 3a4dab4477d6cbb437bafb602f2d6734bb10cb64 Mon Sep 17 00:00:00 2001 From: zoidberg10 Date: Mon, 24 Oct 2011 16:30:09 +0200 Subject: HTTPRequest: skip BOM if response is 'Microsoft-UTF8' encoded --- module/network/HTTPRequest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'module/network') diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index d8d57e76f..99197314f 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -19,7 +19,7 @@ import pycurl -from codecs import getincrementaldecoder +from codecs import getincrementaldecoder, lookup, BOM_UTF8 from urllib import quote, urlencode from httplib import responses from logging import getLogger @@ -230,6 +230,9 @@ class HTTPRequest(): try: #self.log.debug("Decoded %s" % encoding ) + if lookup(encoding).name == 'utf-8' and rep.startswith(BOM_UTF8): + encoding = 'utf-8-sig' + decoder = getincrementaldecoder(encoding)("replace") rep = decoder.decode(rep, True) -- cgit v1.2.3