From 2ae58ff610f7b9c0bb7c19dae67cf8cf681c17c7 Mon Sep 17 00:00:00 2001 From: Luke Macken Date: Fri, 25 Mar 2011 11:31:49 -0400 Subject: [PATCH] Decode URLs to UTF8 before passing them to PyCurl to avoid errors diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py index 219a391..e2cd5c9 100644 --- a/urlgrabber/grabber.py +++ b/urlgrabber/grabber.py @@ -957,6 +957,7 @@ class URLGrabber(object): returned that supports them. The file object can be treated like any other file object. """ + url = _to_utf8(url) opts = self.opts.derive(**kwargs) if DEBUG: DEBUG.debug('combined options: %s' % repr(opts)) (url,parts) = opts.urlparser.parse(url, opts) @@ -970,6 +971,7 @@ class URLGrabber(object): urlgrab returns the filename of the local file, which may be different from the passed-in filename if copy_local == 0. """ + url = _to_utf8(url) opts = self.opts.derive(**kwargs) if DEBUG: DEBUG.debug('combined options: %s' % repr(opts)) (url,parts) = opts.urlparser.parse(url, opts) @@ -1030,6 +1032,7 @@ class URLGrabber(object): "I want the first N bytes" but rather 'read the whole file into memory, but don't use too much' """ + url = _to_utf8(url) opts = self.opts.derive(**kwargs) if DEBUG: DEBUG.debug('combined options: %s' % repr(opts)) (url,parts) = opts.urlparser.parse(url, opts) -- 1.7.4