Sun Jul 11 20:55:25 GMT Daylight Time 2010 david-sarah@jacaranda.org * Allow URIs passed in the initial JSON for t=mkdir-with-children, t=mkdir-immutable to be Unicode. Also pass the name of each child into nodemaker.create_from_cap for error reporting. Mon Jul 12 01:30:15 GMT Daylight Time 2010 david-sarah@jacaranda.org * Rename stringutils to encodingutil, and drop listdir_unicode and open_unicode (since the Python stdlib functions work fine with Unicode paths). Also move some utility functions to fileutil. New patches: [Allow URIs passed in the initial JSON for t=mkdir-with-children, t=mkdir-immutable to be Unicode. Also pass the name of each child into nodemaker.create_from_cap for error reporting. david-sarah@jacaranda.org**20100711195525 Ignore-this: deac32d8b91ba26ede18905d3f7d2b93 ] { hunk ./src/allmydata/web/common.py 14 EmptyPathnameComponentError, MustBeDeepImmutableError, \ MustBeReadonlyError, MustNotBeUnknownRWError from allmydata.mutable.common import UnrecoverableFileError -from allmydata.util import abbreviate # TODO: consolidate +from allmydata.util import abbreviate +from allmydata.util.stringutils import to_str class IOpHandleTable(Interface): pass hunk ./src/allmydata/web/common.py 68 children = {} if children_json: data = simplejson.loads(children_json) - for (name, (ctype, propdict)) in data.iteritems(): - name = unicode(name) - writecap = propdict.get("rw_uri") - if writecap is not None: - writecap = str(writecap) - readcap = propdict.get("ro_uri") - if readcap is not None: - readcap = str(readcap) + for (namex, (ctype, propdict)) in data.iteritems(): + namex = unicode(namex) + writecap = to_str(propdict.get("rw_uri")) + readcap = to_str(propdict.get("ro_uri")) metadata = propdict.get("metadata", {}) hunk ./src/allmydata/web/common.py 73 - childnode = nodemaker.create_from_cap(writecap, readcap) - children[name] = (childnode, metadata) + # name= argument is just for error reporting + childnode = nodemaker.create_from_cap(writecap, readcap, name=namex) + children[namex] = (childnode, metadata) return children def abbreviate_time(data): } [Rename stringutils to encodingutil, and drop listdir_unicode and open_unicode (since the Python stdlib functions work fine with Unicode paths). Also move some utility functions to fileutil. david-sarah@jacaranda.org**20100712003015 Ignore-this: 103b809d180df17a7283077c3104c7be ] { move ./src/allmydata/test/test_stringutils.py ./src/allmydata/test/test_encodingutil.py move ./src/allmydata/util/stringutils.py ./src/allmydata/util/encodingutil.py hunk ./src/allmydata/dirnode.py 19 DeepCheckAndRepairResults from allmydata.monitor import Monitor from allmydata.util import hashutil, mathutil, base32, log -from allmydata.util.stringutils import quote_output +from allmydata.util.encodingutil import quote_output from allmydata.util.assertutil import precondition from allmydata.util.netstring import netstring, split_netstring from allmydata.util.consumer import download_to_data hunk ./src/allmydata/scripts/cli.py 4 import os.path, re, sys, fnmatch from twisted.python import usage from allmydata.scripts.common import BaseOptions, get_aliases -from allmydata.util.stringutils import argv_to_unicode +from allmydata.util.encodingutil import argv_to_unicode NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?") hunk ./src/allmydata/scripts/common.py 5 import os, sys, urllib import codecs from twisted.python import usage -from allmydata.util.stringutils import unicode_to_url, quote_output +from allmydata.util.encodingutil import unicode_to_url, quote_output from allmydata.util.assertutil import precondition class BaseOptions: hunk ./src/allmydata/scripts/common_http.py 6 import urlparse, httplib import allmydata # for __full_version__ -from allmydata.util.stringutils import quote_output +from allmydata.util.encodingutil import quote_output from allmydata.scripts.common import TahoeError hunk ./src/allmydata/scripts/slow_operation.py 7 UnknownAliasError from allmydata.scripts.common_http import do_http, format_http_error from allmydata.util import base32 -from allmydata.util.stringutils import quote_output, is_printable_ascii +from allmydata.util.encodingutil import quote_output, is_printable_ascii import urllib import simplejson hunk ./src/allmydata/scripts/tahoe_add_alias.py 8 from allmydata.scripts.common_http import do_http, check_http_error from allmydata.scripts.common import get_aliases from allmydata.util.fileutil import move_into_place -from allmydata.util.stringutils import unicode_to_output, quote_output +from allmydata.util.encodingutil import unicode_to_output, quote_output def add_line_to_aliasfile(aliasfile, alias, cap): hunk ./src/allmydata/scripts/tahoe_backup.py 12 from allmydata.scripts.common_http import do_http, HTTPError, format_http_error from allmydata.util import time_format from allmydata.scripts import backupdb -from allmydata.util.stringutils import listdir_unicode, open_unicode, quote_output, to_str +from allmydata.util.encodingutil import quote_output, to_str +from allmydata.util.fileutil import open_expanduser from allmydata.util.assertutil import precondition hunk ./src/allmydata/scripts/tahoe_backup.py 170 compare_contents = {} # childname -> rocap try: - children = listdir_unicode(localpath) + children = os.listdir(localpath) except EnvironmentError: self.directories_skipped += 1 self.warn("WARNING: permission denied on directory %s" % quote_output(localpath)) hunk ./src/allmydata/scripts/tahoe_backup.py 175 children = [] + except (UnicodeEncodeError, UnicodeDecodeError): + self.directories_skipped += 1 + self.warn("WARNING: could not list directory %s due to an encoding error" % quote_output(localpath)) + children = [] for child in self.options.filter_listdir(children): assert isinstance(child, unicode), child hunk ./src/allmydata/scripts/tahoe_backup.py 300 if must_upload: self.verboseprint("uploading %s.." % quote_output(childpath)) - infileobj = open_unicode(childpath, "rb") + infileobj = open_expanduser(childpath, "rb") url = self.options['node-url'] + "uri" resp = do_http("PUT", url, infileobj) if resp.status not in (200, 201): hunk ./src/allmydata/scripts/tahoe_check.py 8 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError from allmydata.scripts.common_http import do_http, format_http_error -from allmydata.util.stringutils import quote_output, quote_path +from allmydata.util.encodingutil import quote_output, quote_path class Checker: pass hunk ./src/allmydata/scripts/tahoe_cp.py 11 DefaultAliasMarker, TahoeError from allmydata.scripts.common_http import do_http, HTTPError from allmydata import uri -from allmydata.util.stringutils import unicode_to_url, listdir_unicode, open_unicode, \ - abspath_expanduser_unicode, quote_output, to_str +from allmydata.util.encodingutil import unicode_to_url, quote_output, to_str +from allmydata.util import fileutil +from allmydata.util.fileutil import open_expanduser, abspath_expanduser from allmydata.util.assertutil import precondition hunk ./src/allmydata/scripts/tahoe_cp.py 17 -def _put_local_file(pathname, inf): - # TODO: create temporary file and move into place? - # TODO: move this to fileutil. - outf = open_unicode(pathname, "wb") - try: - while True: - data = inf.read(32768) - if not data: - break - outf.write(data) - finally: - outf.close() - - class MissingSourceError(TahoeError): def __init__(self, name): TahoeError.__init__(self, "No such file or directory %s" % quote_output(name)) hunk ./src/allmydata/scripts/tahoe_cp.py 71 return True def open(self, caps_only): - return open_unicode(self.pathname, "rb") + return open_expanduser(self.pathname, "rb") class LocalFileTarget: hunk ./src/allmydata/scripts/tahoe_cp.py 80 self.pathname = pathname def put_file(self, inf): - _put_local_file(self.pathname, inf) + fileutil.put_file(self.pathname, inf) class LocalMissingTarget: hunk ./src/allmydata/scripts/tahoe_cp.py 89 self.pathname = pathname def put_file(self, inf): - _put_local_file(self.pathname, inf) + fileutil.put_file(self.pathname, inf) class LocalDirectorySource: hunk ./src/allmydata/scripts/tahoe_cp.py 104 if self.children is not None: return self.children = {} - children = listdir_unicode(self.pathname) + children = os.listdir(self.pathname) for i,n in enumerate(children): self.progressfunc("examining %d of %d" % (i, len(children))) pn = os.path.join(self.pathname, n) hunk ./src/allmydata/scripts/tahoe_cp.py 132 if self.children is not None: return self.children = {} - children = listdir_unicode(self.pathname) + children = os.listdir(self.pathname) for i,n in enumerate(children): self.progressfunc("examining %d of %d" % (i, len(children))) n = unicode(n) hunk ./src/allmydata/scripts/tahoe_cp.py 158 def put_file(self, name, inf): precondition(isinstance(name, unicode), name) pathname = os.path.join(self.pathname, name) - _put_local_file(pathname, inf) + fileutil.put_file(pathname, inf) def set_children(self): pass hunk ./src/allmydata/scripts/tahoe_cp.py 515 rootcap, path = get_alias(self.aliases, destination_spec, None) if rootcap == DefaultAliasMarker: # no alias, so this is a local file - pathname = abspath_expanduser_unicode(path.decode('utf-8')) + pathname = abspath_expanduser(path.decode('utf-8')) if not os.path.exists(pathname): t = LocalMissingTarget(pathname) elif os.path.isdir(pathname): hunk ./src/allmydata/scripts/tahoe_cp.py 555 rootcap, path = get_alias(self.aliases, source_spec, None) if rootcap == DefaultAliasMarker: # no alias, so this is a local file - pathname = abspath_expanduser_unicode(path.decode('utf-8')) + pathname = abspath_expanduser(path.decode('utf-8')) name = os.path.basename(pathname) if not os.path.exists(pathname): raise MissingSourceError(source_spec) hunk ./src/allmydata/scripts/tahoe_get.py 6 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError from allmydata.scripts.common_http import do_http, format_http_error -from allmydata.util.stringutils import open_unicode +from allmydata.util.fileutil import open_expanduser def get(options): nodeurl = options['node-url'] hunk ./src/allmydata/scripts/tahoe_get.py 30 resp = do_http("GET", url) if resp.status in (200, 201,): if to_file: - outf = open_unicode(to_file, "wb") + outf = open_expanduser(to_file, "wb") else: outf = stdout while True: hunk ./src/allmydata/scripts/tahoe_ls.py 7 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError from allmydata.scripts.common_http import do_http, format_http_error -from allmydata.util.stringutils import unicode_to_output, quote_output, is_printable_ascii, to_str +from allmydata.util.encodingutil import unicode_to_output, quote_output, is_printable_ascii, to_str def list(options): nodeurl = options['node-url'] hunk ./src/allmydata/scripts/tahoe_manifest.py 9 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError from allmydata.scripts.common_http import do_http, format_http_error -from allmydata.util.stringutils import quote_output, quote_path +from allmydata.util.encodingutil import quote_output, quote_path class FakeTransport: disconnecting = False hunk ./src/allmydata/scripts/tahoe_mkdir.py 5 import urllib from allmydata.scripts.common_http import do_http, check_http_error from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, UnknownAliasError -from allmydata.util.stringutils import quote_output +from allmydata.util.encodingutil import quote_output def mkdir(options): nodeurl = options['node-url'] hunk ./src/allmydata/scripts/tahoe_mv.py 8 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError from allmydata.scripts.common_http import do_http, format_http_error -from allmydata.util.stringutils import to_str +from allmydata.util.encodingutil import to_str # this script is used for both 'mv' and 'ln' hunk ./src/allmydata/scripts/tahoe_put.py 7 from allmydata.scripts.common_http import do_http, format_http_success, format_http_error from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError -from allmydata.util.stringutils import quote_output, open_unicode +from allmydata.util.encodingutil import quote_output +from allmydata.util.fileutil import open_expanduser def put(options): """ hunk ./src/allmydata/scripts/tahoe_put.py 68 if mutable: url += "?mutable=true" if from_file: - infileobj = open_unicode(from_file, "rb") + infileobj = open_expanduser(from_file, "rb") else: # do_http() can't use stdin directly: for one thing, we need a # Content-Length field. So we currently must copy it. hunk ./src/allmydata/test/test_backupdb.py 7 from twisted.trial import unittest from allmydata.util import fileutil -from allmydata.util.stringutils import listdir_unicode, get_filesystem_encoding, unicode_platform +from allmydata.util.encodingutil import get_filesystem_encoding, unicode_platform from allmydata.util.assertutil import precondition from allmydata.scripts import backupdb hunk ./src/allmydata/test/test_backupdb.py 252 self.failUnless(bdb) self.writeto(u"f\u00f6\u00f6.txt", "foo.txt") - files = [fn for fn in listdir_unicode(unicode(basedir)) if fn.endswith(".txt")] + files = [fn for fn in os.listdir(unicode(basedir)) if fn.endswith(".txt")] self.failUnlessEqual(len(files), 1) foo_fn = os.path.join(basedir, files[0]) #print foo_fn, type(foo_fn) hunk ./src/allmydata/test/test_cli.py 34 from twisted.python import usage from allmydata.util.assertutil import precondition -from allmydata.util.stringutils import listdir_unicode, open_unicode, unicode_platform, \ - quote_output, get_output_encoding, get_argv_encoding, get_filesystem_encoding, \ +from allmydata.util.encodingutil import unicode_platform, quote_output, \ + get_output_encoding, get_argv_encoding, get_filesystem_encoding, \ unicode_to_output, to_str timeout = 480 # deep_check takes 360s on Zandr's linksys box, others take > 240s hunk ./src/allmydata/test/test_cli.py 442 fileutil.make_dirs(basedir) for name in filenames: - open_unicode(os.path.join(unicode(basedir), name), "wb").close() + open(os.path.join(unicode(basedir), name), "wb").close() hunk ./src/allmydata/test/test_cli.py 444 - for file in listdir_unicode(unicode(basedir)): + for file in os.listdir(unicode(basedir)): self.failUnlessIn(normalize(file), filenames) hunk ./src/allmydata/test/test_cli.py 977 rel_fn = os.path.join(unicode(self.basedir), u"à trier.txt") # we make the file small enough to fit in a LIT file, for speed DATA = "short file" - f = open_unicode(rel_fn, "wb") - try: - f.write(DATA) - finally: - f.close() + fileutil.write(rel_fn, DATA) d = self.do_cli("create-alias", "tahoe") hunk ./src/allmydata/test/test_cli.py 1348 self.set_up_grid() DATA1 = "unicode file content" - f = open_unicode(fn1, "wb") - try: - f.write(DATA1) - finally: - f.close() + fileutil.write(fn1, DATA1) fn2 = os.path.join(self.basedir, "Metallica") DATA2 = "non-unicode file content" hunk ./src/allmydata/test/test_encodingutil.py 16 # systems. if __name__ == "__main__": - import sys, os - import tempfile - import shutil + import sys import platform if len(sys.argv) != 2: hunk ./src/allmydata/test/test_encodingutil.py 32 print " filesystem_encoding = '%s'" % sys.getfilesystemencoding() print " output_encoding = '%s'" % sys.stdout.encoding print " argv_encoding = '%s'" % (sys.platform == "win32" and 'ascii' or sys.stdout.encoding) - - try: - tmpdir = tempfile.mkdtemp() - for fname in TEST_FILENAMES: - open(os.path.join(tmpdir, fname), 'w').close() - - # Use Unicode API under Windows or MacOS X - if sys.platform in ('win32', 'darwin'): - dirlist = os.listdir(unicode(tmpdir)) - else: - dirlist = os.listdir(tmpdir) - - print " dirlist = %s" % repr(dirlist) - except: - print " # Oops, I cannot write filenames containing non-ascii characters" print hunk ./src/allmydata/test/test_encodingutil.py 34 - shutil.rmtree(tmpdir) sys.exit(0) from twisted.trial import unittest hunk ./src/allmydata/test/test_encodingutil.py 41 import sys from allmydata.test.common_util import ReallyEqualMixin -from allmydata.util.stringutils import argv_to_unicode, unicode_to_url, \ - unicode_to_output, unicode_platform, listdir_unicode, open_unicode, \ - FilenameEncodingError, get_output_encoding, _reload -from allmydata.dirnode import normalize +from allmydata.util.encodingutil import argv_to_unicode, unicode_to_url, \ + unicode_to_output, unicode_platform, get_output_encoding, _reload from twisted.python import usage hunk ./src/allmydata/test/test_encodingutil.py 85 _reload() self.failUnlessRaises(UnicodeEncodeError, unicode_to_output, lumiere_nfc) - @patch('os.listdir') - def test_no_unicode_normalization(self, mock): - # Pretend to run on a Unicode platform. - # We normalized to NFC in 1.7beta, but we now don't. - orig_platform = sys.platform - try: - sys.platform = 'darwin' - mock.return_value = [Artonwall_nfd] - _reload() - self.failUnlessReallyEqual(listdir_unicode(u'/dummy'), [Artonwall_nfd]) - finally: - sys.platform = orig_platform - -# The following tests applies only to platforms which don't store filenames as -# Unicode entities on the filesystem. -class StringUtilsNonUnicodePlatform(unittest.TestCase): - def setUp(self): - # Mock sys.platform because unicode_platform() uses it - self.original_platform = sys.platform - sys.platform = 'linux' - - def tearDown(self): - sys.platform = self.original_platform - _reload() - - @patch('sys.getfilesystemencoding') - @patch('os.listdir') - def test_listdir_unicode(self, mock_listdir, mock_getfilesystemencoding): - # What happens if latin1-encoded filenames are encountered on an UTF-8 - # filesystem? - mock_listdir.return_value = [ - lumiere_nfc.encode('utf-8'), - lumiere_nfc.encode('latin1')] - - mock_getfilesystemencoding.return_value = 'utf-8' - _reload() - self.failUnlessRaises(FilenameEncodingError, - listdir_unicode, - u'/dummy') - - # We're trying to list a directory whose name cannot be represented in - # the filesystem encoding. This should fail. - mock_getfilesystemencoding.return_value = 'ascii' - _reload() - self.failUnlessRaises(FilenameEncodingError, - listdir_unicode, - u'/' + lumiere_nfc) - - @patch('sys.getfilesystemencoding') - def test_open_unicode(self, mock): - mock.return_value = 'ascii' - _reload() - self.failUnlessRaises(FilenameEncodingError, - open_unicode, - lumiere_nfc, 'rb') class StringUtils(ReallyEqualMixin): def setUp(self): hunk ./src/allmydata/test/test_encodingutil.py 130 _reload() self.failUnlessReallyEqual(unicode_platform(), matrix[self.platform]) - @patch('sys.getfilesystemencoding') - @patch('os.listdir') - def test_listdir_unicode(self, mock_listdir, mock_getfilesystemencoding): - if 'dirlist' not in dir(self): - return - - try: - u"test".encode(self.filesystem_encoding) - except (LookupError, AttributeError): - raise unittest.SkipTest("This platform does not support the '%s' filesystem encoding " - "that we are testing for the benefit of a different platform." - % (self.filesystem_encoding,)) - - mock_listdir.return_value = self.dirlist - mock_getfilesystemencoding.return_value = self.filesystem_encoding - - _reload() - filenames = listdir_unicode(u'/dummy') - - self.failUnlessEqual(set([normalize(fname) for fname in filenames]), - set(TEST_FILENAMES)) - - @patch('sys.getfilesystemencoding') - @patch('__builtin__.open') - def test_open_unicode(self, mock_open, mock_getfilesystemencoding): - mock_getfilesystemencoding.return_value = self.filesystem_encoding - fn = u'/dummy_directory/" + lumiere_nfc + ".txt' - - try: - u"test".encode(self.filesystem_encoding) - except (LookupError, AttributeError): - raise unittest.SkipTest("This platform does not support the '%s' filesystem encoding " - "that we are testing for the benefit of a different platform." - % (self.filesystem_encoding,)) - - _reload() - try: - open_unicode(fn, 'rb') - except FilenameEncodingError: - return - - # Pass Unicode string to open() on Unicode platforms - if unicode_platform(): - mock_open.assert_called_with(fn, 'rb') - - # Pass correctly encoded bytestrings to open() on non-Unicode platforms - else: - fn_bytestring = fn.encode(self.filesystem_encoding) - mock_open.assert_called_with(fn_bytestring, 'rb') - class UbuntuKarmicUTF8(StringUtils, unittest.TestCase): uname = 'Linux korn 2.6.31-14-generic #48-Ubuntu SMP Fri Oct 16 14:05:01 UTC 2009 x86_64' hunk ./src/allmydata/test/test_encodingutil.py 139 filesystem_encoding = 'UTF-8' output_encoding = 'UTF-8' argv_encoding = 'UTF-8' - dirlist = ['test_file', '\xc3\x84rtonwall.mp3', 'Blah blah.txt'] class UbuntuKarmicLatin1(StringUtils, unittest.TestCase): uname = 'Linux korn 2.6.31-14-generic #48-Ubuntu SMP Fri Oct 16 14:05:01 UTC 2009 x86_64' hunk ./src/allmydata/test/test_encodingutil.py 148 filesystem_encoding = 'ISO-8859-1' output_encoding = 'ISO-8859-1' argv_encoding = 'ISO-8859-1' - dirlist = ['test_file', 'Blah blah.txt', '\xc4rtonwall.mp3'] class WindowsXP(StringUtils, unittest.TestCase): uname = 'Windows XP 5.1.2600 x86 x86 Family 15 Model 75 Step ping 2, AuthenticAMD' hunk ./src/allmydata/test/test_encodingutil.py 156 filesystem_encoding = 'mbcs' output_encoding = 'cp850' argv_encoding = 'ascii' - dirlist = [u'Blah blah.txt', u'test_file', u'\xc4rtonwall.mp3'] class WindowsXP_UTF8(StringUtils, unittest.TestCase): uname = 'Windows XP 5.1.2600 x86 x86 Family 15 Model 75 Step ping 2, AuthenticAMD' hunk ./src/allmydata/test/test_encodingutil.py 164 filesystem_encoding = 'mbcs' output_encoding = 'cp65001' argv_encoding = 'ascii' - dirlist = [u'Blah blah.txt', u'test_file', u'\xc4rtonwall.mp3'] class WindowsVista(StringUtils, unittest.TestCase): uname = 'Windows Vista 6.0.6000 x86 x86 Family 6 Model 15 Stepping 11, GenuineIntel' hunk ./src/allmydata/test/test_encodingutil.py 172 filesystem_encoding = 'mbcs' output_encoding = 'cp850' argv_encoding = 'ascii' - dirlist = [u'Blah blah.txt', u'test_file', u'\xc4rtonwall.mp3'] class MacOSXLeopard(StringUtils, unittest.TestCase): uname = 'Darwin g5.local 9.8.0 Darwin Kernel Version 9.8.0: Wed Jul 15 16:57:01 PDT 2009; root:xnu-1228.15.4~1/RELEASE_PPC Power Macintosh powerpc' hunk ./src/allmydata/test/test_encodingutil.py 181 filesystem_encoding = 'utf-8' output_encoding = 'UTF-8' argv_encoding = 'UTF-8' - dirlist = [u'A\u0308rtonwall.mp3', u'Blah blah.txt', u'test_file'] class MacOSXLeopard7bit(StringUtils, unittest.TestCase): uname = 'Darwin g5.local 9.8.0 Darwin Kernel Version 9.8.0: Wed Jul 15 16:57:01 PDT 2009; root:xnu-1228.15.4~1/RELEASE_PPC Power Macintosh powerpc' hunk ./src/allmydata/test/test_encodingutil.py 188 filesystem_encoding = 'utf-8' output_encoding = 'US-ASCII' argv_encoding = 'US-ASCII' - dirlist = [u'A\u0308rtonwall.mp3', u'Blah blah.txt', u'test_file'] class OpenBSD(StringUtils, unittest.TestCase): uname = 'OpenBSD 4.1 GENERIC#187 i386 Intel(R) Celeron(R) CPU 2.80GHz ("GenuineIntel" 686-class)' hunk ./src/allmydata/util/encodingutil.py 7 """ import sys -import os import re from allmydata.util.assertutil import precondition from twisted.python import usage hunk ./src/allmydata/util/encodingutil.py 176 Does the current platform handle Unicode filenames natively? """ return is_unicode_platform - -class FilenameEncodingError(Exception): - """ - Filename cannot be encoded using the current encoding of your filesystem - (%s). Please configure your locale correctly or rename this file. - """ - pass - -def listdir_unicode_fallback(path): - """ - This function emulates a fallback Unicode API similar to one available - under Windows or MacOS X. - - If badly encoded filenames are encountered, an exception is raised. - """ - precondition(isinstance(path, unicode), path) - - try: - byte_path = path.encode(filesystem_encoding) - except (UnicodeEncodeError, UnicodeDecodeError): - raise FilenameEncodingError(path) - - try: - return [unicode(fn, filesystem_encoding) for fn in os.listdir(byte_path)] - except UnicodeDecodeError: - raise FilenameEncodingError(fn) - -def listdir_unicode(path): - """ - Wrapper around listdir() which provides safe access to the convenient - Unicode API even under platforms that don't provide one natively. - """ - precondition(isinstance(path, unicode), path) - - # On Windows and MacOS X, the Unicode API is used - # On other platforms (ie. Unix systems), the byte-level API is used - - if is_unicode_platform: - return os.listdir(path) - else: - return listdir_unicode_fallback(path) - -def open_unicode(path, mode): - """ - Wrapper around open() which provides safe access to the convenient Unicode - API even under Unix. - """ - precondition(isinstance(path, unicode), path) - - if is_unicode_platform: - return open(os.path.expanduser(path), mode) - else: - try: - return open(os.path.expanduser(path.encode(filesystem_encoding)), mode) - except UnicodeEncodeError: - raise FilenameEncodingError(path) - -def abspath_expanduser_unicode(path): - precondition(isinstance(path, unicode), path) - - if is_unicode_platform: - return os.path.abspath(os.path.expanduser(path)) - else: - try: - pathstr = path.encode(filesystem_encoding) - return os.path.abspath(os.path.expanduser(pathstr)).decode(filesystem_encoding) - except (UnicodeEncodeError, UnicodeDecodeError): - raise FilenameEncodingError(path) hunk ./src/allmydata/util/fileutil.py 212 finally: rf.close() +def put_file(pathname, inf): + # TODO: create temporary file and move into place? + outf = open_expanduser(pathname, "wb") + try: + while True: + data = inf.read(32768) + if not data: + break + outf.write(data) + finally: + outf.close() + +def open_expanduser(path, mode): + assert isinstance(path, unicode), path + return open(os.path.expanduser(path), mode) + +def abspath_expanduser(path): + assert isinstance(path, unicode), path + return os.path.abspath(os.path.expanduser(path)) + hunk ./src/allmydata/web/common.py 15 MustBeReadonlyError, MustNotBeUnknownRWError from allmydata.mutable.common import UnrecoverableFileError from allmydata.util import abbreviate -from allmydata.util.stringutils import to_str +from allmydata.util.encodingutil import to_str class IOpHandleTable(Interface): pass } Context: [upcase_since_on_welcome terrellrussell@gmail.com**20100708193903] [server_version_on_welcome_page.dpatch.txt freestorm77@gmail.com**20100605191721 Ignore-this: b450c76dc875f5ac8cca229a666cbd0a - The storage server version is 0 for all storage nodes in the Welcome Page ] [NEWS: add NEWS snippets about two recent patches zooko@zooko.com**20100708162058 Ignore-this: 6c9da6a0ad7351a960bdd60f81532899 ] [directory_html_top_banner.dpatch freestorm77@gmail.com**20100622205301 Ignore-this: 1d770d975e0c414c996564774f049bca The div tag with the link "Return to Welcome page" on the directory.xhtml page is not correct ] [tahoe_css_toolbar.dpatch freestorm77@gmail.com**20100622210046 Ignore-this: 5b3ebb2e0f52bbba718a932f80c246c0 CSS modification to be correctly diplayed with Internet Explorer 8 The links on the top of page directory.xhtml are not diplayed in the same line as display with Firefox. ] [runnin_test_tahoe_css.dpatch freestorm77@gmail.com**20100622214714 Ignore-this: e0db73d68740aad09a7b9ae60a08c05c Runnin test for changes in tahoe.css file ] [runnin_test_directory_xhtml.dpatch freestorm77@gmail.com**20100622201403 Ignore-this: f8962463fce50b9466405cb59fe11d43 Runnin test for diretory.xhtml top banner ] [stringutils.py: tolerate sys.stdout having no 'encoding' attribute. david-sarah@jacaranda.org**20100626040817 Ignore-this: f42cad81cef645ee38ac1df4660cc850 ] [quickstart.html: python 2.5 -> 2.6 as recommended version david-sarah@jacaranda.org**20100705175858 Ignore-this: bc3a14645ea1d5435002966ae903199f ] [SFTP: don't call .stopProducing on the producer registered with OverwriteableFileConsumer (which breaks with warner's new downloader). david-sarah@jacaranda.org**20100628231926 Ignore-this: 131b7a5787bc85a9a356b5740d9d996f ] [docs/how_to_make_a_tahoe-lafs_release.txt: trivial correction, install.html should now be quickstart.html. david-sarah@jacaranda.org**20100625223929 Ignore-this: 99a5459cac51bd867cc11ad06927ff30 ] [setup: in the Makefile, refuse to upload tarballs unless someone has passed the environment variable "BB_BRANCH" with value "trunk" zooko@zooko.com**20100619034928 Ignore-this: 276ddf9b6ad7ec79e27474862e0f7d6 ] [trivial: tiny update to in-line comment zooko@zooko.com**20100614045715 Ignore-this: 10851b0ed2abfed542c97749e5d280bc (I'm actually committing this patch as a test of the new eager-annotation-computation of trac-darcs.) ] [docs: about.html link to home page early on, and be decentralized storage instead of cloud storage this time around zooko@zooko.com**20100619065318 Ignore-this: dc6db03f696e5b6d2848699e754d8053 ] [docs: update about.html, especially to have a non-broken link to quickstart.html, and also to comment out the broken links to "for Paranoids" and "for Corporates" zooko@zooko.com**20100619065124 Ignore-this: e292c7f51c337a84ebfeb366fbd24d6c ] [TAG allmydata-tahoe-1.7.0 zooko@zooko.com**20100619052631 Ignore-this: d21e27afe6d85e2e3ba6a3292ba2be1 ] Patch bundle hash: 5dd0b81c02ed648d4eb4ab565a237c53ccbf4ff3