Wed Jul 14 03:20:59 GMT Daylight Time 2010 david-sarah@jacaranda.org * Changes to Tahoe needed to work with new zetuptoolz (that does not use .exe wrappers on Windows), and to support Unicode arguments and stdout/stderr. New patches: [Changes to Tahoe needed to work with new zetuptoolz (that does not use .exe wrappers on Windows), and to support Unicode arguments and stdout/stderr. david-sarah@jacaranda.org**20100714022059 Ignore-this: bae64a497f95e834852de7466ca04066 ] { hunk ./bin/tahoe-script.template 8 where = os.path.realpath(sys.argv[0]) base = os.path.dirname(os.path.dirname(where)) +if sys.platform == "win32": + installed_tahoe = os.path.join(os.path.dirname(sys.executable), 'Scripts', 'tahoe.pyscript') +else: + installed_tahoe = "/usr/bin/tahoe" + whoami = '''\ I am a "bin/tahoe" executable who is only for the convenience of running Tahoe from its source distribution -- I work only when invoked as the "tahoe" hunk ./bin/tahoe-script.template 27 print '''\ I just tried to run and found that I am not living in such a directory, so I am stopping now. To run Tahoe after it has been is installed, please execute -my brother, also named "tahoe", who gets installed into the appropriate place -for executables when you run "make install" (perhaps as /usr/bin/tahoe). -''' +my brother, who gets installed into the appropriate place for executables +when you run "make install" (perhaps as %s). +''' % (installed_tahoe,) sys.exit(1) # we've found our home. Put the tahoe support/lib etc. in our PYTHONPATH. hunk ./bin/tahoe-script.template 49 pp = supportdir os.environ["PYTHONPATH"] = pp -# find the location of the tahoe executable. -bin_dir = "bin" +# find commandline args and the location of the tahoe executable. if sys.platform == "win32": hunk ./bin/tahoe-script.template 51 - bin_dir = "Scripts" -executable = os.path.join(base, "support", bin_dir, "tahoe") + import re + from ctypes import WINFUNCTYPE, POINTER, byref, c_wchar_p, c_int, windll + + GetCommandLineW = WINFUNCTYPE(c_wchar_p)(("GetCommandLineW", windll.kernel32)) + CommandLineToArgvW = WINFUNCTYPE(POINTER(c_wchar_p), c_wchar_p, POINTER(c_int)) \ + (("CommandLineToArgvW", windll.shell32)) + + argc = c_int(0) + argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc)) + + # See src/allmydata/scripts/runner.py for the corresponding unmangler. + def mangle(s): + return str(re.sub(r'[^\x20-\x7E]', lambda m: '\x7F%x;' % (ord(m.group(0)),), s)) + + argv = [mangle(argv_unicode[i]) for i in xrange(1, argc.value)] + local_tahoe = "Scripts\\tahoe.pyscript" +else: + argv = sys.argv + local_tahoe = "bin/tahoe" + +script = os.path.join(base, "support", local_tahoe) try: hunk ./bin/tahoe-script.template 74 - res = subprocess.call([executable] + sys.argv[1:], env=os.environ) + res = subprocess.call([sys.executable, script] + argv[1:], env=os.environ) except (OSError, IOError), le: if le.args[0] == errno.ENOENT: print whoami hunk ./bin/tahoe-script.template 78 + print '''\ I just tried to run and could not find my brother, named hunk ./bin/tahoe-script.template 81 -"../support/bin/tahoe". To run Tahoe when it is installed, please execute my -brother, also named "tahoe", who gets installed into the appropriate place -for executables when you run "make install" (perhaps as /usr/bin/tahoe). -''' +"../support/%s". To run Tahoe when it is installed, please execute my +brother, who gets installed into the appropriate place for executables +when you run "make install" (perhaps as %s). +''' % (local_tahoe, installed_tahoe) raise except Exception, le: print whoami hunk ./bin/tahoe-script.template 89 print '''\ -I just tried to invoke my brother, named "../support/bin/tahoe" and got an +I just tried to invoke my brother, named "../support/%s" and got an exception. hunk ./bin/tahoe-script.template 91 -''' +''' % (local_tahoe,) raise else: sys.exit(res) hunk ./setup.py 238 def run(self): bin_tahoe_template = os.path.join("bin", "tahoe-script.template") - # Create the 'tahoe-script.py' file under the 'bin' directory. The - # 'tahoe-script.py' file is exactly the same as the - # 'tahoe-script.template' script except that the shebang line is - # rewritten to use our sys.executable for the interpreter. On - # Windows, create a tahoe.exe will execute it. On non-Windows, make a - # symlink to it from 'tahoe'. The tahoe.exe will be copied from the - # setuptools egg's cli.exe and this will work from a zip-safe and - # non-zip-safe setuptools egg. + if sys.platform == 'win32': + # 'tahoe' script is needed for cygwin + script_names = ["tahoe.pyscript", "tahoe"] + else: + script_names = ["tahoe"] + + # Create the tahoe script file under the 'bin' directory. This + # file is exactly the same as the 'tahoe-script.template' script + # except that the shebang line is rewritten to use our sys.executable + # for the interpreter. f = open(bin_tahoe_template, "rU") script_lines = f.readlines() f.close() hunk ./setup.py 251 - script_lines[0] = "#!%s\n" % sys.executable - tahoe_script = os.path.join("bin", "tahoe-script.py") - f = open(tahoe_script, "w") - for line in script_lines: - f.write(line) - f.close() - if sys.platform == "win32": - from pkg_resources import require - setuptools_egg = require("setuptools")[0].location - if os.path.isfile(setuptools_egg): - z = zipfile.ZipFile(setuptools_egg, 'r') - for filename in z.namelist(): - if 'cli.exe' in filename: - cli_exe = z.read(filename) - else: - cli_exe = os.path.join(setuptools_egg, 'setuptools', 'cli.exe') - tahoe_exe = os.path.join("bin", "tahoe.exe") - if os.path.isfile(setuptools_egg): - f = open(tahoe_exe, 'wb') - f.write(cli_exe) - f.close() - else: - shutil.copy(cli_exe, tahoe_exe) - else: + script_lines[0] = '#!%s\n' % sys.executable + for script_name in script_names: + tahoe_script = os.path.join("bin", script_name) try: hunk ./setup.py 255 - os.remove(os.path.join('bin', 'tahoe')) - except: + os.remove(tahoe_script) + except Exception: # okay, probably it was already gone pass hunk ./setup.py 259 - os.symlink('tahoe-script.py', os.path.join('bin', 'tahoe')) + f = open(tahoe_script, "wb") + for line in script_lines: + f.write(line) + f.close() + + # chmod +x + old_mode = stat.S_IMODE(os.stat(tahoe_script)[stat.ST_MODE]) + new_mode = old_mode | (stat.S_IXUSR | stat.S_IRUSR | + stat.S_IXGRP | stat.S_IRGRP | + stat.S_IXOTH | stat.S_IROTH ) + os.chmod(tahoe_script, new_mode) hunk ./setup.py 271 - # chmod +x bin/tahoe-script.py - old_mode = stat.S_IMODE(os.stat(tahoe_script)[stat.ST_MODE]) - new_mode = old_mode | (stat.S_IXUSR | stat.S_IRUSR | - stat.S_IXGRP | stat.S_IRGRP | - stat.S_IXOTH | stat.S_IROTH ) - os.chmod(tahoe_script, new_mode) + old_tahoe_exe = os.path.join("bin", "tahoe.exe") + try: + os.remove(old_tahoe_exe) + except Exception: + pass + if os.path.exists(old_tahoe_exe): + print >>sys.stderr, "%s could not be deleted. Please delete this file manually." % (old_tahoe_exe,) class MySdist(sdist.sdist): """ A hook in the sdist command so that we can determine whether this the hunk ./src/allmydata/scripts/runner.py 12 pkg_resources.require('allmydata-tahoe') from allmydata.scripts.common import BaseOptions from allmydata.scripts import debug, create_node, startstop_node, cli, keygen, stats_gatherer +from allmydata.util.encodingutil import quote_output, get_argv_encoding, _reload def GROUP(s): # Usage.parseOptions compares argv[1] against command[0], so it will hunk ./src/allmydata/scripts/runner.py 22 class Options(BaseOptions, usage.Options): - synopsis = "Usage: tahoe [command options]" + synopsis = "\nUsage: tahoe [command options]" subCommands = ( GROUP("Administration") + create_node.subCommands + keygen.subCommands hunk ./src/allmydata/scripts/runner.py 45 def runner(argv, run_by_human=True, - stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, + stdin=None, stdout=None, stderr=None, install_node_control=True, additional_commands=None): hunk ./src/allmydata/scripts/runner.py 48 + stdin = stdin or sys.stdin + stdout = stdout or sys.stdout + stderr = stderr or sys.stderr + config = Options() if install_node_control: config.subCommands.extend(startstop_node.subCommands) hunk ./src/allmydata/scripts/runner.py 70 c = config while hasattr(c, 'subOptions'): c = c.subOptions - print str(c) - print "%s: %s" % (sys.argv[0], e) + print >>stdout, str(c) + try: + msg = e.args[0].decode(get_argv_encoding()) + except Exception: + msg = repr(e) + print >>stdout, "%s: %s\n" % (sys.argv[0], quote_output(msg, quotemarks=False)) return 1 command = config.subCommand hunk ./src/allmydata/scripts/runner.py 110 return rc + def run(install_node_control=True): hunk ./src/allmydata/scripts/runner.py 112 - rc = runner(sys.argv[1:]) + if sys.platform == "win32": + windows_fixups() + + rc = runner(sys.argv[1:], install_node_control=install_node_control) sys.exit(rc) hunk ./src/allmydata/scripts/runner.py 117 + + +def windows_fixups(): + import codecs, re + from ctypes import WINFUNCTYPE, windll, CFUNCTYPE, cdll, POINTER, byref, \ + c_wchar_p, c_char_p, c_void_p, c_int, c_size_t + + # Work around . + codecs.register(lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None) + + # Make Unicode console output work independently of the current code page. + # This also fixes . + try: + STDOUT_FILENO = 1 + STDERR_FILENO = 2 + fix_stdout = hasattr(sys.stdout, 'fileno') and sys.stdout.fileno() == STDOUT_FILENO + fix_stderr = hasattr(sys.stderr, 'fileno') and sys.stderr.fileno() == STDERR_FILENO + + if fix_stdout or fix_stderr: + # FILE * _fdopen(int fd, const char *mode); + # #define _IOLBF 0x0040 + # int setvbuf(FILE *stream, char *buffer, int mode, size_t size); + # #define _O_U8TEXT 0x40000 + # int _setmode(int fd, int mode); + # int fputws(const wchar_t *ws, FILE *stream); + # int fflush(FILE *stream); + + c_runtime = cdll.msvcrt + NULL = None + _fdopen = CFUNCTYPE(c_void_p, c_int, c_char_p)(("_fdopen", c_runtime)) + _IOLBF = 0x0040 + setvbuf = CFUNCTYPE(c_int, c_void_p, c_char_p, c_int, c_size_t)(("setvbuf", c_runtime)) + _O_U8TEXT = 0x40000 + _setmode = CFUNCTYPE(c_int, c_int, c_int)(("_setmode", c_runtime)) + fputws = CFUNCTYPE(c_int, c_wchar_p, c_void_p)(("fputws", c_runtime)); + fflush = CFUNCTYPE(c_int, c_void_p)(("fflush", c_runtime)); + + buffer_chars = 1024 + + class UnicodeOutput: + def __init__(self, fileno, name): + self._stream = _fdopen(fileno, "w") + assert self._stream is not NULL + + # Deep magic. MSVCRT supports writing wide-oriented output to stdout/stderr + # to the console using the Unicode APIs, but it does the conversion in the + # stdio buffer, so you need that buffer to be as large as the maximum amount + # you're going to write in a single call (in bytes, not characters). + setvbuf(self._stream, NULL, _IOLBF, buffer_chars*4 + 100) + _setmode(fileno, _O_U8TEXT) + + self._fileno = fileno + self.closed = False + self.softspace = False + self.mode = 'w' + self.encoding = 'utf-8' + self.name = name + + def isatty(self): + return False + def close(self): + self.closed = True + self.flush() + def fileno(self): + return self._fileno + def flush(self): + fflush(self._stream) + + def write(self, text): + if not isinstance(text, unicode): + text = str(text).decode('utf-8') + for i in xrange(0, len(text), buffer_chars): + fputws(text[i:(i+buffer_chars)], self._stream) + fflush(self._stream) + + def writelines(self, lines): + for line in lines: + self.write(line) + + if fix_stdout: + sys.stdout = UnicodeOutput(STDOUT_FILENO, '') + + if fix_stderr: + sys.stderr = UnicodeOutput(STDERR_FILENO, '') + + _reload() # take account of new output encoding + except Exception, e: + print >>sys.stderr, "Warning: %r" % (e,) + + # Unmangle command-line arguments. + GetCommandLineW = WINFUNCTYPE(c_wchar_p)(("GetCommandLineW", windll.kernel32)) + CommandLineToArgvW = WINFUNCTYPE(POINTER(c_wchar_p), c_wchar_p, POINTER(c_int)) \ + (("CommandLineToArgvW", windll.shell32)) + + argc = c_int(0) + argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc)) + + # The cygwin wrapper script may append '\r' to each argument (bug in cygwin bash). + def unmangle(s): + return re.sub(ur'\x7f[0-9a-fA-F]*\;', lambda m: unichr(int(m.group(0)[1:-1], 16)), + s.rstrip(u'\r')) + + try: + sys.argv = [unmangle(argv_unicode[i]).encode('utf-8') for i in xrange(1, argc.value)] + except Exception, e: + print >>sys.stderr, "%s: could not unmangle Unicode arguments" % (sys.argv[0],) + print >>sys.stderr, repr(e) + print >>sys.stderr, [argv_unicode[i] for i in xrange(1, argc.value)] + return 1 hunk ./src/allmydata/test/test_runner.py 16 import allmydata bintahoe = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(allmydata.__file__))), 'bin', 'tahoe') -if sys.platform == "win32": # TODO: should this include cygwin? - bintahoe += ".exe" +if sys.platform == "win32": + bintahoe += ".pyscript" class SkipMixin: hunk ./src/allmydata/util/encodingutil.py 55 output_encoding = _canonical_encoding(outenc) if sys.platform == 'win32': - # Unicode arguments are not supported on Windows yet; see #565 and #1074. - argv_encoding = 'ascii' + # On Windows we reencode the arguments as UTF-8; see scripts/runner.py. + argv_encoding = 'utf-8' else: argv_encoding = output_encoding is_unicode_platform = sys.platform in ["win32", "darwin"] } Context: [Rename stringutils to encodingutil, and drop open_unicode (since the Python 'open' function works fine with Unicode paths). david-sarah@jacaranda.org**20100713042715 Ignore-this: fa2bb6b5d48ce9ba7ea2b1afc9f3b7b4 ] [Resolve conflicts between NFC normalization changes, and post-1.7 branch. david-sarah@jacaranda.org**20100618021642 Ignore-this: ed3d0d71b761b1317cff9a4c92c2f5cb ] [trivial: a copy of tiny code-cleanups suggested by Kevan in reviewing #967 zooko@zooko.com**20100617045339 Ignore-this: 274b3fdbf1531aa053f484a5d47d7447 ] [minor code clean-up in dirnode.py zooko@zooko.com**20100221052527 Ignore-this: b01bfb47638f5a64256bb306e0022066 Impose micro-POLA by passing only the writekey instead of the whole node object to {{{_encrypt_rw_uri()}}}. Remove DummyImmutableFileNode in nodemaker.py, which is obviated by this. Add micro-optimization by precomputing the netstring of the empty string and branching on whether the writekey is present or not outside of {{{_encrypt_rw_uri()}}}. Add doc about writekey to docstring. ] [Move EncryptedTemporaryFile from SFTP frontend to allmydata.util.fileutil, and make the FTP frontend also use it (fixing #1083). david-sarah@jacaranda.org**20100711213721 Ignore-this: e452e8ca66391aa2a1a49afe0114f317 ] [Add tests of caps from the future that have non-ASCII characters in them (encoded as UTF-8). The changes to test_uri.py, test_client.py, and test_dirnode.py add tests of non-ASCII future caps in addition to the current tests. The changes to test_web.py just replace the tests of all-ASCII future caps with tests of non-ASCII future caps. We also change uses of failUnlessEqual to failUnlessReallyEqual, in order to catch cases where the type of a string is not as expected. david-sarah@jacaranda.org**20100711200252 Ignore-this: c2f193352369d32e06865f8f3e951894 ] [Allow URIs passed in the initial JSON for t=mkdir-with-children, t=mkdir-immutable to be Unicode. Also pass the name of each child into nodemaker.create_from_cap for error reporting. david-sarah@jacaranda.org**20100711195525 Ignore-this: deac32d8b91ba26ede18905d3f7d2b93 ] [docs/logging.txt: note that setting flogging vars might affect tests with race conditions. david-sarah@jacaranda.org**20100712050721 Ignore-this: fc1609d215fcd5561a57fd1226206f27 ] [test_storage.py: potential fix for failures when logging is enabled. david-sarah@jacaranda.org**19700713040546 Ignore-this: 5815693a0df3e64c52c3c6b7be2846c7 ] [SFTP: address some of the comments in zooko's review (#1106). david-sarah@jacaranda.org**20100712025537 Ignore-this: c3921638a2d4f1de2a776ae78e4dc37e ] [upcase_since_on_welcome terrellrussell@gmail.com**20100708193903] [server_version_on_welcome_page.dpatch.txt freestorm77@gmail.com**20100605191721 Ignore-this: b450c76dc875f5ac8cca229a666cbd0a - The storage server version is 0 for all storage nodes in the Welcome Page ] [NEWS: add NEWS snippets about two recent patches zooko@zooko.com**20100708162058 Ignore-this: 6c9da6a0ad7351a960bdd60f81532899 ] [directory_html_top_banner.dpatch freestorm77@gmail.com**20100622205301 Ignore-this: 1d770d975e0c414c996564774f049bca The div tag with the link "Return to Welcome page" on the directory.xhtml page is not correct ] [tahoe_css_toolbar.dpatch freestorm77@gmail.com**20100622210046 Ignore-this: 5b3ebb2e0f52bbba718a932f80c246c0 CSS modification to be correctly diplayed with Internet Explorer 8 The links on the top of page directory.xhtml are not diplayed in the same line as display with Firefox. ] [runnin_test_tahoe_css.dpatch freestorm77@gmail.com**20100622214714 Ignore-this: e0db73d68740aad09a7b9ae60a08c05c Runnin test for changes in tahoe.css file ] [runnin_test_directory_xhtml.dpatch freestorm77@gmail.com**20100622201403 Ignore-this: f8962463fce50b9466405cb59fe11d43 Runnin test for diretory.xhtml top banner ] [stringutils.py: tolerate sys.stdout having no 'encoding' attribute. david-sarah@jacaranda.org**20100626040817 Ignore-this: f42cad81cef645ee38ac1df4660cc850 ] [quickstart.html: python 2.5 -> 2.6 as recommended version david-sarah@jacaranda.org**20100705175858 Ignore-this: bc3a14645ea1d5435002966ae903199f ] [SFTP: don't call .stopProducing on the producer registered with OverwriteableFileConsumer (which breaks with warner's new downloader). david-sarah@jacaranda.org**20100628231926 Ignore-this: 131b7a5787bc85a9a356b5740d9d996f ] [docs/how_to_make_a_tahoe-lafs_release.txt: trivial correction, install.html should now be quickstart.html. david-sarah@jacaranda.org**20100625223929 Ignore-this: 99a5459cac51bd867cc11ad06927ff30 ] [setup: in the Makefile, refuse to upload tarballs unless someone has passed the environment variable "BB_BRANCH" with value "trunk" zooko@zooko.com**20100619034928 Ignore-this: 276ddf9b6ad7ec79e27474862e0f7d6 ] [trivial: tiny update to in-line comment zooko@zooko.com**20100614045715 Ignore-this: 10851b0ed2abfed542c97749e5d280bc (I'm actually committing this patch as a test of the new eager-annotation-computation of trac-darcs.) ] [docs: about.html link to home page early on, and be decentralized storage instead of cloud storage this time around zooko@zooko.com**20100619065318 Ignore-this: dc6db03f696e5b6d2848699e754d8053 ] [docs: update about.html, especially to have a non-broken link to quickstart.html, and also to comment out the broken links to "for Paranoids" and "for Corporates" zooko@zooko.com**20100619065124 Ignore-this: e292c7f51c337a84ebfeb366fbd24d6c ] [TAG allmydata-tahoe-1.7.0 zooko@zooko.com**20100619052631 Ignore-this: d21e27afe6d85e2e3ba6a3292ba2be1 ] Patch bundle hash: 45b4a4597e7297ca2f8868ebd4d1a4d2f9dd9406