diff -rN -u old-ticket1329/Makefile new-ticket1329/Makefile --- old-ticket1329/Makefile 2011-01-17 23:54:16.000000000 -0700 +++ new-ticket1329/Makefile 2011-01-17 23:54:16.000000000 -0700 @@ -179,7 +179,7 @@ pyflakes: - $(PYTHON) -OOu `which pyflakes` src/allmydata |sort |uniq + $(PYTHON) -OOu `which pyflakes` src/allmydata static |sort |uniq check-umids: $(PYTHON) misc/coding_tools/check-umids.py `find src/allmydata -name '*.py'` diff -rN -u old-ticket1329/src/allmydata/__init__.py new-ticket1329/src/allmydata/__init__.py --- old-ticket1329/src/allmydata/__init__.py 2011-01-17 23:54:16.000000000 -0700 +++ new-ticket1329/src/allmydata/__init__.py 2011-01-17 23:54:16.000000000 -0700 @@ -4,57 +4,6 @@ community web site: U{http://tahoe-lafs.org/} """ -# We want to call require_auto_deps() before other imports, because the setuptools -# docs claim that if a distribution is installed with --multi-version, it might not -# be importable until after pkg_resources.require() has been called for it. We don't -# have an example of this happening at this time. It is possible that require() isn't -# actually needed because we set __requires__ in the generated startup script, but -# that would be an undocumented property of the setuptools implementation. - -from allmydata import _auto_deps -_auto_deps.require_auto_deps() - -# This is just to suppress DeprecationWarnings from nevow and twisted. -# See http://allmydata.org/trac/tahoe/ticket/859 and -# http://divmod.org/trac/ticket/2994 . -import warnings -warnings.filterwarnings("ignore", category=DeprecationWarning, - message="the sha module is deprecated; use the hashlib module instead", - append=True) -warnings.filterwarnings("ignore", category=DeprecationWarning, - message="object.__new__\(\) takes no parameters", - append=True) -warnings.filterwarnings("ignore", category=DeprecationWarning, - message="The popen2 module is deprecated. Use the subprocess module.", - append=True) -warnings.filterwarnings("ignore", category=DeprecationWarning, - message="the md5 module is deprecated; use hashlib instead", - append=True) -warnings.filterwarnings("ignore", category=DeprecationWarning, - message="twisted.web.error.NoResource is deprecated since Twisted 9.0. See twisted.web.resource.NoResource.", - append=True) -try: - import nevow - from twisted.persisted import sob - from twisted.python import filepath - hush_pyflakes = (nevow, sob, filepath) - del hush_pyflakes -finally: - warnings.filters.pop() - warnings.filters.pop() - warnings.filters.pop() - warnings.filters.pop() - # Don't pop the filter for the sha module warning because it is also generated - # by pycrypto (which we don't want to import unless needed). - # warnings.filters.pop() - -# This warning is generated by twisted, PyRex, and possibly other packages, -# but can happen at any time, not only when they are imported. See -# http://tahoe-lafs.org/trac/tahoe-lafs/ticket/1129 . -warnings.filterwarnings("ignore", category=DeprecationWarning, - message="BaseException.message has been deprecated as of Python 2.6", - append=True) - __version__ = "unknown" try: from allmydata._version import __version__ @@ -175,88 +124,93 @@ else: return platform.platform() -def get_package_versions_from_setuptools(): - import pkg_resources - return dict([(p.project_name, (p.version, p.location)) for p in pkg_resources.require(__appname__)]) - -def package_dir(srcfile): - return os.path.dirname(os.path.dirname(os.path.normcase(os.path.realpath(srcfile)))) - -def get_package_versions_and_locations(): - # because there are a few dependencies that are outside setuptools's ken - # (Python and platform, and sqlite3 if you are on Python >= 2.5), and - # because setuptools might fail to find something even though import - # finds it: - import OpenSSL, allmydata, foolscap.api, nevow, platform, pycryptopp, setuptools, simplejson, twisted, zfec, zope.interface - pysqlitever = None - pysqlitefile = None - sqlitever = None - try: - import sqlite3 - except ImportError: - try: - from pysqlite2 import dbapi2 - except ImportError: - pass - else: - pysqlitever = dbapi2.version - pysqlitefile = package_dir(dbapi2.__file__) - sqlitever = dbapi2.sqlite_version - else: - pysqlitever = sqlite3.version - pysqlitefile = package_dir(sqlite3.__file__) - sqlitever = sqlite3.sqlite_version - - d1 = { - 'pyOpenSSL': (OpenSSL.__version__, package_dir(OpenSSL.__file__)), - __appname__: (allmydata.__version__, package_dir(allmydata.__file__)), - 'foolscap': (foolscap.api.__version__, package_dir(foolscap.__file__)), - 'Nevow': (nevow.__version__, package_dir(nevow.__file__)), - 'pycryptopp': (pycryptopp.__version__, package_dir(pycryptopp.__file__)), - 'setuptools': (setuptools.__version__, package_dir(setuptools.__file__)), - 'simplejson': (simplejson.__version__, package_dir(simplejson.__file__)), - 'pysqlite': (pysqlitever, pysqlitefile), - 'sqlite': (sqlitever, 'unknown'), - 'zope.interface': ('unknown', package_dir(zope.interface.__file__)), - 'Twisted': (twisted.__version__, package_dir(twisted.__file__)), - 'zfec': (zfec.__version__, package_dir(zfec.__file__)), - 'python': (platform.python_version(), sys.executable), - 'platform': (get_platform(), None), - } - # But we prefer to get all the dependencies as known by setuptools: - import pkg_resources - try: - d2 = get_package_versions_from_setuptools() - except pkg_resources.DistributionNotFound: - # See docstring in _auto_deps.require_auto_deps() to explain why it makes sense to ignore this exception. - pass +from allmydata._auto_deps import get_package_versions_and_locations + +_vers_and_locs = get_package_versions_and_locations() +_vers_and_locs['platform'] = (get_platform(), None) + + +def check_all_requirements(): + import platform + from distutils.version import LooseVersion + from allmydata._auto_deps import install_requires + + # we require 2.4.4 on non-UCS-2, non-Redhat builds to avoid + # we require 2.4.3 on non-UCS-2 Redhat, because 2.4.3 is common on Redhat-based distros and will have patched the above bug + # we require at least 2.4.2 in any case to avoid a bug in the base64 module: + if sys.maxunicode == 65535: + if sys.version_info < (2, 4, 2) or sys.version_info[0] > 2: + raise NotImplementedError("Tahoe-LAFS current requires Python v2.4.2 or greater " + "for a UCS-2 build (but less than v3), not %r" % + (sys.version_info,)) + elif platform.platform().lower().find('redhat') >= 0: + if sys.version_info < (2, 4, 3) or sys.version_info[0] > 2: + raise NotImplementedError("Tahoe-LAFS current requires Python v2.4.3 or greater " + "on Redhat-based distributions (but less than v3), not %r" % + (sys.version_info,)) else: - d1.update(d2) + if sys.version_info < (2, 4, 4) or sys.version_info[0] > 2: + raise NotImplementedError("Tahoe-LAFS current requires Python v2.4.4 or greater " + "for a non-UCS-2 build (but less than v3), not %r" % + (sys.version_info,)) + + def check_requirement(req): + # We only support a single >= or ==. + def _check(s, ok, relation): + name = s[0].strip(' ').partition('[')[0] + required = s[1].strip(' ') + if name not in _vers_and_locs: + raise AssertionError("no version info for %s" % (name,)) + (actual, location) = _vers_and_locs[name] + actual = str(actual) + if actual != 'unknown' and not ok(LooseVersion(actual), LooseVersion(required)): + msg = ("We require %s version %s of %s, but could only find version %s.\n" + % (relation, required, name, actual)) + if location and location != 'unknown': + msg += "The version we found is from %r.\n" % (location,) + msg += ("To resolve this problem, uninstall that version, either using your\n" + "operating system's package manager or by moving aside the directory.") + raise AssertionError(msg) + + s = req.split('>=') + if len(s) == 2: + _check(s, lambda x, y: x >= y, "at least") + return + s = req.split('==') + if len(s) == 2: + _check(s, lambda x, y: x == y, "exactly") + return + if req not in _vers_and_locs: + raise AssertionError("no version info or could not understand requirement for %r" % (req)) + + for requirement in install_requires: + check_requirement(requirement) + +check_all_requirements() - return d1 def get_package_versions(): - return dict([(k, v) for k, (v, l) in get_package_versions_and_locations().iteritems()]) + return dict([(k, v) for k, (v, l) in _vers_and_locs.iteritems()]) def get_package_locations(): - return dict([(k, l) for k, (v, l) in get_package_versions_and_locations().iteritems()]) + return dict([(k, l) for k, (v, l) in _vers_and_locs.iteritems()]) def get_package_versions_string(show_paths=False): - vers_and_locs = get_package_versions_and_locations() res = [] - for p in [__appname__, "foolscap", "pycryptopp", "zfec", "Twisted", "Nevow", "zope.interface", "python", "platform"]: - (ver, loc) = vers_and_locs.get(p, ('UNKNOWN', 'UNKNOWN')) + first = [__appname__, "foolscap", "pycryptopp", "zfec", "Twisted", "Nevow", "zope.interface", "python", "platform"] + for p in first: + (ver, loc) = _vers_and_locs.get(p, ('UNKNOWN', 'UNKNOWN')) info = str(p) + ": " + str(ver) if show_paths: info = info + " (%s)" % str(loc) res.append(info) - if vers_and_locs.has_key(p): - del vers_and_locs[p] - for p, (v, loc) in vers_and_locs.iteritems(): - info = str(p) + ": " + str(v) - if show_paths: - info = info + " (%s)" % str(loc) - res.append(info) + for p, (v, loc) in _vers_and_locs.iteritems(): + if p not in first: + info = str(p) + ": " + str(v) + if show_paths: + info = info + " (%s)" % str(loc) + res.append(info) + return ', '.join(res) diff -rN -u old-ticket1329/src/allmydata/_auto_deps.py new-ticket1329/src/allmydata/_auto_deps.py --- old-ticket1329/src/allmydata/_auto_deps.py 2011-01-17 23:54:16.000000000 -0700 +++ new-ticket1329/src/allmydata/_auto_deps.py 2011-01-17 23:54:16.000000000 -0700 @@ -1,9 +1,10 @@ # Note: do not import any module from Tahoe-LAFS itself in this # file. Also please avoid importing modules from other packages than -# the Python Standard Library if at all possible (exception: we rely -# on importing pkg_resources, which is provided by setuptools, -# zetuptoolz, distribute, and perhaps in the future distutils2, for -# the require_auto_deps() function.) +# the Python Standard Library if at all possible. That includes +# setuptools and pkg_resources. + +# If you add or remove dependencies, please remember to update +# get_package_versions_and_locations() below. install_requires=[ # we require newer versions of setuptools (actually @@ -27,7 +28,7 @@ # Needed for SFTP. pyasn1 is needed by twisted.conch in Twisted >= 9.0. # pycrypto 2.2 doesn't work due to https://bugs.launchpad.net/pycrypto/+bug/620253 - "pycrypto == 2.0.1, == 2.1, >= 2.3", + "pycrypto >= 2.3", "pyasn1 >= 0.0.8a", # Will be needed to test web apps, but not yet. See #1001. @@ -74,49 +75,105 @@ install_requires=[] del sys # clean up namespace -def require_python_version(): - import sys, platform - # we require 2.4.4 on non-UCS-2, non-Redhat builds to avoid - # we require 2.4.3 on non-UCS-2 Redhat, because 2.4.3 is common on Redhat-based distros and will have patched the above bug - # we require at least 2.4.2 in any case to avoid a bug in the base64 module: - if sys.maxunicode == 65535: - if sys.version_info < (2, 4, 2) or sys.version_info[0] > 2: - raise NotImplementedError("Tahoe-LAFS current requires Python v2.4.2 or greater " - "for a UCS-2 build (but less than v3), not %r" % - (sys.version_info,)) - elif platform.platform().lower().find('redhat') >= 0: - if sys.version_info < (2, 4, 3) or sys.version_info[0] > 2: - raise NotImplementedError("Tahoe-LAFS current requires Python v2.4.3 or greater " - "on Redhat-based distributions (but less than v3), not %r" % - (sys.version_info,)) +def get_package_versions_and_locations(): + import warnings, os, sys + def package_dir(srcfile): + return os.path.dirname(os.path.dirname(os.path.normcase(os.path.realpath(srcfile)))) + + # pkg_resources.require returns the distribution that pkg_resources attempted to put + # on sys.path, which can differ from the one that we actually import due to #1258, + # or any other bug that causes sys.path to be set up incorrectly. Therefore we + # must import the packages in order to check their versions and paths. Note that + # this needs to be updated if dependencies are added or removed. + + # This warning is generated by twisted, PyRex, and possibly other packages, + # but can happen at any time, not only when they are imported. See + # http://tahoe-lafs.org/trac/tahoe-lafs/ticket/1129 . + warnings.filterwarnings("ignore", category=DeprecationWarning, + message="BaseException.message has been deprecated as of Python 2.6", + append=True) + + # This is to suppress DeprecationWarnings from nevow, twisted, and pycrypto. + # See http://allmydata.org/trac/tahoe/ticket/859 and + # http://divmod.org/trac/ticket/2994 . + warnings.filterwarnings("ignore", category=DeprecationWarning, + message="the sha module is deprecated; use the hashlib module instead", + append=True) + warnings.filterwarnings("ignore", category=DeprecationWarning, + message="object.__new__\(\) takes no parameters", + append=True) + warnings.filterwarnings("ignore", category=DeprecationWarning, + message="The popen2 module is deprecated. Use the subprocess module.", + append=True) + warnings.filterwarnings("ignore", category=DeprecationWarning, + message="the md5 module is deprecated; use hashlib instead", + append=True) + warnings.filterwarnings("ignore", category=DeprecationWarning, + message="twisted.web.error.NoResource is deprecated since Twisted 9.0. See twisted.web.resource.NoResource.", + append=True) + warnings.filterwarnings("ignore", category=DeprecationWarning, + message="the sets module is deprecated", + append=True) + try: + import nevow + from twisted.persisted import sob + from twisted.python import filepath + import Crypto + import foolscap.api + [sob, filepath] # hush pyflakes + finally: + for n in range(6): + warnings.filters.pop() + + import OpenSSL, allmydata, platform, pycryptopp, simplejson, twisted, zfec, zope.interface, pyasn1, mock + pysqlitever = None + pysqlitefile = None + sqlitever = None + try: + import sqlite3 + except ImportError: + try: + from pysqlite2 import dbapi2 + except ImportError: + pass + else: + pysqlitever = dbapi2.version + pysqlitefile = package_dir(dbapi2.__file__) + sqlitever = dbapi2.sqlite_version else: - if sys.version_info < (2, 4, 4) or sys.version_info[0] > 2: - raise NotImplementedError("Tahoe-LAFS current requires Python v2.4.4 or greater " - "for a non-UCS-2 build (but less than v3), not %r" % - (sys.version_info,)) - -def require_auto_deps(): - """ - The purpose of this function is to raise a pkg_resources exception if any of the - requirements can't be imported. This is just to give earlier and more explicit error - messages, as opposed to waiting until the source code tries to import some module from one - of these packages and gets an ImportError. This function gets called from - src/allmydata/__init__.py . - """ - require_python_version() + pysqlitever = sqlite3.version + pysqlitefile = package_dir(sqlite3.__file__) + sqlitever = sqlite3.sqlite_version + + packages = { + 'pyopenssl': (OpenSSL.__version__, package_dir(OpenSSL.__file__)), + allmydata.__appname__: (allmydata.__version__, package_dir(allmydata.__file__)), + 'foolscap': (foolscap.api.__version__, package_dir(foolscap.__file__)), + 'Nevow': (nevow.__version__, package_dir(nevow.__file__)), + 'pycryptopp': (pycryptopp.__version__, package_dir(pycryptopp.__file__)), + 'simplejson': (simplejson.__version__, package_dir(simplejson.__file__)), + 'pysqlite': (pysqlitever, pysqlitefile), + 'sqlite': (sqlitever, 'unknown'), + 'zope.interface': ('unknown', package_dir(zope.interface.__file__)), + 'Twisted': (twisted.__version__, package_dir(twisted.__file__)), + 'zfec': (zfec.__version__, package_dir(zfec.__file__)), + 'pycrypto': (Crypto.__version__, package_dir(Crypto.__file__)), + 'pyasn1': ('unknown', package_dir(pyasn1.__file__)), + 'mock': (mock.__version__, package_dir(mock.__file__)), + 'python': (platform.python_version(), sys.executable), + } - import pkg_resources - for requirement in install_requires: + if sys.platform == 'win32': try: - pkg_resources.require(requirement) - except pkg_resources.DistributionNotFound: - # there is no .egg-info present for this requirement, which - # either means that it isn't installed, or it is installed in a - # way that pkg_resources can't find it (but regular python - # might). There are several older Linux distributions which - # provide our dependencies just fine, but they don't ship - # .egg-info files. Note that if there *is* an .egg-info file, - # but it shows a too-old version, then we'll get a - # VersionConflict error instead of DistributionNotFound. + import win32api + packages['pywin32'] = ('unknown', package_dir(win32api.__file__)) + except ImportError: pass + + if not hasattr(sys, 'frozen'): + # Don't try to get the setuptools version if using bbfreeze, py2exe etc. See #585. + import setuptools + packages['setuptools'] = (setuptools.__version__, package_dir(setuptools.__file__)) + + return packages diff -rN -u old-ticket1329/src/allmydata/test/test_runner.py new-ticket1329/src/allmydata/test/test_runner.py --- old-ticket1329/src/allmydata/test/test_runner.py 2011-01-17 23:54:16.000000000 -0700 +++ new-ticket1329/src/allmydata/test/test_runner.py 2011-01-17 23:54:16.000000000 -0700 @@ -3,7 +3,7 @@ from twisted.python import usage, runtime from twisted.internet import utils -import os.path, re, sys +import os.path, re, sys, subprocess from cStringIO import StringIO from allmydata.util import fileutil, pollmixin from allmydata.util.encodingutil import unicode_to_argv, unicode_to_output, get_filesystem_encoding @@ -14,24 +14,36 @@ timeout = 240 +def get_root_from_file(src): + srcdir = os.path.dirname(os.path.dirname(os.path.normcase(os.path.realpath(src)))) + + root = os.path.dirname(srcdir) + if os.path.basename(srcdir) == 'site-packages': + if re.search(r'python.+\..+', os.path.basename(root)): + root = os.path.dirname(root) + root = os.path.dirname(root) + elif os.path.basename(root) == 'src': + root = os.path.dirname(root) + + return root + srcfile = allmydata.__file__ -srcdir = os.path.dirname(os.path.dirname(os.path.normcase(os.path.realpath(srcfile)))) +rootdir = get_root_from_file(srcfile) -rootdir = os.path.dirname(srcdir) -if os.path.basename(srcdir) == 'site-packages': - if re.search(r'python.+\..+', os.path.basename(rootdir)): - rootdir = os.path.dirname(rootdir) - rootdir = os.path.dirname(rootdir) -elif os.path.basename(rootdir) == 'src': - rootdir = os.path.dirname(rootdir) - -bintahoe = os.path.join(rootdir, 'bin', 'tahoe') -if sys.platform == "win32": - bintahoe += ".pyscript" - if not os.path.exists(bintahoe): - alt_bintahoe = os.path.join(rootdir, 'Scripts', 'tahoe.pyscript') - if os.path.exists(alt_bintahoe): - bintahoe = alt_bintahoe + +if hasattr(sys, 'frozen'): + bintahoe = os.path.join(rootdir, 'tahoe') + + if sys.platform == "win32" and os.path.exists(bintahoe + '.exe'): + bintahoe += '.exe' +else: + bintahoe = os.path.join(rootdir, 'bin', 'tahoe') + if sys.platform == "win32": + bintahoe += '.pyscript' + if not os.path.exists(bintahoe): + alt_bintahoe = os.path.join(rootdir, 'Scripts', 'tahoe.pyscript') + if os.path.exists(alt_bintahoe): + bintahoe = alt_bintahoe class SkipMixin: @@ -49,16 +61,20 @@ class BinTahoe(common_util.SignalMixin, unittest.TestCase, SkipMixin): - def test_the_right_code(self): + def _check_right_code(self, file_to_check): + root_to_check = get_root_from_file(file_to_check) + if os.path.basename(root_to_check) == 'dist': + root_to_check = os.path.dirname(root_to_check) + cwd = os.path.normcase(os.path.realpath(".")) root_from_cwd = os.path.dirname(cwd) if os.path.basename(root_from_cwd) == 'src': root_from_cwd = os.path.dirname(root_from_cwd) - same = (root_from_cwd == rootdir) + same = (root_from_cwd == root_to_check) if not same: try: - same = os.path.samefile(root_from_cwd, rootdir) + same = os.path.samefile(root_from_cwd, root_to_check) except AttributeError, e: e # hush pyflakes @@ -66,9 +82,9 @@ msg = ("We seem to be testing the code at %r,\n" "(according to the source filename %r),\n" "but expected to be testing the code at %r.\n" - % (rootdir, srcfile, root_from_cwd)) + % (root_to_check, file_to_check, root_from_cwd)) - root_from_cwdu = os.path.normcase(os.path.normpath(os.getcwdu())) + root_from_cwdu = os.path.dirname(os.path.normcase(os.path.normpath(os.getcwdu()))) if os.path.basename(root_from_cwdu) == u'src': root_from_cwdu = os.path.dirname(root_from_cwdu) @@ -81,6 +97,24 @@ msg += "Please run the tests from the root of the Tahoe-LAFS distribution." self.fail(msg) + def test_the_right_code(self): + self._check_right_code(srcfile) + + def test_import_in_repl(self): + self.skip_if_cannot_run_bintahoe() + + args = [bintahoe, "debug", "repl"] + if not hasattr(sys, 'frozen'): + args = [sys.executable] + args + + p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (out, err) = p.communicate("import allmydata; print allmydata.__file__") + + self.failUnlessEqual(p.returncode, 0) + first = out.splitlines()[0] + self.failUnless(first.startswith('>>> ')) + self._check_right_code(first[4:]) + def test_path(self): self.skip_if_cannot_run_bintahoe() d = utils.getProcessOutputAndValue(bintahoe, args=["--version-and-path"], env=os.environ) @@ -112,6 +146,7 @@ else: altverstr = verstr + srcdir = os.path.dirname(os.path.dirname(os.path.normcase(os.path.realpath(srcfile)))) required_ver_and_path = "%s: %s (%s)" % (allmydata.__appname__, verstr, srcdir) alt_required_ver_and_path = "%s: %s (%s)" % (allmydata.__appname__, altverstr, srcdir) @@ -121,6 +156,8 @@ def test_unicode_arguments_and_output(self): self.skip_if_cannot_run_bintahoe() + if hasattr(sys, 'frozen') and sys.platform == "win32": + raise unittest.SkipTest("This test can't currently be made to work for frozen executables on Windows,\ndue to lack of support for Unicode in twisted.internet.utils.getProcessOutputAndValue.") tricky = u"\u2621" try: @@ -139,6 +176,8 @@ def test_run_with_python_options(self): self.skip_if_cannot_run_bintahoe() + if hasattr(sys, 'frozen'): + raise unittest.SkipTest("This test doesn't apply to frozen executables.") # -t is a harmless option that warns about tabs. d = utils.getProcessOutputAndValue(sys.executable, args=['-t', bintahoe, '--version'], diff -rN -u old-ticket1329/src/allmydata/test/test_system.py new-ticket1329/src/allmydata/test/test_system.py --- old-ticket1329/src/allmydata/test/test_system.py 2011-01-17 23:54:16.000000000 -0700 +++ new-ticket1329/src/allmydata/test/test_system.py 2011-01-17 23:54:16.000000000 -0700 @@ -1769,8 +1769,7 @@ if env is None: env = os.environ - d = utils.getProcessOutputAndValue(sys.executable, args=[bintahoe] + argv, - env=env) + d = utils.getProcessOutputAndValue(bintahoe, argv, env=env) return d def _test_checker(self, res):