Fri Sep 10 11:35:20 MDT 2010 zooko@zooko.com * fileutil: copy in the get_disk_stats() and get_available_space() functions from storage/server.py Fri Sep 10 11:36:29 MDT 2010 zooko@zooko.com * storage: use fileutil's version of get_disk_stats() and get_available_space(), use mockery/fakery in tests, enable large share test on platforms with sparse files and if > 4 GiB of disk space is currently available New patches: [fileutil: copy in the get_disk_stats() and get_available_space() functions from storage/server.py zooko@zooko.com**20100910173520 Ignore-this: 8b15569715f710f4fc5092f7ca109253 ] { hunk ./src/allmydata/test/test_util.py 506 finally: os.chdir(saved_cwd) + def test_disk_stats(self): + avail = fileutil.get_available_space('.', 2**14) + if avail == 0: + raise unittest.SkipTest("This test will spuriously fail there is no disk space left.") + + disk = fileutil.get_disk_stats('.', 2**13) + self.failUnless(disk['total'] > 0, disk['total']) + self.failUnless(disk['used'] > 0, disk['used']) + self.failUnless(disk['free_for_root'] > 0, disk['free_for_root']) + self.failUnless(disk['free_for_nonroot'] > 0, disk['free_for_nonroot']) + self.failUnless(disk['avail'] > 0, disk['avail']) + + def test_disk_stats_avail_nonnegative(self): + # This test will spuriously fail if you have more than 2^128 + # bytes of available space on your filesystem. + disk = fileutil.get_disk_stats('.', 2**128) + self.failUnlessEqual(disk['avail'], 0) + class PollMixinTests(unittest.TestCase): def setUp(self): self.pm = pollmixin.PollMixin() hunk ./src/allmydata/util/fileutil.py 308 # there is always at least one Unicode path component. return os.path.normpath(path) +windows = False +try: + import win32api, win32con +except ImportError: + pass +else: + windows = True + # + win32api.SetErrorMode(win32con.SEM_FAILCRITICALERRORS | + win32con.SEM_NOOPENFILEERRORBOX) + +def get_disk_stats(whichdir, reserved_space=0): + """Return disk statistics for the storage disk, in the form of a dict + with the following fields. + total: total bytes on disk + free_for_root: bytes actually free on disk + free_for_nonroot: bytes free for "a non-privileged user" [Unix] or + the current user [Windows]; might take into + account quotas depending on platform + used: bytes used on disk + avail: bytes available excluding reserved space + An AttributeError can occur if the OS has no API to get disk information. + An EnvironmentError can occur if the OS call fails. + + whichdir is a directory on the filesystem in question -- the + answer is about the filesystem, not about the directory, so the + directory is used only to specify which filesystem. + + reserved_space is how many bytes to subtract from the answer, so + you can pass how many bytes you would like to leave unused on this + filesystem as reserved_space. + """ + + if windows: + # For Windows systems, where os.statvfs is not available, use GetDiskFreeSpaceEx. + # + # + # Although the docs say that the argument should be the root directory + # of a disk, GetDiskFreeSpaceEx actually accepts any path on that disk + # (like its Win32 equivalent). + + (free_for_nonroot, total, free_for_root) = win32api.GetDiskFreeSpaceEx(whichdir) + else: + # For Unix-like systems. + # + # + # + s = os.statvfs(whichdir) + + # on my mac laptop: + # statvfs(2) is a wrapper around statfs(2). + # statvfs.f_frsize = statfs.f_bsize : + # "minimum unit of allocation" (statvfs) + # "fundamental file system block size" (statfs) + # statvfs.f_bsize = statfs.f_iosize = stat.st_blocks : preferred IO size + # on an encrypted home directory ("FileVault"), it gets f_blocks + # wrong, and s.f_blocks*s.f_frsize is twice the size of my disk, + # but s.f_bavail*s.f_frsize is correct + + total = s.f_frsize * s.f_blocks + free_for_root = s.f_frsize * s.f_bfree + free_for_nonroot = s.f_frsize * s.f_bavail + + # valid for all platforms: + used = total - free_for_root + avail = max(free_for_nonroot - reserved_space, 0) + + return { 'total': total, 'free_for_root': free_for_root, + 'free_for_nonroot': free_for_nonroot, + 'used': used, 'avail': avail, } + +def get_available_space(whichdir, reserved_space): + """Returns available space for share storage in bytes, or None if no + API to get this information is available. + + whichdir is a directory on the filesystem in question -- the + answer is about the filesystem, not about the directory, so the + directory is used only to specify which filesystem. + + reserved_space is how many bytes to subtract from the answer, so + you can pass how many bytes you would like to leave unused on this + filesystem as reserved_space. + """ + try: + return get_disk_stats(whichdir, reserved_space)['avail'] + except AttributeError: + return None + except EnvironmentError: + log.msg("OS call to get disk statistics failed") + return 0 + } [storage: use fileutil's version of get_disk_stats() and get_available_space(), use mockery/fakery in tests, enable large share test on platforms with sparse files and if > 4 GiB of disk space is currently available zooko@zooko.com**20100910173629 Ignore-this: 1304f1164c661de6d5304f993eb9b27b ] < [fileutil: copy in the get_disk_stats() and get_available_space() functions from storage/server.py zooko@zooko.com**20100910173520 Ignore-this: 8b15569715f710f4fc5092f7ca109253 ] > { hunk ./src/allmydata/storage/server.py 39 implements(RIStorageServer, IStatsProducer) name = 'storage' LeaseCheckerClass = LeaseCheckingCrawler - windows = False - - try: - import win32api, win32con - windows = True - # - win32api.SetErrorMode(win32con.SEM_FAILCRITICALERRORS | - win32con.SEM_NOOPENFILEERRORBOX) - except ImportError: - pass def __init__(self, storedir, nodeid, reserved_space=0, discard_storage=False, readonly_storage=False, hunk ./src/allmydata/storage/server.py 153 def _clean_incomplete(self): fileutil.rm_dir(self.incomingdir) - def get_disk_stats(self): - """Return disk statistics for the storage disk, in the form of a dict - with the following fields. - total: total bytes on disk - free_for_root: bytes actually free on disk - free_for_nonroot: bytes free for "a non-privileged user" [Unix] or - the current user [Windows]; might take into - account quotas depending on platform - used: bytes used on disk - avail: bytes available excluding reserved space - An AttributeError can occur if the OS has no API to get disk information. - An EnvironmentError can occur if the OS call fails.""" - - if self.windows: - # For Windows systems, where os.statvfs is not available, use GetDiskFreeSpaceEx. - # - # - # Although the docs say that the argument should be the root directory - # of a disk, GetDiskFreeSpaceEx actually accepts any path on that disk - # (like its Win32 equivalent). - - (free_for_nonroot, total, free_for_root) = self.win32api.GetDiskFreeSpaceEx(self.storedir) - else: - # For Unix-like systems. - # - # - # - s = os.statvfs(self.storedir) - - # on my mac laptop: - # statvfs(2) is a wrapper around statfs(2). - # statvfs.f_frsize = statfs.f_bsize : - # "minimum unit of allocation" (statvfs) - # "fundamental file system block size" (statfs) - # statvfs.f_bsize = statfs.f_iosize = stat.st_blocks : preferred IO size - # on an encrypted home directory ("FileVault"), it gets f_blocks - # wrong, and s.f_blocks*s.f_frsize is twice the size of my disk, - # but s.f_bavail*s.f_frsize is correct - - total = s.f_frsize * s.f_blocks - free_for_root = s.f_frsize * s.f_bfree - free_for_nonroot = s.f_frsize * s.f_bavail - - # valid for all platforms: - used = total - free_for_root - avail = max(free_for_nonroot - self.reserved_space, 0) - - return { 'total': total, 'free_for_root': free_for_root, - 'free_for_nonroot': free_for_nonroot, - 'used': used, 'avail': avail, } - def get_stats(self): # remember: RIStatsProvider requires that our return dict # contains numeric values. hunk ./src/allmydata/storage/server.py 163 stats['storage_server.latencies.%s.%s' % (category, name)] = v try: - disk = self.get_disk_stats() + disk = fileutil.get_disk_stats(self.storedir, self.reserved_space) writeable = disk['avail'] > 0 # spacetime predictors should use disk_avail / (d(disk_used)/dt) hunk ./src/allmydata/storage/server.py 195 if self.readonly_storage: return 0 - try: - return self.get_disk_stats()['avail'] - except AttributeError: - return None - except EnvironmentError: - log.msg("OS call to get disk statistics failed", level=log.UNUSUAL) - return 0 + return fileutil.get_available_space(self.storedir, self.reserved_space) def allocated_size(self): space = 0 hunk ./src/allmydata/test/test_storage.py 1 +import time, os.path, platform, stat, re, simplejson, struct hunk ./src/allmydata/test/test_storage.py 3 -import time, os.path, stat, re, simplejson, struct +from allmydata.util import log + +import mock from twisted.trial import unittest hunk ./src/allmydata/test/test_storage.py 233 return self._do_test_readwrite("test_readwrite_v2", 0x44, WriteBucketProxy_v2, ReadBucketProxy) -class FakeDiskStorageServer(StorageServer): - DISKAVAIL = 0 - def get_disk_stats(self): - return { 'free_for_nonroot': self.DISKAVAIL, 'avail': max(self.DISKAVAIL - self.reserved_space, 0), } - class Server(unittest.TestCase): def setUp(self): hunk ./src/allmydata/test/test_storage.py 266 sharenums, size, canary) def test_large_share(self): + syslow = platform.system().lower() + if 'cygwin' in syslow or 'windows' in syslow or 'darwin' in syslow: + raise unittest.SkipTest("If your filesystem doesn't support efficient sparse files then it is very expensive (Mac OS X and Windows don't support efficient sparse files).") + + avail = fileutil.get_available_space('.', 2**14) + if avail <= 4*2**30: + raise unittest.SkipTest("This test will spuriously fail if you have less than 4 GiB free on your filesystem.") + ss = self.create("test_large_share") already,writers = self.allocate(ss, "allocate", [0], 2**32+2) hunk ./src/allmydata/test/test_storage.py 288 readers = ss.remote_get_buckets("allocate") reader = readers[shnum] self.failUnlessEqual(reader.remote_read(2**32, 2), "ab") - test_large_share.skip = "This test can spuriously fail if you have less than 4 GiB free on your filesystem, and if your filesystem doesn't support efficient sparse files then it is very expensive (Mac OS X and Windows don't support efficient sparse files)." def test_dont_overfill_dirs(self): """ hunk ./src/allmydata/test/test_storage.py 431 self.failUnlessEqual(already, set()) self.failUnlessEqual(set(writers.keys()), set([0,1,2])) - def test_reserved_space(self): - ss = self.create("test_reserved_space", reserved_space=10000, - klass=FakeDiskStorageServer) - # the FakeDiskStorageServer doesn't do real calls to get_disk_stats - ss.DISKAVAIL = 15000 + @mock.patch('allmydata.util.fileutil.get_disk_stats') + def test_reserved_space(self, mock_get_disk_stats): + reserved_space=10000 + mock_get_disk_stats.return_value = { + 'free_for_nonroot': 15000, + 'avail': max(15000 - reserved_space, 0), + } + + ss = self.create("test_reserved_space", reserved_space=reserved_space) # 15k available, 10k reserved, leaves 5k for shares # a newly created and filled share incurs this much overhead, beyond hunk ./src/allmydata/test/test_storage.py 478 allocated = 1001 + OVERHEAD + LEASE_SIZE - # we have to manually increase DISKAVAIL, since we're not doing real + # we have to manually increase available, since we're not doing real # disk measurements hunk ./src/allmydata/test/test_storage.py 480 - ss.DISKAVAIL -= allocated + mock_get_disk_stats.return_value = { + 'free_for_nonroot': 15000 - allocated, + 'avail': max(15000 - allocated - reserved_space, 0), + } # now there should be ALLOCATED=1001+12+72=1085 bytes allocated, and # 5000-1085=3915 free, therefore we can fit 39 100byte shares hunk ./src/allmydata/test/test_storage.py 497 ss.disownServiceParent() del ss - def test_disk_stats(self): - # This will spuriously fail if there is zero disk space left (but so will other tests). - ss = self.create("test_disk_stats", reserved_space=0) - - disk = ss.get_disk_stats() - self.failUnless(disk['total'] > 0, disk['total']) - self.failUnless(disk['used'] > 0, disk['used']) - self.failUnless(disk['free_for_root'] > 0, disk['free_for_root']) - self.failUnless(disk['free_for_nonroot'] > 0, disk['free_for_nonroot']) - self.failUnless(disk['avail'] > 0, disk['avail']) - - def test_disk_stats_avail_nonnegative(self): - ss = self.create("test_disk_stats_avail_nonnegative", reserved_space=2**64) - - disk = ss.get_disk_stats() - self.failUnlessEqual(disk['avail'], 0) - def test_seek(self): basedir = self.workdir("test_seek_behavior") fileutil.make_dirs(basedir) hunk ./src/allmydata/test/test_storage.py 2459 d = self.render1(page, args={"t": ["json"]}) return d -class NoDiskStatsServer(StorageServer): - def get_disk_stats(self): - raise AttributeError - -class BadDiskStatsServer(StorageServer): - def get_disk_stats(self): - raise OSError - class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): def setUp(self): hunk ./src/allmydata/test/test_storage.py 2500 d = self.render1(page, args={"t": ["json"]}) return d - def test_status_no_disk_stats(self): + @mock.patch('allmydata.util.fileutil.get_disk_stats') + def test_status_no_disk_stats(self, mock_get_disk_stats): + mock_get_disk_stats.side_effect = AttributeError() + # Some platforms may have no disk stats API. Make sure the code can handle that # (test runs on all platforms). basedir = "storage/WebStatus/status_no_disk_stats" hunk ./src/allmydata/test/test_storage.py 2508 fileutil.make_dirs(basedir) - ss = NoDiskStatsServer(basedir, "\x00" * 20) + ss = StorageServer(basedir, "\x00" * 20) ss.setServiceParent(self.s) w = StorageStatus(ss) html = w.renderSynchronously() hunk ./src/allmydata/test/test_storage.py 2519 self.failUnlessIn("Space Available to Tahoe: ?", s) self.failUnless(ss.get_available_space() is None) - def test_status_bad_disk_stats(self): + @mock.patch('allmydata.util.fileutil.get_disk_stats') + def test_status_bad_disk_stats(self, mock_get_disk_stats): + mock_get_disk_stats.side_effect = OSError() + # If the API to get disk stats exists but a call to it fails, then the status should # show that no shares will be accepted, and get_available_space() should be 0. basedir = "storage/WebStatus/status_bad_disk_stats" hunk ./src/allmydata/test/test_storage.py 2527 fileutil.make_dirs(basedir) - ss = BadDiskStatsServer(basedir, "\x00" * 20) + ss = StorageServer(basedir, "\x00" * 20) ss.setServiceParent(self.s) w = StorageStatus(ss) html = w.renderSynchronously() hunk ./src/allmydata/test/test_storage.py 2579 self.failUnlessEqual(w.render_abbrev_space(None, 10e6), "10.00 MB") self.failUnlessEqual(remove_prefix("foo.bar", "foo."), "bar") self.failUnlessEqual(remove_prefix("foo.bar", "baz."), None) - hunk ./src/allmydata/test/test_system.py 1784 d.addCallback(_got_lit_filenode) return d - } Context: [test: make tests stop relying on pyutil version class accepting the string 'unknown' for its version, and make them forward-compatible with the future Python Rational Version Numbering standard zooko@zooko.com**20100910154135 Ignore-this: d051b071f33595493be5df218f5015a6 ] [setup: copy in this fix from zetuptoolz and the accompanying new version number of zetuptoolz: http://tahoe-lafs.org/trac/zetuptoolz/ticket/1 zooko@zooko.com**20100910061411 Ignore-this: cb0ddce66b2a71666df3e22375fa581a ] [immutable download: have the finder inform its share consumer "no more shares" in a subsequent tick, thus avoiding accidentally telling it "no more shares" now and then telling it "here's another share" in a subsequent tick Brian Warner **20100910043038 Ignore-this: 47595fb2b87867d3d75695d51344c484 fixes #1191 Patch by Brian. This patch description was actually written by Zooko, but I forged Brian's name on the "author" field so that he would get credit for this patch in revision control history. ] [immutable downloader: add a test specifically of whether the finder sometimes announces "no more shares ever" and then announces a newly found share zooko@zooko.com**20100909041654 Ignore-this: ec0d5febc499f974b167465290770abd (The current code fails this test, ref #1191.) ] [docs/frontends/FTP-and-SFTP.txt : ftpd and sftpd doesn't listen on loopback interface only marc.doudiet@nimag.net**20100813140853 Ignore-this: 5b5dfd0e5991a2669fe41ba13ea21bd4 ] [tests: assign the storage servers to a fixed order which triggers a bug in new downloader every time this test is run (formerly this test would detect the bug in new-downloader only sporadically) zooko@zooko.com**20100904041515 Ignore-this: 33155dcc03e84217ec5541addd3a16fc If you are investigating the bug in new-downloader, one way to investigate might be to change this ordering to a different fixed order (e.g. rotate by 4 instead of rotate by 5) and observe how the behavior of new-downloader differs in that case. ] [TAG allmydata-tahoe-1.8.0c3 zooko@zooko.com**20100902212140 Ignore-this: e4550de37f57e5c1a591e549a104565d ] Patch bundle hash: c8607a41f8ddd2d945ae8ed6d309413d178c6c84