2 patches for repository zooko@tahoe-lafs.org:/home/source/darcs/tahoe-lafs/trunk: Fri Mar 25 14:35:14 MDT 2011 wilcoxjg@gmail.com * storage: new mocking tests of storage server read and write There are already tests of read and functionality in test_storage.py, but those tests let the code under test use a real filesystem whereas these tests mock all file system calls. Wed Apr 6 14:38:12 MDT 2011 zooko@zooko.com * a bunch of incomplete work on #999, to be unrecorded in arctic's repo New patches: [storage: new mocking tests of storage server read and write wilcoxjg@gmail.com**20110325203514 Ignore-this: df65c3c4f061dd1516f88662023fdb41 There are already tests of read and functionality in test_storage.py, but those tests let the code under test use a real filesystem whereas these tests mock all file system calls. ] { addfile ./src/allmydata/test/test_server.py hunk ./src/allmydata/test/test_server.py 1 +from twisted.trial import unittest + +from StringIO import StringIO + +from allmydata.test.common_util import ReallyEqualMixin + +import mock + +# This is the code that we're going to be testing. +from allmydata.storage.server import StorageServer + +# The following share file contents was generated with +# storage.immutable.ShareFile from Tahoe-LAFS v1.8.2 +# with share data == 'a'. +share_data = 'a\x00\x00\x00\x00xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\x00(\xde\x80' +share_file_data = '\x00\x00\x00\x01\x00\x00\x00\x01\x00\x00\x00\x01' + share_data + +sharefname = 'testdir/shares/or/orsxg5dtorxxeylhmvpws3temv4a/0' + +class TestServerConstruction(unittest.TestCase, ReallyEqualMixin): + @mock.patch('__builtin__.open') + def test_create_server(self, mockopen): + """ This tests whether a server instance can be constructed. """ + + def call_open(fname, mode): + if fname == 'testdir/bucket_counter.state': + raise IOError(2, "No such file or directory: 'testdir/bucket_counter.state'") + elif fname == 'testdir/lease_checker.state': + raise IOError(2, "No such file or directory: 'testdir/lease_checker.state'") + elif fname == 'testdir/lease_checker.history': + return StringIO() + mockopen.side_effect = call_open + + # Now begin the test. + s = StorageServer('testdir', 'testnodeidxxxxxxxxxx') + + # You passed! + +class TestServer(unittest.TestCase, ReallyEqualMixin): + @mock.patch('__builtin__.open') + def setUp(self, mockopen): + def call_open(fname, mode): + if fname == 'testdir/bucket_counter.state': + raise IOError(2, "No such file or directory: 'testdir/bucket_counter.state'") + elif fname == 'testdir/lease_checker.state': + raise IOError(2, "No such file or directory: 'testdir/lease_checker.state'") + elif fname == 'testdir/lease_checker.history': + return StringIO() + mockopen.side_effect = call_open + + self.s = StorageServer('testdir', 'testnodeidxxxxxxxxxx') + + + @mock.patch('time.time') + @mock.patch('os.mkdir') + @mock.patch('__builtin__.open') + @mock.patch('os.listdir') + @mock.patch('os.path.isdir') + def test_write_share(self, mockisdir, mocklistdir, mockopen, mockmkdir, mocktime): + """Handle a report of corruption.""" + + def call_listdir(dirname): + self.failUnlessReallyEqual(dirname, 'testdir/shares/or/orsxg5dtorxxeylhmvpws3temv4a') + raise OSError(2, "No such file or directory: 'testdir/shares/or/orsxg5dtorxxeylhmvpws3temv4a'") + + mocklistdir.side_effect = call_listdir + + class MockFile: + def __init__(self): + self.buffer = '' + self.pos = 0 + def write(self, instring): + begin = self.pos + padlen = begin - len(self.buffer) + if padlen > 0: + self.buffer += '\x00' * padlen + end = self.pos + len(instring) + self.buffer = self.buffer[:begin]+instring+self.buffer[end:] + self.pos = end + def close(self): + pass + def seek(self, pos): + self.pos = pos + def read(self, numberbytes): + return self.buffer[self.pos:self.pos+numberbytes] + def tell(self): + return self.pos + + mocktime.return_value = 0 + + sharefile = MockFile() + def call_open(fname, mode): + self.failUnlessReallyEqual(fname, 'testdir/shares/incoming/or/orsxg5dtorxxeylhmvpws3temv4a/0' ) + return sharefile + + mockopen.side_effect = call_open + # Now begin the test. + alreadygot, bs = self.s.remote_allocate_buckets('teststorage_index', 'x'*32, 'y'*32, set((0,)), 1, mock.Mock()) + print bs + bs[0].remote_write(0, 'a') + self.failUnlessReallyEqual(sharefile.buffer, share_file_data) + + + @mock.patch('os.path.exists') + @mock.patch('os.path.getsize') + @mock.patch('__builtin__.open') + @mock.patch('os.listdir') + def test_read_share(self, mocklistdir, mockopen, mockgetsize, mockexists): + """ This tests whether the code correctly finds and reads + shares written out by old (Tahoe-LAFS <= v1.8.2) + servers. There is a similar test in test_download, but that one + is from the perspective of the client and exercises a deeper + stack of code. This one is for exercising just the + StorageServer object. """ + + def call_listdir(dirname): + self.failUnlessReallyEqual(dirname,'testdir/shares/or/orsxg5dtorxxeylhmvpws3temv4a') + return ['0'] + + mocklistdir.side_effect = call_listdir + + def call_open(fname, mode): + self.failUnlessReallyEqual(fname, sharefname) + self.failUnless('r' in mode, mode) + self.failUnless('b' in mode, mode) + + return StringIO(share_file_data) + mockopen.side_effect = call_open + + datalen = len(share_file_data) + def call_getsize(fname): + self.failUnlessReallyEqual(fname, sharefname) + return datalen + mockgetsize.side_effect = call_getsize + + def call_exists(fname): + self.failUnlessReallyEqual(fname, sharefname) + return True + mockexists.side_effect = call_exists + + # Now begin the test. + bs = self.s.remote_get_buckets('teststorage_index') + + self.failUnlessEqual(len(bs), 1) + b = bs[0] + self.failUnlessReallyEqual(b.remote_read(0, datalen), share_data) + # If you try to read past the end you get the as much data as is there. + self.failUnlessReallyEqual(b.remote_read(0, datalen+20), share_data) + # If you start reading past the end of the file you get the empty string. + self.failUnlessReallyEqual(b.remote_read(datalen+1, 3), '') } [a bunch of incomplete work on #999, to be unrecorded in arctic's repo zooko@zooko.com**20110406203812 Ignore-this: bece4514b60b4a972e57fa50c87c9d0 ] { move ./src/allmydata/test/test_server.py ./src/allmydata/test/test_backends.py hunk ./docs/configuration.rst 637 [storage] enabled = True readonly = True - sizelimit = 10000000000 [helper] hunk ./docs/garbage-collection.rst 16 When a file or directory in the virtual filesystem is no longer referenced, the space that its shares occupied on each storage server can be freed, -making room for other shares. Tahoe currently uses a garbage collection +making room for other shares. Tahoe uses a garbage collection ("GC") mechanism to implement this space-reclamation process. Each share has one or more "leases", which are managed by clients who want the file/directory to be retained. The storage server accepts each share for a hunk ./docs/garbage-collection.rst 34 the ``_ diagram to get an idea for the tradeoffs involved. If lease renewal occurs quickly and with 100% reliability, than any renewal time that is shorter than the lease duration will suffice, but a larger ratio -of duration-over-renewal-time will be more robust in the face of occasional +of lease duration to renewal time will be more robust in the face of occasional delays or failures. The current recommended values for a small Tahoe grid are to renew the leases replace ./docs/garbage-collection.rst [A-Za-z_0-9\-\.] Tahoe Tahoe-LAFS hunk ./src/allmydata/client.py 260 sharetypes.append("mutable") expiration_sharetypes = tuple(sharetypes) + if self.get_config("storage", "backend", "filesystem") == "filesystem": + xyz + xyz ss = StorageServer(storedir, self.nodeid, reserved_space=reserved, discard_storage=discard, hunk ./src/allmydata/interfaces.py 270 store that on disk. """ +class IStorageBackend(Interface): + """ + Objects of this kind live on the server side and are used by the + storage server object. + """ + def get_available_space(self, reserved_space): + """ Returns available space for share storage in bytes, or + None this information is not available or if the available + space is unlimited. + + If the backend is configured for read-only mode then this will + return 0. + + reserved_space is how many bytes to subtract from the answer, so + you can pass how many bytes you would like to leave unused on this + filesystem as reserved_space. """ + class IStorageBucketWriter(Interface): """ Objects of this kind live on the client side. hunk ./src/allmydata/interfaces.py 2472 class EmptyPathnameComponentError(Exception): """The webapi disallows empty pathname components.""" + +class IShareStore(Interface): + pass + hunk ./src/allmydata/storage/crawler.py 68 cpu_slice = 1.0 # use up to 1.0 seconds before yielding minimum_cycle_time = 300 # don't run a cycle faster than this - def __init__(self, server, statefile, allowed_cpu_percentage=None): + def __init__(self, backend, statefile, allowed_cpu_percentage=None): service.MultiService.__init__(self) if allowed_cpu_percentage is not None: self.allowed_cpu_percentage = allowed_cpu_percentage hunk ./src/allmydata/storage/crawler.py 72 - self.server = server - self.sharedir = server.sharedir - self.statefile = statefile + self.backend = backend self.prefixes = [si_b2a(struct.pack(">H", i << (16-10)))[:2] for i in range(2**10)] self.prefixes.sort() hunk ./src/allmydata/storage/crawler.py 446 minimum_cycle_time = 60*60 # we don't need this more than once an hour - def __init__(self, server, statefile, num_sample_prefixes=1): - ShareCrawler.__init__(self, server, statefile) + def __init__(self, statefile, num_sample_prefixes=1): + ShareCrawler.__init__(self, statefile) self.num_sample_prefixes = num_sample_prefixes def add_initial_state(self): hunk ./src/allmydata/storage/expirer.py 15 removed. I collect statistics on the leases and make these available to a web - status page, including:: + status page, including: Space recovered during this cycle-so-far: actual (only if expiration_enabled=True): hunk ./src/allmydata/storage/expirer.py 51 slow_start = 360 # wait 6 minutes after startup minimum_cycle_time = 12*60*60 # not more than twice per day - def __init__(self, server, statefile, historyfile, + def __init__(self, statefile, historyfile, expiration_enabled, mode, override_lease_duration, # used if expiration_mode=="age" cutoff_date, # used if expiration_mode=="cutoff-date" hunk ./src/allmydata/storage/expirer.py 71 else: raise ValueError("GC mode '%s' must be 'age' or 'cutoff-date'" % mode) self.sharetypes_to_expire = sharetypes - ShareCrawler.__init__(self, server, statefile) + ShareCrawler.__init__(self, statefile) def add_initial_state(self): # we fill ["cycle-to-date"] here (even though they will be reset in hunk ./src/allmydata/storage/immutable.py 44 sharetype = "immutable" def __init__(self, filename, max_size=None, create=False): - """ If max_size is not None then I won't allow more than max_size to be written to me. If create=True and max_size must not be None. """ + """ If max_size is not None then I won't allow more than + max_size to be written to me. If create=True then max_size + must not be None. """ precondition((max_size is not None) or (not create), max_size, create) self.home = filename self._max_size = max_size hunk ./src/allmydata/storage/immutable.py 87 def read_share_data(self, offset, length): precondition(offset >= 0) - # reads beyond the end of the data are truncated. Reads that start - # beyond the end of the data return an empty string. I wonder why - # Python doesn't do the following computation for me? + # Reads beyond the end of the data are truncated. Reads that start + # beyond the end of the data return an empty string. seekpos = self._data_offset+offset fsize = os.path.getsize(self.home) actuallength = max(0, min(length, fsize-seekpos)) hunk ./src/allmydata/storage/server.py 7 from twisted.application import service from zope.interface import implements -from allmydata.interfaces import RIStorageServer, IStatsProducer +from allmydata.interfaces import RIStorageServer, IStatsProducer, IShareStore from allmydata.util import fileutil, idlib, log, time_format import allmydata # for __full_version__ hunk ./src/allmydata/storage/server.py 20 from allmydata.storage.crawler import BucketCountingCrawler from allmydata.storage.expirer import LeaseCheckingCrawler +from zope.interface import implements + +# A Backend is a MultiService so that its crawlers (if it has any) can +# be started and stopped. +class Backend(service.MultiService): + implements(RIStorageServer, IStatsProducer) + def __init__(self): + service.MultiService.__init__(self) + +class NullBackend(Backend): + def __init__(self): + Backend.__init__(self) + +class FSBackend(Backend): + def __init__(self, storedir, readonly=False, reserved_space=0): + Backend.__init__(self) + + self._setup_storage(storedir, readonly, reserved_space) + self._setup_corruption_advisory() + self._setup_bucket_counter() + self._setup_lease_checkerf() + + def _setup_storage(self, storedir, readonly, reserved_space): + self.storedir = storedir + self.readonly = readonly + self.reserved_space = int(reserved_space) + if self.reserved_space: + if self.get_available_space() is None: + log.msg("warning: [storage]reserved_space= is set, but this platform does not support an API to get disk statistics (statvfs(2) or GetDiskFreeSpaceEx), so this reservation cannot be honored", + umid="0wZ27w", level=log.UNUSUAL) + + self.sharedir = os.path.join(self.storedir, "shares") + fileutil.make_dirs(self.sharedir) + self.incomingdir = os.path.join(self.sharedir, 'incoming') + self._clean_incomplete() + + def _clean_incomplete(self): + fileutil.rm_dir(self.incomingdir) + fileutil.make_dirs(self.incomingdir) + + def _setup_corruption_advisory(self): + # we don't actually create the corruption-advisory dir until necessary + self.corruption_advisory_dir = os.path.join(self.storedir, + "corruption-advisories") + + def _setup_bucket_counter(self): + statefile = os.path.join(self.storedir, "bucket_counter.state") + self.bucket_counter = BucketCountingCrawler(statefile) + self.bucket_counter.setServiceParent(self) + + def _setup_lease_checkerf(self): + statefile = os.path.join(self.storedir, "lease_checker.state") + historyfile = os.path.join(self.storedir, "lease_checker.history") + self.lease_checker = LeaseCheckingCrawler(statefile, historyfile, + expiration_enabled, expiration_mode, + expiration_override_lease_duration, + expiration_cutoff_date, + expiration_sharetypes) + self.lease_checker.setServiceParent(self) + + def get_available_space(self): + if self.readonly: + return 0 + return fileutil.get_available_space(self.storedir, self.reserved_space) + # storage/ # storage/shares/incoming # incoming/ holds temp dirs named $START/$STORAGEINDEX/$SHARENUM which will hunk ./src/allmydata/storage/server.py 105 name = 'storage' LeaseCheckerClass = LeaseCheckingCrawler - def __init__(self, storedir, nodeid, reserved_space=0, - discard_storage=False, readonly_storage=False, + def __init__(self, nodeid, backend, reserved_space=0, + readonly_storage=False, stats_provider=None, expiration_enabled=False, expiration_mode="age", hunk ./src/allmydata/storage/server.py 117 assert isinstance(nodeid, str) assert len(nodeid) == 20 self.my_nodeid = nodeid - self.storedir = storedir - sharedir = os.path.join(storedir, "shares") - fileutil.make_dirs(sharedir) - self.sharedir = sharedir - # we don't actually create the corruption-advisory dir until necessary - self.corruption_advisory_dir = os.path.join(storedir, - "corruption-advisories") - self.reserved_space = int(reserved_space) - self.no_storage = discard_storage - self.readonly_storage = readonly_storage self.stats_provider = stats_provider if self.stats_provider: self.stats_provider.register_producer(self) hunk ./src/allmydata/storage/server.py 120 - self.incomingdir = os.path.join(sharedir, 'incoming') - self._clean_incomplete() - fileutil.make_dirs(self.incomingdir) self._active_writers = weakref.WeakKeyDictionary() hunk ./src/allmydata/storage/server.py 121 + self.backend = backend + self.backend.setServiceParent(self) log.msg("StorageServer created", facility="tahoe.storage") hunk ./src/allmydata/storage/server.py 125 - if reserved_space: - if self.get_available_space() is None: - log.msg("warning: [storage]reserved_space= is set, but this platform does not support an API to get disk statistics (statvfs(2) or GetDiskFreeSpaceEx), so this reservation cannot be honored", - umin="0wZ27w", level=log.UNUSUAL) - self.latencies = {"allocate": [], # immutable "write": [], "close": [], hunk ./src/allmydata/storage/server.py 136 "renew": [], "cancel": [], } - self.add_bucket_counter() - - statefile = os.path.join(self.storedir, "lease_checker.state") - historyfile = os.path.join(self.storedir, "lease_checker.history") - klass = self.LeaseCheckerClass - self.lease_checker = klass(self, statefile, historyfile, - expiration_enabled, expiration_mode, - expiration_override_lease_duration, - expiration_cutoff_date, - expiration_sharetypes) - self.lease_checker.setServiceParent(self) def __repr__(self): return "" % (idlib.shortnodeid_b2a(self.my_nodeid),) hunk ./src/allmydata/storage/server.py 140 - def add_bucket_counter(self): - statefile = os.path.join(self.storedir, "bucket_counter.state") - self.bucket_counter = BucketCountingCrawler(self, statefile) - self.bucket_counter.setServiceParent(self) - def count(self, name, delta=1): if self.stats_provider: self.stats_provider.count("storage_server." + name, delta) hunk ./src/allmydata/storage/server.py 183 kwargs["facility"] = "tahoe.storage" return log.msg(*args, **kwargs) - def _clean_incomplete(self): - fileutil.rm_dir(self.incomingdir) - def get_stats(self): # remember: RIStatsProvider requires that our return dict # contains numeric values. hunk ./src/allmydata/storage/server.py 219 stats['storage_server.total_bucket_count'] = bucket_count return stats - def get_available_space(self): - """Returns available space for share storage in bytes, or None if no - API to get this information is available.""" - - if self.readonly_storage: - return 0 - return fileutil.get_available_space(self.storedir, self.reserved_space) - def allocated_size(self): space = 0 for bw in self._active_writers: hunk ./src/allmydata/storage/server.py 226 return space def remote_get_version(self): - remaining_space = self.get_available_space() + remaining_space = self.backend.get_available_space() if remaining_space is None: # We're on a platform that has no API to get disk stats. remaining_space = 2**64 hunk ./src/allmydata/storage/server.py 267 max_space_per_bucket = allocated_size - remaining_space = self.get_available_space() + remaining_space = self.backend.get_available_space() limited = remaining_space is not None if limited: # this is a bit conservative, since some of this allocated_size() hunk ./src/allmydata/test/common_util.py 20 def flip_one_bit(s, offset=0, size=None): """ flip one random bit of the string s, in a byte greater than or equal to offset and less - than offset+size. """ + than offset+size. Return the new string. """ if size is None: size=len(s)-offset i = randrange(offset, offset+size) hunk ./src/allmydata/test/test_backends.py 10 import mock # This is the code that we're going to be testing. -from allmydata.storage.server import StorageServer +from allmydata.storage.server import StorageServer, FSBackend, NullBackend # The following share file contents was generated with # storage.immutable.ShareFile from Tahoe-LAFS v1.8.2 hunk ./src/allmydata/test/test_backends.py 21 sharefname = 'testdir/shares/or/orsxg5dtorxxeylhmvpws3temv4a/0' class TestServerConstruction(unittest.TestCase, ReallyEqualMixin): + @mock.patch('time.time') + @mock.patch('os.mkdir') + @mock.patch('__builtin__.open') + @mock.patch('os.listdir') + @mock.patch('os.path.isdir') + def test_create_server_null_backend(self, mockisdir, mocklistdir, mockopen, mockmkdir, mocktime): + """ This tests whether a server instance can be constructed + with a null backend. The server instance fails the test if it + tries to read or write to the file system. """ + + # Now begin the test. + s = StorageServer('testnodeidxxxxxxxxxx', backend=NullBackend()) + + self.failIf(mockisdir.called) + self.failIf(mocklistdir.called) + self.failIf(mockopen.called) + self.failIf(mockmkdir.called) + self.failIf(mocktime.called) + + # You passed! + + @mock.patch('time.time') + @mock.patch('os.mkdir') @mock.patch('__builtin__.open') hunk ./src/allmydata/test/test_backends.py 45 - def test_create_server(self, mockopen): - """ This tests whether a server instance can be constructed. """ + @mock.patch('os.listdir') + @mock.patch('os.path.isdir') + def test_create_server_fs_backend(self, mockisdir, mocklistdir, mockopen, mockmkdir, mocktime): + """ This tests whether a server instance can be constructed + with a filesystem backend. To pass the test, it has to use the + filesystem in only the prescribed ways. """ def call_open(fname, mode): if fname == 'testdir/bucket_counter.state': hunk ./src/allmydata/test/test_backends.py 59 raise IOError(2, "No such file or directory: 'testdir/lease_checker.state'") elif fname == 'testdir/lease_checker.history': return StringIO() + else: + self.fail("Server with FS backend tried to open '%s' in mode '%s'" % (fname, mode)) mockopen.side_effect = call_open # Now begin the test. hunk ./src/allmydata/test/test_backends.py 64 - s = StorageServer('testdir', 'testnodeidxxxxxxxxxx') + s = StorageServer('testnodeidxxxxxxxxxx', backend=FSBackend('teststoredir')) + + self.failIf(mockisdir.called) + self.failIf(mocklistdir.called) + self.failIf(mockopen.called) + self.failIf(mockmkdir.called) + self.failIf(mocktime.called) # You passed! hunk ./src/allmydata/test/test_backends.py 74 -class TestServer(unittest.TestCase, ReallyEqualMixin): +class TestServerFSBackend(unittest.TestCase, ReallyEqualMixin): @mock.patch('__builtin__.open') def setUp(self, mockopen): def call_open(fname, mode): hunk ./src/allmydata/test/test_backends.py 86 return StringIO() mockopen.side_effect = call_open - self.s = StorageServer('testdir', 'testnodeidxxxxxxxxxx') - + self.s = StorageServer('testnodeidxxxxxxxxxx', backend=FSBackend('teststoredir')) @mock.patch('time.time') @mock.patch('os.mkdir') hunk ./src/allmydata/test/test_backends.py 94 @mock.patch('os.listdir') @mock.patch('os.path.isdir') def test_write_share(self, mockisdir, mocklistdir, mockopen, mockmkdir, mocktime): - """Handle a report of corruption.""" + """ Write a new share. """ def call_listdir(dirname): self.failUnlessReallyEqual(dirname, 'testdir/shares/or/orsxg5dtorxxeylhmvpws3temv4a') hunk ./src/allmydata/test/test_backends.py 137 bs[0].remote_write(0, 'a') self.failUnlessReallyEqual(sharefile.buffer, share_file_data) - @mock.patch('os.path.exists') @mock.patch('os.path.getsize') @mock.patch('__builtin__.open') } Context: [test: increase timeout on a network test because Francois's ARM machine hit that timeout zooko@zooko.com**20110317165909 Ignore-this: 380c345cdcbd196268ca5b65664ac85b I'm skeptical that the test was proceeding correctly but ran out of time. It seems more likely that it had gotten hung. But if we raise the timeout to an even more extravagant number then we can be even more certain that the test was never going to finish. ] [docs/configuration.rst: add a "Frontend Configuration" section Brian Warner **20110222014323 Ignore-this: 657018aa501fe4f0efef9851628444ca this points to docs/frontends/*.rst, which were previously underlinked ] [web/filenode.py: avoid calling req.finish() on closed HTTP connections. Closes #1366 "Brian Warner "**20110221061544 Ignore-this: 799d4de19933f2309b3c0c19a63bb888 ] [Add unit tests for cross_check_pkg_resources_versus_import, and a regression test for ref #1355. This requires a little refactoring to make it testable. david-sarah@jacaranda.org**20110221015817 Ignore-this: 51d181698f8c20d3aca58b057e9c475a ] [allmydata/__init__.py: .name was used in place of the correct .__name__ when printing an exception. Also, robustify string formatting by using %r instead of %s in some places. fixes #1355. david-sarah@jacaranda.org**20110221020125 Ignore-this: b0744ed58f161bf188e037bad077fc48 ] [Refactor StorageFarmBroker handling of servers Brian Warner **20110221015804 Ignore-this: 842144ed92f5717699b8f580eab32a51 Pass around IServer instance instead of (peerid, rref) tuple. Replace "descriptor" with "server". Other replacements: get_all_servers -> get_connected_servers/get_known_servers get_servers_for_index -> get_servers_for_psi (now returns IServers) This change still needs to be pushed further down: lots of code is now getting the IServer and then distributing (peerid, rref) internally. Instead, it ought to distribute the IServer internally and delay extracting a serverid or rref until the last moment. no_network.py was updated to retain parallelism. ] [TAG allmydata-tahoe-1.8.2 warner@lothar.com**20110131020101] Patch bundle hash: de6d5cdf0fe735110db2308f55d4e541dd40fd2c