2 patches for repository zooko@tahoe-lafs.org:/home/source/darcs/tahoe-lafs/trunk: Fri Mar 25 14:35:14 MDT 2011 wilcoxjg@gmail.com * storage: new mocking tests of storage server read and write There are already tests of read and functionality in test_storage.py, but those tests let the code under test use a real filesystem whereas these tests mock all file system calls. Wed Apr 6 14:38:12 MDT 2011 zooko@zooko.com * a bunch of incomplete work on #999, to be unrecorded in arctic's repo New patches: [storage: new mocking tests of storage server read and write wilcoxjg@gmail.com**20110325203514 Ignore-this: df65c3c4f061dd1516f88662023fdb41 There are already tests of read and functionality in test_storage.py, but those tests let the code under test use a real filesystem whereas these tests mock all file system calls. ] { addfile ./src/allmydata/test/test_server.py hunk ./src/allmydata/test/test_server.py 1 +from twisted.trial import unittest + +from StringIO import StringIO + +from allmydata.test.common_util import ReallyEqualMixin + +import mock + +# This is the code that we're going to be testing. +from allmydata.storage.server import StorageServer + +# The following share file contents was generated with +# storage.immutable.ShareFile from Tahoe-LAFS v1.8.2 +# with share data == 'a'. +share_data = 'a\x00\x00\x00\x00xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\x00(\xde\x80' +share_file_data = '\x00\x00\x00\x01\x00\x00\x00\x01\x00\x00\x00\x01' + share_data + +sharefname = 'testdir/shares/or/orsxg5dtorxxeylhmvpws3temv4a/0' + +class TestServerConstruction(unittest.TestCase, ReallyEqualMixin): + @mock.patch('__builtin__.open') + def test_create_server(self, mockopen): + """ This tests whether a server instance can be constructed. """ + + def call_open(fname, mode): + if fname == 'testdir/bucket_counter.state': + raise IOError(2, "No such file or directory: 'testdir/bucket_counter.state'") + elif fname == 'testdir/lease_checker.state': + raise IOError(2, "No such file or directory: 'testdir/lease_checker.state'") + elif fname == 'testdir/lease_checker.history': + return StringIO() + mockopen.side_effect = call_open + + # Now begin the test. + s = StorageServer('testdir', 'testnodeidxxxxxxxxxx') + + # You passed! + +class TestServer(unittest.TestCase, ReallyEqualMixin): + @mock.patch('__builtin__.open') + def setUp(self, mockopen): + def call_open(fname, mode): + if fname == 'testdir/bucket_counter.state': + raise IOError(2, "No such file or directory: 'testdir/bucket_counter.state'") + elif fname == 'testdir/lease_checker.state': + raise IOError(2, "No such file or directory: 'testdir/lease_checker.state'") + elif fname == 'testdir/lease_checker.history': + return StringIO() + mockopen.side_effect = call_open + + self.s = StorageServer('testdir', 'testnodeidxxxxxxxxxx') + + + @mock.patch('time.time') + @mock.patch('os.mkdir') + @mock.patch('__builtin__.open') + @mock.patch('os.listdir') + @mock.patch('os.path.isdir') + def test_write_share(self, mockisdir, mocklistdir, mockopen, mockmkdir, mocktime): + """Handle a report of corruption.""" + + def call_listdir(dirname): + self.failUnlessReallyEqual(dirname, 'testdir/shares/or/orsxg5dtorxxeylhmvpws3temv4a') + raise OSError(2, "No such file or directory: 'testdir/shares/or/orsxg5dtorxxeylhmvpws3temv4a'") + + mocklistdir.side_effect = call_listdir + + class MockFile: + def __init__(self): + self.buffer = '' + self.pos = 0 + def write(self, instring): + begin = self.pos + padlen = begin - len(self.buffer) + if padlen > 0: + self.buffer += '\x00' * padlen + end = self.pos + len(instring) + self.buffer = self.buffer[:begin]+instring+self.buffer[end:] + self.pos = end + def close(self): + pass + def seek(self, pos): + self.pos = pos + def read(self, numberbytes): + return self.buffer[self.pos:self.pos+numberbytes] + def tell(self): + return self.pos + + mocktime.return_value = 0 + + sharefile = MockFile() + def call_open(fname, mode): + self.failUnlessReallyEqual(fname, 'testdir/shares/incoming/or/orsxg5dtorxxeylhmvpws3temv4a/0' ) + return sharefile + + mockopen.side_effect = call_open + # Now begin the test. + alreadygot, bs = self.s.remote_allocate_buckets('teststorage_index', 'x'*32, 'y'*32, set((0,)), 1, mock.Mock()) + print bs + bs[0].remote_write(0, 'a') + self.failUnlessReallyEqual(sharefile.buffer, share_file_data) + + + @mock.patch('os.path.exists') + @mock.patch('os.path.getsize') + @mock.patch('__builtin__.open') + @mock.patch('os.listdir') + def test_read_share(self, mocklistdir, mockopen, mockgetsize, mockexists): + """ This tests whether the code correctly finds and reads + shares written out by old (Tahoe-LAFS <= v1.8.2) + servers. There is a similar test in test_download, but that one + is from the perspective of the client and exercises a deeper + stack of code. This one is for exercising just the + StorageServer object. """ + + def call_listdir(dirname): + self.failUnlessReallyEqual(dirname,'testdir/shares/or/orsxg5dtorxxeylhmvpws3temv4a') + return ['0'] + + mocklistdir.side_effect = call_listdir + + def call_open(fname, mode): + self.failUnlessReallyEqual(fname, sharefname) + self.failUnless('r' in mode, mode) + self.failUnless('b' in mode, mode) + + return StringIO(share_file_data) + mockopen.side_effect = call_open + + datalen = len(share_file_data) + def call_getsize(fname): + self.failUnlessReallyEqual(fname, sharefname) + return datalen + mockgetsize.side_effect = call_getsize + + def call_exists(fname): + self.failUnlessReallyEqual(fname, sharefname) + return True + mockexists.side_effect = call_exists + + # Now begin the test. + bs = self.s.remote_get_buckets('teststorage_index') + + self.failUnlessEqual(len(bs), 1) + b = bs[0] + self.failUnlessReallyEqual(b.remote_read(0, datalen), share_data) + # If you try to read past the end you get the as much data as is there. + self.failUnlessReallyEqual(b.remote_read(0, datalen+20), share_data) + # If you start reading past the end of the file you get the empty string. + self.failUnlessReallyEqual(b.remote_read(datalen+1, 3), '') } [a bunch of incomplete work on #999, to be unrecorded in arctic's repo zooko@zooko.com**20110406203812 Ignore-this: bece4514b60b4a972e57fa50c87c9d0 ] { move ./src/allmydata/test/test_server.py ./src/allmydata/test/test_backends.py hunk ./docs/configuration.rst 582 [storage] enabled = True readonly = True - sizelimit = 10000000000 [helper] enabled = True hunk ./docs/garbage-collection.rst 16 When a file or directory in the virtual filesystem is no longer referenced, the space that its shares occupied on each storage server can be freed, -making room for other shares. Tahoe currently uses a garbage collection +making room for other shares. Tahoe uses a garbage collection ("GC") mechanism to implement this space-reclamation process. Each share has one or more "leases", which are managed by clients who want the file/directory to be retained. The storage server accepts each share for a hunk ./docs/garbage-collection.rst 34 the "lease-tradeoffs.svg" diagram to get an idea for the tradeoffs involved. If lease renewal occurs quickly and with 100% reliability, than any renewal time that is shorter than the lease duration will suffice, but a larger ratio -of duration-over-renewal-time will be more robust in the face of occasional +of lease duration to renewal time will be more robust in the face of occasional delays or failures. The current recommended values for a small Tahoe grid are to renew the leases replace ./docs/garbage-collection.rst [A-Za-z_0-9\-\.] Tahoe Tahoe-LAFS hunk ./src/allmydata/client.py 228 sharetypes.append("mutable") expiration_sharetypes = tuple(sharetypes) + if self.get_config("storage", "backend", "filesystem") == "filesystem": + xyz + xyz ss = StorageServer(storedir, self.nodeid, reserved_space=reserved, discard_storage=discard, hunk ./src/allmydata/interfaces.py 250 store that on disk. """ +class IStorageBackend(Interface): + """ + Objects of this kind live on the server side and are used by the + storage server object. + """ + def get_available_space(self, reserved_space): + """ Returns available space for share storage in bytes, or + None this information is not available or if the available + space is unlimited. + + If the backend is configured for read-only mode then this will + return 0. + + reserved_space is how many bytes to subtract from the answer, so + you can pass how many bytes you would like to leave unused on this + filesystem as reserved_space. """ + class IStorageBucketWriter(Interface): """ Objects of this kind live on the client side. hunk ./src/allmydata/interfaces.py 2393 class EmptyPathnameComponentError(Exception): """The webapi disallows empty pathname components.""" + +class IShareStore(Interface): + pass + hunk ./src/allmydata/storage/crawler.py 68 cpu_slice = 1.0 # use up to 1.0 seconds before yielding minimum_cycle_time = 300 # don't run a cycle faster than this - def __init__(self, server, statefile, allowed_cpu_percentage=None): + def __init__(self, backend, statefile, allowed_cpu_percentage=None): service.MultiService.__init__(self) if allowed_cpu_percentage is not None: self.allowed_cpu_percentage = allowed_cpu_percentage hunk ./src/allmydata/storage/crawler.py 72 - self.server = server - self.sharedir = server.sharedir - self.statefile = statefile + self.backend = backend self.prefixes = [si_b2a(struct.pack(">H", i << (16-10)))[:2] for i in range(2**10)] self.prefixes.sort() hunk ./src/allmydata/storage/crawler.py 446 minimum_cycle_time = 60*60 # we don't need this more than once an hour - def __init__(self, server, statefile, num_sample_prefixes=1): - ShareCrawler.__init__(self, server, statefile) + def __init__(self, statefile, num_sample_prefixes=1): + ShareCrawler.__init__(self, statefile) self.num_sample_prefixes = num_sample_prefixes def add_initial_state(self): hunk ./src/allmydata/storage/expirer.py 15 removed. I collect statistics on the leases and make these available to a web - status page, including:: + status page, including: Space recovered during this cycle-so-far: actual (only if expiration_enabled=True): hunk ./src/allmydata/storage/expirer.py 51 slow_start = 360 # wait 6 minutes after startup minimum_cycle_time = 12*60*60 # not more than twice per day - def __init__(self, server, statefile, historyfile, + def __init__(self, statefile, historyfile, expiration_enabled, mode, override_lease_duration, # used if expiration_mode=="age" cutoff_date, # used if expiration_mode=="cutoff-date" hunk ./src/allmydata/storage/expirer.py 71 else: raise ValueError("GC mode '%s' must be 'age' or 'cutoff-date'" % mode) self.sharetypes_to_expire = sharetypes - ShareCrawler.__init__(self, server, statefile) + ShareCrawler.__init__(self, statefile) def add_initial_state(self): # we fill ["cycle-to-date"] here (even though they will be reset in hunk ./src/allmydata/storage/immutable.py 43 sharetype = "immutable" def __init__(self, filename, max_size=None, create=False): - """ If max_size is not None then I won't allow more than max_size to be written to me. If create=True and max_size must not be None. """ + """ If max_size is not None then I won't allow more than + max_size to be written to me. If create=True then max_size + must not be None. """ precondition((max_size is not None) or (not create), max_size, create) self.home = filename self._max_size = max_size hunk ./src/allmydata/storage/immutable.py 86 def read_share_data(self, offset, length): precondition(offset >= 0) - # reads beyond the end of the data are truncated. Reads that start - # beyond the end of the data return an empty string. I wonder why - # Python doesn't do the following computation for me? + # Reads beyond the end of the data are truncated. Reads that start + # beyond the end of the data return an empty string. seekpos = self._data_offset+offset fsize = os.path.getsize(self.home) actuallength = max(0, min(length, fsize-seekpos)) hunk ./src/allmydata/storage/server.py 7 from twisted.application import service from zope.interface import implements -from allmydata.interfaces import RIStorageServer, IStatsProducer +from allmydata.interfaces import RIStorageServer, IStatsProducer, IShareStore from allmydata.util import fileutil, idlib, log, time_format import allmydata # for __full_version__ hunk ./src/allmydata/storage/server.py 20 from allmydata.storage.crawler import BucketCountingCrawler from allmydata.storage.expirer import LeaseCheckingCrawler +from zope.interface import implements + +# A Backend is a MultiService so that its crawlers (if it has any) can +# be started and stopped. +class Backend(service.MultiService): + implements(RIStorageServer, IStatsProducer) + def __init__(self): + service.MultiService.__init__(self) + +class NullBackend(Backend): + def __init__(self): + Backend.__init__(self) + +class FSBackend(Backend): + def __init__(self, storedir, readonly=False, reserved_space=0): + Backend.__init__(self) + + self._setup_storage(storedir, readonly, reserved_space) + self._setup_corruption_advisory() + self._setup_bucket_counter() + self._setup_lease_checkerf() + + def _setup_storage(self, storedir, readonly, reserved_space): + self.storedir = storedir + self.readonly = readonly + self.reserved_space = int(reserved_space) + if self.reserved_space: + if self.get_available_space() is None: + log.msg("warning: [storage]reserved_space= is set, but this platform does not support an API to get disk statistics (statvfs(2) or GetDiskFreeSpaceEx), so this reservation cannot be honored", + umid="0wZ27w", level=log.UNUSUAL) + + self.sharedir = os.path.join(self.storedir, "shares") + fileutil.make_dirs(self.sharedir) + self.incomingdir = os.path.join(self.sharedir, 'incoming') + self._clean_incomplete() + + def _clean_incomplete(self): + fileutil.rm_dir(self.incomingdir) + fileutil.make_dirs(self.incomingdir) + + def _setup_corruption_advisory(self): + # we don't actually create the corruption-advisory dir until necessary + self.corruption_advisory_dir = os.path.join(self.storedir, + "corruption-advisories") + + def _setup_bucket_counter(self): + statefile = os.path.join(self.storedir, "bucket_counter.state") + self.bucket_counter = BucketCountingCrawler(statefile) + self.bucket_counter.setServiceParent(self) + + def _setup_lease_checkerf(self): + statefile = os.path.join(self.storedir, "lease_checker.state") + historyfile = os.path.join(self.storedir, "lease_checker.history") + self.lease_checker = LeaseCheckingCrawler(statefile, historyfile, + expiration_enabled, expiration_mode, + expiration_override_lease_duration, + expiration_cutoff_date, + expiration_sharetypes) + self.lease_checker.setServiceParent(self) + + def get_available_space(self): + if self.readonly: + return 0 + return fileutil.get_available_space(self.storedir, self.reserved_space) + # storage/ # storage/shares/incoming # incoming/ holds temp dirs named $START/$STORAGEINDEX/$SHARENUM which will hunk ./src/allmydata/storage/server.py 105 name = 'storage' LeaseCheckerClass = LeaseCheckingCrawler - def __init__(self, storedir, nodeid, reserved_space=0, - discard_storage=False, readonly_storage=False, + def __init__(self, nodeid, backend, reserved_space=0, + readonly_storage=False, stats_provider=None, expiration_enabled=False, expiration_mode="age", hunk ./src/allmydata/storage/server.py 117 assert isinstance(nodeid, str) assert len(nodeid) == 20 self.my_nodeid = nodeid - self.storedir = storedir - sharedir = os.path.join(storedir, "shares") - fileutil.make_dirs(sharedir) - self.sharedir = sharedir - # we don't actually create the corruption-advisory dir until necessary - self.corruption_advisory_dir = os.path.join(storedir, - "corruption-advisories") - self.reserved_space = int(reserved_space) - self.no_storage = discard_storage - self.readonly_storage = readonly_storage self.stats_provider = stats_provider if self.stats_provider: self.stats_provider.register_producer(self) hunk ./src/allmydata/storage/server.py 120 - self.incomingdir = os.path.join(sharedir, 'incoming') - self._clean_incomplete() - fileutil.make_dirs(self.incomingdir) self._active_writers = weakref.WeakKeyDictionary() hunk ./src/allmydata/storage/server.py 121 + self.backend = backend + self.backend.setServiceParent(self) log.msg("StorageServer created", facility="tahoe.storage") hunk ./src/allmydata/storage/server.py 125 - if reserved_space: - if self.get_available_space() is None: - log.msg("warning: [storage]reserved_space= is set, but this platform does not support an API to get disk statistics (statvfs(2) or GetDiskFreeSpaceEx), so this reservation cannot be honored", - umin="0wZ27w", level=log.UNUSUAL) - self.latencies = {"allocate": [], # immutable "write": [], "close": [], hunk ./src/allmydata/storage/server.py 136 "renew": [], "cancel": [], } - self.add_bucket_counter() - - statefile = os.path.join(self.storedir, "lease_checker.state") - historyfile = os.path.join(self.storedir, "lease_checker.history") - klass = self.LeaseCheckerClass - self.lease_checker = klass(self, statefile, historyfile, - expiration_enabled, expiration_mode, - expiration_override_lease_duration, - expiration_cutoff_date, - expiration_sharetypes) - self.lease_checker.setServiceParent(self) def __repr__(self): return "" % (idlib.shortnodeid_b2a(self.my_nodeid),) hunk ./src/allmydata/storage/server.py 140 - def add_bucket_counter(self): - statefile = os.path.join(self.storedir, "bucket_counter.state") - self.bucket_counter = BucketCountingCrawler(self, statefile) - self.bucket_counter.setServiceParent(self) - def count(self, name, delta=1): if self.stats_provider: self.stats_provider.count("storage_server." + name, delta) hunk ./src/allmydata/storage/server.py 183 kwargs["facility"] = "tahoe.storage" return log.msg(*args, **kwargs) - def _clean_incomplete(self): - fileutil.rm_dir(self.incomingdir) - def get_stats(self): # remember: RIStatsProvider requires that our return dict # contains numeric values. hunk ./src/allmydata/storage/server.py 219 stats['storage_server.total_bucket_count'] = bucket_count return stats - def get_available_space(self): - """Returns available space for share storage in bytes, or None if no - API to get this information is available.""" - - if self.readonly_storage: - return 0 - return fileutil.get_available_space(self.storedir, self.reserved_space) - def allocated_size(self): space = 0 for bw in self._active_writers: hunk ./src/allmydata/storage/server.py 226 return space def remote_get_version(self): - remaining_space = self.get_available_space() + remaining_space = self.backend.get_available_space() if remaining_space is None: # We're on a platform that has no API to get disk stats. remaining_space = 2**64 hunk ./src/allmydata/storage/server.py 267 max_space_per_bucket = allocated_size - remaining_space = self.get_available_space() + remaining_space = self.backend.get_available_space() limited = remaining_space is not None if limited: # this is a bit conservative, since some of this allocated_size() hunk ./src/allmydata/test/common_util.py 20 def flip_one_bit(s, offset=0, size=None): """ flip one random bit of the string s, in a byte greater than or equal to offset and less - than offset+size. """ + than offset+size. Return the new string. """ if size is None: size=len(s) i = randrange(offset, offset+size) hunk ./src/allmydata/test/test_backends.py 10 import mock # This is the code that we're going to be testing. -from allmydata.storage.server import StorageServer +from allmydata.storage.server import StorageServer, FSBackend, NullBackend # The following share file contents was generated with # storage.immutable.ShareFile from Tahoe-LAFS v1.8.2 hunk ./src/allmydata/test/test_backends.py 21 sharefname = 'testdir/shares/or/orsxg5dtorxxeylhmvpws3temv4a/0' class TestServerConstruction(unittest.TestCase, ReallyEqualMixin): + @mock.patch('time.time') + @mock.patch('os.mkdir') + @mock.patch('__builtin__.open') + @mock.patch('os.listdir') + @mock.patch('os.path.isdir') + def test_create_server_null_backend(self, mockisdir, mocklistdir, mockopen, mockmkdir, mocktime): + """ This tests whether a server instance can be constructed + with a null backend. The server instance fails the test if it + tries to read or write to the file system. """ + + # Now begin the test. + s = StorageServer('testnodeidxxxxxxxxxx', backend=NullBackend()) + + self.failIf(mockisdir.called) + self.failIf(mocklistdir.called) + self.failIf(mockopen.called) + self.failIf(mockmkdir.called) + self.failIf(mocktime.called) + + # You passed! + + @mock.patch('time.time') + @mock.patch('os.mkdir') @mock.patch('__builtin__.open') hunk ./src/allmydata/test/test_backends.py 45 - def test_create_server(self, mockopen): - """ This tests whether a server instance can be constructed. """ + @mock.patch('os.listdir') + @mock.patch('os.path.isdir') + def test_create_server_fs_backend(self, mockisdir, mocklistdir, mockopen, mockmkdir, mocktime): + """ This tests whether a server instance can be constructed + with a filesystem backend. To pass the test, it has to use the + filesystem in only the prescribed ways. """ def call_open(fname, mode): if fname == 'testdir/bucket_counter.state': hunk ./src/allmydata/test/test_backends.py 59 raise IOError(2, "No such file or directory: 'testdir/lease_checker.state'") elif fname == 'testdir/lease_checker.history': return StringIO() + else: + self.fail("Server with FS backend tried to open '%s' in mode '%s'" % (fname, mode)) mockopen.side_effect = call_open # Now begin the test. hunk ./src/allmydata/test/test_backends.py 64 - s = StorageServer('testdir', 'testnodeidxxxxxxxxxx') + s = StorageServer('testnodeidxxxxxxxxxx', backend=FSBackend('teststoredir')) + + self.failIf(mockisdir.called) + self.failIf(mocklistdir.called) + self.failIf(mockopen.called) + self.failIf(mockmkdir.called) + self.failIf(mocktime.called) # You passed! hunk ./src/allmydata/test/test_backends.py 74 -class TestServer(unittest.TestCase, ReallyEqualMixin): +class TestServerFSBackend(unittest.TestCase, ReallyEqualMixin): @mock.patch('__builtin__.open') def setUp(self, mockopen): def call_open(fname, mode): hunk ./src/allmydata/test/test_backends.py 86 return StringIO() mockopen.side_effect = call_open - self.s = StorageServer('testdir', 'testnodeidxxxxxxxxxx') - + self.s = StorageServer('testnodeidxxxxxxxxxx', backend=FSBackend('teststoredir')) @mock.patch('time.time') @mock.patch('os.mkdir') hunk ./src/allmydata/test/test_backends.py 94 @mock.patch('os.listdir') @mock.patch('os.path.isdir') def test_write_share(self, mockisdir, mocklistdir, mockopen, mockmkdir, mocktime): - """Handle a report of corruption.""" + """ Write a new share. """ def call_listdir(dirname): self.failUnlessReallyEqual(dirname, 'testdir/shares/or/orsxg5dtorxxeylhmvpws3temv4a') hunk ./src/allmydata/test/test_backends.py 137 bs[0].remote_write(0, 'a') self.failUnlessReallyEqual(sharefile.buffer, share_file_data) - @mock.patch('os.path.exists') @mock.patch('os.path.getsize') @mock.patch('__builtin__.open') } Context: [immutable: use PrefixingLogMixin to organize logging in Tahoe2PeerSelector and add more detailed messages about peer zooko@zooko.com**20100719082000 Ignore-this: e034c4988b327f7e138a106d913a3082 ] [trivial: remove unused import detected by pyflakes, and remove trailing whitespace zooko@zooko.com**20090305223204 Ignore-this: 991001a50da1a357a519c3cb880d7ee1 ] [immutable: extend the tests to check that the shares that got uploaded really do make a sufficiently Happy distribution zooko@zooko.com**20100719045047 Ignore-this: 89c33a7b795e23018667351045a8d5d0 This patch also renames some instances of "find_shares()" to "find_all_shares()" and other instances to "find_uri_shares()" as appropriate -- the conflation between those names confused me at first when writing these tests. ] [test_repairer: add (commented-out) test_each_byte, to see exactly what the Brian Warner **20100110203552 Ignore-this: 8e84277d5304752edeff052b97821815 Verifier misses The results (described in #819) match our expectations: it misses corruption in unused share fields and in most container fields (which are only visible to the storage server, not the client). 1265 bytes of a 2753 byte share (hosting a 56-byte file with an artifically small segment size) are unused, mostly in the unused tail of the overallocated UEB space (765 bytes), and the allocated-but-unwritten plaintext_hash_tree (480 bytes). ] [Verifier: check the full block-hash-tree on each share Brian Warner **20091005214844 Ignore-this: 3f7ccf6d253f32340f1bf1da27803eee Removed the .todo from two test_repairer tests that check this. The only remaining .todos are on the three crypttext-hash-tree tests. ] [Verifier: check the full share-hash chain on each share Brian Warner **20091005213443 Ignore-this: 3d30111904158bec06a4eac22fd39d17 Removed the .todo from two test_repairer tests that check this. ] [immutable/checker.py: rearrange code a little bit, make it easier to follow Brian Warner **20091005200252 Ignore-this: 91cc303fab66faf717433a709f785fb5 ] [Correct harmless indentation errors found by pylint david-sarah@jacaranda.org**20100226052151 Ignore-this: 41335bce830700b18b80b6e00b45aef5 ] [setup: if executing lsb_release doesn't work, fall back to parsing /etc/lsb-release before falling back to platform.dist() zooko@zooko.com**20080923162858 An explanatio of why we do it this way is in the docstring. ] [setup: if invoking lsb_release doesn't work (which it doesn't on our etch buildslave), then fall back to the Python Standard Library's platform.dist() function zooko@zooko.com**20080923154820] [Improvements to test_hung_server, and fix for status updates in download.py david-sarah@jacaranda.org**20100130064303 Ignore-this: dd889c643afdcf0f86d55855aafda6ad ] [immutable: fix bug in tests, change line-endings to unix style, add comment zooko@zooko.com**20100129184237 Ignore-this: f6bd875fe974c55c881e05eddf8d3436 ] [New tests for #928 david-sarah@jacaranda.org**20100129123845 Ignore-this: 5c520f40141f0d9c000ffb05a4698995 ] [immutable: download from the first servers which provide at least K buckets instead of waiting for all servers to reply zooko@zooko.com**20100127233417 Ignore-this: c855355a40d96827e1d0c469a8d8ab3f This should put an end to the phenomenon I've been seeing that a single hung server can cause all downloads on a grid to hang. Also it should speed up all downloads by (a) not-waiting for responses to queries that it doesn't need, and (b) downloading shares from the servers which answered the initial query the fastest. Also, do not count how many buckets you've gotten when deciding whether the download has enough shares or not -- instead count how many buckets to *unique* shares that you've gotten. This appears to improve a slightly weird behavior in the current download code in which receiving >= K different buckets all to the same sharenumber would make it think it had enough to download the file when in fact it hadn't. This patch needs tests before it is actually ready for trunk. ] [M-x whitespace-cleanup zooko@zooko.com**20100718032739 Ignore-this: babfd4af6ad2fc885c957fd5c8b10c3f ] [Improve code coverage of the Tahoe2PeerSelector tests. Kevan Carstensen **20100515032913 Ignore-this: 793151b63ffa65fdae6915db22d9924a ] [immutable: test for #1124 zooko@zooko.com**20100718222907 Ignore-this: 1766e3cbab92ea2a9e246f40eb6e770b ] [immutable: test for #1118 zooko@zooko.com**20100718221537 Ignore-this: 8882aabe2aaec6a0148c87e735d817ad ] [Revisions of the #778 tests, per reviewers' comments Kevan Carstensen **20100514012542 Ignore-this: 735bbc7f663dce633caeb3b66a53cf6e - Fix comments and confusing naming. - Add tests for the new error messages suggested by David-Sarah and Zooko. - Alter existing tests for new error messages. - Make sure that the tests continue to work with the trunk. - Add a test for a mutual disjointedness assertion that I added to upload.servers_of_happiness. - Fix the comments to correctly reflect read-onlyness - Add a test for an edge case in should_add_server - Add an assertion to make sure that share redistribution works as it should - Alter tests to work with revised servers_of_happiness semantics - Remove tests for should_add_server, since that function no longer exists. - Alter tests to know about merge_peers, and to use it before calling servers_of_happiness. - Add tests for merge_peers. - Add Zooko's puzzles to the tests. - Edit encoding tests to expect the new kind of failure message. - Edit tests to expect error messages with the word "only" moved as far to the right as possible. - Extended and cleaned up some helper functions. - Changed some tests to call more appropriate helper functions. - Added a test for the failing redistribution algorithm - Added a test for the progress message - Added a test for the upper bound on readonly peer share discovery. ] [Add tests for the behavior described in #834. Kevan Carstensen **20091123012008 Ignore-this: d8e0aa0f3f7965ce9b5cea843c6d6f9f ] [upload.py: fix #1118 by aborting newly-homeless buckets when reassignment runs. This makes a previously failing assert correct. This version refactors 'abort' into two methods, rather than using a default argument. david-sarah@jacaranda.org**20100719044655 Ignore-this: 142d182c0739986812140bb8387077d5 ] [immutable/upload.py: abort buckets if peer selection fails Kevan Carstensen **20100715231714 Ignore-this: 2a0b643a22284df292d8ed9d91b1fd37 ] [Fix up the behavior of #778, per reviewers' comments Kevan Carstensen **20100514004917 Ignore-this: 9c20b60716125278b5456e8feb396bff - Make some important utility functions clearer and more thoroughly documented. - Assert in upload.servers_of_happiness that the buckets attributes of PeerTrackers passed to it are mutually disjoint. - Get rid of some silly non-Pythonisms that I didn't see when I first wrote these patches. - Make sure that should_add_server returns true when queried about a shnum that it doesn't know about yet. - Change Tahoe2PeerSelector.preexisting_shares to map a shareid to a set of peerids, alter dependencies to deal with that. - Remove upload.should_add_servers, because it is no longer necessary - Move upload.shares_of_happiness and upload.shares_by_server to a utility file. - Change some points in Tahoe2PeerSelector. - Compute servers_of_happiness using a bipartite matching algorithm that we know is optimal instead of an ad-hoc greedy algorithm that isn't. - Change servers_of_happiness to just take a sharemap as an argument, change its callers to merge existing_shares and used_peers before calling it. - Change an error message in the encoder to be more appropriate for servers of happiness. - Clarify the wording of an error message in immutable/upload.py - Refactor a happiness failure message to happinessutil.py, and make immutable/upload.py and immutable/encode.py use it. - Move the word "only" as far to the right as possible in failure messages. - Use a better definition of progress during peer selection. - Do read-only peer share detection queries in parallel, not sequentially. - Clean up logging semantics; print the query statistics whenever an upload is unsuccessful, not just in one case. ] [upload: use WriteBucketProxy_v2 when uploading a large file (with shares larger than 4GiB). This finally closes #346. I think we can now handle immutable files up to 48EiB. warner@allmydata.com**20090113021442] [Alter the signature of set_shareholders in IEncoder to add a 'servermap' parameter, which gives IEncoders enough information to perform a sane check for servers_of_happiness. Kevan Carstensen **20091104033241 Ignore-this: b3a6649a8ac66431beca1026a31fed94 ] [Alter the error message when an upload fails, per some comments in #778. Kevan Carstensen **20091230210344 Ignore-this: ba97422b2f9737c46abeb828727beb1 When I first implemented #778, I just altered the error messages to refer to servers where they referred to shares. The resulting error messages weren't very good. These are a bit better. ] [Alter the error message returned when peer selection fails Kevan Carstensen **20091123002405 Ignore-this: b2a7dc163edcab8d9613bfd6907e5166 The Tahoe2PeerSelector returned either NoSharesError or NotEnoughSharesError for a variety of error conditions that weren't informatively described by them. This patch creates a new error, UploadHappinessError, replaces uses of NoSharesError and NotEnoughSharesError with it, and alters the error message raised with the errors to be more in line with the new servers_of_happiness behavior. See ticket #834 for more information. ] [Eliminate overcounting iof servers_of_happiness in Tahoe2PeerSelector; also reorganize some things. Kevan Carstensen **20091118014542 Ignore-this: a6cb032cbff74f4f9d4238faebd99868 ] [Alter Tahoe2PeerSelector to make sure that it recognizes existing shares on readonly servers, fixing an issue in #778 Kevan Carstensen **20091116192805 Ignore-this: 15289f4d709e03851ed0587b286fd955 ] [trailing-whitespace eradication, no functional changes warner@allmydata.com**20071101222247] [Alter 'immutable/encode.py' and 'immutable/upload.py' to use servers_of_happiness instead of shares_of_happiness. Kevan Carstensen **20091104111222 Ignore-this: abb3283314820a8bbf9b5d0cbfbb57c8 ] [upload: remove Tahoe3 peer-selection algorithm warner@lothar.com**20070916082611] [storage: use fileutil's version of get_disk_stats() and get_available_space(), use mockery/fakery in tests, enable large share test on platforms with sparse files and if > 4 GiB of disk space is currently available zooko@zooko.com**20100910173629 Ignore-this: 1304f1164c661de6d5304f993eb9b27b ] [test_storage.py: change skip note for test_large_share to say that Windows doesn't support sparse files. refs #569 david-sarah@jacaranda.org**20100805022612 Ignore-this: 85c807a536dc4eeb8bf14980028bb05b ] [storage: disable test_large_share again: my linux laptop has less than 4 GiB free zooko@zooko.com**20090131041649 Ignore-this: da931a572a88699371057f932106b375 ] [storage: enable the test of a share larger than 2 GiB; this will probably be too expensive on Mac OS X, but I think it won't be on all the other platforms ; I will probably set it to SkipTest if the sys.platform is Mac after seeing the results of this buildbot run zooko@zooko.com**20090128223312 Ignore-this: 1b08b73b8f9ec4b5f629b734c556f2ed ] [immutable: skip the test of large files, because that is too hard on the host if it doesn't efficiently handle sparse files zooko@zooko.com**20090105230727 Ignore-this: 7d35a6cdb1ea6be2adf0e6dacefe01a7 ] [fileutil: copy in the get_disk_stats() and get_available_space() functions from storage/server.py zooko@zooko.com**20100910173520 Ignore-this: 8b15569715f710f4fc5092f7ca109253 ] [util.fileutil, test.test_util: add abspath_expanduser_unicode function, to work around . util.encodingutil: add a convenience function argv_to_abspath. david-sarah@jacaranda.org**20100721231507 Ignore-this: eee6904d1f65a733ff35190879844d08 ] [util/statistics: add tests, fix mean_repair_cost warner@lothar.com**20090215222326 Ignore-this: c576eabc74c23b170702018fc3c122d9 ] [stringutils.py: Add encoding argument to quote_output. Also work around a bug in locale.getpreferredencoding on older Pythons. david-sarah@jacaranda.org**20100616042012 Ignore-this: 48174c37ad95205997e4d3cdd81f1e28 ] [Rename stringutils to encodingutil, and drop listdir_unicode and open_unicode (since the Python stdlib functions work fine with Unicode paths). Also move some utility functions to fileutil. david-sarah@jacaranda.org**20100712003015 Ignore-this: 103b809d180df17a7283077c3104c7be ] [test_stringutils.py: trivial error in exception message for skipped test. david-sarah@jacaranda.org**20100607061455 Ignore-this: f261a5d4e2b8fe3bcc37e02539ba1ae2 ] [Back out Windows-specific Unicode argument support for v1.7. david-sarah@jacaranda.org**20100609000803 Ignore-this: b230ffe6fdaf9a0d85dfe745b37b42fb ] [Fix for Unicode-related test failures on Zooko's OS X 10.6 machine. david-sarah@jacaranda.org**20100609055448 Ignore-this: 395ad16429e56623edfa74457a121190 ] [test_cli.py: remove invalid 'test_listdir_unicode_bad' test. david-sarah@jacaranda.org**20100607183730 Ignore-this: fadfe87980dc1862f349bfcc21b2145f ] [test_stringutils.py: Fix test failure on CentOS builder, possibly Python 2.4.3-related. david-sarah@jacaranda.org**20100609065056 Ignore-this: 503b561b213baf1b92ae641f2fdf080a ] [More Unicode test fixes. david-sarah@jacaranda.org**20100607053358 Ignore-this: 6a271fb77c31f28cb7bdba63b26a2dd2 ] [Unicode fixes for platforms with non-native-Unicode filesystems. david-sarah@jacaranda.org**20100607043238 Ignore-this: 2134dc1793c4f8e50350bd749c4c98c2 ] [Provisional patch to NFC-normalize filenames going in and out of Tahoe directories. david-sarah@jacaranda.org**20100616031450 Ignore-this: ed08c9d8df37ef0b7cca42bb562c996b ] [dirnode: fix my remarkably-consistent 'metdadata' typo warner@allmydata.com**20081003010845] [trivial: removed unused import noticed by pyflakes zooko@zooko.com**20090709130513 Ignore-this: 63257c9e8481fdcf617f04cc48d95d03 ] [Add must_exist, must_be_directory, and must_be_file arguments to DirectoryNode.delete. This will be used to fixes a minor condition in the SFTP frontend. david-sarah@jacaranda.org**20100527194529 Ignore-this: 6d8114cef4450c52c57639f82852716f ] [stringutils.py: don't NFC-normalize the output of listdir_unicode. david-sarah@jacaranda.org**20100617015537 Ignore-this: 93c9b6f3d7c6812a0afa8d9e1b0b4faa ] [test_stringutils.py: take account of the output of listdir_unicode no longer being normalized. Also use Unicode escapes, not UTF-8. david-sarah@jacaranda.org**20100617034409 Ignore-this: 47f3f072f0e2efea0abeac130f84c56f ] [test_backupdb.py: skip test_unicode if we can't represent the test filenames. david-sarah@jacaranda.org**20100619022620 Ignore-this: 6ee564b6c07f9bb0e89a25dc5b37194f ] [tahoe backup: unicode tests. david-sarah@jacaranda.org**20100618035211 Ignore-this: 88ebab9f3218f083fdc635bff6599b60 ] [Allow URIs passed in the initial JSON for t=mkdir-with-children, t=mkdir-immutable to be Unicode. Also pass the name of each child into nodemaker.create_from_cap for error reporting. david-sarah@jacaranda.org**20100711195525 Ignore-this: deac32d8b91ba26ede18905d3f7d2b93 ] [docs/configuration.rst: fix a typo in the previous correction, and correct another error ('[storage]readonly_storage' should be '[storage]readonly'). david-sarah@jacaranda.org**20110123023955 Ignore-this: 2f9d3fe3c25da1b369618b8cf0867a58 ] [docs/configuration.rst: changes to formatting (mainly putting commands and filenames in monospace). david-sarah@jacaranda.org**20101212181828 Ignore-this: 8a1480e2d5f43bee678476424615b50f ] [Update hyperlinks between docs, and linkify some external references. refs #1225 david-sarah@jacaranda.org**20101212051459 Ignore-this: cd43a4c3d3de1f832abfa88d5fc4ace1 ] [docs: remove the redundant (and therefore bit-rotting) parts of mutable-DSA.txt and instead refer to mutable.txt zooko@zooko.com**20080429225158] [docs/historical: move 'tahoe2' from wiki into source tree warner@allmydata.com**20080603014331] [add docs/proposed/GridID.txt (cleaning out some of my old branches) warner@lothar.com**20090621191204] [docs: fix tab-vs-spaces, make some CLI examples /"literal", wrap some to Brian Warner **20101015060606 Ignore-this: eae08bdf0afb19a2fbf41c31e70a8122 80-cols, remove spurious whitespace. Add rst2html.py rule to Makefile. ] [docs/Makefile: rules to turn our SVG images into .png and .eps forms warner@lothar.com**20080102230022] [docs/known_issues.rst: Add section on traffic analysis. Fix URL for current version of file. david-sarah@jacaranda.org**20101024234259 Ignore-this: f3416e79d3bb833f5118da23e85723ad ] [Release v1.8.1. refs #1242 david-sarah@jacaranda.org**20101128204738 Ignore-this: b0793a8eb0a711cbcaebc566c37920e4 ] [NEWS: note dependency updates to pycryptopp and pycrypto. david-sarah@jacaranda.org**20100924191207 Ignore-this: eeaf5c9c9104f24c450c2ec4482ac1ee ] [minor: remove unused interface declaration, change allmydata.org to tahoe-lafs.org in email address, fix wording in relnotes.txt zooko@zooko.com**20100930153708 Ignore-this: a452969228afed2774de375e29fa3048 ] [interfaces.py: wrap some lines to 80cols Brian Warner **20090702015728 Ignore-this: e2c777c1e89a684b43ceabeb0042456c ] [docs: update how-to-make-a-release doc with a few tweaks from the 1.8.0 process zooko@zooko.com**20101015054413 Ignore-this: ca5e9478531a3393792ae283239549dd ] [docs: a couple of small edits to CREDITS and how_to_make_a_tahoe-lafs_release.txt zooko@zooko.com**20100829222758 Ignore-this: cfdb414f86dfd581b5f1d4d94231b85c ] [how_to_make_a_tahoe-lafs_release.txt: add step to check that release will report itself as the intended version. david-sarah@jacaranda.org**20100807004254 Ignore-this: 7709322e883f4118f38c7f042f5a9a2 ] [docs: update the list of forums to announce Tahoe-LAFS too, add empty checkboxes zooko@zooko.com**20100802063314 Ignore-this: 89d0e8bd43f1749a9e85fcee2205bb04 ] [how_to_make_a_tahoe-lafs_release.txt: reordering, add fuse-sshfs@lists.sourceforge.list as place to send relnotes. david-sarah@jacaranda.org**20100618041854 Ignore-this: 2e380a6e72917d3a20a65ceccd9a4df ] [docs/how_to_make_a_tahoe-lafs_release.txt: trivial correction, install.html should now be quickstart.html. david-sarah@jacaranda.org**20100625223929 Ignore-this: 99a5459cac51bd867cc11ad06927ff30 ] [NEWS: minor wording changes and rewrapping; add entry for #71. david-sarah@jacaranda.org**20101124002122 Ignore-this: 423d31eacce10463f21299860a4fbd4f ] [docs: NEWS: put news items about bugfixes/improvements and packaging before news items about documentation zooko@zooko.com**20101120060716 Ignore-this: 1a4306df950fcdc2ab9771c874d6d0a4 ] [docs: NEWS: add #1255 zooko@zooko.com**20101120071249 Ignore-this: d37ac1a115f6cdebc3dadb32131b6141 ] [docs: NEWS: add #1233 zooko@zooko.com**20101120071634 Ignore-this: 21345072269617999b78405fb314ab28 ] [docs: NEWS: merge two additions zooko@zooko.com**20101111055851 Ignore-this: 13105496bb418ebbd570ba68ef347f2 ] [NEWS: entry for #1242 (tilde expansion in 'tahoe backup --exclude-from'). refs #1242 david-sarah@jacaranda.org**20101104011915 Ignore-this: 1c85e7c74f5a48b4cdae5aa073c6b9fb ] [docs: NEWS: refs #1253 zooko@zooko.com**20101111044118 Ignore-this: 23d1cfbd2d43a68ca496958b55e4688f ] [NEWS: entries for #1190 and #1212, and minor cleanups. refs #1190, #1212 david-sarah@jacaranda.org**20101031051426 Ignore-this: c318dff69296ae1e1a897752b5221870 ] [NEWS: add news entry for #1223 Francois Deppierraz **20101030111130 Ignore-this: 6b6afd4b0f0527a3c9784c1db95d083 ] [NEWS: add a NEWS entry about bug #1045 Francois Deppierraz **20101030101351 Ignore-this: 7e758afbbd0f1d22a5d0b4fc38661c1d ] [NEWS: clarify (strengthen) description of what backdoors.rst declares, and add bugfix entries for 'tahoe cp' and Windows console bugs. refs #1216, #1224, #1232 david-sarah@jacaranda.org**20101028180046 Ignore-this: 1c3eef3cd353b06b6ee00ce87c5ef59a ] [docs: update NEWS ref: #1216 zooko@zooko.com**20101015053719 Ignore-this: 2e0b92e4145d667cdf075e64b7965530 ] [Convert docs/frontends and docs/specifications to reStructuredText format (not including file moves). david-sarah@jacaranda.org**20101212004632 Ignore-this: e3ceb2d832d73875abe48624ddbb5622 ] [update thingA/uri-extension docs warner@lothar.com**20070610033148] [mutable.txt: fix offset math in the SDMF layout warner@allmydata.com**20080213233906] [docs/uri.txt: update to reflect mutable files warner@allmydata.com**20080214235929] [webapi: make the /operations/ 't=status' qualifier optional, remove it from examples warner@lothar.com**20081023225658] [webapi.txt: improve t=deep-size output docs warner@lothar.com**20081022005331] [#514: add meta-refresh=60 tag to t=status page for incomplete operations warner@lothar.com**20081022164842] [webapi.txt: minor edits warner@allmydata.com**20081208213256] [webapi/deep-manifest t=JSON: don't return the (large) manifest/SI/verifycap lists unless the operation has completed, to avoid the considerable CPU+memory cost of creating the JSON (for 330k dirnodes, it could take two minutes to generate 275MB of JSON). They must be paid eventually, but not on every poll warner@allmydata.com**20090109015932] [webapi.txt: explain POST /uri/?t=upload, thanks to Toby Murray for the catch warner@allmydata.com**20090115000803] [CLI.txt: improve docs on 'tahoe manifest' to cover --verify-cap, --repair-cap, and streaming JSON warner@allmydata.com**20090123225939] [Added documentation for '--exclude' and friends cli backup command. Alberto Berti **20090224153049 Ignore-this: bbc791fa56e38535bb82cc3077ffde90 ] [docs: a few edits/updates about dirnodes zooko@zooko.com**20090413160837 Ignore-this: 107fc1796e6c7f5b68b7e2517cce516a ] [dirnodes.txt: update docs a bit, we don't yet do choose-latest-version, just choose-any-version warner@allmydata.com**20080208021405] [update "tahoe backup" docs, and webapi.txt's mkdir-with-children Brian Warner **20091127055900 Ignore-this: defac1fb9a2335b0af3ef9dbbcc67b7e ] [FTP-and-SFTP.txt: fix ssh-keygen pointer Brian Warner **20091207052803 Ignore-this: bc2a70ee8c58ec314e79c1262ccb22f7 ] [Address comments by Kevan on 833 and add test for stripping spaces david-sarah@jacaranda.org**20100127230642 Ignore-this: de36aeaf4afb3ba05dbeb49a5e9a6b26 ] [Patch to accept t=set-children as well as t=set_children david-sarah@jacaranda.org**20100124030020 Ignore-this: 2c061f12af817cdf77feeeb64098ec3a ] [Fix example JSON in webapi.txt that cannot occur in practice david-sarah@jacaranda.org**20100129032742 Ignore-this: 361a1ba663d77169aeef93caef870097 ] [Add mutable field to t=json output for unknown nodes, when mutability is known david-sarah@jacaranda.org**20100129031424 Ignore-this: 1516d63559bdfeb6355485dff0f5c04e ] [Show -IMM and -RO suffixes for types of immutable and read-only unknown nodes in directory listings david-sarah@jacaranda.org**20100128220800 Ignore-this: dc5c17c0a566398f88e4303c41321e66 ] [web/directory.py: remove unused imports warner@allmydata.com**20081007194820] [doc_reformat_CLI.txt freestorm77@gmail.com**20100424121512 Ignore-this: 2d3a59326810adcb20ea232cea405645 - Added heading format begining and ending by "==" - Added Index - Added Title Note: No change are made in paragraphs content ] [CLI.txt: introduce 'create-alias' before 'add-alias', document Unicode argument support, and other minor updates. david-sarah@jacaranda.org**20100610225547 Ignore-this: de7326e98d79291cdc15aed86ae61fe8 ] [doc: add mention of "tahoe create-alias" in the security-warning section of CLI.txt zooko@zooko.com**20081224211646 Ignore-this: 6bb0ab3af59a79e05ebccb800d9a12b0 ] [docs/CLI.txt: add a warning about leaking dircaps through argv in add-alias warner@lothar.com**20080721223309] [FTP-and-SFTP.txt: add Known Issues section. david-sarah@jacaranda.org**20100619004311 Ignore-this: 8d9b1da941cbc24657bb6ec268f984dd ] [FTP-and-SFTP.txt: remove description of public key format that is not actually implemented. Document that SFTP does not support server private keys with passphrases, and that FTP cannot list directories containing mutable files. david-sarah@jacaranda.org**20100619001738 Ignore-this: bf9ef53b85b934822ec76060e1fcb3cb ] [webapi.txt: fix statement about leap seconds. david-sarah@jacaranda.org**20100619035603 Ignore-this: 80b685446e915877a421cf3e31cedf30 ] [docs/specifications/dirnodes.txt: bring layer terminology up-to-date with architecture.txt, and a few other updates (e.g. note that the MAC is no longer verified, and that URIs can be unknown). Also 'Tahoe'->'Tahoe-LAFS'. david-sarah@jacaranda.org**20100723054703 Ignore-this: f3b98183e7d0a0f391225b8b93ac6c37 ] [webapi.txt: fix grammatical error. david-sarah@jacaranda.org**20100810064127 Ignore-this: 64f66aa71682195f82ac1066fe947e35 ] [dirnode.py: stop writing 'ctime' and 'mtime' fields. Includes documentation and test changes. david-sarah@jacaranda.org**20100618230119 Ignore-this: 709119898499769dd64c7977db7c84a6 ] [dirnode.py: Fix bug that caused 'tahoe' fields, 'ctime' and 'mtime' not to be updated when new metadata is present. david-sarah@jacaranda.org**20100602014644 Ignore-this: 5bac95aa897b68f2785d481e49b6a66 ] [trivial: remove trailing whitespace and unused import zooko@zooko.com**20090412021742 Ignore-this: e249172dd0ef51ee034819bc4c62cd9d ] [dirnode.py: fix a bug in the no-write change for Adder, and improve test coverage. Add a 'metadata' argument to create_subdirectory, with documentation. Also update some comments in test_dirnode.py made stale by the ctime/mtime change. david-sarah@jacaranda.org**20100602032641 Ignore-this: 48817b54cd63f5422cb88214c053b03b ] [dirnode.py: Fix #1034 (MetadataSetter does not enforce restriction on setting 'tahoe' subkeys), and expose the metadata updater for use by SFTP. Also, support diminishing a child cap to read-only if 'no-write' is set in the metadata. david-sarah@jacaranda.org**20100601045428 Ignore-this: 14f26e17e58db97fad0dcfd350b38e95 ] [docs: doc of the download status page zooko@zooko.com**20100814054117 Ignore-this: a82ec33da3c39a7c0d47a7a6b5f81bbb ref: http://tahoe-lafs.org/trac/tahoe-lafs/ticket/1169#comment:1 ] [docs/frontends/FTP-and-SFTP.txt: warn users about connecting to the FTP and SFTP servers remotely. Fixes #1192 david-sarah@jacaranda.org**20100910193234 Ignore-this: 7e403e9f349dc38f49197eb0835d47c5 ] [docs/frontends/FTP-and-SFTP.txt : ftpd and sftpd doesn't listen on loopback interface only marc.doudiet@nimag.net**20100813140853 Ignore-this: 5b5dfd0e5991a2669fe41ba13ea21bd4 ] [docs/frontends/webapi.txt: note that 'count-good-share-hosts' is computed incorrectly; refs #1115 david-sarah@jacaranda.org**20100911002548 Ignore-this: 606989661c95a6db109f8fb7bd9a8bf9 ] [docs/frontends/webapi.txt: document that the meaning of the 'healthy' field may change in future to reflect servers-of-happiness; refs #614 david-sarah@jacaranda.org**20100911003147 Ignore-this: 4661d576c145cc2b641481b70e34e357 ] [docs/configuration.rst: correct an error in the Example section ('[helper]run_helper' should be '[helper]enabled'). david-sarah@jacaranda.org**20110123022304 Ignore-this: d16d7c0d5faea3774dc77e7ae4212138 ] [docs: convert all .txt docs to .rst thanks to Ravi Pinjala zooko@zooko.com**20101015052913 Ignore-this: 178a5122423189ecfc45b142314a78ec fixes #1225 ] [docs/filesystem-notes.txt: add notes about enabling the 'directory index' feature on ext3 filesystems for storage server lookup speed warner@allmydata.com**20080821205901] [logging.txt: explain how to put log.err at the end of Deferred chains, explain FLOGTOTWISTED=1 warner@lothar.com**20080920173500] [node.py: remove support for the old BASEDIR/authorized_keys.PORT file warner@allmydata.com**20081029043420] [docs: fix cutnpasto in source:docs/logging.txt zooko@zooko.com**19700105140422 Ignore-this: de0f9ceb8e0ca4c158492ad2f9a6ba6f ] [tahoe.cfg: add tub.location, to override the location hints we include in our FURL. This replaces advertised_ip_addresses, which doesn't remain useful enough to retain it. Helps with #517 (Tor). warner@allmydata.com**20081113004458] [test_node: improve coverage of advertised_ip_addresses a bit warner@allmydata.com**20080930060816] [docs/configuration.txt: fix minor typo warner@lothar.com**20081202215101] [node.py: use NODEDIR/tmp for the 'tempfile' module's temporary directory, so webapi upload tempfiles are put there instead of /tmp . You can set it to something else by setting [node]tempdir in tahoe.cfg warner@allmydata.com**20090115020015] [util/time_format: new routine to parse dates like 2009-03-18, switch expirer to use it. I'd prefer to use 18-Mar-2009, but it is surprisingly non-trivial to build a parser that will take UTC dates instead of local dates warner@allmydata.com**20090319005814 Ignore-this: 4f08b301c780a78084fa02a09b5dda3 ] [add utility function to parse durations, for lease-expiration config warner@lothar.com**20090317070117 Ignore-this: 2f4fa4352c887b4767969f0ea7931644 ] [move GC docs out of proposed/, since it's all implemented now. Add reference to configuration.txt . Add expire.*= suggestions to tahoe.cfg . warner@allmydata.com**20090323230820 Ignore-this: ce0ec977f49c4a3642cf3a7ea7789dc4 ] [storage server: detect disk space usage on Windows too (fixes #637) david-sarah@jacaranda.org**20091121055644 Ignore-this: 20fb30498174ce997befac7701fab056 ] [storage.py: unbreak readonly_storage=True on windows warner@allmydata.com**20081202014946] [storage.py: oops, fix windows again, readonly_storage wasn't getting picked up properly warner@lothar.com**20081203010317] [test_storage.py: more windows-vs-readonly-storage fixes warner@lothar.com**20081203014102] [BucketCountingCrawler: store just the count, not cycle+count, since it's too easy to make usage mistakes otherwise warner@lothar.com**20090221035831 Ignore-this: 573b6f651af74380cdd64059fbbdda4b ] [BucketCountingCrawler: rename status and state keys to use 'bucket' instead of 'share', because the former is more accurate warner@lothar.com**20090221034606 Ignore-this: cf819f63fac9506c878d6c9715ce35b7 ] [stats: don't return booleans: it violates the schema. Add a test. warner@lothar.com**20081204210124] [backupdb.txt: fix ST_CTIME reference Brian Warner **20100114194052 Ignore-this: 5a189c7a1181b07dd87f0a08ea31b6d3 ] [docs: fix references to --no-storage, explanation of [storage] section Brian Warner **20100127200956 Ignore-this: f4be1763a585e1ac6299a4f1b94a59e0 ] [Add create-node CLI command, and make create-client equivalent to create-node --no-storage (fixes #760) david-sarah@jacaranda.org**20100116052055 Ignore-this: 47d08b18c69738685e13ff365738d5a ] [windows installer - changed to update for Mikes recent changes to add AllmydataManager which uses a named pipe to avoid some permissions issues on Vista and fixes some netowkr problems secorp@allmydata.com**20080517011257] [native client - removed unused file (used to be for elevating privileges) and changed installer to start up the AllmydataTray executable directly secorp@allmydata.com**20080328192508] [windows installer - changed to reflect the new StartAllmydata.exe executable that is used to launch the various pieces of the native client. Also verified that the SMB service was stopped when uninstalling. secorp@allmydata.com**20080318174443] [native client - updated to match naming for service and tray executable secorp@allmydata.com**20080222022431] [native client - changing way the executables ask for elevated privileges secorp@allmydata.com**20080328064107] [native client - updated to auto mount drive at start as well as when the drive is opened secorp@allmydata.com**20080327042631] [native client - updated to automatically create a Backup directory, temp directory cleanup, automatic mounting and unmounting of the drive when starting and stopping the service, lots of Vista backup error fixes secorp@allmydata.com**20080327040530] [native client - updated to fix windows vista backup rproblems, edit word documents directly on the drive, requeue files that failed to upload from the node to the helper secorp@allmydata.com**20080326000059] [native client - adding support for special icons for shared and recycling directories secorp@allmydata.com**20080320223026] [native client - adding checks for elevating and managing privileges on Vista secorp@allmydata.com**20080318221259] [native client - added StartAllmydata.exe so that we can start/stop processes in Vista secorp@allmydata.com**20080318171847] [native client - fixes for drive size (now 1TB), running service as adminstrator to allow client stopping and starting in Vista, large number of files fix secorp@allmydata.com**20080317212203] [native client - fixed icon for system tray, improved link batching secorp@allmydata.com**20080313182254] [native client - updated executables and dll's to have green icon, preference for log levels, and force a link every 100 files when uploading large amounts of files secorp@allmydata.com**20080311203126] [native client - updated .exe's and .dll's with better caching, using new multi-child upload call, preferences dialog secorp@allmydata.com**20080311051050] [native client - update to handle large file upload secorp@allmydata.com**20080227061743] [native client - adding ability to mount drives secorp@allmydata.com**20080229201448] [native client - added upload metrics, machine name cfg, better queue resumption secorp@allmydata.com**20080227004331] [native client - added AllmydataTray (replaces TahoeTray) secorp@allmydata.com**20080222023946] [native client - added ability to queue uploads and resume upon service restart secorp@allmydata.com**20080222030207] [native client - renaming things to take Tahoe out of the name, improving caching secorp@allmydata.com**20080222020458] [native client - updating service name to match what the launcher uses secorp@allmydata.com**20080220032034] [WinFUSE updates for beta release secorp@allmydata.com**20080220025837] [doc: warn that unicode might not work, in CLI.txt zooko@zooko.com**20081224211618 Ignore-this: 89355b53aab40af1d45a3746bb90ed10 ] [test_runner.py: revert the CreateNode section to using runner() inline, rather than spawning a process, to get more precise coverage warner@lothar.com**20090209082617 Ignore-this: e7f0ae5c9a2c2d8157289ef39fda371 ] [setup: fix test_runner to assert that lines which are output to stderr must end with a punctuation mark (apparently re.search("x$", "x\r\n") does not match. :-() zooko@zooko.com**20090127203505 Ignore-this: d323b364d7cafb4c5847c0cc8346702e ] [trivial: remove unused imports noticed by pyflakes zooko@zooko.com**20090127211148 Ignore-this: aee8bae8aa6f641fe15c5fe947d92f77 ] [trivial: removed unused imports noticed by pyflakes zooko@zooko.com**20090122215213 Ignore-this: a83aa1f27b31a72aed07caa866abfafe ] [setup: fix test_runner to invoke bin/tahoe.exe instead of bin/tahoe if on Windows (this is what happens when a user invokes bin/tahoe on Windows) zooko@zooko.com**20090127203717 Ignore-this: e8795f2d3c70e871839d893bdfec0186 ] [docs/proposed: new Accounting overview, discuss in #666 warner@allmydata.com**20090324015752 Ignore-this: 7379396536e85194b1c1bec21288adc7 ] [docs: fix the asymptotic network performance of mutable file download in performance.txt, rename the howto-make-a-release file zooko@zooko.com**20100228061439 Ignore-this: c983b2fa7864f717ec17fb556f8a95d2 ] [doc_reformat_backupdb.txt freestorm77@gmail.com**20100424120416 Ignore-this: fed696530e9d2215b6f5058acbedc3ab - Added heading format begining and ending by "==" - Added Index - Added Title Note: No change are made in paragraphs content ] [doc_reformat_debian.txt freestorm77@gmail.com**20100424120537 Ignore-this: 45fe4355bb869e55e683405070f47eff - Added heading format begining and ending by "==" - Added Index - Added Title Note: No change are made in paragraphs content ] [doc_reformat_garbage-collection.txt freestorm77@gmail.com**20100424120830 Ignore-this: aad3e4c99670871b66467062483c977d - Added heading format begining and ending by "==" - Added Index - Added Title Note: No change are made in paragraphs content ] [doc_reformat_helper.txt freestorm77@gmail.com**20100424120649 Ignore-this: de2080d6152ae813b20514b9908e37fb - Added heading format begining and ending by "==" - Added Index - Added Title Note: No change are made in paragraphs content ] [doc_refomat_logging.txt freestorm77@gmail.com**20100424114316 Ignore-this: 593f0f9914516bf1924dfa6eee74e35f - Added heading format begining and ending by "==" - Added Index - Added Title Note: No change are made in paragraphs content ] [doc_reformat_stats.txt freestorm77@gmail.com**20100424114615 Ignore-this: af315db5f7e3a17219ff8fb39bcfcd60 - Added heading format begining and ending by "==" - Added Index - Added Title Note: No change are made in paragraphs content **END OF DESCRIPTION*** Place the long patch description above the ***END OF DESCRIPTION*** marker. The first line of this file will be the patch name. This patch contains the following changes: M ./docs/stats.txt -2 +2 ] [docs/stats.txt: add TOC, notes about controlling gatherer's listening port Brian Warner **20091224202133 Ignore-this: 8eef63b0e18db5aa8249c2eafde02c05 Thanks to Jody Harris for the suggestions. ] [Minor documentation tweaks. david-sarah@jacaranda.org**20100603054458 Ignore-this: e30ae407b0039dfa5b341d8f88e7f959 ] [doc_reformat_architecture.txt freestorm77@gmail.com**20100424120133 Ignore-this: 6e2cab4635080369f2b8cadf7b2f58e - Added heading format begining and ending by "==" - Added Index - Added Title Note: No change are made in paragraphs content ] [doc_reformat_configuration.txt freestorm77@gmail.com**20100424104903 Ignore-this: 4fbabc51b8122fec69ce5ad1672e79f2 - Added heading format begining and ending by "==" - Added Index - Added Title Note: No change are made in paragraphs content ] [doc_reformat_FTP-and-SFTP.txt freestorm77@gmail.com**20100424121334 Ignore-this: 3736b3d8f9a542a3521fbb566d44c7cf - Added heading format begining and ending by "==" - Added Index - Added Title Note: No change are made in paragraphs content ] [setup: organize misc/ scripts and tools and remove obsolete ones zooko@zooko.com**20100607051618 Ignore-this: 161db1158c6b7be8365b0b3dee2e0b28 This is for ticket #1068. ] [misc cleanup: remove old .tac files, move old stuff into misc/ warner@lothar.com**20070407033738] [misc/figleaf.excludes: exclude simplejson too, since we don't run their tests warner@allmydata.com**20070712232625] [figleaf.excludes: ignore allmydata.Crypto, since so little of it gets used by allmydata.test warner@lothar.com**20070708060009] [do not include foolscap or zfec coverage in the figleaf reports for the moment: they have their own test suites warner@allmydata.com**20070505000018] [logtool: rename get-logs.py to logtool.py warner@lothar.com**20071119003055] [misc/boodlegrid.tac: tool to monitor a grid through its flogports warner@allmydata.com**20080326230934] [misc/delete-old-helper.sh: simple script to delete old+abandoned helper files warner@lothar.com**20080528041720] [setup: quote the results of misc/find_trial.py so that they can be passed through a shell even if they contain spaces zooko@zooko.com**20080605225945] [setup: make find_trial self-contained so that we don't have a bootstrapping problem -- if allmydata can't be imported we still want to be able to run find_trial zooko@zooko.com**20080123160426] [mac/Makefile: upload the .dmg file with foolscap xfer-client.py instead of scp warner@allmydata.com**20080908231943] [mac build: fixed permission problem on upload .dmg robk-tahoe@allmydata.com**20080123205118] [misc/fixshebangs.py zooko@zooko.com**20081105000130 Ignore-this: 13b03ea2d2ed8982f8346a827b46bd2e ] [util: copy in pyutil.fileutil.ReopenableNamedTemporaryFile zooko@zooko.com**20081104234715 Ignore-this: f1131e9b8f249b5f10be4cba2aeb6118 ] [shebang: replace "/usr/bin/python" with "/usr/bin/env python" zooko@zooko.com**20081105000306 Ignore-this: 8ae33a8a7828fa7423422e252f2cfd74 ] [misc/count_dirs.py: dev tool to estimate filetree space consumption warner@lothar.com**20070617045513] [Makefile: add 'find-trailing-spaces' tool and target warner@allmydata.com**20071106033208] [add a munin plugin to display introducer stats warner@allmydata.com**20080325201552] [Copy amd-nodememory munin plugin over to tahoe and point at correct pidfile zandr@allmydata.com**20080326005004] [add munin/tahoe-rootdir-space warner@allmydata.com**20080328231809] [munin/tahoe_estimate_files.py: tool to estimate the total number of slots (mutable and immutable combined) in the grid, from a small sample warner@allmydata.com**20080424182835] [misc/cpu-watcher*: add some tools to monitor CPU usage of arbitrary processes, like tahoe nodes warner@allmydata.com**20080507193429] [munin: add tahoe_cpu_watcher.py, to track the data from misc/cpu-watcher.tac warner@allmydata.com**20080507201908] [misc/munin/tahoe_spacetime.py: add a munin plugin to estimate how much time remains before you run out of space warner@allmydata.com**20080528183354] [start work on 'check-grid' target, for use in an automated prodnet tester. Just a docstring so far. warner@allmydata.com**20080618195747] [misc/munin: add server_latency plugin warner@lothar.com**20080714192919] [munin: add plugin for storage-server operations per second warner@lothar.com**20080714201811] [munin: add tahoe_overhead plugin, to measure effectiveness of GC and deleting data from inactive accounts warner@lothar.com**20080807203925] [misc/make_umid: little script and elisp fragment to insert umid= arguments warner@allmydata.com**20080826015918] [munin: add tahoe_diskleft plugin, update spacetime/diskwatcher.tac to support it warner@allmydata.com**20080828203236] [diskwatcher.tac: include total-bytes-used warner@lothar.com**20080807201214] [diskwatcher.tac: add async-GET code, but leave it commented out: urlopen() seems to work better for now warner@lothar.com**20080807050327] [misc/xfer-client.py: small foolscap utility to transfer a file to a waiting server warner@allmydata.com**20080908231903] [misc/make-canary-files.py: tool to create 'canary files', explained in the docstring warner@allmydata.com**20080925004716] [munin/tahoe_disktotal: new plugin to show total disk space (used and unused) in the grid warner@allmydata.com**20081118065101] [munin/tahoe_diskused: new plugin to show total disk space used across the grid warner@allmydata.com**20081118072525] [misc/*: remove RuntimeError too warner@lothar.com**20090222233401 Ignore-this: b76f8a184f75bb28eb9d8002f957936a ] [logtool: add 'gather' and 'dump' modes warner@lothar.com**20071119003204] [setup: add "test_mac_diskimage" command which attempts to mount and use a .dmg to make sure it has a good Tahoe distribution in it zooko@zooko.com**20090712230940 Ignore-this: e889aaf49699429afeb211f7403fec74 ] [setup: if you sdist_dsc (to produce the input files for dpkg-buildpackage) then run darcsver first zooko@zooko.com**20090216201558 Ignore-this: b85be51b3d4a9a19a3366e690f1063e2 ] [setup: make sure you use darcsver whenever you are going to run trial zooko@zooko.com**20090130203819 Ignore-this: 2bd632d7540020c0dd893d337163d396 This fixes the bug Brian had where he ran "python ./setup.py trial" and the allmydata-tahoe version number came out as 0.0.0. ] [new approach for debian packaging, sharing pieces across distributions. Still experimental, still only works for sid. warner@lothar.com**20090818190527 Ignore-this: a75eb63db9106b3269badbfcdd7f5ce1 ] [new experimental deb-packaging rules. Only works for sid so far. Brian Warner **20090818014052 Ignore-this: 3a26ad188668098f8f3cc10a7c0c2f27 ] [Add docs/stats.py, explaining Tahoe stats, the gatherer, and the munin plugins. Brian Warner **20091223052400 Ignore-this: 7c9eeb6e5644eceda98b59a67730ccd5 ] [misc/ringsim.py: tool to discuss #302 Brian Warner **20091226060339 Ignore-this: fc171369b8f0d97afeeb8213e29d10ed ] [code coverage: replace figleaf with coverage.py, should work on py2.6 now. Brian Warner **20100203165421 Ignore-this: 46ab590360be6a385cb4fc4e68b6b42c It still lacks the right HTML report (the builtin report is very pretty, but lacks the "lines uncovered" numbers that I want), and the half-finished delta-from-last-run measurements. ] [figleaf.el: announce annotation/deannotation warner@allmydata.com**20070105035240] [figleaf.el: add some emacs22 comments warner@lothar.com**20070107191015] [figleaf.el: add code to auto-enable the annotation mode for all source files Brian Warner **20070118070011] [makefile: pass COVERAGEDIR= properly to the target makefile warner@allmydata.com**20070702222249] [makefile: change upload-figleaf target to create a 'current' link warner@allmydata.com**20070702221020] [setup: remove some things from .darcs-boringfile which are no longer boring since we no longer use them zooko@zooko.com**20080122234023] [remove simplejson.egg-info from the repo (and boringfile it), it should never have been in the original tarball warner@lothar.com**20070710225158] [boringfile: add simplejson generated files warner@lothar.com**20070710223436] [.darcs-boringfile: ignore some build/ directories for foolscap and zfec warner@lothar.com**20070504033151] [.darcs-boringfile: update to match misc/dependencies setup.py changes warner@allmydata.com**20080111022110] [setup: update the list of files that we think are boring when we are using darcs zooko@zooko.com**20080101054904] [setup: new name for setuptools_darcs plugin is also boring zooko@zooko.com**20071110002809] [setup: setuptools_darcs_plugin is boring zooko@zooko.com**20071015042201] [trial_figleaf.py: make our figleaf code compatible with both Twisted-8.x and Twisted-2.5.x warner@allmydata.com**20080403004855] [Makefile: add figleaf-delta-output, to render figleaf coverage differences with a previous run warner@lothar.com**20090212211829 Ignore-this: 9c78861c4abed3d6c2d013ff40d9b4fb ] [Makefile: use run_with_pythonpath, move windows targets into a separate Makefile warner@allmydata.com**20080912044508] [Makefile: check-memory: run mode=receive along with everything else Brian Warner **20070920033917] [setup.py: add 'setup.py run_with_pythonpath', to run other commands with PYTHONPATH set usefully warner@allmydata.com**20080912044418] [Makefile: include the figleaf pickle in the uploaded coverage data, for later differential analysis warner@allmydata.com**20090212000913 Ignore-this: e31002f4a7d55d8a2ca1131a1066c066 ] [fix quicktest: stop using setuptools, add misc/run-with-pythonpath.py, to make it run faster warner@lothar.com**20090414201400] [makefile: added 'fuse-test' target to makefile, to run 'runtests' robk-tahoe@allmydata.com**20081019132518] [Makefile: use 'setup.py test' for test/quicktest targets (instead of Brian Warner **20090130102536 Ignore-this: da07fae8417bc54a610786bc57959c5e 'setup.py trial'). 'setup.py trial' clobbers the tahoe .egg's PKG-INFO "Version:" field (resetting it to 0.0.0), possibly because it isn't invoking the darcsver subcommand that 'setup.py test' does before it runs the 'trial' subcommand. This slows down quicktest by another couple of seconds (see #591) and adds more noise to its output, but without this change, 'make test' and 'make quicktest' fail on test_runner (which spawns bin/tahoe as a subprocess, and with a mangled Version:, the setuptools-based entry point script refuses to recognize our source tree as a usable version of Tahoe). ] [rollback the #591 fix, since it breaks quicktest entirely warner@allmydata.com**20090123232812] [setup: use "trial" executable instead of the setuptools_trial plugin for "make quicktest" zooko@zooko.com**20090123225830 Ignore-this: f70fb8873e0ac94f0c55e57f0f826334 This is to fix #591 ("make quicktest" could be quicker and less noisy). This means that "make quicktest" won't work unless the user has manually installed Twisted already such that the "trial" executable is on their PATH and the Twisted package is on their PYTHONPATH. This bypasses the behavior of setuptools_trial which builds and checks package dependencies before executing the tests. ] [setup: remove a convenience copy of figleaf, to ease inclusion into Ubuntu Karmic Koala zooko@zooko.com**20090924053215 Ignore-this: a0b0c990d6e2ee65c53a24391365ac8d We need to carefully document the licence of figleaf in order to get Tahoe-LAFS into Ubuntu Karmic Koala. However, figleaf isn't really a part of Tahoe-LAFS per se -- this is just a "convenience copy" of a development tool. The quickest way to make Tahoe-LAFS acceptable for Karmic then, is to remove figleaf from the Tahoe-LAFS tarball itself. People who want to run figleaf on Tahoe-LAFS (as everyone should want) can install figleaf themselves. I haven't tested this -- there may be incompatibilities between upstream figleaf and the copy that we had here... ] [fix pyflakes warnings in figleaf warner@allmydata.com**20070105000443] [hush pyflakes warnings warner@allmydata.com**20070201231301] [figleaf_htmlizer: emit stats to stdout, so buildbot can see it warner@lothar.com**20090212211020 Ignore-this: bf5b503717580561c2fdb6c619159c9 ] [figleaf_htmlizer: oops, re-ignore files that aren't under root, like code in auto-built eggs warner@lothar.com**20090213060022 Ignore-this: b4e67d5e025198d01e373728d1a74b11 ] [figleaf_htmlizer: fix order of summary counters warner@lothar.com**20090213155753 Ignore-this: aa6f0cd37e79857482967496252088da ] [figleaf_htmlizer: render changes in coverage relative to a previous test run using --old-coverage warner@lothar.com**20090212210412 Ignore-this: 6c2bfc21a0671f1a07e19d3f9fd17157 ] [figleaf_htmlizer: more rearranging, behavior should still be unchanged warner@allmydata.com**20090212020515 Ignore-this: 8fb7dd723b2d26bdcf1d57685b113bb8 ] [figleaf_htmlizer: break it up into subfunctions, behavior should still be unchanged warner@allmydata.com**20090212015607 Ignore-this: 9644ec95b0f9950722c13ad3c9654e22 ] [figleaf_htmlizer: rewrite in class form, behavior should be the same as before warner@allmydata.com**20090212014050 Ignore-this: d00b838c0af1ea2d71be246bbcf36c53 ] [figleaf_htmlizer: rewrite with twisted.python.usage, remove logging: should behave the same as before warner@allmydata.com**20090212011643 Ignore-this: 86f961e369625e9ab5188e817b083fe4 ] [figleaf_htmlizer: expand tabs, fix to 4-space indents. No functional changes. warner@allmydata.com**20090212010542 Ignore-this: 5c9aa3704eacf1b42a985ade8e43dd1f ] [sort coverage tables by lines uncovered, rather than percentages warner@allmydata.com**20070424184926] [figleaf: add number-of-uncovered-lines to the HTML output warner@allmydata.com**20070419180933] [improve figleaf: fix some line-numbering bugs, add an alphabetically-sorted index HTML page warner@lothar.com**20070104072643] [figleaf output: include a stats.out for parsing by a munin plugin warner@allmydata.com**20070407004101] [figleaf_htmlizer: when all counts are zero, claim 0% instead of 100%, since it probably means that coverage checking has failed completely warner@allmydata.com**20070501180728] [setup: new improved misc/show-tool-versions.py zooko@zooko.com**20100516050122 Ignore-this: ce9b1de1b35b07d733e6cf823b66335a ] [setup: add sys.maxunicode to the show-tool-versions output in order to investigate http://bugs.python.org/setuptools/issue78 zooko@zooko.com**20090709004533 Ignore-this: dce4420d5c626258e1033d924e506de2 ] [setup: reorder and extend the show-tool-versions script, the better to glean information about our new buildslaves zooko@zooko.com**20100504045643 Ignore-this: 836084b56b8d4ee8f1de1f4efb706d36 ] [setup: copy in misc/show-tools-version.py from zfec -- it prints out platform and setuptools versions zooko@zooko.com**20090621055846 Ignore-this: 4e144886ab02414bbaaf0295ce2b337 ] [setup: fix typos in misc/show-tool-versions.py zooko@zooko.com**20100510063615 Ignore-this: 2181b1303a0e288e7a9ebd4c4855628 ] [setup: show code-coverage tool versions in show-tools-versions.py zooko@zooko.com**20100510062955 Ignore-this: 4b4c68eb3780b762c8dbbd22b39df7cf ] [move show-tool-versions out of setup.py and into a separate script in misc/ , since setuptools is trying to build and install a bunch of stuff first warner@lothar.com**20090219073558 Ignore-this: 9e56bc43026379212e6b6671ed6a1fd4 ] [setup.py: add show_tool_versions command, for the benefit of a new buildbot step warner@lothar.com**20090219062436 Ignore-this: 21d761c76a033e481831584bedc60c86 ] [setup.py: wrap to 80 cols, no functional changes warner@lothar.com**20090219055751 Ignore-this: d29e57c6ee555f2ee435667b7e13e60b ] [setup: stop catching EnvironmentError when attempting to copy ./_auto_deps.py to ./src/allmydata/_auto_deps.py zooko@zooko.com**20080924000402 It is no longer the case that we can run okay without _auto_deps.py being in place in ./src/allmydata, so if that cp fails then the build should fail. ] [setup: a new improved way to create tahoe executables zooko@zooko.com**20090129000716 Ignore-this: d1b038ab8efb949125d7592ebcceccba Create the 'tahoe-script.py' file under the 'bin' directory. The 'tahoe-script.py' file is exactly the same as the 'tahoe-script.template' script except that the shebang line is rewritten to use our sys.executable for the interpreter. On Windows, create a tahoe.exe will execute it. On non-Windows, make a symlink to it from 'tahoe'. The tahoe.exe will be copied from the setuptools egg's cli.exe and this will work from a zip-safe and non-zip-safe setuptools egg. ] [Makefile: convert check-auto-deps target into 'setup.py check_auto_deps' warner@allmydata.com**20080912035904] [setup: require new bundled setuptools-0.6c12dev zooko@zooko.com**20090205152923 Ignore-this: 516bbb2195a20493fa72a0ca11a2361c ] [setup: require and automatically use setuptools-0.6c11dev (our own toothpick of setuptools) which is bundled zooko@zooko.com**20090203042323] [setup: require the latest version of the setuptools bootstrap egg zooko@zooko.com**20081025152858 Ignore-this: c0c9923ba3008f410d5cc56f2236edb9 ] [setup: merge in changes to ez_setup.py from the upstream setuptools project zooko@zooko.com**20080326191128] [setup: import bugfix to ez_setup.py zooko@zooko.com**20071004181846] [setup: require the SVN snapshot of setuptools to build zooko@zooko.com**20081025134959 Ignore-this: f68077dd10d85a71a1e06678365e6753 ] [setup: change ez_setup.py to install setuptools-0.6c9 zooko@zooko.com**20080930200502] [setup: bundle setuptools-0.6c9 zooko@zooko.com**20080930200448] [setup: include _pkgutil.py in setuptools bootstrap egg so that it will work on Python 2.4 zooko@zooko.com**20081025152839 Ignore-this: 38d81a037c1a3413d69d580ccb13fd67 ] [setup: bundle an SVN snapshot of setuptools instead of the most recent stable release of setuptools zooko@zooko.com**20081025134837 Ignore-this: 9a0c9a34b186b972650cf9455edb0d28 This SVN snapshot fixes a problem that prevents the setting up of nevow: http://bugs.python.org/setuptools/issue20 ] [setup: bundle setuptools-0.6c11dev (our own toothpick of setuptools) zooko@zooko.com**20090203041950 Hopefully this one fixes the issue with easy_install not searching the sys.path for packages that were requested to be installed, (setuptools #17), thus allowing us to do away with the "--site-dirs=" kludges, which are currently breaking some of our buildbots. ] [setup: remove a couple of horrible work-arounds in setup.py now that we rely on our own toothpick of setuptools which fixes those issues zooko@zooko.com**20090204052405 Ignore-this: a11bc74a5cb879db42c4fe468c375577 also specify that we need our latest revision (11dev) of our toothpick of setuptools also *always* setup_require setuptools_darcs at module import time. Formerly we added setup_require setuptools_darcs only if the PKG-INFO file were not already created. There is some weird, irreproducible bug to do with setuptool_darcs, and I guess that the fact that whether it is required or not depends on that autogenerated file might have something to do with it. Anyway, this is simpler. ] [setup.py: Require simplejson version >= 2.0.5 francois@ctrlaltdel.ch**20081125171727] [setup: always create a support dir and populate it with a site-packages and add same to the PYTHONPATH, just in case someone is going to do "build", "develop", or "test" or something else which triggers a build zooko@zooko.com**20090129045608 Ignore-this: d3740cd285f1d30a111536863a8e8457 I think there must be a much better solution for this -- probably to fix setuptools #54 and ship our own fork of setuptools and rely on it. ] [setup: temporarily comment-out the horrible kludge to work-around setuptools #17, while I figure out how to solve it better zooko@zooko.com**20090129130000 Ignore-this: 9e42491cfa8042c90a6a96081c00a053 ] [setup: if any of "build", "develop", or "test" appear in the sys.argv then that means we'll be doing a develop, so add the workarounds for setuptools #17 in any case zooko@zooko.com**20090129045534 Ignore-this: 38645dfadf3ba7b42370e795b7c90214 I think there must be a much better solution for this -- probably to fix setuptools #17 and ship our own fork of setuptools and rely on it. ] [prevent --site-dirs from being passed to the 'install' command cgalvan@mail.utexas.edu**20090116195732] [setup: require setuptools_trial >= 0.5, and delegate to it the job of deciding which Twisted reactor to use for the current platform zooko@zooko.com**20090130043133 Ignore-this: 9e354184d6c989ddf16c7e16a3295ef2 ] [setup: hack the sys.argv to set poll reactor if "python ./setup.py test" in addition to if "python ./setup.py trial"; remove another hack which has been moved setup.cfg; remove setup_requires Twisted since now we actually setup_require setuptools_trial and it install_requires Twisted. zooko@zooko.com**20090127044046 Ignore-this: 38c5afd730024cca63bc84f8ab7100f4 ] [setup: fix previous patch to set reactor to poll reactor on linux or cygwin zooko@zooko.com**20090114164022 Ignore-this: 9e37242ca3cfd47c6f69ee2438ec743b ] [setup: use poll reactor for trial if on linux2 or cygwin zooko@zooko.com**20090114151546 Ignore-this: f6c4b45745527811c9f72448d74649f5 ] [setup: undo (for the second time) the use of the --multi-version feature zooko@zooko.com**20090119205352 Ignore-this: 80bdbb488c3e4fb042bcd968a9bc120a When this feature is turned on, then setuptools doesn't create easy-install.pth, setuptools.pth, or site.py in the target site-packages dir. I don't know why not and we should probably open a ticket on the setuptools tracker and/or hack setuptools to create those files anyway. But for now (for the Tahoe-1.3.0 release), we're going to leave --multi-version mode off and require users to manually uninstall any packages which are too old and thus conflict with our newer dependencies. ] [adding multi-version support cgalvan@mail.utexas.edu**20090116230326] [set bin/tahoe executable permissions and leave build_tahoe in sys.argv cgalvan@mail.utexas.edu**20090109210640] [fix bin/tahoe executable for Windows cgalvan@mail.utexas.edu**20090109184222] [setup: execute ../support/bin/tahoe from ./bin/tahoe zooko@zooko.com**20080410214037] [bin/allmydata-tahoe: fix handling of PYTHONPATH, we were missing an os.pathsep Brian Warner **20070917104322 which would cause a pre-existing PYTHONPATH to get mangled ] [bin/tahoe: rename 'allmydata-tahoe' in some comments warner@lothar.com**20071011103929] [run build_tahoe command with trial commmand cgalvan@mail.utexas.edu**20090117000047] [setup: bundle setuptools-0.6c12dev (our own toothpick of setuptools) this version completes my patch to fix http://bugs.python.org/setuptools/issue54 , which is necessary for tahoe to build with --prefix=support without doing a lot of PYTHONPATH gymnastics around the call to setup.py zooko@zooko.com**20090205152818 Ignore-this: da7b1587ee91180c4a1a56f217311de3 ] [run_trial.darcspath freestorm77@gmail.com**20100510232829 Ignore-this: 5ebb4df74e9ea8a4bdb22b65373d1ff2 ] [Remove firewall section from running.html and say to read configuration.txt instead. david-sarah@jacaranda.org**20100617004513 Ignore-this: d2e46fffa4855b01093e8240b5fd1eff ] [running.html: fix overeager replacement of 'tahoe' with 'Tahoe-LAFS', and some simplifications. david-sarah@jacaranda.org**20100617000952 Ignore-this: 472b4b531c866574ed79f076b58495b5 ] [Add a note about running Tahoe-LAFS on a small grid to running.html zooko@zooko.com**20100616140227 Ignore-this: 14dfbff0d47144f7c2375108c6055dc2 also Change "tahoe" and "Tahoe" to "Tahoe-LAFS" in running.html author: Kevan Carstensen ] [docs running.html - "tahoe run ." does not work with the current installation, replaced with "tahoe start ." secorp@allmydata.com**20100206165320 Ignore-this: fdb2dcb0e417d303cd43b1951a4f8c03 ] [Change running.html to describe 'tahoe run' david-sarah@jacaranda.org**20100112044409 Ignore-this: 23ad0114643ce31b56e19bb14e011e4f ] [docs: wording fix, thanks to Jeremy Visser, fix #987 francois@ctrlaltdel.ch**20100609081103 Ignore-this: 6d2e627e0f1cd58c0e1394e193287a4b ] [Debian documentation update jacob@appelbaum.net**20100305003004] [debian-docs-patch-final jacob@appelbaum.net**20100304085955] [desert-island-build-with-proper-versions jacob@appelbaum.net**20100304013858] [docs/frontends/FTP-and-SFTP.txt: docs/performance.txt, architecture.txt: updates taking into account new downloader (revised). refs #798 david-sarah@jacaranda.org**20100910195422 Ignore-this: 5774da17f734231fefe6454a80e81455 ] [docs: mention default values of K, H, and M zooko@zooko.com**20100924020245 Ignore-this: ab825b7415142b4394599f909ea31934 ] [configuration.txt and servers-of-happiness.txt: 1 <= happy <= N, not k <= happy <= N. Also minor wording changes. david-sarah@jacaranda.org**20100618050710 Ignore-this: edac0716e753e1f1c4c755c85bec9a19 ] [Note that servers of happiness only applies to immutable files for the moment Kevan Carstensen **20100524042836 Ignore-this: cf83cac7a2b3ed347ae278c1a7d9a176 ] [Update 'docs/configuration.txt' to reflect the servers_of_happiness behavior. Kevan Carstensen **20091205033813 Ignore-this: 5e1cb171f8239bfb5b565d73c75ac2b8 ] [docs: update docs/architecture.txt to more fully and correctly explain the upload procedure zooko@zooko.com**20100514043458 Ignore-this: 538b6ea256a49fed837500342092efa3 ] [Alter the wording in docs/architecture.txt to more accurately describe the servers_of_happiness behavior. Kevan Carstensen **20100428002455 Ignore-this: 6eff7fa756858a1c6f73728d989544cc ] [Update 'docs/architecture.txt' to reflect readonly share discovery kevan@isnotajoke.com**20100514003852 Ignore-this: 7ead71b34df3b1ecfdcfd3cb2882e4f9 ] [architecture.txt: remove trailing whitespace, wrap lines: no content changes Brian Warner **20100202055304 Ignore-this: 1662f37d1162858ac2619db27bcc411f ] [Change stray "shares_of_happiness" to "servers_of_happiness" Kevan Carstensen **20091116212459 Ignore-this: 1c971ba8c3c4d2e7ba9f020577b28b73 ] [Add a specification for servers of happiness. Kevan Carstensen **20100524003508 Ignore-this: 982e2be8a411be5beaf3582bdfde6151 ] [docs/specifications: add an outline of the spec documents we'd like to have some day warner@lothar.com**20090208234748 Ignore-this: b591ad0361810e5aae37cba07fdfbd43 ] [docs: timestamp the 1.8.0 release zooko@zooko.com**20100924021552 Ignore-this: dbacb97c0f9994532f38a5612cecef65 ] [NEWS, docs/known_issues.txt for 1.8.0 release david-sarah@jacaranda.org**20100919044412 Ignore-this: d8901578b0c0c20e42daaae23879b091 ] [relnotes.txt and docs/quickstart.html for 1.8.0 release david-sarah@jacaranda.org**20100919050335 Ignore-this: 9ef6499236d07309fb4df983f9a0a5cd ] [relnotes.txt: update revision of NEWS. david-sarah@jacaranda.org**20100810063243 Ignore-this: cf9eb342802d19f3a8004acd123fd46e ] [docs: update relnotes.txt, NEWS, and quickstart.html for the 1.8.0c4 release zooko@zooko.com**20100912061423 Ignore-this: bb17f4c54ba390fdcc74eb5d5017373 ] [doc_reformat_performance.txt freestorm77@gmail.com**20100424114444 Ignore-this: 55295ff5cd8a5b67034eb661a5b0699d - Added heading format begining and ending by "==" - Added Index - Added Title Note: No change are made in paragraphs content ] [docs/performance.txt: split out CPU from network, expand on mutable costs Brian Warner **20100224043813 Ignore-this: 4779e78ca0eed1dcbd1652e6287219f1 ] [quickstart.html: update tarball link. david-sarah@jacaranda.org**20100810073832 Ignore-this: 4fcf9a7ec9d0de297c8ed4f29af50d71 ] [docs: update NEWS a bit about New-Downloader zooko@zooko.com**20100819021446 Ignore-this: 31a6e2fb0a6e3d19f73481e99070da7a ] [docs: trivial naming change zooko@zooko.com**20090121025042 Ignore-this: a9c2fe6119c43683c6f88662e60de306 ] [docs: NEWS: edit English usage, remove ticket numbers for regressions vs. 1.7.1 that were fixed again before 1.8.0c2 zooko@zooko.com**20100811071758 Ignore-this: 993f5a1e6a9535f5b7a0bd77b93b66d0 ] [docs: NEWS: more detail about new-downloader zooko@zooko.com**20100811071303 Ignore-this: 9f07da4dce9d794ce165aae287f29a1e ] [docs: update relnotes.txt for v1.8.0c3 zooko@zooko.com**20100902212111 Ignore-this: 7211f79f4c388c9e8ff0d05f22eb3ba2 ] [NEWS, relnotes and known-issues for 1.8.0c2. david-sarah@jacaranda.org**20100810062851 Ignore-this: bf319506558f6ba053fd896823c96a20 ] [NEWS, quickstart.html and known_issues.txt for 1.8.0c1 release. david-sarah@jacaranda.org**20100806235111 Ignore-this: 777cea943685cf2d48b6147a7648fca0 ] [relnotes.txt: 1.8.0c1 release david-sarah@jacaranda.org**20100807003646 Ignore-this: 1994ffcaf55089eb05e96c23c037dfee ] [update NEWS and other docs in preparation for 1.8.0rc1 Brian Warner **20100806080228 Ignore-this: 6ebdf11806f6dfbfde0b61115421a459 in particular, merge the various 1.8.0b1/b2 sections, and remove the datestamp. NEWS gets updated just before a release, doesn't need to precisely describe pre-release candidates, and the datestamp gets updated just before the final release is tagged Also, I removed the BOM from some files. My toolchain made it hard to retain, and BOMs in UTF-8 don't make a whole lot of sense anyway. Sorry if that messes anything up. ] [NEWS: remove XXX comment and separate description of #890. david-sarah@jacaranda.org**20100803050827 Ignore-this: 6d308f34dc9d929d3d0811f7a1f5c786 ] [docs: more updates to NEWS for 1.8.0β zooko@zooko.com**20100803044618 Ignore-this: 8193a1be38effe2bdcc632fdb570e9fc ] [test_util.py: use SHA-256 from pycryptopp instead of MD5 from hashlib (for uses in which any hash will do), since hashlib was only added to the stdlib in Python 2.5. david-sarah@jacaranda.org**20100806050051 Ignore-this: 552049b5d190a5ca775a8240030dbe3f ] [Add a byte-spans utility class, like perl's Set::IntSpan for .newsrc files. Brian Warner **20100804072600 Ignore-this: bbad42104aeb2f26b8dd0779de546128 Also a data-spans class, which records a byte (instead of a bit) for each index. ] [Clean up log.err calls, for one of the issues in #889. Brian Warner **20100112013343 Ignore-this: f58455ce15f1fda647c5fb25d234d2db allmydata.util.log.err() either takes a Failure as the first positional argument, or takes no positional arguments and must be invoked in an exception handler. Fixed its signature to match both foolscap.logging.log.err and twisted.python.log.err . Included a brief unit test. ] [util.log: send log.err to Twisted too, so that Trial tests are flunked warner@lothar.com**20080920173427] [NEWS and docs/quickstart.html for 1.8.0beta2. david-sarah@jacaranda.org**20100806035112 Ignore-this: 3a593cfdc2ae265da8f64c6c8aebae4 ] [docs: incomplete beginnings of a NEWS update for v1.8β zooko@zooko.com**20100802072840 Ignore-this: cb00fcd4f1e0eaed8c8341014a2ba4d4 ] [NEWS: describe #1055 zooko@zooko.com**20100801034338 Ignore-this: 3a16cfa387c2b245c610ea1d1ad8d7f1 ] [tests, NEWS, CREDITS re: #1117 zooko@zooko.com**20100718203225 Ignore-this: 1f08be2c692fb72cc0dd023259f11354 Give Brian and Kevan promotions, move release date in NEWS to the 18th, commit Brian's test for #1117. fixes #1117 ] [Re-work 'test_upload.py' to be more readable; add more tests for #778 Kevan Carstensen **20091116192334 Ignore-this: 7e8565f92fe51dece5ae28daf442d659 ] [Alter tests to use the new form of set_shareholders Kevan Carstensen **20091104033602 Ignore-this: 3deac11fc831618d11441317463ef830 ] [Minor tweak to an existing test -- make the first server read-write, instead of read-only Kevan Carstensen **20091104034232 Ignore-this: a951a46c93f7f58dd44d93d8623b2aee ] [Add more tests for comment:53 in ticket #778 Kevan Carstensen **20091104112849 Ignore-this: 3bb2edd299a944cc9586e14d5d83ec8c ] [Test Tahoe2PeerSelector to make sure that it recognizeses existing shares on readonly servers Kevan Carstensen **20091109003735 Ignore-this: 12f9b4cff5752fca7ed32a6ebcff6446 ] [Add a test for upload.shares_by_server Kevan Carstensen **20091104111324 Ignore-this: f9802e82d6982a93e00f92e0b276f018 ] [Refactor some behavior into a mixin, and add tests for the behavior described in #778 "Kevan Carstensen" **20091030091908 Ignore-this: a6f9797057ca135579b249af3b2b66ac ] [docs: update NEWS zooko@zooko.com**20100718053225 Ignore-this: 63d5c782ef84812e6d010f0590866831 ] [docs: tidy up NEWS a little zooko@zooko.com**20100718032434 Ignore-this: 54f2820fd1a37c8967609f6bfc4e5e18 ] [NEWS: add snippet about #1083 zooko@zooko.com**20100718020653 Ignore-this: d353a9d93cbc5a5e6ba4671f78d1e22b ] [NEWS: Forward-compatibility improvements for non-ASCII caps (#1051). david-sarah@jacaranda.org**20100718143622 Ignore-this: 1edfebc4bd38a3b5c35e75c99588153f ] [iputil.py: Add support for FreeBSD 7,8 and 9 francois@ctrlaltdel.ch**20100718022832 Ignore-this: 1829b4cf4b91107f4cf87841e6167e99 committed by: zooko@zooko.com date: 2010-07-17 and I also patched: NEWS and CREDITS ] [support freebsd 6 ben@links.org**20080428074140] [NEWS: reorder NEWS snippets to be in descending order of interestingness zooko@zooko.com**20100718015929 Ignore-this: 146c42e88a9555a868a04a69dd0e5326 ] [CLI: add 'tahoe unlink' as an alias to 'tahoe rm', for forward-compatibility. david-sarah@jacaranda.org**20100717220411 Ignore-this: 3ecdde7f2d0498514cef32e118e0b855 ] [small change to CREDITS david-sarah@jacaranda.org**20100603062421 Ignore-this: 2909cdbedc19da5573dec810fc23243 ] [docs: CREDITS and NEWS zooko@zooko.com**20100714060150 Ignore-this: dc83e612f77d69e50ee975f07f6b16fe ] [CREDITS: jsgf zooko@zooko.com**20100608143052 Ignore-this: 10abe06d40b88e22a9107d30f1b84810 ] [docs: CREDITS for Jeremy Visser zooko@zooko.com**20100524081829 Ignore-this: d7c1465fd8d4e25b8d46d38a1793465b ] [CREDITS to Jacob Appelbaum zooko@zooko.com**20100304015616 Ignore-this: 70db493abbc23968fcc8db93f386ea54 ] [NEWS: add UTF-8 coding declaration. david-sarah@jacaranda.org**20100609234851 Ignore-this: 3e6ef125b278e0a982c88d23180a78ae ] [docs: more CREDITS for Kevan, plus utf-8 BOM zooko@zooko.com**20100619045809 Ignore-this: ee9c3b7cf7e385c8ca396091cebc9ca6 ] [NEWS: add NEWS snippets about two recent patches zooko@zooko.com**20100708162058 Ignore-this: 6c9da6a0ad7351a960bdd60f81532899 ] [docs: update NEWS for release 1.7.0 zooko@zooko.com**20100619045750 Ignore-this: 112c352fd52297ebff8138896fc6353d ] [CREDITS: more creds for Kevan, plus utf-8 BOM zooko@zooko.com**20100619045503 Ignore-this: 72d02bdd7a0f324f1cee8cd399c7c6de ] [fix typo in CREDITS file "Zooko O'Whielacronx "**20070506212642] [relnotes.txt and docs/known_issues.txt for 1.8.0beta2. david-sarah@jacaranda.org**20100806040823 Ignore-this: 862ad55d93ee37259ded9e2c9da78eb9 ] [docs/known_issues.txt: update release version and date. david-sarah@jacaranda.org**20100718235940 Ignore-this: dbbb42dbfa6c0d205a0b8e6e58eee9c7 ] [docs: update known_issues.txt with more detail about web browser "safe-browsing" features and slightly tweaked formatting zooko@zooko.com**20100619051734 Ignore-this: afc10be0da2517ddd0b58e42ef9aa46d ] [doc_reformat_known_issues.txt freestorm77@gmail.com**20100424114118 Ignore-this: 9577c3965d77b7ac18698988cfa06049 - Added heading format begining and ending by "==" - Added Index - Added Title Note: No change are made in paragraphs content ] [Document leakage of cap URLs via phishing filters in known_issues.txt david-sarah@jacaranda.org**20100202015238 Ignore-this: 78e668dbca77c0e3a73e10c0b74cf024 ] [docs: relnotes.txt for 1.8.0β zooko@zooko.com**20100803154913 Ignore-this: d9101f72572b18da3cfac3c0e272c907 ] [relnotes.txt: updated for v1.7.1 release! zooko@zooko.com**20100719083059 Ignore-this: 9f10eb19b65a39d652b546c57481da45 ] [relnotes.txt, docs/quickstart.html: prepare for 1.7.1 release. Don't claim to work on Cygwin (this might work but is untested). david-sarah@jacaranda.org**20100718235437 Ignore-this: dfc7334ee4bb76c04ee19304a7f1024b ] [docs: quickstart.html: link to 1.7.0 zip file and add UTF-8 BOM zooko@zooko.com**20100619050124 Ignore-this: 5104fc90af542b97662b4016da975f34 ] [docs: a few tweaks to NEWS and CREDITS and make quickstart.html point to 1.7.0β! zooko@zooko.com**20100609142927 Ignore-this: f8097d3062f41f06c4420a7c84a56481 ] [quickstart.html: We haven't released 1.7beta yet. david-sarah@jacaranda.org**20100606220301 Ignore-this: 4e18898cfdb08cc3ddd1ff94d43fdda7 ] [quickstart.html: link to snapshots page, sorted with most recent first. david-sarah@jacaranda.org**20100606221127 Ignore-this: 93ea7e6ee47acc66f6daac9cabffed2d ] [Remove the 'tahoe debug consolidate' subcommand. david-sarah@jacaranda.org**20100607183757 Ignore-this: 4b14daa3ae557cea07d6e119d25dafe9 ] [consolidate: remove pointless 'else' after for loop warner@allmydata.com**20090313082751 Ignore-this: 517f41316f7ae40b92b9eef8269a4b69 ] [Change relative imports to absolute david-sarah@jacaranda.org**20100226071433 Ignore-this: 32e6ce1a86e2ffaaba1a37d9a1a5de0e ] [confwiz: update the config wiz to open the welcome page robk-tahoe@allmydata.com**20080215021258 regardless of platform, the confwiz now opens the welcoe page upon writing a config. it also provides a 'plat' argument (from python's sys.platform) to help disambiguate our instructions by platform. ] [confwiz: reworked confwiz look and feel robk-tahoe@allmydata.com**20080318231536 this changes the confwiz to have a look and feel much more consistent with that of the innosetup installer it is launched within the context of. this applies, naturally, primarily to windows. ] [confwiz: revise layout robk-tahoe@allmydata.com**20080128203603 fix the make-confwiz-match-installer-size changes, to eliminate some weird layout/rendering bugs. also tweaked the layout slightly to add space between the warning label and the newsletter subscribe checkbox. ] [tweak config wizard window size robk-tahoe@allmydata.com**20080128200713 adjust the confiwiz frames to display at a size comparable to the innosetup installer window ] [installer and config - name changes secorp@allmydata.com**20080227013439] [native client - renaming a few more instances to be consistent with Allmydata naming scheme for the release, maybe should parameterize this secorp@allmydata.com**20080222024657] [native client - updated system tray name, missed it at first secorp@allmydata.com**20080222023811] [config wizard - changing the name secorp@allmydata.com**20080227015554] [gui/macapp: improve 'about' box robk-tahoe@allmydata.com**20080925135415 adds exactly 1 metric dollop of professionalism to the previously rather amateurish looking about box. ] [macapp: changed to remove 'Tahoe' from .app name robk-tahoe@allmydata.com**20080611003145 Change the build product from 'Allmydata Tahoe' to 'Allmydata' more inkeeping with the branding of the Allmydata product ] [macapp: new mac icon robk-tahoe@allmydata.com**20080308004828 this provides a new icon for the .app bundle also removes the setting of the dock icon from within wx (which previously used a different icon) ] [better mac .app icon robk-tahoe@allmydata.com**20080125022347] [util: copy in nummedobj from pyutil zooko@zooko.com**20081104195550] [util: copy in dictutil from pyutil zooko@zooko.com**20081104195327] [trivial: whitespace cleanup zooko@zooko.com**20090106172058 Ignore-this: 50ee40d42cc8d8f39d2f8ed15f6790d4 ] [checker: don't let failures in add-lease affect checker results. Closes #875. Brian Warner **20091229230108 Ignore-this: ef1a367b93e4d01298c2b1e6ca59c492 Mutable servermap updates and the immutable checker, when run with add_lease=True, send both the do-you-have-block and add-lease commands in parallel, to avoid an extra round trip time. Many older servers have problems with add-lease and raise various exceptions, which don't generally matter. The client-side code was catching+ignoring some of them, but unrecognized exceptions were passed through to the DYHB code, concealing the DYHB results from the checker, making it think the server had no shares. The fix is to separate the code paths. Both commands are sent at the same time, but the errback path from add-lease is handled separately. Known exceptions are ignored, the others (both unknown-remote and all-local) are logged (log.WEIRD, which will trigger an Incident), but neither will affect the DYHB results. The add-lease message is sent first, and we know that the server handles them synchronously. So when the checker is done, we can be sure that all the add-lease messages have been retired. This makes life easier for unit tests. ] [cli: suppress DeprecationWarnings emitted from importing nevow and twisted. Fixes #859 david-sarah@jacaranda.org**20100201004429 Ignore-this: 22d7216921cd5f04381c0194ed501bbe ] [setup: don't catch ImportError when importing _auto_deps in allmydata/__init__.py zooko@zooko.com**20080430202204 Nowadays pkg_resources is a runtime requirement, and if there is something screwed up in the installation, we want an explicit ImportError exception as early as possible. ] [versioning: include an "appname" in the application version string in the versioning protocol, and make that appname be controlled by setup.py zooko@zooko.com**20090211231816 Ignore-this: 6a0e62492bd271cdaaf696c4a35bc919 It is currently hardcoded in setup.py to be 'allmydata-tahoe'. Ticket #556 is to make it configurable by a runtime command-line argument to setup.py: "--appname=foo", but I suddenly wondered if we really wanted that and at the same time realized that we don't need that for tahoe-1.3.0 release, so this patch just hardcodes it in setup.py. setup.py inspects a file named 'src/allmydata/_appname.py' and assert that it contains the string "__appname__ = 'allmydata-tahoe'", and creates it if it isn't already present. src/allmydata/__init__.py import _appname and reads __appname__ from it. The rest of the Python code imports allmydata and inspects "allmydata.__appname__", although actually every use it uses "allmydata.__full_version__" instead, where "allmydata.__full_version__" is created in src/allmydata/__init__.py to be: __full_version__ = __appname + '-' + str(__version__). All the code that emits an "application version string" when describing what version of a protocol it supports (introducer server, storage server, upload helper), or when describing itself in general (introducer client), usese allmydata.__full_version__. This fixes ticket #556 at least well enough for tahoe-1.3.0 release. ] [setup: require darcsver >= 1.2.0 and rely exclusively on darcsver to set the version string zooko@zooko.com**20090129185640 Ignore-this: b7ed63526015c0769812f3c6f2342b8c ] [setup: require darcsver always, and not just when we see the string "darcsver" in sys.argv, because the new aliases hack means that the string might not appear in sys.argv zooko@zooko.com**20090120184229 Ignore-this: beecca08d9f59704be7ef1aeca6fd779 ] [setup: setup_requires darcsver >= 1.1.5 zooko@zooko.com**20080621210109] [setup: a tiny tweak to setup to avoid requiring darcsver package if the user isn't invoking "./setup.py darcsver" zooko@zooko.com**20080418050752 The real reason for this patch is to test our patch management infrastructure. ] [setup: setup_require darcsver >= 1.1.2 zooko@zooko.com**20080311025707] [setup: bundle darcsver-1.1.2.tar zooko@zooko.com**20080311025647] [setup: invoke darcsver whenever doing an sdist zooko@zooko.com**20090129165125 Ignore-this: 88b9bf4fae0303250ada810d351b566 ] [setup: always run "build" before running "test" zooko@zooko.com**20090126233240 Ignore-this: 8cf76347ba24f02023f1690a470569df ] [setup: always run build_tahoe before running tests zooko@zooko.com**20090126233024 Ignore-this: da31145fa86a61a307dc5dcc4debc0bb ] [test_runner.py: remove test_client_no_noise: the issue in question is warner@lothar.com**20090601225007 ticketed in http://divmod.org/trac/ticket/2830 and doesn't need a Tahoe-side change, plus this test fails on win32 for unrelated reasons (and test_client is the place to think about the win32 issue). ] [setup: go ahead and check for noise in test_client_no_noise zooko@zooko.com**20090126234616 Ignore-this: dff1a3511fdfad1a61fe73e4277c8981 ] [test_runner: skip all spawnProcess-using tests on cygwin, since spawnProcess just hangs forever warner@lothar.com**20090209083400 Ignore-this: e8c2c85650b61cf084cb8a8852118b86 ] [setup: find a "bin/tahoe" executable to test based on allmydata.__file__ instead of based on the CWD zooko@zooko.com**20090124003437 Ignore-this: 25282068ce695c12a2b1f23c6fd2b205 This means that the tests still work if you are executing them from a CWD other than the src dir -- *if* the "bin/tahoe" is found at os.path.dirname(os.path.dirname(os.path.dirname(allmydata.__file__))). If no file is found at that location, then just skip the tests of executing the "tahoe" executable, because we don't want to accidentally run those tests against an executable from a different version of tahoe. ] [setup: add test that the tests are testing the right source code zooko@zooko.com**20090122215240 Ignore-this: f56c1bc525924154042fefa5d30b04f6 This is a test of #145, and I think that now the code passes this test. ] [setup: add a test for a warning message from importing nevow, marked as TODO, comment-out the assertion of no-noise inside other test_runner tests zooko@zooko.com**20090126233046 Ignore-this: 6b1554ed9268988fd65b8e8aac75ed4e ] [setup: change test_runner to invoke "bin/tahoe" in a subprocess instead of executing runner.runner() zooko@zooko.com**20090122213818 Ignore-this: f7ef67adf1b9508617c9a7d305191627 This is necessary because loading allmydata code now depends on PYTHONPATH manipulation which is done in the "bin/tahoe" script. Unfortunately it makes test_runner slower since it launches and waits for many subprocesses. ] [startstop_node.py: improve test coverage a little bit warner@lothar.com**20070919085027] [startstop_node.py: refactor find_twistd() out so it is only run when you need to start a node warner@lothar.com**20070711021355] [test_runner.RunNode: pass an explicit webport, to avoid using 8123 (which might be in used by a running node). Closes #175. warner@lothar.com**20071013230639] [test_runner.py: test launching an introducer too warner@lothar.com**20080218062856] [#542 'tahoe create-key-generator': fix the .tac file this creates to be compatible with modern code, add a test warner@allmydata.com**20081201234721] [key_generator: make default key size be a constructor argument instead of a class variable, pass default key size of 522 (the smallest that we can do) in unit tests to make them faster zooko@zooko.com**20080422192818] [key_generator: fix timing, make tests more robust robk-tahoe@allmydata.com**20080404014346 previously there was an edge case in the timing of expected behaviour of the key_generator (w.r.t. the refresh delay and twisted/foolscap delivery). if it took >6s for a key to be generated, then it was possible for the pool refresh delay to transpire _during_ the synchronous creation of a key in remote_get_rsa_key_pair. this could lead to the timer elapsing during key creation and hence the pool being refilled before control returned to the client. this change ensures that the time window from a get key request until the key gen reactor blocks to refill the pool is the time since a request was answered, not since a request was asked. this causes the behaviour to match expectations, as embodied in test_keygen, even if the delay window is dropped to 0.1s ] [key_generator: fix a typo in the .tac generating create-key-generator robk-tahoe@allmydata.com**20080408180606 verbose is an object attribute, no longer settable via init args ] [test: extend timeout on the hotline file that prevents the client from stopping itself zooko@zooko.com**20081222030629 Ignore-this: 391f48caef9d6ad558e540ded56a8075 The 20-second timeout was apparently tripped on my Powerbook G4 "draco". ] [client.py: increase hotline timeout, the check_memory test is failing warner@allmydata.com**20070926022233] [Unicode fixes. david-sarah@jacaranda.org**20100607010215 Ignore-this: d58727b5cd2ce00e6b6dae3166030138 ] [test_cli: pass rc out of do_cli() too warner@lothar.com**20081203020828] [cli: if response code from wapi server is not 200 then stop instead of proceeding zooko@zooko.com**20081220134918 Ignore-this: 907481c941fc5696630b9c118137fb52 Also, include the data that failed to json parse in an exception raised by the json parser. ] [cli: tests: skip symlink test if there is no os.symlink zooko@zooko.com**20090115001010 Ignore-this: 4987fea4fe070c2dd5ff75401fbf89e1 ] [Added tests for the fixed alias related command's synopsis Alberto Berti **20090222163732 Ignore-this: 4432b4e88e990ba53a5b3fe0f12db2ac ] [Use failUnlessEqual instead of failUnless(a == b) Alberto Berti **20090222224214 Ignore-this: 8f9144632e3ac9acb4726fb48a083bf4 ] [scripts: stop using RuntimeError, for #639 warner@lothar.com**20090222233106 Ignore-this: 686a424442670fffbd4d1816c284a601 ] [consolidator: add progress to scan-old-directory passes warner@allmydata.com**20090313054728 Ignore-this: adc67a34f4f19fd58c5bc76301b3df36 ] [consolidate: tolerate unicode dirnames warner@allmydata.com**20090313065402 Ignore-this: 7e65703ed3d12d4bd5ec14b693e5f61f ] [consolidate: add eta, flush stdout warner@allmydata.com**20090313082451 Ignore-this: 845f63adccc32557c6864ae6120ba836 ] [consolidator: fix cycle detection to not trigger on merely shared directories, add snapshot counter to progress warner@allmydata.com**20090313042229 Ignore-this: eba2cf9f1b1364b8e4c5ae4fa030a99f ] [consolidator: re-use more directories, add total directories seen-vs-used counts warner@allmydata.com**20090313034801 Ignore-this: 6e743d2940a9fe129cee31008c894d70 ] [consolidate: create multiple numbered backups of the original Archives directory, not just the first time warner@allmydata.com**20090312230427 Ignore-this: e4985f76969b584d099b050781aa561c ] [consolidator: add more verbose traversal of directories warner@allmydata.com**20090312232900 Ignore-this: 8ff0e17c6174566832a566a111032db4 ] [tahoe_get: don't create the output file on error. Closes #121. Brian Warner **20091227220404 Ignore-this: 58d5e793a77ec6e87d9394ade074b926 ] [tahoe backup: skip all symlinks, with warning. Fixes #850, addresses #641. Brian Warner **20100127223517 Ignore-this: ab5cf05158d32a575ca8efc0f650033f ] [Improve behaviour of 'tahoe ls' for unknown objects, addressing kevan's comments david-sarah@jacaranda.org**20100220061313 Ignore-this: 6205025c477f1c999473a4ae67e1c83 ] [CLI: Support for https url in option --node-url Francois Deppierraz **20100430185609 Ignore-this: 1717176b4d27c877e6bc67a944d9bf34 This patch modifies the regular expression used for verifying of '--node-url' parameter. Support for accessing a Tahoe gateway over HTTPS was already present, thanks to Python's urllib. ] [test_stringutils.py: Fix a trivial Python 2.4 syntax incompatibility Francois Deppierraz **20100521093345 Ignore-this: 9297e3d14a0dd37d0c1a4c6954fd59d3 ] [test_stringutils.py: Mock the open() call in test_open_unicode Francois Deppierraz **20100521135817 Ignore-this: d8be4e56a6eefe7d60f97f01ea20ac67 This test ensure that open(a_unicode_string) is used on Unicode platforms (Windows or MacOS X) and that open(a_correctly_encoded_bytestring) on other platforms such as Unix. ] [test_stringutils.py: Add a test class for OpenBSD 4.1 with LANG=C Francois Deppierraz **20100521140053 Ignore-this: 63f568aec259cef0e807752fc8150b73 ] [test_stringutils.py: Skip test_listdir_unicode on mocked platform which cannot store non-ASCII filenames Francois Deppierraz **20100521160559 Ignore-this: b93fde736a8904712b506e799250a600 ] [test_stringutils.py: Skip test test_listdir_unicode_good if filesystem supports only ASCII filenames Francois Deppierraz **20100521160839 Ignore-this: f2ccdbd04c8d9f42f1efb0eb80018257 ] [fix flakes zooko@zooko.com**20100604075845 Ignore-this: 3e6a84b78771b0ad519e771a13605f0 ] [setup: fix "tahoe start" to work on Windows even when a Tahoe base dir hasn't been configured in the Windows registry zooko@zooko.com**20090121184720 Ignore-this: ba147a8f75e8aa9cdc3ee0a56dbf7413 ] [change default node-directory on windows to do registry lookup, not ~/.tahoe robk-tahoe@allmydata.com**20080111013218] [runner: make most commands use ~/.tahoe by default (create-client, start/stop/restart, all CLI tools, but *not* create-introducer warner@lothar.com**20071011085423] [SFTP: changes for #1063 ('no-write' field) including comment:1 (clearing owner write permission diminishes to a read cap). Includes documentation changes, but not tests for the new behaviour. david-sarah@jacaranda.org**20100601051139 Ignore-this: eff7c08bd47fd52bfe2b844dabf02558 ] [docs/FTP: the Twisted patch (t3462) has landed, will be in the next release Brian Warner **20100223210402 Ignore-this: ddc5c8da8c95d8c19380d8c7ecbaf18 ] [ftpd: update docs, point to Twisted ticket for the proposed patch Brian Warner **20090731183226 Ignore-this: f1e93258a0700a529d9fef6ff93847a4 ] [SFTP: fix silly bug in _sync_heisenfiles ('f is not ignore' vs 'not (f is ignore)'). david-sarah@jacaranda.org**20100530053807 Ignore-this: 71c4bc62613bf8fef835886d8eb61c27 ] [SFTP: another try at fixing the _sync_heisenfiles bug. david-sarah@jacaranda.org**20100530055254 Ignore-this: c15f76f32a60083a6b7de6ca0e917934 ] [SFTP: fix bug in previous logging patch. david-sarah@jacaranda.org**20100530050000 Ignore-this: 613e4c115f03fe2d04c621b510340817 ] [SFTP: more logging to track down OpenOffice hang. david-sarah@jacaranda.org**20100530040809 Ignore-this: 6c11f2d1eac9f62e2d0f04f006476a03 ] [SFTP: avoid blocking close on a heisenfile that has been abandoned or never changed. Also, improve the logging to help track down a case where OpenOffice hangs on opening a file with FXF_READ|FXF_WRITE. david-sarah@jacaranda.org**20100530025544 Ignore-this: 9919dddd446fff64de4031ad51490d1c ] [SFTP: the same bug as in _sync_heisenfiles also occurred in two other places. david-sarah@jacaranda.org**20100530060127 Ignore-this: 8d137658fc6e4596fa42697476c39aa3 ] [SFTP: further improvements to test coverage. david-sarah@jacaranda.org**20100602234422 Ignore-this: 87eeee567e8d7562659442ea491e187c ] [SFTP: cater to clients that assume a file is created as soon as they have made an open request; also, fix some race conditions associated with closing a file at about the same time as renaming or removing it. david-sarah@jacaranda.org**20100529045253 Ignore-this: 2404076b2154ff2659e2b10e0b9e813c ] [SFTP: fix pyflakes warnings; drop 'noisy' versions of eventually_callback and eventually_errback; robustify conversion of exception messages to UTF-8. david-sarah@jacaranda.org**20100523140905 Ignore-this: 420196fc58646b05bbc9c3732b6eb314 ] [SFTP: avoid logging all data passed to callbacks. david-sarah@jacaranda.org**20100519000651 Ignore-this: ade6d69a473ada50acef6389fc7fdf69 ] [SFTP: Increase test_sftp timeout to cater for francois' ARM buildslave. david-sarah@jacaranda.org**20100522191639 Ignore-this: a5acf9660d304677048ab4dd72908ad8 ] [SFTP: log tracebacks for RAISEd exceptions. david-sarah@jacaranda.org**20100523221535 Ignore-this: c76a7852df099b358642f0631237cc89 ] [SFTP: more logging to investigate behaviour of getAttrs(path). david-sarah@jacaranda.org**20100523204236 Ignore-this: e58fd35dc9015316e16a9f49f19bb469 ] [SFTP: fix time handling to make sure floats are not passed into twisted.conch, and to print times in the future less ambiguously in directory listings. david-sarah@jacaranda.org**20100524230412 Ignore-this: eb1a3fb72492fa2fb19667b6e4300440 ] [SFTP: 'sync' any open files at a direntry before opening any new file at that direntry. This works around the sshfs misbehaviour of returning success to clients immediately on close. david-sarah@jacaranda.org**20100525230257 Ignore-this: 63245d6d864f8f591c86170864d7c57f ] [SFTP: handle removing a file while it is open. Also some simplifications of the logout handling. david-sarah@jacaranda.org**20100525184210 Ignore-this: 660ee80be6ecab783c60452a9da896de ] [SFTP: fixes and test cases for renaming of open files. david-sarah@jacaranda.org**20100523032549 Ignore-this: 32e0726be0fc89335f3035157e202c68 ] [SFTP: Fix error in support for getAttrs on an open file, to index open files by directory entry rather than path. Extend that support to renaming open files. Also, implement the extposix-rename@openssh.org extension, and some other minor refactoring. david-sarah@jacaranda.org**20100522035836 Ignore-this: 8ef93a828e927cce2c23b805250b81a4 ] [SFTP: allow getAttrs to succeed on a file that has been opened for creation but not yet uploaded or linked (part of #1050). david-sarah@jacaranda.org**20100520035613 Ignore-this: 2f59107d60d5476edac19361ccf6cf94 ] [SFTP: fixed bugs that caused hangs during write (#1037). david-sarah@jacaranda.org**20100517044228 Ignore-this: b8b95e82c4057367388a1e6baada993b ] [SFTP: add tests for more combinations of open flags. david-sarah@jacaranda.org**20100519053933 Ignore-this: b97ee351b1e8ecfecabac70698060665 ] [SFTP: allow FXF_WRITE | FXF_TRUNC (#1050). david-sarah@jacaranda.org**20100519043240 Ignore-this: bd70009f11d07ac6e9fd0d1e3fa87a9b ] [SFTP: improve logging so that results of requests are (usually) logged. david-sarah@jacaranda.org**20100520003652 Ignore-this: 3f59eeee374a3eba71db9be31d5a95 ] [SFTP: change error code returned for ExistingChildError to FX_FAILURE (fixes gvfs with some picky programs such as gedit). david-sarah@jacaranda.org**20100518004205 Ignore-this: c194c2c9aaf3edba7af84b7413cec375 ] [SFTP: fixes related to reporting of permissions (needed for sshfs). david-sarah@jacaranda.org**20100518054521 Ignore-this: c51f8a5d0dc76b80d33ffef9b0541325 ] [Eliminate Windows newlines from sftpd.py. david-sarah@jacaranda.org**20100515005656 Ignore-this: cd54fd25beb957887514ae76e08c277 ] [SFTP: work around a probable bug in twisted.conch.ssh.session:loseConnection(). Also some minor error handling cleanups. david-sarah@jacaranda.org**20100517012606 Ignore-this: 5d3da7c4219cb0c14547e7fd70c74204 ] [SFTP: avoid race condition where .write could be called on an OverwriteableFileConsumer after it had been closed. david-sarah@jacaranda.org**20100523233830 Ignore-this: 55d381064a15bd64381163341df4d09f ] [SFTP: Support statvfs extensions, avoid logging actual data, and decline shell sessions politely. david-sarah@jacaranda.org**20100516154347 Ignore-this: 9d05d23ba77693c03a61accd348ccbe5 ] [SFTP: implement execCommand to interoperate with clients that issue a 'df -P -k /' command. Also eliminate use of Zope adaptation. david-sarah@jacaranda.org**20100516012754 Ignore-this: 2d0ed28b759f67f83875b1eaf5778992 ] [sftpd.py: 'log.OPERATIONAL' should be just 'OPERATIONAL'. david-sarah@jacaranda.org**20100515155533 Ignore-this: f2347cb3301bbccc086356f6edc685 ] [Attempt to fix #1040 by making SFTPUser implement ISession. david-sarah@jacaranda.org**20100515005719 Ignore-this: b3baaf088ba567e861e61e347195dfc4 ] [Update SFTP implementation and tests: fix #1038 and switch to foolscap logging; also some code reorganization. david-sarah@jacaranda.org**20100514043113 Ignore-this: 262f76d953dcd4317210789f2b2bf5da ] [New SFTP implementation: mutable files, read/write support, streaming download, Unicode filenames, and more david-sarah@jacaranda.org**20100512055407 Ignore-this: 906f51c48d974ba9cf360c27845c55eb ] [sftpd: minor debug-logging tweak warner@allmydata.com**20081105194511] [ftpd/sftpd: stop using RuntimeError, for #639 warner@lothar.com**20090222232426 Ignore-this: 97001362c4ba9e94b2e254e229b79987 ] [rollback [20090226150237-b2345-1e916a746a7f4627b050f02f0e442fae5caf69d4] for 1.4.0 release; #645 zooko@zooko.com**20090411181906 Ignore-this: 15aa9ce6d1d49e9447f32e233d136bab ] [Fix for bug #645, correct path handling logic so that it works from sshfs Alberto Berti **20090226150237 Ignore-this: e9c1b2d48ebf4ba68100d76e54154a78 ] [Tests for new SFTP implementation david-sarah@jacaranda.org**20100512060552 Ignore-this: 20308d4a59b3ebc868aad55ae0a7a981 ] [unicode: make test_cli test a non-ascii argument, and make the fallback term encoding be locale.getpreferredencoding() zooko@zooko.com**20100604141251 Ignore-this: b2bfc07942f69141811e59891842bd8c ] [test_cli.py: Fix tests when sys.stdout.encoding=None and refactor this code into functions Francois Deppierraz **20100520084447 Ignore-this: cf2286e225aaa4d7b1927c78c901477f ] [More cleanups to test_cli using new utilities for reading and writing files. david-sarah@jacaranda.org**20100206013855 Ignore-this: 9fd2294406b346bfe9144fff6a61f789 ] [debug catalog-shares: tolerate even more errors on bad files/directories warner@allmydata.com**20081030215447] [unicode: always decode json manifest as utf-8 then encode for stdout zooko@zooko.com**20100604084840 Ignore-this: ac481692315fae870a0f3562bd7db48e pyflakes pointed out that the exception handler fallback called an un-imported function, showing that the fallback wasn't being exercised. I'm not 100% sure that this patch is right and would appreciate François or someone reviewing it. ] [Test behaviour of 'tahoe ls' for unknown objects (#837) david-sarah@jacaranda.org**20100224025913 Ignore-this: b999f6239796a90cadb41e8650aa3782 ] [addendum to "Fix 'tahoe ls' on files (#771)" Brian Warner **20091227232149 Ignore-this: 6dd5e25f8072a3153ba200b7fdd49491 tahoe_ls.py: tolerate missing metadata web/filenode.py: minor cleanups test_cli.py: test 'tahoe ls FILECAP' ] [Fix 'tahoe ls' on files (#771). Patch adapted from Kevan Carstensen. Brian Warner **20091227225443 Ignore-this: 8bf8c7b1cd14ea4b0ebd453434f4fe07 web/filenode.py: also serve edge metadata when using t=json on a DIRCAP/childname object. tahoe_ls.py: list file objects as if we were listing one-entry directories. Show edge metadata if we have it, which will be true when doing 'tahoe ls DIRCAP/filename' and false when doing 'tahoe ls FILECAP' ] [unicode tests: fix missing import zooko@zooko.com**20100604142630 Ignore-this: db437fe8009971882aaea9de05e2bc3 ] [stringutils.py: Unicode helper functions + associated tests Francois Deppierraz **20100520004105 Ignore-this: 7a73fc31de2fd39d437d6abd278bfa9a This file contains a bunch of helper functions which converts unicode string from and to argv, filenames and stdout. ] [tests: drastically increase timeout of this very time-consuming test in honor of François's ARM box zooko@zooko.com**20100607115929 Ignore-this: bf1bb52ffb6b5ccae71d4dde14621bc8 ] [add 'tahoe debug consolidate' command, to merge directories created by repeated 'tahoe cp -r' or the allmydata win32 backup tool, into the form that would have been created by 'tahoe backup'. warner@allmydata.com**20090312205606 Ignore-this: 66569ca2190aa7b0f9199bcf09dcb27e ] [CREDITS: update François's Description zooko@zooko.com**20100608155513 Ignore-this: a266b438d25ca2cb28eafff75aa4b2a ] [docs: Update NEWS file with new features and bugfixes in 1.7.0 francois@ctrlaltdel.ch**20100609091120 Ignore-this: 8c1014e4469ef530e5ff48d7d6ae71c5 ] [docs: update relnotes.txt for Tahoe-LAFS v1.7.0! zooko@zooko.com**20100619052048 Ignore-this: 1dd2c851f02adf3ab5a33040051fe05a ... and remove relnotes-short.txt (just use the first section of relnotes.txt for that purpose) ] [docs: update relnote.txt for Tahoe-LAFS v1.7.0β zooko@zooko.com**20100609054602 Ignore-this: 52e1bf86a91d45315960fb8806b7a479 ] [docs: update relnotes.txt for v1.6.1 zooko@zooko.com**20100224065755 Ignore-this: 6d078e94425462ac8d074e3e7c82da28 ] [quickstart.html: python 2.5 -> 2.6 as recommended version david-sarah@jacaranda.org**20100705175858 Ignore-this: bc3a14645ea1d5435002966ae903199f ] [Raise Python version requirement to 2.4.4 for non-UCS-2 builds, to avoid a critical Python security bug. david-sarah@jacaranda.org**20100605031713 Ignore-this: 2df2b6d620c5d8191c79eefe655059e2 ] [setup: fix bug (wrong import) in error message, as noticed by pyflakes zooko@zooko.com**20090519195642 Ignore-this: f1b9f8c00b46c1b5f2f20e5fc424f341 ] [setup: fix trivial bug in recent patch to test base64.py at startup zooko@zooko.com**20090519195129 Ignore-this: f6be038f74b53ca69e7109fe34adfbc ] [setup: make Tahoe exit at startup with a useful error message if the base64.py module is buggy (fixes part of #710) zooko@zooko.com**20090519194555 Ignore-this: aa4d398235ddca8d417d61c9688e154 ] [setup: doc string describing what the require_auto_deps() function is for zooko@zooko.com**20080815172234] [setup: remove the try: except: around the import of pkg_resources -- we now require setuptools at run time and at build time zooko@zooko.com**20080418202459] [setup: don't try __import__(name) in _auto_deps.py zooko@zooko.com**20080418191722 This happens to work, because all of our "distribution" (i.e. distributable packaged Python code) names to coincide with all of their "package" (i.e. a directory with a __init__.py in it, which is "import"-able) names, except, I think for Twisted on Brian's debian sid system. But there's no reason why it should always work, and the only reason for that __import__() was to give us an explicit error message indicating missing requirements in the case that pkg_resources isn't importable or that the requirements don't have correct .egg-info metadata. So, by removing this stanza we may allow certain places to get a more ad-hoc failure message, i.e. an ImportError from somewhere, instead of an ImportError from _auto_deps.py, but that's okay. Note that dependencies which do not have their .egg-info metadata with them are increasingly rare, since Python 2.5 distutils creates the .egg-info file by default, and Linux distributions have stopped their former practice of actively deleting the .egg-info files. ] [_auto_deps.py: update comment warner@allmydata.com**20080129195321] [_auto_deps: tolerate DistributionNotFound (but not VersionConflict), to accomodate distributions (i.e. gutsy) which provide our dependencies but don't include .egg-info files warner@allmydata.com**20080129195237] [quickstart.html: warn against installing Python at a path containing spaces. david-sarah@jacaranda.org**20100604032413 Ignore-this: c7118332573abd7762d9a897e650bc6a ] [docs: install.html: link into Python 2.5.5 download page zooko@zooko.com**20100202065852 Ignore-this: 1a9471b8175b7de5741d8445a7ede29d ] [docs: install.html: recommend Python 2.5 (because I can build extension modules for it with mingw), architecture.txt: point out that our Proof of Retrievability feature is client-side-only zooko@zooko.com**20100202053842 Ignore-this: e33fd413a91771c77b17d7de0f215bea ] [Clarify quickstart instructions for installing pywin32 david-sarah@jacaranda.org**20100511180300 Ignore-this: d4668359673600d2acbc7cd8dd44b93c ] [docs: install.html -> quickstart.html zooko@zooko.com**20100421155757 Ignore-this: 6084e203909306bed93efb09d0e6181d It is not called "installing" because that implies that it is going to change the configuration of your operating system. It is not called "building" because that implies that you need developer tools like a compiler. Also I added a stern warning against looking at the "InstallDetails" wiki page, which I have renamed to "AdvancedInstall". ] [docs: a few small edits to try to guide newcomers through the docs zooko@zooko.com**20100303231902 Ignore-this: a6aab44f5bf5ad97ea73e6976bc4042d These edits were suggested by my watching over Jake Appelbaum's shoulder as he completely ignored/skipped/missed install.html and also as he decided that debian.txt wouldn't help him with basic installation. Then I threw in a few docs edits that have been sitting around in my sandbox asking to be committed for months. ] [docs/debian.txt: add notes on how to build Tahoe on a debian system warner@allmydata.com**20080617204132] [add misc/simulate_load.py in an attempt to understand why permuting the peerlist per each storage index matters zooko@zooko.com**20080712212622] [docs: a few small edits to performance.txt and README zooko@zooko.com**20100202052750 Ignore-this: bf8b1b7438e8fb6da09eec9713c78533 ] [setup: update README to point to known_issues.txt zooko@zooko.com**20080722010229] [docs: a few edits to architecture.txt, most significantly highlighting "future work" to avoid confusing it with the current version, and adding a "future work" about a random-sampling Proof of Retrievability verifier zooko@zooko.com**20100202045117 Ignore-this: 81122b3042ea9ee6bc12e795c2386d59 ] [Fill in 'docs/performance.txt' with some performance information Kevan Carstensen **20100202005914 Ignore-this: c66b255b2bd2e7e11f5707b25e7b38be ] [Add 'docs/performance.txt', which (for the moment) describes mutable file performance issues Kevan Carstensen **20100115204500 Ignore-this: ade4e500217db2509aee35aacc8c5dbf ] [Change install.html to reference 1.6.1 instead of 1.6.0 david-sarah@jacaranda.org**20100228061941 Ignore-this: 4738440e66a12dcf2cadf968fba5337 ] [Fix handling of correctly encoded unicode filenames (#534) Francois Deppierraz **20100520004356 Ignore-this: 8a3a7df214a855f5a12dc0eeab6f2e39 Tahoe CLI commands working on local files, for instance 'tahoe cp' or 'tahoe backup', have been improved to correctly handle filenames containing non-ASCII characters. In the case where Tahoe encounters a filename which cannot be decoded using the system encoding, an error will be returned and the operation will fail. Under Linux, this typically happens when the filesystem contains filenames encoded with another encoding, for instance latin1, than the system locale, for instance UTF-8. In such case, you'll need to fix your system with tools such as 'convmv' before using Tahoe CLI. All CLI commands have been improved to support non-ASCII parameters such as filenames and aliases on all supported Operating Systems except Windows as of now. ] [Fix an filename encoding issue with "tahoe cp" francois@ctrlaltdel.ch**20081111200803] [test_cli.py: Ensure that we can read our uploaded files back francois@ctrlaltdel.ch**20081114134458] [cli: undo the effects of [http://allmydata.org/trac/tahoe/changeset/20081222235453-92b7f-f841e18afb94e1fd95e6dafb799a3d876dd85c69] zooko@zooko.com**20081224155317 Ignore-this: d34ee20d89221357e32872d721d7685f We're just going to mark unicode in the cli as unsupported for tahoe-lafs-1.3.0. Unicode filenames on the command-line do actually work for some platforms and probably only if the platform encoding is utf-8, but I'm not sure, and in any case for it to be marked as "supported" it would have to work on all platforms, be thoroughly tested, and also we would have to understand why it worked. :-) ] [cli: decode all cli arguments, assuming that they are utf-8 encoded zooko@zooko.com**20081222235453 Ignore-this: d92b4d146e1dc9848c6a4b6aaaa3d1e9 Also encode all args to urllib as utf-8 because urllib doesn't handle unicode objects. I'm not sure if it is appropriate to *assume* utf-8 encoding of cli args. Perhaps the Right thing to do is to detect the platform encoding. Any ideas? This patch is mostly due to François Deppierraz. ] [util/base32: allow unicode inputs to a2b() or could_be_base32_encoded(), and encode them with utf-8 before processing them zooko@zooko.com**20081222234713 Ignore-this: e1eb4caed2f78b2fef0df4bbf8bb26f7 ] [util/base32: loosen the precondition forbidding unicode and requiring str -- now it requires either unicode or str zooko@zooko.com**20081222222237 Ignore-this: 3481d644bdc5345facbc199d33653f37 Hopefully this will make it so that tests pass with François Deppierraz's patch to fix the tahoe cli's handling of unicode argument. ] [idlib: make failures much clearer when encountering unicode robk-tahoe@allmydata.com**20080214232307 while investigating fuse related stuff, I spent quite a while staring at very cryptic explosions I got from idlib. it turns out that unicode objects and str objects have .translate() methods with differing signatures. to save anyone else the headache, this makes it very clear if you accidentally try to pass a unicode object in to a2b() etc. ] [cli: mark unicode filenames as unsupported -- see #534 for details zooko@zooko.com**20081224192802 Ignore-this: b209ccbd838f633ec201e2e97156847c ] [test_cli: use explicit (and stable) testdirs, instead of using self.mktemp warner@lothar.com**20090307090428 Ignore-this: 7c58d159e4f33d01635c3445d9e591f9 ] [Fixed tests again so they will pass on windows. Alberto Berti **20090223003502 Ignore-this: 80d5074e7153642a2fa2a77958bfb50d ] [Fixed tests so that they pass also on buildbots. Alberto Berti **20090222224311 Ignore-this: fcb91cd6acf028382411d23d380a4576 ] [Added tests for the cse when listdir is an iterator Alberto Berti **20090222224356 Ignore-this: 218fb2aba02c28b4b1e5324bdb5adeaa ] [tests: raise the timeout for test_cli since Zandr's ARM machine totally burst through the old one zooko@zooko.com**20090609210509] [tahoe_add_alias.py: minor refactoring Brian Warner **20100115064220 Ignore-this: 29910e81ad11209c9e493d65fd2dab9b ] [tahoe add-alias/create-alias: don't corrupt non-newline-terminated alias Brian Warner **20100114210246 Ignore-this: 9c994792e53a85159d708760a9b1b000 file. Closes #741. ] [Alter CLI utilities to handle nonexistent aliases better Kevan Carstensen **20100211024318 Ignore-this: e698ea4a57f5fe27c24336581ca0cf65 ] [tahoe_ls: improve error message when the target is missing warner@allmydata.com**20080522003452] [Implement more clearly defined moving semantics in tahoe_mv.py kevan@isnotajoke.com**20090720034523 Ignore-this: aaa592156f6fa93cb087824a74b0f2cb ] [Prevent mutable objects from being retrieved from an immutable directory, and associated forward-compatibility improvements. david-sarah@jacaranda.org**20100127064430 Ignore-this: 5ef6a3554cf6bef0bf0712cc7d6c0252 ] [test/common: oops, forgot the FakeMutableFileNode.get_readonly fix warner@allmydata.com**20080520015219] [web: add test for unicode POST when the name comes from name=, not the filename attribute warner@allmydata.com**20080604000939] [fuse/blackmatch: added asynchronous (background) file download robk-tahoe@allmydata.com**20081020233333 previously, upon opening a file for reading, the open() call would block while the entire file was retrieved from tahoe into the cache directory. This change adds a DownloaderWithReadQueue class, and associated plumbing, such that an open() will return promptly with the download initiated 'in the background'. Subsequent read() operations will block until enough data has been downloaded to satisfy that request. This provides a behaviour similar to streaming, i.e. the client application will be able to read data from the fuse interface while the remainder of the file is still being downloaded. ] [fuse/blackmatch: add readability to some logging, fix a permissions problem robk-tahoe@allmydata.com**20081017004421 adds a couple of functions to unpack 'mode' and 'flags' for open() calls, to facilitate debugging. adds a fix to ensure that all tmp files created for writing are opened with permissions 0600 - one problem I had with testing with the Finder was that files were being opened write only (0200) and were then failing to upload to tahoe due to internal permission denied errors. there remain a variety of problems with finder access which I'm unable to comprehend at this time. sometimes copies to tahoe will work fine, sometimes they yield "the finder cannot complete the operation because some data ... could not be read or written. (Error code -36)" sometimes "You may need to enter the name and password for an administrator on this computer to change the item" sometimes "The operation cannot be completed because an item with the name ... already exists." and sometimes "The operation cannot be completed because the item ... is locked." What seems to be absent is rhyme or reason. unix operations (cp, mv) work fine, rsync works fine. ] [fuse/blackmatch: log exception in server startup robk-tahoe@allmydata.com**20081017014650 humphf. my build runs the fuse stuff fine, but the build from the buildslave doesn't seem to start up properly. hopefully this will elicit some useful info ] [fuse/blackmatch: split into client/server (twisted server) robk-tahoe@allmydata.com**20081016150846 This implements a client/server split for blackmatch, where the client implements the fuse_main bindings and a simple blocking rpc client mechanism. The server implements the other half of that rpc mechanism, and contains all the actual logic for interpreting fuse requests in the context of the on disk cache and requests to the tahoe node. The server is based on a twisted reactor. The rpc mechanism implements a simple method dispatch including marshalling, using json, of basic inert data types, in a flat namespace (no objects). The client side is written in a blocking idiom, to interface with the threading model used by the fuse_main bindings, whereas the server side is written for a twisted reactor-based environment, intended to facilitate implementing more sophisticated logic in that paradigm. The two communicate over a unix domain socket, allocated within the nodedir. Command line usage is unchanged; the server is launched automatically by the client. The server daemonizes itself, to avoid preventing the original parent process (e.g. 'runtests') from waiting upon the server exiting. The client keeps open a 'keepalive' connection to the server; upon loss thereof the server will exit. This addresses the fact that the python-fuse bindings provide no notification of exit of the client process upon unmount. The client thus provides a relatively thin 'shim' proxying requests from the fuse_main bindings across the rpc to the server process, which handles the logic behind each request. For the time being, a '--no-split' option is provided to surpress the splitting into client/server, yielding the prior behaviour. Once the server logic gets more complex and more entrenched in a twisted idiom, this might be removed. The 'runtests' test harness currently tests both modes, as 'impl_c' and 'impl_c_no_split' ] [macfuse: fix unicode handling robk-tahoe@allmydata.com**20080306234325 at one point I'd thrown in a 'str' since fuse api bits required a str instance but tahoe returns unicode objects from its json parsing. that, naturally enough should really be a utf8 encoded str of the unicode object... ] [tahoefuse: return bogus but useful data to statfs call robk-tahoe@allmydata.com**20080507234009 previously tahoefuse returned the fs stat for the filesystem the fuse plugin was running upon (e.g. '/'). this works ok until you need to copy more to tahoe than the local machine has free disk space, at which point Finder will refuse to copy 'too much' data. this changes it so that tahoe always reports 2TiB used of an 8TiB filesystem this is entirely bogus, but allows copies of up to 2TiB to be initiated. ] [fuse/impl_c: UNDO --auto-fsid option robk-tahoe@allmydata.com**20080925134730 rolling back: Thu Sep 25 14:42:23 BST 2008 robk-tahoe@allmydata.com * fuse/impl_c: add --auto-fsid option this was inspired by reading the fuse docs and discovering the 'fsid' option to fuse_main, and was _intended_ to support a sort of 'stability' to the filesystem (specifically derived from the root-uri mounted, whether directly or via an alias) to support mac aliases across unmount/remount etc. some experimentation shows that that doesn't actually work, and that, at least for mac aliases in my testing, they're tied to path-to-mountpoint and not to the fsid - which seems to have no bearing. perhaps the 'local' flag is causing weirdness therein. at any rate, I'm recording it simply for posterity, in case it turns out to be useful after all somewhere down the road. M ./contrib/fuse/impl_c/blackmatch.py +13 ] [fuse/impl_c: add --auto-fsid option robk-tahoe@allmydata.com**20080925134223 this was inspired by reading the fuse docs and discovering the 'fsid' option to fuse_main, and was _intended_ to support a sort of 'stability' to the filesystem (specifically derived from the root-uri mounted, whether directly or via an alias) to support mac aliases across unmount/remount etc. some experimentation shows that that doesn't actually work, and that, at least for mac aliases in my testing, they're tied to path-to-mountpoint and not to the fsid - which seems to have no bearing. perhaps the 'local' flag is causing weirdness therein. at any rate, I'm recording it simply for posterity, in case it turns out to be useful after all somewhere down the road. ] [fuse/runtests: added --tests, renamed --suites robk-tahoe@allmydata.com**20081016142836 changed the --tests option to be --suites, as it takes a prefix, e.g. 'read' 'write' (or 'all', the default) and runs those suites which are applicable to each implementation being tested. added a --tests option, which takes a list of tests, e.g. 'read_file_contents' 'write_overlapping_large_writes' and runs all tests specified without regard to whether the implementation(s) under test are declared to support them. this is basically to allow a specific test or two to be run, saving time during development and debugging by not running the entire suite ] [fuse/impl_c: move mac tahoefuse impl out into contrib/fuse robk-tahoe@allmydata.com**20080925014214 For a variety of reasons, high amongst them the fact that many people interested in fuse support for tahoe seem to have missed its existence, the existing fuse implementation for tahoe, previously 'mac/tahoefuse.py' has been renamed and moved. It was suggested that, even though the mac build depends upon it, that the mac/tahoefuse implementation be moved into contrib/fuse along with the other fuse implementations. The fact that it's not as extensively covered by unit tests as mainline tahoe was given as corroboration. In a bid to try and stem the confusion inherent in having tahoe_fuse, tfuse and tahoefuse jumbled together (not necessarily helped by referring to them as impl_a, b and c respectively) I'm hereby renaming tahoefuse as 'blackmatch' (black match is, per wikipedia "a type of crude fuse" hey, I'm a punny guy) Maybe one day it'll be promoted to be 'quickmatch' instead... Anyway, this patch moves mac/tahoefuse.py out to contrib/fuse/impl_c/ as blackmatch.py, and makes appropriate changes to the mac build process to transclude blackmatch therein. this leaves the extant fuse.py and fuseparts business in mac/ as-is and doesn't attempt to address such issues in contrib/fuse/impl_c. it is left as an exercise to the reader (or the reader of a message to follow) as to how to deal with the 'fuse' python module on the mac. as of this time, blackmatch should work on both mac and linux, and passes the four extant tests in runtests. (fwiw neither impl_a nor impl_b have I managed to get working on the mac yet) since blackmatch supports a read-write and caching fuse interface to tahoe, some write tests obviously need to be added to runtests. ] [macfuse: move macfuse files around to simplify pythonpath robk-tahoe@allmydata.com**20080219231817 the mac/macfuse subdirectory needed to be added to the pythonpath in order to build a binary incorporating the mac fuse system. this change should make those modules accessible relative to the mac/ directory which is implicitly included in the .app build process. ] [mac build: updates to respect UPLOAD_DEST argument to make robk-tahoe@allmydata.com**20080226230353 the make mac-upload target now requires an UPLOAD_DEST argument to be given, which is the rsync destination (including trailing '/') to which the version stamped directory containing the .dmg should be placed. the account the build is running as (e.g. 'buildslave') should have ssh access to the account specified in that dest. one might also consider locking the key down to the target directory by adding something like command="rsync --server -vlogDtpr . /home/amduser/public_html/dist/mac-blah/" to the corresponding authorized_key entry on the target machine. ] [Makefile: split mac 'make .dmg' and 'upload' into separate steps warner@allmydata.com**20080125222913] [mac build: ahem. fix makefile probs robk-tahoe@allmydata.com**20080227004822 oops. I screwed up the makefile syntax further. buildslave would spend a lot of fruitless time trawling the entire drive. this fixes that. and a stray -n. ahem. [looks down sheepishly] ] [mac build: fix makefile bug robk-tahoe@allmydata.com**20080227002010 blah $( foo ) is more explicit than blah ` foo ` in a bash-like context unfortunately it doesn't translate very well to makefiles, for which $( means something else entirely ] [mac build: tweaks to build fuse for 10.4 and 10.5 robk-tahoe@allmydata.com**20080227000844 rather than trying to build a single .app with both 10.4 and 10.5 fuse libraries embedded within it, for the time being, we're just going to have independant 10.4 and 10.5 builds. this provides a 10.5 _fusemodule.so, and build changes to copy the appropriate versions of files for 10.4 or 10.5 from sub dirs of mac/ into the build tree before triggering py2app ] [fuse/runtests: added a --web-open option robk-tahoe@allmydata.com**20081003172026 similar to the --debug-wait option which causes the test harness to pause at various stages of the process to facilitate debugging, this option simplifies that debugging by automatically opening a web browser to the root dir of that implementation's tests when tests are commenced. in addition, if --web-open is specfied but --debug-wait is not, the harness will still pause after running tests but before tearing down the tahoe grid - this allows all tests to run to completion, but provide a debugging hook to investigate the end state of the grid's contents thereafter. ] [fuse/tests: slew of changes to fuse 'runtests' robk-tahoe@allmydata.com**20080924183601 This patch makes a significant number of changes to the fuse 'runtests' script which stem from my efforts to integrate the third fuse implementation into this framework. Perhaps not all were necessary to that end, and I beg nejucomo's forebearance if I got too carried away. - cleaned up the blank lines; imho blank lines should be empty - made the unmount command switch based on platform, since macfuse just uses 'umount' not the 'fusermount' command (which doesn't exist) - made the expected working dir for runtests the contrib/fuse dir, not the top-level tahoe source tree - see also discussion of --path-to-tahoe below - significantly reworked the ImplProcManager class. rather than subclassing for each fuse implementation to be tested, the new version is based on instantiating objects and providing relevant config info to the constructor. this was motivated by a desire to eliminate the duplication of similar but subtly different code between instances, framed by consideration of increasing the number of platforms and implementations involved. each implementation to test is thus reduced to the pertinent import and an entry in the 'implementations' table defining how to handle that implementation. this also provides a way to specify which sets of tests to run for each implementation, more on that below. - significantly reworked the command line options parsing, using twisted.usage; what used to be a single optional argument is now represented by the --test-type option which allows one to choose between running unittests, the system tests, or both. the --implementations option allows for a specific (comma-separated) list of implemenations to be tested, or the default 'all' the --tests option allows for a specific (comma-separated) list of tests sets to be run, or the default 'all'. note that only the intersection of tests requested on the command line and tests relevant to each implementation will be run. see below for more on tests sets. the --path-to-tahoe open allows for the path to the 'tahoe' executable to be specified. it defaults to '../../bin/tahoe' which is the location of the tahoe script in the source tree relative to the contrib/fuse dir by default. the --tmp-dir option controls where temporary directories (and hence mountpoints) are created during the test. this defaults to /tmp - a change from the previous behaviour of using the system default dir for calls to tempfile.mkdtemp(), a behaviour which can be obtained by providing an empty value, e.g. "--tmp-dir=" the --debug-wait flag causes the test runner to pause waiting upon user input at various stages through the testing, which facilitates debugging e.g. by allowing the user to open a browser and explore or modify the contents of the ephemeral grid after it has been instantiated but before tests are run, or make environmental adjustments before actually triggering fuse mounts etc. note that the webapi url for the first client node is printed out upon its startup to facilitate this sort of debugging also. - the default tmp dir was changed, and made configurable. previously the default behaviour of tempfile.mkdtemp() was used. it turns out that, at least on the mac, that led to temporary directories to be created in a location which ultimately led to mountpoint paths longer than could be handled by macfuse - specifically mounted filesystems could not be unmounted and would 'leak'. by changing the default location to be rooted at /tmp this leads to mountpoint paths short enough to be supported without problems. - tests are now grouped into 'sets' by method name prefix. all the existing tests have been moved into the 'read' set, i.e. with method names starting 'test_read_'. this is intended to facilitate the fact that some implementations are read-only, and some support write, so the applicability of tests will vary by implementation. the 'implementations' table, which governs the configuration of the ImplProcManager responsible for a given implementation, provides a list of 'test' (i.e test set names) which are applicable to that implementation. note no 'write' tests yet exist, this is merely laying the groundwork. - the 'expected output' of the tahoe command, which is checked for 'surprising' output by regex match, can be confused by spurious output from libraries. specfically, testing on the mac produced a warning message about zope interface resolution various multiple eggs. the 'check_tahoe_output()' function now has a list of 'ignorable_lines' (each a regex) which will be discarded before the remainder of the output of the tahoe script is matched against expectation. - cleaned up a typo, and a few spurious imports caught by pyflakes ] [fuse: runtests: Move exception classes to top scope. nejucomo@gmail.com**20080607070600] [tahoe_fuse: system test: Verify file contents can be properly read. nejucomo@gmail.com**20080130091448] [fuse: runtests.py: Fix bug in polling_operation error that always referred to introducer.furl. nejucomo@gmail.com**20080607061719] [tahoe_fuse: system test: Replace repeated attempts at webapi calls with single calls, abstract webapi calls into a single function. nejucomo@gmail.com**20080130085625] [tahoe_fuse: system test: webapi connection: bug fix and small log output change. nejucomo@gmail.com**20080121021031] [fuse: runtests: Create an interface for setup/cleanup of the two implementations... nejucomo@gmail.com**20080607070825 The impl_b cleanup appears incorrect. I'm not sure what the proper behavior is. ] [tahoe_fuse: system test: Populate a testdir with files and empty children directories, then test the fuse interface for proper listings and size metadata. nejucomo@gmail.com**20080130085943] [tahoe_fuse: system test: Make output checking into non-fatal warnings, and make patterns looser. nejucomo@gmail.com**20080130071053] [fuse: runtests.py: Fix a typo bug in fusermount output checking. nejucomo@gmail.com**20080607061815] [fuse: runtests: Make test numbers (and everything in general) 0-indexed for consistency. nejucomo@gmail.com**20080607061915] [fuse/blackmatch: 'flatten' the fuse api implementation robk-tahoe@allmydata.com**20081016143547 the previous revision of blackmatch used a file_class to delegate all fuse api operations on files to a specific per-file class, which is an option given by the python-fuse bindings. this is a pre-cursor to the 'split' client/server version, which uses a simple, moreover flat, rpc mechanism to broker access to methods. ] [test_web.py: one more line of test coverage warner@allmydata.com**20081029050015] [uri: add abbrev_si() method, which returns the abbreviated storage index warner@allmydata.com**20090131013110 Ignore-this: bb3d9483570dbe0dc9ecdc1f31d8d79f ] [web: make sure that PUT /uri?mutable=false really means immutable, fixes #675 warner@lothar.com**20090408021340] [Tweak wording in directory page: not-read-only is "modifiable", mention creating a directory _in this directory_. Kevin Reid **20090526232414 Ignore-this: f006ec52ba2051802e025a60bcface56 ] [directories: fix semantic conflict between my "keep track of position" optimization patch and Kevan's "cache serialized entries" optimization patch zooko@zooko.com**20090710032028 Ignore-this: 46f8b00fd3eca4adf89dec437e65d696 ] [directories: keep track of your position as you decode netstring after netstring from an input buffer instead of copying the trailing part zooko@zooko.com**20090705025109 Ignore-this: bee1ae76060fbc920bddb6e839b7dd1a This makes decoding linear in the number of netstrings instead of O(N^2). ] [netstring: add required_trailer= argument warner@allmydata.com**20080926165754] [test_netstring.py: move netstring tests to a separate file warner@allmydata.com**20080926165526] [webapi.txt: document t=set_children, other small edits Brian Warner **20091009200446 Ignore-this: 4d7e76b04a7b8eaa0a981879f778ea5d ] [webapi: fix t=check for DIR2-LIT (i.e. empty immutable directories) Brian Warner **20091126232731 Ignore-this: 8513c890525c69c1eca0e80d53a231f8 ] [control.py: fix speedtest: use download_best_version (not read) on mutable nodes Brian Warner **20091207060512 Ignore-this: 7125eabfe74837e05f9291dd6414f917 ] [Simplify immutable download API: use just filenode.read(consumer, offset, size) Brian Warner **20091201225330 Ignore-this: bdedfb488ac23738bf52ae6d4ab3a3fb * remove Downloader.download_to_data/download_to_filename/download_to_filehandle * remove download.Data/FileName/FileHandle targets * remove filenode.download/download_to_data/download_to_filename methods * leave Downloader.download (the whole Downloader will go away eventually) * add util.consumer.MemoryConsumer/download_to_data, for convenience (this is mostly used by unit tests, but it gets used by enough non-test code to warrant putting it in allmydata.util) * update tests * removes about 180 lines of code. Yay negative code days! Overall plan is to rewrite immutable/download.py and leave filenode.read() as the sole read-side API. ] [SFTP/FTP: merge user/account code, merge docs warner@allmydata.com**20081106012558] [ftpd: make sure we're using a patched/fixed Twisted, to avoid confusion later warner@allmydata.com**20081007011411] [ftpd: hush pyflakes warner@allmydata.com**20081007014513] [ftpd: add native_client.php -based HTTP authentication scheme warner@allmydata.com**20081006231511] [docs/ftp.txt: correct Twisted dependency: we don't need VFS, we can use a release, as long as you apply the patch warner@allmydata.com**20081104235840] [ftp/sftp: move to a new frontends/ directory in preparation for factoring out password-auth component warner@allmydata.com**20081105200733] [ftp: change the twisted hack necessary for async-write-close, to one more agreeable to the twisted-dev folks, add a copy of the necessary patch to docs/ftp.txt warner@allmydata.com**20081007010605] [ftpd: remove debug messages warner@allmydata.com**20081006231620] [ftpd: add ftp.accounts checker, remove InMemoryPasswordChecker warner@allmydata.com**20081006225124] [trivial: M-x whitespace-cleanup on src/immutable/download.py zooko@zooko.com**20090108164901 Ignore-this: bb62daf511e41a69860be657cde8df04 ] [immutable/download.py: wrap to 80cols, no functional changes Brian Warner **20091005192542 Ignore-this: 6b05fe3dc6d78832323e708b9e6a1fe ] [immutable: when downloading an immutable file, use primary shares if they are available zooko@zooko.com**20081220131456 Ignore-this: f7b8b76fd7df032673ab072384eaa989 Primary shares require no erasure decoding so the more primary shares you have, the less CPU is used. ] [immutable: handle another form of share corruption with LayoutInvalid exception instead of AssertionError zooko@zooko.com**20090105234645 Ignore-this: fee5f6572efca5435ef54ed32552ca9d ] [contrib: fix fuse_impl_c to use new Python API zooko@zooko.com**20100109174956 Ignore-this: 51ca1ec7c2a92a0862e9b99e52542179 original patch by Thomas Delaet, fixed by François, reviewed by Brian, committed by me ] [hush pyflakes-0.4.0 warnings: remove trivial unused variables. For #900. Brian Warner **20100114221529 Ignore-this: e96106c8f1a99fbf93306fbfe9a294cf ] [provisioning: more repair/survivability data warner@lothar.com**20070907055453] [test_system: assert that BASEDIR/node.url is created properly warner@allmydata.com**20080107234622] [mutable/servermap.py: fix needs_merge(), it was incorrectly claiming that mixed shares with distinct seqnums needed a merge, causing repair(force=False) to fail warner@lothar.com**20081024040024] [reliability.py: fix the numpy conversion, it was completely broken. Thanks to Terrell Russell for the help. warner@lothar.com**20090219195515 Ignore-this: f2b1eb65855111b338e1487feee1bbcf ] [reliability: switch to NumPy, since Numeric is deprecated warner@lothar.com**20090219074435 Ignore-this: f588a68e9bcd3b0bc3653570882b6fd5 ] [storage/immutable: raise a specific error upon seeing a bad version number, instead of using assert. Also wrap to 80cols. warner@lothar.com**20090309030732 Ignore-this: 5331d9680ffceff029fbbbcdece7f282 ] [storage/mutable: raise a specific error upon seeing bad magic, instead of using assert warner@lothar.com**20090309020201 Ignore-this: 8daa77362902f5d6ef793e9602a1383b ] [hashtree: fix O(N**2) behavior, to improve fatal alacrity problems in a 10GB file (#670). Also improve docstring. warner@lothar.com**20090331202127 Ignore-this: a04f72ed2b783fc880932fc5c482182b ] [Fix broken link from Provisioning to Reliability page. Kevin Reid **20090501191050 Ignore-this: 56dc1a5e659b70cc02dc4df7b5d518cd ] [dirnode.set_children: take a dict, not a list Brian Warner **20091013002440 Ignore-this: 540ce72ce2727ee053afaae1ff124e21 ] [dirnode.set_uri/set_children: change signature to take writecap+readcap Brian Warner **20091012235126 Ignore-this: 5df617b2d379a51c79148a857e6026b1 instead of a single cap. The webapi t=set_children call benefits too. ] [tolerate simplejson-2.0.0 and newer, which frequently return bytestrings instead of unicode objects. Closes #523 warner@allmydata.com**20080930222106] [check_load: add stats-gathering warner@allmydata.com**20071218200737] [dirnode.pack_children(): add deep_immutable= argument Brian Warner **20091026162809 Ignore-this: d5a2371e47662c4bc6eff273e8181b00 This will be used by DIR2:CHK to enforce the deep-immutability requirement. ] [repairer: add deterministic test for #819, mark as TODO zooko@zooko.com**20100110013619 Ignore-this: 4cb8bb30b25246de58ed2b96fa447d68 ] [test_repairer: rename Verifier test cases to be more precise and less verbose Brian Warner **20091005201115 Ignore-this: 64be7094e33338c7c2aea9387e138771 ] [Fix webapi t=mkdir with multpart/form-data, as on the Welcome page. Closes #919. Brian Warner **20100121065052 Ignore-this: 1f20ea0a0f1f6d6c1e8e14f193a92c87 ] [Fix race conditions and missing callback in allmydata.test.test_cli.Cp.test_copy_using_filecap, add utilities for one-liner reading and writing of files, and fix cases in test_cli where files were not being closed after writing. david-sarah@jacaranda.org**20100206013727 Ignore-this: 49da6c33190d526a4ae84c472f04d5f4 ] [fix test_cli to put the root_dir.cap in the private subdir zooko@zooko.com**20080103234853] [test_cli.py: use str objects instead of unicode ones francois@ctrlaltdel.ch**20081114134137 This will hopefully fix failing tests with LC_ALL=C ] [docs: NEWS and relnotes-short.txt and CREDITS for v1.6.1 zooko@zooko.com**20100224065231 Ignore-this: 41c056ae48c639e5a934d4c1983bc118 ] [docs: CREDITS: where due zooko@zooko.com**20100202053831 Ignore-this: 11646dd603ac715ae8277a4bb9562215 ] [docs: further CREDITS level-ups for Nils, Kevan, David-Sarah zooko@zooko.com**20100126170021 Ignore-this: 1e513e85cf7b7abf57f056e6d7544b38 ] [docs: more CREDITS for François, Kevan, and David-Sarah zooko@zooko.com**20100126132133 Ignore-this: f37d4977c13066fcac088ba98a31b02e ] [docs: CREDITS zooko@zooko.com**20090213034228 Ignore-this: d6bb651d657ed8967ca1dfb23afbd00e ] [CREDITS: format to <= 79 columns, add Marc Tooley, update Kevan Carstensen zooko@zooko.com**20090720131354 Ignore-this: 3b8bbf952e69eb26597c7bce15e830e0 ] [docs: CREDITS for Nathan and for Armin Rigo zooko@zooko.com**20080610231424] [docs: CREDITS: add David-Sarah to the CREDITS file zooko@zooko.com**20100109060435 Ignore-this: 896062396ad85f9d2d4806762632f25a ] [docs: edits for docs/running.html from Sam Mason zooko@zooko.com**20090809201416 Ignore-this: 2207e80449943ebd4ed50cea57c43143 ] [docs: a couple of small edits to release notes (thanks Peter) zooko@zooko.com**20100202054832 Ignore-this: 1d0963c43ff19c92775b124c49c8a88a ] [docs: a few edits and updates to relnotes.txt, relnotes-short.txt, and NEWS in preparation for v1.6.0 zooko@zooko.com**20100202043222 Ignore-this: d90c644fa61d78e33cbdf0be428bb07a ] [docs: updates to relnotes.txt, NEWS, architecture, historical_known_issues, install.html, etc. zooko@zooko.com**20100201181809 Ignore-this: f4fc924652af746862c8ee4d9ba97bf6 ] [architecture.txt: explain the introducer SPOF and why it really isn't that bad. Closes #323. warner@allmydata.com**20080530015111] [docs: mention issues using flogtool on Windows zooko@zooko.com**20090204033410 Ignore-this: 6122bcb82eea32d2a936a59d77233743 ] [docs: some small edits to install.html zooko@zooko.com**20090413160414 Ignore-this: 1e7142ea444fef61c684c089407d675 ] [setup: simplify install.html a tad zooko@zooko.com**20090119210447 Ignore-this: 529b2f225b3d98ed3bc99a4962e781ee ] [setup: use setup.cfg aliases to map "setup.py test" to "setup.py trial" and "setup.py build" to "setup.py darcsver --count-all-patches build_tahoe" zooko@zooko.com**20090120183723 Ignore-this: f390676787f4d521c17fbe96fb2cd2a6 Thanks to dpeterson for the suggestion. ] [setup: attempt to remove the custom setuptools-ish logic in setup.py -- the result works on my Windows box but doesn't yield a working ./bin/tahoe on Windows, and hasn't been tested yet on other platforms zooko@zooko.com**20081205233054 Ignore-this: 843e7514870d7a4e708646acaa7c9699 ] [setup: require setuptools >= v0.6c8 zooko@zooko.com**20080326191302] [setup: fix the md5sum of the bundled setuptools egg zooko@zooko.com**20080206183529] [setup: use a customized version of ez_setup.py which bootstraps from Python-version-agnostic setuptools bootstrap eggs zooko@zooko.com**20080122170056] [setup: add a setuptools bootstrap egg that works on all versions of Python zooko@zooko.com**20080122170012 For versions of Python >= 2.3. ] [setup: bundle setuptools-0.6c8, we need a bugfix in it zooko@zooko.com**20080326191234] [setup: fix site-dirs to find system installed twisted on mac. robk-tahoe@allmydata.com**20080924174255 zooko helped me unravel a build weirdness today. somehow the system installed twisted (/System/Library) was pulling in parts of the other twisted (/Library) which had been installed by easy_install, and exploding. getting rid of the latter helped, but it took this change to get the tahoe build to stop trying to rebuild twisted and instead use the one that was already installed. c.f. tkt #229 ] [setup: turn off --multi-version until I can figure out why it breaks test_runner zooko@zooko.com**20081121043645 Ignore-this: 36bf5db4122e6bc4e12588d9717a1e32 ] [setup: use "setup.py develop --multi-version" so that if there is a too-old version of a dependency installed this doesn't prevent Tahoe's "develop" and run-in-place from working zooko@zooko.com**20081120201545 Ignore-this: 898f21fc1b16ae39c292fdd1ef42c446 ] [setup.py,Makefile: move the 'chmod +x bin/tahoe' into setup.py warner@lothar.com**20080917230756] [Makefile: touch .built on every build, so other targets can depend upon .built and avoid redundant rebuilds warner@lothar.com**20080130073257] [setup: remove custom Trial class inside our setup.py and use the setuptools_trial plugin zooko@zooko.com**20081205232207 Ignore-this: e0f68169e8ac1b5a54b796e8905c7b80 ] [setup: integrate the bundled setuptools_trial plugin with Chris Galvan's patch to use that plugin zooko@zooko.com**20081201174804 Ignore-this: 5d03e936cf45f67a39f993704024788c ] [setup: bundle setuptools_trial in misc/dependencies/ zooko@zooko.com**20081201174438 Ignore-this: f13a4a1af648f9ab9b3b3438cf94053f ] [use_setuptools_trial.patch cgalvan@mail.utexas.edu**20081121205759] [setup.py trial: improve --verbose suggestion a bit warner@lothar.com**20080919193922] [setup: pretend the tahoe requires twisted to set up, so that twisted will be there for nevow zooko@zooko.com**20081025135042 Ignore-this: 4e6c7e580f7e30df571e2e63be663734 ] [docs: remove extra

from install.html (thanks, David-Sarah Hopwood) zooko@zooko.com**20090726142436 Ignore-this: a6fcab5e6524505b5b8514f62d9a97f3 ] [docs: update install.html to point to 1.5.0 and edit the instructions (broadening the recommendation on Python versions to bless >= v2.4.2 <= v2.6.x) zooko@zooko.com**20090802030523 Ignore-this: 6aabf53148df1bc7a5dd25b1d290829a ] [docs: mention pywin32 earlier zooko@zooko.com**20090726133452 Ignore-this: 67e0d5b32af136113ec5b4e6c6d6c37 ] [docs: remove warning about inability to build modules on py2.6 on Windows with mingw, differentiate between clients and servers, reflow to a consistent column width (79), add hint about firewall/NAT docs. zooko@zooko.com**20090621175005 Ignore-this: 85e7c1ccb258317ca4dd37917afb48f5 ] [docs: edit running.html zooko@zooko.com**20080215170219] [docs: shorter running.html zooko@zooko.com**20080506222904] [docs: edits to [source:docs/install.html] and [source:docs/running.html] zooko@zooko.com**20080611022200] [docs: mention configuration, suggested by ben hyde's question about storage servers zooko@zooko.com**20080506203935] [docs: explain better how to invoke the tahoe executable when creating and starting nodes zooko@zooko.com**20080611021923] [docs: lowercase "introducer" zooko@zooko.com**20080611022314 He's not that important. ] [setup: edit install.html to warn Windows users away from Python v2.6 zooko@zooko.com**20090611225506 Ignore-this: 89ad63eab49ede883ef92f2de5b5fc54 ] [docs: edit install.html regarding versions of Python zooko@zooko.com**20090413160612 Ignore-this: 1de165ad7645be32ef671ece3fcae9ea ] [docs: setup: Norm Hardy suggested that it would be easier if users realized that they already had Python (especially true for Mac users) zooko@zooko.com**20090325035459 Ignore-this: e1bb76a1be4d6d541090d8d9e7e73db9 ] [docs: suggest Python 2.5 -- Python 2.6 is not as well tested yet zooko@zooko.com**20090210054421 Ignore-this: 3ef6988c693330d4937b4d8e1a996c39 ] [doc: specify Python >= 2.4.2 zooko@zooko.com**20090204213840 Ignore-this: 108c60b69fdb1d0fcb95810703ce415a ] [docs: update install.html to recommend Python v2 instead of Python v2.5.2 zooko@zooko.com**20090103183100 Ignore-this: 5dbea379c59e0d9be817cdd9c8393d65 ] [docs: install.html: instruct Debian users to use this document and not to go find the DownloadDebianPackages page, ignore the warning at the top of it, and try it zooko@zooko.com**20090804123840 Ignore-this: 49da654f19d377ffc5a1eff0c820e026 http://allmydata.org/pipermail/tahoe-dev/2009-August/002507.html ] [docs: add note about pywin32 to install.html zooko@zooko.com**20090413185210 Ignore-this: d386abfccfdc1015b8f1216d95b4792f ] [docs: fix helper.txt to describe new config style zooko@zooko.com**20091224223522 Ignore-this: 102e7692dc414a4b466307f7d78601fe ] [NEWS: improve "tahoe backup" notes, mention first-backup-after-upgrade duration Brian Warner **20100111190132 Ignore-this: 10347c590b3375964579ba6c2b0edb4f Thanks to Francois Deppierraz for the suggestion. ] [NEWS: update with all recent user-visible changes Brian Warner **20100127222209 Ignore-this: 277d24568018bf4f3fb7736fda64eceb ] [NEWS: update with all user-visible changes since the last release Brian Warner **20091127224217 Ignore-this: 741da6cd928e939fb6d21a61ea3daf0b ] [docs: add a couple of details to NEWS, change date and a bit of formatting, name of 'Tahoe-LAFS' project zooko@zooko.com**20090802022601 Ignore-this: 9c35cdaaec10613570225b19c902b0ef ] [NEWS: more minor edits Brian Warner **20090722024522 Ignore-this: e2c199cfdbaa32a125819f14df971d45 ] [docs: update relnotes.txt for Tahoe-LAFS v1.6 zooko@zooko.com**20100128171257 Ignore-this: 920df92152aead69ef861b9b2e8ff218 ] [Miscellaneous documentation, test, and code formatting tweaks. david-sarah@jacaranda.org**20100127070309 Ignore-this: 84ca7e4bb7c64221ae2c61144ef5edef ] [fuse/impl_c: reworking of mac/tahoefuse, command line options, test integration robk-tahoe@allmydata.com**20080925001535 a handful of changes to the tahoefuse implementation used by the mac build, to make command line option parsing more flexible and robust, and moreover to facilitate integration of this implementation with the 'runtests' test harness used to test the other two implementations. this patch includes; - improvements to command line option parsing [ see below ] - support for 'aliases' akin to other tahoe tools - tweaks to support linux (ubuntu hardy) the linux support tweaks are, or at least seem to be, a result of the fact that hardy ships with fuse 0.2pre3, as opposed to the fuse0.2 that macfuse is based upon. at least the versions I was working with have discrepencies in their interfaces, but on reflection this is probably a 'python-fuse' version issue rather than fuse per se. At any rate, the fixes to handling the Stat objects should be safe against either version, it's just that the bindings on hardy lacked code that was in the 'fuse' python module on the mac... command line options: the need for more flexible invocation in support of the runtests harness led me to rework the argument parsing from some simple positional hacks with a pass-through of the remainder to the fuse binding's 'fuse_main' to a system using twisted.usage to parse arguments, and having just one option '-o' being explicitly a pass-through for -o options to fuse_main. the options are now: --node-directory NODEDIR : this is used to look up the node-url to connect to if that's not specified concretely on the command line, and also used to determine the location of the cache directory used by the implementation, specifically '_cache' within the nodedir. default value: ~/.tahoe --node-url NODEURL : specify a node-url taking precendence over that found in the node.url file within the nodedir --alias ALIAS : specifies the named alias should be mounted. a lookup is performed in the alias table within 'nodedir' to find the root dir cap the named alias must exist in the alias table of the specified nodedir --root-uri ROOTURI : specifies that the given directory uri should be mounted at least one of --alias and --root-uri must be given (which directory to mount must be specified somehow) if both are given --alias takes precedence. --cache-timeout TIMEOUTSECS : specifies the number of seconds that cached directory data should be considered valid for. this tahoefuse implementation implements directory caching for a limited time; largely because the mac (i.e. the Finder in particular) tends to make a large number of requests in quick successsion when browsing the filesystem. on the flip side, the 'runtests' unit tests fail in the face of such caching because the changes made to the underlying tahoe directories are not reflected in the fuse presentation. by specifying a cache-timeout of 0 seconds, runtests can force the fuse layer into refetching directory data upon each request. any number of -oname=value options may be specified on the command line, and they will all be passed into the underlying fuse_main call. a single non-optional argument, the mountpoint, must also be given. ] [macfuse: slew of updates robk-tahoe@allmydata.com**20080301021241 various updates to improve the functionality of the mac fuse plugin 1. caching previously, the experimental tahoefuse plugin pre-loaded the whole structure of the specified mount into memory at launch time. changes which were made through that fuse plugin would be remembered, but any changes made through other tahoe clients would not be reflected. now directory contents are only loaded when needed, and the data is cached for a limited time. any use of Directory objects should first call maybe_refresh() which will check the time since the cache was last loaded, and if the data is older than some validity period (currently 26s) then the directory's contents will be refetched and reloaded. this replaces the 'load_dir()' method of TFS whenever a local change is made to a Directory object, or when the aforementioned cache reloading notices a change in directory data, the mtime of the directory is automatically updated. 2. stat / metadata the retrieval of 'stat' information for getattr(), and the way that metadata is handled, has been refactored to better reflect the fact that metadata in tahoe is only represented by 'edges' (i.e entries in directories) not on 'nodes' (files or directories themselves) hence a stat lookup should be a query to the parent directory (specifically the parent specified by the path being queried in the case that a node has multiple parents) for details known by that directory for the given child, rather than a query to the child itself. the TStat utility class for returning stat information to the python- fuse layer has been extended to accept a 'metadata' argument in its constructor. any fields found in the metadata dict which match the names of the stat attributes are loaded into the TStat object. the 'ctime' and 'mtime' attributes are translated to st_ctime and st_mtime to mesh with the existing timestamp handling code. any fields specified by kwargs to the constructor override things that might be loaded from the metadata dict. Directory objects now track their children as a dict mapping name to (child_obj, metadata) tuples. This is because the metadata in tahoe will be stored exclusively on the edges of the graph. each Directory maintains its own mtime however, and get_stat() calls will report the mtime of a directory based on the last modification of the Directory object, not based on any mtime records from the parent directory's metadata for that child. This addresses the fact that since directories may be shared, a given parent may or may not reflect the latest changes, however one of the Finder's behaviours is to examine the stat of a directory, and not to bother doing a readdir() if the stat is unchanged. i.e. unless directories report their changes in their stat info, the Finder will not show changes within that directory. 3. refactoring reporting of many error codes has been refactored to raise IOError subclasses with the appropriate errno. this exploits python-fuse's built-in mechanism for catching IOError and reporting the errno embedded within it automatically, while simplifying the code within the plugin. the add_child() method on TFS was removed in favour of simply having an add_child() method on Directory objects. this provides a more OO approach in that Directory is responsible for maintaining its own in memory state and also writing changes back to the node. similarly for remove_child() these changes, along with the new tfs.compose_url() method, significantly simplify and improve readability of mkdir, rename methods along with the newer link and unlink. these also get improved error reporting. various operations (chmod, chown, truncate, utime) are now ignored. previously they would report an unsupported operation (EOPNOTSUPP) but now are simply logged and ignored. this surpresses errors caused by some client programs which try to use these operations, but at the moment those operations are meaningless to the tahoe filesystem anyway. 4. link / unlink / rmdir link, symlink calls are now supported, though with semantics differing from posix, both equivalent. unlink, rmdir calls are now supported, also equivalent. link or symlink calls duplicate the uri of the named source and adds it as a child of another directory according to the destination path. for directories, this creates a 'hard' link, i.e. the same directory will appear in multiple locations within the filesystem, and changes in any place will be reflected everywhere. for files, by contrast, since the uri being duplicated is an immutable CHK uri, link/symlink for files is equivalent to a copy - though significantly cheaper. (a file copy with the fuse plugin is likely to cause a new file to be written and uploaded, the link command simply adds an entry referring to an existing uri) in testing, the 'ln' command is unable to make hard links (i.e. call link()) for directories, though symlink ('ln -s') is supported. either forms works equivalently for files. unlink and rmdir both remove the specified entry from its parent directory. 5. logging the 'tfuse.log' file now only reports launches of the fuse plugin. once the plugin has parsed the options, it reopens the log file with the name of the mount, e.g. tfuse.root_dir.log, so that multiple instances running concurrently will not interfere with each others' logging. 6. bug fixes the tmp_file in the cache dir backing files opened for write was intermittently failing to open the file. added O_CREAT to the os.open call so that files will be created if missing, not throw errors. a failure to correctly parse arguments if no mount (dir_cap) name was given but also no fuse options were given has been fixed. now the command 'tahoe fuse mountpoint' will correctly default to root_dir also when running from source, arguments to tahoefuse were not handled to correctly match the 'tahoe fuse ...' behaviour. ] [macfuse: rework fuse initialisation, integrate with 'tahoe' robk-tahoe@allmydata.com**20080219231608 this provides a variety of changes to the macfuse 'tahoefuse' implementation. most notably it extends the 'tahoe' command available through the mac build to provide a 'fuse' subcommand, which invokes tahoefuse. this addresses various aspects of main(argv) handling, sys.argv manipulation to provide an appropriate command line syntax that meshes with the fuse library's built- in command line parsing. this provides a "tahoe fuse [dir_cap_name] [fuse_options] mountpoint" command, where dir_cap_name is an optional name of a .cap file to be found in ~/.tahoe/private defaulting to the standard root_dir.cap. fuse_options if given are passed into the fuse system as its normal command line options and the mountpoint is checked for existence before launching fuse. the tahoe 'fuse' command is provided as an additional_command to the tahoe runner in the case that it's launched from the mac .app binary. this also includes a tweak to the TFS class which incorporates the ctime and mtime of files into the tahoe fs model, if available. ] [uri: generalize regexp that recognizes tahoe URLs to work for any host and port zooko@zooko.com**20081216234930 Ignore-this: 4a7716b8034c8e5ed9698a99f1ec5cb4 ] [Touch up #705 changes: Brian Warner **20090720153803 Ignore-this: 583517a3d80c2c1c6a397b6934b78b73 webapi.txt: clarify replace=only-files argument, mention replace= on POST t=uri test_cli.py: insert whitespace between logical operations web.common.parse_replace_arg: make it case-insensitive, to match the docs ] [Update webapi docs to reference new PUT behavior. kevan@isnotajoke.com**20090720034447 Ignore-this: 981c43767ee4d7d3e7711dfbea89b590 ] [Add a function to parse arguments for the replace parameter kevan@isnotajoke.com**20090720034723 Ignore-this: f27aae3befa76b7bec1b697b5588332 ] [docs: relnotes.txt: reflow to 63 chars wide because google groups and some web forms seem to wrap to that zooko@zooko.com**20090802135016 Ignore-this: 53b1493a0491bc30fb2935fad283caeb ] [docs: relnotes.txt: fix edits noticed by Amber zooko@zooko.com**20090802031003 Ignore-this: 5ea62f161924d2ce8477b59c50d9ecc0 ] [docs: update relnotes.txt, relnotes-short.txt, and others documentation bits for v1.5.0 release! zooko@zooko.com**20090802025710 Ignore-this: cd95f569a2c0b4fada453e409f101679 ] [amdlib.util: merge in changes to humanreadable.py that were made in pyutil "Zooko O'Whielacronx "**20070525224957] [trivial: source code metadata zooko@zooko.com**20090403233315 Ignore-this: 23858d0320b9b7ba0e6d2fe4adeccea8 ] [clean up debian packaging: we have control files for etch/lenny/sid, and "Brian Warner "**20090703072804 everything else uses one of those. Add dependency on python-pysqlite2 for platforms that use py2.4 by default. Update foolscap dependency to 0.4.1. ] [debian/changelog: remove all versions but 0.0.1, so debchange can always override it. The 'deb-X' Makefile targets should not be used; run 'make deb-X-head' instead warner@allmydata.com**20080715222341] [debian: oops, set debian/changelog version to 0.6.0-1+, to be less than everything the buildbot is creating warner@allmydata.com**20070927011513] [debian: put a version string of '0.6.0+' in the debian changelog warner@allmydata.com**20070927005842] [debian/control: update dependencies to match _auto_deps: foolscap-0.3.0, pycryptopp-0.5 warner@lothar.com**20080806013222] [debian: add python-setuptools to the debian install-time dependencies. Should close #382. warner@allmydata.com**20080424230104] [debian: include misc/cpu-watcher.tac in the debian package warner@allmydata.com**20080827223026] [misc/spacetime: add munin plugins, add everything to .deb warner@lothar.com**20080807060003] [debian/rules: put munin plugins in /usr/share/PACKAGENAME/munin/, make them +x, remove packagename from rules to make branches easier to manage warner@allmydata.com**20080716014741] [debian: use our own /usr/bin/tahoe, remove runtime dependency on setuptools (since it required egg-aware versions of all dependencies too) warner@allmydata.com**20080410232959] [debian: use setuptools-generated support/bin/tahoe instead of bin/tahoe, to match Zooko's change that makes our in-tree bin/tahoe spawn support/bin/tahoe warner@allmydata.com**20080410213627] [debian: we now require setuptools at build time, and that or pkg_resources at runtime warner@allmydata.com**20080410224356] [oops, change debian dependency on zfec to 1.1, not 1.1.0 warner@allmydata.com**20080410004552] [make debian dependencies match _auto_deps.py ones, for foolscap and zfec warner@allmydata.com**20080409172301] [debian/sid: add more docs to package, including munin plugins warner@lothar.com**20080714195534] [copy debian/sid changes to debian/feisty warner@lothar.com**20080714195638] [disk-watcher: first draft of a daemon to use the HTTP stats interface and its new storage_server.disk_avail feature, to track changes in disk space over time warner@lothar.com**20080807042222] [setup: update the debian/copyright text to reflect the current licences zooko@zooko.com**20090311152952 Ignore-this: 806a95b1b79d6bb20507db5c7201af45 ] [setup: specify in the debian/control files that tahoe is compatible with Python 2.6 zooko@zooko.com**20090311225902 Ignore-this: d0793013e4c868d92793d932ef92a62d ] [sid/control: set python versions to 2.4,2.5 , to match feisty/control, since sid has had 2.5 available forever now warner@allmydata.com**20080716014238] [Makefile: add jaunty support, rearrange debian sections in order of release warner@lothar.com**20090618050502] [makefile: fix deb-edgy-head and deb-etch-head targets warner@allmydata.com**20070711202623] [setup: create a "make deb-lenny-head" target zooko@zooko.com**20090306191057 Ignore-this: 4b2ff187a3d08dcfe9318980ca92f097 I made this patch by copying [20090305220021-92b7f-89d987c7d05306b5cb03a64f2956a652c10a7296] and changing the name from "intrepid" to "lenny". I haven't tested it. ] [docs: edit about.html, add P.S. about expansion of LAFS, add Andrew Orlowski to media list zooko@zooko.com**20090722022430 Ignore-this: 6717610239104d273a417769c8cf66a5 ] [docs: how_to_make_a_tahoe_release.txt: a couple of small edits zooko@zooko.com**20090507214932 Ignore-this: ae92aa835ad369f4b9e6e49d681957a3 ] [docs: modify how-to-relase notes a tiny bit warner@lothar.com**20090407021135] [docs: add "darcs pull" to how_to_make_a_tahoe_release.txt, and renumber zooko@zooko.com**20090414024342 Ignore-this: d54d33e5f7e170eea12b6d6a16d0fc87 ] [doc: update how_to_make_a_tahoe_release.txt zooko@zooko.com**20090222175739 Ignore-this: 6a2e1592741b362bc170167a9cadc0b ] [docs: add a note about the process of making a new Tahoe release zooko@zooko.com**20080917170839] [docs: add some notes about things to do for a Tahoe release on pypi, freshmeat, and launchpad zooko@zooko.com**20081001210703] [docs: update NEWS, about.html, relnotes-short.txt, and known_issues.txt in preparation for v1.5.0 zooko@zooko.com**20090721234311 Ignore-this: bc81915d7d4f48c231fa86f0b8308d83 Especially note that strong claims of specialness that I've added, e.g. in about.html . ] [docs: known_issues.txt: my version of #615, remove "issue numbers", edits, move tahoe-1.1.0 issues to historical zooko@zooko.com**20090213041621 Ignore-this: 58dee952a3139791ba0fe03f03fcf8bb ] [known_issues.txt: edits suggested by Brian zooko@zooko.com**20080721174406] [docs: editing changes and updated news in known_issues.txt zooko@zooko.com**20081230070116 Ignore-this: e5dddc4446e3335a6c4eee7472e0670e ] [docs: known_issues.txt: edit to emphasize that other sorts of local-filesystem-unwritability will lead to the same problems zooko@zooko.com**20080611193857] [docs: split historical/historical_known_issues.txt out of known_issues.txt zooko@zooko.com**20081230065226 Ignore-this: 9b6d0d679294110deeb0ea18b4ad7ac8 All issues which are relevant to users of v1.1, v1.2, or v1.3 go in known_issues.txt. All issues which are relevant to users of v1.0 go in historical/historical_known_issues.txt. ] [move historical docs from wiki pages into the source tree, clearly marked as historical warner@allmydata.com**20080603013832] [docs: known_issues.txt: change the release data of Tahoe v1.1.0 to 2008-06-11 from 2008-06-10 zooko@zooko.com**20080611194033] [docs/known_issues: mention #615 javascript-vs-frames, for zooko to improve/rewrite warner@allmydata.com**20090211201453 Ignore-this: b4805670c3700d90db39fb008b0f2c92 ] [known_issues.txt: fix up the argv leakage issue -- it applies to Tahoe 1.2.0. Other editing corrections. zooko@zooko.com**20080722010249] [known_issues.txt: command-line arguments are leaked to other processes zooko@zooko.com**20080722004334] [known_issues.txt: add issue #491 and renumber issues zooko@zooko.com**20080721172101] [docs: known_issues.txt: add the issue of files > 12 GiB being silently corrupted zooko@zooko.com**20080611195159] [docs: known_issues.txt: add the security issue concerning leakage of file cap by active content or referrer-bearing hyperlinks embedded in the file zooko@zooko.com**20080611193937] [docs: reformat for 70 columns plus a few small edits zooko@zooko.com**20080610233725] [docs: explain exactly what false alarms are caused in the unit tests by Twisted v8 and pyOpenSSL v0.7 zooko@zooko.com**20080610233126] [docs: [source:docs/known_issues.txt] zooko@zooko.com**20080610232425] [docs: relnotes-short.txt zooko@zooko.com**20090215163510 Ignore-this: 683649bb13499bbe0e5cea2e1716ff59 linkedin.com imposed a strict limit on the number of characters I could post. This forced me to prune and prune and edit and edit until relnotes.txt was a quarter of its former size. Here's the short version. ] [docs: small edit to about.html zooko@zooko.com**20090528233422 Ignore-this: 1cfbb1f8426ed6d63b2d3952e4464ddc ] [docs: add links to Tahoe-LAFS for Paranoids and Tahoe-LAFS for Corporates in about.html zooko@zooko.com**20090528232717 Ignore-this: 7b70baa700d6b6f6e9ceec4132efe5 ] [docs: edit about.html and include network-and-reliance-topology.png (loaded from http://allmydata.org ) zooko@zooko.com**20090527150916 Ignore-this: 44adc61cde8ced8be2f0a7dfc7d95dad ] [docs: small edit to about.html zooko@zooko.com**20090210170219 Ignore-this: fa79838f4cdac17c09b6c3332e8a68b5 ] [setup: tidy up formatting and comments in _auto_deps.py zooko@zooko.com**20090727193008 Ignore-this: 99fcb61a27caae0e63ae8ce8d7505c05 ] [setup: increase requirement on pycryptopp to >= 0.5.15 zooko@zooko.com**20090706140815 Ignore-this: f3839c7c1f9ebff1fcf2eea47ed3c48b ] [setup: require pycryptopp>=0.5.14 if on Windows and with Python>=2.6 zooko@zooko.com**20090630184807 Ignore-this: f7e9beeb5d5613a7c0ffed14d1dda3c6 ] [setup: require pycryptopp >= v0.5 zooko@zooko.com**20080506181747] [setup: loosen our requirement on pycryptopp from >= 0.2.9 to >= 0.2.8 zooko@zooko.com**20080123170035 Again, tahoecs2 has pycryptopp v0.2.8, and reviewing the pycryptopp change history shows that there were no important bugfixes added since 0.2.8. ] [setup: require pysqlite >= v2.0.5. if we are running on Python < 2.5 zooko@zooko.com**20090604154548 Ignore-this: cf04f46079821df209d01dad2e24b40b ] [setup: remove attempt to automatically satisfy dependency on pywin32 zooko@zooko.com**20090213234939 Ignore-this: ac02d54a956f7cc58bd3c0802764005f ] ["tahoe webopen": add --info flag, to get ?t=info Brian Warner **20100424233003 Ignore-this: 126b0bb6db340fabacb623d295eb45fa Also fix some trailing whitespace. ] [change docs and --help to use "grid" instead of "virtual drive": closes #892. Brian Warner **20100114201119 Ignore-this: a20d4a4dcc4de4e3b404ff72d40fc29b Thanks to David-Sarah Hopwood for the patch. ] [cli.py: fix typo in synopsis warner@allmydata.com**20070817004724] [add a simple load-generating tool to do random reads and writes warner@allmydata.com**20071218030607] [architecture.txt: make it clear that accounting/leases are not yet implemented warner@allmydata.com**20080310192519] [test_client.py: improve test coverage a bit warner@allmydata.com**20081029044335] [CLI: fix examples in tahoe put --help warner@allmydata.com**20090127213909 Ignore-this: 1fe319f70c3791482bb381c06d4a066b ] [test_cli: add test coverage for help strings warner@lothar.com**20090216210833 Ignore-this: d2020849107f687448e159a19d0e5dab ] [Add missing synopsis and descriptions for alias commands. Alberto Berti **20090221003106 Ignore-this: 8aedd03d36d92d912102c7f29e4ca697 ] [Removed '.hgrags' from vcs excludes Alberto Berti **20090222223946 Ignore-this: 3e94c22fc9d85f380ee11fb8bdb4d1e9 ] [tahoe cp -r: add --caps-only flag, to write filecaps into local files instead of actual file contents. Used only for debugging and as a quick tree-comparison tool. warner@lothar.com**20090315231958 Ignore-this: 8ecdf2b08601ae9e9fec5885bf640262 ] [test_system: even more 'cp -r' coverage warner@allmydata.com**20080522014049] [test_system.py: improve 'cp -r' coverage: exercise copy from tahoe to local disk warner@allmydata.com**20080522013625] [backup: remove the --no-backupdb command, the handling of "can't import sqlite", and the related tests, and change an error message to more correctly indicate failure to load the database from disk rather than failure to import sqlite module zooko@zooko.com**20090604173131 Ignore-this: 8200a9fdfc49243c280ecd1d0c44fa19 Fixes #728. ] [backupdb: cosmetic: capitalize the no-pysqlite instructions properly. Thanks to Terrell Russell for the catch. warner@allmydata.com**20090211212830 Ignore-this: a17b34a12bbe96ad5b531ef5d293471e ] [cli: add some --help text to 'tahoe cp' warner@lothar.com**20090625235751] [cli: webopen: when called with no arguments, open the Welcome page Brian Warner **20090701200548 Ignore-this: ae7d6cb42165d0c751926065378343dd ] [Update tahoe mv help text. kevan@isnotajoke.com**20090720034503 Ignore-this: 9124164acf459a4aa030c25e286bcb19 ] [docs: reflow architecture.txt to 78-char lines zooko@zooko.com**20091208232943 Ignore-this: 88f55166415f15192e39407815141f77 ] [docs: warn that the "garbage-collection and accounting" section of architecture.txt is out of date, and clarify that "deleted" therein means ciphertext getting garbage-collected zooko@zooko.com**20080822154605] [docs: update architecture.txt 's section on the vdrive a.k.a. filesystem layer zooko@zooko.com**20081006210500 Remove some obsolete parts (correct at the time, now incorrect), change terminology to reflect my preference: s/vdrive/filesystem/ and s/dirnode/directory/, and make a few other small changes. ] [docs: a couple of minor edits to NEWS and docs/architecture.txt zooko@zooko.com**20090721014112 Ignore-this: f82d77a46e442d38d5a17609f4b3dfa5 ] [more minor architecture.txt changes warner@allmydata.com**20080214022043] [update NEWS to cover all recent changes, sort by end-user importance Brian Warner **20090703014303 Ignore-this: 6ddac78075d7547a19712d505818949c ] [edit NEWS Brian Warner **20090630174115 Ignore-this: c4461a2304fcd45bee95e11418693a18 ] [docs: start updating the NEWS and relnotes.txt files, add Kevan to CREDITS zooko@zooko.com**20090621055114 Ignore-this: 35e05a5739549ffa693d55df51ffcfd ] [setup: enable build of .debs for Ubuntu Intrepid, thanks to DarKNesS_WolF zooko@zooko.com**20090305220021 Ignore-this: 88dbb3f72c2446b7734ac437189b67df ] [Makefile: add ubuntu/hardy deb targets warner@allmydata.com**20080617222618] [Makefile: add ubuntu 'gutsy' as a .deb target warner@allmydata.com**20071017203511] [cease producing .debs for dapper, since they'd depend upon a library (simplejson) that isn't packaged for dapper. Feisty++ are fine. Dapper users are encouraged to build from source. warner@allmydata.com**20070711213104] [update debian/copying to reflect GPLv2+12months license warner@lothar.com**20070426083833] [docs: CREDITS to Alberto Berti zooko@zooko.com**20090222193314 Ignore-this: 74d370ada3234cce9e58aec15d739f71 ] [NEWS: list all user-visible changes since 1.4.1 . Needs lots of editing. Brian Warner **20090630170734 Ignore-this: f606a5d678d0db8065b9f84e796d59b0 ] [docs: update NEWS, relnotes.txt, CREDITS to mention WUI Style zooko@zooko.com**20090526233654 Ignore-this: 72d16ec833bc4a22af23d29ea1d5ff8b ] [docs: CREDITS zooko@zooko.com**20090213201245 Ignore-this: 5d3101e680739e6cdacb4351b518ae33 ] [doc: add Toby Murray to the CREDITS zooko@zooko.com**20090120043857 Ignore-this: eedb7e9d47ddee5cbe189b55251d0859 ] [doc: add Larry Hosken to CREDITS zooko@zooko.com**20090117164943 Ignore-this: f2433a296ab2485872d22538bd0f64b2 ] [New credit file entry francois@ctrlaltdel.ch**20081114140548] [CREDITS: thanks to Chris Galvan zooko@zooko.com**20080827183950] [docs: CREDITS for Justin zooko@zooko.com**20080611020547] [CREDITS: add Paul Gerhardt, who submitted a small patch for make check-deps to be more newbie-friendly zooko@zooko.com**20080325184739] [CREDITS zooko@zooko.com**20080313160444] [CREDITS for nejucomo zooko@zooko.com**20080108165417] [CREDITS: more credit to nejucomo since we accepted a doc patch zooko@zooko.com**20071107160107] [CREDITS: Nathan Wilcox ++ zooko@zooko.com**20071015021312] [docs: edit relnotes.txt and promote Tahoe from filesystem to cloud storage thingie zooko@zooko.com**20090414021913 Ignore-this: 78cc79078c234d0467f6290dcae456b9 ] [docs: inaugurate Tahoe-1.4.1, since I left out a handful of patches from the Tahoe-1.4.0 release zooko@zooko.com**20090414025430 Ignore-this: 12d5ff7dc842668bdf318c5539272089 ] [docs: update relnotes.txt, NEWS for Tahoe-1.4.0 release! zooko@zooko.com**20090413041405 Ignore-this: d2eacb26b359a020956ee14b63d95dc5 ] [docs: edit about.html zooko@zooko.com**20090210080102 Ignore-this: d96f9b21f88d4c7a552f9ed3db5c6af4 ] [docs: about.html: a couple of edits suggested by kpreid's comments zooko@zooko.com**20080227150138] [setup: relnotes.txt mention the iPhone app and CIFS/SMB (tahoe-w32-client) zooko@zooko.com**20090213044121 Ignore-this: 2c9b8720579c4c146e4416c5a02c77a5 ] [doc: a few edits to docs made after the 1.3.0 release zooko@zooko.com**20090216201539 Ignore-this: dbff3b929d88134d862f1dffd1ef068a ] [NEWS: format some (but not all) items warner@lothar.com**20090407211514] [NEWS: add unformatted list of code changes since previous release warner@lothar.com**20090407021155] [docs: change install.html to point to the 1.4.0 release zip file instead of 1.3.0 zooko@zooko.com**20090413160649 Ignore-this: ec6b177d6689894b9842a012da04e5dc ] [docs: a few last-minute edits to the docs for 1.3.0 (also this patch will accompany the tag and conveniently trigger the buildbots to build a 1.3.0 version) zooko@zooko.com**20090214000500 Ignore-this: 879c9b10f0e5b9ed0031236e0714ddfa ] [docs/install.html: reference InstallDetails instead of debian-specific stuff warner@lothar.com**20080917225742] [setup: add link to the DownloadDebianPackages page zooko@zooko.com**20080908215451 Because I want that link off of the front page of the wiki... ] [merge_install.patch cgalvan@mail.utexas.edu**20090102164434 Ignore-this: aa6d4c05d583a0724eb218fef04c3940 ] [remove_sumo_install.patch cgalvan@mail.utexas.edu**20090102162347 Ignore-this: f328570b1da1ccfbaebc770d40748046 ] [setup: new install doc -- doesn't require GNU make or a C++ compiler any more! zooko@zooko.com**20081201180933 Ignore-this: 753e8d1e6f32e2ddcd7a082050114725 ] [docs: edit to install.html suggested by Brian zooko@zooko.com**20080506193115] [doc: mention that "Mac Developer Tools" is the way to get gcc/g++ for Mac zooko@zooko.com**20080610231934] [setup: edit the text of install.html zooko@zooko.com**20080908215549] [docs: a couple of tiny docs updates zooko@zooko.com**20080409225759] [docs: update install.html to reflect Justin's user test zooko@zooko.com**20080611020458] [Make the purpose and location of the tahoe executable more explicit in install.html. nejucomo@gmail.com**20080108173326 ] [Makefile,docs: tahoe-deps.tar.gz now lives in separate source/deps/ directory on http://allmydata.org warner@lothar.com**20080917204452] [#249: get dependent libs from tahoe-deps and ../tahoe-deps warner@lothar.com**20080917013627] [setup: add a setup.cfg file which instructs setuptools to install all eggs in unzipped form and to always copy them into the target directory (even if they are already installed somewhere else on the path that setuptools searches, which includes the CWD) zooko@zooko.com**20080122194647] [rewrite parts of the Makefile in setup.py. Add 'build_tahoe' and 'trial' subcommands. warner@allmydata.com**20080912010321 The 'make build' target now runs 'setup.py build_tahoe', which figures out where the target 'supportlib' directory should go, and invokes 'setup.py develop' with the appropriate arguments. The 'make test' target now runs 'setup.py trial', which manages sys.path and runs trial as a subroutine instead of spawning an external process. This simplifies the case where Twisted was built as a dependent library (and thus the 'trial' executable is not on PATH). setup.py now manages sys.path and PYTHONPATH for its internal subcommands, so the $(PP) prefix was removed from all Makefile targets that invoke setup.py . For the remaining ones, the 'setup.py -q show_pythonpath' subcommand was added to compute this prefix with python rather than with fragile shell/Makefile syntax. ] [setup: remove obsolete makefile target build-deps zooko@zooko.com**20080422190712] [setup: test depends on build, which means it invokes setup.py every time, which is slower but does "the right thing" more often zooko@zooko.com**20080424165704 There is a new target "quicktest" which depends on the .built and .checked-deps files. test-figleaf also depends on the build target now. ] [tests: test depends on _version.py zooko@zooko.com**20080228202924 because there is a test that asserts that our version is not "unknown" ] [Makefile: add quicktest-figleaf: this is in my edit-test-repeat loop, and I need it to be fast warner@allmydata.com**20080424183038] [setup: don't use "python" in Makefile, use $(PYTHON) everywhere zooko@zooko.com**20080721162849] [setup: whoops, really remove the default reactor=poll this time zooko@zooko.com**20080730032358] [setup: pass --reactor=poll to trial unless REACTOR variable is set, in which case pass --reactor=$(REACTOR) zooko@zooko.com**20080730023906 This hopefully works around the problem that Twisted v8.1.0 has a bug when used with pyOpenSSL v0.7 which bug causes some unit tests to spuriously fail -- see known_issues.txt r2788: http://allmydata.org/trac/tahoe/browser/docs/known_issues.txt?rev=2788#L122 Also it matches with the fact that --reactor=poll is required on cygwin. ] [setup: fix bug in Makefile -- ifeq, not ifneq -- so that now it sets poll reactor only if the user hasn't specified a REACTOR variable, instead of setting poll reactor only if the user has specified a REACTOR variable zooko@zooko.com**20080730160429] [setup: turn back on reactor=poll for cygwin trial (else it runs out of fds) zooko@zooko.com**20080730181217] [setup: instead of setting --reactor=poll for trial in all cases (which fails on platforms that don't have poll reactor, such as Windows and some Mac OS X), just set --reactor=poll for linux2. zooko@zooko.com**20080730031656 ] [setup: patch from Chris Galvan to build sdists with no deps in them normally, but include deps if --sumo zooko@zooko.com**20080827182644] [setup: don't assert that trial is present when the Makefile is evaluated zooko@zooko.com**20080903171837 This should fix #506, but it means that if (for some weird reason) Twisted can't be auto-installed and the find_trial.py script doesn't work, the user will get a weird failure message instead of a clean failure message explaining that trial couldn't be found. Oh well. Chris Galvan is working on a much nicer fix to all these issues -- see #505. ] [Makefile: avoid bare quotes, since the emacs syntax-highlighter gets confused by them warner@lothar.com**20080807183001] [setup: don't attempt to escape quote marks, just delete them. Ugly, but it works okay. zooko@zooko.com**20080806232742] [setup: remove accidentally duplicated lines from Makefile zooko@zooko.com**20080807193029] [setup: if the user passes a TRIALOPT env var then pass that on to trial zooko@zooko.com**20080730205806 This is useful for --reporter=bwverbose, for example. ] [setup: don't quote REACTOROPT -- when it is empty then we want no argument at all to be passed to trial, rather than the empty-string-argument zooko@zooko.com**20080609185324] [setup: don't quote TRIALCMD in Makefile -- it can be a pair of (python executable, path to command) paths zooko@zooko.com**20080605233912] [setup: quote variables which are going to be passed through a shell and which might contain spaces zooko@zooko.com**20080605221951] [setup: escape any double-quote chars in the PATH before using the PATH to find and invoke trial zooko@zooko.com**20080806231143] [setup: quote another place where spaces in paths cause shell command misparsing zooko@zooko.com**20080609232150] [Makefile: give setup.py develop a '--site-dirs' arg to work around the #249 setuptools bug which causes us to unnecessarily rebuild pyopenssl and other support libs installed via debian's python-support. Should be harmless on other platforms. warner@allmydata.com**20080910233432] [setup: indentation zooko@zooko.com**20080605210249] [Makefile: build twice, since sometimes the Nevow build fails the first time. See #455. This ought to be undone once that ticket is fixed by a new release of setuptools warner@allmydata.com**20080609230629] [Makefile: desert-island: don't re-fetch tahoe-deps.tar.gz if it's already there, remove the tahoe-deps/ before untarring directory to avoid unpacking weirdness warner@lothar.com**20080917052204] [#249: add 'test-desert-island', to assert that a tahoe-deps.tar.gz -enabled build does not download anything warner@lothar.com**20080917013702] [add 'tarballs' target, to generate compressed source tarballs warner@allmydata.com**20080131024514] [docs: mention -SUMO tarballs, point users at release tarballs instead of development ones warner@lothar.com**20080917203631] [setup: change URL from which to get source tarballs zooko@zooko.com**20080908215409 So that when you look at that directory you won't see distracting other things such as darcs repositories. ] [docs: install.html: link to http://allmydata.org/source/tahoe/ instead of http://allmydata.org/source/tahoe/tarballs/ zooko@zooko.com**20080611213522] [docs: edit install.html a tad zooko@zooko.com**20080826154929] [setup: remove the developer note about doing without GNU make (the GNU make requirement is about to hurt Peter if he tries to follow this doc, by the way) zooko@zooko.com**20081021163200 add classifiers showing with which versions of Python it is known to work. ] [setup.py: cosmetic, remove trailing whitespace warner@allmydata.com**20080714215325] [setup: update licensing information in setup.py comments and metadata zooko@zooko.com**20080108174500] [comment-out the 'license' field because PyPI rejects upload if you have an invalid value therein zooko@zooko.com**20071016034809 This means that we have no machine-readable licence for now. I will make the human-readable licensing. ] [setup: stop claiming that we are under GPL in the "license" field of the PyPI database zooko@zooko.com**20071016025742 Unfortunately, there is no way to claim that we are under a Free Software/Open Source licence without also claiming to be under a licence that we are not or claiming to have approval from DFSG or OSI, which we haven't. Until now, I erred on the side of choosing the licence that is closest to our from the list (GPL), but that was a bad idea and now I'm erring on the side of not including a machine-readable licensing claim at all. Hopefully humans who are interested will quickly find out that we are actually under a Real Free Software Licence. But really, this underscores that we need to talk to FSF, edit our licence for clarity of intent, and submit it to DFSG/OSI. ] [fix the 'license' field of the PyPI db (not the Trove Classifiers that I was changing in recent patches) to describe our licence and link to it czooko@zooko.com**20071016035510 The earlier patches were changing the Trove Classifiers, which is a different thing to this 'license' field. ] [setup: add excited DEVELOPER NOTE to install.html zooko@zooko.com**20080908215603 It should be removed before 1.3.0 release, of course... ] [docs: relnotes.txt final (!?) update for 1.3.0! zooko@zooko.com**20090213042814 Ignore-this: 7a959eba00115474ff048cd84ecab495 ] [docs: not-quite-final version of relnotes.txt for tahoe-1.3.0 zooko@zooko.com**20090210170227 Ignore-this: 64e11f3619d537eae28f4d33977bd7ab ] [docs: a couple of tiny edits zooko@zooko.com**20080619192619] [docs: relnotes.txt: reflow to 70 cols zooko@zooko.com**20080611230256] [relnotes.txt: update and edit for the 1.2.0 release! zooko@zooko.com**20080722010403] [docs: relnotes.txt: re-arrange sections a bit zooko@zooko.com**20080611195234] [docs: start updating the relnotes.txt in preparation for the next release zooko@zooko.com**20080701201936] [docs: relnotes.txt: trivial change (really just to trigger the buildbot when it comes in a bundle along with the 1.1.0 tag) zooko@zooko.com**20080611213202] [docs: relnotes.txt: update the release notes for the v1.1.0 release! zooko@zooko.com**20080611194055] [docs: fix name of docs dir in relnotes.txt zooko@zooko.com**20080218220805] [docs: small edit to relnotes.txt zooko@zooko.com**20080313185655] [docs: update relnotes.txt for Tahoe v1.0! zooko@zooko.com**20080326012800] [docs: fix typo in relnotes.txt zooko@zooko.com**20080313190147] [docs: link to the current CREDITS file from relnotes.txt zooko@zooko.com**20080313190420] [docs: small edit to relnotes.txt zooko@zooko.com**20080313191326] [docs: update relnotes.txt for allmydata.org "Tahoe" v0.9.0 ! zooko@zooko.com**20080313184326 Whee! ] [docs: update relnotes.txt zooko@zooko.com**20080215233533] [docs: update relnotes.txt for v0.8.0! zooko@zooko.com**20080214150026] [docs: update relnotes, running.html zooko@zooko.com**20080214145434] [docs: beginning of update to relnotes.txt for v0.8 zooko@zooko.com**20080213234302] [docs: relnotes for 0.7.0 zooko@zooko.com**20080108170144] [relnotes.txt: fix bug in relnotes.txt for v0.6.1 -- it incorrectly described v0.6 as having been released in August; it was actually September czooko@zooko.com**20071016043325] [relnote.txt: update relnotes.txt for the v0.6.1 release zooko@zooko.com**20071015215602] [relnotes.txt a few tiny branding edits zooko@zooko.com**20070817203303 Uncertain about the branding, but what the heck. ] [relnotes.txt: tweak #129 description a bit Brian Warner **20070920062433] [relnotes.txt: link to the final version of the README for v0.6 zooko@zooko.com**20070924214238] [relnotes.txt: add Performance section, fix link to foolscap zooko@zooko.com**20070924213231] [relnotes.txt: line-wrap to 70-chars and a small edit zooko@zooko.com**20070922030327] [relnotes.txt: correct description of leases zooko@zooko.com**20070919022416] [relnotes.txt: a few final touch-ups for v0.6 zooko@zooko.com**20070923170804] [relnotes.txt: add ticket #129 zooko@zooko.com**20070919212646] [relnotes.txt: update for v0.6 (not complete) zooko@zooko.com**20070918220430] [relnotes.txt: v0.5.1 zooko@zooko.com**20070823205141] [webapi: don't accept zero-length childnames during traversal. Closes #358, #676. Brian Warner **20091227201043 Ignore-this: a9119dec89e1c7741f2289b0cad6497b This forbids operations that would implicitly create a directory with a zero-length (empty string) name, like what you'd get if you did "tahoe put local /oops/blah" (#358) or "POST /uri/CAP//?t=mkdir" (#676). The error message is fairly friendly too. Also added code to "tahoe put" to catch this error beforehand and suggest the correct syntax (i.e. without the leading slash). ] [interface name cleanups: IFileNode, IImmutableFileNode, IMutableFileNode Brian Warner **20091120075255 Ignore-this: e3d193c229e2463e1d0b0c92306de27f The proper hierarchy is: IFilesystemNode +IFileNode ++IMutableFileNode ++IImmutableFileNode +IDirectoryNode Also expand test_client.py (NodeMaker) to hit all IFilesystemNode types. ] [interfaces.py: minor improvement to IDirectoryNode.set_node warner@allmydata.com**20080909233416] [Add t=mkdir-immutable to the webapi. Closes #607. Brian Warner **20091118070900 Ignore-this: 311e5fab9a5f28b9e8a28d3d08f3c0d * change t=mkdir-with-children to not use multipart/form encoding. Instead, the request body is all JSON. t=mkdir-immutable uses this format too. * make nodemaker.create_immutable_dirnode() get convergence from SecretHolder, but let callers override it * raise NotDeepImmutableError instead of using assert() * add mutable= argument to DirectoryNode.create_subdirectory(), default True ] [webapi: use t=mkdir-with-children instead of a children= arg to t=mkdir . Brian Warner **20091026011321 Ignore-this: 769cab30b6ab50db95000b6c5a524916 This is safer: in the earlier API, an old webapi server would silently ignore the initial children, and clients trying to set them would have to fetch the newly-created directory to discover the incompatibility. In the new API, clients using t=mkdir-with-children against an old webapi server will get a clear error. ] [make get_size/get_current_size consistent for all IFilesystemNode classes Brian Warner **20091118191624 Ignore-this: bd3449cf96e4827abaaf962672c1665a * stop caching most_recent_size in dirnode, rely upon backing filenode for it * start caching most_recent_size in MutableFileNode * return None when you don't know, not "?" * only render None as "?" in the web "more info" page * add get_size/get_current_size to UnknownNode ] [mutable: add get_size_of_best_version to the interface, to simplify the web HEAD code, and tests warner@allmydata.com**20080813020252] [class name cleanups: s/FileNode/ImmutableFileNode/ Brian Warner **20091120072239 Ignore-this: 4b3218f2d0e585c62827e14ad8ed8ac1 also fix test/bench_dirnode.py for recent dirnode changes ] [nodemaker: implement immutable directories (internal interface), for #607 Brian Warner **20091112002233 Ignore-this: d09fccf41813fdf7e0db177ed9e5e130 * nodemaker.create_from_cap() now handles DIR2-CHK and DIR2-LIT * client.create_immutable_dirnode() is used to create them * no webapi yet ] [dirnode.py: move pack_children() out to a function, for eventual use by others Brian Warner **20091017180707 Ignore-this: 6a823fb61f2c180fd38d6742d3196a7a ] [dirnode.py/_encrypt_rwcap: rename IV to "salt", which is more accurate Brian Warner **20090712235025 Ignore-this: 1b8d6a4e8667655f52abe2b3be46a0ed ] [dirnode.py: security bug: also use child writecap to derive child enc key, Brian Warner **20090712234750 Ignore-this: 13867ebc123b521df60e4013b75716e not just the dirnode writecap. The previous code (which only hashed the dirnode writecap) would use the same key for all children, which is very bad. This is the correct implementation of #750. ] [directories: make the IV for the writecaps in directory entries be computed from the secure hash of the writecap itself zooko@zooko.com**20090705024815 Ignore-this: cb9cc29f8f0687f2545e95d5b7b42d44 This makes encoding of directory entries deterministic, and it is also a tad faster on Macbook Pro than getting a random IV with os.urandom(16). ] [move dirnode.CachingDict to dictutil.AuxValueDict, generalize method names, Brian Warner **20091017180005 Ignore-this: b086933cf429df0fcea16a308d2640dd improve tests. Let dirnode _pack_children accept either dict or AuxValueDict. ] [Add CachingDict dict subclass to dirnode.py kevan@isnotajoke.com**20090705212345 Ignore-this: 484bdcecbc1ae25e04bf659abcfcf834 ] [stop using IURI()/etc as an adapter Brian Warner **20091111224542 Ignore-this: 9611da7ea6a4696de2a3b8c08776e6e0 ] [upload: fix #758 recursion-loop in peer-selection when servers report errors. Brian Warner **20090717050709 Ignore-this: 9c28ef13649c4475ede23815b69e51fd The bug was in the code that handles a third-or-later pass, and was previously untested. ] [hush pyflakes warner@lothar.com**20090625021809] [Split out NoSharesError, stop adding attributes to NotEnoughSharesError, change humanize_failure to include the original exception string, update tests, behave better if humanize_failure fails. warner@lothar.com**20090625021707] [test_system.py minor typo warner@allmydata.com**20070926190737] [nodemaker.create_new_mutable_directory: pack_children() in initial_contents= Brian Warner **20091020005118 Ignore-this: bd43c4eefe06fd32b7492bcb0a55d07e instead of creating an empty file and then adding the children later. This should speed up mkdir(initial_children) considerably, removing two roundtrips and an entire read-modify-write cycle, probably bringing it down to a single roundtrip. A quick test (against the volunteergrid) suggests a 30% speedup. test_dirnode: add new tests to enforce the restrictions that interfaces.py claims for create_new_mutable_directory(): no UnknownNodes, metadata dicts ] [update many dirnode interfaces to accept dict-of-nodes instead of dict-of-caps Brian Warner **20091017192829 Ignore-this: b35472285143862a856bf4b361d692f0 interfaces.py: define INodeMaker, document argument values, change create_new_mutable_directory() to take dict-of-nodes. Change dirnode.set_nodes() and dirnode.create_subdirectory() too. nodemaker.py: use INodeMaker, update create_new_mutable_directory() client.py: have create_dirnode() delegate initial_children= to nodemaker dirnode.py (Adder): take dict-of-nodes instead of list-of-nodes, which updates set_nodes() and create_subdirectory() web/common.py (convert_initial_children_json): create dict-of-nodes web/directory.py: same web/unlinked.py: same test_dirnode.py: update tests to match ] [dirnode.set_nodes: change return value: fire with self instead of None Brian Warner **20091013014546 Ignore-this: b75b3829fb53f7399693f1c1a39aacae ] [webapi: t=mkdir now accepts initial children, using the same JSON that t=json Brian Warner **20091013023444 Ignore-this: 574a46ed46af4251abf8c9580fd31ef7 emits. client.create_dirnode(initial_children=) now works. ] [replace Client.create_empty_dirnode() with create_dirnode(), in anticipation Brian Warner **20091012224506 Ignore-this: cbdaa4266ecb3c6496ffceab4f95709d of adding initial_children= argument. Includes stubbed-out initial_children= support. ] [test_web: improve test coverage of PUT DIRURL t=uri replace=false warner@allmydata.com**20081029045744] [test_web: test behavior of broken-dirnode GET, both html and json warner@lothar.com**20090307105707 Ignore-this: c0e5b45eee28959f899efa1bd189d6bd ] [The initial_children= argument to nodemaker.create_new_mutable_directory is Brian Warner **20091013031922 Ignore-this: 72e45317c21f9eb9ec3bd79bd4311f48 now enabled. ] [replace dirnode.create_empty_directory() with create_subdirectory(), which Brian Warner **20091013021520 Ignore-this: 6b57cb51bcfcc6058d0df569fdc8a9cf takes an initial_children= argument ] [test_dirnode: improve coverage of not-mutable-error a bit warner@allmydata.com**20080508235335] [CLI: modify 'tahoe manifest' and 'tahoe deep-check' to report ERROR: properly. For #590. warner@allmydata.com**20090225054415 Ignore-this: 99162f894fdd24112a869e14848c3dea ] [test_dirnode.py: add tests of initial_children= args to client.create_dirnode Brian Warner **20091017194159 Ignore-this: 2e2da28323a4d5d815466387914abc1b and nodemaker.create_new_mutable_directory ] [tests: remove obsolete test (it tests functionality that is long gone) which occasionally ERRORs now that we have more metadata (since [20090411225205-92b7f-7adfb89cb4db4ac7d28427934dea3d2c108f6476]) zooko@zooko.com**20090413023621 Ignore-this: b9f1b1225015f59ffd7e0ee1633e4098 ] [dirnode: don't check MAC on entries in dirnodes zooko@zooko.com**20081221233518 Ignore-this: efacb56d18259219c910cf5c84b17340 In an ancient version of directories, we needed a MAC on each entry. In modern times, the entire dirnode comes with a digital signature, so the MAC on each entry is redundant. With this patch, we no longer check those MACs when reading directories, but we still produce them so that older readers will accept directories that we write. ] [clean up uri-vs-cap terminology, emphasize cap instances instead of URI strings Brian Warner **20091111222619 Ignore-this: 93626385f6e7f039ada71f54feefe267 * "cap" means a python instance which encapsulates a filecap/dircap (uri.py) * "uri" means a string with a "URI:" prefix * FileNode instances are created with (and retain) a cap instance, and generate uri strings on demand * .get_cap/get_readcap/get_verifycap/get_repaircap return cap instances * .get_uri/get_readonly_uri return uri strings * add filenode.download_to_filename() for control.py, should find a better way * use MutableFileNode.init_from_cap, not .init_from_uri * directory URI instances: use get_filenode_cap, not get_filenode_uri * update/cleanup bench_dirnode.py to match, add Makefile target to run it ] [Makefile: add simple 'repl' target to start a python interpreter with a useful PYTHONPATH warner@allmydata.com**20071103053255] [filenode: add get_repair_cap(), which uses the read-write filecap for immutable files, and the verifycap for immutable files warner@allmydata.com**20090123033836] [dirnode: add get_repair_cap() warner@allmydata.com**20090123034449] [directories: in bench_dirnode.py, use a real CacheDirectoryManager instead of a fake one (because CacheDirectoryManager is a significant user of CPU and/or time) zooko@zooko.com**20090707034119 Ignore-this: 207a2dc346ca2c01dc7b341e88a0ca0a ] [Modify bench_dirnode.py to use CachingDict. kevan@isnotajoke.com**20090705223142 Ignore-this: 9ba62a16fd37ef281368715a887fd9f8 ] [immutable.Downloader: pass StorageBroker to constructor, stop being a Service Brian Warner **20090815192543 Ignore-this: af5ab12dbf75377640a670c689838479 child of the client, access with client.downloader instead of client.getServiceNamed("downloader"). The single "Downloader" instance is scheduled for demolition anyways, to be replaced by individual filenode.download calls. ] [test/common.py: update FakeMutableFileNode to new contents= callable scheme Brian Warner **20091013052154 Ignore-this: 62f00a76454a2190d1c8641c5993632f ] [Overhaul IFilesystemNode handling, to simplify tests and use POLA internally. Brian Warner **20090815112846 Ignore-this: 1db1b9c149a60a310228aba04c5c8e5f * stop using IURI as an adapter * pass cap strings around instead of URI instances * move filenode/dirnode creation duties from Client to new NodeMaker class * move other Client duties to KeyGenerator, SecretHolder, History classes * stop passing Client reference to dirnode/filenode constructors - pass less-powerful references instead, like StorageBroker or Uploader * always create DirectoryNodes by wrapping a filenode (mutable for now) * remove some specialized mock classes from unit tests Detailed list of changes (done one at a time, then merged together) always pass a string to create_node_from_uri(), not an IURI instance always pass a string to IFilesystemNode constructors, not an IURI instance stop using IURI() as an adapter, switch on cap prefix in create_node_from_uri() client.py: move SecretHolder code out to a separate class test_web.py: hush pyflakes client.py: move NodeMaker functionality out into a separate object LiteralFileNode: stop storing a Client reference immutable Checker: remove Client reference, it only needs a SecretHolder immutable Upload: remove Client reference, leave SecretHolder and StorageBroker immutable Repairer: replace Client reference with StorageBroker and SecretHolder immutable FileNode: remove Client reference mutable.Publish: stop passing Client mutable.ServermapUpdater: get StorageBroker in constructor, not by peeking into Client reference MutableChecker: reference StorageBroker and History directly, not through Client mutable.FileNode: removed unused indirection to checker classes mutable.FileNode: remove Client reference client.py: move RSA key generation into a separate class, so it can be passed to the nodemaker move create_mutable_file() into NodeMaker test_dirnode.py: stop using FakeClient mockups, use NoNetworkGrid instead. This simplifies the code, but takes longer to run (17s instead of 6s). This should come down later when other cleanups make it possible to use simpler (non-RSA) fake mutable files for dirnode tests. test_mutable.py: clean up basedir names client.py: move create_empty_dirnode() into NodeMaker dirnode.py: get rid of DirectoryNode.create remove DirectoryNode.init_from_uri, refactor NodeMaker for customization, simplify test_web's mock Client to match stop passing Client to DirectoryNode, make DirectoryNode.create_with_mutablefile the normal DirectoryNode constructor, start removing client from NodeMaker remove Client from NodeMaker move helper status into History, pass History to web.Status instead of Client test_mutable.py: fix minor typo ] [added is_uri() function to allmydata.uri robk-tahoe@allmydata.com**20080111024342] [mutable WIP: merge in patches from current trunk warner@allmydata.com**20080417200922] [mutable.py: catch errors during publish.. previously they were ignored. oops. warner@allmydata.com**20080412055102] [test_mutable.py: remove spurious Retrieve during a publish test warner@allmydata.com**20080415182038] [mutable: improve test coverage in Retrieve, when shares change after mapupdate warner@allmydata.com**20080423002514] [test_mutable.py: add tests for no-servers conditions, closes #463. warner@lothar.com**20080707191810] [dirnode: cleanup, make get_verifier() always return a URI instance, not a string warner@lothar.com**20080910083755] [trivial: remove unused imports; thanks, pyflakes zooko@zooko.com**20080925180422] [mutable: respect the new tahoe.cfg 'shares.needed' and 'shares.total' settings warner@allmydata.com**20081119200501] [trivial: whitespace zooko@zooko.com**20090118165815 Ignore-this: 6f97042f221da3ad931c6b545edc6a30 Ran "M-x whitespace-cleanup" on files that Toby's recent patch touched, even though they didn't have trailing whitespace. ] [test_dirnode: #625 run deep-check on a readonly dirnode too warner@lothar.com**20090213205337 Ignore-this: 2a18d33a7cc99c9959b7182e37b35077 ] [tests: bump up the timeout on a bunch of tests that took longer than the default timeout (120s) on François Lenny-armv5tel zooko@zooko.com**20090605031444 Ignore-this: 84d67849b1f8edc88bf7001e31b5f7f3 ] [remove upper limit on SDMF filesize kevan@isnotajoke.com**20090620213130 Ignore-this: 5bc48c7421c73827909a17e651799d0c ] [big rework of introducer client: change local API, split division of responsibilites better, remove old-code testing, improve error logging warner@lothar.com**20090623021047] [test_introducer.py: increase timeouts on poll() calls warner@allmydata.com**20080205223758] [introducer: fix bug in recent simplification caught by Brian's sharp code-reviewing eye zooko@zooko.com**20081208231634 Ignore-this: 29854954577018d658be49142177edf2 ] [#620: storage: allow mutable shares to be deleted, with a writev where new_length=0 warner@allmydata.com**20090211053756 Ignore-this: 81f79e0d72f7572bdc1c9f01bb91620 ] [storage #596: announce 'tolerates-immutable-read-overrun' to the version announcement, to indicate that a read() on an immutable share where offset+length is beyond the end of the file will return a truncated string instead of raising an exception warner@lothar.com**20090209015602 Ignore-this: cbd07102909449da55067184a63fc0d1 ] [test_introducer.py: add a test for the python2.4.0/2.4.1 bug in base64.b32decode warner@lothar.com**20090519034101] [switch to using RemoteException instead of 'wrapped' RemoteReferences. Should fix #653, the rref-EQ problem warner@lothar.com**20090522004632] [trivial: remove unused import -- thanks, pyflakes zooko@zooko.com**20081231212556 Ignore-this: a70cd39a7d633bde2bb5275dfd4d3781 ] [immutable: do not catch arbitrary exceptions/failures from the attempt to get a crypttext hash tree -- catch only ServerFailure, IntegrityCheckReject, LayoutInvalid, ShareVersionIncompatible, and DeadReferenceError zooko@zooko.com**20090108042551 Ignore-this: 35f208af1b9f8603df25ed69047360d1 Once again I inserted a bug into the code, and once again it was hidden by something catching arbitrary exception/failure and assuming that it means the server failed to provide valid data. ] [immutable/checker.py: trap ShareVersionIncompatible too. Also, use f.check warner@lothar.com**20090224041405 Ignore-this: b667e8d3192116293babcacdeed42898 instead of examining the value returned by f.trap, because the latter appears to squash exception types down into their base classes (i.e. since ShareVersionIncompatible is a subclass of LayoutInvalid, f.trap(Failure(ShareVersionIncompatible)) == LayoutInvalid). All this resulted in 'incompatible' shares being misclassified as 'corrupt'. ] [immutable/checker: wrap comments to 80cols, my laptop does not have a wide screen. No functional changes. warner@lothar.com**20090207200439 Ignore-this: ad8f03eb17b217987268f76d15fa5655 ] [servermap add-lease: fix the code that's supposed to catch remote IndexErrors, I forgot that they present as ServerFailures instead. This should stop the deluge of Incidents that occur when you do add-lease against 1.3.0 servers warner@allmydata.com**20090227070426 Ignore-this: 3d7bc87d9587b51d44b27317d9eded23 ] [immutable checker add-lease: catch remote IndexError here too warner@allmydata.com**20090227071724 Ignore-this: 94ee6064ce26409381c6451cd50fdade ] [rrefutil: add check_remote utility function warner@allmydata.com**20090227065957 Ignore-this: d859d0fae87b7e84ad3c2893350ed519 ] [util/pipeline.py: new utility class to manage size-limited work pipelines, for #392 warner@lothar.com**20090518234326] [rrefutil: add trap_remote utility and friends warner@allmydata.com**20090227065524 Ignore-this: a594050cdd9bcca073d8029819dbc35 ] [test_util: get almost full test coverage of dictutil, starting with the original pyutil tests as a base. The remaining three uncovered lines involve funny cases of ValueOrderedDict that I can't figure out how to get at warner@lothar.com**20090216023210 Ignore-this: dc1f0c6d8c003c0ade38bc8f8516b04d ] [switch all foolscap imports to use foolscap.api or foolscap.logging warner@lothar.com**20090522003823] [add in-line doc that Josh wrote as he was trying to understand this code "Zooko O'Whielacronx "**20070523221123] [add OneShotObserverList from the amdlib tree warner@lothar.com**20070308210738] [confwiz: add command line options robk-tahoe@allmydata.com**20080215014429 adds command line option parsing to the confwiz. the previous --uninstall option behaves as before, but it parsed more explicitly with the twisted usage library. added is a --server option, which controls which web site the backend script for configuration is to be found on. (it is looked for at /native_client.php on the given server) this option can be used on conjunction with --uninstall to control where the uninstall is recorded Options: -u, --uninstall record uninstall -s, --server= url of server to contact [default: https://beta.allmydata.com/] e.g. confwiz.py -s https://www-test.allmydata.com/ ] [confwiz: update to record install and uninstall events. robk-tahoe@allmydata.com**20080130225207] [confwiz: use get_config call to backend robk-tahoe@allmydata.com**20080126010132 this will write an arbitrary number of config files, instead of being restricted to just the introducer.furl, based on the response of the php backend. the get_config is passed username/password ] [updating installer for beta release secorp@allmydata.com**20080214033609] [windows installer: remove uninstall tracking, add welcome page robk-tahoe@allmydata.com**20080214001716 in justin's testing, the uninstall tracking was hanging the uninstall process (on vista) for now, until we see enough need for it to warrant more detailed testing/debugging/tweaks, I'm simply disabling the call to confwiz --uninstall also this adds a 'welcome page' to the install process. once the user has installed the windows build, then the installer will open a web browser to the 'welcome page' on the website ('/welcome_install') ] [windows: include latest windown build, winfuse and tray.exe robk-tahoe@allmydata.com**20080205001250] [windows: track uninstalls robk-tahoe@allmydata.com**20080206201249 the confwiz and the native_client backend both gained hooks to track uninstall events. however that somehow didn't make it to the uninstaller :-) ] [servermap: don't log late arrivals, and don't log DeadReferenceError at log.WEIRD warner@allmydata.com**20080827003729] [various: use util.log.err instead of twisted.log.err, so we get both Incidents and trial-test-flunking warner@lothar.com**20080920173545] [update assertutil to use twisted log instead of amdlib Log robk@allmydata.com**20061130212408] [setup: when using the foolscap "what versions are here?" feature, use allmydata.get_package_versions() instead of specifically importing allmydata, pycryptopp, and zfec zooko@zooko.com**20080923000351] [trivial: fix redefinition of name "log" in imports (pyflakes) zooko@zooko.com**20090107040829 Ignore-this: cdcf7ff84082323ebc022b186127e678 ] [setup: refactor versions-and-paths and use pkg_resources to find them zooko@zooko.com**20090119210435 Ignore-this: b368d8ede7531f1d79ee3c2c1a2cc116 Using pkg_resources is probably better if it works -- zope.interface doesn't have a __version__ attribute that we can query, but pkg_resources knows zope.interface's version number, for one thing. This code falls back to the old way -- looking at the __version__ attributes and __file__ attributes -- if the pkg_resources way doesn't answer. Note that this patch also changes the capitalization of "Nevow", "Twisted", and "pyOpenSSL", and the spelling of "allmydata-tahoe". These changes are not frivolous: they are reflecting the fact that we are naming Python packages (technically called Python "distributions") instead of Python modules (technically and confusingly called Python "packages") here. The package ("distribution") is named "allmydata-tahoe". The module ("package") is named "allmydata". ] [setup: fix missing import -- thanks, pyflakes zooko@zooko.com**20081125155528 Ignore-this: 1fc042da2882b7b2f71cde93eb234a47 ] [setup: simplify the implementation of allmydata.get_package_versions() and add "platform" which is a human-oriented summary of the underlying operating system and machine zooko@zooko.com**20080922235354] [setup: simplify parsing of python version number zooko@zooko.com**20080829000045] [setup: emit the version of python in the list of versions zooko@zooko.com**20080828220454] [remove runtime dependency upon setuptools (which crept into allmydata.get_package_versions) warner@allmydata.com**20080105025341] [add option to show version and path to the tahoe executable cgalvan@mail.utexas.edu**20090116184751] [dirnode deep_traverse: insert a turn break (fireEventually) at least once every 100 files, otherwise a CHK followed by more than 158 LITs can overflow the stack, sort of like #237. warner@allmydata.com**20090313233135 Ignore-this: 39b78faa947ed9461f2d120f6843e59f ] [Modify markup of Tahoe web pages to be more amenable to styling; some minor changes of wording. Kevin Reid **20090526232545 Ignore-this: 8845937f0df6c7ddc07abe3211428a6f ] [welcome page: add link to statistics page warner@allmydata.com**20080710003722] [wui: fix bug in which empty directory is marked as "unreadable", add test, remove exclamation point zooko@zooko.com**20090407182834 Ignore-this: 2623a7ecd9c7c46b3c984fbaddf43ad0 ] [WUI: fix display of empty directories, it threw an exception before warner@allmydata.com**20090320235809 Ignore-this: e598bb806d75411d202ba90fc251ad2b ] [web: when a dirnode can't be read, emit a regular HTML page but with the child-table and upload-forms replaced with an apologetic message. Make sure to include the 'get info' links so the user can do a filecheck warner@lothar.com**20090307105601 Ignore-this: f949d6bd58c0c2fd60fd5fa730115f3f ] [web/directory: add a link from readwrite directories to a read-only version, and fix the 'SI=xxx' header to actually use the storage index, not the writekey warner@allmydata.com**20090131013205 Ignore-this: a1f21f81e6dbf88e591085efd1a57740 ] [wui: edit some of the human-readable parts of the wui such as button labels zooko@zooko.com**20090407185459 Ignore-this: 145722f4627271ea1d43107a0c7ce0e1 (The word "parent" suggests that you can go up a directory hierarchy -- perhaps that word is vestigial.) ] [tests: bump up timeouts so that the tests can finish before timeout on Francois's little arm box zooko@zooko.com**20090608225557 Ignore-this: fb83698338b2f12546cd3e1dcb896d34 ] [tests: increase timeouts on some other tests that timed-out on Francois's arm box zooko@zooko.com**20090605143437 Ignore-this: 2903cc20d914fc074c8d7a6c47740ba6 ] [unit tests: bump up a timeout which I encountered when running on a very slow machine zooko@zooko.com**20071129204735] [tests: bump up timeout on a test that timed out on draco zooko@zooko.com**20090610044628 Ignore-this: f598b98cbae44dc947937c6ca54c10cb ] [mutable/filenode.py: set _writekey to None, rather than leaving it missing Brian Warner **20090626062022 Ignore-this: be111c37dabd6c7aa47abd7bf160926e This will at least turn the really really weird error when a repair of a readonly mutable file is attempted into a merely really weird assertion that mentions "repair currently requires a writecap". ] [remove trailing whitespace Brian Warner **20090629200358 Ignore-this: 7a3756618dcfca0a40acb4c3d15f6440 ] [More lossmodel work, on repair. Shawn Willden **20090116025648] [Loss model work (temp1) Shawn Willden **20090115030058] [Statistics module Shawn Willden **20090114021235 Added a statistics module for calculating various facets of share survival statistics. ] [util/cachedir.py: add a cache-directory manager class, which expires+deletes unused files after a while warner@allmydata.com**20081030200120] [change max filesize limit tests kevan@isnotajoke.com**20090620212822 Ignore-this: 38e7c62a308c3c93e79df4bf72f4f675 Instead of testing to see that the previous SDMF filesize limit was being obeyed, we now test to make sure that we can insert files larger than that limit. ] [repairer: raise a better exception when faced with a readonly filenode. Still Brian Warner **20090626063230 Ignore-this: a100005b973a6a57566b943073352828 produces an error, though. ] [repairer.py: wrap to 80cols. No code changes. Brian Warner **20090701000047 Ignore-this: 4a84ac95a849be0656d362882876082a ] [clean up storage_broker interface: should fix #732 warner@lothar.com**20090621235119 Ignore-this: fb93cd670e809eed2bc123142dd8d4ff ] [client.py: improve docstring warner@lothar.com**20090216231532 Ignore-this: bbaa9e3f63fdb0048e3125c4681b2d1f ] [client: add get_servers() zooko@zooko.com**20081208230400 Ignore-this: 1b9b3ff483849563342f467c39fdd15d ] [introducer: simplify get_permuted_peers() implementation and add get_peers() zooko@zooko.com**20081208225725 Ignore-this: 8299c0dc187521f34187e54c72e57dc9 ] [tests/no_network: move GET into the GridTestMixin class warner@allmydata.com**20090225003300 Ignore-this: 7779ad38c2d687ae328ba3cb6164a7a4 ] [test_repairer: change to use faster no_network.GridTestMixin, split Verifier tests into separate cases, refactor judgement funcs into shared methods warner@lothar.com**20090224041506 Ignore-this: 584ce72d6276da5edc00562793d4ee53 ] [immutable: tests: the real WRITE_LEEWAY is 35 (it was a mistake to move it from 10 to 35 earlier -- I had seen a failure in which it took 35 times as many writes as I thought were optimal, but I misread and thought it took only 20 times as many) zooko@zooko.com**20090210055348 Ignore-this: e81c34d31fe2e3fd641a284a300352cc ] [immutable: tests: sigh, raise, again the limit of how many extra writes you can do and still pass this test zooko@zooko.com**20090210020931 Ignore-this: 91faf5d6919ca27f8212efc8d19b04c5 Obviously requiring the code under test to perform within some limit isn't very meaningful if we raise the limit whenever the test goes outside of it. But I still don't want to remove the test code which measures how many writes (and, elsewhere, how many reads) a client does in order to fulfill these duties. Let this number -- now 20 -- stand as an approximation of the inefficiency of our code divided by my mental model of how many operations are actually optimal for these duties. ] [test_repairer: change Repairer to use much-faster no_network.GridTestMixin. As a side-effect, fix what I think was a bug: some of the assert-minimal-effort-expended checks were mixing write counts and allocate counts warner@lothar.com**20090223234227 Ignore-this: d58bd0a909f9939775730cda4a858cae ] [test_repairer.py: hush pyflakes: remove duplicate/shadowed function name, by using the earlier definition (which is identical) warner@allmydata.com**20090112214509] [immutable: test: add a test after attempting to repair from corruption: does a full verify run give the file a clean bill of health? If not, the you haven't successfully repaired it. zooko@zooko.com**20090210010149 Ignore-this: 43faea747e7afccaae230d50c067adc6 This will make the repairer tests more consistent -- less accidentally passing due to getting lucky. ] [immutable: tests: assert that verifier gives a clean bill of health after corruption and repair (the previous patch mistakenly did this only after deletion and repair), and also test whether deleting seven other shares and then downloading works. Also count the number of shares stored in the local filesystem. zooko@zooko.com**20090210020841 Ignore-this: ac803d0599f336c308fe74a2582e6aa4 ] [test_repairer: wrap comments to 80cols, my laptop does not have a wide screen. No functional changes. warner@lothar.com**20090207200626 Ignore-this: f539c156f3b79cfe49c7cf0fa788994e ] [immutable: tests: put shares back to their pristine condition in between each test of corrupting-and-repairing them zooko@zooko.com**20090210002956 Ignore-this: 45de680a6ac69b1845c0c74534913dec This is important, because if the repairer doesn't completely repair all kinds of corruption (as the current one doesn't), then the successive tests get messed up by assuming that the shares were uncorrupted when the test first set about to corrupt them. ] [test_repairer: disable repair-from-corruption tests until other things are improved well enough to make it useful warner@allmydata.com**20090211210159 Ignore-this: bf2c780be028f0f5556f1aed04cc29b9 ] [immutable repairer: errback any pending readers of DownUpConnectorwhen it runs out of bytes, and test that fact zooko@zooko.com**20090212021129 Ignore-this: efd1fac753dad541fe5a0f232bcbf161 ] [immutable: repairer: add a simple test to exercise the "leftover" code path, fix the bug (and rename the variable "leftover" to "extra") zooko@zooko.com**20090210181245 Ignore-this: 8426b3cd55ff38e390c4d1d5c0b87e9d ] [immutable: tighten preconditions -- you can write empty strings or read zero bytes, and add the first simple unit test of DownUpConnector zooko@zooko.com**20090210065647 Ignore-this: 81aad112bb240b23f92e38f06b4ae140 ] [add more information to NotEnoughSharesError, split out new exceptions for no-servers and no-source-of-ueb-hash warner@lothar.com**20090304013715] [upload: when using a Helper, insist that it provide protocols/helper/v1 . Related to #538. warner@allmydata.com**20081122022932] [more refactoring: move get_all_serverids() and get_nickname_for_serverid() from Client to storage_broker warner@lothar.com**20090602030750] [test/no_network.py: add a basic stats provider warner@lothar.com**20090223233937 Ignore-this: c9f3cc4eed99cfc36f68938ceff4162c ] [test/no_network: do startService on the storage servers, make it easier to customize the storage servers warner@lothar.com**20090220022254 Ignore-this: e62f328721c007e4c5ee023a6efdf66d ] [more storage_broker refactoring: downloader gets a broker instead of a client, warner@lothar.com**20090602022511 use Client.get_storage_broker() accessor instead of direct attribute access. ] [immutable/download: instrument do-you-have-block responses to investigate #732 warner@lothar.com**20090621041209] [start to factor server-connection-management into a distinct 'StorageServerFarmBroker' object, separate from the client and the introducer. This is the starting point for #467: static server selection warner@lothar.com**20090601210604] [introducer: add get_nickname_for_peerid warner@allmydata.com**20080906050700] [web checker_results: include a table of servers in permuted order, so you can see the places where new servers have been inserted warner@allmydata.com**20081205080309] [mutable publish: if we are surprised by shares that match what we would have written anyways, don't be surprised. This should fix one of the two #546 problems, in which we re-use a server and forget that we already sent them a share. warner@allmydata.com**20081210044449] [test_upload: add test of maximum-immutable-share-size, to complete the last item of #538 warner@lothar.com**20090209014127 Ignore-this: 943b9b11812ad784ec824db7a8a7aff9 ] [upload: don't use servers which can't support the share size we need. This ought to avoid #439 problems. Some day we'll have a storage server which advertises support for a larger share size. No tests yet. warner@allmydata.com**20081122022812] [test_checker: improve test coverage for checker results warner@lothar.com**20090223201943 Ignore-this: 83e173602f0f4c811a7a9893d85385df ] [mutable repairer: skip repair of readcaps instead of throwing an exception. Brian Warner **20090701011343 Ignore-this: 2c24493426cdc1db8f0e3815ee2c5f87 This should improve the behavior of #625 a bit: at least all the files will get repaired. ] [hush pyflakes with recent FileTooLarge removal warner@lothar.com**20090621231757 Ignore-this: 4231b38c7e9091b0577b07ec99ac2df0 ] [directories: make the profiling behavior of bench_dirnode.py accessible by adding '--profile' to the cmdline zooko@zooko.com**20090707033035 Ignore-this: 159c36ac1cafaa4e9a6239025ef9d57b ] [directories: update the directory benchmarks to exercise the unpack-and-repack functionality, and add optional profiling zooko@zooko.com**20090705162953 Ignore-this: 4a1b11c9b1880772c923b3c03e10770b ] [directories: make initialization of the download cache lazy zooko@zooko.com**20090708004040 Ignore-this: 3c3714ccc09ae1de811664d52211e143 If you open up a directory containing thousands of files, it currently computes the cache filename and checks for the cache file on disk immediately for each immutble file in that directory. With this patch, it delays those steps until you try to do something with an immutable file that could use the cache. ] [Add tests for CachingDict, _pack_contents, _unpack_contents kevan@isnotajoke.com**20090704034328 Ignore-this: 12f3e989244288c211ba393d3a205111 ] [Use CachingDict instead of dict in dirnode.py kevan@isnotajoke.com**20090704034301 Ignore-this: 53f12260176a5170b3599eda54f38e98 ] [Alter Adder + Adder tests to look for 'only-files' instead of 'only_files' kevan@isnotajoke.com**20090720034318 Ignore-this: 65d66133f4db6c082e716864bc273e13 ] [Add 'only_files' option to the overwrite field in Adder kevan@isnotajoke.com**20090718030010 Ignore-this: 56605f6740f692549acdd9b236ce6443 ] [Add unit tests for the Adder in dirnode.py kevan@isnotajoke.com**20090718195049 Ignore-this: 93434af3656249962cf9bc6d7ac5bc01 ] [Add tests for new PUT behavior kevan@isnotajoke.com**20090720034632 Ignore-this: a64a8e8767b4d03d87445104475b045d ] [add parser for immutable directory caps: DIR2-CHK, DIR2-LIT, DIR2-CHK-Verifier Brian Warner **20091104181351 Ignore-this: 854398cc7a75bada57fa97c367b67518 ] [rename NewDirectoryNode to DirectoryNode, NewDirectoryURI to DirectoryURI Brian Warner **20090717221549 Ignore-this: 5e6226e8d9fba824bf45f67ea5821e0e ] [interfaces: remove spurious line that counted against the figleaf coverage warner@allmydata.com**20080206224126] [rename "get_verifier()" to "get_verify_cap()" zooko@zooko.com**20081208184411 Ignore-this: 3ea4d7a78c802b23f628a37cc643c11a ] [test_dirnode.py: convert Deleter to new no-network gridtest warner@lothar.com**20090216232348 Ignore-this: 8041739442ec4db726675e48f9775ae9 ] [mutable.modify(): after UCWE, publish even if the second invocation of the modifier didn't modify anything. For #551. warner@allmydata.com**20081206044923] [test/benchmark: benchmark the time to pack and unpack dirnodes zooko@zooko.com**20090704224300 Ignore-this: cd8f6a6ded44a3f6f102f9cd0b60ca62 See also the prof_benchmarks() function in this file which will run the benchmarks under profiling. ] [Allow tests to pass with -OO by turning some AssertionErrors (the ones that Brian Warner **20090715064510 Ignore-this: db08d38b720a5260b5d1dc6d6a9878c1 we actually exercise during tests) into more specific exceptions, so they don't get optimized away. The best rule to follow is probably this: if an exception is worth testing, then it's part of the API, and AssertionError should never be part of the API. Closes #749. ] [Tolerate unknown URI types in directory structures. Part of #683. Brian Warner **20090703010749 Ignore-this: afd0e15e2e39d3b87743ec7ccd87054d The idea is that future versions of Tahoe will add new URI types that this version won't recognize, but might store them in directories that we *can* read. We should handle these "objects from the future" as best we can. Previous releases of Tahoe would just explode. With this change, we'll continue to be able to work with everything else in the directory. The code change is to wrap anything we don't recognize as an UnknownNode instance (as opposed to a FileNode or DirectoryNode). Then webapi knows how to render these (mostly by leaving fields blank), deep-check knows to skip over them, deep-stats counts them in "count-unknown". You can rename and delete these things, but you can't add new ones (because we wouldn't know how to generate a readcap to put into the dirnode's rocap slot, and because this lets us catch typos better). ] [web/directory: t=manifest output=html: make the caps into clickable hrefs warner@allmydata.com**20081007201845] [web/info: don't let an unrecoverable file break the page (show ? instead of a size) warner@allmydata.com**20081107045117] [dirnode.py: dirnode.delete which hits UCWE should not fail with NoSuchChildError. Fixes #550. warner@allmydata.com**20081206040837] [test_dirnode: add an explainError call warner@allmydata.com**20081119220212] [MutableFileNode.modify: pass first_time= and servermap= to the modifier callback warner@allmydata.com**20081206040710] [test_mutable.py: test replacing a file that has one new outlier share present: closes #272 warner@allmydata.com**20080514201041] [web/directory.py: really really fix #553. Unfortunately it's tricky to simulate the behavior of a brower's relative-url handling in a unit test. warner@allmydata.com**20081206051412] [web: fix moreinfo link zooko@zooko.com**20081205212939 Ignore-this: 89913601a159437a2c151dd3652e6a94 ] [web: "More Info" link describes the same file that the "file" link points to, rather than to the file under the same name in this directory zooko@zooko.com**20081205210502 Ignore-this: 5017754e11749b376c7fa66d1acb2a58 It's a subtle but real difference. Fixes #553 -- "More Info" link should point to a file/dir, not a dir+childname . ] [webapi: pass client through constructor arguments, remove IClient, should make it easier to test web renderers in isolation warner@lothar.com**20090220181554 Ignore-this: e7848cd1bee8faf2ce7aaf040b9bf8e3 ] [webish: make /cap/ equivalent to /uri/, accepting both with the same meanings. Closes #428 warner@allmydata.com**20080603213400] [web: make nickname more visible in the welcome page, closes #361 warner@allmydata.com**20080603220210] [webish: add an extra newline to JSON output warner@lothar.com**20080915204314] [web: make t=json stats pages use text/plain, instead of leaving it at text/html warner@allmydata.com**20080726002427] [web: add /status/?t=json, with active upload/download ops. Addresses #493. warner@allmydata.com**20080726004110] [web/directory.py: slight shuffle to improve test coverage warner@allmydata.com**20081029045406] [web: test (and fix) PUT DIRURL t=uri, which replaces a directory in-place with some other cap warner@allmydata.com**20081029045446] [webapi: introducer stats: add 'announcement_distinct_hosts' to the t=json form, to show how many distinct hosts are providing e.g. storage services warner@allmydata.com**20081118213015] [rollback the feature of making "ambient upload authority" configurable zooko@zooko.com**20090121024735 Ignore-this: 3fcea1b8179e6278adc360414b527b8b This reverses some, but not all, of the changes that were committed in the following set of patches. rolling back: Sun Jan 18 09:54:30 MST 2009 toby.murray * add 'web.ambient_upload_authority' as a paramater to tahoe.cfg M ./src/allmydata/client.py -1 +3 M ./src/allmydata/test/common.py -7 +9 A ./src/allmydata/test/test_ambient_upload_authority.py M ./src/allmydata/web/root.py +12 M ./src/allmydata/webish.py -1 +4 Sun Jan 18 09:56:08 MST 2009 zooko@zooko.com * trivial: whitespace I ran emacs's "M-x whitespace-cleanup" on the files that Toby's recent patch had touched that had trailing whitespace on some lines. M ./src/allmydata/test/test_ambient_upload_authority.py -9 +8 M ./src/allmydata/web/root.py -2 +1 M ./src/allmydata/webish.py -2 +1 Mon Jan 19 14:16:19 MST 2009 zooko@zooko.com * trivial: remove unused import noticed by pyflakes M ./src/allmydata/test/test_ambient_upload_authority.py -1 Mon Jan 19 21:38:35 MST 2009 toby.murray * doc: describe web.ambient_upload_authority M ./docs/configuration.txt +14 M ./docs/frontends/webapi.txt +11 Mon Jan 19 21:38:57 MST 2009 zooko@zooko.com * doc: add Toby Murray to the CREDITS M ./CREDITS +4 ] [trivial: whitespace zooko@zooko.com**20090118165608 Ignore-this: 8539e7e73e43f459f7b82e84dface95c I ran emacs's "M-x whitespace-cleanup" on the files that Toby's recent patch had touched that had trailing whitespace on some lines. ] [trivial: remove unused import noticed by pyflakes zooko@zooko.com**20090119211619 Ignore-this: 4999f513a5c8d73ed8f79c2b012fea6b ] [doc: describe web.ambient_upload_authority toby.murray**20090120043835 Ignore-this: cc1920b2c5d4d587af84c4d251ad0e4b ] [add 'web.ambient_upload_authority' as a paramater to tahoe.cfg toby.murray**20090118165430 Ignore-this: 2c6ed484009c03fe9db1bb6eb67500ff ] [web/root.py: fix minor typo warner@lothar.com**20080707071816] [web: fix handling of reliability page when Numeric is not available warner@lothar.com**20090217015658 Ignore-this: 9d329182f1b2e5f812e5e7eb5f4cf2ed ] [webapi: modify streaming deep-manifest/deep-checker to emit an ERROR: line if they encounter an unrecoverable+untraversable directory. For #590. warner@allmydata.com**20090225051335 Ignore-this: e6bc49368fb0e7ada7cff477fd63ffe6 ] [make streaming-manifest stop doing work after the HTTP connection is dropped warner@allmydata.com**20090124013908] [test_web: add (disabled) test to see what happens when deep-check encounters an unrecoverable directory. We still need code changes to improve this behavior. warner@lothar.com**20090224214017 Ignore-this: e839f1b0ec40f53fedcd809c2a30d5f9 ] [test_deepcheck: switch deep-check tests to use no-network too. This cuts the runtime down by about 50% warner@allmydata.com**20090225030457 Ignore-this: b3a98ed18c5752c9016c047e95d42b ] [test_deepcheck: convert MutableChecker to no-network GridTest warner@allmydata.com**20090225020010 Ignore-this: eccba7fda129330b642886271a61a573 ] [dirnode.py: when doing deep-traverse, walk each directory in alphabetical order, to make things like 'manifest' more predictable warner@allmydata.com**20090313065046 Ignore-this: 9a80055a93a6b11853d4e8202bacec14 ] [expirer: clean up constructor args, add tahoe.cfg controls, use cutoff_date instead of date_cutoff warner@allmydata.com**20090319010009 Ignore-this: 2b6aaa6d5e6ff9fd417f32978b443fd2 ] [change StorageServer to take nodeid in the constructor, instead of assigning it later, since it's cleaner and because the original problem (Tubs not being ready until later) went away warner@lothar.com**20090218222301 Ignore-this: 740d582f20c93bebf60e21d9a446d3d2 ] [expirer: change setup, config options, in preparation for adding tahoe.cfg controls warner@allmydata.com**20090319002138 Ignore-this: b23a53e97f2a9fb7a005e9fe40e83fac ] [expirer: track mutable-vs-immutable sharecounts and sizes, report them on the web status page for comparison warner@allmydata.com**20090318202504 Ignore-this: 87e809bf5dedef3f0bc8f4a7b90e42d2 ] [test_storage: solaris doesn't appear to give the same block count as other platforms, so don't assert as much about 'diskbytes' recovered warner@lothar.com**20090307084518 Ignore-this: 55b35c094ce78c50c8ede42062c5ea13 ] [storage.expirer: exercise the last missing line of webstatus code warner@lothar.com**20090309033828 Ignore-this: fc4aa34734cae32eec1db623ca0b145b ] [expirer: tolerate corrupt shares, add them to the state and history for future examination warner@lothar.com**20090309030840 Ignore-this: 5ae7e68471ed700cc68beb408e0f303 ] [expirer: add mode to expire only-mutable or only-immutable shares warner@lothar.com**20090317065118 Ignore-this: b0b25427e3b1516bdfe293528b8e4a4e ] [GC: add date-cutoff -based expiration, add proposed docs warner@lothar.com**20090317051041 Ignore-this: a5c0ecbcc2666eb04f2daa67331d1948 ] [expirer: make web display a bit more consistent warner@lothar.com**20090307221442 Ignore-this: 7fec9d9bffc0bddeb51c1baa8e7ea020 ] [web/storage.py: tolerate unknown-future displays, I'm not sure why LeaseCrawler.test_unpredictable_future didn't catch this warner@lothar.com**20090307220243 Ignore-this: 3d4e5baa8cc6d5d26edcea29fda8593d ] [dirnode: add 'tahoe'/'linkcrtime' and 'tahoe'/'linkmotime' to take the place of what 'mtime'/'ctime' originally did, and make the 'tahoe' subdict be unwritable through the set_children API zooko@zooko.com**20090411225205 Ignore-this: b48b0812f353891c62f371bedb3e9880 Also add extensive documentation in docs/frontends/webapi.txt about the behaviors of these values. See ticket #628. ] [docs/dirnodes.txt: add notes on dirnode sizes warner@allmydata.com**20080213234045] [docs/dirnodes.txt: rewrite to reflect 0.7.0's RSA-based SDMF dirnodes warner@allmydata.com**20080130011358] [dirnodes.txt: minor edits warner@allmydata.com**20070703201648] [docs: add not to dirnode.txt that it is obsolete zooko@zooko.com**20080108165025] [document our current directory node (dirnode) design warner@allmydata.com**20070703003224] [docs/webapi.txt: update to discuss tahoe.cfg, not BASEDIR/webport warner@lothar.com**20081203010612] [docs/webapi.txt: update helper section to discuss tahoe.cfg warner@lothar.com**20081203010726] [docs: rename wapi.txt to webapi.txt zooko@zooko.com**20090114195348 Ignore-this: 419685f2807714bab4069fbaa3a02c1c Because Brian argues that the file contains a description of the wui as well as of the wapi, and because the name "webapi.txt" might be more obvious to the untrained eye. ] [wui/wapi: change the default port number from 8123 to 3456 to avoid conflict with TorButton zooko@zooko.com**20081125235737 Ignore-this: 47ea30bafd5917a7e1dbc88aa0190f8e See ticket #536 for details. ] [macfuse: another tahoe fuse implementation robk-tahoe@allmydata.com**20080215003510 This is the result of various experimentation done into using python-fuse to provide access to tahoe on the mac. It's rough in quite a few places, and is really the result of investigation more than a thorough implemenation of the fuse api. upon launch, it looks for the users root_dir by opening ~/.tahoe/node.url and ~/.tahoe/private/root_dir.cap it then proceeds to cache the directory structure found by walking the users tahoe drive (safely in the face of directory loops) into memory and then mounts that filesystem. when a file is read, it calls the tahoe node to first download the file into a cache directory (~/.tahoe/_cache) and then serves up the file from there. when a file is written, a temporary file is allocated within the tmp dir of the cache, and upon close() (specifically upon release()) the file is uploaded to the tahoe node, and the new directory entry written. note that while the durectory structure is cached into memory only when the filesystem is mounted, that it is 'write through' i.e. changes made via fuse are reflected into the underlying tahoe fs, even though changes made to the tahoe fs otherwise show up only upon restart. in addition to opening files for read and write, the mkdir() and rename() calls are supported. most other file system operations are not yet supported. notably stat() metadata is not currently tracked by tahoe, and is variably reported by this fs depending on write cache files. also note that this version does not fully support Finder. access through normal unix commands such as cat, cp, mv, ls etc works fine, and read access to file from within finder (including preview images and double- click to open) work ok. but copies to the tahoe drive from within finder may or may not succeed, but will always report an error. This is still under investigation. also note that this does not include any build integration. the included _fusemodule.so was built on mac os 10.4 against macfuse 1.3.0, and is known to not work against 10.5-1.3.1 it's possible it may also contain dependencies upon parts of macports used to build the python that it was built against. this will be cleaned up later. usage: python tahoefuse.py /Path/to/choice/of/mountpoint or optionally python tahoefuse.py -ovolicon=/Path/to/icon.icns /Path/to/mountpoint upon startup, tahoefuse will walk the tahoe directory, then print a summary of files and folders found, and then daemonise itself. to exit, either eject the 'drive' (note: 10.5 doesn't show it as a drive, since it considers fuse to be a connected server instead) or unmount it via umount /Path/to/mountpoint etc. ] [uri.py: get 100% test coverage, fix a few bugs in the process warner@allmydata.com**20080304202745] [docs/helper.txt: explain more about the helper warner@allmydata.com**20080506204901] [architecture.txt: fix some things that have changed a lot in recent releases warner@allmydata.com**20080214021429] [fuse: Reorganize directory tree and modify runtests.py to run against both implementations... nejucomo@gmail.com**20080607051923 Currently, fuse impl_b does not support a --basedir argument, and always uses ~/.tahoe, which makes it incompatible with these system tests. ] [Rename the unittest script for tahoe-fuse. nejucomo@gmail.com**20080119061612] [tahoe_fuse.py: system test: setup: fixed a bug in which the mointpoint was not created before mounting. nejucomo@gmail.com**20080129043913] [contrib: add a note about Armin Rigo's fuse implementation zooko@zooko.com**20080428140544] [fuse_a: Fix the expected path in runtests.py. nejucomo@gmail.com**20080531074202] [fuse_a: Remove unused webport files... nejucomo@gmail.com**20080601020351 This prevents the third client from failing to start due to a port collision with the second client. The first client, which is used for testing has a random high port written to webport, and thus does not interfere. ] [fuse_a: runtests.py: The current ubuntu python-fuse ignores the -f option and always forks, so this updates runtests to use fusermount for clean shutdown. nejucomo@gmail.com**20080601031605] [tahoe_fuse: system test: Move test summary to end of output. nejucomo@gmail.com**20080130084624] [tahoe_fuse.py: system test: Distinguish between TestFailures and unexpected exceptions during testing (and fix a typo). nejucomo@gmail.com**20080129044228] [tahoe_fuse: system test: Remove some needless comments. nejucomo@gmail.com**20080130085553] [tahoe_fuse: system test: Create a separate directory for each test and pass the cap and local path to each test. Add two basic sanity tests for empty directories. nejucomo@gmail.com**20080130085754] [tahoe_fuse.py: system test: setup: lexically sort test names, create a TestFailure class, implement an empty directory listing test. nejucomo@gmail.com**20080129044047] [Individual tests run after all the setup layers are in place. nejucomo@gmail.com**20080129042511] [fuse_a: Fix a bug in test cleanup code. nejucomo@gmail.com**20080601020541] [tahoe_fuse.py: system test: Many changes to framework... nejucomo@gmail.com**20080129042719 The flow control has been de-obfuscated a bit. Some output changes. The test framework has quite a few race conditions, but it does a reasonable job of setting up and cleaning up. ] [tahoe_fuse: system test: Manage multiple clients for test grid... System test setup is almost complete. nejucomo@gmail.com**20080121020220 This is a little convoluted because of the "layer" design, but it appears to function correctly and do properly ordered cleanup. Before system test setup is complete, tahoe_fuse.py needs to be modified to allow arbitrary client base directories. ] [tahoe_fuse: system test: factor out some cleanup code. nejucomo@gmail.com**20080120235448] [Small log output change. nejucomo@gmail.com**20080121021853] [tahoe_fuse: cmdline args & system test: Allow nonstandard client basedirs to be specified and update the system tests to use this feature... nejucomo@gmail.com**20080121025627 The commandline option handling of the version of python-fuse I use is arcane. This is an ugly hack. ] [tahoe_fuse: system test: Add FIXME comments. nejucomo@gmail.com**20080121020619] [tahoe_fuse: system test: Attempt to create a dirnode to place in /private/root_dir.cap, but this fails because the network is too small... nejucomo@gmail.com**20080121004747 This patch also factors out the "polling_operation" pattern. ] [tahoe_fuse: system test: Launch the fuse interface. nejucomo@gmail.com**20080120235551] [tahoe_fuse: system test: Copy the introducer.furl with a possible race condition due to timeout. nejucomo@gmail.com**20080120230944] [A start at adding a system test for tahoe_fuse. Incomplete... nejucomo@gmail.com**20080120225456] [The start of unit tests for tahoe_fuse.py. nejucomo@gmail.com**20080113015603] [A patch to make tahoe-fuse.py work with 0.7.0 plus a howto README. nejucomo@gmail.com**20080112230639] [User friendly error messages, and updates to use new URI formats. nejucomo@gmail.com**20080108182121 ] [tahoe-fuse: print out helpful error messages if the caller didn't give the right context zooko@zooko.com**20080108164035] [Use "my_vdrive.uri" for the root. The old "fuse-bookmarks.uri" served exactly the same purpose. nejucomo@gmail.com**20071120200001] [docs: rename frontends/webapi.txt to frontends/wapi.txt zooko@zooko.com**20090114025143 Ignore-this: c35acd8dc7c1106ca31104b6db43e5ad rename CLI.txt to frontends/CLI.txt change a few mentions of "webapi" to "wapi" fixes #582 ] [docs/using.html: update CLI section to reflect the new alias: scheme. Closes #431 warner@allmydata.com**20080603010016] [docs: change example capability zooko@zooko.com**20080219213419] [docs/logging.txt: explain tahoe/foolscap logging. Addresses #239. warner@allmydata.com**20080904002531] [webapi: serve the /static URL tree from /public_html (configurable) warner@allmydata.com**20081029223431] [fix webish unit tests by making node.url file optional robk-tahoe@allmydata.com**20080108183614] [test_web.py: localdir=/localfile= is going away, so remove the tests that exercise it warner@allmydata.com**20080519193209] [fix small bug in unit tests which caused spurious failures on Windows zooko@zooko.com**20070816211441] [test_web.py: survive localdir/localfile= names with spaces. Should close #223 warner@allmydata.com**20071212014704] [test_web: remove leftover import to hush pyflakes warner@allmydata.com**20080519212839] [NEWS: update with all user-visible changes since the last update warner@allmydata.com**20081030213604] [docs: move webapi/ftp/sftp into a new frontends/ directory warner@allmydata.com**20081105233050] [#531: implement an SFTP frontend. Mostly works, still lots of debug messages. Still needs tests and auth-by-pubkey in accounts.file warner@allmydata.com**20081105000022] [doc: use the term "filesystem" rather than "virtual drive" in CLI.txt zooko@zooko.com**20081224211614 Ignore-this: c9541955201671c1a3a8c6ca7be4e7d ] [webapi: add verifycap (spelled 'verify_url') to the t=json output on files and directories. Closes #559. warner@allmydata.com**20090204012248 Ignore-this: 7da755304f6708b4973e4a7c1bcf8a43 ] [web t=json: add 'mutable' key to the information dict warner@allmydata.com**20080520224049] [docs: fix example JSON in webapi.txt to be legal JSON. ;-) zooko@zooko.com**20080301003925] [web: fix JSON output for mutable files warner@allmydata.com**20080520221419] [tahoe_ls.py: add comment about error cases to improve warner@lothar.com**20090317051206 Ignore-this: 7c678b92a55b2f7c6f0d96fb6ece74ed ] [tahoe_ls: CLI command should return rc=0, not None warner@allmydata.com**20090203030720 Ignore-this: 18993c782cf84edc01e4accb6f8bf31a ] [cli scripts: remove the for-educational-purposes standalone clauses. Closes #261. warner@lothar.com**20080116060851] [trailing-whitespace eradication, no functional changes warner@allmydata.com**20071101222854] [trailing-whitespace eradication, no functional changes warner@allmydata.com**20071101222858] [trailing-whitespace eradication, no functional changes warner@allmydata.com**20071101222912] [docs: webapi.txt edits to explain a few things better, adjust indentation, editing zooko@zooko.com**20090411224828 Ignore-this: 3a8a5559c7f0c8a585a4c0b5a2c80451 ] [use 522-bit RSA keys in all unit tests (except one) Brian Warner **20090629223124 Ignore-this: 7a4c3685683ff9da5ceb2d8cb7b19b7 This reduces the total test time on my laptop from 400s to 283s. * src/allmydata/test/test_system.py (SystemTest.test_mutable._test_debug): Remove assertion about container_size/data_size, this changes with keysize and was too variable anyways. * src/allmydata/mutable/filenode.py (MutableFileNode.create): add keysize= * src/allmydata/dirnode.py (NewDirectoryNode.create): same * src/allmydata/client.py (Client.DEFAULT_MUTABLE_KEYSIZE): add default, this overrides the one in MutableFileNode ] [test_system: split off checker tests to test_deepcheck.py, this file is too big warner@lothar.com**20090218214234 Ignore-this: 82bf8db81dfbc98224bbf694054a8761 ] [test_system: make 'where' strings more helpful, to track down test failures better warner@allmydata.com**20081119002950] [test_system: oops, re-enable some tests that got bypassed warner@lothar.com**20080910060245] [oops, update tests to match 'tahoe stats' change warner@allmydata.com**20081119023259] [#509: test_system.py: add test for streamed-manifest warner@allmydata.com**20090123223247] [#509: remove non-streaming 'tahoe manifest' CLI form warner@allmydata.com**20090123230002] [#509 CLI: add 'tahoe manifest --stream' warner@allmydata.com**20090123223321] [cli: factor out slow-http-operation to a separate module warner@allmydata.com**20081119011113] [test_system: rearrange DeepCheckWebGood to make it easier to add CLI tests warner@allmydata.com**20090123221306] [test_system.py: fix new 'tahoe manifest' tests to not break on windows, by providing --node-directory instead of --node-url warner@allmydata.com**20081113212748] [cli: tahoe stats/manifest: change --verbose to --raw, since I want -v for --verify for check/deep-check/repair warner@allmydata.com**20081119003608] [cli: add tests for 'tahoe stats --verbose' warner@allmydata.com**20081118041114] [cli: add --verbose to 'tahoe manifest', to show the raw JSON data warner@allmydata.com**20081118040219] [create_node_from_uri: take both writecap+readcap, move logic out of dirnode.py Brian Warner **20090702222537 Ignore-this: 93051498076e90d3f1dc85161ce8247a ] [test_web: add get_permuted_peers, to unbreak recent checker_results change warner@allmydata.com**20081205081210] [dirnode deep-traversal: remove use of Limiter, stick with strict depth-first-traversal, to reduce memory usage during very large (300k+ dirnode) traversals warner@allmydata.com**20090109014116] [dirnode.build_manifest: include node.list in the limiter, that's the most important thing to slow down warner@allmydata.com**20081007201929] [dirnode.py: prepare to preserve both rwcap+rocap when copying Brian Warner **20090702211254 Ignore-this: f128c02da32f86d7e39527a35dfc2e02 This will make it easier to tolerate unknown nodes safely. ] [Add tests for #939 Kevan Carstensen **20100212062137 Ignore-this: 5459e8c64ba76cca70aa720e68549637 ] [scripts/common: fix alias handling on windows again, emit slightly nicer error message in response to an unknown alias warner@allmydata.com**20090225042136 Ignore-this: 76df800d131aed6701b5c7408105b134 ] [test_cli: exercise the recent tolerate-'c:\dir\file.txt' fix in scripts/common, recorded in a separate match to make it easier to merge the fix to prod warner@allmydata.com**20090224235620 Ignore-this: 2cae196dd4ccb578b2abae085376e0d7 ] [scripts/common: on windows, tolerate paths like 'c:\dir\file.txt', by treating single-letter aliases on windows/cygwin as non-aliases warner@allmydata.com**20090224235522 Ignore-this: 96d37644b7f81ac768ff4a1d1915eb46 ] ['tahoe stats': tolerate empty directories. Closes #693. Brian Warner **20090715075109 Ignore-this: a713325132e05d5d122111f978fe5e14 ] [cli: 'tahoe stats': add abbreviated size to the histogram. Not sure this actually improves things. warner@allmydata.com**20081119021736] [test_cli: validate non-HTML error response of 'tahoe get' on an unrecoverable file warner@lothar.com**20090304041146] [Add tests for tahoe mv behavior kevan@isnotajoke.com**20090720034609 Ignore-this: 9f20cc5c19ded743c4b129cdf16e04d9 ] [test_cli.py: assert that 'ls' on an unrecoverable file now gives a better error message warner@lothar.com**20090307110815 Ignore-this: 18e9758e4d0ca7faeaf5bd481c2d72ff ] [web/common: split out exception-to-explanation+code mapping to a separate humanize_failure() function, so it can be used by other code. Add explanation for mutable UnrecoverableFileError. warner@lothar.com**20090307105408 Ignore-this: 92f326804ba73bb446c5df5992b7d72e ] [web/common.py: use 'Accept:' header to control HTML-vs-text/plain traceback renderings warner@lothar.com**20090304035457] [web: full patch for HTML-vs-plaintext traceback renderings, improve test coverage of exception rendering warner@lothar.com**20090304035630] [storage: add a lease-checker-and-expirer crawler, plus web status page. warner@allmydata.com**20090307044517 Ignore-this: 4355224f89b959c6f1a256a7e6c88c3b This walks slowly through all shares, examining their leases, deciding which are still valid and which have expired. Once enabled, it will then remove the expired leases, and delete shares which no longer have any valid leases. Note that there is not yet a tahoe.cfg option to enable lease-deletion: the current code is read-only. A subsequent patch will add a tahoe.cfg knob to control this, as well as docs. Some other minor items included in this patch: tahoe debug dump-share has a new --leases-only flag storage sharefile/leaseinfo code is cleaned up storage web status page (/storage) has more info, more tests coverage space-left measurement on OS-X should be more accurate (it was off by 2048x) (use stat .f_frsize instead of f_bsize) ] ['tahoe debug dump-share': add --offsets, to show section offsets warner@allmydata.com**20080812214656] [storage.py: announce a maximum-immutable-share-size based upon a 'df' of the disk. Fixes #569, and this should be the last requirement for #346 (remove 12GiB filesize limit) warner@allmydata.com**20090110013736] [storage: also report space-free-for-root and space-free-for-nonroot, since that helps users understand the space-left-for-tahoe number better warner@lothar.com**20090221032856 Ignore-this: 9fdf0475f758acd98b73026677170b45 ] [stop using RuntimeError in unit tests, for #639 warner@lothar.com**20090222232722 Ignore-this: 475ce0c0dcd7a1f5ed83ef460312efea ] [test_util.py: fix problems warner@allmydata.com**20070406233622] [test_util.py: sigh, one last minor python-2.5 issue warner@allmydata.com**20070407002125] [test_util.py: fix another minor python-2.5 issue warner@allmydata.com**20070407001226] [crawler: add ETA to get_progress() warner@allmydata.com**20090227014248 Ignore-this: 27ae76c0530b83323209be70df3d8a4b ] [crawler: load state from the pickle in init, rather than waiting until startService, so get_state() can be called early warner@lothar.com**20090221035720 Ignore-this: ecd128a5f4364c0daf4b72d791340b66 ] [crawler: fix performance problems: only save state once per timeslice (not after every bucket), don't start the crawler until 5 minutes after node startup warner@lothar.com**20090221205649 Ignore-this: e6551569982bd31d19779ff15c2d6f58 ] [crawler: tolerate low-resolution system clocks (i.e. windows) warner@lothar.com**20090221061533 Ignore-this: 57286a3abcaf44f6d1a78c3c1ad547a5 ] [storage: add bucket-counting share crawler, add its output (number of files+directories maintained by a storage server) and status to the webapi /storage page warner@lothar.com**20090221030408 Ignore-this: 28761c5e076648026bc5f518506db65c ] [trivial: "M-x whitespace-cleanup", and also remove an unused variable zooko@zooko.com**20081231214233 Ignore-this: 54c33c205aa88de8655e4232d07f083e ] [crawler: provide for one-shot crawlers, which stop after their first full cycle, for share-upgraders and database-populaters warner@lothar.com**20090220211911 Ignore-this: fcdf72c5ffcafa374d376388be6fa5c5 ] [web/storage: make sure we can handle platforms without os.statvfs too warner@lothar.com**20090220220353 Ignore-this: 79d4cb8482a8543b9759dc949c86c587 ] [web: add Storage status page, improve tests warner@lothar.com**20090220202926 Ignore-this: e34d5270dcf0237fe72f573f717c7a4 ] [test_storage: fix pyflakes warnings warner@allmydata.com**20080115032648] [storage: rename the latency key names so they sort properly warner@lothar.com**20080712045102] [add 1%,10% percentiles to the storage server latency output warner@lothar.com**20080712043436] [more #514 log-webop status/cancel: add handle-expiration, test coverage warner@lothar.com**20081022051354] [#514: improve test coverage warner@lothar.com**20081022005256] [storage: include reserved_space in stats warner@lothar.com**20090220202920 Ignore-this: b5b480fe0abad0148ecad0c1fb47ecae ] [crawler: add get_progress, clean up get_state warner@lothar.com**20090221002743 Ignore-this: 9bea69f154c75b31a53425a8ea67789b ] [crawler: modify API to support upcoming bucket-counting crawler warner@lothar.com**20090220013142 Ignore-this: 808f8382837b13082f8b245db2ebee06 ] [storage: move si_b2a/si_a2b/storage_index_to_dir out of server.py and into common.py warner@lothar.com**20090221030309 Ignore-this: 645056428ab797f0b542831c82bf192a ] [crawler: use fileutil.move_info_place in preference to our own version warner@lothar.com**20090219051342 Ignore-this: ee4e46f3de965610503ba36b28184db9 ] [crawler: fix problems on windows and our slow cygwin slave warner@lothar.com**20090219042431 Ignore-this: 8019cb0da79ba00c536183a6f57b4cab ] [#633: first version of a rate-limited interruptable share-crawler warner@lothar.com**20090219034633 Ignore-this: 5d2d30c743e3b096a8e775d5a9b33601 ] [break storage.py into smaller pieces in storage/*.py . No behavioral changes. warner@lothar.com**20090218204655 Ignore-this: 312d408d1cacc5a764d791b53ebf8f91 ] [fix a few unused imports and suchlike, discovered by pyflakes zooko@zooko.com**20080213133808] [helper: add SI to logged progress messages warner@allmydata.com**20080415022653] [storage.py: leave the storage/shares/incoming/ directory in place when the bucket is closed warner@allmydata.com**20080626180757] [#538: add remote_get_version() to four main Referenceable objects: Introducer Service, Storage Server, Helper, CHK Upload Helper. Remove unused storage-server get_versions(). warner@allmydata.com**20081121234352] [storage.py: remove unused import warner@allmydata.com**20080610200544] [storage: make storage servers declare oldest supported version == 1.0, and storage clients declare oldest supported version == 1.0 zooko@zooko.com**20080730225107 See comments in patch for intended semantics. ] [storage servers announce that they will support clients as old as v0.8.0 zooko@zooko.com**20080313161011 Not that anyone pays attention to what storage servers claim about what versions they will support. ] [doc: remove notes to self that I accidentally included in a recent patch zooko@zooko.com**20090102041457 Ignore-this: d0039512dbde09811fdec48a2e00dc4 ] [trivial: a few improvements to in-line doc and code, and renaming of test/test_immutable_checker.py to test/test_immutable.py zooko@zooko.com**20090102224941 Ignore-this: 27b97a06c3edad1821f43876b4350f3 That file currently tests checker and verifier and repairer, and will soon also test downloader. ] [trivial: another place where I accidentally committed a note-to-self about the lease fields in the server-side share file zooko@zooko.com**20090103172941 Ignore-this: c23c7095ffccdf5aa033ed434b50582b ] [storage.py : replace 4294967295 with 2**32-1: python does constant folding, I measured this statement as taking 50ns, versus the 400ns for the call to min(), or the 9us required for the 'assert not os.path.exists' syscall warner@allmydata.com**20090110015222] [storage.py: explain what this large and hard-to-recognize 4294967295 number is warner@allmydata.com**20090106195721] [immutable: fix the writing of share data size into share file in case the share file is used by a < v1.3.0 storage server zooko@zooko.com**20090106182404 Ignore-this: 7d6025aba05fe8140bb712e71e89f1ba Brian noticed that the constant was wrong, and in fixing that I noticed that we should be saturating instead of modding. This code would never matter unless a server downgraded or a share migrated from Tahoe >= v1.3.0 to Tahoe < v1.3.0. Even in that case, this bug would never matter unless the share size were exactly 4,294,967,296 bytes long. Brian, for good reason, wanted this to be spelled "2**32" instead of "4294967296", but I couldn't stand to see a couple of more Python bytecodes interpreted in the middle of a core, frequent operation on the server like immutable share creation. ] [hush pyflakes by removing unused imports warner@allmydata.com**20090112214120] [storage: make add-lease work, change default ownernum=1 since 0 is reserved to mean 'no lease here' warner@allmydata.com**20090211053938 Ignore-this: 9437ec1529c34351f2725df37e0859c ] [add --add-lease to 'tahoe check', 'tahoe deep-check', and webapi. warner@lothar.com**20090218013243 Ignore-this: 176b2006cef5041adcb592ee83e084dd ] [test_dirnode.py: oops, missed a Monitor(), unbreak tests warner@lothar.com**20081022085054] [provisioning/reliability: add tests, hush pyflakes, remove dead code, fix web links warner@lothar.com**20090215222451 Ignore-this: 7854df3e0130d9388f06efd4c797262f ] [build a 'reliability' web page, with a simulation of file decay and repair over time warner@lothar.com**20090213234234 Ignore-this: 9e9623eaac7b0637bbd0071f082bd345 ] [test_upload: rewrite in terms of no-network GridTestMixin, improve no_network.py as necessary warner@lothar.com**20090216234457 Ignore-this: 80a341d5aa3036d24de98e267499d70d ] [test_web.Grid: change the CHECK() function to make it easier to test t= values with hyphens in them warner@lothar.com**20090217050034 Ignore-this: 410c08735347c2057df52f6716520228 ] [test_web: improve checker-results coverage with a no-network -based test, enhance no-network harness to assist, fix some bugs in web/check_results.py that were exposed warner@lothar.com**20090217041242 Ignore-this: fe54bb66a9ae073c002a7af51cd1e18 ] [test_download: rewrite in terms of no-network GridTestMixin, improve no_network.py as necessary warner@lothar.com**20090216233658 Ignore-this: ec2febafd2403830519120fb3f3ca04e ] [test_download: test both mutable and immutable pre-generated shares warner@lothar.com**20081203003007] [test_download.py: added 'known-answer-tests', to make sure current code can download a file that was created by earlier code warner@lothar.com**20081203002208] [tests: fix no_network framework to work with upload/download and checker warner@lothar.com**20090216231947 Ignore-this: 74b4dbd66b8384ae7c7544969fe4f744 ] [test/no_network: new test harness, like system-test but doesn't use the network so it's faster warner@lothar.com**20090216205844 Ignore-this: 31678f7bdef30b0216fd657fc6145534 ] [mutable: move recent operation history management code (MutableWatcher) into history.py, have History provide stats warner@allmydata.com**20090114233620] [mutable stats: track mutable bytes published too warner@allmydata.com**20080430012005] [web/statistics: fix typo that make immutable-download stats missing warner@allmydata.com**20080415182004] [mutable WIP: publish status doesn't know its size early enough to update the stats_provider warner@allmydata.com**20080417005517] [test_mutable: update notify_publish() to match new signature warner@allmydata.com**20080430012457] [mutable/checker: announce the mapupdate op on the 'recent uploads+downloads' page warner@lothar.com**20081023230319] [upload: move upload history into History object warner@allmydata.com**20090114224106] [immutable/download.py move recent-downloads history out of Downloader and into a separate class. upload/etc will follow soon. warner@allmydata.com**20090114221424] [webapi #590: add streaming deep-check. Still need a CLI tool to use it. warner@lothar.com**20090217053553 Ignore-this: a0edd3d2a531c48a64d8397f7e4b208c ] [test_web: improve test coverage of web.common utility code warner@allmydata.com**20080520222146] [remove unimplemented and skipped test for feature that we don't plan to implement any time soon (XML-RPC interface) zooko@zooko.com**20071213020605] [more information SkipTest for XMLRPC zooko@zooko.com**20071004180746] [#590: add webish t=stream-manifest warner@allmydata.com**20090123040136] [fileutil: add move_into_place(), to perform the standard unix trick of atomically replacing a file, with a fallback for windows warner@lothar.com**20090219051310 Ignore-this: c1d35e8ca88fcb223ea194513611c511 ] [Implement more coherent behavior when copying with dircaps/filecaps (closes #761). Patch by Kevan Carstensen. "Brian Warner "**20091130211009] [test_cli.py: modify to use the new 'no-network' gridtest instead of SystemTestMixin, which speeds it up from 73s to 43s on my system warner@lothar.com**20090216232005 Ignore-this: ec6d010c9182aa72049d1fb894cf890e ] [CLI: check for pre-existing aliases in 'tahoe create-alias' and 'tahoe add-alias' warner@lothar.com**20081203022022] [CLI: rework webopen, and moreover its tests w.r.t. path handling robk-tahoe@allmydata.com**20080924164523 in the recent reconciliation of webopen patches, I wound up adjusting webopen to 'pass through' the state of the trailing slash on the given argument to the resultant url passed to the browser. this change removes the requirement that arguments must be directories, and allows webopen to be used with files. it also broke the tests that assumed that webopen would always normalise the url to have a trailing slash. in fixing the tests, I realised that, IMHO, there's something deeply awry with the way tahoe handles paths; specifically in the combination of '/' being the name of the root path within an alias, but a leading slash on paths, e.g. 'alias:/path', is catagorically incorrect. i.e. 'tahoe:' == 'tahoe:/' == '/' but 'tahoe:/foo' is an invalid path, and must be 'tahoe:foo' I wound up making the internals of webopen simply spot a 'path' of '/' and smash it to '', which 'fixes' webopen to match the behaviour of tahoe's path handling elsewhere, but that special case sort of points to the weirdness. (fwiw, I personally found the fact that the leading / in a path was disallowed to be weird - I'm just used to seeing paths qualified by the leading / I guess - so in a debate about normalising path handling I'd vote to include the /) ] [CLI: reconcile webopen changes robk-tahoe@allmydata.com**20080924152002 I think this is largely attributable to a cleanup patch I'd made which never got committed upstream somehow, but at any rate various conflicting changes to webopen had been made. This cleans up the conflicts therein, and hopefully brings 'tahoe webopen' in line with other cli commands. ] [cli: cleanup webopen command robk-tahoe@allmydata.com**20080618201940 moved the body of webopen out of cli.py into tahoe_webopen.py made its invocation consistent with the other cli commands, most notably replacing its 'vdrive path' with the same alias parsing, allowing usage such as 'tahoe webopen private:Pictures/xti' ] [test_cli.Backup: insert some stalls to make sure two successive backups get distinct timestamps, avoiding intermittent failures warner@allmydata.com**20090211023709 Ignore-this: a146380e9adf94c2d301b59e4d23f02 ] [test_cli.backup: oops, fix test to work even when sqlite is unavailable warner@allmydata.com**20090206041042 Ignore-this: 8a550049c8eb27c34ab2440263e07593 ] [#619: make 'tahoe backup' complain and refuse to run if sqlite is unavailable and --no-backupdb is not passed warner@allmydata.com**20090211004910 Ignore-this: 46ca8be19011b0ec11e7a10cb2556386 ] [test_cli.Backup: capture stderr when sqlite is unavailable warner@lothar.com**20090207211440 Ignore-this: 4978b1a149e32bd31e66d5bc4269bc55 ] [test_cli: improve test coverage slightly warner@lothar.com**20090216030451 Ignore-this: e01ccc6a6fb44aaa4fb14fe8669e2065 ] [test for bug #534, unicode filenames francois@ctrlaltdel.ch**20081113111951 This test assure that uploading a file whose name contains unicode character doesn't prevent further uploads in the same directory. ] [tahoe_cp.py: return 0 for success, instead of None warner@allmydata.com**20090312205345 Ignore-this: 808f6d8617f8c4e7fde455e6c7639fab ] [tahoe_cp.py: improve error reporting slightly: don't json-interpret HTTP errors, pass through tahoe webapi error messages warner@lothar.com**20090307114051 Ignore-this: a8beccb67adb082a92509d439b27d68e ] [tahoe_backup.py: display warnings on errors instead of stopping the whole backup. Fix #729. francois@ctrlaltdel.ch**20100120094249 Ignore-this: 7006ea4b0910b6d29af6ab4a3997a8f9 This patch displays a warning to the user in two cases: 1. When special files like symlinks, fifos, devices, etc. are found in the local source. 2. If files or directories are not readables by the user running the 'tahoe backup' command. In verbose mode, the number of skipped files and directories is printed at the end of the backup. Exit status returned by 'tahoe backup': - 0 everything went fine - 1 the backup failed - 2 files were skipped during the backup ] [Added --exclude, --exclude-from and --exclude-vcs options to backup command. Alberto Berti **20090222170829 Ignore-this: 4912890229cd54a2f61f14f06bc4afcc It is still impossible to specify absolute exclusion path, only relative. I must check with tar or rsync how they allow them to be specified. ] [Added tests for the --exclude* options of backup command. Alberto Berti **20090222165106 Ignore-this: f1b931cf2e7929ce47b737c022bca707 ] [CLI #590: convert 'tahoe deep-check' to streaming form, improve display, add tests warner@lothar.com**20090217231511 Ignore-this: 6d88eb94b1c877eacc8c5ca7d0aac776 ] [test_cli.py: remove unused imports warner@allmydata.com**20081007004204] [cli: add 'tahoe check' and 'tahoe deep-check' commands, with primitive reporting code warner@allmydata.com**20081119011210] [deep-check-and-repair: improve results and their HTML representation warner@allmydata.com**20090113005619] [webapi deep-check: show the root as , rather than an empty path string warner@lothar.com**20081023230359] [immutable repairer zooko@zooko.com**20090112170022 Ignore-this: f17cb07b15a554b31fc5203cf4f64d81 This implements an immutable repairer by marrying a CiphertextDownloader to a CHKUploader. It extends the IDownloadTarget interface so that the downloader can provide some metadata that the uploader requires. The processing is incremental -- it uploads the first segments before it finishes downloading the whole file. This is necessary so that you can repair large files without running out of RAM or using a temporary file on the repairer. It requires only a verifycap, not a readcap. That is: it doesn't need or use the decryption key, only the integrity check codes. There are several tests marked TODO and several instances of XXX in the source code. I intend to open tickets to document further improvements to functionality and testing, but the current version is probably good enough for Tahoe-1.3.0. ] [immutable: refactor uploader to do just encoding-and-uploading, not encryption zooko@zooko.com**20090107034822 Ignore-this: 681f3ad6827a93f1431d6e3f818840a9 This makes Uploader take an EncryptedUploadable object instead of an Uploadable object. I also changed it to return a verify cap instead of a tuple of the bits of data that one finds in a verify cap. This will facilitate hooking together an Uploader and a Downloader to make a Repairer. Also move offloaded.py into src/allmydata/immutable/. ] [upload: fix up some log messages warner@allmydata.com**20080301020045] [upload.py: the 'skipping encryption' message was emitted exactly backwards warner@allmydata.com**20080129003838] [immutable: add a monitor API to CiphertextDownloader with which to tell it to stop its work zooko@zooko.com**20090108204215 Ignore-this: f96fc150fa68fc2cec46c943171a5d48 ] [naming: Rename a few things which I touched or changed in the recent patch to download-without-decrypting. zooko@zooko.com**20090108181307 Ignore-this: 495ce8d8854c5db5a09b35b856809fba Rename "downloadable" to "target". Rename "u" to "v" in FileDownloader.__init__(). Rename "_uri" to "_verifycap" in FileDownloader. Rename "_downloadable" to "_target" in FileDownloader. Rename "FileDownloader" to "CiphertextDownloader". ] [immutable: refactor download to do only download-and-decode, not decryption zooko@zooko.com**20090108175349 Ignore-this: 1e4f26f6390a67aa5714650017c4dca1 FileDownloader takes a verify cap and produces ciphertext, instead of taking a read cap and producing plaintext. FileDownloader does all integrity checking including the mandatory ciphertext hash tree and the optional ciphertext flat hash, rather than expecting its target to do some of that checking. Rename immutable.download.Output to immutable.download.DecryptingOutput. An instance of DecryptingOutput can be passed to FileDownloader to use as the latter's target. Text pushed to the DecryptingOutput is decrypted and then pushed to *its* target. DecryptingOutput satisfies the IConsumer interface, and if its target also satisfies IConsumer, then it forwards and pause/unpause signals to its producer (which is the FileDownloader). This patch also changes some logging code to use the new logging mixin class. Check integrity of a segment and decrypt the segment one block-sized buffer at a time instead of copying the buffers together into one segment-sized buffer (reduces peak memory usage, I think, and is probably a tad faster/less CPU, depending on your encoding parameters). Refactor FileDownloader so that processing of segments and of tail-segment share as much code is possible. FileDownloader and FileNode take caps as instances of URI (Python objects), not as strings. ] [immutable download: remove dead LiteralDownloader, now that we use filenodes for download warner@allmydata.com**20080716233147] [download.py: set up self._paused before registering the producer, since they might call pauseProducing right away warner@lothar.com**20080728215731] [util: log: allow empty msgs (because downloader is using the "format" alternative with no "msg" argument) zooko@zooko.com**20090107175411 Ignore-this: 832c333bf027a30a2fcf96e462297ac5 ] [immutable: define a new interface IImmutableFileURI and declare that CHKFileURI and LiteralFileURI provide it zooko@zooko.com**20090107182451 Ignore-this: 12c256a0d20655cd73739d45fff0d4d8 ] [hush pyflakes warner@allmydata.com**20071219003722] [immutable: ValidatedExtendedURIProxy computes and stores the tail data size as a convenience to its caller. zooko@zooko.com**20090108164139 Ignore-this: 75c561d73b17418775faafa60fbbd45b The "tail data size" is how many of the bytes of the tail segment are data (as opposed to padding). ] [immutable: Make more parts of download use logging mixins and know what their "parent msg id" is. zooko@zooko.com**20090108172530 Ignore-this: a4296b5f9b75933d644fd222e1fba079 ] [trivial: fix a bunch of pyflakes complaints zooko@zooko.com**20090106140054 Ignore-this: 9a515a237248a148bcf8db68f70566d4 ] [setup: we require pywin32 if building on Windows (plus some formatting and comment fixes) zooko@zooko.com**20081205231911 Ignore-this: c1d1966cfe458a6380bfd5dce09010ff ] [fix build breakage caused by auto_deps setuptools stuff robk-tahoe@allmydata.com**20080123013255 zooko recently added a runtime check, via setuptools, that specific versions of various packages were reported as available through setuptools at runtime. however exe and app builds run with collected egg contents, not linked against entire eggs, i.e. the code is transcluded into a single library.zip thus setuptools reports that those specific version cannot be reported as available, though they are in fact available built into the library this disables that runtime check if the app is running 'frozen' ] [setup: require setuptools >= 0.6c7 to run zooko@zooko.com**20081121043611 Ignore-this: e92e07c7e8edbaadcd44db7e8f4a028 ] [setup: we require setuptools > 0.6a9 in order to parse requirements that have a dot in them such as "zope.interface" zooko@zooko.com**20081120151503 Ignore-this: a6304de8f1f44defc50438d72a13e58f In the near future we might start actually relying on setuptools's pkg_resources's "require()" function to make modules importable, so we can't just skip zope.interface. ] [setup: reorder dependencies to be sort of increasing order of how much they depend on other stuff zooko@zooko.com**20081025134739 Ignore-this: 6d636aaf5deb37cbf18172824b0bbf87 Not that the order makes any different to how it gets installed, as far as I can tell. ] [setup: loosen our version requirement on zfec to require >= 1.1 instead of >= 1.3 zooko@zooko.com**20080122233538 I see that we have .deb's only for v1.1. ] [_auto_deps.py: relax our simplejson dependency to 1.4, since I think it works and because that's what feisty offers warner@allmydata.com**20080123190309] [setup: loosen requirement on simplejson from 1.7.3 to 1.7.1 zooko@zooko.com**20080123155420 Since apparently 1.7.1 is what we use on tahoecs2, and it works. ] [setup: trivial change: the name of the "Nevow" distribution is capitalized zooko@zooko.com**20080610231537] [immutable: make the web display of upload results more human-friendly, like they were before my recent change to the meaning of the "sharemap" zooko@zooko.com**20090110200209 Ignore-this: 527d067334f982cb2d3e185f72272f60 ] [immutable: redefine the "sharemap" member of the upload results to be a map from shnum to set of serverids zooko@zooko.com**20090110174623 Ignore-this: 10300a2333605bc26c4ee9c7ab7dae10 It used to be a map from shnum to a string saying "placed this share on XYZ server". The new definition is more in keeping with the "sharemap" object that results from immutable file checking and repair, and it is more useful to the repairer, which is a consumer of immutable upload results. ] [webish: add more share information to upload status, including assisted uploads warner@allmydata.com**20080306015151] [trivial: minor changes to in-line comments -- mark plaintext-hash-tree as obsolete zooko@zooko.com**20090110205601 Ignore-this: df286154e1acde469f28e9bd00bb1068 ] [immutable: separate tests of immutable upload/download from tests of immutable checking/repair zooko@zooko.com**20090110210739 Ignore-this: 9e668609d797ec86a618ed52602c111d ] [tahoe.cfg: add controls for k and N (and shares-of-happiness) warner@allmydata.com**20081118062944] [immutable: tests: verifier doesn't always catch corrupted share hashes zooko@zooko.com**20090106190449 Ignore-this: a9be83b8e2350ae9af808476015fe0e4 Maybe it already got one of the corrupted hashes from a different server and it doesn't double-check that the hash from every server is correct. Or another problem. But in any case I'm marking this as TODO because an even better (more picky) verifier is less urgent than repairer. ] [rename "checker results" to "check results", because it is more parallel to "check-and-repair results" zooko@zooko.com**20090106193703 Ignore-this: d310e3d7f42a76df68536650c996aa49 ] [mutable: rename mutable/node.py to mutable/filenode.py and mutable/repair.py to mutable/repairer.py zooko@zooko.com**20081207142008 Ignore-this: ecee635b01a21e6f866a11bb349712a3 To be more consistent with the immutable layout that I am working on. ] [mutable WIP: rename NotEnoughPeersError to NotEnoughSharesError warner@allmydata.com**20080415230832] [trivial: tiny changes to test code zooko@zooko.com**20090108172048 Ignore-this: b1a434cd40a87c3d027fef4ce609d25c ] [immutable: fix error in validation of ciphertext hash tree and add test for that code zooko@zooko.com**20090108054012 Ignore-this: 3241ce66373ebc514ae6e6f086f6daa2 pyflakes pointed out to me that I had committed some code that is untested, since it uses an undefined name. This patch exercises that code -- the validation of the ciphertext hash tree -- by corrupting some of the share files in a very specific way, and also fixes the bug. ] [download: make sure you really get all the crypttext hashes zooko@zooko.com**20090108022638 Ignore-this: c1d5ebb048e81f706b9098e26876e040 We were not making sure that we really got all the crypttext hashes during download. If a server were to return less than the complete set of crypttext hashes, then our subsequent attempt to verify the correctness of the ciphertext would fail. (And it wouldn't be obvious without very careful debugging why it had failed.) This patch makes it so that you keep trying to get ciphertext hashes until you have a full set or you run out of servers to ask. ] [immutable: new checker and verifier zooko@zooko.com**20090106002818 Ignore-this: 65441f8fdf0db8bcedeeb3fcbbd07d12 New checker and verifier use the new download class. They are robust against various sorts of failures or corruption. They return detailed results explaining what they learned about your immutable files. Some grotesque sorts of corruption are not properly handled yet, and those ones are marked as TODO or commented-out in the unit tests. There is also a repairer module in this patch with the beginnings of a repairer in it. That repairer is mostly just the interface to the outside world -- the core operation of actually reconstructing the missing data blocks and uploading them is not in there yet. This patch also refactors the unit tests in test_immutable so that the handling of each kind of corruption is reported as passing or failing separately, can be separately TODO'ified, etc. The unit tests are also improved in various ways to require more of the code under test or to stop requiring unreasonable things of it. :-) ] [immutable/checker: make log() tolerate the format= form warner@lothar.com**20080908030308] [checker: make the log() function of SimpleCHKFileVerifier compatible with the log() function of its superclasses and subclasses zooko@zooko.com**20080825214407] [immutable/filenode.py: add TODO note about the #514 monitor to check(), rather than going through the checker/verifier code and adding it, since Zooko is currently working on that code warner@lothar.com**20081022084237] [dirnode manifest/stats: process more than one LIT file per tree; we were accidentally ignoring all but the first warner@allmydata.com**20081115045049] [webapi: add 'summary' string to checker results JSON warner@allmydata.com**20081119002826] [try to tidy up uri-as-string vs. uri-as-object zooko@zooko.com**20081219143924 Ignore-this: 4280727007c29f5b3e9273a34519893f I get confused about whether a given argument or return value is a uri-as-string or uri-as-object. This patch adds a lot of assertions that it is one or the other, and also changes CheckerResults to take objects not strings. In the future, I hope that we generally use Python objects except when importing into or exporting from the Python interpreter e.g. over the wire, the UI, or a stored file. ] [web/filenode: oops, fix test failures, not everything has a storage index warner@allmydata.com**20081029011720] [web/filenode: add Accept-Ranges and ETag (for immutable files) headers to GET responses warner@allmydata.com**20081029010103] [#330: convert stats-gatherer into a .tac file service, add 'tahoe create-stats-gatherer' warner@allmydata.com**20081118074620] [dirnode manifest: add verifycaps, both to internal API and to webapi. This will give the manual-GC tools more to work with, so they can estimate how much space will be freed. warner@allmydata.com**20081124204046] [CLI: add 'tahoe stats', to run start-deep-stats and print the results warner@allmydata.com**20081114014350] [manifest: add storage-index strings to the json results warner@allmydata.com**20081119220027] [manifest: include stats in results. webapi is unchanged. warner@allmydata.com**20081119210347] [doc: sundry amendments to docs and in-line code comments zooko@zooko.com**20081228225954 Ignore-this: a38057b9bf0f00afeea1c468b2237c36 ] [NEWS: minor edits warner@allmydata.com**20081106223356] [setup: one more address to send release announcements to zooko@zooko.com**20081203015040 Ignore-this: 87cb7a9c3a1810ff0c87908548027ac5 ] [setup: another note about the process of making a tahoe release: mail to duplicity-talk@nongnu.org zooko@zooko.com**20081203014414 Ignore-this: 77ffd6f7412cdc3283c1450cfde9fdf1 ] [docs: add a note that when you make a new tahoe release, you should send the announcement to fuse-devel@lists.sourceforge.net zooko@zooko.com**20081023213658] [docs: how_to_make_a_tahoe_release.txt zooko@zooko.com**20080828202109 Just some cryptic notes to self, but if I get hit by a truck then someone else might be able to decode them. ] [immutable: stop reading past the end of the sharefile in the process of optimizing download -- Tahoe storage servers < 1.3.0 return an error if you read past the end of the share file zooko@zooko.com**20090105194057 Ignore-this: 365e1f199235a55c0354ba6cb2b05a04 ] [immutable: refactor downloader to be more reusable for checker/verifier/repairer (and better) zooko@zooko.com**20090105155145 Ignore-this: 29a22b1eb4cb530d4b69c12aa0d00740 The code for validating the share hash tree and the block hash tree has been rewritten to make sure it handles all cases, to share metadata about the file (such as the share hash tree, block hash trees, and UEB) among different share downloads, and not to require hashes to be stored on the server unnecessarily, such as the roots of the block hash trees (not needed since they are also the leaves of the share hash tree), and the root of the share hash tree (not needed since it is also included in the UEB). It also passes the latest tests including handling corrupted shares well. ValidatedReadBucketProxy takes a share_hash_tree argument to its constructor, which is a reference to a share hash tree shared by all ValidatedReadBucketProxies for that immutable file download. ValidatedReadBucketProxy requires the block_size and share_size to be provided in its constructor, and it then uses those to compute the offsets and lengths of blocks when it needs them, instead of reading those values out of the share. The user of ValidatedReadBucketProxy therefore has to have first used a ValidatedExtendedURIProxy to compute those two values from the validated contents of the URI. This is pleasingly simplifies safety analysis: the client knows which span of bytes corresponds to a given block from the validated URI data, rather than from the unvalidated data stored on the storage server. It also simplifies unit testing of verifier/repairer, because now it doesn't care about the contents of the "share size" and "block size" fields in the share. It does not relieve the need for share data v2 layout, because we still need to store and retrieve the offsets of the fields which come after the share data, therefore we still need to use share data v2 with its 8-byte fields if we want to store share data larger than about 2^32. Specify which subset of the block hashes and share hashes you need while downloading a particular share. In the future this will hopefully be used to fetch only a subset, for network efficiency, but currently all of them are fetched, regardless of which subset you specify. ReadBucketProxy hides the question of whether it has "started" or not (sent a request to the server to get metadata) from its user. Download is optimized to do as few roundtrips and as few requests as possible, hopefully speeding up download a bit. ] [download: oops, NotEnoughHashesError comes from hashtree, not hashutil warner@allmydata.com**20070418033751] [debug/test_cli: fix error handling for catalog-shares, to make the test stop failing on windows warner@allmydata.com**20081030190651] [catalog-shares command: tolerate errors, log them to stderr, handle v2-immutable shares warner@allmydata.com**20081029221010] [minor: fix unused imports -- thanks, pyflakes zooko@zooko.com**20081205190723 Ignore-this: 799f6a16360ac1aee8f6e0eb35a28a88 ] [finish renaming 'subshare' to 'block' in immutable/encode.py and in docs/ zooko@zooko.com**20081209223318 Ignore-this: 3d1b519f740c3d1030cb733f76fdae61 ] [docs: add a bunch of .svg pictures warner@allmydata.com**20070424012526] [docs/file-encoding.txt: move this over from the wiki warner@allmydata.com**20080603025827] [immutable: refactor ReadBucketProxy a little zooko@zooko.com**20081216235325 Ignore-this: b3733257769eff3b3e9625bd04643fd6 ] [debug: pass empty optional arguments to ReadBucketProxy zooko@zooko.com**20081216235145 Ignore-this: 7132cdc6a52767fbbcca03b242a16982 because those arguments are about to become non-optional (for other code than test/debug code) ] [immutable: remove the last bits of code (only test code or unused code) which did something with plaintext hashes or plaintext hash trees zooko@zooko.com**20081219141807 Ignore-this: d10d26b279794383f27fa59ec4a50219 ] [interfaces: loosen a few max-size constraints which would limit us to a mere 1.09 TB maximum file size zooko@zooko.com**20081009191357 These constraints were originally intended to protect against attacks on the storage server protocol layer which exhaust memory in the peer. However, defending against that sort of DoS is hard -- probably it isn't completely achieved -- and it costs development time to think about it, and it sometimes imposes limits on legitimate users which we don't necessarily want to impose. So, for now we forget about limiting the amount of RAM that a foolscap peer can cause you to start using. ] [interfaces: move signatures into docstrings, to reduce lines of code and improve code-coverage numbers warner@allmydata.com**20070725024321] [raise the limit on block hashes and share hashes from 30 to 2**20 "Zooko O'Whielacronx "**20070430065115] [immutable: remove unused code to produce plaintext hashes zooko@zooko.com**20081209224546 Ignore-this: 1ff9b6fa48e0617fea809998a0e3b6e ] [encode.py: also record the size, along with plaintext_hash and SI warner@allmydata.com**20080325020815] [encode: log a plaintext hash and SI for each upload. This will allow the log gatherer to correlate the two, to better measure the benefits of convergence warner@allmydata.com**20080325015537] [immutable: ValidatedExtendedURIProxy computes and stores block_size and share_size for the convenience of its users zooko@zooko.com**20090102174317 Ignore-this: 2bab64048fffc05dc6592d617aeb412f ] [immutable: fix name change from BadOrMissingShareHash to BadOrMissingHash zooko@zooko.com**20090102192709 Ignore-this: 3f22ca1ee045beabb11559512ba130f4 One of the instances of the name accidentally didn't get changed, and pyflakes noticed. The new downloader/checker/verifier/repairer unit tests would also have noticed, but those tests haven't been rolled into a patch and applied to this repo yet... ] [immutable: download.py: Raise the appropriate type of exception to indicate the cause of failure, e.g. BadOrMissingHash, ServerFailure, IntegrityCheckReject (which is a supertype of BadOrMissingHash). This helps users (such as verifier/repairer) catch certain classes of reasons for "why did this download not work". The tests of verifier/repairer test this code and rely on this code. zooko@zooko.com**20090102185858 Ignore-this: 377bf621bbb6e360a98fd287bb1593f1 ] [trivial: remove unused import (thanks, pyflakes) zooko@zooko.com**20081219194629 Ignore-this: 96e15d6de43dd1204a8933171f194189 ] [immutable: don't catch all exception when downloading, catch only DeadReferenceError and IntegrityCheckReject zooko@zooko.com**20081221234135 Ignore-this: 1abe05c3a5910378abc3920961f19aee ] [immutable: invent download.BadOrMissingHashError which is raised if either hashtree.BadHashError, hashtree.NotEnoughHashesError, and which is a subclass of IntegrityCheckReject zooko@zooko.com**20081221234130 Ignore-this: 1b04d7e9402ebfb2cd4c7648eb16af84 ] [mutable: merge renaming with test patches zooko@zooko.com**20081207144519 Ignore-this: a922a8b231090fb35b9ef84d99e9dba3 ] [rrefutil: generically wrap any errback from callRemote() in a ServerFailure instance zooko@zooko.com**20081231202830 Ignore-this: c949eaf8589ed4c3c232f17808fdce6a This facilitates client code to easily catch ServerFailures without also catching exceptions arising from client-side code. See also: http://foolscap.lothar.com/trac/ticket/105 # make it easy to distinguish server-side failures/exceptions from client-side ] [#538: fetch version and attach to the rref. Make IntroducerClient demand v1 support. warner@allmydata.com**20081122020727] [upload: if we lose the helper, go back to doing direct-to-server uploads instead of causing all subsequent uploads to fail warner@allmydata.com**20080207232659] [introducer: move the relevant interfaces out to introducer/interfaces.py warner@allmydata.com**20080619000441] [note that setting k=1 is equivalent to replication warner@allmydata.com**20070712232212] [immutable: ReadBucketProxy defines classes of exception: LayoutInvalid and its two subtypes RidiculouslyLargeURIExtensionBlock and ShareVersionIncompatible. This helps users (such as verifier/repairer) catch certain classes of reasons for "why did this download not work". This code gets exercised by the verifier/repairer unit tests, which corrupt the shares on disk in order to trigger problems like these. zooko@zooko.com**20090102181554 Ignore-this: 2288262a59ee695f524859ed4b0b39d5 ] [immutable: fix detection of truncated shares to take into account the fieldsize -- either 4 or 8 zooko@zooko.com**20090103005745 Ignore-this: 710184bd90f73dc18f3899d90ec6e972 ] [immutable: raise LayoutInvalid instead of struct.error when a share is truncated zooko@zooko.com**20090103004806 Ignore-this: 346c779045f79725965a0f2d3eea41f9 To fix this error from the Windows buildslave: [ERROR]: allmydata.test.test_immutable.Test.test_download_from_only_3_remaining_shares Traceback (most recent call last): File "C:\Documents and Settings\buildslave\windows-native-tahoe\windows\build\src\allmydata\immutable\download.py", line 135, in _bad raise NotEnoughSharesError("ran out of peers, last error was %s" % (f,)) allmydata.interfaces.NotEnoughSharesError: ran out of peers, last error was [Failure instance: Traceback: : unpack requires a string argument of length 4 c:\documents and settings\buildslave\windows-native-tahoe\windows\build\support\lib\site-packages\foolscap-0.3.2-py2.5.egg\foolscap\call.py:667:_done c:\documents and settings\buildslave\windows-native-tahoe\windows\build\support\lib\site-packages\foolscap-0.3.2-py2.5.egg\foolscap\call.py:53:complete c:\Python25\lib\site-packages\twisted\internet\defer.py:239:callback c:\Python25\lib\site-packages\twisted\internet\defer.py:304:_startRunCallbacks --- --- c:\Python25\lib\site-packages\twisted\internet\defer.py:317:_runCallbacks C:\Documents and Settings\buildslave\windows-native-tahoe\windows\build\src\allmydata\immutable\layout.py:374:_got_length C:\Python25\lib\struct.py:87:unpack ] =============================================================================== ] [storage: introduce v2 immutable shares, with 8-byte offsets fields, to remove two of the three size limitations in #346. This code handles v2 shares but does not generate them. We'll make a release with this v2-tolerance, wait a while, then make a second release that actually generates v2 shares, to avoid compatibility problems. warner@allmydata.com**20081010011327] [trivial: remove unused import (pyflakes) zooko@zooko.com**20090103182215 Ignore-this: 4a29a14fa4580460a5e61fa0aa88b9b2 ] [immutable: fix test for truncated reads of URI extension block size zooko@zooko.com**20090103174427 Ignore-this: d9ff9dfff88b4cc7aa6751ce2e9088a6 ] [immutable: more detailed tests for checker/verifier/repairer zooko@zooko.com**20081231201838 Ignore-this: dd16beef604b0917f4493bc4ef35ab74 There are a lot of different ways that a share could be corrupted, or that attempting to download it might fail. These tests attempt to exercise many of those ways and require the checker/verifier/repairer to handle each kind of failure well. ] [immutable: mark a failing download test as "todo", because I think it is revealing a limitation of the current downloader's handling of corrupted shares zooko@zooko.com**20090103190003 Ignore-this: 1d429912dda92d986e2ee366d73e088c ] [immutable: add more detailed tests of download, including testing the count of how many reads different sorts of downloads take zooko@zooko.com**20090102225459 Ignore-this: d248eb3982fdb05b43329142a723f5a1 ] [immutable: fix think-o in previous patch which caused all reads to return "", and also optimize by not opening the file when the answer is going to be "" zooko@zooko.com**20090103200245 Ignore-this: 8ac4d0b0399cd74e8a424ffbcf3d9eb9 ] [immutable: when storage server reads from immutable share, don't try to read past the end of the file (Python allocates space according to the amount of data requested, so if there is corruption and that number is huge it will do a huge memory allocation) zooko@zooko.com**20090103192222 Ignore-this: e533a65d74437676d5116369fd7c663b ] [immutable: storage servers accept any size shares now zooko@zooko.com**20081231214226 Ignore-this: 28669d591dddaff69088cba4483da61a Nathan Wilcox observed that the storage server can rely on the size of the share file combined with the count of leases to unambiguously identify the location of the leases. This means that it can hold any size share data, even though the field nominally used to hold the size of the share data is only 32 bits wide. With this patch, the storage server still writes the "size of the share data" field (just in case the server gets downgraded to an earlier version which requires that field, or the share file gets moved to another server which is of an earlier vintage), but it doesn't use it. Also, with this patch, the server no longer rejects requests to write shares which are >= 2^32 bytes in size, and it no longer rejects attempts to read such shares. This fixes http://allmydata.org/trac/tahoe/ticket/346 (increase share-size field to 8 bytes, remove 12GiB filesize limit), although there remains open a question of how clients know that a given server can handle large shares (by using the new versioning scheme, probably). Note that share size is also limited by another factor -- how big of a file we can store on the local filesystem on the server. Currently allmydata.com typically uses ext3 and I think we typically have block size = 4 KiB, which means that the largest file is about 2 TiB. Also, the hard drives themselves are only 1 TB, so the largest share is definitely slightly less than 1 TB, which means (when K == 3), the largest file is less than 3 TB. This patch also refactors the creation of new sharefiles so that only a single fopen() is used. This patch also helps with the unit-testing of repairer, since formerly it was unclear what repairer should expect to find if the "share data size" field was corrupted (some corruptions would have no effect, others would cause failure to download). Now it is clear that repairer is not required to notice if this field is corrupted since it has no effect on download. :-) ] [storage.py: improve some precondition() error messages warner@allmydata.com**20081010011425] [storage.py: assert that immutable share size will fit in the 4-byte v1 container (see #346). The struct module in py2.4 raises an error on overflow, but py2.5 merely emits a warning warner@lothar.com**20081020172208] [storage: replace sizelimit with reserved_space, make the stats 'disk_avail' number incorporate this reservation warner@lothar.com**20081201232421] [storage: change service name from 'storageserver' to 'storage' warner@allmydata.com**20080206022859] [test_cli: remove windows-worrying newlines from test data warner@lothar.com**20080802024734] [storage: include disk-free information in the stats-gatherer output warner@lothar.com**20080806210602] [storage: record latency stats in a flat dict, not nested, to conform to RIStatsProvider warner@allmydata.com**20080625002118] [#518: replace various BASEDIR/* config files with a single BASEDIR/tahoe.cfg, with backwards-compatibility of course warner@allmydata.com**20080930232149] [node.py: multi-class exception calls need parentheses warner@allmydata.com**20070601013221] [create_node.py: allow config['webport'] to be missing, for check_memory warner@lothar.com**20071011091959] [change our default HTTP port to 8123 warner@allmydata.com**20071011201723] [docs: trivial edit zooko@zooko.com**20080307003230] [docs: document the private/convergence configuration file zooko@zooko.com**20080325182241] [docs/configuration.txt: wrap to 80 cols warner@allmydata.com**20080205203512] [configuration.txt: describe helper config warner@allmydata.com**20080506225906] [run a stats provider even if there's no gatherer, since the HTTP /statistics page is then useful. Only run the once-per-second load-monitor if there is a gatherer configured warner@allmydata.com**20080508183730] [docs/configuration.txt: explain the current limitations of readonly_storage warner@allmydata.com**20080604004708] [BASEDIR/nickname is now UTF-8 encoded warner@lothar.com**20080920183713] [docs/configuration.txt: document nickname, no_storage, readonly_storage warner@allmydata.com**20080205203329] [docs: update configuration.txt to mention the private subdir and edit the description of webport zooko@zooko.com**20080108174407] [use foolscap's new app_versions API, require foolscap-0.3.1 warner@lothar.com**20080920183853] [bump foolscap dependency to 0.3.0, for the new incident-gathering interfaces warner@lothar.com**20080805235828] [setup: require secure_connections from foolscap zooko@zooko.com**20080730021041 This causes a problem on debian sid, since the pyOpenSSL v0.6 .deb doesn't come with .egg-info, so setuptools will not know that it is already installed and will try to install pyOpenSSL, and if it installs pyOpenSSL v0.7, then this will trigger the bug in Twisted v8.1.0 when used with pyOpenSSL v0.7. http://twistedmatrix.com/trac/ticket/3218 Now the comments in twisted #3218 suggest that it happens only with the select reactor, so maybe using --reactor=poll will avoid it. ] [start using Foolscap's 'incident-logging' feature, which requires foolscap-0.2.9 warner@allmydata.com**20080703004029] [remove unused import: thanks, pyflakes zooko@zooko.com**20071213020530] [node.py: remove the provoke-logport-furlfile-creation hack now that foolscap-0.2.3 does it for us, and add bridge-twisted-logs warner@lothar.com**20071224232440] [node: provoke foolscap-0.2.2 into saving logport.furl, so we can attach to it with 'flogtool dump'. Also, place it in private/, since logs are considered somewhat private warner@allmydata.com**20071219052702] [macapp: simplify node startup failure reporting robk-tahoe@allmydata.com**20080306210904 1. changed the node's exit-on-error behaviour. rather than logging debug and then delegating to self for _abort_process() instead simply delegate to self _service_startup_failed(failure) to report failures in the startup deferred chain. subclasses then have complete control of handling and reporting any failures in node startup. 2. replace the convoluted wx.PostEvent() glue for posting an event into the gui thread with the simpler expedient of wx.CallAfter() which is much like foolscap's eventually() but also thread safe for inducing a call back on the gui thread. ] [macapp: report failure of node startup to the user robk-tahoe@allmydata.com**20080306195321 in certain cases (e.g. the node.pem changed but old .furls are in private/) the node will abort upon startup. previously it used os.abort() which in these cases caused the mac gui app to crash on startup with no explanation. this changes that behaviour from calling os.abort() to calling node._abort_process(failure) which by default calls os.abort(). this allows that method to be overridden in subclasses. the mac app now provides and uses such a subclass of Client, so that failures are reported to the user in a message dialog before the process exits. this uses wx.PostEvent() with a custom event type to signal from the reactor thread into the gui thread. ] [node.py: when calling os.abort(), announce it to stdout as well as the log warner@lothar.com**20080116090132] [catch failures in startService() and abort process robk-org@allmydata.com**20070605014637] [mac: added 'mount filesystem' action to the mac gui robk-tahoe@allmydata.com**20080220005659 this adds an action to the dock menu and to the file menu (when visible) "Mount Filesystem". This action opens a windows offering the user an opportunity to select from any of the named *.cap files in their .tahoe/private directory, and choose a corresponding mount point to mount that at. it launches the .app binary as a subprocess with the corresponding command line arguments to launch the 'tahoe fuse' functionality to mount that file system. if a NAME.icns file is present in .tahoe/private alonside the chosen NAME.cap, then that icon will be used when the filesystem is mounted. this is highly unlikely to work when running from source, since it uses introspection on sys.executable to find the relavent binary to launch in order to get the right built .app's 'tahoe fuse' functionality. it is also relatively likely that the code currently checked in, hence linked into the build, will have as yet unresolved library dependencies. it's quite unlikely to work on 10.5 with macfuse 1.3.1 at the moment. ] [macapp.py: cosmetic, remove trailing whitespace warner@allmydata.com**20080128184654] [confwiz: fix mac confwiz w.r.t. recent confwiz changes robk-tahoe@allmydata.com**20080215021446] [Reworked mac gui to not lock up upon launch robk-tahoe@allmydata.com**20080125020028 Previously, once the node itself was launched, the UI event loop was no longer running. This meant that the app would sit around seemingly 'wedged' and being reported as 'Not Responding' by the os. This chnages that by actually implementing a wxPython gui which is left running while the reactor, and the node within it, is launched in another thread. Beyond 'quit' -> reactor.stop, there are no interactions between the threads. The ui provides 'open web root' and 'open account page' actions, both in the file menu, and in the (right click) dock icon menu. Something weird in the handling of wxpython's per-frame menubar stuff seems to mean that the menu bar only displays the file menu and about etc (i.e. the items from the wx menubar) if the focus changes from and back to the app while the frame the menubar belongs to is displayed. Hence a splash frame comes up at startup to provide an opportunity. It also seems that, in the case that the file menu is not available, that one can induce it to reappear by choosing 'about' from the dock menu, and then closing the about window. ] [cleanup mac and windows build code robk-tahoe@allmydata.com**20080124030641 this moves some of the code common to both windows and mac builds into the allmydata module hierarchy, and cleans up the windows and mac build directories to import the code from there. ] [update confwiz to include account creation ui robk-tahoe@allmydata.com**20080121211310 this changes the confwiz so that hitting the 'create account' button, rather than opening a webbrowser to the register page, instead provides a simple account creation ui directly, along with changes to the backend (native_client.php) to support that. also added a 'connecting...' message so the user sees a response when they hit login or create account, since the getBasedir call can sometimes take up to ~5min (which is unacceptable for a user product, but this at least somewhat ameliorates the issue of the ui locking up and not giving the user any feedback while it's happening) ] [add winfuse plugin to installer robk-tahoe@allmydata.com**20080117011535 this adds the latest build of mike's winfuse plugins, now also running as a windows service (and using the node.url, private/root_dir.cap files from the noderoot specified by the registry) into the install process. ] [add files from allmydata/web to py2exe distribution robk-tahoe@allmydata.com**20080110213446 when building the py2exe package, glob src/allmydata/web/* into web/ within the dist ] [implement a very simple, wxpython based, config wizard robk-tahoe@allmydata.com**20080112015315 This implements a very small app using a wx ui to log a user in. it takes a username and password, and submits them to a backend on the web site (currently the allmydata test net webserver) to authenticate them. It returns the 'root_cap' uri of the user's virtual drive. Also the introducer.furl is retrieved. These are then written into the default noderoot basedir in their usual files (private/root_dir.cap and introducer.furl) a button is provided which will direct the user to the web site in the event that they need to register in order to have an account to use. once the user is successfully authenticated and the files are written, then on win32 the tahoe service will be started. ] [added a small script as a stub for a config wizard robk-tahoe@allmydata.com**20080111023718 this doesn't implement any config wizard ui, but does a simple http fetch of root_cap and introducer.furl from a php backend stub. ] [more minor build tweaks for windows robk-tahoe@allmydata.com**20080115233806 tweaking version number display, and fixing a couple of small bugs ] [fix tahoe script installation logic robk-tahoe@allmydata.com**20080124000556 refine the logic in the .app which tries to install the 'tahoe' script. now it will do nothing if 'tahoe' is found anywhere on the user's path, and only if it's not present will it try to install it in each of the candidate paths (/usr/local/bin ~/bin ~/Library/bin) which are on the user's path ] [have mac app write a tahoe upon startup robk-tahoe@allmydata.com**20080123023501 upon startup, the .app will look in '/usr/local/bin', '~/bin', '~/Library/bin' if it finds one of these dirs, and can write into it, and there isn't already a 'tahoe' present, it will write a small bach script which will launch the binary contained within the .app bundle this allows the .app bundle to offer the services of the 'tahoe' script easily and simply ] [further fixes to windows build (pkg_resources / web templates) robk-tahoe@allmydata.com**20080124005243 now that web templates are found via pkg_resources, then the windows build should in fact _use_ pkg_resources, rather than exclude it from the build to prevent nevow exploding upon import due to the zip provider exception, so that the pkgreshook can do install location based lookups ] [add build dependencies to support py2exe's modulefinder robk-tahoe@allmydata.com**20080110012538 adds windows/depends.py as a container for modules which are needed at runtime but which py2exe's modulefinder dependency analysis fails to find as requisites. ] [fix nevow build prob for py2exe robk-tahoe@allmydata.com**20080110212619 nevow attempts to use pkg_resources to find the formless css file upon import, if pkg_resources is available. unfortunately using pkg_resources to find files is not supported if the files are being loaded from a zip archive (i.e. only source and egg), and further py2exe uses a zip bundle for all the code and dependent libraries. hence having both pkg_resources and nevow built into an exe causes nevow to explode upon import. this tells py2exe not to link pkg_resources into the target, so that this behaviour isn't stimulated. the side effect being that pkg_resources isn't available. ] [_auto_deps.py: per #456, don't require the 'secure_connections' feature from Foolscap, to avoid (failing) builds of pyopenssl warner@allmydata.com**20080609235504] [setup and docs: various improvements to setup and docs zooko@zooko.com**20080605205505 Remove docs/install-details.html and README.win32 for now (see #282). Remove checks for pywin32 and pyopenssl in Makefile -- that is (or will be) automated by setuptools. Remove twisted from setup_requires. This causes the problem in which Nevow doesn't declare its dependency on Twisted (#440) to yield a clear ImportError mentioning Twisted and to fail repeatedly, rather than yielding a weird ImportError and working on the second identical attempt. Fix Makefile to set PATH so that trial and twistd can be found by "make test" after Twisted was installed into support/ during "make" ] [short note about building cryptopp under cywin/native on win robk-tahoe@allmydata.com**20080107235020] [add a note to README.win32 about building cryptopp etc on cygwin robk-tahoe@allmydata.com**20080107213545] [docs: a few updates, edits, and formatting tweaks to README.win32 zooko@zooko.com**20071230113812] [README.win32: clarify layout a bit zooko@zooko.com**20071211232558] [Win32 openSSH info booker@allmydata.com**20071019180241] [README.win32: add note showing MikeB where this file is making false statements :-) zooko@zooko.com**20071211232610] [docs: edit install.html to point to about.html zooko@zooko.com**20080123140810] [docs/install-details.html: wrap to 80 cols, no content changes warner@allmydata.com**20080208035004] [install-details.html: debian 3.1 is better known as 'sarge' warner@allmydata.com**20080130230342] [docs: add note that Debian 3.1 seems to have the same problem as Ubuntu Dapper with regard to Nevow and Twisted version compatibility zooko@zooko.com**20080130182117] [docs: mention some tips of how to resolve a certain dependency on Dapper zooko@zooko.com**20080110213238] [docs: start updating install-details.html to reflect current auto-dependency and setuptools requirements, and to be better written zooko@zooko.com**20080110200337] [auto_deps: require foolscap >= 2.5 robk-tahoe@allmydata.com**20080403230646 the key_generator depends upon tub.setLocationAutomatically() which is only available in foolscap > 0.2.5 ] [bump foolscap dependency to 0.2.4, since we now use log.err warner@allmydata.com**20080205212714] [debian: update dependencies to match calc-deps.py, mainly pycryptopp-0.2.8 warner@allmydata.com**20080102211434] [debian: add Depends: on python-pycryptopp, now that it's been packaged warner@allmydata.com**20071119190450] [docs: yay! We can remove Twisted from the list of "Manual Dependencies" that users have to be aware of when installing zooko@zooko.com**20080411014407] [docs: fix anchor text of hyperlink to tarball zooko@zooko.com**20080328020129] [docs: link to the 1.0.0 tarball in docs/install.html zooko@zooko.com**20080326032229] [docs: link from install.html to the (imminent) location of allmydata-tahoe-0.9.0.tar.gz zooko@zooko.com**20080313200237] [docs: link straight to the release tar.gz in install.html zooko@zooko.com**20080214175414] [setup: fix spacing in error output messages from makefile zooko@zooko.com**20080414135221 Looking over Brian's shoulder the other day, I saw a message that said "Please seedocs/install.html". Looking at the source code of the Makefile, it seems like it should say @echo "ERROR: Not all of Tahoe's dependencies are in place. Please see \ docs/install.html for help on installing dependencies." instead of its current: @echo "ERROR: Not all of Tahoe's dependencies are in place. Please see\ docs/install.html for help on installing dependencies." But, I remember changing this more than once in the past, so either there is a different version of make somewhere which interprets trailing backslashes differently, or someone (possibly me) has repeatedly gotten confused about this issue. Anyway, this time around, I'm trying: @echo "ERROR: Not all of Tahoe's dependencies are in place. Please see docs/install.html for help on installing dependencies." Even though it is > 80 chars. ] [Makefile: figleaf2el.py needs PYTHONPATH to get allmydata.util warner@lothar.com**20070917081027] [setup: pyOpenSSL is now easy_installable, and pycryptopp now includes Crypto++, so we can remove those two from the Manual Dependencies zooko@zooko.com**20080424172917] [setup: trivial formatting change in _auto_deps.py zooko@zooko.com**20080506193056] [back our runtime setuptools dependency down to 0.6a9 . We need a newer version to build, but can handle an older version to simply run a pre-built package warner@allmydata.com**20080410233159] [setup: we now require setuptools at run-time zooko@zooko.com**20080410224610] [setup: require twisted >= 2.4.0 zooko@zooko.com**20080409011200] [setup: weaken the requirement on zope.interface from >= 3.1.0 to "any" zooko@zooko.com**20080123172604 We've never heard of a version of zope.interface that *wasn't* compatible, and there is a bug in Ubuntu's packaging of zope.interface which causes it to report its version number as 0.0.0: https://bugs.launchpad.net/zope.interface/+bug/185418 ] [setup: tiny fix to syntax in makefile zooko@zooko.com**20080507123711] [setup: remove specific checks for twisted dependency in makefile zooko@zooko.com**20080506190900 Now that the twisted dependency is handled by the automatic dependency mechanism. ] [setup: update some docs, metadata, and docstrings zooko@zooko.com**20080122162251] [setup: update doc in __init__.py zooko@zooko.com**20071222164650] [setup: simplify makefile's path manipulation now that we rely on setup.py develop zooko@zooko.com**20080326170033] [setup: fix bug in bugfix to patch to include .egg's found in CWD zooko@zooko.com**20071222171952] [Makefile: put an existing PYTHONPATH in front of our generated EGGSPATH, to make it easier to test tahoe against development versions of dependent libraries warner@lothar.com**20071224232153] [setup: whoops, fix the use of source-tree-root-dir eggs in our Makefile zooko@zooko.com**20080101075331] [setup: fix bin/tahoe to include .egg's from the source tree root dir as well zooko@zooko.com**20080101075128 This is necessary, as we can't prevent setuptools from respecting any such eggs, therefore we need to respect them in order to maintain consistency. However, we don't normally install any "install_requires" eggs into the source tree root dir. ] [bin/allmydata-tahoe: update to new src/ + support/ directories, remove instdir/bin check warner@lothar.com**20070915022428] [bin/allmydata-tahoe: also update PYTHONPATH so that child processes (like twistd) will work warner@allmydata.com**20070606183648] [setup: fix bug in previous patch to include .egg's from CWD zooko@zooko.com**20071222171427] [setup: we also need to include .egg's in the CWD in our search path, because if we install a 3rd party library into support/, and *it* installs a library that *it* requires, that one will appear in CWD zooko@zooko.com**20071222170424 It would be nice to figure out a way to force them to all appear in support/ where they belong. ] [setup: don't echo the echo of EGGSPATH zooko@zooko.com**20080307001856] [setup: for reasons that I do not understand "show-eggspath" gives me a GNUmake error unless I move it down a couple of stanzas (until after the stanza that sets PYTHONPATH) zooko@zooko.com**20080122232238] [setup: generate a unique revision number for each build zooko@zooko.com**20080311022602] [setup: bundle darcsver-1.1.1.tar zooko@zooko.com**20080311022116 I forgot that we're doing uncompressed misc/deps at the moment. We don't think this is a long-term good approach... ] [makefile - unreverted Zooko's change to setup.py, this originally was to see if it was causing the build to create a non-running installer on windows, but it wasn't the problem. secorp@allmydata.com**20080311181038] [reverting a change zooko made (2258) to see if it clears up problems in the windows build secorp@allmydata.com**20080310183749] [setup: don't install, just "develop" in setup.py in build target zooko@zooko.com**20080307002820] [setup: put back "chmod +x bin/tahoe" in the build target zooko@zooko.com**20080123224020] [setup: don't echo "signal-error" to stdout when testing for errors zooko@zooko.com**20080325184555 This patch is thanks to Paul Gerhardt. ] [setup: require specific versions of dependencies, both at run-time (if pkg_resources is available) and at build-time, and make there be only once place where we specify those versions zooko@zooko.com**20080122232433 Using pkg_resources.require() like this also apparently allows people to install multiple different versions of packages on their system and tahoe (if pkg_resources is available to it) will import the version of the package that it requires. I haven't tested this feature. ] [setup: use setuptools (if it is present) at run-time to give a specific error message on startup if a too-old version of a dependency is installed zooko@zooko.com**20080122234254] [Makefile: define TRIALCMD with '=' not ':=', to fix make-clean test. Closes #180 warner@allmydata.com**20071015220159] [setup: use the new find_exe module to find trial zooko@zooko.com**20071015185226] [Makefile: use --reactor=poll on cygwin, since select() is insufficient Brian Warner **20070914103344] [setup: generalize the kludge of finding an executable (i.e. trial or twistd) when there might be only a .py script version of it available zooko@zooko.com**20071015172504] [setup: simplify the setup by removing the "tahoe dependencies" fake project zooko@zooko.com**20080122143538 Now we use "./setup.py develop" to ensure that changes to our source code are immediately used without requiring a "make" step. This simplification will hopefully pave the way for easier py2exe and py2app, solving the "Unit tests test the installed version" bug (#145), and perhaps also #164 and #176. This patch also conditionalizes the use of setuptools_darcs on the absence of a PKG-INFO file, which is part of fixing #263. ] [doc: emphasize in our "description" field that we are under a Free Software licence czooko@zooko.com**20071016043509] [setup: fix scheme ("file:") for download base for boostrapping setuptools zooko@zooko.com**20071220221814] [setup: fix typo in name of download base for bootstrapping setuptools zooko@zooko.com**20071220221630] [setup: small tidy-up of Make rules zooko@zooko.com**20071222164631] [setup: make dependency failures more helpful (thanks to Priyanka) zooko@zooko.com**20071120060744] [Makefile: check-deps: check for pycryptopp warner@allmydata.com**20071108015046] [setup: fix formatting of error messages from makefile zooko@zooko.com**20071109191339] [setup: formatting of dependency-missing errors zooko@zooko.com**20070921214012] [upgrade to foolscap-0.2.3 warner@lothar.com**20071224232327] [setup: foolscap bundle .tar instead of .tar.gz because multiple bundled .tar's compress together much more nicely than multiple bundled .tar.gz's do. :-) zooko@zooko.com**20071222164920] [setup: remove the hack to determine if we can avoid the explicit setuptools-managed dependency on nevow (which was useful for building on dapper) zooko@zooko.com**20080110195800 For simplicity, and to avoid weird failure modes that result from importing nevow during the build process, we now simply require nevow >= 0.6.0. We currently bundle in misc/dependencies nevow v0.9.18, which will not work on Dapper, since it requires Twisted >= 2.4.0, and Dapper comes with Twisted 2.2.0. Dapper users can (a) install a newer Twisted, (b) install nevow 0.6.0 in egg form so that setuptools can tell that it is installed (without importing it), (c) beg us to start shipping nevow 0.6.0 instead of nevow 0.9.18 in our bundle. ] [setup: update the version numbers of packages that we require, add zope.interface to our requirements, make nevow >= 0.6.0 always be a requirement zooko@zooko.com**20080110195639] [docs: add require version numbers of deps to install.html, move pywin32 from install.html to install-details.html, change ref to install-details.html in install.html zooko@zooko.com**20080110193530] [quick hacks to make install-details.html viewable as html robk-tahoe@allmydata.com**20080104233415] [docs: make it so that people will stop experiencing build failure due to g++ not being installed zooko@zooko.com**20080108170329] [docs: fix hyperlinks from install.html to the Win32 and Cygwin notes zooko@zooko.com**20080108173848] [setup: shebang usr bin env python zooko@zooko.com**20080110200131] [setup: remove hard import of ez_setup -- we can proceed even if ez_setup can't be imported zooko@zooko.com**20080110200152] [setup: require setuptools >= v0.6c6 on all platforms zooko@zooko.com**20080110200213 Technically, we could get away with v0.6c5 or v0.6c4 on non-cygwin platforms, but if someone currently doesn't have setuptools >= v0.6c6 installed then our setup process will just use our bundled setuptools v0.6c7 anyway, so it will still work, and this makes the setup.py and the accompanying documentation simpler. ] [setup: fix the name of "misc/dependencies" for bootstrapping setuptools zooko@zooko.com**20071220221310] [setup: use os.path.join('misc', 'dependencies') instead of "misc/dependencies" zooko@zooko.com**20071220220717 In the hopes that this will make the boostrapping of setuptools from its bundled egg work on Windows. ] [build-deps-setup.py: import twisted early, to make setuptools on dapper use the right version warner@allmydata.com**20080111021502] [setup: fix name of setup script again zooko@zooko.com**20080112004603] [setup: fix name of setup script zooko@zooko.com**20080112004448] [setup: switch back from using "misc/dependencies/setup.py easy_install --always-unzip misc/dependencies" to using "misc/dependencies/setup.py install" zooko@zooko.com**20080112004043 because I don't fully understand the former, I suspect it of being implicated in the current buildslave redness, and we require --always-unzip solely for py2exe. ] [setup: if the build fails, make returns a failure exit code zooko@zooko.com**20080111204331] [resolve makefile conflicts robk-tahoe@allmydata.com**20080110032115 and surpress echo of echoes ] [setup: fix it to direct the user to install.html in case of build failure zooko@zooko.com**20080108163949] [setup: attempt to work-around the problem that paths might end with trailing back-slashes (on Windows) by appending a PATHSEP (i.e. ":" or ";") instead of an OSSEP (i.e. "/" or "\") zooko@zooko.com**20071004211116 I don't know what will happen if the path ends up with something like "C:\Programs and Files\Whatever\;" on Windows, and then that gets passed to cygwin bash. This reminds me of Brian's suggestion to use Python helper scripts (c.f. misc/find-dep-eggs.py) instead of writing this stuff in the GNUmake language. And *that* reminds me of the idea of writing the whole damn thing in Python instead of in GNUmake, i.e. make all of our build tools be plugins for setuptools instead of being GNUmake targets. ] [Makefile: attempt to workaround problem caused by workaround for backslashes glomming onto the following double-quote. ... zooko@zooko.com**20070921000254] [silence warning when building zooko@zooko.com**20070411155059] [Makefile: end PYTHONPATH with "." because the string might end with "\", which will cause shell to later escape whatever character comes after the string zooko@zooko.com**20070920032654] [setup: direct user to doc/install.html if the build fails zooko@zooko.com**20080107232302] [Makefile: don't run darcsver if we already have _version.py. Ought to fix building on non-darcs checkouts, and close #257 warner@allmydata.com**20080104232546] [Makefile: run 'make-version' at least once, in the 'build' target, to make sure we populate src/allmydata/_version.py warner@allmydata.com**20080103203333] [setup: "make" now defaults to "simple-build", which depends on build-deps zooko@zooko.com**20080101064430 This is for conformance with the simple new install.html. People who don't want build-deps can run "make build". ] [Makefile: add suggestion about how to use the distutils config file to select mingw32 compiler zooko@zooko.com**20070914012000] [Makefile: make it clear that it requires GNU Make warner@allmydata.com**20070711203955] [setup: we needn't depend on make-version targets because setup.py always attempts to make a version whenever it is executed zooko@zooko.com**20071004211448] [tweaks to build process to support py2exe robk-tahoe@allmydata.com**20080110010253 py2exe is unable to handle .eggs which are packaged as zip files in preference it will pull in other versions of libraries if they can be found in the environment. this changes causes .eggs to be built as .egg directories, which py2exe can handle. ] [setup: add back "build-deps" as an alias for "build-auto-deps" since I don't know how to configure the buildmaster zooko@zooko.com**20080101075802 And I am very tired. ] [setup: rename build-deps to build-auto-deps zooko@zooko.com**20080101074921] [setup: finish renaming of docs/testnet to docs/testgrid zooko@zooko.com**20080101053659] [setup: setup_requires twisted, because foolscap <= 0.2.5 imports twisted in its setup.py and because we want trial to be available at build time zooko@zooko.com**20080409183053] [setup_require pyutil >= 1.3.16, as the Windows installer builder's misc/sub-ver.py relies on it zooko@zooko.com**20080311031321] [setup: use darcsver instead of pyutil for darcsver, use setup.py plugin instead of executable for darcsver zooko@zooko.com**20080101052831 This hopefully fixes the deb builders. ] [setup: refactor ez_setup.py and setup.py to satisfy the Desert Island scenario, to find and use setuptools egg in-place in misc/dependencies, and make it setup_require pyutil (for darcsver) zooko@zooko.com**20071222164447] [setup: leave the "file:" off the front of your URLs and setuptools (v0.6c7) will treat them as not-URLs which means it will prefer them to HTTP: URLs zooko@zooko.com**20070920222912] [setup: automatically discover files to include in packages zooko@zooko.com**20071110000419 (Because they are python packages or because they are registered under darcs revision control.) ] [setup.py: don't install allmydata.Crypto.PublicKey either Brian Warner **20070816075452] [setup: setup_requires setuptools_darcs_plugin. Without it the "./setup.py sdist upload" will silently upload the wrong package contents. zooko@zooko.com**20071013203818 ] [setup.py: arg, another stupid paste error, affecting cygwin warner@allmydata.com**20071211021734] [setup.py: fix stupid cut-and-paste error warner@allmydata.com**20071211020838] [build-deps: require setuptools 0.6c4 or later, because older ones don't allow foolscap to use os.random at import time warner@allmydata.com**20071211020659] [move to foolscap-0.2.2 warner@lothar.com**20071213022145] [upgrade to foolscap-0.2.1, with a new logging framework warner@lothar.com**20071211003508] [upgrade to foolscap-0.1.7 warner@allmydata.com**20070927012451] [add foolscap tarball to misc/dependencies zooko@zooko.com**20070913215023] [setup: bundle setuptools_darcs plugin in misc/dependencies zooko@zooko.com**20071213012042] [setup: copy in the latest version of ez_setup.py, which works even if setuptools is already imported, or can be imported, into the current Python interpreter, but can't be imported into a new Python interpreter in a subprocess zooko@zooko.com**20071222052620 This actually happens in practice -- this fixes the Desert Island scenario. Thanks to PJE. ] [setup: make ez_setup.py work to upgrade setuptools even if there is already a setuptools installed which is too old zooko@zooko.com**20071013055937 This works only if setup.py is invoked as "./setup.py install" (or "python ./setup.py install" or whatever). It doesn't work if it is invoked by easy_install. On the other hand, I don't know why easy_install would execute ez_setup.py anyway -- I thought that it didn't execute the setup.py scripts. See this mailing list thread for details: http://mail.python.org/pipermail/distutils-sig/2007-October/008339.html ] [setup: patch to fix bug in our latest ez_setup.py if pkg_resources can't be imported zooko@zooko.com**20071004200920] [setup: bundle pyutil-1.3.16.tar zooko@zooko.com**20080311025818] [node.py: add BASEDIR/keepalive_timeout and BASEDIR/disconnect_timeout, to set/enable the foolscap timers, for #521 warner@allmydata.com**20080924175112] [manhole: be more tolerant of authorized_keys. files in .tahoe robk-tahoe@allmydata.com**20080925031149 both peter and I independently tried to do the same thing to eliminate the authorized_keys file which was causing problems with the broken mac build (c.f. #522) namely mv authorized_keys.8223{,.bak} but the node is, ahem, let's say 'intolerant' of the trailing .bak - rather than disable the manhole as one might expect, it instead causes the node to explode on startup. this patch makes it skip over anything that doesn't pass the 'parse this trailing stuff as an int' test. ] [storage: add remote_advise_corrupt_share, for clients to tell storage servers about share corruption that they've discovered. The server logs the report. warner@lothar.com**20081024185248] [remove unused import (thanks, pyflakes) zooko@zooko.com**20080131230059] [repairer: test all different kinds of corruption that can happen to share files on disk zooko@zooko.com**20081014230920] [storage: remove update_write_enabler method, it won't serve the desired purpose, and I have a better scheme in mind. See #489 for details warner@lothar.com**20080722002828] [repairer: fix swapped docstrings; thanks Brian zooko@zooko.com**20080925182436] [#527: expire the cached files that are used to support Range: headers, every hour, when the file is unused and older than an hour warner@allmydata.com**20081030203909] [immutable, checker, and tests: improve docstrings, assertions, tests zooko@zooko.com**20081221210752 Ignore-this: 403ed5ca120d085d582cd5695d8371f No functional changes, but remove unused code, improve or fix docstrings, etc. ] [immutable: use new logging mixins to simplify logging zooko@zooko.com**20081217000450 Ignore-this: 7d942905d1ea8f34753dbb997e1857f3 ] [download.py: make logging safe in ValidatedBucket warner@allmydata.com**20080206085034] [download: refactor handling of URI Extension Block and crypttext hash tree, simplify things zooko@zooko.com**20081205141754 Ignore-this: 51b9952ea2406b0eea60e8d72654fd99 Refactor into a class the logic of asking each server in turn until one of them gives an answer that validates. It is called ValidatedThingObtainer. Refactor the downloading and verification of the URI Extension Block into a class named ValidatedExtendedURIProxy. The new logic of validating UEBs is minimalist: it doesn't require the UEB to contain any unncessary information, but of course it still accepts such information for backwards compatibility (so that this new download code is able to download files uploaded with old, and for that matter with current, upload code). The new logic of validating UEBs follows the practice of doing all validation up front. This practice advises one to isolate the validation of incoming data into one place, so that all of the rest of the code can assume only valid data. If any redundant information is present in the UEB+URI, the new code cross-checks and asserts that it is all fully consistent. This closes some issues where the uploader could have uploaded inconsistent redundant data, which would probably have caused the old downloader to simply reject that download after getting a Python exception, but perhaps could have caused greater harm to the old downloader. I removed the notion of selecting an erasure codec from codec.py based on the string that was passed in the UEB. Currently "crs" is the only such string that works, so "_assert(codec_name == 'crs')" is simpler and more explicit. This is also in keeping with the "validate up front" strategy -- now if someone sets a different string than "crs" in their UEB, the downloader will reject the download in the "validate this UEB" function instead of in a separate "select the codec instance" function. I removed the code to check plaintext hashes and plaintext Merkle Trees. Uploaders do not produce this information any more (since it potentially exposes confidential information about the file), and the unit tests for it were disabled. The downloader before this patch would check that plaintext hash or plaintext merkle tree if they were present, but not complain if they were absent. The new downloader in this patch complains if they are present and doesn't check them. (We might in the future re-introduce such hashes over the plaintext, but encrypt the hashes which are stored in the UEB to preserve confidentiality. This would be a double- check on the correctness of our own source code -- the current Merkle Tree over the ciphertext is already sufficient to guarantee the integrity of the download unless there is a bug in our Merkle Tree or AES implementation.) This patch increases the lines-of-code count by 8 (from 17,770 to 17,778), and reduces the uncovered-by-tests lines-of-code count by 24 (from 1408 to 1384). Those numbers would be more meaningful if we omitted src/allmydata/util/ from the test-coverage statistics. ] [remove left-over early encode mechanism zooko@zooko.com**20070328070603] [encode: delay completion until all our messages have been delivered warner@lothar.com**20061203065338] [download.py: don't truncate tail segments that are the same size as all the others warner@allmydata.com**20070417203935] [hashtree.BadHashError: mention which leaf caused the problem warner@lothar.com**20070607193822] [disable plaintext hashes in shares, but leave a switch to turn it back on warner@allmydata.com**20080324203951] [immutable file download: make the ciphertext hash tree mandatory zooko@zooko.com**20080721163102 This fixes #491 (URIs do not refer to unique files in Allmydata Tahoe). Fortunately all of the versions of Tahoe currently in use are already producing this ciphertext hash tree when uploading, so there is no backwards-compatibility problem with having the downloader require it to be present. ] [download: make plaintext and ciphertext hashes in the UEB optional. warner@lothar.com**20080323214649 Removing the plaintext hashes can help with the guess-partial-information attack. This does not affect compatibility, but if and when we actually remove any hashes from the share, that will introduce a forwards-compatibility break: tahoe-0.9 will not be able to read such files. ] [upload: stop putting plaintext and ciphertext hashes in shares. warner@lothar.com*-20080323223554 This removes the guess-partial-information attack vector, and reduces the amount of overhead that we consume with each file. It also introduces a forwards-compability break: older versions of the code (before the previous download-time "make hashes optional" patch) will be unable to read files uploaded by this version, as they will complain about the missing hashes. This patch is experimental, and is being pushed into trunk to obtain test coverage. We may undo it before releasing 1.0. ] [upload: stop putting plaintext and ciphertext hashes in shares. warner@lothar.com**20080323223554 This removes the guess-partial-information attack vector, and reduces the amount of overhead that we consume with each file. It also introduces a forwards-compability break: older versions of the code (before the previous download-time "make hashes optional" patch) will be unable to read files uploaded by this version, as they will complain about the missing hashes. This patch is experimental, and is being pushed into trunk to obtain test coverage. We may undo it before releasing 1.0. ] [dump-cap: include UEB_hash in output warner@allmydata.com**20080206184819] [move netstring() and split_netstring() into a separate util.netstring module warner@allmydata.com**20080926043824] [#527: respond to GETs with early ranges quickly, without waiting for the whole file to download. Fixes the alacrity problems with the earlier code. Still needs cache expiration. warner@allmydata.com**20081029005618] [web: add 'Repair' button to checker results when they indicate unhealthyness. Also add the object's uri to the CheckerResults instance. warner@allmydata.com**20081030010917] [util: logging: refactor some common logging behavior into mixins zooko@zooko.com**20081216233807 Ignore-this: d91408bc984d1cf1fae30134f6cddb13 ] [log.py: update log.err() to take advantage of current foolscap's log.err warner@allmydata.com**20080131004723] [util.log: add levels like UNUSUAL warner@allmydata.com**20080107233245] [util: add gatherResults which is a deferred-list-like thing that doesn't wrap failures in a FirstError zooko@zooko.com**20090104165202 Ignore-this: a284fb8ab8a00a39416a67dc5d9a451e ] [util: dictutil: add DictOfSets.union(key, values) and DictOfSets.update(otherdictofsets) zooko@zooko.com**20090112165539 Ignore-this: 84fb8a2793238b077a7a71aa03ae9d2 ] [util.dictutil: move DictOfSets out to a separate utility module warner@allmydata.com**20080521164349] [test_cli: increase timeout on test_backup, since our dapper buildslave is really slow warner@lothar.com**20090206081753] [Add dirnodes to backupdb and "tahoe backup", closes #606. Brian Warner **20091126234257 Ignore-this: fa88796fcad1763c6a2bf81f56103223 * backups now share dirnodes with any previous backup, in any location, so renames and moves are handled very efficiently * "tahoe backup" no longer bothers reading the previous snapshot * if you switch grids, you should delete ~/.tahoe/private/backupdb.sqlite, to force new uploads of all files and directories ] [Raise a more explanatory exception for errors encountered during backup processing. Alberto Berti **20090222170252 Ignore-this: f6b8ffe2a903ba07a2c1c59130dac1e4 ] [Add elapsed timestamp to cli backup command final summary. Alberto Berti **20090224171425 Ignore-this: 9a042d11f95ee9f6858a5096d513c0bc ] [Two small fixes on documentation for cli backup command. Alberto Berti **20090224223634 Ignore-this: 5634a6dadad6e4e43a112de7fe5c74c ] [tahoe_backup.py: tolerate more time formats warner@allmydata.com**20090313011600 Ignore-this: ca74f56f0dce7d19810c5a7a75bc623c ] [hashutil: add constant-time comparison function, to avoid timing attacks when python's short-circuiting data-dependent == operator is used to, say, check a write-enabler warner@lothar.com**20090323032055 Ignore-this: c5c1f5e529ab1b352c949f3e0d9abf20 ] [util/abbreviate: little utility to abbreviate seconds and bytes warner@allmydata.com**20081119021142] [move testutil into test/common_util.py, since it doesn't count as 'code under test' for our pyflakes numbers warner@allmydata.com**20081029042831] [mutable WIP: clean up status handling, shrink the code a lot, improve test coverage warner@allmydata.com**20080417200222] [web-status: client methods like list_all_uploads() return Upload instances, warner@allmydata.com**20080327012007 not status instances. Fix this. The symptom was that following a link like 'up-123' that referred to an old operation (no longer in memory) while an upload was active would get an ugly traceback instead of a "no such resource" message. ] [download: fix stopProducing failure ('self._paused_at not defined'), add tests warner@allmydata.com**20080714222521] [download status: add time spent paused by the client (when we're serving over a slow HTTP link) warner@allmydata.com**20080421191917] [test_web: more test coverage warner@lothar.com**20081024001118] [interfaces.py: promote immutable.encode.NotEnoughSharesError.. it isn't just for immutable files any more warner@lothar.com**20081027203449] [encode: actually define the UploadAborted exception warner@allmydata.com**20080115032702] [storage: split WriteBucketProxy and ReadBucketProxy out into immutable/layout.py . No behavioral changes. warner@allmydata.com**20081010000800] [storage.py: turn some assertions into preconditions warner@lothar.com**20070714023048] [WriteBucketProxy: improve __repr__ warner@allmydata.com**20080129005351] [upload: abort the bucket upon any write error, and do it with callRemoteOnly to avoid double errors warner@allmydata.com**20080610185528] [storage.ReadBucketProxy: avoid double-start, this only affected tests warner@allmydata.com**20080618000129] [storage: add add_lease/update_write_enabler to remote API, revamp lease handling warner@lothar.com**20080710010655] [storage: remove unused delete_bucket() method, lease-cancellation covers it warner@lothar.com**20070902220029] [storage: always record lease expiration times as integers warner@lothar.com**20070911215331] [docs: fix a few stale comments in code zooko@zooko.com**20080507153903] [increase remote-interface size limits to 16EiB by not casually using 'int' as a constraint warner@lothar.com**20080311175031] [storage: improve stats, make them accessible via webport /statistics warner@allmydata.com**20080616233559] [stats gathering: fix storage server stats if not tracking consumed robk-tahoe@allmydata.com**20080410012306 the RIStatsProvider interface requires that counter and stat values be ChoiceOf(float, int, long) the recent changes to storage server to not track 'consumed' led to returning None as the value of a counter. this causes violations to be experienced by nodes whose stats are being gathered. this patch simply omits that stat if 'consumed' is not being tracked. ] [test_storage: add coverage for discard_storage warner@allmydata.com**20080617005240] [test_storage: add coverage for readonly_storage warner@allmydata.com**20080617005213] [test_storage.py: improve test coverage warner@allmydata.com**20080618000142] [storage: ignore shares in incoming/, to make clients use other servers during simultaneous uploads warner@allmydata.com**20080610185310] [storage: measure latency-per-operation, calculate mean/median/percentiles warner@allmydata.com**20080616222155] [storage: emit log messages on bucket allocate/read and mutable writev warner@allmydata.com**20080328003358] [don't do a du on startup if there is no size limit configured zooko@zooko.com**20080408183656 This also turns off the production of the "space measurement done" log message, if there is no size limit configured. ] [scripts/debug: split out dump_immutable_share warner@allmydata.com**20080812205517] [scripts/debug: clean up use of stdout/stderr warner@allmydata.com**20080812205242] [dump-share: tweak formatting a little bit, to make dumping multiple shares in a row easier to read warner@allmydata.com**20080206193743] [dump-share: clarify the label on the size of the original file warner@allmydata.com**20070926220059] ['tahoe dump-share': show verify-cap too warner@lothar.com**20080707211102] [debug.py: add share-overhead size info to dump-uri-extension warner@lothar.com**20070827064239] [stop using 'as' as an identifier: as with 'with', 'as' has become a reserved word in python 2.6 warner@allmydata.com**20081003002749] [dirnode lookup: use distinct NoSuchChildError instead of the generic KeyError when a child can't be found warner@lothar.com**20081027201525] [dirnode: add get_child_and_metadata_at_path warner@allmydata.com**20081003005203] [util: move PollMixin to a separate file (pollmixin.py), so testutil can be moved into test/ warner@allmydata.com**20081029041548] [client: add 'node.uptime' to the stats we collect warner@allmydata.com**20080417181339] [test_stats.py: improve test coverage warner@allmydata.com**20080430185231] [stats: add tests for CPUUsageMonitor, modify it a bit to facilitate testing warner@allmydata.com**20080430183913] [stats: add CPU-percentage monitor, with 1min/5min/15min moving-window averages, using time.clock() warner@allmydata.com**20080430011253] [introducer: add old (V1) introducer code, add test framework for compatibility testing warner@allmydata.com**20080618235834] [test_introducer.py: don't log nodeids as binary goop warner@allmydata.com**20080422195416] [repair: fix test to map from storage index to directory structure properly (thanks, cygwin buildbot, for being so kloodgey that you won't accept random binary filenames and thus making me notice this bug) zooko@zooko.com**20080926224913] [testutil.PollMixin: set default timeout (to 100s), emit a more helpful error when the timeout is hit warner@allmydata.com**20080930052309] [testutil.PollMixin: use a custom exception (and convert it) to avoid the ugly 'stash' cycle warner@allmydata.com**20080903033251] [util/time_format.py: accept space separator, add unit tests warner@allmydata.com**20081013225258] [copy pyutil.time_format into src/allmydata/util zooko@zooko.com**20070814163349 time_format() provides a good format for logging timestamps ] [add a basic concurrency limiter utility warner@allmydata.com**20080507235330] [test_util.py: get 100% test coverage for hashutil.py warner@allmydata.com**20080304204225] [trivial: fix comment zooko@zooko.com**20090413174138 Ignore-this: d45a9786c44793bc830dab1d8c9dd57c ] [Use DIR-IMM and t=mkdir-immutable for "tahoe backup", for #828 Brian Warner **20091118192813 Ignore-this: a4720529c9bc6bc8b22a3d3265925491 ] [#598: add cli+backupdb tests, improve user display, update docs, move docs out of proposed/ warner@allmydata.com**20090206040701 Ignore-this: 7a795db5573247471c6a268fb0aa23c0 ] [docs: move files that are about future plans into docs/proposed/, to clearly separate them from descriptions of the present codebase warner@allmydata.com**20080603060702] [docs/mutable-DSA.svg: add a picture of the upcoming DSA-based mutable file structure warner@allmydata.com**20080109020852] [docs/mutable-DSA.txt: update mutable.txt to reflect our proposed DSA-based mutable file scheme (#217) warner@lothar.com**20080111103058] [docs: add some accounting proposals warner@lothar.com**20080320191841] [docs/backupdb.txt: preliminary sketch of our plans for the duplicate-upload-avoidance database warner@allmydata.com**20080528232013] [tests: fix comment zooko@zooko.com**19700105101055 Ignore-this: fabedea917895568b1fca75a480111b9 ] [tests: add tahoe_cp to the list of scripts that we don't actually have tests for yet zooko@zooko.com**19700105100058 Ignore-this: ac89583992fb1b48d9a4680344569d91 ] [#598: add backupdb to 'tahoe backup' command, enable it by default warner@allmydata.com**20090206015640 Ignore-this: 4e6a158d97549c55dbc49f6d69be8c44 ] [#598: first cut of 'tahoe backup' command: no backupdb, but yes shared-unchanged-directories and Archives/TIMESTAMP and Latest/ warner@allmydata.com**20090203030902 Ignore-this: 650df5631523b63dd138978b8f3aa372 ] [CLI: make 'tahoe webopen' command accept aliases like 'tahoe ls' warner@allmydata.com**20080812012023] [test_cli.py: factor out CLITestMixin warner@lothar.com**20080802022938] [CLI: add create-alias command, to merge mkdir and add-alias into a single (secure-from-argv-snooping) step warner@lothar.com**20080802021041] [docs: CLI.txt: rewrite the way that "root directories" (now called "starting directories") and aliases are introduced zooko@zooko.com**20080611193459] ['tahoe cp -r', upon encountering a dangling symlink, would assert out. Larry Hosken **20090108055114 Ignore-this: 46e75845339faa69ffb3addb7ce74f28 This was somewhat sad; the assertion didn't say what path caused the error, what went wrong. So... silently skip over things that are neither dirs nor files. ] [CLI: remove 'tahoe admin generate-keypair', since the pycryptopp ecdsa API is about to change incompatibly. We'll undo this once pycryptopp is updated warner@allmydata.com**20081007002320] [test_cli: disable generate-keypair test on OS-X, pycryptopp still has a bug warner@lothar.com**20080919193855] [scripts/admin: split up generate_keypair code so that unit tests can use it more easily warner@allmydata.com**20081001235238] [CLI: add 'tahoe admin generate-keypair' command warner@lothar.com**20080919001133] [test_cli: more coverage for 'tahoe put' modifying a mutable file in-place, by filename, closes #441 warner@lothar.com**20080804202643] [CLI: change one-arg forms of 'tahoe put' to make an unlinked file, fix replace-mutable #441 warner@lothar.com**20080802022729] [CLI: improve docs w.r.t. aliases, add examples to 'tahoe put' and 'tahoe get' help output. Addresses part of #431 warner@allmydata.com**20080603005456] [tests: simplify CLI tests that use stdin, now that runner supports it warner@lothar.com**20080801220514] [test_cli: add system-based tests for PUT, including a mutable put that fails/todo (#441) warner@lothar.com**20080801221009] [add sqlite-based backupdb, for #598 and others (including 'tahoe cp'). Not enabled yet. warner@allmydata.com**20090206001756 Ignore-this: 36d9a56b257e481091fd1a105318cc25 ] [docs/CLI: document 'tahoe backup' warner@allmydata.com**20090206041445 Ignore-this: 60dade71212f2a65d3c0aaca7fb8ba00 ] [CLI: add 'tahoe manifest', which takes a directory and returns a list of things you can reach from it warner@allmydata.com**20081113021725] [checker: add is_recoverable() to checker results, make our stub immutable-verifier not throw an exception on unrecoverable files, add tests warner@allmydata.com**20081107043547] [deep-check: add webapi links to detailed per-file/dir results warner@lothar.com**20081023230031] [test_system: update test to match web checker results warner@lothar.com**20081023233202] [#527: support HTTP 'Range:' requests, using a cachefile. Adds filenode.read(consumer, offset, size) method. Still needs: cache expiration, reduced alacrity. warner@lothar.com**20081028204104] [web: handle PUT mutable=true properly warner@allmydata.com**20080520193602] [web: for GET save=true, don't interpret the filename= arg with any character set, just copy the bytes back into the Content-Disposition header. This seems to make it maximally compatible with Firefox and IE7 warner@allmydata.com**20080719010650] [docs: update webapi.txt with write-coordination issues, add TODO note to recovery section of mutable.txt warner@allmydata.com**20080603060321] [test_web: test that save=true filename=unicode doesn't crash warner@allmydata.com**20080719015857] [web: improve test coverage, remove some dead code warner@allmydata.com**20080520181312] [test_web: oops, actually use HEAD (instead of GET) in the HEAD test warner@allmydata.com**20080813020451] [test_web: workaround broken HEAD behavior in twisted-2.5.0 and earlier warner@allmydata.com**20080813024520] [test_system: add test coverage for immutable download.ConsumerAdapter, remove debug messages warner@allmydata.com**20081006225037] [ftp server: initial implementation. Still needs unit tests, custom Twisted patches. For #512 warner@allmydata.com**20081006195236] [immutable: refactor immutable filenodes and comparison thereof zooko@zooko.com**20080923185249 * the two kinds of immutable filenode now have a common base class * they store only an instance of their URI, not both an instance and a string * they delegate comparison to that instance ] [filenode: add is_mutable to non-MutableFileNode classes warner@allmydata.com**20080519200300] [immutable: remove unused imports (thanks, pyflakes) zooko@zooko.com**20080923192610] [more #514: pass a Monitor to all checker operations, make mutable-checker honor the cancel flag warner@lothar.com**20081022083818] [mutable: more repair tests, one with force=True to check out merging warner@lothar.com**20080806190607] [test_mutable: factor out common setup code warner@lothar.com**20080806173804] [test_mutable.py: add more tests of post-mapupdate corruption, to support #474 testing warner@allmydata.com**20080624180810] [testutil.shouldFail: mention the 'which' string in substring failures too warner@allmydata.com**20080516230838] [test_mutable: add comment about minimal-bandwidth repairer, comma lack of warner@lothar.com**20080806173850] [mutable: start adding Repair tests, fix a simple bug warner@lothar.com**20080806061239] [web: use get_size_of_best_version for HEAD requests, provide correct content-type warner@allmydata.com**20080813020410] [repairer: assert that the test code isn't accidentally allowing the repairer code which is being tested to do impossible things zooko@zooko.com**20080926222353] [repairer: enhance the repairer tests zooko@zooko.com**20080926174719 Make sure the file can actually be downloaded afterward, that it used one of the deleted and then repaired shares to do so, and that it repairs from multiple deletions at once (without using more than a reasonable amount of calls to storage server allocate). ] [repairer: remove a test that doesn't apply to the repair-from-corruption case zooko@zooko.com**20080925220954] [repairer: add a test that repairer fixes corrupted shares (in addition to the test that it fixes deleted shares) zooko@zooko.com**20080925220712] [dirnode.py: check for cancel during deep-traverse operations, and don't initiate any new ones if we've been cancelled. Gets us closer to #514. warner@lothar.com**20081022075552] [Change deep-size/stats/check/manifest to a start+poll model instead of a single long-running synchronous operation. No cancel or handle-expiration yet. #514. warner@lothar.com**20081022000307] [web: factor out identical renderHTTP methods warner@allmydata.com**20080519221925] [test/common: add ShouldFailMixin warner@lothar.com**20080806190552] [repairer: add basic test of repairer, move tests of immutable checker/repairer from test_system to test_immutable_checker, remove obsolete test helper code from test_filenode zooko@zooko.com**20080925171653 Hm... "Checker" ought to be renamed to "CheckerRepairer" or "Repairer" at some point... ] [tests: use the handy dandy TestCase.mktemp() function from trial to give unique and nicely named directories for each testcase zooko@zooko.com**20080730224920] [tests: don't use SignalMixin zooko@zooko.com**20080730223536 It seems like we no longer need it, and it screws up something internal in trial which causes trial's TestCase.mktemp() method to exhibit wrong behavior (always using a certain test method name instead of using the current test method name), and I wish to use TestCase.mktemp(). Of course, it is possible that the buildbot is about to tell me that we do still require SignalMixin on some of our platforms... ] [test_system: rename Checker to ImmutableChecker, to make room for a mutable one warner@allmydata.com**20080812225932] [setup: remove a few minimal unit tests from test_filenode which have been obviated by much better tests in test_mutable and test_system zooko@zooko.com**20080925161544] [disallow deep-check on non-directories, simplifies the code a bit warner@allmydata.com**20080910204458] [trivial: remove unused imports -- thanks, pyflakes zooko@zooko.com**20080925173453] [test_system: factor out find_shares/replace_shares to a common class, so they can be used by other tests warner@lothar.com**20080806014958] [web: rewrite t=deep-size in terms of deep-stats, update test to match inclusion of directory sizes warner@allmydata.com**20081007043539] [web: change t=manifest to return a list of (path,read/writecap) tuples, instead of a list of verifycaps. Add output=html,text,json. warner@allmydata.com**20081007043618] [webapi.txt: explain that t=manifest gives verifycaps warner@allmydata.com**20080907192950] [dirnode: refactor recursive-traversal methods, add stats to deep_check() method results and t=deep-check webapi warner@lothar.com**20080910084504] [dirnode: use the concurrency limiter in t=manifest and t=deep-size, allow 10 retrievals in parallel warner@allmydata.com**20080508013637] [web: add 'more info' pages for files and directories, move URI/checker-buttons/deep-size/etc off to them warner@lothar.com**20080918050041] [directory.xhtml: oops, missed a comma warner@lothar.com**20070708074408] [directory.xhtml: remove the leftover XML link warner@lothar.com**20070708073320] [web: more mutable-file coverage warner@allmydata.com**20080520183547] [hush pyflakes warner@allmydata.com**20080910025017] [test_system: add deep-check-JSON tests, fix a bug warner@lothar.com**20080910061416] [test_system: check t=deep-stats too warner@lothar.com**20080910065457] [test_system: add deep-stats test warner@lothar.com**20080910055634] [web: fix output=JSON, add buttons for repair/json to the 'run deep-check' form warner@allmydata.com**20080910211137] [checker results: add output=JSON to webapi, add tests, clean up APIs warner@allmydata.com**20080910024517 to make the internal ones use binary strings (nodeid, storage index) and the web/JSON ones use base32-encoded strings. The immutable verifier is still incomplete (it returns imaginary healty results). ] [immutable verifier: provide some dummy results so deep-check works, make the tests ignore these results until we finish it off warner@allmydata.com**20080910010827] [mutable checker: even more tests. Everything in ICheckerResults should be covered now, except for immutable-verify which is incomplete warner@allmydata.com**20080910005706] [checker results: more tests, update interface docs warner@allmydata.com**20080910003010] [checker results: more tests, more results. immutable verifier tests are disabled until they emit more complete results warner@allmydata.com**20080910001546] [checker: add tests, add stub for immutable check_and_repair warner@allmydata.com**20080909233449] [checker: overhaul checker results, split check/check_and_repair into separate methods, improve web displays warner@allmydata.com**20080907194456] [hush pyflakes warning about code that got moved in the recent StallMixin refactoring warner@allmydata.com**20080423001426] [test_client.py: validate more versioning code warner@allmydata.com**20080103203824] [test_client.py: assert allmydata.__version__ is not unknown warner@allmydata.com**20080103203459] [dirnode deep-check: add tests of cycles, fix failures warner@allmydata.com**20080717213704] [oops, fix import/pyflakes problems warner@allmydata.com**20080718000620] [checker_results.problems: don't str the whole Failure, just extract the reason string warner@allmydata.com**20080812042306] [dirnode: add some deep-check logging warner@allmydata.com**20080812042338] [hush a pyflakes warning warner@allmydata.com**20080812042423] [checker: add information to results, add some deep-check tests, fix a bug in which unhealthy files were not counted warner@allmydata.com**20080812040326] [IFilesystemNode: add get_storage_index(), it makes tests easier warner@allmydata.com**20080812231407] [web/deep-check: show the webapi runtime at the bottom of the page warner@allmydata.com**20080813033426] [immutable checker: add a status_report field warner@allmydata.com**20080813033530] [logging: add 'unique-message-ids' (or 'umids') to each WEIRD-or-higher log.msg call, to make it easier to correlate log message with source code warner@allmydata.com**20080826015759] [download: DownloadStopped isn't SCARY, lower the log severity warner@allmydata.com**20080415230609] [client: don't start the IntroducerClient until the Tub is ready, otherwise we will sometimes connect to the introducer (or other clients) before we've done Tub.setLocation, which loses some information on the introducer status page warner@allmydata.com**20080423215234] [web: add 'report incident' button at the bottom of the welcome page warner@lothar.com**20080805190921] [wui: reorganize the welcome.xhtml page zooko@zooko.com**20080429221014 Jake Edge tried Tahoe out and didn't notice the /status page. Hopefully with this new organization people like he will see that link more easily. This also addresses drewp's suggestion that the controls appear above the list of servers instead of below. (I think that was his suggestion.) I also reordered the controls. ] [mutable/checker: log a WEIRD-level event when we see a hash failure, to trigger an Incident warner@allmydata.com**20080813035020] [logging cleanups: lower DeadReferenceError from WEIRD (which provokes Incidents) to merely UNUSUAL, don't pre-format Failures in others warner@allmydata.com**20080826005155] [mutable read: enable the cache (written during mapupdate, read during retrieve). This speeds up small-file reads by about 30% over a link with an average 25ms RTT warner@allmydata.com**20080422002750] [mutable: make mutable-repair work for non-verifier runs, add tests warner@allmydata.com**20080826233454] [mutable/checker: rearrange a bit, change checker-results to have a status_report string warner@allmydata.com**20080812032033] [first pass at a mutable repairer. not tested at all yet, but of course all existing tests pass warner@allmydata.com**20080718040923] [interfaces: add IRepairable warner@allmydata.com**20080718003217] [mutable/servermap: add summarize_version warner@allmydata.com**20080812031930] [CLI: add 'tahoe debug corrupt-share', and use it for deep-verify tests, and fix non-deep web checker API to pass verify=true into node warner@allmydata.com**20080813000501] [tests: add test that verifier notices any (randomly chosen) bit flipped in the verifiable part of any (randomly chosen) share zooko@zooko.com**20080731002015 The currently verifier doesn't (usually) pass this randomized test, hence the TODO. ] [tests: test that checker doesn't cause reads on the storage servers zooko@zooko.com**20080730235420 It would still pass the test if it noticed a corrupted share. (It won't notice, of course.) But it is required to do its work without causing storage servers to read blocks from the filesystem. ] [tests: add test_system.Checker which tests basic checking (without verification) functionality zooko@zooko.com**20080728234317] [test_system.py: factor SystemTestMixin out of SystemTest warner@allmydata.com**20080725223349] [test_system.py: add a log message to help track down the occasional cygwin failure warner@allmydata.com**20070531190114] [offloaded: fix failure in unit test on windows robk-tahoe@allmydata.com**20080118025729 in trying to test my fix for the failure of the offloaded unit test on windows (by closing the reader before unlinking the encoding file - which, perhaps disturbingly doesn't actually make a difference in my windows environment) I was unable too because the unit test failed every time with a connection lost error. after much more time than I'd like to admit it took, I eventually managed to track that down to a part of the unit test which is supposed to be be dropping a connection. it looks like the exceptions that get thrown on unix, or at least all the specific environments brian tested in, for that dropped connection are different from what is thrown on my box (which is running py2.4 and twisted 2.4.0, for reference) adding ConnectionLost to the list of expected exceptions makes the test pass. though curiously still my test logs a NotEnoughWritersError error, and I'm not currently able to fathom why that exception isn't leading to any overall failure of the unit test itself. for general interest, a large part of the time spent trying to track this down was lost to the state of logging. I added a whole bunch of logging to try and track down where the tests were failing, but then spent a bunch of time searching in vain for that log output. as far as I can tell at this point the unit tests are themselves logging to foolscap's log module, but that isn't being directed anywhere, so all the test's logging is being black holed. ] [test_system.py: refactor bounce_client, probably make it stop failing on cygwin warner@allmydata.com**20080211212658] [break introducer up into separate modules in the new allmydata.introducer package warner@allmydata.com**20080618192416] [introducer: remove PeerCountObserver, tests are managing with purely poll-for-connected approachers warner@allmydata.com**20080205201549] [introducer.py: accelerate reconnection after being offline. Closes #374. warner@allmydata.com**20080331222845 When we establish any new connection, reset the delays on all the other Reconnectors. This will trigger a new batch of connection attempts. The idea is to detect when we (the client) have been offline for a while, and to connect to all servers when we get back online. By accelerating the timers inside the Reconnectors, we try to avoid spending a long time in a partially-connected state (which increases the chances of causing problems with mutable files, by not updating all the shares that we ought to). ] [introducer: only record one announcement per (tubid,service) tuple. Fixes #343. warner@allmydata.com**20080423220539] [introducer: record a timestamp with each announcement, and display it on the introducer's web page warner@allmydata.com**20080312023319] [introweb.py: add ?t=json, to provide machine-readable subscriber counts warner@allmydata.com**20080325195612] [introducer: record a timestamp with each subscriber, and display it on the introducer's web page warner@allmydata.com**20080312022837] [introducer: remove encoding-parameter config, for now warner@allmydata.com**20080213005954] [introweb: combine announcement and subscriber information to show version+nickname for each client warner@allmydata.com**20080312022129] [oops, add introducer.xhtml warner@allmydata.com**20080312004103] [introweb.py: tolerate non-setLocationed client tubs warner@allmydata.com**20080312010913] [test_system.py: modify system-test setup code in preparation for merge with common.SystemTestMixin warner@allmydata.com**20080725222931] [tests: make it so that you can use common.py's SystemTestMixin.set_up_nodes() more than once with the same introducer zooko@zooko.com**20080728234029] [test/common.py: use pre-computed Tub certificates for the system-test mixin, to speed such tests up by maybe 15%. The goal is to encourage more full-grid tests. warner@allmydata.com**20080728194421] [test_system.py: move SystemTestMixin out into common.py, where further improvements will occur warner@allmydata.com**20080725221758] [test_system.py: create SystemTestMixin, with less cruft, for faster system-like tests warner@allmydata.com**20080725221300] [test: add testutil.flip_one_bit which flips a randomly chosen bit of the input string zooko@zooko.com**20080728234217] [testutil.py: remove unused import, appease pyflakes Brian Warner **20070427151134] [web/directory: enable verify=true in t=deep-check warner@allmydata.com**20080812042409] [deep-check: add webapi, add 'DEEP-CHECK' button to wui, add tests, rearrange checker API a bit warner@allmydata.com**20080717234709] [web: transform FileTooLargeError into a friendlier '413 Request Entity Too Large' error warner@allmydata.com**20080603070316] [dirnode: return to 'delete fails if the child wasn't actually there' semantics, to make tests pass. There's a switch to enable/disable this warner@allmydata.com**20080418030606] [test_web: add HEAD coverage warner@allmydata.com**20080520184743] [checker: re-enable checker web results (although they just say 'Healthy' right now) warner@allmydata.com**20080716224256] [webapi.txt: overhaul documentation. API changes are as follows: warner@allmydata.com**20080519194746 * download/upload localdir=/localfile= has been removed. This sort of ambient authority was unsafe to expose over the web (CSRF), and at some point soon we'll have 'cp -r' in the CLI to replace it. * GET save=filename -> GET filename=filename&save=true * GET t=download removed * side-effect causing operations now use POST where appropriate, not PUT * to create multiple directories, either use * POST /uri/DIRCAP/parent?t=mkdir&name=child (more form/browser oriented) * POST /uri/DIRCAP/parent/child?t=mkdir (more machine oriented) The t=mkdir-p form is still accepted, but not preferred (since it leaks the child name queryarg into the logs) * use PUT /uri/MUTABLEFILECAP or PUT /uri/DIRCAP/child (on a mutable file) to replace its contents, or POST /same?t=upload from forms * response bodies and codes are better specified than before ] [webapi.txt: minor edits zooko@zooko.com**20070823200944] [webapi.txt: update webapi.txt to reflect the security fix from #98 zooko@zooko.com**20071015192902] [webapi.txt: document POST /uri?t=mkdir warner@allmydata.com**20080208021028] [doc: change example filename extension back because it is more recognizable and because I love Brian zooko@zooko.com**20080227205405] [docs: tweak wording per kpreid and tweak example filename extension per me zooko@zooko.com**20080227204157] [docs: merge conflicts between the patch to document "127.0.0.1" instead of "localhost" and some other patches (precisely which, I don't know) zooko@zooko.com**20080418035741] [docs: clarify which webport value is the default zooko@zooko.com**20080227171003] [fix a typo in webapi.txt robk-org@allmydata.com**20070712234551] [docs: use "127.0.0.1" instead of "localhost" zooko@zooko.com**20080418034534 Unfortunately there are occasionally configurations in the real world where "localhost" does not resolve to 127.0.0.1, and if a user has such a configuration then using 'localhost' could lead to an authority leak. ] [docs/webapi.txt: mention that we default to a --webport of 8123 warner@allmydata.com**20071011201911] [docs: tiny change in webapi.txt zooko@zooko.com**20080418040051 (I'm actually committing this patch only in order to test our patch management infrastructure.) ] [docs: tiny update to webapi.txt zooko@zooko.com**20080418040912 I'm actually committing this just to test our patch management infrastructure. ] [web: even more test coverage warner@allmydata.com**20080520183314] [web/directory: fix rw_uri output in t=JSON to reflect mutable files properly warner@allmydata.com**20080520013728] [test_web.py: minor cleanups, improved error reporting warner@allmydata.com**20080519193339] [test_web/test_system: improve test coverage warner@lothar.com**20080520062852] [web: improve test coverage warner@allmydata.com**20080520013839] [test_web.py: hush pyflakes warner@allmydata.com**20080206043751] [dirnode deep-check: rearrange traversal approach, simplify code a bit warner@allmydata.com**20080717212504] [first pass at deep-checker, no webapi yet, probably big problems with it, only minimal tests warner@allmydata.com**20080717012057] [move encode/upload/download/checker.py into a new immutable/ directory. No behavior changes expected. warner@allmydata.com**20080716201439] [rename encode_new.py to encode.py, now that there isn't an old one anymore warner@lothar.com**20070406041742] [move FileTooLargeError out to a common location warner@allmydata.com**20080603070115] [Don't allow uploads of large files (about 12GiB or larger), since they're doomed to be corrupted. Closes #439 warner@allmydata.com**20080602235701] [storage.py: handle num_segments != power-of-two without an assertion warner@lothar.com**20070714023021] [test_upload.py: hush pyflakes warner@allmydata.com**20080207020431] [mutable/publish.py: raise FileTooLargeError instead of an ugly assertion when the SDMF restrictions are exceeded warner@allmydata.com**20080603070210] [overhaul checker invocation warner@allmydata.com**20080716002325 Removed the Checker service, removed checker results storage (both in-memory and the tiny stub of sqlite-based storage). Added ICheckable, all check/verify is now done by calling the check() method on filenodes and dirnodes (immutable files, literal files, mutable files, and directory instances). Checker results are returned in a Results instance, with an html() method for display. Checker results have been temporarily removed from the wui directory listing until we make some other fixes. Also fixed client.create_node_from_uri() to create LiteralFileNodes properly, since they have different checking behavior. Previously we were creating full FileNodes with LIT uris inside, which were downloadable but not checkable. ] [interfaces: clarify IChecker.checker_results_for a bit warner@allmydata.com**20071023011046] [checker.checker_results_for: ignore uris of 'None' warner@allmydata.com**20071027013837] [checker: improve test coverage a little bit warner@allmydata.com**20071205000012] [mutable.node: avoid reentrancy problems in Deferred code on twisted2.5, by adding an eventual-send call warner@lothar.com**20080418074329] [use a weakref cache in the client to manage singleton filenodes/dirnodes, to avoid autocollision. Should close #391. warner@allmydata.com**20080509010255] [dirnode: add overwrite= to most API calls, defaulting to True. When False, this raises ExistingChildError rather than overwriting an existing child warner@allmydata.com**20080516230947] [dirnode: update to use MutableFileNode.modify warner@allmydata.com**20080418025704] [deep-stats: add file-size histogram warner@allmydata.com**20080508231942] [dirnode: refactor deep-stats a bit warner@allmydata.com**20080508203307] [interfaces: add verify= and repair= args to check() warner@lothar.com**20080707213736] [implement a mutable checker+verifier. No repair yet. Part of #205. warner@lothar.com**20080708003600] [test_mutable: test that all servers refusing our share means a publish fails warner@allmydata.com**20080423015320] [mutable/servermap: improve test coverage warner@allmydata.com**20080422234752] [mutable: test write failures, uncoordinated write detection warner@allmydata.com**20080422184953] [mutable WIP: use fireOnOneErrback when using a DeferredList warner@allmydata.com**20080417201148] [test_mutable: factor out ShouldFailMixin warner@allmydata.com**20080418025551] [test_mutable: hush pyflakes warner@allmydata.com**20080418025755] [mutable: improve test coverage, fix bug in privkey fetching, add .finished to stats, remove dead code warner@allmydata.com**20080419025512] [mutable: improve test coverage slightly warner@allmydata.com**20080421221050] [mutable: implement MutableFileNode.modify, plus tests warner@allmydata.com**20080418021242] [testutil: factor stall() out into a common location warner@allmydata.com**20080422234715] [test_system: stall for a second while bouncing the client, it might help windows warner@allmydata.com**20070629022028] [setup: finish switching from Tahoe's versions of autoversioning tools to pyutil's versions zooko@zooko.com**20071221204238] [don't try to use bindann zooko@zooko.com**20070914021446 It causes a mysterious misbehavior in Python import which causes the previous patch to fail (the patch to not run trial tests if dependencies can't be imported) ] [version_class.py: if you don't have pkg_resources for comparing version numbers, use distutils.version.LooseVersion zooko@zooko.com**20070816231641] [setup.py use sys.executable instead of hard-coded 'python' to run make-version.py warner@allmydata.com**20070924193859] [setup: attempt to invoke make-version.py whenever setup.py is evaluated zooko@zooko.com**20070924014336] [setup.py: fix move to _version.py warner@lothar.com**20070912230223] [trailing-whitespace eradication, no functional changes warner@allmydata.com**20071101223435] [setup: continue running setup.py even if ez_setup.py can't be imported zooko@zooko.com**20071109205803] [setup: use ez_setup.py without a "download delay" zooko@zooko.com**20071003221414] [change ez_setup.py to find tarballs in misc/dependencies zooko@zooko.com**20070913215119] [add setuptools eggs for py2.4 and py2.5 to misc/dependencies zooko@zooko.com**20070913215043] [setup: import the latest version of ez_setup.py with my patches zooko@zooko.com**20071003221319] [ez_setup.py: put back the warning about downloading, but only if the URL that you are using is not "file:" zooko@zooko.com**20070914031451] [ez_setup.py: don't warn about the need to download packages (because we actually bundle them all with Tahoe) zooko@zooko.com**20070914030027] [use ez_setup.py to bootstrap setuptools zooko@zooko.com**20070913015710] [setup: remove misc/make-version.py and invoke "darcsver" from the pyutil library zooko@zooko.com**20071221001755 misc/make-version.py has a limitation which prevents it from generating version stamps from our current darcs history. This limitation has been fixed in pyutil's "darcsver". Rather than copy the fix from there to misc/make-version.py, I'm making it so that you have to install pyutil if you want to automatically generate _version.py files from the current darcs history. ] [fix version class to preferred format and correct parsing zooko@zooko.com**20070816223801] [hush some pyflakes warnings warner@lothar.com**20070915220721] [shebang usr bin env python zooko@zooko.com**20070613015525] [misc/storage-overhead.py: tool to estimate storage-space overhead per filesize warner@allmydata.com**20070716204331] [make-version.py: when _darcs doesn't exist, make the warning less scary-looking warner@allmydata.com**20070924193837] [PollMixin: add timeout= argument, rewrite to avoid tail-recursion problems warner@allmydata.com**20080205023507] [CLI: move the 'repl' command to 'tahoe debug repl' warner@allmydata.com**20080812204017] [CLI: simplify argument-passing, use options= for everthing, including stdout warner@lothar.com**20080801184624] [added a 'repl' command to tahoe.exe robk-tahoe@allmydata.com**20080110011952 this is probably not of very high utility in the unix case of bin/tahoe but is useful when working with native builds, e.g. py2exe's tahoe.exe, to examine and debug the runtime environment, linking problems etc. ] [tahoe_ls: improve output formatting warner@allmydata.com**20080520200750] [CLI: add 'ln', just like move but without the delete warner@allmydata.com**20080520203031] [CLI mv: if we can't created the new child, don't delete the old one warner@allmydata.com**20080520194947] [CLI: add 'list-aliases', factor out get_aliases warner@allmydata.com**20080520213604] [tahoe_cp: rewrite, make --recursive work warner@allmydata.com**20080522003521] [tahoe_cp.py: fix pyflakes complaint warner@allmydata.com**20080521190913] [cli: initial implementation of 'cp -r', probably doesn't work yet warner@allmydata.com**20080521184922] [CLI: implement the easy part of cp (no -r, only two arguments) warner@allmydata.com**20080520235603] [CLI ls: add --readonly-uri to display readonly URIs for all children warner@allmydata.com**20080520194911] [CLI: add put --mutable, enhance ls to show mutable vs immutable as rw/r- warner@allmydata.com**20080520193655] [overhaul CLI: not quite complete but it works a lot better than it used to. The new scheme uses 'tahoe add-alias' and rsync/scp-style 'alias:foo/bar.txt' arguments warner@allmydata.com**20080520022850] [tahoe_get.py: remove unused import warner@allmydata.com**20071012024740] [cli: use urllib.quote() on vdrive-path arguments before passing them through HTTP zooko@zooko.com**20071027013044] [tahoe_ls: list individual files warner@lothar.com**20071021193306] [test_system: write test data in 'b' verbatim mode, since on windows the default text-mode is different. Addresses one of the failures in #223 warner@allmydata.com**20071212011633] [fix unit test to pass forward-slashes to the CLI since it demands that the CLI emit forward-slashes zooko@zooko.com**20071212020344] [remove the slash-to-bang conversion from CLI tools and webapi.txt warner@allmydata.com**20071218022226] [docs: change our default HTTP port to 8123 warner@allmydata.com**20071011201733] [webapi.txt: reinstate documentation of the unpleasant URI-escaping needed for Brian Warner **20071011135808 slashes in dirnode URIs, to be resolved some day by #102. ] [webapi.txt edits (thanks to Brian Warner) zooko@zooko.com**20070823200606] [cli: simplify code by using stdlib's httplib module warner@allmydata.com**20071012052923] [command-line: fix ticket #111 by requiring input to be a local file and sending Content-Length header zooko@zooko.com**20070817215949] [cli: use urllib.escape on all URIs warner@lothar.com**20071011083444] [tahoe_put.py: hush pyflakes by removing unused 'sys' import warner@lothar.com**20070817232950] [create_node.py: need to create private/ dir in create-client so we can touch the my_private_dir.cap file warner@allmydata.com**20071218214218] [oops, touch private/my_private_dir.cap when creating the client warner@allmydata.com**20071218213659] [a few formatting tidy-ups zooko@zooko.com**20080103231419] [key_generator: service related cleanups, incorporation into system test robk-tahoe@allmydata.com**20080403225707 this cleans up KeyGenerator to be a service (a subservice of the KeyGeneratorService as instantiated by the key-generator.tac app) this means that the timer which replenishes the keypool will be shutdown cleanly when the service is stopped. adds checks on the key_generator service and client into the system test 'test_mutable' such that one of the nodes (clients[3]) uses the key_generator service, and checks that mutable file creation in that node, via a variety of means, are all consuming keys from the key_generator. ] [doc: slightly clarify an error message zooko@zooko.com**20080414184305] [CLI.txt: document proposed scp:-based CLI syntax warner@allmydata.com**20080510010629] [test_system: match webapi change, making new files now returns 201 Created, not 200 OK warner@allmydata.com**20080519210931] [CLI: make 'tahoe webopen' use the 'tahoe:' alias properly, instead of the old --dir-cap option warner@allmydata.com**20080603005554] [added "tahoe webopen" command robk-tahoe@allmydata.com**20080105003410 taking the same arguments as tahoe ls, it does a webbrowser.open to the page specified by those args. hence "tahoe webopen" will open a browser to the root dir specified in private/root_dir.cap by default. this might be a good alternative to the start.html page. ] [CLI: move all debug commands (dump-share, dump-cap, find-shares, catalog-shares) into a 'debug' subcommand, and improve --help output warner@allmydata.com**20080812203732] [tahoe dump-share: remove --filename option, just take it from argv warner@allmydata.com**20080206201533] [docs/CLI.txt: provide an overview of bin/tahoe subcommands warner@allmydata.com**20080509193619] [web/directory: factor out the get_root function warner@allmydata.com**20081007201742] [web: stop using absolute links (or url.here) in forms and pages, since they break behind proxies. Partially addresses #461 warner@allmydata.com**20080618024940] [test_web.py: use /uri?t=mkdir instead of /uri/?t=mkdir, and confirm that the redirection target is correct warner@lothar.com**20071224234652] [wapi/wui: add a trailing slash to the targets of hyperlinks of children of the current directory when those targets are directories zooko@zooko.com**20080611025938 This is intended to fix #458 '"other representations" broken in webish ui'. ] [webish: complete rewrite, break into smaller pieces, auto-create directories, improve error handling warner@allmydata.com**20080519195704] [web: show the root name of the vdrive warner@lothar.com**20070708043102] [web: remove more dead code warner@lothar.com**20070708052003] [webish: cosmetic: fix missing whitespace in HTML warner@lothar.com**20070710173319] [webish.DirnodeWalkerMixin: comment out the code that demonstrates what we Brian Warner **20070713062809 would do if we could just use generators, since we don't use it. ] [fix bug in arg handling around 'delete' button in directory view robk-org@allmydata.com**20070712234654 the code composing the form providing the 'delete' button in a dir view was broken in that it tried to put some of the arguments into the url query, rather than as the form's post args. worse, nevow was kind enough to escape the querystring making it invalid. ] [webish: handle PUTs to direct children of the root warner@lothar.com**20070714023152] [webish: oops, handle POST without localfile= too warner@allmydata.com**20070716190054] [change name of the query "allow_local_access()?" to "local_access_is_allowed()" zooko@zooko.com**20070822173200 So as to avoid confusing it with the setter method "allow_local_access(allowed=True)". ] [webish.py: handle asynchronous checker results. warner@allmydata.com**20071031000037 Thanks to robk for pointing out that Nevow will accept a Deferred almost everywhere. In this case, we just pass a Deferred into ctx.fillSlots(). One quirk: nevow doesn't evaluate all rows of the table in parallel: using a slow Deferred in a slot in one row seems to stall the next row until that one has fired, probably to simplify the flattening of the HTML. ] [webish: tolerate not having a checker, since some unit tests don't make one warner@allmydata.com**20071024003531] [don't provide the "overwrite" button if the file is readonly to you zooko@zooko.com**20071110220705] [webish: oops, unit tests don't have an Uploader, don't rely upon it for helper-status display warner@allmydata.com**20080128200329] [webish: upload+localdir=missing should give an error warner@allmydata.com**20080128204806] [webish: upload: when the localdir= doesn't exist, say so in the HTTP response warner@lothar.com**20070915194907] [webish: remove 'URI-link' from directory page, now that we only use URI-links warner@allmydata.com**20080130001109] [webish: condense display of nickname a little bit warner@allmydata.com**20080206002928] [webish: show nickname too warner@allmydata.com**20080206002605] [webish.py: fix for #237: when listing large directories, insert a turn break once every 100 children, to work around non-optimized tail recursion Deferreds warner@allmydata.com**20080213032852] [add a mkdir-p POST handler robk-tahoe@allmydata.com**20080318011301 this adds a t=mkdir-p call to directories (accessed by their uri as /uri/?t=mkdir=p&path=/some/path) which returns the uri for a directory at a specified path before the given uri, regardless of whether the directory exists or whether intermediate directories need to be created to satisfy the request. this is used by the migration code in MV to optimise the work of path traversal which was other wise done on every file PUT ] [webish: fix 'not running helper' status indicator on welcome page, add tests warner@allmydata.com**20080414232811] [test_system.py: improve coverage of webish.py warner@lothar.com**20080215100250] [dirnode: add a deep_stats(), like deep-size but with more information. webish adds t=deeps-size too. warner@allmydata.com**20080508202114] [add GET /uri/URI/?t=deep-size, to compute the total size of immutable files reachable from a given directory warner@allmydata.com**20080327183342] [mutable: replace MutableFileNode API, update tests. Changed all callers to use overwrite(), but that will change soon warner@allmydata.com**20080418005138] [mutable WIP: add servermap update status pages warner@allmydata.com**20080417020541] [mutable WIP: merge conflicts in test_system.py warner@allmydata.com**20080415225728] [web: don't break status page when there is no helper running warner@lothar.com**20080415064220] [stats: add /statistics web page to show them, add tests warner@allmydata.com**20080414211708] [stats_gatherer: verbose debug logging robk-tahoe@allmydata.com**20080409231053 one of the storage servers is throwing foolscap violations about the return value of get_stats(). this adds a log of the data returned to the foolscap log event stream at the debug level '12' (between NOISY(10) and OPERATIONAL(20)) hopefully this will facilitate finding the cause of this problem. ] [helper status: include percentage fetched+pushed, add helper-uploads to the upload/download list warner@allmydata.com**20080415013627] [stats_gatherer: fix typo in helper stats gathering. robk-tahoe@allmydata.com**20080411004142] [helper stats: fix the /helper_status page, the recent conflict merging missed some uses. Added tests, updated the munin plugins to match warner@allmydata.com**20080414201853] [helper: add another munin plugin warner@allmydata.com**20080327235030] [stats_gatherer: reconcile helper stats gathering robk-tahoe@allmydata.com**20080411002544 I'd implemented stats gathering hooks in the helper a while back. Brian did the same without reference to my changes. This reconciles those two changes, encompassing all the stats in both changes, implemented through the stats_provider interface. this also provide templates for all 10 helper graphs in the tahoe-stats munin plugin. ] [munin: added a series of munin graphs to report upload helper state robk-tahoe@allmydata.com**20080326013046] [conflict reconciliation (part 1, stats gathering in helper) robk-tahoe@allmydata.com**20080328002516] [helper: add more stats to webapi, at /helper_status warner@allmydata.com**20080327234608] [helper: add stats for the gatherer, show some on the webish welcome page warner@allmydata.com**20080327225532] [webish: show storage sizelimit, abbreviate current usage warner@allmydata.com**20080307031638] [webish: add storage-consumed estimate on welcome page warner@allmydata.com**20080206022939] [stats: added stats reporting to the upload helper robk-tahoe@allmydata.com**20080326011908 adds a stats_producer for the upload helper, which provides a series of counters to the stats gatherer, under the name 'chk_upload_helper'. it examines both the 'incoming' directory, and the 'encoding' dir, providing inc_count inc_size inc_size_old enc_count enc_size enc_size_old, respectively the number of files in each dir, the total size thereof, and the aggregate size of all files older than 48hrs ] [offloaded: add fetched-percentage to the log message warner@allmydata.com**20080414211638] [mutable WIP: re-enable publish/retrieve status warner@allmydata.com**20080417004906] [key_generator: added a unit test robk-tahoe@allmydata.com**20080403200143 implemented a unit test of basic KeyGenService functionality, fixed a bug in the timing of pool refreshes ] [mutable WIP: improve logging a bit warner@allmydata.com**20080416222230] [mutable WIP: rewrite ServerMap data structure, add tests warner@allmydata.com**20080416214947] [mutable WIP: if corrupt shares cause a retrieve to fail, restart it once, ignoring those shares and using different ones warner@allmydata.com**20080415225802] [mutable WIP: split mutable.py into separate files. All tests pass. warner@lothar.com**20080411213116] [direct the user to docs/write_coordination.html in case of an UncoordinatedWriteError zooko@zooko.com**20080108171506] [test_filenode.py : improve coverage of mutable filenode, fix a bug in __hash__ warner@allmydata.com**20080304200128] [add a webserver for the Introducer, showing service announcements and subscriber lists warner@allmydata.com**20080312003625] [reinstate creation of node.url files upon startup robk-tahoe@allmydata.com**20080108000456 a recent purge of the start.html code also took away the logic that wrote 'node.url' into the node root. this is required for the tahoe cli tool to find the node. this puts back a limited fraction of that code, so that the node writes out a node.url file upon startup. ] [stats gathering: added counters to upload,download,mutablewatcher robk-tahoe@allmydata.com**20080410010859 counting number of operations, and for immutable files, bytes transferred ] [use added secret to protect convergent encryption zooko@zooko.com**20080324164606 Now upload or encode methods take a required argument named "convergence" which can be either None, indicating no convergent encryption at all, or a string, which is the "added secret" to be mixed in to the content hash key. If you want traditional convergent encryption behavior, set the added secret to be the empty string. This patch also renames "content hash key" to "convergent encryption" in a argument names and variable names. (A different and larger renaming is needed in order to clarify that Tahoe supports immutable files which are not encrypted content-hash-key a.k.a. convergent encryption.) This patch also changes a few unit tests to use non-convergent encryption, because it doesn't matter for what they are testing and non-convergent encryption is slightly faster. ] [upload: Data should use convergence by default warner@allmydata.com**20080131010256] [offloaded: oops, need more tricks to make the unit tests pass warner@allmydata.com**20080206235111] [helper: return full uri-extension data to the client, so it can compare encoding parameters warner@allmydata.com**20080206233058] [test_upload.py: add test to exercise CHK hashing variations warner@allmydata.com**20080207020335] [Merge patch which switches to SHA-256d with patch that adds punctuation and capitalization to the comment about the hash value. zooko@zooko.com**20080215191643] [docs: update install and usage docs, improve cli "usage" output, make new example directories, add unit test that fails code which prints out sentences that don't end with punctuation marks zooko@zooko.com**20080215191102] [docs: update the example link in using.html zooko@zooko.com**20080108172345] [docs: 10 blocks by default, not 12 amber@yukyuk**20080213105719] [docs: update docs/about.html with Amber zooko@zooko.com**20080201173923] [doc: fix typos and otherwise edit about.html zooko@zooko.com**20080121215443] [docs: about.html: edit thanks to nej zooko@zooko.com**20080122035201] [doc: add an overview to about.html zooko@zooko.com**20080121211925 Hopefully this overview has the right combination of generality and precision to satisfy The Norm Hardy Request: http://allmydata.org/pipermail/tahoe-dev/2007-November/000222.html ] [docs: some documentation updates for 0.7.0 zooko@zooko.com**20080108163241] [more introductory doc cleanup zooko@zooko.com**20080105000919 mv README to docs/about.html and reformat it as HTML add a new README which is a text file pointing to docs/{about,install,running}.html include the Transitive Grace Period Public Licence in its HTML form (it is too big) ] [new licences, move details from README to doc/install-details.html zooko@zooko.com**20080104182742] [README: explain when you need to download source and when you don't zooko@zooko.com**20070921185356] [README: reflow to 80 cols zooko@zooko.com**20070924200732] [README: edit to clarify that you can't use "make" if you installed it the easy_install way zooko@zooko.com**20071013062153] [DEPENDENCIES: recommend the 'build-essential' package instead of gcc+make, Brian Warner **20071018033331 since it includes important things like libc6-dev ] [setup: pass INCLUDE_DIRS and LIBRARY_DIRS variables, if any to setup.py from Makefile zooko@zooko.com**20071114021659 ] [setup: README: a few clarifications thanks to Priyanka zooko@zooko.com**20071120050853] [setup: README: warn against Python 2.4.1, recommend easy_install, link to easy_install download page, edit for clarity zooko@zooko.com**20071120193922] [README: fix whitespace zooko@zooko.com**20071107160057] [README: it works on Python 2.4 on Windows zooko@zooko.com**20070914021730] [README: advise 'make clean' before 'make build-deps' (after update), otherwise old versions of the dependent libaries can be used in preference to the newer one warner@allmydata.com**20071113202449] [README: update required version numbers, fix small cut-and-paste error warner@lothar.com**20080102033646] [doc: describe how to start client first in running.html zooko@zooko.com**20080106071657] [doc: even simpler running.html zooko@zooko.com**20080104031159] [running.html: fix usage of 'tahoe create-client' and 'tahoe start', explain ~/.tahoe, add closing tags warner@lothar.com**20080102035046] [docs: update install.html and update and format running.html zooko@zooko.com**20080101232007] [docs: a couple of improvements to install.html zooko@zooko.com**20080101075250] [docs: even further simplify and reformat install.html zooko@zooko.com**20071231143907] [docs: format install.html into HTML format zooko@zooko.com**20071231020118] [docs: a bunch of updates to simplify the process of installing from source and running Tahoe zooko@zooko.com**20071230114717 These changes are a work-in-progress -- there are many incomplete and incorrect parts, but the install.html and running.html files are complete and should work (and they are delightfully concise!). I'm pushing this just to let people see incremental progress and to solicit feedback. "Testing out" the install.html and running.html files and submitting patches or bug reports would be quite welcome. More to come. ] [README: edits zooko@zooko.com**20070929180520] [a few documentation and naming convention updates zooko@zooko.com**20071213013408 Notable: the argument to make REPORTER has been renamed to TRIALARGS. ] [docs: edits and updates to architecture.txt, with Amber zooko@zooko.com**20080213162452] [docs: edit "grid of storage servers" section with Amber zooko@zooko.com**20080128174821] [docs: architecture.txt: some edits with Amber zooko@zooko.com**20080201183906] [docs: edit architecture.txt with Amber's help zooko@zooko.com**20080128173346] [doc: architecture.txt: start updating architecture.txt zooko@zooko.com**20080121235303 I chose to remove mention of non-convergent encoding, not because I dislike non-convergent encoding, but because that option isn't currently expressed in the API and in order to shorten architecture.txt. I renamed "URI" to "Capability". I did some editing, including updating a few places that treated all capabilities as CHK-capabilities and that mentioned that distributed SSKs were not yet implemented. ] [architecture.txt: small edits zooko@zooko.com**20070809053105] [mention ticket #22 "Zooko O'Whielacronx "**20070502033322] [cleaning grammar wilcoxjg@gmail.com**20070809041154] [architecture.txt: a few small edits zooko@zooko.com**20070919212704] [docs: architecture.txt: reflow to 77 cols zooko@zooko.com**20080121232628 Experiment showed that reflowing to 77 cols changed the fewest lines. ] [BIG COMPATIBILITY BREAK: update hash tags, switch to SHA-256d everywhere warner@allmydata.com**20080215015801] [mutable: storage_index is always 16 bytes warner@allmydata.com**20071107005434] [hashutil.py: switch from pycrypto to pycryptopp SHA256 zooko@zooko.com**20071109204013] [hashutil: replace pycrypto's SHA256 with pycryptopp's SHA256 zooko@zooko.com*-20071108000239] [hashutil: replace pycrypto's SHA256 with pycryptopp's SHA256 zooko@zooko.com**20071108000239] [stats: make StatsGatherer happy about sharing a process with other services, add one during system test to get some test coverage warner@allmydata.com**20080304055558] [test_system: fix pyflakes warnings warner@allmydata.com**20080115032628] [stats: fix service issues robk-tahoe@allmydata.com**20080202005731 having moved inititalisation into startService to handle tub init cleanly, I neglected the up-call to startService, which wound up not starting the load_monitor. also I changed the 'running' attribute to 'started' since 'running' is the name used internally by MultiService itself. ] [stats: added IStatsProducer interface, fixed stats provider startup robk-tahoe@allmydata.com**20080201031015 this adds an interface, IStatsProducer, defining the get_stats() method which the stats provider calls upon and registered producer, and made the register_producer() method check that interface is implemented. also refine the startup logic, so that the stats provider doesn't try and connect out to the stats gatherer until after the node declares the tub 'ready'. this is to address an issue whereby providers would attach to the gatherer without providing a valid furl, and hence the gatherer would be unable to determine the tubid of the connected client, leading to lost samples. ] [mutable WIP: all tests pass, but publish/retrieve status is still stubbed out warner@allmydata.com**20080411022635] [test_system: improve test coverage of publish/retrieve status warner@allmydata.com**20080304072435] [test_system: add test coverage for download-status and upload-status warner@allmydata.com**20080304033717] [mutable: WIP. make Publish work, remove some test scaffolding. test_system still fails. warner@allmydata.com**20080411014406] [test_mutable: remove dead code warner@allmydata.com**20071107005639] [added offloaded key generation robk-tahoe@allmydata.com**20080402014513 this adds a new service to pre-generate RSA key pairs. This allows the expensive (i.e. slow) key generation to be placed into a process outside the node, so that the node's reactor will not block when it needs a key pair, but instead can retrieve them from a pool of already generated key pairs in the key-generator service. it adds a tahoe create-key-generator command which initialises an empty dir with a tahoe-key-generator.tac file which can then be run via twistd. it stashes its .pem and portnum for furl stability and writes the furl of the key gen service to key_generator.furl, also printing it to stdout. by placing a key_generator.furl file into the nodes config directory (e.g. ~/.tahoe) a node will attempt to connect to such a service, and will use that when creating mutable files (i.e. directories) whenever possible. if the keygen service is unavailable, it will perform the key generation locally instead, as before. ] [more new-pyflakes warnings fixed warner@allmydata.com**20071219005133] [runner: tweaked runner to make it easier to extend with additional subcommands robk-tahoe@allmydata.com**20080219230514 runner provides the main point of entry for the 'tahoe' command, and provides various subcommands by default. this provides a hook whereby additional subcommands can be added in in other contexts, providing a simple way to extend the (sub)commands space available through 'tahoe' ] [tweak running to make node start/stop code optional robk-tahoe@allmydata.com**20080109015118 add a 'install_node_control' flag to runner.run(), default True this enables the start/stop node commands which are not too useful on windows ] [client: publish a 'stub client' announcement to the introducer, to provide version/nickname information for each client warner@allmydata.com**20080312022010] [client.py: remove unused import warner@allmydata.com**20070629010100] [refactor node startup, remove tub_ready() warner@allmydata.com**20080206015838] [Client.tub_ready: upcall to Node warner@allmydata.com**20071102002712] [merge patch to integrate decentralized directories with "introducer_and_vdrive.py: use logpublisher too" zooko@zooko.com**20071203212904] [mutable.py: checkpointing #303 work part 2, Publish is sketched out warner@allmydata.com**20080408020128] [mutable.py: fix padding/shape-of-input-data to zfec zooko@zooko.com**20071110000625] [mutable.py: oops, our logging wrappers can't take posargs yet warner@allmydata.com**20071219053616] [mutable.py: update comment about uncoordinated writes appearing as not-enough-peers(unable to get privkey) errors warner@allmydata.com**20080107204805] [mutable: increase max segsize to 3.5MB, to allow dirnodes with about 10k entries warner@allmydata.com**20080311033955] [mutable: cheap padding hack to make zfec tolerate short files warner@lothar.com**20071108103037] [mutable: revise a couple of error messages robk-tahoe@allmydata.com**20080324224628 at brian and zooko's suggestion, reword an error message encountered when multiple writers are racing to make overlapping changes to a directory ] [mutable.py: checkpointing #303 work: retrieve does what I want, now starting in on publish warner@allmydata.com**20080405000926] [mutable: when retrieving, don't try to grab the encprivkey, to save a roundtrip warner@allmydata.com**20080305030824] [mutable-retrieve: only record server response times for queries that we don't ignore warner@allmydata.com**20080305031052] [test_mutable: exercise short reads too warner@allmydata.com**20080311010823] [mutable.py: add comments to justify initial-read size choices warner@allmydata.com**20080213234120] [test_mutable: improve multiple-encodings test coverage warner@lothar.com**20080311064735] [mutable: tolerate multiple encodings, using whichever version is recoverable first. Closes #312 warner@lothar.com**20080311072600] [test_mutable: test all hash-failure cases except a corrupted encrypted private key warner@allmydata.com**20080311004652] [mutable: validate share_hash_chain for each inbound share warner@allmydata.com**20071114212646] [test_mutable: make test-multiple-encodings work warner@lothar.com**20080311061628] [test_mutable: more test coverage, building up a framework to cause reads to occur in a specific order warner@allmydata.com**20080311051543] [test_mutable: add Roundtrip test, suitable for new share-mangling tests warner@allmydata.com**20080310231408] [mutable: minor refactoring of _do_read, to make other tests easier warner@allmydata.com**20080310224256] [test_mutable.py: accomodate changes to mutable.py logging warner@allmydata.com**20080111041834] [webish: add publish status warner@allmydata.com**20080306004110] [mutable: cosmetic changes warner@allmydata.com**20080310224405] [mutable WIP: oops, fix test_mutable warner@allmydata.com**20080417010654] [mutable.py: split replace() into update() and overwrite(). Addresses #328. warner@allmydata.com**20080313010043] [speedcheck: track SSK creation time separately warner@allmydata.com**20080130024432] [mutable: oops, .download *is* in use, by the speedtest. Restore it and add a test. warner@allmydata.com**20080304211140] [mutable.py: remove unused 'download' method (we only have download_to_data for now) warner@allmydata.com**20080304200155] [webish: add /file links, change directory page to use them. This fixes filenames in wget. Closes #221. warner@allmydata.com**20080514213221] [webish: display timestamps warner@allmydata.com**20080211211318] [refactor one of the dispatch routines in webish.py zooko@zooko.com**20080320191109 The behavior is intended to be unchanged by this refactoring. Unit tests show no change in behavior. ] [webish: link to directory URIs rather than a child path. Addresses #103. warner@allmydata.com**20080130000432] [webish: split out 'unlinked' operations warner@allmydata.com**20080305211242] [wui/wapi/webish: HTML form checkboxes send the value "on", so let's interpret that as boolean true zooko@zooko.com**20080301022942] [webish: this file is too big, start breaking it into pieces, beginning with status warner@allmydata.com**20080305205956] [webish: more upload stats: total encode-and-push rate, already-in-grid existence check time warner@allmydata.com**20080206083901] [UploadResults: add more helper timing stats (ciphertext fetch times) warner@allmydata.com**20080206081235] [webish: download-results: add per-server response times warner@allmydata.com**20080304025345] [webish: download-results: add server_problems warner@allmydata.com**20080304023035] [test_web: improve upload/download status coverage warner@allmydata.com**20080304035623] [webish: add more mutable-retrieve timing status warner@allmydata.com**20080305030436] [mutable.py: reject shares with different k/N than we expect. Quick fix for #312: avoids data corruption but has availability problems. warner@allmydata.com**20080213193420] [webish download results: add servermap, decrypt time warner@allmydata.com**20080304020932] [web: status: add 'started' timestamps to all operations warner@allmydata.com**20080305005044] [webish: add 'download results', with some basic timing information warner@allmydata.com**20080304011921] [log more peerinfo in download/upload/checker problems warner@allmydata.com**20080226233314] [switch from base62 to base32 for storage indices, switch from z-base-32 to rfc 3548 base-32 for everything, separate out base32 encoding from idlib zooko@zooko.com**20080215012747] [mutable.py: log more information during publish, specifically the sharemap, and the reason for an UncoordinatedWriteError warner@allmydata.com**20080111041623] [hashutil: add tagged_hash_256d and tagged_hasher_256d warner@allmydata.com**20080207013643] [trailing-whitespace eradication, no functional changes warner@allmydata.com**20071101222509] [use base62 encoding for storage indexes, on disk and in verifier caps, and in logging and diagnostic tools zooko@zooko.com**20080213024837 base62 encoding fits more information into alphanumeric chars while avoiding the troublesome non-alphanumeric chars of base64 encoding. In particular, this allows us to work around the ext3 "32,000 entries in a directory" limit while retaining the convenient property that the intermediate directory names are leading prefixes of the storage index file names. ] [storage: clean up use of si_s vs si_dir, add test for BadWriterEnabler message, add some logging warner@allmydata.com**20080131234848] [storage: make two levels of share directories so as not to exceed certain filesystems's limitations on directory size zooko@zooko.com**20080131222628 The filesystem which gets my vote for most undeservedly popular is ext3, and it has a hard limit of 32,000 entries in a directory. Many other filesystems (even ones that I like more than I like ext3) have either hard limits or bad performance consequences or weird edge cases when you get too many entries in a single directory. This patch makes it so that there is a layer of intermediate directories between the "shares" directory and the actual storage-index directory (the one whose name contains the entire storage index (z-base-32 encoded) and which contains one or more share files named by their share number). The intermediate directories are named by the first 14 bits of the storage index, which means there are at most 16384 of them. (This also means that the intermediate directory names are not a leading prefix of the storage-index directory names -- to do that would have required us to have intermediate directories limited to either 1024 (2-char), which is too few, or 32768 (3-chars of a full 5 bits each), which would overrun ext3's funny hard limit of 32,000.)) This closes #150, and please see the "convertshares.py" script attached to #150 to convert your old tahoe-0.7.0 storage/shares directory into a new tahoe-0.8.0 storage/shares directory. ] [test_cli: oops, need to update this when the CHK hash changes warner@allmydata.com**20080207015853] ['tahoe dump-cap': accept http:// -prefixed URLs too warner@allmydata.com**20080114201227] [webish: censor all caps before logging the HTTP request, to preserve user privacy warner@allmydata.com**20080213013123] ['tahoe catalog-shares': add SDMF filesize to the output, update misc/find-share-anomalies.py to match warner@allmydata.com**20080213211206] [catalog-shares: add expiration time to output warner@allmydata.com**20080212004454] [add 'tahoe catalog-shares' tool, to make a one-line summary of each share file. This can help do cross-server correlation of sharefiles, looking for anomalies warner@allmydata.com**20080212001701] [add 'tahoe find-shares' command, to locate share files on a local node's disk warner@allmydata.com**20080206191951] [add 'tahoe dump-cap' command, to show storage index, lease secrets, etc warner@allmydata.com**20080114194325] [test_cli.py: hush pyflakes with a dummy usage, until we get some real CLI tests warner@lothar.com**20071011085529] [misc/find-share-anomalies.py: tool to analyze 'tahoe catalog-shares' output and look for problems warner@allmydata.com**20080212015336] [webish: add primitive publish/retrieve status pages warner@allmydata.com**20080304070744] [webish: make upload timings visible on the recent uploads/downloads status page warner@allmydata.com**20080303204852] [webish: add when_done= to POST /uri?t=upload . I did not add a 'recent uploads' section to the welcome page, but I think the new upload-results page provides the desired data warner@allmydata.com**20080206083816] [webish: add per-file upload/download status pages warner@allmydata.com**20080301050300] [retain 10 most recent upload/download status objects, show them in /status . Prep for showing individual status objects warner@allmydata.com**20080301041903] [webish: display tahoe import path on the welcome page, to help figure out where the code is coming from warner@allmydata.com**20080206020849] [make current upload/download status objects available from the client warner@allmydata.com**20080212213945] [webish status: distinguish active uploads/downloads from recent ones warner@allmydata.com**20080226213528] [add test coverage for the /stats web page warner@allmydata.com**20080213195739] [current-downloads status: add SI, size, make numsegs 1-based warner@allmydata.com**20080226210235] [webish: add /status page to display current uploads/downloads warner@allmydata.com**20080212214005] [download status: refactor into a separate object, so we don't need to keep the Download itself around for a long time warner@allmydata.com**20080213010103] [add download-status objects, to track download progress warner@allmydata.com**20080212213839] [add upload-status objects, to track upload progress warner@allmydata.com**20080212213605] [offloaded: close the local filehandle after encoding is done, otherwise windows fails warner@lothar.com**20080117075233] [encode.py: don't record BAD log event unless there is actually a problem warner@allmydata.com**20080128175910] [encode.py: improve error message when segment lengths come out wrong warner@allmydata.com**20080125035109] [encode.py: don't allow a shareholder which dies in start() to kill the whole upload warner@allmydata.com**20080128181448] [add upload-results timing info for helper uploads. This changes the Helper protocol, and introduces a compatibility break warner@allmydata.com**20080206075225] [upload.py: start removing wait_for_numpeers code warner@allmydata.com**20080110032518] [offloaded: reinstate fix for windows tests robk-tahoe@allmydata.com**20080121212515 in a discussion the other day, brian had asked me to try removing this fix, since it leads to double-closing the reader. since on my windows box, the test failures I'd experienced were related to the ConnectionLost exception problem, and this close didn't see to make a difference to test results, I agreed. turns out that the buildbot's environment does fail without this fix, even with the exception fix, as I'd kind of expected. it makes sense, because the reader (specifically the file handle) must be closed before it can be unlinked. at any rate, I'm reinstating this, in order to fix the windows build ] [offloaded: close reader before removing its file robk-tahoe@allmydata.com*-20080117233628 unlinking a file before closing it is not portable. it works on unix, but fails since an open file holds a lock on windows. this closes the reader before trying to unlink the encoding file within the CHKUploadHelper. ] [offloaded: close reader before removing its file robk-tahoe@allmydata.com**20080117233628 unlinking a file before closing it is not portable. it works on unix, but fails since an open file holds a lock on windows. this closes the reader before trying to unlink the encoding file within the CHKUploadHelper. ] [offloaded.py: delete encoding tempfile when upload is complete warner@lothar.com**20080117071554] [offloaded upload: avoid tail-recursion problem that would break large files warner@allmydata.com**20080125035134] [offloaded: when uploading a file that failed to upload before, ask for the last byte of ciphertext, so the reader is prepared to give us the plaintext hashes warner@allmydata.com**20080129010543] [upload: add log message when AssistedUploader is done warner@allmydata.com**20080129233812] [offloaded.py: fix logging a bit warner@allmydata.com**20080131194501] [add upload timings and rates to the POST /uri?t=upload results page warner@allmydata.com**20080206064151] [encode.py: update logging levels warner@allmydata.com**20080128181527] [encode.py: log the contents of the uri_extension block warner@allmydata.com**20080124000804] [introducer: remove remaining bits of 'push-to-myself' flags. The uploading/downloading node is no longer special. warner@allmydata.com**20080205201601] [webish: make POST /uri?t=upload deposit you on an 'Upload Results' page warner@allmydata.com**20080206050137] [webish: add extra introducer data (version, timestamps) to Welcome page warner@allmydata.com**20080205233227] [tweak webish to use resource_filename to find css and html files robk-tahoe@allmydata.com**20080122234458 using sibpath to find web template files relative to source code is functional when running from source environments, but not especially flexible when running from bundled built environments. the more 'orthodox' mechanism, pkg_resources, in theory at least, knows how to find resource files in various environments. this makes the 'web' directory in allmydata into an actual allmydata.web module (since pkg_resources looks for files relative to a named module, and that module must be importable) and uses pkg_resources.resource_filename to find the files therein. ] [added tweaked sibpath implementation robk-tahoe@allmydata.com**20080110212341 use of twisted.python.util.sibpath to find files relative to modules doesn't work when those modules are bundled into a library by py2exe. this provides an alternative implementation (in allmydata.util.sibpath) which checks for the existence of the file, and if it is not found, attempts to find it relative to sys.executable instead. ] [provisioning: add some drive failure and repair rate info warner@allmydata.com**20070907014741] [display the Helper FURL and our connection status on the welcome page. Closes #285. warner@allmydata.com**20080128195622] [upload: rework passing of default encoding parameters: move more responsibility into BaseUploadable warner@allmydata.com**20080207003903] [upload.py: make it easier to have an IUploadable that overrides encoding parameters: just set an attribute instead of subclassing warner@lothar.com**20080117071742] [change encryption-key hash to include encoding parameters. This is a minor compatibility break: CHK files encoded (with convergence) before and after this will have different keys and ciphertexts. Also switched to SHA-256d for both the data-to-key hash and the key-to-storageindex hash warner@allmydata.com**20080207015047] [rename storage_index_chk_hash() to storage_index_hash() and add TODO about how our use of it now includes keys that are not CHKs zooko@zooko.com**20080201182737] [rename "dir-uri" to "dir-cap" zooko@zooko.com**20080108164127] [cmdline: give useful error messages about the --dir-uri and ~/.tahoe/private/root_dir.cap zooko@zooko.com**20080103233535] [test_util: add full coverage for allmydata.util.deferredutil warner@allmydata.com**20080206224104] [remove tests of logging functionality that's been subsumed by foolscap logging warner@lothar.com**20071213022353] [test_node.py: more coverage of Node.log warner@lothar.com**20071031075659] [logging: only test log.err when Twisted is new enough to let us ignore the generated errors warner@allmydata.com**20071120003700] [test_system: remove the hackish debug_interrupt= attribute magic used to exercise interrupted-upload resumption, instead just make the Uploadable bounce the helper halfway through the upload warner@allmydata.com**20080208021537] [offloaded uploader: don't use a huge amount of memory when skipping over previously-uploaded data warner@allmydata.com**20080124232533] [test_system.py: remove that ugly debug_stash_RemoteencryptedUploadable hack, now that UploadResults give us a better approach warner@allmydata.com**20080207232730] [offloaded: upload.py: handle forward skips, to allow resumed uploads to send less than all the data. We still read all the data (to hash it, 'paranoid mode'), but we don't send it over the wire warner@lothar.com**20080117071656] [big introducer refactoring: separate publish+subscribe. Addresses #271. warner@allmydata.com**20080205190513] [assert that only dicts get passed to _got_response() zooko@zooko.com**20070331010040] [bump timeout up because it timed out even on my super fast MacBook Pro zooko@zooko.com**20070501061606 Hm. This probably means that it is never going to finish... ] [Client.get_permuted_peers: use self.nodeid now that it's fixed warner@lothar.com**20070812232451] [storage: add version number to share data. Closes #90. warner@allmydata.com**20070904160024] [storageserver.py: remove unused import warner@allmydata.com**20070418174333] [storageserver.ReadBucketProxy: break out _parse_offsets, for debug tools warner@lothar.com**20070713235217] [client.py: add the missing remote_get_nodeid() method claimed in interfaces.py warner@allmydata.com**20070926192048] [client.py: remove unused code warner@allmydata.com**20070629005513] [fix test_introducer to wait for the right things in order to avoid intermittent test failures zooko@zooko.com**20071211200815] [fix IntroducerClient.when_enough_peers() zooko@zooko.com**20071211022259 add IntroducerClient.when_few_enough_peers(), fix and improve test_introducer ] [fix import error, detected thanks to "make test" zooko@zooko.com**20071211232301] [fix missing import, thanks to pyflakes zooko@zooko.com**20071211230029] [mutable: improve logging: mark events with level=log.WEIRD and log.UNUSUAL warner@allmydata.com**20080107230916] [mutable.Publish: improve logging warner@allmydata.com**20071219053050] [mutable: verify incoming share signatures during Publish, it's not that expensive and it's a good idea warner@allmydata.com**20071108200236] [mutable.py: one more logging fix warner@allmydata.com**20071219054241] [storage: improve logging a bit warner@allmydata.com**20080114175858] [storage.py: add a little logging (disabled) warner@allmydata.com**20071107201454] [storage.py: factor out a common compare() routine warner@allmydata.com**20071205062034] [test_upload.py: implement remote_abort on our fake BucketWriter warner@allmydata.com**20080124000734] [stats: add a simple stats gathering system robk-tahoe@allmydata.com**20080131021107 We have a desire to collect runtime statistics from multiple nodes primarily for server monitoring purposes. This implements a simple implementation of such a system, as a skeleton to build more sophistication upon. Each client now looks for a 'stats_gatherer.furl' config file. If it has been configured to use a stats gatherer, then it instantiates internally a StatsProvider. This is a central place for code which wishes to offer stats up for monitoring to report them to, either by calling stats_provider.count('stat.name', value) to increment a counter, or by registering a class as a stats producer with sp.register_producer(obj). The StatsProvider connects to the StatsGatherer server and provides its provider upon startup. The StatsGatherer is then responsible for polling the attached providers periodically to retrieve the data provided. The provider queries each registered producer when the gatherer queries the provider. Both the internal 'counters' and the queried 'stats' are then reported to the gatherer. This provides a simple gatherer app, (c.f. make stats-gatherer-run) which prints its furl and listens for incoming connections. Once a minute, the gatherer polls all connected providers, and writes the retrieved data into a pickle file. Also included is a munin plugin which knows how to read the gatherer's stats.pickle and output data munin can interpret. this plugin, tahoe-stats.py can be symlinked as multiple different names within munin's 'plugins' directory, and inspects argv to determine which data to display, doing a lookup in a table within that file. It looks in the environment for 'statsfile' to determine the path to the gatherer's stats.pickle. An example plugins-conf.d file is provided. ] [add some munin plugins warner@allmydata.com**20070705203815] [add mac native build robk-tahoe@allmydata.com**20080123013226 This patch adds support for a mac native build. At the moment it's a fairly simple .app - i.e. so simple as to be unacceptable for a shipping product, but ok for testing and experiment at this point. notably once launched, the app's ui does not respond at all, although its dock icon does allow it to be force-quit. this produces a single .app bundle, which when run will look for a node basedir in ~/.tahoe. If one is not found, one will be created in ~/Library/Application Support/Allmydata Tahoe, and that will be symlinked to ~/.tahoe if the basedir is lacking basic config (introducer.furl and root_dir.cap) then the wx config wizard will be launched to log into an account and to set up those files. if a webport file is not found, the default value of 8123 will be written into it. once the node has started running, a webbrowser will be opened to the webish interface at the users root_dir note that, once configured, the node runs as the main thread of the .app, no daemonisation is done, twistd is not involved. the binary itself, from within the .app bundle, i.e. "Allmydata Tahoe.app/Contents/MacOS/Allmydata Tahoe" can be used from the command line and functions as the 'tahoe' executable would in a unix environment, with one exception - when launched with no args it triggers the default behaviour of running a node, and if necessary config wizard, as if the user had launched the .app one other gotcha to be aware of is that symlinking to this binary from some other place in ones $PATH will most likely not work. when I tried this, something - wx I believe - exploded, since it seems to use argv[0] to figure out where necessary libraries reside and fails if argv[0] isn't in the .app bundle. it's pretty easy to set up a script a la #!/bin/bash /Blah/blah/blah/Allmydata\ Tahoe.app/Contents/MacOS/Allmydata\ Tahoe "${@}" ] [simplify buildbot upload of windows installer robk-tahoe@allmydata.com**20080117022930 since the installer upload got more complex (needing to chmod files before rsyncing) I promoted it to a makefile target, simplifying the buildbot steps involved ] [Makefile: move use of 'cygpath' into win32-conditionalized section warner@allmydata.com**20080115002236] [setup: check for the pywin32 dep only on Windows zooko@zooko.com**20070921211116] [Makefile: use absolute path to src/ in PP, since check-memory needs it (it chdirs then imports allmydata) warner@lothar.com**20070915031743] [Makefile: prepend src/ to the PYTHONPATH zooko@zooko.com**20070914024315] [windows installer build refinements robk-tahoe@allmydata.com**20080114235354 this resolves problems of py2exe's modulefinder collection of sources from .zipped egg files, not by using easy_install to reach the --always-unzip option, but rather with a small tool which unpacks any zipped egg files found in misc/dependencies. this fixes the py2exe build given rollback of the easy_install stuff which had broken the unix builds. misc/hatch-eggs.py performs the honours. this also includes a misc/sub-ver.py tool which substitutes elements of the verion number for the current code base (by importing allmydata.__version__ hence make-version should be run first, and the python path carefully managed) into template files using python's string interpolation of named args from a dict as the templating syntax. i.e. %(major)d %(minor)d %(point)d %(nano)d each expand to the individual components of the version number as codified by the pyutil.version_class.Version class. there is also a %(build)s tag which expands to the string form of the whole version number. This tool is used to interpolate the automatically generated version information into the innosetup source file in a form consistent with innosetup/windows' restrictions ] [add windows installer target to build robk-tahoe@allmydata.com**20080112024121 add 'windows-installer' target to top level makefile to build a windows setup.exe package using innosetup. this assumes innosetup 5 is installed in program files as normal. this doesn't include any logic to manage version numbers at this point, it's just a simple experiment to test out building an installer as yet. ] [add windows-exe target to makefile robk-tahoe@allmydata.com**20080110010628] [first stab at windows build details. robk-tahoe@allmydata.com**20080110010156 there are many and various fiddly details that were involved in this process on mountain view. This is a stripped down version of the build process used there. there's hence a good chance that one or two necessary details got stripped down through the cracks. this provides a py2exe setup.py to build a tahoe.exe and a tahoesvc.exe the former is equivalent to bin/tahoe, but without the start/stop commands. the latter is a windows service that instantiates a client whose basedir is found in the registry. ] [introducer: allow nodes to refrain from publishing themselves, by passing furl=None. This would be useful for clients who do not run storage servers. warner@allmydata.com**20080202014838] [client.py: touch BASEDIR/no_storage to not publish a storage server. Addresses #271 warner@allmydata.com**20080202020708] [client.py: hush pyflakes warner@allmydata.com**20080202022815] [test_web: fix webapi test around redirection issues robk-tahoe@allmydata.com**20080611221917 this fixes the test_web test test_POST_upload_mutable which chdir's into a /foo subdirectory, but then later asserts that the web ui redirects the user back to /foo, which should really be /foo/ since it's a directory. ] [test_web: implement API changes from the recent webapi.txt overhaul warner@allmydata.com**20080519195602] [webish: test error cases more thoroughly by looking inside the response text warner@allmydata.com**20070716190119] [make the unit test of the overwrite button more general zooko@zooko.com**20071220212307] [web: return a proper error upon POST with a bad t= value warner@allmydata.com**20080415181129] [wapi: add POST /uri/$DIRECTORY?t=set_children zooko@zooko.com**20080301004027 Unfinished bits: doc in webapi.txt, test handling of badly formed JSON, return reasonable HTTP response, examination of the effect of this patch on code coverage -- but I'm committing it anyway because MikeB can use it and I'm being called to dinner... ] [test_web.py: remove last use of fake_* methods, remove dead code warner@allmydata.com**20071205051100] [editing: change names like "MyThing" to "FakeThing" for fake objects in unit tests zooko@zooko.com**20071212001029] [webish.py: refactor /uri handlers, one rend.Page class per operation warner@allmydata.com**20080206043820] [webish: factor out queryargs-or-formargs extraction, make /uri?t=mkdir work and redirect to a valid url warner@lothar.com**20071224232652] [test_web: more coverage of URIPOSTHandler error cases warner@lothar.com**20071225044935] [webish: add PUT /uri?mutable=true warner@allmydata.com**20080206041802] [webish: improve test coverage warner@lothar.com**20070917085346] [webish: add POST /uri?t=upload&mutable=true warner@allmydata.com**20080206041022] [tests: put back skipped and todo tests zooko@zooko.com**20080115030241 closes #258 -- "put back skipped and todo tests" ] [suppress the skipped and the todo unit tests in order to make unit test results prettier (the install.html instructs users to run the unit tests) zooko@zooko.com**20080104063618] [refactor webish.py slightly, improve the test coverage a bit. One test is marked SKIP pending improvements in webish.py warner@lothar.com**20071225094857] [webish: add upload/view-uri forms (not associated with any particular directory) to the welcome page. Document POST /uri?t=upload . warner@allmydata.com**20080206034440] [webish: add button to make directories and unit test of it zooko@zooko.com**20071220213112 Unfortunately although it passes the unit tests, it doesn't work, because the unit tests and the implementation use the "encode params into URL" technique but the button uses the "encode params into request body" technique. ] [web: replace welcome-page download-URI form with new version warner@lothar.com**20070708050622] [test: refactor webist.POSTHandler() to have a method for each "?t=" command zooko@zooko.com**20080229191118 Input validation and decoding is still done in the body of POSTHandler.renderHTTP(). ] [test_web.py: add coverage for POST t=check warner@allmydata.com**20071205054938] [upload: return an UploadResults instance (with .uri) instead of just a URI warner@allmydata.com**20080206030138] [check-speed: test SSK upload/download speed too. SDMF imposes a limit on the file sizes, no 10MB or 100MB test warner@lothar.com**20071214080531] [check_speed: add optional 100MB test, if the 10MB test finished fast enough Brian Warner **20070922070446] [check_speed.py: use more small-file tests to improve accuracy of per-file time Brian Warner **20070926015736] [check_speed: measure RTT, report per-file times as a multiple of RTT warner@allmydata.com**20070926200733] [check_speed.py: minor comment warner@allmydata.com**20070926030703] [check_speed: average multiple pings when measuring RTT warner@allmydata.com**20070927011615] [upload-helper: avoid duplicate uploads: check the grid to see if the file already exists warner@allmydata.com**20080131004902] [remove upload.upload_(data,filename,filehandle) convenience functions warner@allmydata.com**20080131010319] [CHK upload helper: don't let one failed upload prevent us from trying again warner@allmydata.com**20080128185813] [offloaded.py: hush pyflakes warner@lothar.com**20080111110514] [offloaded: improve logging across the board warner@lothar.com**20080117071135] [download: use hierarchical logging warner@allmydata.com**20071120010710] [make content-hash-key encryption a parameter of uploading zooko@zooko.com**20080130182450 fixes #293 ] [offloaded: update unit tests: assert that interrupt/resume works, and that the helper deletes tempfiles warner@lothar.com**20080117071810] [megapatch: overhaul encoding_parameters handling: now it comes from the Uploadable, or the Client. Removed options= too. Also move helper towards resumability. warner@lothar.com**20080116090335] [storage.py: add a test for the next_power_of_k fix I made a few hours ago, basically do an upload with a non-power-of-two number of segments warner@lothar.com**20070714052406] [reduce MAX_SEGMENT_SIZE from 2MB to 1MB, to compensate for the large blocks that 3-of-10 produces warner@allmydata.com**20070716204834] [interfaces: increase ShareSize now that our default k is smaller (hence blocks are bigger) warner@lothar.com**20070714022931] [upload.py: fix signature of NonConvergentUploadMixin.get_encryption_key warner@lothar.com**20070919063811] [update a few documents, comments, and defaults to mention 3-of-10 instead of 25-of-100 zooko@zooko.com**20071016025359] [architecture.txt: update to include tahoe2, dirnodes, leases warner@allmydata.com**20070918012448] [tests: make test_encode specify the erasure coding params it wants instead of expecting the defaults to be what it wants zooko@zooko.com**20071016030742] [logging: enable flogging in more places, replace Node.log with flogging warner@allmydata.com**20080115031658] [upload: pass options through to the encoder warner@allmydata.com**20080115031732] [upload: add Encoder.abort(), to abandon the upload in progress. Add some debug hooks to enable unit tests. warner@allmydata.com**20080115032255] [storage: remove the leftover incoming/XYZ/ directory when we're done with it warner@lothar.com**20070915213404] [upload: improve logging warner@allmydata.com**20080115031920] [offloaded: cleanup to handle multiple simultaneous uploaders gracefully warner@allmydata.com**20080115042003] [offloaded: improve logging, pass through options, get ready for testing interrupted uploads. test_system: add (disabled) interrupted-upload test warner@allmydata.com**20080115032426] [offloaded: more test coverage on client side, change interfaces a bit warner@lothar.com**20080111105337] [offloaded: add a system test, make it pass. files are now being uploaded through the helper. warner@lothar.com**20080111114255] [test_system: slight refactoring to eventually make it easier to configure some nodes with the output of others warner@allmydata.com**20080110022354] [upload: oops, fix breakage after removing upload_file/upload_data/etc warner@allmydata.com**20080131014143] [interfaces.py: remove spurious 'pass' statements (which, incidentally, were counted as uncovered code) warner@allmydata.com**20070418224637] [webish: flogify the remaining log messages warner@allmydata.com**20080213013233] [unicode handling: declare dirnodes to contain unicode child names, update webish to match warner@allmydata.com**20080214214556] [improve test coverage on FileNode.check warner@allmydata.com**20071204215527] [fix the overwrite button and add unit test for it zooko@zooko.com**20071214235205] [webish: fix overwrite form display warner@allmydata.com**20071205061513] [test_web.py: add coverage for directory listings that include mutable files warner@allmydata.com**20071205055740] [the new pyflakes is stricter, complaining about function definitions that overshadow earlier definitions or imports. Fix some of its complaints. warner@allmydata.com**20071219004728] [webish: use get_args() everywhere, put .fields=None in MyRequest warner@lothar.com**20071225060704] [webish.POSTHandler: fix typo that meant we didn't look for 'name' in req.fields warner@lothar.com**20070811002528] [webish: strip leading/tailing whitespace from user-provided URIs warner@lothar.com**20070825190506] [webish: append save=true to a GET URL to save-to-disk. Closes #222. warner@allmydata.com**20071212000444] [download.py: use producer/consumer to reduce memory usage, closes #129. warner@lothar.com**20070919073447 If the DownloadTarget is also an IConsumer, give it control of the brakes by offering ourselves to target.registerProducer(). When they tell us to pause, set a flag, which is checked between segment downloads and decodes. webish.py: make WebDownloadTarget an IConsumer and pass control along to the http.Request, which already knows how to be an IConsumer. This reduces the memory footprint of stalled HTTP GETs to a bare minimum, and thus closes #129. ] [make the wapi/wui [*] support creation of a new directory without there already being an existing directory to link the new one into zooko@zooko.com**20071220185817 [*] WebAPI/WebUI ] [dirnode.py: add metadata= to add_file(), add tests warner@allmydata.com**20080211205328] [test_dirnode.py: assert that we update mtime and preserve ctime warner@allmydata.com**20080211211255] [test_dirnode.py: add diag output to test-ctime/mtime tests warner@allmydata.com**20080211201307] [test_dirnode.py: simplejson-1.7.1 incorrectly rounds floats to two decimal places. Don't let this bug flunk the timestamp test. warner@allmydata.com**20080211233517] [dirnode: add ctime/mtime to metadata, update metadata-modifying APIs. Needs more testing and sanity checking. warner@allmydata.com**20080209004347] [remove wait_for_numpeers and the when_enough_peers call in mutable.Publish warner@allmydata.com**20080114205559] [merge patch to integrate decentralized directories with patch "download: use hierarchical logging" zooko@zooko.com**20071203212721] [the wait_for_numpeers= argument to client.upload() is optional: make both the code and the Interface reflect this warner@allmydata.com**20071207003658] [tidy-up some comments and logging messages zooko@zooko.com**20071215002446] [dirnode: add set_uris() and set_nodes() (plural), to set multiple children at once. Use it to set up a new webapi test for issue #237. warner@allmydata.com**20071219053002] [mutable: always include ourselves in the querylist when Publishing, because 0.7.0 usually stores our root dirnode only on ourselves, and our nodeid might not appear in the first N+epsilon peers warner@allmydata.com**20071219060653] [merge patch to integrate decentralized directories with patch to "only test log.err when Twisted is new enough to let us ignore the generated errors" zooko@zooko.com**20071203212514] [merge patch to integrate decentralized directories with patch to handle bad hashes zooko@zooko.com**20071203212114] [offloaded: move interfaces to interfaces.py, start implementing backend warner@allmydata.com**20080110032547] [control: add measure_peer_response_time(), to estimate RTT for the mesh warner@allmydata.com**20070926192115] [more hierarchical logging: download/upload/encode warner@allmydata.com**20071120013341] [encode.py: add a reactor turn barrier between segments, to allow Deferreds to retire and free their arguments, all in the name of reducing memory footprint warner@allmydata.com**20070810012617] [Encoder.__repr__: mention the file being encoded warner@allmydata.com**20070810012656] [encode.py: log a percentage complete as well as the raw byte counts warner@allmydata.com**20070810012845] [tidiness: return res from logging of progress "Zooko O'Whielacronx "**20070402172324] [trailing-whitespace eradication, no functional changes warner@allmydata.com**20071101222238] [fix representation of node ids in PeerTracker objects zooko@zooko.com**20071219235528] [offloaded: more code, fix pyflakes problems, change IEncryptedUploader a bit warner@allmydata.com**20080109235847] [use AES from pycryptopp instead of pycrypto, also truncate the keys slightly differently warner@allmydata.com**20071203232746] [docs/mutable.txt: put fingerprint in read-write URI too, it makes everything easier warner@allmydata.com**20071031233015] [setup: require pycryptopp >= 0.2.6 zooko@zooko.com**20071110001500] [setup: require pycryptopp >= v0.2.5 zooko@zooko.com**20071109190315] [setup: tell setuptools that we depend on pycryptopp >= 0.2.3 zooko@zooko.com**20071107235518] [setup: don't include zope.interface in our automatically-satisfiable dependencies for now zooko@zooko.com**20070927220617] [setup.py: name zope.interface >= 3.0 as one of our dependencies zooko@zooko.com**20070920174650] [setup: upgrade bundled pycryptopp from v0.2.5 to v0.2.6 zooko@zooko.com**20071110001436] [setup: upgrade the bundled pycryptopp tarball from pycryptopp v0.2.3 to pycryptopp v0.2.5 zooko@zooko.com**20071109190249] [setup: upgrade to pycryptopp v0.2.3 zooko@zooko.com**20071107235446] [mutable: fix control flow to allow good+bad shares from a peer. Fixes #211. warner@allmydata.com**20071116231233] [mutable: add more logging to retrieval process warner@allmydata.com**20071115202256] [test_mutable: workaround: use more peers to avoid random test failures. warner@allmydata.com**20071115205500 The underlying issue is recorded in #211: one corrupt share in a query response will cause us to ignore the remaining shares in that response, even if they are good. In our tests (with N=10 but only 5 peers), this can leave us with too few shares to recover the file. The temporary workaround is to use 10 peers, to make sure we never get multiple shares per response. The real fix will be to fix the control flow. This fixes #209. ] [offloaded: create a Helper if 'run_helper' is non-empty warner@allmydata.com**20080110022505] [remove automatic private dir zooko@zooko.com**20080103230205 * rename my_private_dir.cap to root_dir.cap * move it into the private subdir * change the cmdline argument "--root-uri=[private]" to "--dir-uri=[root]" ] [web: change title to say 'Tahoe', not tahoe2 warner@lothar.com**20070708043038] [remove leftover defer.setDebugging(), to speed up tests from 200s to 83s warner@allmydata.com**20071203231002] [we no longer need to replace "/" with "!" in urls zooko@zooko.com**20071215002550] [add POST /uri?t=upload and tests thereof zooko@zooko.com**20071206231702 Hm... I refactored processing of segments in a way that I marked as "XXX HELP I AM YUCKY", and then I ran out of time for rerefactoring it before I committed. At least all the tests pass. ] [webish: add POST t=mutable, make it replace files in-place, add t=overwrite warner@allmydata.com**20071205054254] [docs/webapi.txt: document the POST t=upload&mutable=on command used to create mutable files warner@lothar.com**20071109100507] [fix unit tests to assert that we do *not* link to start.html when there is no private dir, instead of asserting that we *do* zooko@zooko.com**20071218005116] [do not put a link from the welcoÂme page to the start.html if there is not a private directory zooko@zooko.com**20071218003559] [rename "my_private_dir.uri" to "my_private_dir.cap" zooko@zooko.com**20071218003525] [put all private state in $BASEDIR/private zooko@zooko.com**20071217223954 fixes #219 The only part of #219 that this doesn't include is the part about logpublisher, which has been moved out of tahoe into foolscap. ] [node: change get_or_create_config to strip whitespace and accept a filemode= argument warner@lothar.com**20070828022350] [node.py: change get_or_create_config() to accept a function warner@lothar.com**20070828020712] [remove test in test_client -- we can't easily assert that files aren't readable by others, on Windows zooko@zooko.com**20070914031226] [node.py: chmod the foolscap private key (node.pem) to 0600, since it's secret warner@allmydata.com**20070921235255] [node.py: use Tub's certFile= argument instead of doing it ourselves warner@allmydata.com**20070523194123] [don't test for existence of certfile and then try to open it -- instead try to open it and catch exception "Zooko O'Whielacronx "**20070522210416 This avoids a race condition, also known as time-of-check-to-time-of-use. ] [move increase_rlimits() into iputil and make it a no-op on Windows zooko@zooko.com**20071207140343] [fix iputil so that it doesn't launch dozens of processes when you give it a full path and so that it tries executables in preference order and stops as soon as one gives out a dotted-quad string zooko@zooko.com**20070809175647] [iputil.get_local_ip_for: tolerate running on a disconnected host Brian Warner **20070608022333] [refactor iputil and make it return addresses in descending order of goodness instead of in a set "Zooko O'Whielacronx "**20070522210637 Actually of course iputil can't tell exactly how good they are, and a wise user of iputil will try all of them. But you can't try all of them simultaneously, so you might as well try the best ones first. ] [node.py: try to fix rlimit-setting again warner@allmydata.com**20071116221238] [node.py: try rlimit fix again warner@allmydata.com**20071116050902] [node.py: try to fix RLIMIT_NOFILE on solaris too warner@allmydata.com**20071116050644] [remove logpublisher, use the Foolscap version now that this functionality has been moved into Foolscap-0.2.2 warner@lothar.com**20071213023101] [introducer_and_vdrive.py: use logpublisher too warner@allmydata.com**20071120202226] [refactor the feature of getting versions of packages, include the version numbers of more of the packages that we use zooko@zooko.com**20071213013737] [setup: print out the version number of pycryptopp in "tahoe --version" zooko@zooko.com**20071107161156] [setup: add Crypto++ and pycryptopp to dependencies zooko@zooko.com**20071107160013 Crypto++ is a new manual dependency (boo hoo), and pycryptopp is a new automatic dependency. ] [setup: split off README.win32 from README and paste in Mike Booker's notes about building OpenSSL zooko@zooko.com**20071015160841] [CREDITS: nejucomo zooko@zooko.com**20070918214313] [update CREDITS file zooko@zooko.com**20070629212230] [add CREDITS file where people who contribute source code, docs, web page updates, etc., are recorded "Zooko O'Whielacronx "**20070506211143] [README: OpenSSL is bundled with pyOpenSSL on Windows-native zooko@zooko.com**20071004151253 ] [README: add note about how to build on Windows-native using gcc, and a bit of editing zooko@zooko.com**20071004193721] [setup: add misc/dependencies/pycryptopp-0.2.1.tar.gz zooko@zooko.com**20071107155951] [node.py: raise RLIMIT_NOFILE on bsd/cygwin to more than 256 warner@allmydata.com**20071116045355] [patch the LogObserver in a more modern, forward-compatible way and update the in-line comments about it czooko@zooko.com**20071022235255] [a slightly nicer method of computing our timestamp format zooko@zooko.com**20071015034651] [logpublisher: hush pyflakes warning warner@allmydata.com**20071117021446] [logpublisher: implement subscribe/publish for log, add a sample client warner@lothar.com**20071117020750] [implement preliminary log publisher/gatherer warner@allmydata.com**20071102002915 This creates a Referenceable object that will eventually be able to publish log events to a remote subscriber (at present all it can do is provide version information). The FURL for this logport is written to 'logport.furl'. In addition, if a file named 'log_gatherer.furl' is present, the given target will be contacted and offered access to the logport. This can be used by a centralized logging agent to subscribe to logs, e.g. from all the nodes in a centrally-maintained storage grid. (think syslog -r, but with all the security properties of FURLs, and permitting non-printable strings and structured data). Once this framework matures a bit, it will be moved into Foolscap. ] [test_node.py: hush pyflakes warnings warner@allmydata.com**20070524005504] [update docs about webport (fixes #185) zooko@zooko.com**20071023005052] [README: use 8123 instead of 8080/8443 as the example port numbers zooko@zooko.com**20070924185524 If people follow the example, I'd like for them to land on an otherwise little-claimed port number in case we standardize on it in order to facilitate exchange of URLs. ] [change another example to use port 8123 zooko@zooko.com**20071011231213] [README: give a link to the TestGrid page zooko@zooko.com**20071015215553] [test_node.py: improve test coverage of node.py warner@lothar.com**20071031075442] [hierarchical logging: add numbered messages and parent= args warner@allmydata.com**20071120002318] [add unit test for "advertised_ip_addresses" feature and fix bug in that feature uncovered by this unit test "Zooko O'Whielacronx "**20070523220855] [Add some passing unit tests for testutil.PollMixin. nejucomo@gmail.com**20070907231541] [fileutil: add du() function warner@allmydata.com**20070703224945] [test_util: improve test coverage of allmydata.util.fileutil warner@lothar.com**20070703181505] [test_util: add more coverage for assertutil.py warner@lothar.com**20070408200213] [test_util: add full coverage for mathutil.py warner@lothar.com**20070408194301] [install our custom timestamp formats in a less disruptive way zooko@zooko.com**20071015034311 The unit tests on Windows fail because trial is attempting to remove its own log observer during teardown. This patch customizes the extant log observer object by replacing its formatTime method with our own. I first tried the approach of storing their log observer object and putting it back during teardown, but it didn't work (perhaps because our node object doesn't get a chance to do its deferred stopService behavior in time), and anyway I generally prefer the "fail-safe", or "crash-only" design. ] [remove unused imports (thanks, pyflakes) zooko@zooko.com**20071015153221] [node.py: fix timestamps (add ms and Z) by replacing the FileLogObserver. #171. warner@allmydata.com**20071012003007] [node.py: log twisted version along with tahoe/foolscap/zfec versions warner@allmydata.com**20070531182106] [node.py: don't append 'Z' to the timestamp, since it's really localtime. We need deeper changes to make it be UTC warner@lothar.com**20071011092417] [node.py: set logging timestamp to '2007-10-11 02:11:14.000Z', per ticket #171. No milliseconds yet, though warner@lothar.com**20071011091305] [trying to introduce old style humanreadablied logs hopefully without breaking the existing ones tahoe@arnowaschk.de**20070811215237] [mutable: handle bad hashes, improve test coverage, rearrange slightly to facilitate these warner@allmydata.com**20071114050815] [mutable: make error handling more robust warner@allmydata.com**20071107234545] [rename "secret" to "lease_secret" and change its size from 16 to 32 bytes zooko@zooko.com**20071218003411] [my_private_dir.cap: add newline for readability, open with mode 'w' not 'w+' since we don't need to read from this filehandle warner@allmydata.com**20071218024848] [move my_private_dir.cap into private/ warner@allmydata.com**20071218025740] [remove some no-longer needed replacements of "/" with "!" in uris zooko@zooko.com**20071219235440] [A hastily written single-threaded read-only fuse client. nejucomo@gmail.com**20071120072150] [add human-encodings of caps, refactor encodings of caps, tighten checks, fix unit tests to use values of the right size zooko@zooko.com**20080103225543] [unit test that POST /uri/?t=mkdir works zooko@zooko.com**20071218234749] [refactor constructing URI objects from strings and add human-encoding of WriteableSSKFileURI zooko@zooko.com**20080103041215] [remove the DirnodeURI foolscap schema and mv those regexes into uri.py zooko@zooko.com**20071218234424 We currently do not pass dirnode uris over foolscap. ] [make more precise regexp for WriteableSSKFileURI and DirnodeURI and use it in unit tests zooko@zooko.com**20071218191508 Also allow an optional leading "http://127.0.0.1:8123/uri/". Also fix a few unit tests to generate bogus Dirnode URIs of the modern form instead of the former form. ] [remove most (maybe all?) traces of old Dirnode class. Yay for negative code days. warner@allmydata.com**20071204230058] [uri.py: improve test coverage warner@allmydata.com**20071204233831] [trailing-whitespace eradication, no functional changes warner@allmydata.com**20071101222504] [refactor test_uri and add a test of the empty file zooko@zooko.com**20070726174830] [refactor web tests, and interfaces.IFileNode warner@allmydata.com**20071205050137] [remove PyCrypto, now we only use pycrypto++ warner@allmydata.com**20071204001001] [hush pyflakes warning in Crypto.Util.number warner@allmydata.com**20070814211712] [set the zip_safe flag to False zooko@zooko.com**20070913223755 This means that by default the allmydata-tahoe egg will be a directory with a tree of files instead of a zip file containing files. I prefer the former because it makes it easier for people to hack into it. Unfortunately the files therein are still going to be .pyc's instead of .py's, if I understand correctly. I would prefer for them to be .py's. See also discussion on the distutils-sig mailing list: http://mail.python.org/pipermail/distutils-sig/2007-July/007827.html ] [rename bin/allmydata-tahoe to bin/tahoe. Closes #155. warner@lothar.com**20071011103824] [README: mention strports-ness of NODE/webport, closes #55 warner@allmydata.com**20070606164918] [bin/allmydata-tahoe: add a sys.path-modifying preamble to make it easy to run from source warner@allmydata.com**20070606182400] [/twistd.log -> /logs/twistd.log wilcoxjg@gmail.com**20070809035328] [test_runner: try harder to work on slow buildslaves and cygwin warner@allmydata.com**20070919205600] [client.py: make a note in the logs when the auto-shutdown feature is in use Brian Warner **20070814091230] [test_runner: better diagnostics in case test_client fails warner@allmydata.com**20070919030318] [test_runner.py: fix race conditions in start/stop node, should run on cygwin now warner@allmydata.com**20070918221726] [test_runner.py: skip the start/stop test when sys.platform is win32/cygwin. Brian Warner **20070918045608 The previous twisted.python.runtime.platformType approach didn't catch cygwin for some reason. ] [test_runner.py: add tests for startstop_node.py warner@lothar.com**20070917092531] [Makefile: check-speed: leave the client node stopped when we're done warner@allmydata.com**20070921020316] [setup: use the setuptools "console_scripts" feature to get an allmydata-tahoe.exe on Windows zooko@zooko.com**20070921205627] [README: fix/update description of allmydata-tahoe zooko@zooko.com**20070924185417] [debian/feisty: use our original bin/allmydata-tahoe instead of setuptools's warner@allmydata.com**20070927005230 because the setuptools "entry points" form asserts that there are setuptools-visible packages like nevow/zope.interface (i.e. they have .egg-info metadata). Until very recently, most debian systems did not install this metadata. Instead, we rely upon the usual debian dependency checking as expressed in debian/control . ] [debian/sid: copy rules from feisty, the same can be used for both warner@allmydata.com**20070927005524] [debian: remove the lines that install the now-removed .tac files warner@lothar.com**20070407035132] [debian: depend upon python-zfec, stop including zfec in the tahoe .deb warner@allmydata.com**20070821205451] [debian: make all .debs dependent upon python-simplejson. We must create the dapper package, the others are available upstream warner@allmydata.com**20070711210156] [debian: add python-twisted, python-nevow, python-pyopenssl to the dapper dependencies warner@allmydata.com**20070329192214] [debian: add python-nevow to the sid dependencies warner@allmydata.com**20070329192241] [README: refactor README zooko@zooko.com**20071002201907 fixes #163 hopefully fixes #148 -- but somebody else will have to try it (maybe Mike Booker) to find out! ] [README: update the wording of the "LICENCE" section to more closely follow FSF recommendations zooko@zooko.com**20070912183157] [README: update wording of licence zooko@zooko.com**20070823205130] [README: point to new location of wiki/SetuptoolsAndGNUStow zooko@zooko.com**20070921164345] [README: demote The Debian Way zooko@zooko.com**20070921204257] [README: update on how to test and how to run tahoe after installation zooko@zooko.com**20070921205725] [README: add notes about how to use easy_install for dependencies zooko@zooko.com**20070923124711] [README: fix bug in which pywin32 was mentioned as easy_installable zooko@zooko.com**20070924184124] [README: fix bug in which How To Build was omitted from The Running-In-Place Way to install. zooko@zooko.com**20070924184140] [README: further clarity about when you need to acquire source and when you don't zooko@zooko.com**20070921190401] [README: updated debian-packaging section a bit, replace dapper with etch warner@allmydata.com**20070822000725] [README: update/clarify the build-depends packages need to create .debs Brian Warner **20070817033007] [README: add dependency on OpenSSL zooko@zooko.com**20070929180525] [setup: alas, pywin32 doesn't actually work as an easy_install; also make dependency missing errors more useful zooko@zooko.com**20070921204028] [Makefile: use .built to only require one tahoe-compile pass warner@allmydata.com**20070919192830] [boringfile: ignore .checked-deps warner@lothar.com**20070916082654] [Makefile: fix and-vs-or bug in check-deps, hide errmsg in non-failing cases warner@allmydata.com**20070919232355] [Makefile: give explicit failure message to user if dependencies aren't found zooko@zooko.com**20070920000436] [Makefile: don't re-check dependencies on each test run, and fix clean target warner@lothar.com**20070915221755] [Makefile: clean: remove _trial_temp/ and _test_memory/ too warner@lothar.com**20070915205559] [README: explain the new packaging system zooko@zooko.com**20070921023254 Happily, the README is now shorter and simpler. ] [fix error in README about windows builds "Zooko O'Whielacronx "**20070502054433] [added notes on pywin32 version support to README robk-org@allmydata.com**20070613200936] [require setuptools >= 0.6c6 on cygwin, and >= 0.6a9 on other platforms zooko@zooko.com**20070807211554 Earlier I tried 0.6a9 (which comes in .deb format on Dapper) and something didn't work, so I raised it to 0.6c3. However, I didn't make a note of what failed, and so now I'm hoping that what failed was cygwin-specific. Anyway, everyone look out for setuptools compatibility problems on the your favorite platform (and I'll check the buildslaves), and we'll raise the required version as little as possible and only on the problematic platform. ] [update README to reflect requirement of setuptools 0.6c6 zooko@zooko.com**20070807191259] [setuptools upgrade to 0.6c6 tahoe@arnowaschk.de**20070731231811] [change the 'ez_setup.py' script to have distinct desired & minimum required versions of setuptools robk-org@allmydata.com**20070606194903 and change zfec/setup.py's invocation of ez_setup to require 0.6a9 (which happens to be the default version installed by apt-get on dapper machines) while leaving the default (desired) version at 0.6c5 ] [zfec: switch back to minimum setuptools of 0.6c3 "Zooko O'Whielacronx "**20070430201838 This is newer than the one that comes with Dapper Ubuntu 6.06, *but* if there is no setuptools at all installed, then the bootstrap script will download one itself. So with this patch, this will install cleanly on a Dapper system as long as the Dapper version of python-setuptools is *not* installed. ] [zfec: see if we can get by with the version of setuptools that comes with Ubuntu 6.06 Dapper "Zooko O'Whielacronx "**20070430181647] [zfec: see if we can get by with the version of setuptools that is already in Ubuntu 6.10 "Zooko O'Whielacronx "**20070430180830] [zfec: complete the removal of the 15-second delay when downloading setuptools "Zooko O'Whielacronx "**20070501005643] [zfec: add the setuptools bootstrap script to download setuptools automatically "Zooko O'Whielacronx "**20070430062613] [zfec: update licence, contact info, description "Zooko O'Whielacronx "**20070427181434] [fec: add test for mathutil warner@allmydata.com**20070416190158] [zfec: use setuptools to construct executables, fix bug on Windows -- forgot "BINARY" flag to output files zooko@zooko.com**20070421020612] [rename bin/fec and bin/unfec to zfec and zunfec warner@lothar.com**20070418192333] [zfec: mv the cmdline tools from "bin/" to "cmdline/", because setuptools overwrites the contents of "bin/" when creating executables "Zooko O'Whielacronx "**20070420190914] [zfec: update docs, metadata, version number to 1.0.0a2-0-STABLE zooko@zooko.com**20070418230103] [pyfec: bump version number to 1.0.0a1-2-STABLE "Zooko O'Whielacronx "**20070415190812] [pyfec: bump version number to 1.0.0a1-1-STABLE "Zooko O'Whielacronx "**20070415005040] [zfec: tweak licence text for clarity zooko@zooko.com**20070426225216] [zfec: change formatting of copyright timestamps, licence etc. "Zooko O'Whielacronx "**20070420175254] [rename top-level package from 'fec' to 'zfec' warner@lothar.com**20070418191812] [rename the C extension from "fec" to "_fec" zooko@zooko.com**20070127015804] [pyfec: add -f option to fec, add more user-friendly handling of filesystem errors and user errors "Zooko O'Whielacronx "**20070415004832] [pyfec: more progress indicators, handling of already-existent outfile "Zooko O'Whielacronx "**20070414230059] [pyfec: fix up docs, version numbers, bump version to 1.0.0a1-0-STABLE "Zooko O'Whielacronx "**20070414195451] [pyfec: version number bump to v0.99 "Zooko O'Whielacronx "**20070201215235] [pyfec: improve package metadata zooko@zooko.com**20070126000829] [pyfec: add bin/fec and bin/unfec, do better handling and reporting of various errors "Zooko O'Whielacronx "**20070414190010] [pyfec: import a copy of the argparse module "Zooko O'Whielacronx "**20070414180554] [pyfec: update README and bump version number to 0.9.9-0-STABLE "Zooko O'Whielacronx "**20070414190218] [pyfec: more thanks to Brian zooko@zooko.com**20070328222251] [give it a version number -- v0.9 "Zooko O'Whielacronx "**20070411174933 I intend to bump it to 1.0 after adding the cmdline tools. ] [pyfec: add fec/__init__.py to import all the right names into the fec module namespace zooko@zooko.com**20070127022536] [pyfec: new filefec with compressed metadata, better error handling, much better unit tests "Zooko O'Whielacronx "**20070414181924] [pyfec: add ACK and TODO "Zooko O'Whielacronx "**20070201214915] [pyfec: licensing tweak "Zooko O'Whielacronx "**20070131195420] [pyfec: documentation and licensing "Zooko O'Whielacronx "**20070130163642] [pyfec: COPYING zooko@zooko.com**20070130162826] [pyfec: add TODO "Zooko O'Whielacronx "**20070201163748] [pyfec: rename and clarify -- "blocks" are the units of input/output of the codec, "shares" are sequences of blocks (used to process arbitrary-length files) zooko@zooko.com**20070330185243] [add dummy function to see how fast we can read in a file and invoke a Python function on each segment zooko@zooko.com**20070125235026] [pyfec: add documentation, assertion, licence information zooko@zooko.com**20070328031406] [pyfec: make randomized unit tests more comprehensive "Zooko O'Whielacronx "**20070201010224] [merge changes and fix wrong type -- k and m need more than 8 bits (because they are the count rather than the index, i.e. they are 1-indexed) "Zooko O'Whielacronx "**20070411173427] [pyfec: loosen preconditions -- you can have up to 256 total shares, not up to 255 total shares "Zooko O'Whielacronx "**20070131225316] [pyfec: fix precondition checks on k and m to actually check the value before coercing it into a smaller type (oops) "Zooko O'Whielacronx "**20070201010140] [pyfec: tighten internal C types "Zooko O'Whielacronx "**20070131195531] [pyfec: make unit test failures print out a few useful bytes instead of kibibytes "Zooko O'Whielacronx "**20070201010257] [pyfec: add easyfec.Decoder(), which is easier to use than fec.Decoder() "Zooko O'Whielacronx "**20070414175245] [pyfec: add easyfec wrapper which takes a single string and splits it into input shares and pads, then passes it on to the inner fec object zooko@zooko.com**20070328031430] [add copyright notices, license, attributions zooko@zooko.com**20070126000216] [add verbosity option to bench "Zooko O'Whielacronx "**20070328004259] [pyfec: import a copy of fileutil.py from the pyutil library "Zooko O'Whielacronx "**20070414175402] [pyfec: import a copy of mathutil.py from the pyutil library "Zooko O'Whielacronx "**20070414175443] [import Version from pyutil "Zooko O'Whielacronx "**20070411174844] [simplejson: lower our minimum-required-version of setuptools to 0.6a9, since that seems to work, and it's the version that comes with dapper warner@lothar.com**20070714024756] [patch simplejson's ezsetup to take a min_version, and patch setup.py to use it, to allow installs on edgy machines with setuptools-0.6c3 warner@lothar.com**20070710231111] [import simplejson-1.7.1 into src/simplejson warner@lothar.com**20070710222253] [README: add note about what "darcs get" will do and how to run "darcs pull" zooko@zooko.com**20070809032832] [README: point people to allmydata.org if they are looking for precompiled packages zooko@zooko.com**20070809034403] [README: nevow is now automatically handled by build-deps/install warner@lothar.com**20070915211945] [setup: add pywin32 to our dependencies if sys.platform == "win32" zooko@zooko.com**20070920180540] [setup.py: factor out dependency stuff, add workaround for nevow-0.6.0-on-dapper problem warner@lothar.com**20070915220535] [edit README and require Nevow 0.9.18 zooko@zooko.com**20070629223338 If anybody out there is running with Nevow < v0.9.18 and wants to vouch for it, then we can relax that requirement. ] [build-deps-setup.py: provide dependency_links too warner@lothar.com**20070915210457] [build-deps-setup.py: use ez_setup here too, for platforms without setuptools Brian Warner **20070915093140] [setup.py: minor reformatting, use explicit file: URLs in dependency-links warner@lothar.com**20070915210532] [change setup.py to find dependency tarballs in misc/dependencies zooko@zooko.com**20070913223727] [setup.py: require simplejson>=1.4, since we use the indent= argument Brian Warner **20070914102415] [setup.py: add Nevow to our dependency list warner@lothar.com**20070915211807] [setup.py: remove nevow dependency: we need it, but easy_install can't install it warner@lothar.com**20070912235538] [Makefile build-deps: use a fake package named 'tahoe-deps', to avoid a double-build of tahoe warner@lothar.com**20070915022355] [insert misc/dependencies zooko@zooko.com**20070913215010] [Makefile: improve 'clean' behavior: rm ez_setup leftovers, ignore _version.py Brian Warner **20070914102856 in the test-clean target ] [Makefile: fix race condition in test-clean that caused occasional failures Brian Warner **20070822042851] [makefile: add a test-clean target warner@allmydata.com**20070524005748] [remove parts of pycrypto that we are no longer going to use: SHA256 and RSA zooko@zooko.com**20071110002112] [crypto: fix compiler warnings in the .c files warner@lothar.com**20070816004852] [setup.py: oops, I inserted a typo by commenting out the wrong line Brian Warner **20070816075658] [setup.py: disable building RSA for now, since it requires GMP. We'll refrain Brian Warner **20070816075221 from adding it to the build-dependencies until we actually want to use RSA. ] [trailing-whitespace eradication, no functional changes warner@allmydata.com**20071101223341] [copy RSA from PyCrypto into the allmydata/ tree, we'll use it eventually warner@lothar.com**20070816003639] [test_dirnode.py: obtain full coverage of dirnode.py warner@allmydata.com**20071204203204] [test_mutable.py: hush pyflakes warner@lothar.com**20071106073239] [fix several bugs and warnings -- thanks, pyflakes zooko@zooko.com**20071203214235] [test_mutable: improve test coverage a bit, add test_filenode.py warner@allmydata.com**20071204033754] [fix idlib.could_be_base32_encoded_l() to accept *only* valid strings zooko@zooko.com**20080103174311] [add regexes to idlib to match certain kinds of base32-encoded values zooko@zooko.com**20080103224446] [offloaded: basic test for client-side of AssistedUploader warner@allmydata.com**20080110022550] [offloaded: early code: most of client-side, defined the RemoteInterfaces warner@allmydata.com**20080109031854] [webish: add edge metadata to t=json output, including timestamps warner@allmydata.com**20080212011410] [rename dirnode2.py to dirnode.py warner@allmydata.com**20071204174520] [decentralized directories: integration and testing zooko@zooko.com**20071203205242 * use new decentralized directories everywhere instead of old centralized directories * provide UI to them through the web server * provide UI to them through the CLI * update unit tests to simulate decentralized mutable directories in order to test other components that rely on them * remove the notion of a "vdrive server" and a client thereof * remove the notion of a "public vdrive", which was a directory that was centrally published/subscribed automatically by the tahoe node (you can accomplish this manually by making a directory and posting the URL to it on your web site, for example) * add a notion of "wait_for_numpeers" when you need to publish data to peers, which is how many peers should be attached before you start. The default is 1. * add __repr__ for filesystem nodes (note: these reprs contain a few bits of the secret key!) * fix a few bugs where we used to equate "mutable" with "not read-only". Nowadays all directories are mutable, but some might be read-only (to you). * fix a few bugs where code wasn't aware of the new general-purpose metadata dict the comes with each filesystem edge * sundry fixes to unit tests to adjust to the new directories, e.g. don't assume that every share on disk belongs to a chk file. ] [fix test cases to not use camelCase warner@lothar.com**20061201022000] [test_introducer: flushEventualQueue at the end of the test run warner@allmydata.com**20070328001613] [introducer: add some comments about separating nodeid from tubid warner@allmydata.com**20070329181629] [test_system: turn off test_connections, since it is slow and subsumed by the other system tests warner@allmydata.com**20070418230657] [rename all "*PBURL*" to "*FURL*" "Zooko O'Whielacronx "**20070522210830 This breaks backwards compatibility with Tahoe v0.2 -- the first public release of Tahoe. ] [node.py: use 'node.pem' for all nodes warner@allmydata.com**20070523194852 Rather than use separate client.pem and introducer.pem files, use 'node.pem' for all nodes regardless of what type it is. This is slightly cleaner, but introduces a compatibility. Users who upgrade to this change should do 'mv client.pem node.pem' to avoid generating a new certificate and thus changing their TubID. ] [node.py: add logging of startup/shutdown, for the cygwin test_system failure warner@allmydata.com**20070531204422] [add a missing up-call in introducer.startService() robk-org@allmydata.com**20070605014540] [webish: mark read-only directories as such when listing their parent warner@lothar.com**20070626193700] [change IVirtualDrive.get_node_at_path to accept either a list or a single slash-separated string warner@allmydata.com**20070629004614] [interfaces.py: remove some unused 'pass' lines Brian Warner **20070629075000] [interfaces: remove spurious 'self' from interface declarations warner@lothar.com**20070707023947] [web: use KeyError (rather than IndexError) to signal a missing child warner@lothar.com**20070707024008] [web: missed a IndexError-to-KeyError conversion warner@lothar.com**20070707024303] [interfaces.py: increase RIVirtualDriveServer.list constraint from 100 entries to 1000, for now warner@allmydata.com**20070712202352] [webish.py: test that _get_or_create_directories fix I made a few hours ago warner@lothar.com**20070714051916] [introducer: don't log.err() an initial connection failure, since that flunks tests. Use self.log() instead. Also improve test_client.py to always trigger this case, before it was finishing to quickly to always hit the problem. warner@allmydata.com**20070809195344] [log a prominent warning message in the case that the introducer cannot be reached robk-org@allmydata.com**20070605014853] [shorten ids tahoe@arnowaschk.de**20070812185318] [test_system.py: add coverage for get_permuted_peers() warner@lothar.com**20070812232934] [IDirectoryNode: add has_child() method warner@allmydata.com**20070815202201] [webish: modify JSON to match zooko's proposed API changes in #118 warner@allmydata.com**20070823200039] [node.py: refactor config-file getting and setting warner@lothar.com**20070828015839] [webish: implement 'PUT /uri?t=mkdir' (to create anonymous dirnodes) warner@allmydata.com**20070906002306] [fix test_web refactoring so that the WebMixin class isn't a TestCase zooko@zooko.com**20070810154002 Thanks, Brian. ] [refactor test_web so that other tests can use the part of test_web that sets up a simple filesystem zooko@zooko.com**20070809200842] [vdrive: log an error if we weren't able to use the vdrive the way we wanted to warner@lothar.com**20070915221729] [upload: make peer-selection a bit more uniform. Closes #132. warner@lothar.com**20070917000834] [peer-selection: if we must loop, send a minimal number of queries (by asking for more than one share per peer on the second pass) warner@lothar.com**20070916085300] [upload.py: implement Tahoe2 peer-selection algorithm warner@lothar.com**20070916082407] [upload: switch to Tahoe2, add test for uniform share allocation warner@lothar.com**20070916082503] [provisioning.py: get full test coverage warner@lothar.com**20070917083854] [test_web: improve provisioning.py test coverage a bit by using a live web hit warner@lothar.com**20070828002639] [provisioning.py: add file/server availability numbers warner@allmydata.com**20070906011621] [add a provisioning utility page which shows necessary storage space and transfer rates for grids of various sizes warner@lothar.com**20070827064424] [welcome.xhtml: add a link to allmydata.org warner@lothar.com**20070714001544] [test_system.py: do a large-file test (1.5MB) to trigger pauseProducing warner@lothar.com**20070919084344] [test_system.py: verify that we can replace files in place warner@allmydata.com**20070817000350] [test_system.py: many (failing) web tests were accidentally bypassed, fix those. Add some PUT tests. warner@allmydata.com**20070816234940] [test_system.py: change/remove the tests that currently fail due to web changes warner@lothar.com**20070708030644] [test_system.py: match change to /global_vdrive URL warner@lothar.com**20070615083855] [introducer.py: add test coverage of _disconnected() warner@allmydata.com**20070919185013] [check_memory.py: record initial memory usage (before any connections are made) warner@allmydata.com**20070920193627] [check_memory.py: fix benign-but-noisy vdrive.furl handling bug warner@allmydata.com**20070717023452] [a few edits to architecture.txt and related docs zooko@zooko.com**20070921211226] [update architecture.txt a little bit warner@lothar.com**20070723033005] [in --> across wilcoxjg@gmail.com**20070809041754] [test_system.py: do one upload, then test debug scripts, then do other uploads warner@allmydata.com**20070925011237] [fix test_vdrive (fixes #144) zooko@zooko.com**20070922222627 It turns out that we actually have *two* files in our storage servers at the time that test_vdrive asserts things about the shares. I suppose that test_vdrive happens to pass on all other operating systems because the filesystem happens to return the right share as the first one in a "listdir()". The fix in this patch is slightly kludgey -- allow either share to pass -- but good enough. ] [webish: write node.url, for the benefit of CLI tools warner@lothar.com**20071011083804] [create_node: use a webport by default, on localhost:8011 warner@lothar.com**20071011090123] [rename client.tac to tahoe-client.tac, so that 'ps ax|grep tahoe' works. Closes #156. warner@lothar.com**20071011094806] [check_memory.py: preserve client.log and stats.out in _test_memory/ Brian Warner **20070916035306 Put the nodes in _test_memory/test/, which is clobbered on each test. Also kill the client with SIGINT instead of SIGKILL. Also don't daemonize the client, since we're going to kill it at the end of the test anyways: this cleans up shutdown a bit. ] [check_memory.py: have all clients write their logs to _test_memory/client.log instead of a separate file per client warner@lothar.com**20070915193405] [add public testnet .furls to docs/testnet/, and copy into .deb . Closes #157. warner@allmydata.com**20071011215523] [README: update link to test grid on web page zooko@zooko.com**20070924223719] [deb: add docs/* to the debian package warner@allmydata.com**20071011213729] [boringfile: exclude some foolscap.deb-generated files warner@allmydata.com**20070711202842] [update .darcs-boringfile to match the debian-moves-to-misc/ change warner@allmydata.com**20070705215511] [update feisty packaging warner@allmydata.com**20070329214842] [Makefile: stop producing foolscap .debs warner@allmydata.com**20070821210101] [Makefile: factor out deb-foolscap-ARCH to a separate target warner@allmydata.com**20070711210047] [make the anchor text in the WUI be descriptive of the target -- not "Click Here!"! czooko@zooko.com**20071022235630] [web: oops, forgot to add start.html warner@allmydata.com**20070822220800] [cli: improve test coverage warner@lothar.com**20071021193317] [cli: implement 'mv'. Closes #162. warner@allmydata.com**20071012033148] [move NotMutableError from dirnode.py into interfaces.py warner@allmydata.com**20071101220307] [trailing-whitespace eradication, no functional changes warner@allmydata.com**20071101223401] [docs/configuration.txt: expand the 'sizelimit' docs warner@lothar.com**20071108080842] [Add "sizelimit" to configuration doc. nejucomo@gmail.com**20071105074642] [docs/configuration.txt: explain the files in the node's basedir, which ones are useful to modify, etc warner@allmydata.com**20070813202840] [mutable: add basic test coverage of new-dirnodes-using-mutable-files warner@lothar.com**20071108103100] [test_system: RSA keys are even more variable than I thought, 2044..2049 warner@lothar.com**20071108110411] [test_system: RSA keys vary in size, expand valid ranges in test warner@lothar.com**20071108090113] [consolidate dirnode/filenode-creation code into Client warner@lothar.com**20071109085451] [mutable: fix multiple-versions-interfering-with-each-other bug. replace() tests now pass. warner@lothar.com**20071108100733] [mutable: grab encprivkey when necessary during publish, fix test_mutable warner@lothar.com**20071108084627] [mutable: fix usage of NeedMoreDataError warner@allmydata.com**20071107235209] [mutable: rearrange order of Publish to allow replace() to work. Doesn't work yet. Also test_mutable is disabled for a while. warner@allmydata.com**20071108030139] [stabilize on 20-byte nodeids everywhere, printed with foolscap's base32 warner@allmydata.com**20071107004959] [mutable.txt: more updates: record offset of extra lease count instead of the actual extra leases warner@allmydata.com**20071031024658] [mutable: fix use of storage API warner@allmydata.com**20071107005334] [mutable: wire in RSA for real, using pycryptopp warner@allmydata.com**20071107235135] [checker: remember checker results, but only in ram for now warner@allmydata.com**20071023004624] [checker.py: rearrange classes a little bit warner@allmydata.com**20071022231918] [add an equally-simple file-verifier warner@allmydata.com**20071016192509] [checker: return more information per CHK file, including the shareholder list warner@lothar.com**20071017092550] [mutable: stub out pubkey creation until we wire in pycryptopp properly warner@allmydata.com**20071107012706] [mutable: test roundtrip, make it work warner@allmydata.com**20071107201901] [storage: rewrite slot API, now use testv_and_readv_and_writev or readv warner@allmydata.com**20071106021714] [storage.py: fix tests, timestamps get updated when leases are renewed warner@allmydata.com**20071031193133] [storage.py: more test coverage, make sure leases survive resizing warner@allmydata.com**20071031190747] [storage.py: improve test coverage even more warner@lothar.com**20071031084401] [storage.py: more mutable-slot coverage, renewing/cancelling leases warner@lothar.com**20071031083156] [mutable slots: add some test coverage for lease-addition warner@lothar.com**20071031073830] [storage: add readv_slots: get data from all shares warner@lothar.com**20071105063701] [mutable slots: finish up basic coding on server-side containers, add some tests. Remove all caching from MutableShareFile. warner@lothar.com**20071031071040] [checkpointing mutable-file work. Storage layer is 80% in place. warner@allmydata.com**20071031024736] [dirnode: change the defined behavior of RIVirtualDriveServer.set to allow replace-in-place without raising an exception warner@allmydata.com**20070817000319] [storage: fill alreadygot= with all known shares for the given storageindex, not just the ones they asked about warner@lothar.com**20070917074840] [storage: don't add a duplicate lease, renew the old one instead Brian Warner **20070903043947] [storage: handle simultanous uploads: add a lease for the pre-empted client warner@lothar.com**20070902215749] [storage: replace sqlite with in-share lease records warner@lothar.com**20070902214715] [storage: use sqlite from either python2.5's stdlib or the pysqlite2 package warner@lothar.com**20070829062852] [deletion phase3: add a sqlite database to track renew/cancel-lease secrets, implement renew/cancel_lease (but nobody calls them yet). Also, move the shares from BASEDIR/storage/* down to BASEDIR/storage/shares/* warner@lothar.com**20070828064140] [update README concerning dependent libraries that the user has to install "Zooko O'Whielacronx "**20070516153433] [client.py: add a 'debug_no_storage' option to throw out all share data warner@allmydata.com**20070717010703] [MANIFEST.in: include .html and .css files from allmydata/web/ too warner@allmydata.com**20070822220841] [mutable: move IV into signed prefix, add more retrieval code warner@allmydata.com**20071106210446] [mutable.txt: need offset of EOF too warner@allmydata.com**20071103052807] [mutable.txt: more notes warner@allmydata.com**20071103035341] [docs/mutable.txt: add IV, define a place for it in the data structure Brian Warner **20071030025112] [mutable.txt: use merkle trees on blocks, since it probably won't be that hard (the code is all being copied from the CHK classes anyways), and that keeps the storage format identical to the MDMF case, for better forward-compatibility warner@allmydata.com**20071026232501] [docs/mutable.txt: we need readkey IV Brian Warner **20071030011451] [mutable.txt: fix everybody-gets-read bug, define WE-update protocol, add accepting-nodeid to leases to allow updating lease tokens warner@allmydata.com**20071026231550] [mutable.txt: we're now sort of ready to handle lease-migration warner@lothar.com**20071031070408] [docs: add writeup of our mutable-file plans warner@lothar.com**20071026092656] [test_system.mutable: make sure we exercise FEC padding warner@allmydata.com**20071107015033] [add container_size to mutable dump-share output warner@allmydata.com**20071107013122] [dump-share: emit SDMF information too warner@allmydata.com**20071107014631] [test_system.mutable: add test coverage for the 'dump-share' debug command warner@allmydata.com**20071107011049] [debug: add mutable-slot support to 'dump-share' command warner@allmydata.com**20071107005555] [debug: remove the 'dump-share-leases' command, it is subsumed by 'dump-share' warner@lothar.com**20070917084439] [debug: 'dump-uri-extension' command becomes 'dump-share', add 'dump-share-leases'. Both display leases. warner@lothar.com**20070902214820] [test_system: add early test for mutable slots, currently publish-only warner@allmydata.com**20071107005711] [bump test timeout even higher zooko@zooko.com**20070502223105 It does indeed take longer than 2400 seconds to run test_upload_and_download on a virtual windows machine when the underlying real machine is heavily loaded down with filesystem analysis runs... ] [WUI: hook up an "overwrite" button on mutable files zooko@zooko.com**20071110165345] [webish: add preliminary mutable file support: upload, download, listings, JSON, URI, RO-URI. No replace yet. warner@lothar.com**20071109095427] [webish: implement replace= for POST commands warner@allmydata.com**20070815222138] [webish: look for when_done= in POST fields as well as queryargs. Closes #101. warner@allmydata.com**20070814004502 We need to look in the fields because that's how we build the mkdir/upload forms. Without this, uploading or creating directories would leave us on a page that had just a URI, instead of something actually useful to a human. ] [web: replace FILE links with /uri -based ones, to prevent an XSS attack against the secret vdrive URI contained in the current URL warner@allmydata.com**20070823003501] [webish: strip leading/tailing whitespace from user-provided filenames, to avoid confusion warner@lothar.com**20070825190524] [webish.py: allow users to delete (but not create) empty-named files. Closes #94. Brian Warner **20070816061405] [webish: implement 'PUT /uri' (to create anonymous files) warner@allmydata.com**20070906001227] [webish: implement replace= for PUT commands warner@allmydata.com**20070815202223] [vdrive.py: oops, forgot an import warner@lothar.com**20070915222157] [improve code coverage by removing untested raise RuntimeError and replacing it with tested assert zooko@zooko.com**20071027013740] [webish.Directory: add a 'text/plain' link for files, to download as plaintext. Brian Warner **20070714064535 This is useful to view (e.g.) .py files that would otherwise be served as some weird text/x-python that browsers will just want to save to disk. ] [mutable: rearrange classes, putting MutableFileNode at the bottom warner@allmydata.com**20071106211948] [mutable: get most of the retrieve-side code written. no tests yet. warner@lothar.com**20071106094729] [mutable.Publish: create a dispatch_map for the benefit of recovery code, and pull pack/unpack methods out into functions warner@allmydata.com**20071106041459] [mutable: parameterize Publish/Retrieve classes in MutableFileNode, for tests warner@allmydata.com**20071106211809] [mutable.Publish: rearrange create() to use more of it in unit tests warner@lothar.com**20071106063340] [mutable.Publish: more tests warner@allmydata.com**20071106043843] [test_mutable: hush pyflakes warning warner@lothar.com**20071105064153] [mutable: use proper enable/renew/cancel secrets warner@allmydata.com**20071106035108] [mutable: added send-messages-to-peers code, about 70% done. No recovery code yet. warner@allmydata.com**20071106032947] [mutable: add peer-selection to Publish, and some basic unit tests warner@lothar.com**20071105063807] [mutable.py: add share-unpacking code, use it for more tests warner@allmydata.com**20071103052831] [mutable.py: start writing share-mapping code warner@allmydata.com**20071103055902] [mutable: implement filenode share-packing, still pretty rough warner@allmydata.com**20071103035139] [mutable.py: sketch out data-structure packing/unpacking methods warner@lothar.com**20071102070328] [mutable: split dirnode stuff out to dirnode2.py, will be renamed later warner@lothar.com**20071102064647] [mutable: improve NewDirectoryNode test coverage warner@allmydata.com**20071102013554] [mutable: implement most remaining dirnode methods. No tests yet. warner@allmydata.com**20071101235758] [IMutableFileNode is a subtype of IFileNode zooko@zooko.com**20071110223718 I'm not 100% sure that this is correct, but it looks reasonable, it passes unit tests (although note that unit tests are currently not covering the new mutable files very well), and it makes the "view JSON" link on a directory work instead of raising an assertion error. ] [mutable: first pass at dirnodes, filenodes, new URIs. Some test coverage. warner@allmydata.com**20071101221529 The URI typenames need revision, and only a few dirnode methods are implemented. Filenodes are non-functional, but URI/key-management is in place. There are a lot of classes with names like "NewDirectoryNode" that will need to be rename once we decide what (if any) backwards compatibility want to retain. ] [test_uri.py: remove tiny whitespace warner@lothar.com**20070829064003] [created DirnodeURI schema wilcoxjg@gmail.com**20070829062826] [uri.py: improve test coverage a bit warner@lothar.com**20070917080947] [add a simple checker, for both files and directories warner@allmydata.com**20071015231639] [dirnode.build_manifest(): tolerate cycles in the directory graph warner@lothar.com**20070721224013] [uri.py: get keys and index from the URI instance warner@lothar.com**20070722004500] [fix pyflakes warnings from recent b32decode change warner@lothar.com**20070812233351] [client.py: only import webish.py if 'webport' is set, to save 3MB of footprint Brian Warner **20070904233306] [cli: add test coverage warner@allmydata.com**20071012022041] [tahoe_put.py: don't treat "^HTTP 200 OK" appearing somewhere in the body as if it appeared in the header zooko@zooko.com**20070817190641] [test_system.py: add coverage for allmydata.control warner@allmydata.com**20070926190655] [don't over-encode the nodeid many times with ascii-encoding zooko@zooko.com**20070812172938] [don't check for existence of portnum file and then try to open it -- instead try to open it and catch exception "Zooko O'Whielacronx "**20070522210600 This avoids a race condition, also known as time-of-check-to-time-of-use. ] [remove unused imports: hush pyflakes warnings warner@allmydata.com**20070725011358] [web: remove /vdrive/private, replace with a start.html file that points at the /uri/PRIVATE_URI, to prevent XSRF attacks warner@allmydata.com**20070822215434] [test_web.test_welcome: give the rooturl a trailing slash, otherwise older versions of twisted complain warner@lothar.com**20070708062252] [vdrive: eventually create public/private dirnodes event if the vdrive server isn't available at start up warner@allmydata.com**20070725024425] [vdrive: make sure we really record global/private root directory uris warner@allmydata.com**20070628175951] [vdrive.py: log a note when we successfully retrieve the global root directory warner@allmydata.com**20070717031706] [refactor reading of configuration files in client.py zooko@zooko.com**20070822172957 This makes it so that an optional file which is unreadable or is rm'ed at the wrong moment will be ignored instead of raising an exception. It also bums out a couple of unnecessary lines of code (the explicit ".close()" call). ] [client.py: write control.furl in the right place warner@allmydata.com**20070717010627] [webish: localfile=/localdir= are now disabled by default, a special switch is required to enable them warner@lothar.com**20070811012122] [test_web.py: remove spurious '# YES' lines, now that all the tests pass warner@allmydata.com**20070725031621] [webish.py: disallow slashes in POSTed filenames. Closes #75. warner@allmydata.com**20070716185312] [added unit test to webish's rename function robk-org@allmydata.com**20070717000501 added unit tests to test various permutations of the rename function, and some sanity checks on the rename-form function. also added a guard to prevent '/' in from_/to_name in rename calls. ] [webish: improve test coverage of WebDownloadTarget warner@lothar.com**20070717191645] [webish: improve test coverage further warner@lothar.com**20070717192829] [webish: reduce POST memory footprint by overriding http.Request warner@lothar.com**20070811002533 The original twisted.web.http.Request class has a requestReceived method that parses the form body (in the request's .content filehandle) using the stdlib cgi.parse_multipart() function. parse_multipart() consumes a lot of memory when handling large file uploads, because it holds the arguments entirely in RAM. Nevow's subclass of Request uses cgi.FieldStorage instead, which is much more memory-efficient. This patch uses a local subclass of Request and a modified copy of the requestReceived method. It disables the cgi.parse_multipart parsing and instead relies upon Nevow's use of FieldStorage. Our code must look for form elements (which come from the body of the POST request) in req.fields, rather than assuming that they will be copied into req.args (which now only contains the query arguments that appear in the URL). As a result, memory usage is no longer inflated by the size of the file being uploaded in a POST upload request. Note that cgi.FieldStorage uses temporary files (tempfile.TemporaryFile) to hold the data. This closes #29. ] [client.py: add a newline to "myself.furl" contents Brian Warner **20070818062822] [check_speed: do both upload and download tests warner@allmydata.com**20070921015244] [check_speed: upload multiple files, measure Ax+B warner@allmydata.com**20070920235533] [check_memory: fix race condition for startup of in-process server nodes warner@allmydata.com**20070920223358] [deletion phase2b: create renew/cancel secrets for real. warner@lothar.com**20070828023026] [deletion phase2a: improve creation of renew/cancel secrets. Still fake though. warner@lothar.com**20070828020018] [deletion phase1: send renew/cancel-lease secrets, but my_secret is fake, and the StorageServer discards them warner@lothar.com**20070828002851] [check-memory: add 'receive' mode, for #97 (consumption during share receive warner@allmydata.com**20070919195932] [check_memory: oops, silly bug make 'upload' push to ourselves, raising the usage Brian Warner **20070919111448] [check_memory: don't accept shares for download/download-GET test, since that hits bug #97 warner@allmydata.com**20070919024029] [Makefile: check-memory-once: add a convenience target for focussed testing warner@allmydata.com**20070919192916] [check_memory: add download-GET-slow, to simulate memory usage of a node feeding downloaded data via HTTP GET to a slow client warner@allmydata.com**20070919033527] [check_memory: add download, download-GET warner@allmydata.com**20070919015605] [#96: add flag to enable pushing data to ourselves, defaulting to False warner@allmydata.com**20070810013024] [node: enable Tub.logLocalFailures now that we require foolscap-0.1.2 warner@lothar.com**20070407035559] [activate storage size limits in the client. Closes #34. warner@allmydata.com**20070704002707 To use this, write a number like 10MB or 5Gb or 5000000000 to a file named 'sizelimit' in the client's base directory. The node will not grant leases for shares that would take it much beyond this many bytes of storage. Note that metadata is not included in the allocation count until a restart, so the actual space consumed may grow beyond the limit if the node is not restarted very frequently and the amount of metadata is significant. ] [introducer.py: minor rearrangement of methods warner@allmydata.com**20070717024742] [refactor upload/encode, to split encrypt and encode responsibilities warner@allmydata.com**20070724023153] [document a proposed IDecoder interface, still pretty bare-bones warner@allmydata.com**20070328183131] [truncate storage index to 128 bits, since it's derived from a 128 bit AES key warner@lothar.com**20070723024844] [CHK: remove the storage index from the URI, deriving it from the key instead warner@lothar.com**20070722012315] [uri: implement URI-processing classes, IFileURI/IDirnodeURI, use internally warner@lothar.com**20070721224036] [webish: provide a valid Content-Length header on downloads warner@allmydata.com**20070703220900] [webish: improve reporting of web download errors that occur early enough warner@allmydata.com**20070703204737 If the error occurs before any data has been sent, we can give a sensible error message (code 500, stack trace, etc). This will cover most of the error cases. The ones that aren't covered are when we run out of good peers after successfully decoding the first segment, either because they go away or because their shares are corrupt. ] [webish.py: handle errors during download better. Addresses #65. warner@allmydata.com**20070703201814 Previously, exceptions during a web download caused a hang rather than some kind of exception or error message. This patch improves the situation by terminating the HTTP download rather than letting it hang forever. The behavior still isn't ideal, however, because the error can occur too late to abort the HTTP request cleanly (i.e. with an error code). In fact, the Content-Type header and response code have already been set by the time any download errors have been detected, so the browser is committed to displaying an image or whatever (thus any error message we put into the stream is unlikely to be displayed in a meaningful way). ] [web: improve test coverage warner@lothar.com**20070708054718] [web: add a 'return to welcome page' link warner@lothar.com**20070708043004] [web: make sure we present read-only directories properly warner@lothar.com**20070708052120] [import simplejson directly, and remove our local copies of the component .py files from util/json* warner@lothar.com**20070710224932] [web: indent the JSON to make it easier for humans to read, but slightly larger warner@lothar.com**20070708072323] [import the decoder portion of simplejson-1.7.1 too warner@lothar.com**20070710201240] [fix several methods to handle LIT URIs correctly, rather than assuming that all filenodes are CHK URIs warner@allmydata.com**20070712231749] [test_system.py: more vdrive coverage warner@allmydata.com**20070628220533] [vdrive: add get_node_at_path(pathlist) warner@allmydata.com**20070628180003] [web: survive bogus URIs when displaying sizes warner@lothar.com**20070708043118] [webish.py: remove dead code warner@lothar.com**20070708051246] [storage: we must truncate short segments. Now most tests pass (except uri_extension) warner@lothar.com**20070713233825] [rename storageserver.py to just storage.py, since it has both server and client sides now warner@lothar.com**20070714002545] [make it possible to download LIT uris. oops. warner@allmydata.com**20070712231659] [fix dump-uri-extension warner@lothar.com**20070713235808] [upload: finish refactoring, all unit tests pass now warner@lothar.com**20070720055329] [upload: refactor to enable streaming upload. not all tests pass yet warner@allmydata.com**20070720012144] [rename chunk.py to hashtree.py warner@allmydata.com**20070412201325] [comment out some verbose log messages, add commented-out new ones warner@allmydata.com**20070418032552] [handle foolscap's annoying tendency to turn everything into a sets.Set, which are not interoperable with 2.4's builtin 'set' type warner@allmydata.com**20070330235504] [encode: fix multi-segment uploads: lambdas inside for loops require special attention to make sure you are capturing the *value* of the loop variable and not just the slot it lives in warner@allmydata.com**20070418032908] [encode.py: fix generation of plaintext/crypttext merkle trees warner@lothar.com**20070607201414] [refactor URI_extension handlers out of encode/download and into uri.py warner@allmydata.com**20070612012518] [handle uri_extension with a non-bencode serialization scheme warner@lothar.com**20070608231754] [upload.py: minor refactoring warner@allmydata.com**20070612022151] [allow the introducer to set default encoding parameters. Closes #84. warner@allmydata.com**20070712223330 By writing something like "25 75 100" into a file named 'encoding_parameters' in the central Introducer's base directory, all clients which use that introducer will be advised to use 25-out-of-100 encoding for files (i.e. 100 shares will be produced, 25 are required to reconstruct, and the upload process will be happy if it can find homes for at least 75 shares). The default values are "3 7 10". For small meshes, the defaults are probably good, but for larger ones it may be appropriate to increase the number of shares. ] [test_introducer_and_vdrive: remove the assumption that startService returns a Deferred warner@allmydata.com**20070524003720] [more #85 work, system test still fails warner@lothar.com**20070713220901] [implement URI:LIT, which stores small (<55B) files inside the URI itself. Fixes #81. warner@allmydata.com**20070712202236] [bump some unit tests up to very high timeouts because my poor G4 867 MHz PowerBook, which is busy doing video iChat, takes a long time to run these tests "Zooko O'Whielacronx "**20070501040343] [test_system: increase timeouts to almost 20 minutes: we have buildslaves that take more than 5 minutes to finish these tests. warner@allmydata.com**20070501031458] [test_system: bump up timeout again, the new extra download forms take more time on poor overloaded slave1 warner@allmydata.com**20070416225325] [consolidate multiple definitions of NotEnoughPeersError warner@allmydata.com**20070608052055] [storage: use one file per share instead of 7 (#85). work-in-progress, tests still fail warner@allmydata.com**20070713210449] [rename fileid/verifierid to plaintext_hash/crypttext_hash warner@lothar.com**20070610034604] [docs/thingA.txt: describe what this does, even if it doesn't yet have a name warner@allmydata.com**20070608045058] [storageserver: include metadata in the sizelimit, at least after the bucket has been closed warner@allmydata.com**20070704003849] [storageserver: implement size limits. No code to enable them yet, though warner@allmydata.com**20070704000802] [test_storage: test StorageServer code too: allocation, lookup, multiple-writer behavior warner@allmydata.com**20070418224234] [storageserver: assert that blocks are written in-order, clean up tests a bit warner@allmydata.com**20070418032105] [storageserver: the last segment is not always going to be the same size as the rest, so don't assert such a thing warner@allmydata.com**20070417204145] [test_storage: add (failing) test of the BucketWriter/BucketReader implementation warner@allmydata.com**20070418030344] [don't clobber existing storefile every put block "Zooko O'Whielacronx "**20070418031444] [pyutil: fileutil.open_or_create() "Zooko O'Whielacronx "**20070418031426] [import fileutil, some of which came from amdlib.util and some of which came from the pyutil library zooko@zooko.com**20070330191223] [storage: wrap buckets in a local proxy warner@lothar.com**20070709062746 This will make it easier to change RIBucketWriter in the future to reduce the wire protocol to just open/write(offset,data)/close, and do all the structuring on the client end. The ultimate goal is to store each bucket in a single file, to reduce the considerable filesystem-quantization/inode overhead on the storage servers. ] [remove several leftover defintions of netstring() warner@allmydata.com**20070608051318] [rename thingA to 'uri extension' warner@lothar.com**20070608225916] [docs/uri.txt: document current URI formats warner@lothar.com**20070722002901] [switch from rfc 3548 base-32 to z-base-32 except for tubids/nodeids zooko@zooko.com**20070724204606] [record our nodeid (for reference) in 'my_nodeid' warner@allmydata.com**20061205224048] [more allmydata.util test coverage, rename the test case to be more correct warner@allmydata.com**20070406233322] [add test coverage for stuff in allmydata/utils warner@allmydata.com**20070406232925] [check_memory.py: Disable the 100MB test for now: our buildslave can't currently handle it because the testnet/framework processes uses something like 600M of RSS. warner@allmydata.com**20070815195511] [check_memory.py: include a single 100MB test warner@lothar.com**20070809183033] [now that the buildslave is moved to a new machine, enable the 50MB test Brian Warner **20070809083252] [check_memory: oops, only bail if the process ended abnormally warner@lothar.com**20070915184729] [check-memory: if the child process fails to start, shut down instead of hanging forever warner@lothar.com**20070915031657] [check_speed.py: run two 1MB uploads and measure the time it takes warner@allmydata.com**20070920014018] [control.py: make get_memory_usage() callable from elsewhere warner@lothar.com**20070709020754] [started framework for an automated speed-checking tool. Doesn't do much yet. warner@allmydata.com**20070920012747] [Makefile: check_memory.py now manages the combined stats.out itself warner@lothar.com**20070915205336] [Makefile: insert a linebreak after use of PP, since it is long and obscures the real command warner@lothar.com**20070915193211] [Makefile: add 'upload-self' pass to check-memory Brian Warner **20070810084134] [Makefile/check-memory: put all stats in ./memstats.out warner@lothar.com**20070717174141] [check_memory.py: test POST-based uploads as well as from-disk ones warner@allmydata.com**20070717031751] [makefile: add a test-darcs-boringfile target warner@allmydata.com**20070524005128] [check_memory: update it, write stats to a file, disable 50MB test for now warner@allmydata.com**20070717010855] [check_memory: getting closer, now we have memusage numbers for uploads of 10kB and 10MB files warner@lothar.com**20070530003939] [testutil.py: hush a pyflakes warning warner@lothar.com**20070424042150] [re-port Makefile to Windows and remove a bunch of no-longer-needed hacks zooko@zooko.com**20070914012035] [use select reactor instead of iocp reactor on Windows zooko@allmydata.com**20070420030757 Good: this makes it pass all the current unit tests Bad: this makes it limited to around 100 simultaneous peers ] [--rterrors is a nice flag for trial zooko@zooko.com**20070427011152] [GNUmakefile: detect trial's location better on Windows zooko@zooko.com**20070427004901] [Makefile: add build_ext target, for convenience warner@lothar.com**20070816004710] [another dummy record to trigger the buildbot warner@allmydata.com**20070426012223] [dummy record to trigger the buildbot warner@allmydata.com**20070426011348] [fix Makefile not to assume that ":" is the PYTHONPATH separator zooko@zooko.com**20070913223734 (It is ";" on Windows.) ] [Makefile: re-order targets and let user's PYTHONPATH be included in the PP zooko@zooko.com**20070913015633] [import version class and make-version script from pyutil -- fixes win32 build, improves error handling, and eliminates unused features zooko@zooko.com**20070816210930] [manually merge recent changes to pyutil's copy of "version.py" with allmydata's copy of "version.py" "Zooko O'Whielacronx "**20070509230405] [version.py: handle both '0.2.0' and '0.2.0-1', remove 'tags' for now warner@lothar.com**20070504033127] [make-version.py: remove the strip-until- workaround warner@allmydata.com**20070516190258] [make-version.py: accomodate windows by ignoring the junk that appears at the start of 'darcs changes' warner@allmydata.com**20070516001042] [test_client.py: improve test coverage a bit warner@allmydata.com**20070608050902] [catch EnvironmentError from attempt to invoke darcs using subprocess module zooko@zooko.com**20070611185957] [make-version.py invokes darcs as "darcs" instead of "realdarcs" zooko@zooko.com**20070629212250 Some other people might use the official Windows build of darcs, and people who use my cygwin wrapper for darcs will be compatible with this patch as long as they use the latest version of the wrapper. ] [make-version.py: it is sys.platform, not os.platform warner@allmydata.com**20070613010610] [make make-version.py exec darcs correctly on windows robk-org@allmydata.com**20070613005655] [make-version.py: use 'subprocess' module instead of 'commands', to improve windows compatibility warner@allmydata.com**20070516185720] [packaging: add 'build-deps' target, to automatically build and install (locally, in ./support) necessary dependencies. All such installed files are used during tests. warner@lothar.com**20070912234845] [packaging: move to setuptools warner@lothar.com**20070912230253] [transfer any PYTHONPATH environment variable to the environment of the foolscap tests zooko@zooko.com**20070501061947] [add note to GNUmakefile that we append instdir instead of prepending for a good reason "Zooko O'Whielacronx "**20070501021751] [fix: don't pass the PYTHONPATH that features the instdir to the invocation of trial for foolscap zooko@allmydata.com**20070501043728 ] [add warning that "make test" can take a long time "Zooko O'Whielacronx "**20070502055240] [fix one last use of "--root" in building which was corrupting the pathnames in .pyc files "Zooko O'Whielacronx "**20070523220149] [makefile: use --single-version-externally-managed for simplejson warner@lothar.com**20070711023710] [makefile: refactor .deb-producing rules warner@allmydata.com**20070711194500] [fix debian targets in makefile "Zooko O'Whielacronx "**20070501194646] [move debian files down into misc/ warner@allmydata.com**20070705214909] [README: edit to arch_o_median's contribution re: installing the Python way zooko@zooko.com**20070809033245] [README: clarify/edit build, install, and test instructions zooko@zooko.com**20070809034458] [changed README install "Python Way" section wilcoxjg@gmail.com**20070809030035] [update README to reflect existence of "make install" target zooko@zooko.com**20070807191324] [Makefile: improve 'clean' behavior by removing foolscap .pycs and _version.py Brian Warner **20070818063800] [Makefile: use a different rm *.so command for the 'clean' target warner@allmydata.com**20070815222812] [Makefile: more portable way to make clean succeed when there's nothing to rm zooko@zooko.com**20070815211851 xargs doesn't have a "-r" option on Mac OS X. ] [Makefile: fix 'clean' target to work even if there's nothing to clean warner@allmydata.com**20070814212448] [Makefile: oops, fix clean-simplejson target. Might fix #10. Brian Warner **20070818065037] [Makefile: improve 'clean' behavior even more, maybe even completely. Brian Warner **20070818064514 Now we remove downloaded setuptools-*.egg files, and *.egg-info directories ] [remove PyCrypto, copy AES/SHA256/Util.number into the allmydata/ tree warner@allmydata.com**20070814205741] [allmydata.Crypto: fix all internal imports warner@lothar.com**20061214092956] [rename Crypto to allmydata.Crypto "Zooko O'Whielacronx "**20070130223305 This kind of makes me think that there is a general principle that things shouldn't know their own names. After all, a name is what *other* people use to refer to you. And in the general case, some other people might refer to you in ways incompatible with the ways that other people refer to you. ] [.darcs-boringfile: ignore zfec files instead of pyfec files warner@allmydata.com**20070418175355] [modify debian/rules files to handle zfec+setuptools correctly warner@allmydata.com**20070426234517] [README: mention the additional included libraries (zfec, foolscap, pycrypto) warner@allmydata.com**20070516011302] [makefile: build simplejson along with zfec/foolscap/etc warner@lothar.com**20070710222838] [add deb-etch target, rearrange make-version a bit, closes #23 warner@lothar.com**20070504070706] [update version-number handling, pull release tags from darcs history warner@lothar.com**20070504031407] [changed MakeFile to install simplejson wilcoxjg@gmail.com**20070809034126] [add install target, works at lest on sudo-free cygwin tahoe@arnowaschk.de**20070731235545] [cleanup new "instdir" on "make clean" "Zooko O'Whielacronx "**20070130223252] [Makefile: improve 'clean' target to remove debian/ symlink too warner@lothar.com**20061214100809] [README: formatting (line-wrapping) of arch_o_median's edits to README zooko@zooko.com**20070809034742] [edited README "The Python Way" installs to /usr/bin wilcoxjg@gmail.com**20070809033209] [changed README to inslude simplejson in "The Python Way" install wilcoxjg@gmail.com**20070809034340] [some edits to the README zooko@zooko.com**20070611164054] [re-re-factored the download and install instructions "Zooko O'Whielacronx "**20070502195426 Now the instructions about how to download debian packages live on a separate page of the wiki instead of on the front page or in the README. The README is only about building from source. The front page contains pointers to those two other pages (the debiandownloadpage and the README). ] [two more fixes for README "Zooko O'Whielacronx "**20070502054024] [add another note about where to run "make" "Zooko O'Whielacronx "**20070502053735] [fix README on how to install The Debian Way "Zooko O'Whielacronx "**20070502063839] [clarify instructions on how to get build dependencies for debian "Zooko O'Whielacronx "**20070502061606 My test subject here didn't used "locate" to find something named "debian/control" somewhere on his filesystem... :-) ] [move the debian packaging dependencies to the place where they are needed in the README "Zooko O'Whielacronx "**20070502061105] [tweak README to be more helpful about a small detail "Zooko O'Whielacronx "**20070502051149] [fix the TESTING THAT IT IS PROPERLY INSTALLED section "Zooko O'Whielacronx "**20070502064201 My oh-so-useful test subject here skipped that section because he thought it was part of a section that he was finished with. ] [boringfile: ignore the setuptools .egg that ez_setup.py likes to download/build/install during compilation warner@allmydata.com**20070810024518] [.darcs-boringfile: ignore the buildbot's source-stampfile warner@allmydata.com**20070524011242] [Makefile: don't attempt to run trial tests if the dependencies can't be imported zooko@zooko.com**20070914021255] [don't use --root, instead use --single-version-externally-managed "Zooko O'Whielacronx "**20070521174714 fixes #35 ] [makefile: delete .pycs before running tests, since our build process puts the wrong filenames into them warner@lothar.com**20070504041215] [makefile: change the 'test' target to exclude foolscap, add 'test-all' to do both warner@allmydata.com**20070501180839] [Makefile: add an upload-figleaf target, rather than having the buildbot create the rsync command itself warner@lothar.com**20070308212813] [make separate makefile targets for test-foolscap and test-TEST zooko@allmydata.com**20070501060245 So that if you want to run just one test, you can run "make test-TEST TEST=that.one.test" and not get the whole foolscap suite thrown in for the bargain. ] [run foolscap tests from src instead of from instdir since .pyc's in instdir have funny filenames encoded into them that cause foolscap test failures "Zooko O'Whielacronx "**20070501021806] [turn foolscap unit tests back on in the "make test" target "Zooko O'Whielacronx "**20070430180413] [don't run foolscap unit tests "Zooko O'Whielacronx "**20070430032528] [cli: add --node-directory and --root-uri to all commands warner@lothar.com**20071011073036] [command-line: remove the --vdrive option (it is now hardcoded to "global") zooko@zooko.com**20070823202700] [command-line: fix a few bugs in the "execute this python file" way to execute rm zooko@zooko.com**20070817211731] [command-line: add "rm", and tidy-up variable names, and make it so "allmydata-tahoe spam" prints a usage message instead of returning silently zooko@zooko.com**20070817202316] [relnotes.txt: edit and update relnotes and clarify licence zooko@zooko.com**20070817192509] [small edit of relnotes zooko@zooko.com**20070723012337] [in relnotes.txt, make the safety warnings slightly less scary zooko@zooko.com**20070807212323 and update the reference to the wiki's UseCases page ] [clarify licence zooko@zooko.com**20070809034442] [make big distinction between downloading precompiled packages and building yourself "Zooko O'Whielacronx "**20070502053433 Both of my two test subjects here stumbled on this by thinking that building on debian was related to building debian packages was related to downloading precompiled packages. ] [update README to reflect dependency on python-dev "Zooko O'Whielacronx "**20070430063349] [README: mention the names of the debian packages that provide our dependencies warner@allmydata.com**20070430194830] [add the apt-get information to the README "Zooko O'Whielacronx "**20070502050458] [clarify licence zooko@zooko.com**20070629222815] [update licence to allow linking with OpenSSL zooko@zooko.com**20070430045605] [relnotes.txt: a few more updates to relnotes.txt zooko@zooko.com**20070816235222] [relnotes.txt: incomplete update to describe v0.5 zooko@zooko.com**20070816230302 more to come... ] [update relnotes.txt to reflect NJS's comments zooko@zooko.com**20070723012129] [relnotes.txt: reflow to 70-char width zooko@zooko.com**20070705015926] [fix small typo in relnotes.txt "Zooko O'Whielacronx "**20070502205211] [relnotes.txt update for v0.4 release zooko@zooko.com**20070629235805] [fix the relnotes.txt -- don't incorrectly attribute a compatibility break to foolscap zooko@zooko.com**20070611172014] [relnotes.txt reflow column width and editing zooko@zooko.com**20070629235827] [relnotes.txt: mention the private filesystem warner@allmydata.com**20070630000149] [relnotes.txt: formatting for thinner columns zooko@zooko.com**20070629235538] [trivial formatting tweak in relnotes.txt "Zooko O'Whielacronx "**20070502040505] [update the relnotes.txt for v0.3 zooko@zooko.com**20070611162142] [edit: fix typo and formatting in relnotes.txt "Zooko O'Whielacronx "**20070430155700] [edit description of Foolscap in relnotes.txt "Zooko O'Whielacronx "**20070501004306] [better description of Foolscap in README zooko@zooko.com**20070501003451] [edits to relnotes.txt "Zooko O'Whielacronx "**20070501152647] [relnotes.txt: editing for English usage "Zooko O'Whielacronx "**20070419210915] [edit relnotes.txt "Zooko O'Whielacronx "**20070427233752] [relnotes.txt: all platforms pass unit tests. Whee! "Zooko O'Whielacronx "**20070420035301] [relnotes.txt: a bit of editing "Zooko O'Whielacronx "**20070420035319] [small edit to target market "Zooko O'Whielacronx "**20070502200614] [edit release notes to give proper user expectations "Zooko O'Whielacronx "**20070430042455] [tahoe v0.2.0-0-UNSTABLE "Zooko O'Whielacronx "**20070502232323] [v0.2.0b2-0-UNSTABLE "Zooko O'Whielacronx "**20070501224619] [grand unified version numbers "Zooko O'Whielacronx "**20070501202315 Now the allmydata python package, the setup.py script, and the debian packages all get their tahoe version number from the same place. ] [v0.2.0b1-0-UNSTABLE "Zooko O'Whielacronx "**20070501061123] [v0.1.4b2-0-UNSTABLE "Zooko O'Whielacronx "**20070430170007] [v0.1.3-0-UNSTABLE "Zooko O'Whielacronx "**20070430130243] [v0.1.2-0-UNSTABLE "Zooko O'Whielacronx "**20070430060008] [allmydata-tahoe v0.1.1-1-UNSTABLE "Zooko O'Whielacronx "**20070427230722] [tahoe v0.1.1-0-UNSTABLE "Zooko O'Whielacronx "**20070426182515] [assign version number tahoe 0.1.0-0-UNSTABLE "Zooko O'Whielacronx "**20070419204759] [copy version.py from pyutil "Zooko O'Whielacronx "**20070419204736] [make relnotes.txt point to the latest source tarball "Zooko O'Whielacronx "**20070430175635] [update URL pointing to source code tarball "Zooko O'Whielacronx "**20070430060539] [update URLs in relnotes.txt "Zooko O'Whielacronx "**20070427231244] [command-line: remove some redundant options checking zooko@zooko.com**20070817200643] [command-line: fix all three commands and all two ways to invoke them to require node-url and give a useful usage string if node-url is absent or of the wrong form zooko@zooko.com**20070817195447] [tahoe_ls: remove a couple of vestigial or debugging bits zooko@zooko.com**20070711024640] [scripts: rearrange Options, make --version behave the same for all commands warner@allmydata.com**20070816195019] [put now exits after doing its work, and it prints a terse message if 200 or 201, and a full dump of response elsewise zooko@zooko.com**20070816233039] [incomplete version of tahoe-put.py zooko@zooko.com**20070816230101 It doesn't exit properly afterward, and it might not do the best things with non-success responses from the server. (See tahoe-put-web2ish.py for an example of better response handling.) ] [tahoe_put.py: require node-url to be provided zooko@zooko.com**20070817190611] [cli: use port 8123 as the example port number zooko@zooko.com**20070924201727] [cmdline: change "--server" to "--node-url" and make it have no default value zooko@zooko.com**20070816235327] [a first crack at the "put" command-line zooko@zooko.com**20070816191538 There are actually two versions in this patch, one of which requires twisted.web2 and the other of which uses the Python standard library's socket module. The socketish one doesn't know when the web server is done so it hangs after doing its thing. (Oh -- maybe I should add an HTTP header asking the web server to close the connection when finished.) The web2ish one works better in that respect. Neither of them handle error responses from the server very well yet. After lunch I intend to finish the socketish one. To try one, mv src/allmydata/scripts/tahoe_put-{socketish,web2ish}.py src/allmydata/scripts/tahoe_put.py . If you want to try the web2ish one, and you can't find a web2 package to install, you can get one from: http://allmydata.org/~zooko/repos/twistedweb2snarf/ ] [cli: implement 'get' warner@allmydata.com**20070711172619] [tahoe-get.py: add optional target-local-file argument zooko@zooko.com**20070710012711] [tahoe-get.py: fix bug: actually respect the optional server argument zooko@zooko.com**20070710014716] [tahoe-get.py: accept vdrive and server options (using optparse) zooko@zooko.com**20070710012002] [tahoe-get.py: the first, most primitive command-line client zooko@zooko.com**20070710011953] [cli: implement 'allmydata-tahoe ls' warner@lothar.com**20070711023737] [tahoe-ls.py: hush a pyflakes warning warner@lothar.com**20070710232605] [tahoe-ls.py: initial version of an 'ls' CLI utility warner@lothar.com**20070710223455] [runner.py: further refactoring warner@lothar.com**20070711020518] [fix 'allmydata restart' warner@lothar.com**20061205183758] [runner.py: add --force flag to restart, to restart a not-already-running node warner@lothar.com**20070707181732] [runner.py: start to refactor into separate files warner@lothar.com**20070711014152] [runner.py: make 'allmydata-tahoe --version' emit version numbers of everything warner@allmydata.com**20070716215836] [runner.py: improve test coverage further: implement --quiet with StringIOs warner@lothar.com**20070626231918] [node creation: be willing to utilize a pre-existing (but empty) directory warner@allmydata.com**20070329213228] [runner.py: add --quiet, use it from test cases warner@lothar.com**20070424041713] [webish: add checker results and a 'Check' button to the web interface warner@allmydata.com**20071024002357] [include the filename on URI-link urls for files robk-org@allmydata.com**20070711222647 this is likely to induce browsers to do more useful things with the result than they would when given neither content-type nor filename. (i.e. they can guess that a .jpg file is an image, even with a bogus content type) ] [docs: webapi: Add concise shorthand for options, input, output, and statuses to the operation descriptions... nejucomo@gmail.com**20080130072742 I'm not convinced if this is the best way to do this, but I find it handy to have a conscise "quick reference" for the webapi operations which summarize all related I/O. Another possibility is to reject this patch, but create a separate "webapi_quickref.html" with a concise table. ] [new improved webapi.txt zooko@zooko.com**20070823200326 As per ticket #118, this refactors the explanation of URIs and paths and changes the JSON metadata schema. http://allmydata.org/trac/tahoe/ticket/118 ] [webapi.txt: shorter and hopefully clearer description of names vs. identifiers zooko@zooko.com**20070815192804 Brian (and anyone who has an interest in the API and documentation): please review. ] [webapi.txt: minor clarifications and examples warner@lothar.com**20070810225227] [webapi.txt: add URI-based GET variants zooko@zooko.com**20070810193329] [webapi.txt: clear up underspecified items, replace 'webpassword' paragraph Brian Warner **20070816020447 with a section about our expected plans for #98, add more introductory text to the sections on manipulate-file vs manipulate-directory. ] [webapi.txt: specify replace= behavior on all PUT and POST commands Brian Warner **20070816023149] [webapi.txt: s/dirnodes/directories/ zooko@zooko.com**20070816225353] [webapi.txt: separate out debug/test commands, indicate that localfile=/localdir= requires special activation warner@lothar.com**20070811012022] [webapi.txt: put back the localfile feature zooko@zooko.com**20070810195237 So that we can compare versions webapi.txt with and without this documentation side by side. ] [webapi.txt: some editing, and remove the localfile feature and the manifest feature zooko@zooko.com**20070810192413 My motivation to remove these features is as per: http://allmydata.org/pipermail/tahoe-dev/2007-August/000067.html However, I haven't heard back from Brian yet, so I'm actually going to put them back in the next patch so that I can compare the two versions of webapi.txt side by side. ] [webapi.txt: put back the manifest feature zooko@zooko.com**20070810195833 So that we can compare versions of webapi.txt with and without this documentation, side by side. ] [webapi.txt: further refactoring and add a section explaining TOCTTOU bugs and how to avoid them by using URIs zooko@zooko.com**20070810190430] [webapi.txt: further refactoring and editing to clarify the fact that you don't know whether a thing is a file or a directory before you fetch it zooko@zooko.com**20070810171927] [add a 'rename' button to the webish dir view robk-org@allmydata.com**20070712235354 alongside the 'del' button is now presented a 'rename' button, which takes the user to a new page, the 't=rename-form' page, which asks ther user for the new name of the child and ultimately submits a POST request to the dir for 't=rename' to perform the actual rename i.e. an attach followed by a delete of children. ] [webapi.txt: update rfc reference warner@lothar.com**20070809182435] [webapi.txt: add "?t=file" flag and reorganize doc to discourage people from thinking that they know before hand the file-or-dir type of the thing that they are naming zooko@zooko.com**20070810164352] [web: /uri/ must escape slashes, we use bangs for this warner@lothar.com**20070708050652] [web: remove t=XML, and other dead code warner@lothar.com**20070708055515] [webish.py: add links to JSON/etc representations of directory contents to the listing warner@lothar.com**20070707183107] [webapi: normalized API: use t=upload or t=download when providing localdir= or localfile= warner@lothar.com**20070710202410] [webapi updates warner@lothar.com**20070707173707] [web: use real JSON instead of the fake stubs warner@lothar.com**20070708071711 Also include the encoder portion of Bob Ippolito's simplejson-1.7.1 as allmydata.util.json_encoder . simplejson is distributed under a more liberal license than Tahoe (looks to be modified BSD), so redistributing it should be ok. ] [web: add when_done to all POST operations, use it from upload/mkdir/mount/delete forms warner@lothar.com**20070708041748] [web: change per-directory forms to match new POST scheme warner@lothar.com**20070708033547] [webish: when mounting a shared directory, don't automatically move to it warner@lothar.com**20070615093424] [webish: log dirname in mkdir warner@lothar.com**20070615094819] [web: make 'delete' buttons work again warner@lothar.com**20070708034630] [web: remove debug prints warner@lothar.com**20070708031130] [web: more test work, now all tests either pass or are skipped (POST, XMLRPC, and URI/) warner@lothar.com**20070707173405] [add IDirectoryNode.get_child_at_path warner@lothar.com**20070707023837] [dirnode: add build_manifest() and introduce 'refresh capabilities' warner@lothar.com**20070627024120] [web: more test work, now all tests pass, POST too, only XMLRPC left to implement warner@lothar.com**20070708030658] [webapi: checkpointing more test progress warner@lothar.com**20070707071636] [webish.py: put the URI in a narrower auto-scrolling box (with CSS) warner@lothar.com**20070615083220] [webish: fix ?t=manifest, ?t=xml so they don't throw exceptions, prune directory.xhtml warner@lothar.com**20070707181531] [checkpointing new webapi: not all tests pass yet warner@lothar.com**20070707024355] [webish: display program/library versions on the welcome page warner@allmydata.com**20070611175111] [node.py: hush pyflakes warnings warner@allmydata.com**20070524005448] [make stopService() defer until startService() completes (fixes a problem with the new not-yet-committed unit test) "Zooko O'Whielacronx "**20070523220803] [update docs, remove extraneous licence text, sort module names in import statement "Zooko O'Whielacronx "**20070521204251 closes #46 ? ] [node.py: switch to using get_local_addresses_async, which is slightly more portable (most forms of unix) warner@lothar.com**20070308211252] [control: add RIControlClient, fix some pyflakes warnings warner@lothar.com**20070308012027] [fix handling of local_ip file and rename it to advertised_ip_addresses and document it in README "Zooko O'Whielacronx "**20070522210140] [make it print out version numbers when it constructs a Node instance "Zooko O'Whielacronx "**20070427204738] [some English usage edits to README, thanks to Stephen Hill "Zooko O'Whielacronx "**20070502040323] [add some simple create-client/start-client targets, document htem briefly warner@allmydata.com**20070427031648] [update README in response to some criticisms from some European grad student on IRC "Zooko O'Whielacronx "**20070428001325] [install libs into instdir/lib and scripts into instdir/bin zooko@allmydata.com**20070501004910] [don't include test code itself in the test-coverage numbers warner@allmydata.com**20070416193245] [remove "." from setup.py invocation because it seems to be screwing up paths in the .pyc's "Zooko O'Whielacronx "**20070430214336] [build foolscap from the bundled snapshot zooko@zooko.com**20070430024418 and fix PYTHONPATH usage in a few places in GNUmakefile ] [GNUmakefile: clean up better zooko@zooko.com**20070418230954] [GNUmakefile: rm -rf ./src/pyfec/build on clean "Zooko O'Whielacronx "**20070418164947] [GNUmakefile: don't stop clean if clean-pyfec fails "Zooko O'Whielacronx "**20070418164824] [GNUmakefile: 'test' should use zfec, not fec warner@allmydata.com**20070419005918] [port GNUmakefile to Windows, cygwin, Linux, MacOSX/PPC, MacOSX/Intel "Zooko O'Whielacronx "**20070410192432 And of course it may well work on lots of other modern unixes, too, especially the more GNUish ones. ] [include pyfec in the trial tests warner@allmydata.com**20070407011723] [pyfec: trial-ize the unit tests, making sure to keep working if trial is unavailable warner@allmydata.com**20070407011650] [pyfec: fix some error-checking, add more unit tests warner@allmydata.com**20070328195312] [pyfec: correctly handle wrongly typed input by raising exception zooko@zooko.com**20070328065332] [pyfec: fix preconditions and typing, remove unused error-checking, tidy-up naming and documentation "Zooko O'Whielacronx "**20070201060325] [fix tests to not require pyutil and other tweaks zooko@zooko.com**20070125212534] [pyfec: add precondition checks on the values of k and m to constructors zooko@zooko.com**20070126004704] [pyfec: remove (optimize) unused code zooko@zooko.com**20070127021028] [add comment "Zooko O'Whielacronx "**20070124212145] [pyfec: trivial formatting tweak zooko@zooko.com**20070127021053] [pyfec: fix preconditions, tighten internal C types, fix bugs in the file-encoding and benchmarking utility functions "Zooko O'Whielacronx "**20070130163735] [use buffers as generic "read" buffers instead of char buffers zooko@zooko.com**20070125212401 This is a typing kludge -- if your buffers have elements of size > 1 then we will be processing only a subset of the elements and treating each byte of the element as a separate entry. Oh well. ] [pyfec: make README.txt much more detailed and rename some internal variables and add some docstrings zooko@zooko.com**20070127020850] [stricter typing -- using unsigned char for indexes into shares "Zooko O'Whielacronx "**20070124211751] [add README.txt zooko@zooko.com**20070126004637] [pyfec: add utility functions to encode into a bunch of temp files and to decode from a bunch of tempfiles zooko@zooko.com**20070127021527] [pyfec: delete m-k of the tempfiles at random in the benchmark of the to/from files zooko@zooko.com**20070127022213] [pyfec: move benchmark code from test_pyfec.py into the new bench_pyfec.py and add more benchmark code zooko@zooko.com**20070127021549] [add benchmark zooko@zooko.com**20070125233348] [add utility method file_fec zooko@zooko.com**20070125212519] [pyfec: fix typo in unit test "Zooko O'Whielacronx "**20070201162846] [pyfec: execute tests on import of test_pyfec.py only if it is the __main__ zooko@zooko.com**20070127021759] [use zfec's setuptools installer in the nice controllable way zooko@allmydata.com**20070426224016] [GNUmakefile: remove test-zfec target, since the normal 'test' target provides zfec coverage warner@allmydata.com**20070418184100] [pyflakes: do not assume that pyflakes lives in /usr/local, just assume that it is on the PATH warner@allmydata.com**20070427020048] [make pyflakes run faster and with more alacrity zooko@zooko.com**20070427011235] [edit README "Zooko O'Whielacronx "**20070501195724] [improve "the Debian way" build process to produce foolscap .debs "Zooko O'Whielacronx "**20070501194135] [add .deb targets for edgy warner@allmydata.com**20070404233925] [large update to the install instructions in the README, and add a link to allmydata.org zooko@zooko.com**20070430051633] [update README to reflect simpler run-in-place procedure "Zooko O'Whielacronx "**20070428001751] [update README to show that foolscap is bundled and to fix the module name of zfec zooko@zooko.com**20070430045623] [update README to mention which dependencies are python packages "Zooko O'Whielacronx "**20070427224313 In particular, my brother is an Ubuntu/Debian user who said it would be nice if we had mentioned that those packages were named "python-foo" in the apt package namespace. ] [more README updates in response to user feedback "Zooko O'Whielacronx "**20070427230351 The user in this case is my brother Josh. ] [webish.py: add a web page to display the manifest for any particular directory warner@lothar.com**20070627025521] [Add the 'vdrive' service, for clients to access the public/private root dirs. warner@allmydata.com**20070628001106 These allow client-side code to conveniently retrieve the IDirectoryNode instances for both the global shared public root directory, and the per-user private root directory. ] [client.py: use persistent FURLs for our Node, to reduce spurious connection attempts (#26) warner@allmydata.com**20070607223221] [merge vdrive.py and filetable.py into a single dirnode.py warner@lothar.com**20070627001658] [remove old filetree code warner@lothar.com**20070626033419] [filetree: mark leaf nodes by adding is_leaf_subtree(), stop traversing when we hit them, to make vdrive._get_file_uri() work warner@lothar.com**20070121193116] [test_filetree: more vdrive._get_file_uri() coverage warner@lothar.com**20070121193940] [hush pyflakes warner@allmydata.com**20070124212922] [use the "binary" flag on open() for files that shouldn't have line-endings automatically converted "Zooko O'Whielacronx "**20070404231230] [vdrive.py: let put_file return the URI that was used for the file itself warner@lothar.com**20070424084140] [add licence, update metadata and URLs "Zooko O'Whielacronx "**20070427204715] [rename bin/allmydata to bin/allmydata-tahoe, so the package can co-exist with the regular (amdlib) package warner@allmydata.com**20070120020533] [relnotes.txt: English usage (more) "Zooko O'Whielacronx "**20070419211648] [relnotes.txt: update probable project URLs warner@allmydata.com**20070420031121] [further update the notes about the dependencies zooko@zooko.com**20070427051613] [update README re: dependencies "Zooko O'Whielacronx "**20070427040049] [add setuptools to required dependencies in README "Zooko O'Whielacronx "**20070426225654] [README: mention 'fakeroot' and other build dependencies to creat debian packages warner@allmydata.com**20070426220022] [update README "Zooko O'Whielacronx "**20070426212815] [tweak README zooko@zooko.com**20070409231444] [tahoe readme - updated to included all packages and clarify text secorp@allmydata.com**20070329190232] [update relnotes.txt "Zooko O'Whielacronx "**20070427051920] [a couple of minor edits to relnotes.txt "Zooko O'Whielacronx "**20070426182457] [relnotes.txt: fix typo "Zooko O'Whielacronx "**20070419210520] [update tarball URL "Zooko O'Whielacronx "**20070427045436] [globally search and replace "mesh" with "grid" and adjust description of the effect of NAT on the topology "Zooko O'Whielacronx "**20070430200609] [vdrive: protect dirnode contents with an HMAC warner@lothar.com**20070626193621] [more runner.py test coverage: don't bypass argv parsing warner@lothar.com**20070626235138] [improve test coverage of runner.py warner@lothar.com**20070626223646] [test_runner.py: improve test coverage a little bit warner@lothar.com**20070424042819] [dump_uri_extension: improve test coverage of runner.py warner@lothar.com**20070626225500] [testutil: make SignalMixin actually be a mixin (and not inherit from TestCase), use it from all tests that start notes and thus exec ifconfig warner@lothar.com**20070424041502] [iputil/testutil: fix pyflakes errors/warnings warner@allmydata.com**20070419013337] [iputil.list_async_addresses now "works" on cygwin zooko@zooko.com**20070419003008] [util.iputil: try to survive not having a global network connection at all warner@lothar.com**20070308002142] [test_iputil: remove the test that only works on linux, since we're using the cross-unix 'get_local_addresses_async' anyways. This should allow the tests to pass on OS-X warner@allmydata.com**20070329182117] [oops -- the previous commit of iputil wasn't the right version "Zooko O'Whielacronx "**20070416221201 Too bad synchronizing pyutil and allmydata.util includes a manual step. ] [test_iputil: improve error message warner@allmydata.com**20070416220525] [pyutil: iputil: fix netbsd, irix, sunos zooko@zooko.com**20070418144026] [iputil.py: remove unused import warner@allmydata.com**20070417000800] [port iputil to Windows (and Irix, and NetBSD, and Solaris 2, ...) "Zooko O'Whielacronx "**20070416215913] [iputil: use the subprocess module instead of os.popen warner@lothar.com**20070308012900] [iputil: use explicit /sbin/ifconfig, to avoid depending upon PATH warner@lothar.com**20070308004740] [iputil: switch to a scapy-inspired SIOCGIFADDR approach, very linux-specific now warner@lothar.com**20070308020347] [copy testutil from pyutil zooko@zooko.com**20070419002836] [copy repeatable_random from pyutil zooko@zooko.com**20070419002733] [remove unused/obsoleted workqueue.py warner@lothar.com**20070627002523] [fix some of the filetree/workqueue interface definitions warner@allmydata.com**20070126233124] [extirpate all references the "queen" and "metatracker" "Zooko O'Whielacronx "**20070430165752 This is a potentially disruptive and potentially ugly change to the code base, because I renamed the object that serves in both roles from "Queen" to "IntroducerAndVdrive", which is a bit of an ugly name. However, I think that clarity is important enough in this release to make this change. All unit tests pass. I'm now darcs recording this patch in order to pull it to other machines for more testing. ] [rename "pyfec" to "zfec" within tahoe build system and source code "Zooko O'Whielacronx "**20070418171123] [probably fix the debian dapper packaging to include pyfec and crypto.. sid still needs to be fixed warner@allmydata.com**20070329190724] [debian: change name of debian packages to 'allmydata-tahoe' to avoid conflicts warner@allmydata.com**20061205204002] [fix the sid debian packaging too warner@allmydata.com**20070329191639] [first attempt at adding debian packaging support for ubuntu 'feisty' warner@allmydata.com**20070329213615] [debian: give incrementing version numbers to .deb packages, using timestamps warner@allmydata.com**20070106020651] [dummy line to show off buildbot warner@lothar.com**20070202224545] [add some USAGE notes to the release notes, capitalize section headers, some small edits warner@allmydata.com**20070420003006] [relnotes.txt: English usage "Zooko O'Whielacronx "**20070419211100] [rough draft of release announcement for 0.1 "Zooko O'Whielacronx "**20070419205513] [more architecture docs, this is fun warner@allmydata.com**20070420081429] [add architecture/code-layout documents describing our current architecture and a bit of our future plans warner@allmydata.com**20070420064347] [add a couple of todos to roadmap.txt (one of which is to eliminate roadmap.txt) "Zooko O'Whielacronx "**20070426230134] [remove a couple of items from the todo list in roadmap.txt "Zooko O'Whielacronx "**20070419204824] [update roadmap warner@allmydata.com**20070419174437] [merge TODO into roadmap.txt "Zooko O'Whielacronx "**20070410173338] [update roadmap: add download-peer-selection warner@allmydata.com**20070405193422] [roadmap: connection management v1 milestone done warner@lothar.com**20061201021924] [update notes about lease deletion logic zooko@zooko.com**20061204072512] [update roadmap with webish UI progress warner@lothar.com**20061205015622] [got read/write webish interface working Brian Warner **20061204182252] [roadmap.txt: editing "Zooko O'Whielacronx "**20070403173234] [roadmap: updated who is working on what "Zooko O'Whielacronx "**20070403171620 Namely that we don't know who is working on what at the moment... ] [update roadmap more warner@lothar.com**20061204015505] [add Operations/Deployment for first open source release "Zooko O'Whielacronx "**20070403173247 As per discussion I had with Peter this morning, we should do these three things and then release the first open source version. ] [add TODO: port to Windows "Zooko O'Whielacronx "**20070410162758] [update TODO (add operational TODOs) zooko@zooko.com**20070409232034] [TODOs zooko@zooko.com**20070330175211] [add TODO zooko@zooko.com**20070330031938] [webish: fix link to private_vdrive warner@allmydata.com**20070629020752] [webish: make each link in the directory name an href to that directory warner@allmydata.com**20070629181718] [vdrive: switch to URI:DIR and URI:DIR-RO, providing transitive readonlyness warner@lothar.com**20070625202351] [filetable: oops, os.listdir() does not guarantee sorted results warner@lothar.com**20070120190226] [rename queen control files to 'introducer' warner@allmydata.com**20070420012415] [switch from FieldStorage.value to FieldStorage.file "Zooko O'Whielacronx "**20070525230019 Unfortunately this doesn't make the O(n) memory usage go away. It might reduce the constants -- I'm not sure. I look forward to enhancement #54 -- memory usage tests! ] [test_filetable: fix to match new vdrive approach warner@allmydata.com**20070615070101] [runner.py: add 'dump-directory-node' command warner@lothar.com**20070615074705] [add 'allmydata-tahoe dump-uri-extension' utility command warner@allmydata.com**20070612013821] [runner.py: allow --multiple to enable starting/stopping/creating multiple nodes at once warner@allmydata.com**20070606210657] [test_runner.py: add some coverage for allmydata.scripts.runner, to create nodes warner@allmydata.com**20070420015645] [runner.py: allow all directory-using commands (create/start/stop) to accept argv[-1] as well as --basedir warner@allmydata.com**20070606183719] [runner.py: spoke too soon. Really fix #51 this time. warner@allmydata.com**20070524182039] [make runner find twistd and invoke it with python zooko@zooko.com**20070427061413 This makes allmydata-tahoe work on Windows, and probably doesn't make it fail on other platforms. ] [runner.py: expanduser() basedirs, so '~' works. Closes #51. warner@allmydata.com**20070524181019] [webish: enable deletion of directories warner@lothar.com**20070615083257] [create a personal (non-shared) vdrive, in addition to the global shared one warner@lothar.com**20070615083324] [minor change to test buildbot triggering for new repository warner@allmydata.com**20061207210520] [check_memory.py: finish the failsafe-shutdown code warner@allmydata.com**20070525003442] [make new vdrive work, implement convenience wrapper, passes all tests warner@lothar.com**20070615073732] [webish: present real URI in directory listing, not an unnecessarily b2a'ed form warner@allmydata.com**20070117204602] [vdrive/webish: finish verifierid/uri transition warner@allmydata.com**20070117014313] [add some vdrive logging warner@lothar.com**20061204064636] [webish: show vdrive and introducer connectedness separately warner@lothar.com**20070610040357 Also don't offer a link to the vdrive webpages if we don't have a vdrive.furl ] [fix some python2.5 incompatibilities, and remove an old webish display that suggested we might know about peers but not be connected to them warner@allmydata.com**20070328004449] [replace 'queen' with 'introducer' in a lot of places, but not all warner@allmydata.com**20070420003021] [fix return value of 'allmydata restart' Brian Warner **20061206014352] [add more useful text to the README warner@allmydata.com**20070405005535] [port the makefile to Windows and update the README zooko@allmydata.com**20070404181838] [Makefile: add a default target to just build the tree warner@lothar.com**20070107190937] [add test-pyfec target, not added to the main 'test' target because it adds 20 seconds to the test run warner@lothar.com**20070308235438] [fix purely syntactic merge conflict "Zooko O'Whielacronx "**20070201220707] [Makefile: $(PWD) doesn't always work, in particular the buildslaves don't Brian Warner **20070201093913 update environment variables like $(PWD) when the launch commands. ] ["make clean" recursively runs cleanup in subprojects (pyfec and Crypto) "Zooko O'Whielacronx "**20070201214952] [Makefile: update figleaf code to use INSTDIR instead of now-obsolete builddir.py warner@allmydata.com**20070306014606] [Makefile: parameterize 'trial' and 'python' to enable a python2.5 builder warner@allmydata.com**20070201011233] [modify figleaf2html to show module names instead of .py filenames, also add a --root argument to restrict coverage to a specific parent directory warner@lothar.com**20070104042325] [move figleaf2html and figleaf_htmlizer.py into our tree, for easier customization warner@lothar.com**20070104040651] [figleaf: delete coverage file (.figleaf) before each test run, otherwise I suspect coverage data will accumulate from one test run to the next warner@allmydata.com**20070109032808] [figleaf.el: when converting to the elisp-readable format, use our in-tree version of figleaf rather than one found on the system. Also change the keybinding to toggle annotations to C-cC-a, which is easier to type warner@allmydata.com**20070109032903] [more figleaf emacs work: enable the minor mode, then type C-cA to toggle annotations warner@lothar.com**20070104072536] [forget about old peers (closes #26) warner@allmydata.com*-20070508021024 Add a new method to RIIntroducer, to allow the central introducer node to remove peers from the active set after they've gone away. Without this, client nodes accumulate stale peer FURLs forever. This introduces a compatibility break, as old introducers won't know about the 'lost_peers' message, although the errors produced are probably harmless. ] [forget about old peers (closes #26) warner@allmydata.com**20070508021024 Add a new method to RIIntroducer, to allow the central introducer node to remove peers from the active set after they've gone away. Without this, client nodes accumulate stale peer FURLs forever. This introduces a compatibility break, as old introducers won't know about the 'lost_peers' message, although the errors produced are probably harmless. ] [revamp vdrive: nodes with furls. tests still fail. warner@allmydata.com**20070615031434] [test_system: improve webish.py coverage warner@lothar.com**20070424084154] [add RIClient.get_versions, in the hopes of enabling backwards-compatibility code in the future warner@allmydata.com**20070426190125] [enable private upload, in which the file is inserted at the grid layer but not at the vdrive layer "Zooko O'Whielacronx "**20070516154019 This patch is actually by Faried Nawaz, as per ticket #33: http://allmydata.org/trac/tahoe/ticket/33 ] [test_system.py: minor reformatting warner@allmydata.com**20070601013101] [client.py: allow operation without vdrive.furl, for storage-only no-UI nodes warner@allmydata.com**20070608005549] [move almost all hashing to SHA256, consolidate into hashutil.py warner@allmydata.com**20070608044721 The only SHA-1 hash that remains is used in the permutation of nodeids, where we need to decide if we care about performance or long-term security. I suspect that we could use a much weaker hash (and faster) hash for this purpose. In the long run, we'll be doing thousands of such hashes for each file uploaded or downloaded (one per known peer). ] [hashutil: convenience methods for tagged and encoded hashes zooko@zooko.com**20070330011130 In various cases, including Merkle Trees, it is useful to tag and encode the inputs to your secure hashes to prevent security flaws due to ambiguous meanings of hash values. ] [interfaces: use explicit TupleOf and ChoiceOf constraints, since the upcoming version of Foolscap changes the meaning of bare tuples (from ChoiceOf to TupleOf) warner@lothar.com**20070414020438] [raise constraint on FURLs from 150 chars to 1000 chars "Zooko O'Whielacronx "**20070522205917] [encode/download: reduce memory footprint by deleting large intermediate buffers as soon as possible, improve hash tree usage warner@lothar.com**20070607201558] [encode.py: hush pyflakes warnings warner@lothar.com**20070607201855] [encode.py: fix pyflakes warning Brian Warner **20070607095616] [test_encode.py: even more testing of merkle trees, getting fairly comprehensive now warner@allmydata.com**20070608042439] [change #!/usr/bin/python to #!/usr/bin/env python zooko@zooko.com**20070329210128 Note that using "whatever version of python the name 'python' maps to in the current shell environment" is more error-prone that specifying which python you mean, such as by executing "/usr/bin/python setup.py" instead of executing "./setup.py". When you build tahoe (by running "make") it will make a copy of bin/allmydata-tahoe in instdir/bin/allmydata-tahoe with the shebang line rewritten to execute the specific version of python that was used when building instead of to execute "/usr/bin/env python". However, it seems better that the default for lazy people be "whatever 'python' means currently" instead of "whatever 'python' meant to the manufacturer of your operating system". ] [tool to generate an overhead/alacrity table for various hashing schemes warner@lothar.com**20061022031004] [added a simulator tool warner@lothar.com**20061128222708] [control.py: fix get_memory_usage, add a sample client tool warner@lothar.com**20070308023149] [add tests for bad/inconsistent plaintext/crypttext merkle tree hashes warner@allmydata.com**20070608023229] [download.py: refactor get-thingA-from-somebody to reuse the logic for other things warner@lothar.com**20070607065002] [fetch plaintext/crypttext merkle trees during download, but don't check the segments against them yet warner@lothar.com**20070607071541] [use absolute import of 'allmydata.Crypto' rather than a relative import of just 'Crypto', to make it clear that we're using our own form rather than relying upon the system version warner@allmydata.com**20070106031226] [encode: add plaintext/crypttext merkle trees to the shares, and the thingA block. Still needs tests and download-side verification. warner@lothar.com**20070607024020] [encode.py: clean up handling of lost peers during upload, add some logging warner@allmydata.com**20070606194016] [test_encode.py: further refactoring of send_and_recover warner@allmydata.com**20070608013625] [test_encode.py: refactor send_and_recover a bit warner@allmydata.com**20070608012426] [encode: tolerate lost peers, as long as we still get enough shares out. Closes #17. warner@allmydata.com**20070606173240] [move validation data to thingA, URI has storage_index plus thingA hash warner@allmydata.com**20070602014801 This (compatibility-breaking) change moves much of the validation data and encoding parameters out of the URI and into the so-called "thingA" block (which will get a better name as soon as we find one we're comfortable with). The URI retains the "storage_index" (a generalized term for the role that we're currently using the verifierid for, the unique index for each file that gets used by storage servers to decide which shares to return), the decryption key, the needed_shares/total_shares counts (since they affect peer selection), and the hash of the thingA block. This shortens the URI and lets us add more kinds of validation data without growing the URI (like plaintext merkle trees, to enable strong incremental plaintext validation), at the cost of maybe 150 bytes of alacrity. Each storage server holds an identical copy of the thingA block. This is an incompatible change: new messages have been added to the storage server interface, and the URI format has changed drastically. ] [fix BucketWriter to not create a finalhome until it is complete, and to clean up the empty $VERIFIERID dir under incoming/ when it moves the last share out of it zooko@zooko.com**20070331010156] [storageserver: ignore files in verifierdir whose filenames aren't of the right form for shares zooko@zooko.com**20070418144156] [fix storage server to handle the case that it has no directory at all when someone asks for buckets "Zooko O'Whielacronx "**20070331001207] [fix bug in storage-server: yes, "0" is a number, Mr. storage server "Zooko O'Whielacronx "**20070331000704] [factor out the tagged hash function used for subshares/blocks warner@allmydata.com**20070418032756] [encode.py: remove an unused import warner@allmydata.com**20070413030932] [log a running total of how much of your file has been uploaded zooko@zooko.com**20070331010137] [download: remove unused import warner@allmydata.com**20070418041120] [make test_encode less CPU-intense by using 4-out-of-10 encoding instead of 25-out-of-100 warner@allmydata.com**20070419175615] [download: validate handling of missing sharehashes too warner@allmydata.com**20070417001544] [interfaces: allow URIs to be up to 300 chars long, we're just crossing over the edge now warner@allmydata.com**20070427010829] [allmydata.interfaces: remove some of the placeholders now that we require foolscap-0.1.2 warner@lothar.com**20070404225936] [added a README explaining how to get started and what library dependencies are needed warner@allmydata.com**20070328174829] [raise the limit on the number of hashes when retrieving them, too "Zooko O'Whielacronx "**20070430130010] [change uri-packer-unpacker to deal with dictionaries, not fragile tuples warner@allmydata.com**20070523181849] [uri.py: share counts are not base32-encoded warner@allmydata.com**20070330193616] [use real encryption, generate/store/verify verifierid and fileid warner@allmydata.com**20070426005310] [download: remove some leftover (and not very useful) debug logging warner@allmydata.com**20070417001757] [test_encode.Encode: cover more combinations of data size relative to segment size and number of block hash tree leaves warner@allmydata.com**20070417192956] [test_encode.Roundtrip: cover more combinations of data size relative to segment size and number of block hash tree leaves warner@allmydata.com**20070417195755] [encode: make MAX_SEGMENT_SIZE controllable, to support tests which force the use of multiple segments. Also, remove not-very-useful upload-side debug messages warner@allmydata.com**20070417022957] [test_encode: test filesizes which are an exact multiple of the segment size. This test fails right now. warner@allmydata.com**20070417025503] [download: more test coverage warner@allmydata.com**20070417002137] [download: verify that bad blocks or hashes are caught by the download process warner@allmydata.com**20070416233021] [download: log more information when hashtree checks fail warner@allmydata.com**20070416200819] [encode: handle uploads of the same file multiple times. Unfortunately we have to do almost as much work the second time around, to compute the full URI warner@allmydata.com**20070419012910] [encode: clean up some weirdness that was there to make unit tests easier to write warner@lothar.com**20070406053618] [download: improve test coverage on our IDownloadTarget classes, make FileHandle return the filehandle when its done so that it is easier to close warner@allmydata.com**20070416200736] [system_test: exercise multiple segments warner@allmydata.com**20070417204047] [test_system: bump up timeouts for the sake of slow slave1, give each test a separate base directory warner@lothar.com**20070407033432] [test_system.py: remove the lowered (20s) timeouts, since some buildslaves require more like 30 or 40 seconds to complete the test warner@lothar.com**20070308233009] [webish: add 'my nodeid' to the page warner@allmydata.com**20070329213155] [tests: clean up tearDown to use flushEventualQueue instead of hacking fixed-time delays warner@allmydata.com**20070404230913] [display file size in directory.xhtml Faried Nawaz **20070504200732] [webish: more verifierid-to-uri transition warner@allmydata.com**20070117015553] [download.py: refactor bucket_failed() a bit, add some docs warner@allmydata.com**20070601013136] [hash trees: further cleanup, to make sure we're validating the right thing warner@allmydata.com**20070413024148 hashtree.py: improve the methods available for finding out which hash nodes are needed. Change set_hashes() to require that every hash provided can be validated up to the root. download.py: validate from the top down, including the URI-derived roothash in the share hash tree, and stashing the thus-validated share hash for use in the block hash tree. ] [encode: add more logging to investigate occasional test failures warner@allmydata.com**20070407010438] [encode/upload: add more logging, to understand the test failure on a slow buildslave warner@allmydata.com**20070406224545] [hashtree.py: reindent from 2-spaces to 4-spaces. No functional changes. warner@allmydata.com**20070412212411] [encode: start to fix a few problems, still a lot of work left to go warner@lothar.com**20061214103117] [download: always validate the blockhash, and don't let the bucket trick us into not validating hashes warner@allmydata.com**20070412221846] [verify hash chains on incoming blocks warner@allmydata.com**20070412200740 Implement enough of chunk.IncompleteHashTree to be usable. Rearrange download: all block/hash requests now go through a ValidatedBucket instance, which is responsible for retrieving and verifying hashes before providing validated data. Download was changed to use ValidatedBuckets everywhere instead of unwrapped RIBucketReader references. ] [chunk.py: remove unused non-tagged hash code warner@allmydata.com**20070330183247] [chunk.py: fix a pyflakes warning warner@allmydata.com**20070330024940] [finish making the new encoder/decoder/upload/download work warner@allmydata.com**20070330235050] [small tweaks to test_storage.py zooko@zooko.com**20061204073559] [encode_new: use tagged (sha256) hashes everywhere warner@allmydata.com**20070330183213] [upload: change _compute_uri a bit, get infile in a different way warner@allmydata.com**20070330193014] [add unit tests and fix bugs in upload zooko@zooko.com**20070330215433] [hush pyflakes warnings warner@allmydata.com**20070117033434] [switch upload to use encode_new, fix a few things (but not nearly all of them) warner@allmydata.com**20070330185303] [encode_new.py: add comments, make variable names more illuminating warner@allmydata.com**20070328180619] [ICodecEncoder: resolve some questions about the API, still more to examine warner@allmydata.com**20070328031445] [update the docs on ICodecEncoder and ICodecDecoder warner@allmydata.com**20070328020509] [add some questions to allmydata.interfaces warner@allmydata.com**20070306025738] [encode_new.py: rearrange methods into the order in which they should be called warner@allmydata.com**20070328203017] [chunk: add IncompleteHashTree for download purposes, plus tests warner@lothar.com**20070406160957] [change HashTree to use new hashutil convenience methods, thus fixing a security flaw zooko@zooko.com**20070330011235] [download: retrieve share hashes when downloading. We don't really do much validation with them yet, though. warner@lothar.com**20070407055119] [add new test for doing an encode/decode round trip, and make it almost work warner@allmydata.com**20070330202001] [finish storage server and write new download zooko@zooko.com**20070330175219] [fix counting of bytes written robk@allmydata.com**20061201023736] [fix another read_attr bug robk@allmydata.com**20061201024907] [fix bug in bucketstore read/write _attr robk@allmydata.com**20061201023657] [mv amdlib/util/* to allmydata/util/ zooko@zooko.com**20061204080325] [pulled in assertutil and with it humanreadable from amdlib.util robk@allmydata.com**20061130212225] [fix pyflakes warning in debugshell, by providing a dummy value for 'app' that will be overwritten when the manhole connection is established warner@allmydata.com**20070106031559] [hush pyflakes warnings warner@allmydata.com**20070116033110] [update the use of the encoder API in download.py "Zooko O'Whielacronx "**20070201223013] [test_encode: make sure encode_new can produce the data it is supposed to warner@allmydata.com**20070330183257] [new upload and storage server zooko@zooko.com**20070330031952] [make initial simple encode_new test pass warner@lothar.com**20061214101701] [change upload to push 2 shares instead of 3 warner@allmydata.com**20070116211526 Now that peers can talk to themselves, the 5-node system test won't fail just because one of the shares was hosted on the downloader (and thus inaccessible until recently). The 3-share push was there to avoid this problem. ] [upload: fix typo in debug messages warner@allmydata.com**20070117014228] [encode_new.py: recent Foolscap accepts 'None' as a constraint warner@allmydata.com**20070117015630] [figleaf doesn't like the last line of a file to be a comment warner@allmydata.com**20061214023512] [filetree: make delete() work warner@allmydata.com**20070124211053] [filetree: change the way addpath works, now we add workqueue steps for all involved subtrees at about the same time, rather than letting one step add the next when it runs. Finally add a (passing) test for uploading files to CHK-based directories warner@lothar.com**20070122070609] [filetree: add vdrive upload/download test, change workqueue relative-filename semantics warner@lothar.com**20070121220315] [workqueue: more tests, coverage now at 63.4%, yay warner@lothar.com**20070109073612] [workqueue: more tests warner@lothar.com**20070109075850] [filetree: test NoSuchDirectoryError in vdrive.list() warner@lothar.com**20070121105037] [complete the Introducer changes, separate out vdrive access, make everything work again warner@allmydata.com**20070327231211] [make sure the StorageServer goes underneath the client's basedir warner@lothar.com**20061203065228] [clients now get the queen's pburl from a file named roster_pburl, not hardcoded in the .tac file warner@lothar.com**20061204102740] [fix bin/allmydata argument-parsing warner@lothar.com**20061205183515] [webish: add queen pburl and connection status to welcome page warner@lothar.com**20061205185132] [queen.sendOnly: ignore DeadReferenceError too warner@lothar.com**20070121220109] [sendOnly: oops, I keep forgetting that you can't really use f.trap in a lambda, because it returns the failure it trapped warner@allmydata.com**20070110031305] [hush a pyflakes warning Brian Warner **20070323052200] [more work on a memory-footprint test program warner@lothar.com**20070312232837] [add new build/instdir directories to the boringfile warner@allmydata.com**20070131000345] [add some experimental emacs test-coverage-annotation tools, still in development warner@lothar.com**20070102054842] [start work on a memory-measuring test tool warner@lothar.com**20070309001224] [Makefile: count-lines: ignore build/* files, also add total number of .py files warner@allmydata.com**20070131001037] [Makefile: add target to count lines of code and TODOs warner@lothar.com**20070122071739] [merge incomplete stuff with other patches "Zooko O'Whielacronx "**20070323232026] [add a local foolscap control interface, to upload/download files and check memory usage warner@lothar.com**20070308011606] [move IWorkQueue into allmydata.interfaces, give VirtualDrive an uploader warner@lothar.com**20070121211531] [more filetree, workqueue-boxes now hold serialized Nodes, move NodeMaker out to a separate module warner@lothar.com**20070121101854] [workqueue: more improvements, more tests warner@allmydata.com**20070109042942] [workqueue: start adding tests warner@allmydata.com**20070109032933] [move upload/download interfaces to allmydata.interfaces, let SubTreeMaker assert IDownloader-ness of its 'downloader' argument warner@lothar.com**20070121210134] [download.py: fix IDownloader to take a URI warner@lothar.com**20070119081748] [filetree: start testing IVirtualDrive, beginning with list() warner@lothar.com**20070121031441] [filetree: put SubTreeMaker and NodeMaker in separate classes warner@lothar.com**20070120230456] [more filetree work, more tests now pass warner@lothar.com**20070120215021] [more filetree, it's actually starting to make sense now warner@lothar.com**20070120204151] [filetree: more tests, still very early warner@lothar.com**20070120111315] [filetree: refactor INode serialization, start on tests warner@lothar.com**20070120105253] [more filetree hacking, still too early to test warner@lothar.com**20070120102520] [checkpont more filetree stuff warner@allmydata.com**20070120052239] [snapshot filetree work: fix pyflakes complaints warner@lothar.com**20070119083536] [snapshot filetree work: it's getting close warner@lothar.com**20070119082303] [workqueue: fix pyflakes warnings, code is still quite incomplete warner@allmydata.com**20070106031119] [filetree.interfaces: remove not-really-code to improve coverage stats warner@allmydata.com**20070117230047] [rearrange service startup a bit, now Node.startService() returns a Deferred that fires when the tub is actually ready, and there is also a Node.when_tub_ready() hook. This allows get_local_addresses() to be slow and not break everything. Changed all necessary test cases to accomodate this slow startup. warner@lothar.com**20070308211036] [change node startup to put all local addresses in the PBURL, including 127.0.0.1. This should facilitate testing on both connected and disconnected systems. warner@lothar.com**20070308004317] [iputil: add get_local_addresses(), an attempt to enumerate all IPv4 addresses on this host. This is pretty unix-specific for right now (it calls ifconfig) warner@lothar.com**20070308002230] [incomplete work to be finished elsewhere "Zooko O'Whielacronx "**20070323231557] [client: closures confuse me, late binding bites me yet again warner@lothar.com**20061203065307] [add simple metadata (a single string) to the storage protocol warner@allmydata.com**20070115200122] [remove a couple of unused methods of buckets zooko@zooko.com**20061204081401] [now that foolscap-0.0.7 accepts connect-to-self, allow peers to know about themselves. We now require foolscap-0.0.7 warner@allmydata.com**20070116211249] [test_system.py: exercise queen.Roster._lost_node too warner@allmydata.com**20070110014036] [webish: add PBURL to the all-peers table on the welcome page warner@allmydata.com**20070117030118] [make logging multiple nodes at once easier to follow warner@lothar.com**20061203022750] [separate queen's service of introduction-to-the-network from queen's service of providing-access-to-the-vdrive "Zooko O'Whielacronx "**20070322213930] [fix test_codec and test_upload to handle current API zooko@zooko.com**20070328055715] [upload: rearrange peer-selection code to be more readable, and fix a silly bug warner@allmydata.com**20070117013512] [oops, fix that NotEnoughPeersError instrumentation warner@allmydata.com**20070117010723] [add better error reporting to NotEnoughPeersError warner@allmydata.com**20070117004752] [update ReplicatingEncoder and decoder to match the current ICodecEncoder/decoder interface warner@lothar.com**20070328051731] [switch to pyfec zooko@zooko.com**20070328070516] [update roadmap warner@lothar.com**20070119081724] [update roadmap warner@lothar.com**20061204015133] [more roadmap updates warner@lothar.com**20061204104333] [document IEncoder, add Encoder.set_landlords() warner@allmydata.com**20070328182453] [use pyfec instead of py_ecc for erasure coding and update API to codec "Zooko O'Whielacronx "**20070201220700] [disable figleaf tracing during py_ecc, since it takes *forever*, especially on the slow buildslave warner@allmydata.com**20070106001204] [figleaf: move a copy into allmydata.util.figleaf, update Makefile/trial stuff warner@lothar.com**20070104033829] [Makefile: allow 'make test TEST=something.else' warner@lothar.com**20061214101329] [Makefile: fix location of figleaf.excludes warner@lothar.com**20061214101643] [Makefile: add pyflakes target, only covering allmydata code (not pycrypto) warner@lothar.com**20061214100522] [add figleaf-output target warner@allmydata.com**20061207012402] [encode.py: add some timing comments warner@allmydata.com**20070105064842] [update URI format, include codec name warner@allmydata.com**20070117032959] [update interfaces and docs for codec "Zooko O'Whielacronx "**20070124213402 It now takes a sequence of buffers instead of a single string for both encode and decode, and it also takes a separate sequence of shareids for decode instead of a sequence of tuples, and it returns a sequence of buffers instead of a single string. ] [change IEncoder to ICodecEncoder, to match the previous change warner@allmydata.com**20070112025714] [nicer API -- you don't have to shuffle the shares into place before calling decode_all() "Zooko O'Whielacronx "**20070124212053 Instead it does that shuffling in C inside fecmodule. ] [pyfec v0.9 "Zooko O'Whielacronx "**20070122231731 Here is the change history from the first darcs era, in reverse chronological order: Mon Jan 22 16:12:56 MST 2007 "Zooko O'Whielacronx " * move everything into a subdirectory so that I can merge this darcs repo with the tahoe darcs repo ./fec -> ./pyfec/fec ./setup.py -> ./pyfec/setup.py A ./pyfec/ Mon Jan 22 16:10:17 MST 2007 "Zooko O'Whielacronx " * clean up and minimize fec.c * strip out unused code * hard-code GF_BITS to 8 * reindent and reformat curly bracket placement M ./fec/fec.c -655 +324 M ./fec/fec.h -25 Mon Jan 22 14:24:32 MST 2007 "Zooko O'Whielacronx " * change API to allow a subset of the shares to be produced, and to just pass back pointers to primary shares instead of copying them M ./fec/fec.c -24 +40 M ./fec/fec.h -5 +17 M ./fec/fecmodule.c -63 +144 M ./fec/test/test_pyfec.py -16 +25 M ./setup.py -2 +27 Tue Jan 16 23:01:44 MST 2007 "Zooko O'Whielacronx " * split encoder from decoder M ./fec/fecmodule.c -48 +161 M ./fec/test/test_pyfec.py -3 +4 Tue Jan 16 14:35:25 MST 2007 "Zooko O'Whielacronx " * it compiles now! ./fec.c -> ./pyfec/fec.c ./fec.h -> ./pyfec/fec.h ./fecmodule.c -> ./pyfec/fecmodule.c ./pyfec -> ./fec M ./fec/fec.c -109 +85 r13 M ./fec/fec.h -3 +2 r13 M ./fec/fecmodule.c -23 +241 r13 A ./fec/test/ A ./fec/test/test_pyfec.py A ./pyfec/ A ./setup.py Tue Jan 9 10:47:58 MST 2007 zooko@zooko.com * start of new fecmodule.c A ./fecmodule.c Mon Jan 1 15:00:04 MST 2007 zooko@zooko.com * tidy up error handling M ./fec.c -26 +16 Mon Jan 1 14:06:30 MST 2007 zooko@zooko.com * remove the on-the-fly encoding option We don't currently need it. M ./fec.c -68 M ./fec.h -22 Mon Jan 1 13:53:28 MST 2007 zooko@zooko.com * original import from Mnet project A ./fec.c A ./fec.h ] [download: update all users to match Zooko's change to ICodecDecoder.decode (as it now returns a list instead of a single string) warner@allmydata.com**20070124232322] [change build system to use subpackages pristinely and ask them to install themselves into an "instdir" "Zooko O'Whielacronx "**20070130205759] [add __init__ and setup.py glue for py_ecc, so we can import it warner@lothar.com**20070102054324] [Makefile: use absolute paths when setting PYTHONPATH warner@allmydata.com**20070117030200] [Makefile: add correct generated build/lib.linux-i686-2.4 directory to PYTHONPATH for tests warner@lothar.com**20061214095951] [tests: add support for figleaf code-coverage gathering warner@allmydata.com**20061206212612] [add preliminary debian packaging warner@lothar.com**20061205080044] [split filetree_new.py up into smaller pieces, in a new subpackage warner@allmydata.com**20070117195438] [filetable_new: fix the test warner@lothar.com**20061224184703] [checkpoint work-in-progress for WorkQueue, a disk-persistent list of work to be done warner@lothar.com**20070102054716] [rename the new filetable code to 'filetree', since robk astutely pointed out that 'table' is misleading and implies a flat list of files in a single directory warner@allmydata.com**20070108222914] [add mathutil.next_power_of_k() and mathutil.ave() "Zooko O'Whielacronx "**20070201215526] [filetable: switch to new approach with anonymous nodes warner@allmydata.com**20070615002456] [improve test coverage a bit warner@allmydata.com**20070117213429] [add RIMutableDirectoryNode.get, to get a specific child file or directory warner@lothar.com**20061204100329] [more filetable_new tests warner@lothar.com**20061225065618] [rearrange encode/upload, add URIs, switch to ReplicatingEncoder warner@allmydata.com**20070116032222 Added metadata to the bucket store, which is used to hold the share number (but the bucket doesn't know that, it just gets a string). Modified the codec interfaces a bit. Try to pass around URIs to/from download/upload instead of verifierids. URI format is still in flux. Change the current (primitive) file encoder to use a ReplicatingEncoder because it provides ICodecEncoder. We will be moving to the (less primitive) file encoder (currently in allmydata.encode_new) eventually, but for now this change lets us test out PyRS or zooko's upcoming C-based RS codec in something larger than a single unit test. This primitive file encoder only uses a single segment, and has no merkle trees. Also added allmydata.util.deferredutil for a DeferredList wrapper that errbacks (but only when all component Deferreds have fired) if there were any errors, which unfortunately is not a behavior available from the standard DeferredList. ] [more upload unit tests warner@lothar.com**20061204064621] [increase the maximum size of ShareData, since currently it is also a limit on uploaded file size warner@lothar.com**20061204111431] [only run a single (short) py_ecc test on slave3, since it is so slow the tests timeout warner@allmydata.com**20070105064252] [test_encode_share.py: fix some pyflakes warnings warner@allmydata.com**20070106005322] [add some (disabled) encoder benchmarking code warner@allmydata.com**20070106004603] [use the word 'codec' for erasure coding, for now. 'encode' is used for file-level segmentation/hashing warner@allmydata.com**20070112025127] [encoding: fix the last py_ecc problem, tests pass now warner@allmydata.com**20070105060642] [fix our use of py_ecc (set log2FieldSize=8 explicitly) warner@allmydata.com**20070105055021] [establish IEncoder/IDecoder, create suitable interfaces for both the simple replicating encoder and the py_ecc one, add a (failing) unit test for it warner@allmydata.com**20070105035251] [figleaf gets confused when the last line of a file is a comment warner@allmydata.com**20061207205823] [webish: improve download, now you can just append the vdrive path to the base URL to get at the contents of the file. Also added a download-by-URI box warner@allmydata.com**20061207204837] [webish: add mesh stats, peer list. improve distribution of client services within rend_* methods warner@lothar.com**20061205015435] [add mkdir to webish interface, switch to new bind_* approach warner@lothar.com**20061205004924] [allow webfront to use a strports port specification warner@allmydata.com**20061207184740] [add a basic read-only webfront test warner@allmydata.com**20061207184806] [change Encoder to use py_ecc, now that it works warner@allmydata.com**20070106001245] [more pyflakes cleanups warner@allmydata.com**20070105000620] [checkpointing new filetable work.. tests don't pass yet warner@lothar.com**20061224183924] [filetable: shuffle lines a bit to appease figleaf's confusion warner@lothar.com**20061214103357] [record some WIP structure for filetable warner@allmydata.com**20061214023700] [change encode_new to use IEncoder warner@allmydata.com**20070105055135] [filetable_new: import py_ecc, make the tests get a little bit further warner@lothar.com**20070102054412] [add padding to segments and convert type from string to list of integers before passing to py_ecc "Zooko O'Whielacronx "**20070104235814 But then I get an error that I don't understand about ECC math... ] [make encode_new use py_ecc for real live erasure coding zooko@zooko.com**20061229195053 (This patch is not tested -- I'm working on a Mac which doesn't have gcc installed... (gcc is necessary for the crypto module.) I will now attempt to connect to a better development computer over my mom's staticy, failure-prone, 14.4 K modem...) ] [encode_new: fix a bug in subshare hashes and tidy-up a couple of things zooko@zooko.com**20061229184010] [start work on new encoder, with merkle trees and subshares and stuff warner@allmydata.com**20061214023235] [import py_ecc, a pure python fec by Emin Martinian, which is under a permissive licence zooko@zooko.com**20061229194645 It is too slow for a real product, but is a quick way to get a working prototype, and also is freely redistributable by us... ] [move all packages into src/, fix allmydata.Crypto build. Now you must perform a 'setup.py build' before using anything, and you must add the build directory (build/lib.linux-i686-2.4) to your PYTHONPATH before doing anything warner@lothar.com**20061214093950] [first cut at creating allmydata.Crypto, starting with python-amycrypto-2.0.1.allmydata2 (with working CTR mode) warner@lothar.com**20061214092530] [add bin/allmydata to create/stop/start nodes warner@lothar.com**20061205182523] [setup.py: oops, add sub-packages, needed since I went back to distutils Brian Warner **20061206010622] [add distutils-based packaging warner@lothar.com**20061205072926] [ignore .buildbot options directory warner@lothar.com**20061204031049] [import mathutil from pyutil zooko@zooko.com**20061229195042] [webish: add option to export/import shared directories (always read-write) warner@lothar.com**20070615093123] [webish: implement delete (for files only, not directories) warner@lothar.com**20061205012738] [add 'make directory' button to webish warner@lothar.com**20061204180329] [implemented upload side of webish warner@lothar.com**20061204111536] [add a (read-only) web frontend. Put a 'webport' file in your base directory to activate it. warner@lothar.com**20061204100609] [add download code to vdrive, add system-level test for vdrive functionality, refactor DownloadTargets warner@lothar.com**20061204044219] [unit tests for vdrive warner@lothar.com**20061204031126] [initial file-table support, VirtualDrive service, rearrange Storage somewhat warner@lothar.com**20061204010741] [added run-client2 target to makefile robk@allmydata.com**20061201015119] [added create_dirs makefile target to create initial dirs robk@allmydata.com**20061130231606] [create a Makefile to drive things warner@lothar.com**20061130214005] [fix pyflakes warnings/errors warner@lothar.com**20061202222846] [added 'debugshell' module a convenient dumping ground for tools for manhole environment robk@allmydata.com**20061201015308] [misc upload fixes and improvements warner@lothar.com**20061203023143] [add more logging warner@lothar.com**20061203065353] [avoid race conditions and multiple-notify in the Roster warner@lothar.com**20061203022909] [deal with the face that peerids are always binary in tests warner@lothar.com**20061203023102] [implement/test download, modify Storage to match warner@lothar.com**20061203090143] [added read and cross check to storage unit test, minor cleanups robk@allmydata.com**20061201082111] [split 'Bucket' into separate subclasses for read and write robk@allmydata.com**20061201090454] [allow buckets written by previous runs to be read robk@allmydata.com**20061201084555] [prevent reading unfinished buckets robk@allmydata.com**20061201084827] [rearrange names, add more RemoteInterfaces to make tests pass again warner@lothar.com**20061202232557] [prototype encoder zooko@zooko.com**20061202233126] [make the system test work warner@lothar.com**20061203065627] [improving system test, still broken, possibly a Foolscap problem warner@lothar.com**20061203023208] [rearrange node startup again, allowing Tub.registerReference to happen in startService rather than in some later turn. Also add a 'local_ip' file with which you can override the published IP address of the node warner@lothar.com**20061203013731] [rearrange client setup, factor out common Node functionality, add Uploader service to client warner@lothar.com**20061203002718] [point client.tac at yumyum's queen warner@lothar.com**20061130231631] [remove that queen_host stuff warner@lothar.com**20061130231340] [change manhole setup for queen too warner@lothar.com**20061201013838] [add Manhole to queen (port 8021), also change manhole namespace to just have 'app' (for both queen and client) warner@lothar.com**20061130233045] [fix up StorageServer so that unit tests will run again warner@lothar.com**20061201021829] [add Client.permute_peers warner@lothar.com**20061201021851] [change bucket writer protocol, give Encoder a RemoteReference instead of a file-like object warner@lothar.com**20061202011726] [upload: add WriterProxy warner@lothar.com**20061201100611] [standardize on keeping nodeids (and all other SHA1-derived values as binary everywhere, only doing idlib.b2a() when interacting with a human or the filesystem warner@lothar.com**20061202222626] [fix losing-client-connection handler warner@lothar.com**20061201001346] [set Client.nodeid to our (ascii, base32) tubid warner@lothar.com**20061201012017] [implement upload peer selection warner@lothar.com**20061201085428] [add Client.get_remote_service utility method warner@lothar.com**20061201001736] [improve RemoteInterface specifications warner@lothar.com**20061202220309] [added storage test robk@allmydata.com**20061201043842] [add RemoteInterfaces (foolscap schemas). some tests break. warner@lothar.com**20061202011750] [add roadmap warner@lothar.com**20061130231619] [rerecord all the storageserver patches in one go robk@allmydata.com**20061201021423 darcs was dying trying to deal with the conflict resolution patches. this adds a (very rough) bucketstore and storageserver. probably needs lots of work both in api and implementation. ] [implement more Roster stuff: add_peer, lost_peer. Changed Client service registration scheme. warner@lothar.com**20061201000957] [start developing Roster, track all active peers warner@lothar.com**20061130234315] [help the queen have a persistent PBURL, have the client connect to it warner@lothar.com**20061130223938] [change manhole setup for client: create authorized_keys.8022 (or other portnum) warner@lothar.com**20061201013425] [add Manhole functionality to the client: port 8022, add an authorized_keys file to the client's basedir to enable it warner@lothar.com**20061130232641] [start on client what-is-my-ipaddress functionality warner@lothar.com**20061130222339] [create a stub Storage service, start work on publishing it warner@lothar.com**20061130212952] [add test infrastructure, use 'make test' to run it, please run before pushing warner@lothar.com**20061130215526] [have client running, no queen to connect to yet warner@lothar.com**20061130212706] [add darcs boringfile warner@lothar.com**20061130213733] [add beginning queen code warner@lothar.com**20061130213924] [beginnings of a system test, with 5 nodes and a queen warner@lothar.com**20061203003018] [webapi.txt: note that the 'curl' utility can be used to exercise most of this interface warner@lothar.com**20070710173637] [add webapi.txt: explain our plans for the node's webserver warner@allmydata.com**20070705203603] [NEWS: update to summarize all changes since the last update warner@lothar.com**20081020164047] [NEWS: more edits, almost done warner@lothar.com**20080919010036] [NEWS: finish editing for the upcoming 1.3.0 release warner@lothar.com**20080919193053] [NEWS: describe all changes since the last release. Still needs editing. warner@lothar.com**20080919002755] [NEWS: add user-visible changes since the previous release warner@lothar.com**20080721232930] [docs: NEWS: a couple of small edits zooko@zooko.com**20080611195140] [NEWS: description of user-visible changes in the new release warner@allmydata.com**20080611193935] [docs: add a copy of the NEWS file into docs/ since I sent out a release announcement which links to it there zooko@zooko.com**20080612024150] [docs: add statement on our refusal to insert backdoors zooko@zooko.com**20101006051147 Ignore-this: 644d308319a7b80c4434bdff9760404a ] [add the 'Denver Airport' design doc, for Chord-based peer selection warner@lothar.com**20061202010914] [actually add tests warner@lothar.com**20061130222301] [start client framework warner@lothar.com**20061130211447] Patch bundle hash: 5953c6bfe7ed2f5971a54a57bbba2e17660a14ab