4 patches for repository http://tahoe-lafs.org/source/tahoe/trunk: Fri Oct 7 03:30:01 BST 2011 david-sarah@jacaranda.org * misc/simulators/hashbasedsig.py: simplify by removing unnecessary local function that captured a variable declared in a for loop (this was not a bug, but the code was unclear). Also fix a pyflakes warning about an import. refs #1556 Fri Oct 7 03:34:43 BST 2011 david-sarah@jacaranda.org * Fix some potential bugs in test code exposed by check-miscaptures.py. refs #1556 Fri Oct 7 04:24:44 BST 2011 david-sarah@jacaranda.org * Fix some potential bugs (in non-test code) exposed by check-miscaptures.py. refs #1556 Fri Oct 7 04:38:47 BST 2011 david-sarah@jacaranda.org * Fix some more potential bugs in test code exposed by check-miscaptures.py. refs #1556 New patches: [misc/simulators/hashbasedsig.py: simplify by removing unnecessary local function that captured a variable declared in a for loop (this was not a bug, but the code was unclear). Also fix a pyflakes warning about an import. refs #1556 david-sarah@jacaranda.org**20111007023001 Ignore-this: 446c94efae02ded5e85eb3335ca5e69 ] { hunk ./misc/simulators/hashbasedsig.py 22 Mcycles_per_block = cycles_per_byte * L_block / (8 * 1000000.0) -from math import floor, ceil, log, log1p, pow, e, sqrt +from math import floor, ceil, log, log1p, pow, e from sys import stderr from gc import collect hunk ./misc/simulators/hashbasedsig.py 141 for x in xrange(1, j): lg_px[x] = lg_px[x-1] - lg(x) + lg_px_step - def find_min_q(): - for q in xrange(1, q_max+1): - lg_q = lg(q) - lg_pforge = [lg_px[x] + (lg_q*x - lg_K2)*q for x in xrange(1, j)] - if max(lg_pforge) < -L_hash + lg(j) and lg_px[j-1] + 1.0 < -L_hash: - #print "K = %d, K1 = %d, K2 = %d, L_hash = %d, lg_K2 = %.3f, q = %d, lg_pforge_1 = %.3f, lg_pforge_2 = %.3f, lg_pforge_3 = %.3f" \ - # % (K, K1, K2, L_hash, lg_K2, q, lg_pforge_1, lg_pforge_2, lg_pforge_3) - return q - return None + q = None + # Find the minimum acceptable value of q. + for q_cand in xrange(1, q_max+1): + lg_q = lg(q_cand) + lg_pforge = [lg_px[x] + (lg_q*x - lg_K2)*q_cand for x in xrange(1, j)] + if max(lg_pforge) < -L_hash + lg(j) and lg_px[j-1] + 1.0 < -L_hash: + #print "K = %d, K1 = %d, K2 = %d, L_hash = %d, lg_K2 = %.3f, q = %d, lg_pforge_1 = %.3f, lg_pforge_2 = %.3f, lg_pforge_3 = %.3f" \ + # % (K, K1, K2, L_hash, lg_K2, q, lg_pforge_1, lg_pforge_2, lg_pforge_3) + q = q_cand + break hunk ./misc/simulators/hashbasedsig.py 152 - q = find_min_q() if q is None or q == last_q: # if q hasn't decreased, this will be strictly worse than the previous candidate continue } [Fix some potential bugs in test code exposed by check-miscaptures.py. refs #1556 david-sarah@jacaranda.org**20111007023443 Ignore-this: e48b2c2d200521d6f28c737994ce3a2a ] { hunk ./src/allmydata/test/test_cli.py 760 self.failUnlessIn("cannot contain", stderr) for invalid in ['foo:bar', 'foo bar', 'foobar::']: - d.addCallback(lambda res: self.do_cli("create-alias", invalid)) + d.addCallback(lambda res, invalid=invalid: self.do_cli("create-alias", invalid)) d.addCallback(_check_invalid) hunk ./src/allmydata/test/test_cli.py 762 - d.addCallback(lambda res: self.do_cli("add-alias", invalid, self.two_uri)) + d.addCallback(lambda res, invalid=invalid: self.do_cli("add-alias", invalid, self.two_uri)) d.addCallback(_check_invalid) def _test_urls(junk): hunk ./src/allmydata/test/test_dirnode.py 1713 n.raise_error() for (i, n) in unknown_rw: - self.failUnlessRaises(MustNotBeUnknownRWError, lambda: n.raise_error()) + self.failUnlessRaises(MustNotBeUnknownRWError, lambda n=n: n.raise_error()) for (i, n) in must_be_ro: hunk ./src/allmydata/test/test_dirnode.py 1716 - self.failUnlessRaises(MustBeReadonlyError, lambda: n.raise_error()) + self.failUnlessRaises(MustBeReadonlyError, lambda n=n: n.raise_error()) for (i, n) in must_be_imm: hunk ./src/allmydata/test/test_dirnode.py 1719 - self.failUnlessRaises(MustBeDeepImmutableError, lambda: n.raise_error()) + self.failUnlessRaises(MustBeDeepImmutableError, lambda n=n: n.raise_error()) for (i, n) in bad_uri: hunk ./src/allmydata/test/test_dirnode.py 1722 - self.failUnlessRaises(uri.BadURIError, lambda: n.raise_error()) + self.failUnlessRaises(uri.BadURIError, lambda n=n: n.raise_error()) for (i, n) in ok: self.failIf(n.get_readonly_uri() is None, i) hunk ./src/allmydata/test/test_download.py 1118 for i,which,substring in corrupt_me: # All these tests result in a failed download. d.addCallback(self._corrupt_flip_all, imm_uri, i) - d.addCallback(lambda ign: + d.addCallback(lambda ign, which=which, substring=substring: self.shouldFail(NoSharesError, which, substring, _download, imm_uri)) } [Fix some potential bugs (in non-test code) exposed by check-miscaptures.py. refs #1556 david-sarah@jacaranda.org**20111007032444 Ignore-this: bac9ed65b21c2136c4db2482b3c093f7 ] { hunk ./src/allmydata/mutable/filenode.py 1139 start_segments = {} # shnum -> start segment end_segments = {} # shnum -> end segment blockhashes = {} # shnum -> blockhash tree - for (shnum, data) in update_data.iteritems(): - data = [d[1] for d in data if d[0] == self._version] + for (shnum, original_data) in update_data.iteritems(): + data = [d[1] for d in original_data if d[0] == self._version] # Every data entry in our list should now be share shnum for # a particular version of the mutable file, so all of the hunk ./src/allmydata/mutable/filenode.py 1146 # entries should be identical. datum = data[0] - assert filter(lambda x: x != datum, data) == [] + assert [x for x in data if x != datum] == [] blockhashes[shnum] = datum[0] start_segments[shnum] = datum[1] hunk ./src/allmydata/mutable/servermap.py 683 d.addCallback(lambda results, shnum=shnum, peerid=peerid: self._try_to_set_pubkey(results, peerid, shnum, lp)) # XXX: Make self._pubkey_query_failed? - d.addErrback(lambda error, shnum=shnum, peerid=peerid: + d.addErrback(lambda error, shnum=shnum, peerid=peerid, data=data: self._got_corrupt_share(error, shnum, peerid, data, lp)) else: # we already have the public key. hunk ./src/allmydata/mutable/servermap.py 698 # bytes of the share on the storage server, so we # shouldn't need to fetch anything at this step. d2 = reader.get_verinfo() - d2.addErrback(lambda error, shnum=shnum, peerid=peerid: + d2.addErrback(lambda error, shnum=shnum, peerid=peerid, data=data: self._got_corrupt_share(error, shnum, peerid, data, lp)) # - Next, we need the signature. For an SDMF share, it is # likely that we fetched this when doing our initial fetch hunk ./src/allmydata/mutable/servermap.py 706 # the end of the share, so unless the file is quite small, # we'll need to do a remote fetch to get it. d3 = reader.get_signature() - d3.addErrback(lambda error, shnum=shnum, peerid=peerid: + d3.addErrback(lambda error, shnum=shnum, peerid=peerid, data=data: self._got_corrupt_share(error, shnum, peerid, data, lp)) # Once we have all three of these responses, we can move on # to validating the signature hunk ./src/allmydata/mutable/servermap.py 717 d4 = reader.get_encprivkey() d4.addCallback(lambda results, shnum=shnum, peerid=peerid: self._try_to_validate_privkey(results, peerid, shnum, lp)) - d4.addErrback(lambda error, shnum=shnum, peerid=peerid: + d4.addErrback(lambda error, shnum=shnum, peerid=peerid, data=data: self._privkey_query_failed(error, shnum, data, lp)) else: d4 = defer.succeed(None) } [Fix some more potential bugs in test code exposed by check-miscaptures.py. refs #1556 david-sarah@jacaranda.org**20111007033847 Ignore-this: aec8a543e9b5c3563b60692c647439a8 ] { hunk ./src/allmydata/test/test_hung_server.py 165 def test_10_good_sanity_check(self): d = defer.succeed(None) for mutable in [False, True]: - d.addCallback(lambda ign: self._set_up(mutable, "test_10_good_sanity_check")) + d.addCallback(lambda ign, mutable=mutable: self._set_up(mutable, "test_10_good_sanity_check")) d.addCallback(lambda ign: self._download_and_check()) return d hunk ./src/allmydata/test/test_hung_server.py 172 def test_10_good_copied_share(self): d = defer.succeed(None) for mutable in [False, True]: - d.addCallback(lambda ign: self._set_up(mutable, "test_10_good_copied_share")) + d.addCallback(lambda ign, mutable=mutable: self._set_up(mutable, "test_10_good_copied_share")) d.addCallback(lambda ign: self._copy_all_shares_from(self.servers[2:3], self.servers[0])) d.addCallback(lambda ign: self._download_and_check()) return d hunk ./src/allmydata/test/test_hung_server.py 180 def test_3_good_7_noshares(self): d = defer.succeed(None) for mutable in [False, True]: - d.addCallback(lambda ign: self._set_up(mutable, "test_3_good_7_noshares")) + d.addCallback(lambda ign, mutable=mutable: self._set_up(mutable, "test_3_good_7_noshares")) d.addCallback(lambda ign: self._delete_all_shares_from(self.servers[3:])) d.addCallback(lambda ign: self._download_and_check()) return d hunk ./src/allmydata/test/test_hung_server.py 188 def test_2_good_8_broken_fail(self): d = defer.succeed(None) for mutable in [False, True]: - d.addCallback(lambda ign: self._set_up(mutable, "test_2_good_8_broken_fail")) + d.addCallback(lambda ign, mutable=mutable: self._set_up(mutable, "test_2_good_8_broken_fail")) d.addCallback(lambda ign: self._break(self.servers[2:])) d.addCallback(lambda ign: self._should_fail_download()) return d hunk ./src/allmydata/test/test_hung_server.py 196 def test_2_good_8_noshares_fail(self): d = defer.succeed(None) for mutable in [False, True]: - d.addCallback(lambda ign: self._set_up(mutable, "test_2_good_8_noshares_fail")) + d.addCallback(lambda ign, mutable=mutable: self._set_up(mutable, "test_2_good_8_noshares_fail")) d.addCallback(lambda ign: self._delete_all_shares_from(self.servers[2:])) d.addCallback(lambda ign: self._should_fail_download()) return d hunk ./src/allmydata/test/test_hung_server.py 204 def test_2_good_8_broken_copied_share(self): d = defer.succeed(None) for mutable in [False, True]: - d.addCallback(lambda ign: self._set_up(mutable, "test_2_good_8_broken_copied_share")) + d.addCallback(lambda ign, mutable=mutable: self._set_up(mutable, "test_2_good_8_broken_copied_share")) d.addCallback(lambda ign: self._copy_all_shares_from(self.servers[2:3], self.servers[0])) d.addCallback(lambda ign: self._break(self.servers[2:])) d.addCallback(lambda ign: self._download_and_check()) hunk ./src/allmydata/test/test_hung_server.py 213 def test_2_good_8_broken_duplicate_share_fail(self): d = defer.succeed(None) for mutable in [False, True]: - d.addCallback(lambda ign: self._set_up(mutable, "test_2_good_8_broken_duplicate_share_fail")) + d.addCallback(lambda ign, mutable=mutable: self._set_up(mutable, "test_2_good_8_broken_duplicate_share_fail")) d.addCallback(lambda ign: self._copy_all_shares_from(self.servers[1:2], self.servers[0])) d.addCallback(lambda ign: self._break(self.servers[2:])) d.addCallback(lambda ign: self._should_fail_download()) hunk ./src/allmydata/test/test_mutable.py 193 reader = MDMFSlotReadProxy(None, None, shnum, data) # We need to get the offsets for the next part. d = reader.get_verinfo() - def _do_corruption(verinfo, data, shnum): + def _do_corruption(verinfo, data, shnum, shares): (seqnum, root_hash, IV, hunk ./src/allmydata/test/test_mutable.py 218 else: f = flip_bit shares[shnum] = f(data, real_offset) - d.addCallback(_do_corruption, data, shnum) + d.addCallback(_do_corruption, data, shnum, shares) ds.append(d) dl = defer.DeferredList(ds) dl.addCallback(lambda ignored: res) hunk ./src/allmydata/test/test_mutable.py 286 self.nodemaker.default_encoding_parameters['n'] = 1 d = defer.succeed(None) for v in (SDMF_VERSION, MDMF_VERSION): - d.addCallback(lambda ignored: + d.addCallback(lambda ignored, v=v: self.nodemaker.create_mutable_file(version=v)) def _created(n): self.failUnless(isinstance(n, MutableFileNode)) } Context: [no_network.py: Clean up whitespace around code changed by previous patch. david-sarah@jacaranda.org**20111004010407 Ignore-this: 647ec8a9346dca1a41212ab250619b72 ] [no_network.py: Fix potential bugs in some tests due to capture of slots in for loops. david-sarah@jacaranda.org**20111004010231 Ignore-this: 9c496877613a3befd54979e5de6e63d2 ] [docs: fix the rst formatting of COPYING.TGPPL.rst zooko@zooko.com**20111003043333 Ignore-this: c5fbc83f4a3db81a0c95b27053c463c5 Now it renders correctly both on trac and with rst2html --verbose from docutils v0.8.1. ] [MDMF: remove extension fields from caps, tolerate arbitrary ones. Fixes #1526 Brian Warner **20111001233553 Ignore-this: 335e1690aef1146a2c0b8d8c18c1cb21 The filecaps used to be produced with hints for 'k' and segsize, but they weren't actually used, and doing so had the potential to limit how we change those filecaps in the future. Also the parsing code had some problems dealing with other numbers of extensions. Removing the existing fields and making the parser tolerate (and ignore) extra ones makes MDMF more future-proof. ] [test/test_runner.py: BinTahoe.test_path has rare nondeterministic failures; this patch probably fixes a problem where the actual cause of failure is masked by a string conversion error. david-sarah@jacaranda.org**20110927225336 Ignore-this: 6f1ad68004194cc9cea55ace3745e4af ] [docs/configuration.rst: add section about the types of node, and clarify when setting web.port enables web-API service. fixes #1444 zooko@zooko.com**20110926203801 Ignore-this: ab94d470c68e720101a7ff3c207a719e ] [TAG allmydata-tahoe-1.9.0a2 warner@lothar.com**20110925234811 Ignore-this: e9649c58f9c9017a7d55008938dba64f ] Patch bundle hash: 058e6f84c9bbf9869cd770ce646f57166c7a9ce9