Tue Feb 23 23:45:49 MST 2010 zooko@zooko.com * directories: add immutable directories to test_deepcheck.py New patches: [directories: add immutable directories to test_deepcheck.py zooko@zooko.com**20100224064549 Ignore-this: 679f2b3cbbf02d59a44a7727c6eeb16d ] { hunk ./src/allmydata/nodemaker.py 48 return DirectoryNode(filenode, self, self.uploader) def create_from_cap(self, writecap, readcap=None, deep_immutable=False, name=u""): + """ The 'name' argument is solely for debugging -- it is not + necessarily the name of this node in any Tahoe-LAFS namespace. """ # this returns synchronously. It starts with a "cap string". assert isinstance(writecap, (str, type(None))), type(writecap) assert isinstance(readcap, (str, type(None))), type(readcap) hunk ./src/allmydata/test/test_deepcheck.py 149 if not unit: # stream should end with a newline, so split returns "" continue - yield simplejson.loads(unit) + try: + yield simplejson.loads(unit) + except ValueError, le: + le.args = tuple(le.args + (unit,)) + raise def web(self, n, method="GET", **kwargs): # returns (data, url) hunk ./src/allmydata/test/test_deepcheck.py 207 class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): # construct a small directory tree (with one dir, one immutable file, one - # mutable file, one LIT file, and a loop), and then check/examine it in - # various ways. + # mutable file, one LIT file, one DIR2:LIT empty dir, one DIR2:LIT tiny + # dir, and a loop), and then check/examine it in various ways. def set_up_tree(self): # 2.9s hunk ./src/allmydata/test/test_deepcheck.py 213 - # root - # mutable - # large - # small - # small2 - # loop -> root c0 = self.g.clients[0] d = c0.create_dirnode() def _created_root(n): hunk ./src/allmydata/test/test_deepcheck.py 247 self.small2_uri = n.get_uri() d.addCallback(_created_small2) + empty_litdir_uri = "URI:DIR2-LIT:" + tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT + + d.addCallback(lambda ign: self.root._create_and_validate_node(None, empty_litdir_uri, name=u"test_deepcheck empty_lit_dir")) + def _created_empty_lit_dir(n): + self.empty_lit_dir = n + self.empty_lit_dir_uri = n.get_uri() + self.root.set_node(u"empty_lit_dir", n) + d.addCallback(_created_empty_lit_dir) + + d.addCallback(lambda ign: self.root._create_and_validate_node(None, tiny_litdir_uri, name=u"test_deepcheck tiny_lit_dir")) + def _created_tiny_lit_dir(n): + self.tiny_lit_dir = n + self.tiny_lit_dir_uri = n.get_uri() + self.root.set_node(u"tiny_lit_dir", n) + d.addCallback(_created_tiny_lit_dir) + d.addCallback(lambda ign: self.root.set_node(u"loop", self.root)) return d hunk ./src/allmydata/test/test_deepcheck.py 348 return d def check_stats_good(self, s): - self.failUnlessEqual(s["count-directories"], 1) - self.failUnlessEqual(s["count-files"], 4) + self.failUnlessEqual(s["count-directories"], 3) + self.failUnlessEqual(s["count-files"], 5) self.failUnlessEqual(s["count-immutable-files"], 1) hunk ./src/allmydata/test/test_deepcheck.py 351 - self.failUnlessEqual(s["count-literal-files"], 2) + self.failUnlessEqual(s["count-literal-files"], 3) self.failUnlessEqual(s["count-mutable-files"], 1) # don't check directories: their size will vary # s["largest-directory"] hunk ./src/allmydata/test/test_deepcheck.py 356 # s["size-directories"] - self.failUnlessEqual(s["largest-directory-children"], 5) + self.failUnlessEqual(s["largest-directory-children"], 7) self.failUnlessEqual(s["largest-immutable-file"], 13000) # to re-use this function for both the local # dirnode.start_deep_stats() and the webapi t=start-deep-stats, we hunk ./src/allmydata/test/test_deepcheck.py 364 # returns a list of tuples, but JSON only knows about lists., so # t=start-deep-stats returns a list of lists. histogram = [tuple(stuff) for stuff in s["size-files-histogram"]] - self.failUnlessEqual(histogram, [(11, 31, 2), + self.failUnlessEqual(histogram, [(4, 10, 1), (11, 31, 2), (10001, 31622, 1), ]) self.failUnlessEqual(s["size-immutable-files"], 13000) hunk ./src/allmydata/test/test_deepcheck.py 368 - self.failUnlessEqual(s["size-literal-files"], 48) + self.failUnlessEqual(s["size-literal-files"], 56) def do_web_stream_manifest(self, ignored): d = self.web(self.root, method="POST", t="stream-manifest") hunk ./src/allmydata/test/test_deepcheck.py 381 files = [u for u in units if u["type"] in ("file", "directory")] assert units[-1]["type"] == "stats" stats = units[-1]["stats"] - self.failUnlessEqual(len(files), 5) - # [root,mutable,large] are distributed, [small,small2] are not + self.failUnlessEqual(len(files), 8) + # [root,mutable,large] are distributed, [small,small2,empty_litdir,tiny_litdir] are not self.failUnlessEqual(len([f for f in files if f["verifycap"] is not None]), 3) self.failUnlessEqual(len([f for f in files hunk ./src/allmydata/test/test_deepcheck.py 386 - if f["verifycap"] is None]), 2) + if f["verifycap"] is None]), 5) self.failUnlessEqual(len([f for f in files if f["repaircap"] is not None]), 3) self.failUnlessEqual(len([f for f in files hunk ./src/allmydata/test/test_deepcheck.py 390 - if f["repaircap"] is None]), 2) + if f["repaircap"] is None]), 5) self.failUnlessEqual(len([f for f in files if f["storage-index"] is not None]), 3) self.failUnlessEqual(len([f for f in files hunk ./src/allmydata/test/test_deepcheck.py 394 - if f["storage-index"] is None]), 2) + if f["storage-index"] is None]), 5) # make sure that a mutable file has filecap==repaircap!=verifycap mutable = [f for f in files if f["cap"] is not None hunk ./src/allmydata/test/test_deepcheck.py 437 d.addCallback(self.failUnlessEqual, None, "small") d.addCallback(lambda ign: self.small2.check(Monitor())) d.addCallback(self.failUnlessEqual, None, "small2") + d.addCallback(lambda ign: self.empty_lit_dir.check(Monitor())) + d.addCallback(self.failUnlessEqual, None, "empty_lit_dir") + d.addCallback(lambda ign: self.tiny_lit_dir.check(Monitor())) + d.addCallback(self.failUnlessEqual, None, "tiny_lit_dir") # and again with verify=True d.addCallback(lambda ign: self.root.check(Monitor(), verify=True)) hunk ./src/allmydata/test/test_deepcheck.py 453 d.addCallback(self.failUnlessEqual, None, "small") d.addCallback(lambda ign: self.small2.check(Monitor(), verify=True)) d.addCallback(self.failUnlessEqual, None, "small2") + d.addCallback(lambda ign: self.empty_lit_dir.check(Monitor(), verify=True)) + d.addCallback(self.failUnlessEqual, None, "empty_lit_dir") + d.addCallback(lambda ign: self.tiny_lit_dir.check(Monitor(), verify=True)) + d.addCallback(self.failUnlessEqual, None, "tiny_lit_dir") # and check_and_repair(), which should be a nop d.addCallback(lambda ign: self.root.check_and_repair(Monitor())) hunk ./src/allmydata/test/test_deepcheck.py 463 d.addCallback(self.check_and_repair_is_healthy, self.root, "root") d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor())) d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable") - #TODO d.addCallback(lambda ign: self.large.check_and_repair(Monitor())) - #TODO d.addCallback(self.check_and_repair_is_healthy, self.large, "large") - #TODO d.addCallback(lambda ign: self.small.check_and_repair(Monitor())) - #TODO d.addCallback(self.failUnlessEqual, None, "small") - #TODO d.addCallback(lambda ign: self.small2.check_and_repair(Monitor())) - #TODO d.addCallback(self.failUnlessEqual, None, "small2") + d.addCallback(lambda ign: self.large.check_and_repair(Monitor())) + d.addCallback(self.check_and_repair_is_healthy, self.large, "large") + d.addCallback(lambda ign: self.small.check_and_repair(Monitor())) + d.addCallback(self.failUnlessEqual, None, "small") + d.addCallback(lambda ign: self.small2.check_and_repair(Monitor())) + d.addCallback(self.failUnlessEqual, None, "small2") + d.addCallback(lambda ign: self.empty_lit_dir.check_and_repair(Monitor())) + d.addCallback(self.failUnlessEqual, None, "empty_lit_dir") + d.addCallback(lambda ign: self.tiny_lit_dir.check_and_repair(Monitor())) # check_and_repair(verify=True) d.addCallback(lambda ign: self.root.check_and_repair(Monitor(), verify=True)) hunk ./src/allmydata/test/test_deepcheck.py 478 d.addCallback(self.check_and_repair_is_healthy, self.root, "root") d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor(), verify=True)) d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable") - #TODO d.addCallback(lambda ign: self.large.check_and_repair(Monitor(), verify=True)) - #TODO d.addCallback(self.check_and_repair_is_healthy, self.large, "large", - #TODO incomplete=True) - #TODO d.addCallback(lambda ign: self.small.check_and_repair(Monitor(), verify=True)) - #TODO d.addCallback(self.failUnlessEqual, None, "small") - #TODO d.addCallback(lambda ign: self.small2.check_and_repair(Monitor(), verify=True)) - #TODO d.addCallback(self.failUnlessEqual, None, "small2") + d.addCallback(lambda ign: self.large.check_and_repair(Monitor(), verify=True)) + d.addCallback(self.check_and_repair_is_healthy, self.large, "large", incomplete=True) + d.addCallback(lambda ign: self.small.check_and_repair(Monitor(), verify=True)) + d.addCallback(self.failUnlessEqual, None, "small") + d.addCallback(lambda ign: self.small2.check_and_repair(Monitor(), verify=True)) + d.addCallback(self.failUnlessEqual, None, "small2") + d.addCallback(self.failUnlessEqual, None, "small2") + d.addCallback(lambda ign: self.empty_lit_dir.check_and_repair(Monitor(), verify=True)) + d.addCallback(self.failUnlessEqual, None, "empty_lit_dir") + d.addCallback(lambda ign: self.tiny_lit_dir.check_and_repair(Monitor(), verify=True)) # now deep-check the root, with various verify= and repair= options hunk ./src/allmydata/test/test_deepcheck.py 631 d.addCallback(self.json_check_lit, self.small, "small") d.addCallback(lambda ign: self.web_json(self.small2, t="check")) d.addCallback(self.json_check_lit, self.small2, "small2") + d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check")) + d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir") + d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check")) + d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir") # check and verify d.addCallback(lambda ign: hunk ./src/allmydata/test/test_deepcheck.py 653 d.addCallback(lambda ign: self.web_json(self.small2, t="check", verify="true")) d.addCallback(self.json_check_lit, self.small2, "small2+v") + d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", verify="true")) + d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+v") + d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", verify="true")) + d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+v") # check and repair, no verify d.addCallback(lambda ign: hunk ./src/allmydata/test/test_deepcheck.py 674 d.addCallback(lambda ign: self.web_json(self.small2, t="check", repair="true")) d.addCallback(self.json_check_lit, self.small2, "small2+r") + d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", repair="true")) + d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+r") + d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", repair="true")) + d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+r") # check+verify+repair d.addCallback(lambda ign: hunk ./src/allmydata/test/test_deepcheck.py 695 d.addCallback(lambda ign: self.web_json(self.small2, t="check", repair="true", verify="true")) d.addCallback(self.json_check_lit, self.small2, "small2+vr") + d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", repair="true", verify=True)) + d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+vr") + d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", repair="true", verify=True)) + d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+vr") # now run a deep-check, with various verify= and repair= flags d.addCallback(lambda ign: hunk ./src/allmydata/test/test_deepcheck.py 723 d.addCallback(lambda ign: self.web(self.large, t="info")) d.addCallback(lambda ign: self.web(self.small, t="info")) d.addCallback(lambda ign: self.web(self.small2, t="info")) + d.addCallback(lambda ign: self.web(self.empty_lit_dir, t="info")) + d.addCallback(lambda ign: self.web(self.tiny_lit_dir, t="info")) return d hunk ./src/allmydata/test/test_deepcheck.py 780 self.failUnlessEqual(caps[self.large.get_uri()], "large") self.failUnlessEqual(caps[self.small.get_uri()], "small") self.failUnlessEqual(caps[self.small2.get_uri()], "small2") + self.failUnlessEqual(caps[self.empty_lit_dir.get_uri()], "empty_lit_dir") + self.failUnlessEqual(caps[self.tiny_lit_dir.get_uri()], "tiny_lit_dir") d.addCallback(_check) return d hunk ./src/allmydata/uri.py 197 STRING_RE=re.compile('^URI:LIT:'+base32.BASE32STR_anybytes+'$') HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'LIT'+SEP+base32.BASE32STR_anybytes+'$') - def __init__(self, data=None): - if data is not None: - assert isinstance(data, str) - self.data = data + def __init__(self, data): + assert isinstance(data, str) + self.data = data @classmethod def init_from_human_encoding(cls, uri): } Context: [More cleanups to test_cli using new utilities for reading and writing files. david-sarah@jacaranda.org**20100206013855 Ignore-this: 9fd2294406b346bfe9144fff6a61f789 ] [Fix race conditions and missing callback in allmydata.test.test_cli.Cp.test_copy_using_filecap, add utilities for one-liner reading and writing of files, and fix cases in test_cli where files were not being closed after writing. david-sarah@jacaranda.org**20100206013727 Ignore-this: 49da6c33190d526a4ae84c472f04d5f4 ] [setup: comment-out the dependency on pycrypto, see #953 zooko@zooko.com**20100215050844 Ignore-this: 2751120921ff35b8189d8fcd896da149 ] [Add tests for #939 Kevan Carstensen **20100212062137 Ignore-this: 5459e8c64ba76cca70aa720e68549637 ] [Alter CLI utilities to handle nonexistent aliases better Kevan Carstensen **20100211024318 Ignore-this: e698ea4a57f5fe27c24336581ca0cf65 ] [adding pycrypto to the auto dependencies secorp@allmydata.com**20100206054314 Ignore-this: b873fc00a6a5b001d30d479e6053cf2f ] [docs running.html - "tahoe run ." does not work with the current installation, replaced with "tahoe start ." secorp@allmydata.com**20100206165320 Ignore-this: fdb2dcb0e417d303cd43b1951a4f8c03 ] [web/storage.py: display total-seen on the last-complete-cycle line. For #940. Brian Warner **20100208002010 Ignore-this: c0ed860f3e9628d3171d2b055d96c5aa ] [code coverage: replace figleaf with coverage.py, should work on py2.6 now. Brian Warner **20100203165421 Ignore-this: 46ab590360be6a385cb4fc4e68b6b42c It still lacks the right HTML report (the builtin report is very pretty, but lacks the "lines uncovered" numbers that I want), and the half-finished delta-from-last-run measurements. ] [More comprehensive changes and ticket references for NEWS david-sarah@jacaranda.org**20100202061256 Ignore-this: 696cf0106e8a7fd388afc5b55fba8a1b ] [docs: install.html: link into Python 2.5.5 download page zooko@zooko.com**20100202065852 Ignore-this: 1a9471b8175b7de5741d8445a7ede29d ] [TAG allmydata-tahoe-1.6.0 zooko@zooko.com**20100202061125 Ignore-this: dee6ade7ac1452cf5d1d9c69a8146d84 ] Patch bundle hash: 7d1641ff84c408c3275fe709337123b0c4a988e9