Sun May 30 18:43:46 PDT 2010 Kevan Carstensen * Code cleanup - Change 'readv' to 'readvs' in remote_slot_readv in the storage server, to more adaquately convey what the argument is. Mon May 31 15:49:00 PDT 2010 Kevan Carstensen * Add tests for new MDMF proxies Mon May 31 16:10:09 PDT 2010 Kevan Carstensen * Add objects for MDMF shares in support of a new segmented uploader New patches: [Code cleanup Kevan Carstensen **20100531014346 Ignore-this: 697378037e83290267f108a4a88b8776 - Change 'readv' to 'readvs' in remote_slot_readv in the storage server, to more adaquately convey what the argument is. ] { hunk ./src/allmydata/storage/server.py 569 self) return share - def remote_slot_readv(self, storage_index, shares, readv): + def remote_slot_readv(self, storage_index, shares, readvs): start = time.time() self.count("readv") si_s = si_b2a(storage_index) hunk ./src/allmydata/storage/server.py 590 if sharenum in shares or not shares: filename = os.path.join(bucketdir, sharenum_s) msf = MutableShareFile(filename, self) - datavs[sharenum] = msf.readv(readv) + datavs[sharenum] = msf.readv(readvs) log.msg("returning shares %s" % (datavs.keys(),), facility="tahoe.storage", level=log.NOISY, parent=lp) self.add_latency("readv", time.time() - start) } [Add tests for new MDMF proxies Kevan Carstensen **20100531224900 Ignore-this: 34c9b2afba71cb63228d48d4d2a72195 ] { hunk ./src/allmydata/test/test_storage.py 2 -import time, os.path, stat, re, simplejson, struct +import time, os.path, stat, re, simplejson, struct, shutil from twisted.trial import unittest hunk ./src/allmydata/test/test_storage.py 22 from allmydata.storage.expirer import LeaseCheckingCrawler from allmydata.immutable.layout import WriteBucketProxy, WriteBucketProxy_v2, \ ReadBucketProxy +from allmydata.mutable.layout import MDMFSlotWriteProxy, MDMFSlotReadProxy, \ + LayoutInvalid from allmydata.interfaces import BadWriteEnablerError hunk ./src/allmydata/test/test_storage.py 25 -from allmydata.test.common import LoggingServiceParent +from allmydata.test.common import LoggingServiceParent, ShouldFailMixin from allmydata.test.common_web import WebRenderingMixin from allmydata.web.storage import StorageStatus, remove_prefix hunk ./src/allmydata/test/test_storage.py 1285 self.failUnless(os.path.exists(prefixdir)) self.failIf(os.path.exists(bucketdir)) + +class MDMFProxies(unittest.TestCase, ShouldFailMixin): + def setUp(self): + self.sparent = LoggingServiceParent() + self._lease_secret = itertools.count() + self.ss = self.create("MDMFProxies storage test server") + self.rref = RemoteBucket() + self.rref.target = self.ss + self.secrets = (self.write_enabler("we_secret"), + self.renew_secret("renew_secret"), + self.cancel_secret("cancel_secret")) + self.block = "aa" + self.salt = "a" * 16 + self.block_hash = "a" * 32 + self.block_hash_tree = [self.block_hash for i in xrange(6)] + self.share_hash = self.block_hash + self.share_hash_chain = dict([(i, self.share_hash) for i in xrange(6)]) + self.signature = "foobarbaz" + self.verification_key = "vvvvvv" + self.encprivkey = "private" + self.root_hash = self.block_hash + self.salt_hash = self.root_hash + self.block_hash_tree_s = self.serialize_blockhashes(self.block_hash_tree) + self.share_hash_chain_s = self.serialize_sharehashes(self.share_hash_chain) + + + def tearDown(self): + self.sparent.stopService() + shutil.rmtree(self.workdir("MDMFProxies storage test server")) + + + def write_enabler(self, we_tag): + return hashutil.tagged_hash("we_blah", we_tag) + + + def renew_secret(self, tag): + return hashutil.tagged_hash("renew_blah", str(tag)) + + + def cancel_secret(self, tag): + return hashutil.tagged_hash("cancel_blah", str(tag)) + + + def workdir(self, name): + basedir = os.path.join("storage", "MutableServer", name) + return basedir + + + def create(self, name): + workdir = self.workdir(name) + ss = StorageServer(workdir, "\x00" * 20) + ss.setServiceParent(self.sparent) + return ss + + + def write_test_share_to_server(self, + storage_index, + tail_segment=False): + """ + I write some data for the read tests to read to self.ss + + If tail_segment=True, then I will write a share that has a + smaller tail segment than other segments. + """ + write = self.ss.remote_slot_testv_and_readv_and_writev + # Start with the checkstring + data = struct.pack(">BQ32s32s", + 1, + 0, + self.root_hash, + self.salt_hash) + self.checkstring = data + # Next, the encoding parameters + if tail_segment: + data += struct.pack(">BBQQ", + 3, + 10, + 6, + 33) + else: + data += struct.pack(">BBQQ", + 3, + 10, + 6, + 36) + # Now we'll build the offsets. + # The header -- everything up to the salts -- is 143 bytes long. + # The shares come after the salts. + salts = self.salt * 6 + share_offset = 143 + len(salts) + if tail_segment: + sharedata = self.block * 6 + else: + sharedata = self.block * 6 + "a" + # The encrypted private key comes after the shares + encrypted_private_key_offset = share_offset + len(sharedata) + # The blockhashes come after the private key + blockhashes_offset = encrypted_private_key_offset + len(self.encprivkey) + # The sharehashes come after the blockhashes + sharehashes_offset = blockhashes_offset + len(self.block_hash_tree_s) + # The signature comes after the share hash chain + signature_offset = sharehashes_offset + len(self.share_hash_chain_s) + # The verification key comes after the signature + verification_offset = signature_offset + len(self.signature) + # The EOF comes after the verification key + eof_offset = verification_offset + len(self.verification_key) + data += struct.pack(">LQQQQQQ", + share_offset, + encrypted_private_key_offset, + blockhashes_offset, + sharehashes_offset, + signature_offset, + verification_offset, + eof_offset) + # Next, we'll add in the salts, + data += salts + # the share data, + data += sharedata + # the private key, + data += self.encprivkey + # the block hash tree, + data += self.block_hash_tree_s + # the share hash chain, + data += self.share_hash_chain_s + # the signature, + data += self.signature + # and the verification key + data += self.verification_key + + # Finally, we write the whole thing to the storage server in one + # pass. + testvs = [(0, 1, "eq", "")] + tws = {} + tws[0] = (testvs, [(0, data)], None) + readv = [(0, 1)] + results = write(storage_index, self.secrets, tws, readv) + self.failUnless(results[0]) + + + def test_read(self): + self.write_test_share_to_server("si1") + mr = MDMFSlotReadProxy(self.rref, self.secrets, "si1", 0) + # Check that every method equals what we expect it to. + d = defer.succeed(None) + def _check_block_and_salt((block, salt)): + self.failUnlessEqual(block, self.block) + self.failUnlessEqual(salt, self.salt) + + for i in xrange(6): + d.addCallback(lambda ignored, i=i: + mr.get_block_and_salt(i)) + d.addCallback(_check_block_and_salt) + + d.addCallback(lambda ignored: + mr.get_encprivkey()) + d.addCallback(lambda encprivkey: + self.failUnlessEqual(self.encprivkey, encprivkey)) + + d.addCallback(lambda ignored: + mr.get_blockhashes()) + d.addCallback(lambda blockhashes: + self.failUnlessEqual(self.block_hash_tree, blockhashes)) + + d.addCallback(lambda ignored: + mr.get_sharehashes()) + d.addCallback(lambda sharehashes: + self.failUnlessEqual(self.share_hash_chain, sharehashes)) + + d.addCallback(lambda ignored: + mr.get_signature()) + d.addCallback(lambda signature: + self.failUnlessEqual(signature, self.signature)) + + d.addCallback(lambda ignored: + mr.get_verification_key()) + d.addCallback(lambda verification_key: + self.failUnlessEqual(verification_key, self.verification_key)) + + d.addCallback(lambda ignored: + mr.get_seqnum()) + d.addCallback(lambda seqnum: + self.failUnlessEqual(seqnum, 0)) + + d.addCallback(lambda ignored: + mr.get_root_hash()) + d.addCallback(lambda root_hash: + self.failUnlessEqual(self.root_hash, root_hash)) + + d.addCallback(lambda ignored: + mr.get_salt_hash()) + d.addCallback(lambda salt_hash: + self.failUnlessEqual(self.salt_hash, salt_hash)) + + d.addCallback(lambda ignored: + mr.get_seqnum()) + d.addCallback(lambda seqnum: + self.failUnlessEqual(0, seqnum)) + + d.addCallback(lambda ignored: + mr.get_encoding_parameters()) + def _check_encoding_parameters((k, n, segsize, datalen)): + self.failUnlessEqual(k, 3) + self.failUnlessEqual(n, 10) + self.failUnlessEqual(segsize, 6) + self.failUnlessEqual(datalen, 36) + d.addCallback(_check_encoding_parameters) + + d.addCallback(lambda ignored: + mr.get_checkstring()) + d.addCallback(lambda checkstring: + self.failUnlessEqual(checkstring, checkstring)) + return d + + + def test_read_with_different_tail_segment_size(self): + self.write_test_share_to_server("si1", tail_segment=True) + mr = MDMFSlotReadProxy(self.rref, self.secrets, "si1", 0) + d = mr.get_block_and_salt(5) + def _check_tail_segment(results): + block, salt = results + self.failUnlessEqual(len(block), 1) + self.failUnlessEqual(block, "a") + d.addCallback(_check_tail_segment) + return d + + + def test_get_block_with_invalid_segnum(self): + self.write_test_share_to_server("si1") + mr = MDMFSlotReadProxy(self.rref, self.secrets, "si1", 0) + d = defer.succeed(None) + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "test invalid segnum", + None, + mr.get_block_and_salt, 7)) + return d + + + def test_get_encoding_parameters_first(self): + self.write_test_share_to_server("si1") + mr = MDMFSlotReadProxy(self.rref, self.secrets, "si1", 0) + d = mr.get_encoding_parameters() + def _check_encoding_parameters((k, n, segment_size, datalen)): + self.failUnlessEqual(k, 3) + self.failUnlessEqual(n, 10) + self.failUnlessEqual(segment_size, 6) + self.failUnlessEqual(datalen, 36) + d.addCallback(_check_encoding_parameters) + return d + + + def test_get_seqnum_first(self): + self.write_test_share_to_server("si1") + mr = MDMFSlotReadProxy(self.rref, self.secrets, "si1", 0) + d = mr.get_seqnum() + d.addCallback(lambda seqnum: + self.failUnlessEqual(seqnum, 0)) + return d + + + def test_get_root_hash_first(self): + self.write_test_share_to_server("si1") + mr = MDMFSlotReadProxy(self.rref, self.secrets, "si1", 0) + d = mr.get_root_hash() + d.addCallback(lambda root_hash: + self.failUnlessEqual(root_hash, self.root_hash)) + return d + + + def test_get_salt_hash_first(self): + self.write_test_share_to_server("si1") + mr = MDMFSlotReadProxy(self.rref, self.secrets, "si1", 0) + d = mr.get_salt_hash() + d.addCallback(lambda salt_hash: + self.failUnlessEqual(salt_hash, self.salt_hash)) + return d + + + def test_get_checkstring_first(self): + self.write_test_share_to_server("si1") + mr = MDMFSlotReadProxy(self.rref, self.secrets, "si1", 0) + d = mr.get_checkstring() + d.addCallback(lambda checkstring: + self.failUnlessEqual(checkstring, self.checkstring)) + return d + + + def test_write_read_vectors(self): + # When writing for us, the storage server will return to us a + # read vector, along with its result. If a write fails because + # the test vectors failed, this read vector can help us to + # diagnose the problem. This test ensures that the read vector + # is working appropriately. + mw = self._make_new_mw("si1", 0) + d = defer.succeed(None) + + # Write one share. This should return a checkstring of nothing, + # since there is no data there. + d.addCallback(lambda ignored: + mw.put_block(self.block, 0, self.salt)) + def _check_first_write(results): + result, readvs = results + self.failUnless(result) + self.failIf(readvs) + d.addCallback(_check_first_write) + # Now, there should be a different checkstring returned when + # we write other shares + d.addCallback(lambda ignored: + mw.put_block(self.block, 1, self.salt)) + def _check_next_write(results): + result, readvs = results + self.failUnless(result) + self.expected_checkstring = mw.get_checkstring() + self.failUnlessIn(0, readvs) + self.failUnlessEqual(readvs[0][0], self.expected_checkstring) + d.addCallback(_check_next_write) + # Add the other four shares + for i in xrange(2, 6): + d.addCallback(lambda ignored, i=i: + mw.put_block(self.block, i, self.salt)) + d.addCallback(_check_next_write) + # Add the encrypted private key + d.addCallback(lambda ignored: + mw.put_encprivkey(self.encprivkey)) + d.addCallback(_check_next_write) + # Add the block hash tree and share hash tree + d.addCallback(lambda ignored: + mw.put_blockhashes(self.block_hash_tree)) + d.addCallback(_check_next_write) + d.addCallback(lambda ignored: + mw.put_sharehashes(self.share_hash_chain)) + d.addCallback(_check_next_write) + # Add the root hash and the salt hash. This should change the + # checkstring, but not in a way that we'll be able to see right + # now, since the read vectors are applied before the write + # vectors. + d.addCallback(lambda ignored: + mw.put_root_and_salt_hashes(self.root_hash, self.salt_hash)) + def _check_old_testv_after_new_one_is_written(results): + result, readvs = results + self.failUnless(result) + self.failUnlessIn(0, readvs) + self.failUnlessEqual(self.expected_checkstring, + readvs[0][0]) + new_checkstring = mw.get_checkstring() + self.failIfEqual(new_checkstring, + readvs[0][0]) + d.addCallback(_check_old_testv_after_new_one_is_written) + # Now add the signature. This should succeed, meaning that the + # data gets written and the read vector matches what the writer + # thinks should be there. + d.addCallback(lambda ignored: + mw.put_signature(self.signature)) + d.addCallback(_check_next_write) + # The checkstring remains the same for the rest of the process. + return d + + + def test_blockhashes_after_share_hash_chain(self): + mw = self._make_new_mw("si1", 0) + d = defer.succeed(None) + # Put everything up to and including the share hash chain + for i in xrange(6): + d.addCallback(lambda ignored, i=i: + mw.put_block(self.block, i, self.salt)) + d.addCallback(lambda ignored: + mw.put_encprivkey(self.encprivkey)) + d.addCallback(lambda ignored: + mw.put_blockhashes(self.block_hash_tree)) + d.addCallback(lambda ignored: + mw.put_sharehashes(self.share_hash_chain)) + # Now try to put a block hash tree after the share hash chain. + # This won't necessarily overwrite the share hash chain, but it + # is a bad idea in general -- if we write one that is anything + # other than the exact size of the initial one, we will either + # overwrite the share hash chain, or give the reader (who uses + # the offset of the share hash chain as an end boundary) a + # shorter tree than they know to read, which will result in them + # reading junk. There is little reason to support it as a use + # case, so we should disallow it altogether. + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "test same blockhashes", + None, + mw.put_blockhashes, self.block_hash_tree)) + return d + + + def test_encprivkey_after_blockhashes(self): + mw = self._make_new_mw("si1", 0) + d = defer.succeed(None) + # Put everything up to and including the block hash tree + for i in xrange(6): + d.addCallback(lambda ignored, i=i: + mw.put_block(self.block, i, self.salt)) + d.addCallback(lambda ignored: + mw.put_encprivkey(self.encprivkey)) + d.addCallback(lambda ignored: + mw.put_blockhashes(self.block_hash_tree)) + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "out of order private key", + None, + mw.put_encprivkey, self.encprivkey)) + return d + + + def test_share_hash_chain_after_signature(self): + mw = self._make_new_mw("si1", 0) + d = defer.succeed(None) + # Put everything up to and including the signature + for i in xrange(6): + d.addCallback(lambda ignored, i=i: + mw.put_block(self.block, i, self.salt)) + d.addCallback(lambda ignored: + mw.put_encprivkey(self.encprivkey)) + d.addCallback(lambda ignored: + mw.put_blockhashes(self.block_hash_tree)) + d.addCallback(lambda ignored: + mw.put_sharehashes(self.share_hash_chain)) + d.addCallback(lambda ignored: + mw.put_root_and_salt_hashes(self.root_hash, self.salt_hash)) + d.addCallback(lambda ignored: + mw.put_signature(self.signature)) + # Now try to put the share hash chain again. This should fail + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "out of order share hash chain", + None, + mw.put_sharehashes, self.share_hash_chain)) + return d + + + def test_signature_after_verification_key(self): + mw = self._make_new_mw("si1", 0) + d = defer.succeed(None) + # Put everything up to and including the verification key. + for i in xrange(6): + d.addCallback(lambda ignored, i=i: + mw.put_block(self.block, i, self.salt)) + d.addCallback(lambda ignored: + mw.put_encprivkey(self.encprivkey)) + d.addCallback(lambda ignored: + mw.put_blockhashes(self.block_hash_tree)) + d.addCallback(lambda ignored: + mw.put_sharehashes(self.share_hash_chain)) + d.addCallback(lambda ignored: + mw.put_root_and_salt_hashes(self.root_hash, self.salt_hash)) + d.addCallback(lambda ignored: + mw.put_signature(self.signature)) + d.addCallback(lambda ignored: + mw.put_verification_key(self.verification_key)) + # Now try to put the signature again. This should fail + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "signature after verification", + None, + mw.put_signature, self.signature)) + return d + + + def test_uncoordinated_write(self): + # Make two mutable writers, both pointing to the same storage + # server, both at the same storage index, and try writing to the + # same share. + mw1 = self._make_new_mw("si1", 0) + mw2 = self._make_new_mw("si1", 0) + d = defer.succeed(None) + def _check_success(results): + result, readvs = results + self.failUnless(result) + + def _check_failure(results): + result, readvs = results + self.failIf(result) + + d.addCallback(lambda ignored: + mw1.put_block(self.block, 0, self.salt)) + d.addCallback(_check_success) + d.addCallback(lambda ignored: + mw2.put_block(self.block, 0, self.salt)) + d.addCallback(_check_failure) + return d + + + def test_invalid_salt_size(self): + # Salts need to be 16 bytes in size. Writes that attempt to + # write more or less than this should be rejected. + mw = self._make_new_mw("si1", 0) + invalid_salt = "a" * 17 # 17 bytes + another_invalid_salt = "b" * 15 # 15 bytes + d = defer.succeed(None) + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "salt too big", + None, + mw.put_block, self.block, 0, invalid_salt)) + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "salt too small", + None, + mw.put_block, self.block, 0, + another_invalid_salt)) + return d + + + def test_write_test_vectors(self): + # If we give the write proxy a bogus test vector at + # any point during the process, it should fail to write. + mw = self._make_new_mw("si1", 0) + mw.set_checkstring("this is a lie") + # The initial write should be expecting to find the improbable + # checkstring above in place; finding nothing, it should fail. + d = defer.succeed(None) + d.addCallback(lambda ignored: + mw.put_block(self.block, 0, self.salt)) + def _check_failure(results): + result, readv = results + self.failIf(result) + d.addCallback(_check_failure) + # Now set the checkstring to the empty string, which + # indicates that no share is there. + d.addCallback(lambda ignored: + mw.set_checkstring("")) + d.addCallback(lambda ignored: + mw.put_block(self.block, 0, self.salt)) + def _check_success(results): + result, readv = results + self.failUnless(result) + d.addCallback(_check_success) + # Now set the checkstring to something wrong + d.addCallback(lambda ignored: + mw.set_checkstring("something wrong")) + # This should fail to do anything + d.addCallback(lambda ignored: + mw.put_block(self.block, 1, self.salt)) + d.addCallback(_check_failure) + # Now set it back to what it should be. + d.addCallback(lambda ignored: + mw.set_checkstring(mw.get_checkstring())) + for i in xrange(1, 6): + d.addCallback(lambda ignored, i=i: + mw.put_block(self.block, i, self.salt)) + d.addCallback(_check_success) + d.addCallback(lambda ignored: + mw.put_encprivkey(self.encprivkey)) + d.addCallback(_check_success) + d.addCallback(lambda ignored: + mw.put_blockhashes(self.block_hash_tree)) + d.addCallback(_check_success) + d.addCallback(lambda ignored: + mw.put_sharehashes(self.share_hash_chain)) + d.addCallback(_check_success) + def _keep_old_checkstring(ignored): + self.old_checkstring = mw.get_checkstring() + mw.set_checkstring("foobarbaz") + d.addCallback(_keep_old_checkstring) + d.addCallback(lambda ignored: + mw.put_root_and_salt_hashes(self.root_hash, self.salt_hash)) + d.addCallback(_check_failure) + d.addCallback(lambda ignored: + self.failUnlessEqual(self.old_checkstring, mw.get_checkstring())) + def _restore_old_checkstring(ignored): + mw.set_checkstring(self.old_checkstring) + d.addCallback(_restore_old_checkstring) + d.addCallback(lambda ignored: + mw.put_root_and_salt_hashes(self.root_hash, self.salt_hash)) + # The checkstring should have been set appropriately for us on + # the last write; if we try to change it to something else, + # that change should cause the verification key step to fail. + d.addCallback(lambda ignored: + mw.set_checkstring("something else")) + d.addCallback(lambda ignored: + mw.put_signature(self.signature)) + d.addCallback(_check_failure) + d.addCallback(lambda ignored: + mw.set_checkstring(mw.get_checkstring())) + d.addCallback(lambda ignored: + mw.put_signature(self.signature)) + d.addCallback(_check_success) + d.addCallback(lambda ignored: + mw.put_verification_key(self.verification_key)) + d.addCallback(_check_success) + return d + + + def test_offset_only_set_on_success(self): + # The write proxy should be smart enough to detect when a write + # has failed, and to temper its definition of progress based on + # that. + mw = self._make_new_mw("si1", 0) + d = defer.succeed(None) + for i in xrange(1, 6): + d.addCallback(lambda ignored, i=i: + mw.put_block(self.block, i, self.salt)) + def _break_checkstring(ignored): + self._old_checkstring = mw.get_checkstring() + mw.set_checkstring("foobarbaz") + + def _fix_checkstring(ignored): + mw.set_checkstring(self._old_checkstring) + + d.addCallback(_break_checkstring) + + # Setting the encrypted private key shouldn't work now, which is + # to be expected and is tested elsewhere. We also want to make + # sure that we can't add the block hash tree after a failed + # write of this sort. + d.addCallback(lambda ignored: + mw.put_encprivkey(self.encprivkey)) + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "test out-of-order blockhashes", + None, + mw.put_blockhashes, self.block_hash_tree)) + d.addCallback(_fix_checkstring) + d.addCallback(lambda ignored: + mw.put_encprivkey(self.encprivkey)) + d.addCallback(_break_checkstring) + d.addCallback(lambda ignored: + mw.put_blockhashes(self.block_hash_tree)) + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "test out-of-order sharehashes", + None, + mw.put_sharehashes, self.share_hash_chain)) + d.addCallback(_fix_checkstring) + d.addCallback(lambda ignored: + mw.put_blockhashes(self.block_hash_tree)) + d.addCallback(_break_checkstring) + d.addCallback(lambda ignored: + mw.put_sharehashes(self.share_hash_chain)) + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "out-of-order root hash", + None, + mw.put_root_and_salt_hashes, + self.root_hash, self.salt_hash)) + d.addCallback(_fix_checkstring) + d.addCallback(lambda ignored: + mw.put_sharehashes(self.share_hash_chain)) + d.addCallback(_break_checkstring) + d.addCallback(lambda ignored: + mw.put_root_and_salt_hashes(self.root_hash, self.salt_hash)) + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "out-of-order signature", + None, + mw.put_signature, self.signature)) + d.addCallback(_fix_checkstring) + d.addCallback(lambda ignored: + mw.put_root_and_salt_hashes(self.root_hash, self.salt_hash)) + d.addCallback(_break_checkstring) + d.addCallback(lambda ignored: + mw.put_signature(self.signature)) + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "out-of-order verification key", + None, + mw.put_verification_key, + self.verification_key)) + d.addCallback(_fix_checkstring) + d.addCallback(lambda ignored: + mw.put_signature(self.signature)) + d.addCallback(_break_checkstring) + d.addCallback(lambda ignored: + mw.put_verification_key(self.verification_key)) + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "out-of-order finish", + None, + mw.finish_publishing)) + return d + + + def serialize_blockhashes(self, blockhashes): + return "".join(blockhashes) + + + def serialize_sharehashes(self, sharehashes): + return "".join([struct.pack(">H32s", i, sharehashes[i]) + for i in sorted(sharehashes.keys())]) + + + def test_write(self): + # This translates to a file with 6 6-byte segments, and with 2-byte + # blocks. + mw = self._make_new_mw("si1", 0) + mw2 = self._make_new_mw("si1", 1) + # Test writing some blocks. + read = self.ss.remote_slot_readv + def _check_block_write(i, share): + self.failUnlessEqual(read("si1", [share], [(239 + (i * 2), 2)]), + {share: [self.block]}) + self.failUnlessEqual(read("si1", [share], [(143 + (i * 16), 16)]), + {share: [self.salt]}) + d = defer.succeed(None) + for i in xrange(6): + d.addCallback(lambda ignored, i=i: + mw.put_block(self.block, i, self.salt)) + d.addCallback(lambda ignored, i=i: + _check_block_write(i, 0)) + # Now try the same thing, but with share 1 instead of share 0. + for i in xrange(6): + d.addCallback(lambda ignored, i=i: + mw2.put_block(self.block, i, self.salt)) + d.addCallback(lambda ignored, i=i: + _check_block_write(i, 1)) + + def _spy_on_results(results): + print read("si1", [], [(0, 40000000)]) + return results + + # Next, we make a fake encrypted private key, and put it onto the + # storage server. + d.addCallback(lambda ignored: + mw.put_encprivkey(self.encprivkey)) + # So far, we have: + # header: 143 bytes + # salts: 16 * 6 = 96 bytes + # blocks: 2 * 6 = 12 bytes + # = 251 bytes + expected_private_key_offset = 251 + self.failUnlessEqual(len(self.encprivkey), 7) + d.addCallback(lambda ignored: + self.failUnlessEqual(read("si1", [0], [(251, 7)]), + {0: [self.encprivkey]})) + + # Next, we put a fake block hash tree. + d.addCallback(lambda ignored: + mw.put_blockhashes(self.block_hash_tree)) + # The block hash tree got inserted at: + # header + salts + blocks: 251 bytes + # encrypted private key: 7 bytes + # = 258 bytes + expected_block_hash_offset = 258 + self.failUnlessEqual(len(self.block_hash_tree_s), 32 * 6) + d.addCallback(lambda ignored: + self.failUnlessEqual(read("si1", [0], [(expected_block_hash_offset, 32 * 6)]), + {0: [self.block_hash_tree_s]})) + + # Next, put a fake share hash chain + d.addCallback(lambda ignored: + mw.put_sharehashes(self.share_hash_chain)) + # The share hash chain got inserted at: + # header + salts + blocks + private key = 258 bytes + # block hash tree: 32 * 6 = 192 bytes + # = 450 bytes + expected_share_hash_offset = 450 + d.addCallback(lambda ignored: + self.failUnlessEqual(read("si1", [0],[(expected_share_hash_offset, (32 + 2) * 6)]), + {0: [self.share_hash_chain_s]})) + + # Next, we put what is supposed to be the root hash of + # our share hash tree but isn't, along with the flat hash + # of all the salts. + d.addCallback(lambda ignored: + mw.put_root_and_salt_hashes(self.root_hash, self.salt_hash)) + # The root hash gets inserted at byte 9 (its position is in the header, + # and is fixed). The salt is right after it. + def _check(ignored): + self.failUnlessEqual(read("si1", [0], [(9, 32)]), + {0: [self.root_hash]}) + self.failUnlessEqual(read("si1", [0], [(41, 32)]), + {0: [self.salt_hash]}) + d.addCallback(_check) + + # Next, we put a signature of the header block. + d.addCallback(lambda ignored: + mw.put_signature(self.signature)) + # The signature gets written to: + # header + salts + blocks + block and share hash tree = 654 + expected_signature_offset = 654 + self.failUnlessEqual(len(self.signature), 9) + d.addCallback(lambda ignored: + self.failUnlessEqual(read("si1", [0], [(expected_signature_offset, 9)]), + {0: [self.signature]})) + + # Next, we put the verification key + d.addCallback(lambda ignored: + mw.put_verification_key(self.verification_key)) + # The verification key gets written to: + # 654 + 9 = 663 bytes + expected_verification_key_offset = 663 + self.failUnlessEqual(len(self.verification_key), 6) + d.addCallback(lambda ignored: + self.failUnlessEqual(read("si1", [0], [(expected_verification_key_offset, 6)]), + {0: [self.verification_key]})) + + def _check_signable(ignored): + # Make sure that the signable is what we think it should be. + signable = mw.get_signable() + verno, seq, roothash, salthash, k, n, segsize, datalen = \ + struct.unpack(">BQ32s32sBBQQ", + signable) + self.failUnlessEqual(verno, 1) + self.failUnlessEqual(seq, 0) + self.failUnlessEqual(roothash, self.root_hash) + self.failUnlessEqual(salthash, self.salt_hash) + self.failUnlessEqual(k, 3) + self.failUnlessEqual(n, 10) + self.failUnlessEqual(segsize, 6) + self.failUnlessEqual(datalen, 36) + d.addCallback(_check_signable) + # Next, we cause the offset table to be published. + d.addCallback(lambda ignored: + mw.finish_publishing()) + expected_eof_offset = 669 + + # The offset table starts at byte 91. Happily, we have already + # worked out most of these offsets above, but we want to make + # sure that the representation on disk agrees what what we've + # calculated. + # + # (we don't have an explicit offset for the AES salts, because + # we know that they start right after the header) + def _check_offsets(ignored): + # Check the version number to make sure that it is correct. + expected_version_number = struct.pack(">B", 1) + self.failUnlessEqual(read("si1", [0], [(0, 1)]), + {0: [expected_version_number]}) + # Check the sequence number to make sure that it is correct + expected_sequence_number = struct.pack(">Q", 0) + self.failUnlessEqual(read("si1", [0], [(1, 8)]), + {0: [expected_sequence_number]}) + # Check that the encoding parameters (k, N, segement size, data + # length) are what they should be. These are 3, 10, 6, 36 + expected_k = struct.pack(">B", 3) + self.failUnlessEqual(read("si1", [0], [(73, 1)]), + {0: [expected_k]}) + expected_n = struct.pack(">B", 10) + self.failUnlessEqual(read("si1", [0], [(74, 1)]), + {0: [expected_n]}) + expected_segment_size = struct.pack(">Q", 6) + self.failUnlessEqual(read("si1", [0], [(75, 8)]), + {0: [expected_segment_size]}) + expected_data_length = struct.pack(">Q", 36) + self.failUnlessEqual(read("si1", [0], [(83, 8)]), + {0: [expected_data_length]}) + # 91 4 The offset of the share data + expected_offset = struct.pack(">L", 239) + self.failUnlessEqual(read("si1", [0], [(91, 4)]), + {0: [expected_offset]}) + # 95 8 The offset of the encrypted private key + expected_offset = struct.pack(">Q", expected_private_key_offset) + self.failUnlessEqual(read("si1", [0], [(95, 8)]), + {0: [expected_offset]}) + # 103 8 The offset of the block hash tree + expected_offset = struct.pack(">Q", expected_block_hash_offset) + self.failUnlessEqual(read("si1", [0], [(103, 8)]), + {0: [expected_offset]}) + # 111 8 The offset of the share hash chain + expected_offset = struct.pack(">Q", expected_share_hash_offset) + self.failUnlessEqual(read("si1", [0], [(111, 8)]), + {0: [expected_offset]}) + # 119 8 The offset of the signature + expected_offset = struct.pack(">Q", expected_signature_offset) + self.failUnlessEqual(read("si1", [0], [(119, 8)]), + {0: [expected_offset]}) + # 127 8 The offset of the verification key + expected_offset = struct.pack(">Q", expected_verification_key_offset) + self.failUnlessEqual(read("si1", [0], [(127, 8)]), + {0: [expected_offset]}) + # 135 8 offset of the EOF + expected_offset = struct.pack(">Q", expected_eof_offset) + self.failUnlessEqual(read("si1", [0], [(135, 8)]), + {0: [expected_offset]}) + # = 143 bytes in total. + d.addCallback(_check_offsets) + return d + + def _make_new_mw(self, si, share, datalength=36): + # This is a file of size 36 bytes. Since it has a segment + # size of 6, we know that it has 6 byte segments, which will + # be split into blocks of 2 bytes because our FEC k + # parameter is 3. + mw = MDMFSlotWriteProxy(share, self.rref, si, self.secrets, 0, 3, 10, + 6, datalength) + return mw + + + def test_write_rejected_with_too_many_blocks(self): + mw = self._make_new_mw("si0", 0) + + # Try writing too many blocks. We should not be able to write + # more than 6 + # blocks into each share. + d = defer.succeed(None) + for i in xrange(6): + d.addCallback(lambda ignored, i=i: + mw.put_block(self.block, i, self.salt)) + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "too many blocks", + None, + mw.put_block, self.block, 7, self.salt)) + return d + + + def test_write_rejected_with_invalid_salt(self): + # Try writing an invalid salt. Salts are 16 bytes -- any more or + # less should cause an error. + mw = self._make_new_mw("si1", 0) + bad_salt = "a" * 17 # 17 bytes + d = defer.succeed(None) + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "test_invalid_salt", + None, mw.put_block, self.block, 7, bad_salt)) + return d + + + def test_write_rejected_with_invalid_salt_hash(self): + # Try writing an invalid salt hash. These should be SHA256d, and + # 32 bytes long as a result. + mw = self._make_new_mw("si2", 0) + invalid_salt_hash = "b" * 31 + d = defer.succeed(None) + # Before this test can work, we need to put some blocks + salts, + # a block hash tree, and a share hash tree. Otherwise, we'll see + # failures that match what we are looking for, but are caused by + # the constraints imposed on operation ordering. + for i in xrange(6): + d.addCallback(lambda ignored, i=i: + mw.put_block(self.block, i, self.salt)) + d.addCallback(lambda ignored: + mw.put_encprivkey(self.encprivkey)) + d.addCallback(lambda ignored: + mw.put_blockhashes(self.block_hash_tree)) + d.addCallback(lambda ignored: + mw.put_sharehashes(self.share_hash_chain)) + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "invalid root hash", + None, mw.put_root_and_salt_hashes, + self.root_hash, invalid_salt_hash)) + return d + + + def test_write_rejected_with_invalid_root_hash(self): + # Try writing an invalid root hash. This should be SHA256d, and + # 32 bytes long as a result. + mw = self._make_new_mw("si2", 0) + # 17 bytes != 32 bytes + invalid_root_hash = "a" * 17 + d = defer.succeed(None) + # Before this test can work, we need to put some blocks + salts, + # a block hash tree, and a share hash tree. Otherwise, we'll see + # failures that match what we are looking for, but are caused by + # the constraints imposed on operation ordering. + for i in xrange(6): + d.addCallback(lambda ignored, i=i: + mw.put_block(self.block, i, self.salt)) + d.addCallback(lambda ignored: + mw.put_encprivkey(self.encprivkey)) + d.addCallback(lambda ignored: + mw.put_blockhashes(self.block_hash_tree)) + d.addCallback(lambda ignored: + mw.put_sharehashes(self.share_hash_chain)) + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "invalid root hash", + None, mw.put_root_and_salt_hashes, + invalid_root_hash, self.salt_hash)) + return d + + + def test_write_rejected_with_invalid_blocksize(self): + # The blocksize implied by the writer that we get from + # _make_new_mw is 2bytes -- any more or any less than this + # should be cause for failure, unless it is the tail segment, in + # which case it may not be failure. + invalid_block = "a" + mw = self._make_new_mw("si3", 0, 33) # implies a tail segment with + # one byte blocks + # 1 bytes != 2 bytes + d = defer.succeed(None) + d.addCallback(lambda ignored, invalid_block=invalid_block: + self.shouldFail(LayoutInvalid, "test blocksize too small", + None, mw.put_block, invalid_block, 0, + self.salt)) + invalid_block = invalid_block * 3 + # 3 bytes != 2 bytes + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "test blocksize too large", + None, + mw.put_block, invalid_block, 0, self.salt)) + for i in xrange(5): + d.addCallback(lambda ignored, i=i: + mw.put_block(self.block, i, self.salt)) + # Try to put an invalid tail segment + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "test invalid tail segment", + None, + mw.put_block, self.block, 5, self.salt)) + valid_block = "a" + d.addCallback(lambda ignored: + mw.put_block(valid_block, 5, self.salt)) + return d + + + def test_write_enforces_order_constraints(self): + # We require that the MDMFSlotWriteProxy be interacted with in a + # specific way. + # That way is: + # 0: __init__ + # 1: write blocks and salts + # 2: Write the encrypted private key + # 3: Write the block hashes + # 4: Write the share hashes + # 5: Write the root hash and salt hash + # 6: Write the signature and verification key + # 7: Write the file. + # + # Some of these can be performed out-of-order, and some can't. + # The dependencies that I want to test here are: + # - Private key before block hashes + # - share hashes and block hashes before root hash + # - root hash before signature + # - signature before verification key + mw0 = self._make_new_mw("si0", 0) + # Write some shares + d = defer.succeed(None) + for i in xrange(6): + d.addCallback(lambda ignored, i=i: + mw0.put_block(self.block, i, self.salt)) + # Try to write the block hashes before writing the encrypted + # private key + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "block hashes before key", + None, mw0.put_blockhashes, + self.block_hash_tree)) + + # Write the private key. + d.addCallback(lambda ignored: + mw0.put_encprivkey(self.encprivkey)) + + + # Try to write the share hash chain without writing the block + # hash tree + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "share hash chain before " + "block hash tree", + None, + mw0.put_sharehashes, self.share_hash_chain)) + + # Try to write the root hash and salt hash without writing either the + # block hashes or the share hashes + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "root hash before share hashes", + None, + mw0.put_root_and_salt_hashes, + self.root_hash, self.salt_hash)) + + # Now write the block hashes and try again + d.addCallback(lambda ignored: + mw0.put_blockhashes(self.block_hash_tree)) + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "root hash before share hashes", + None, mw0.put_root_and_salt_hashes, + self.root_hash, self.salt_hash)) + + # We haven't yet put the root hash on the share, so we shouldn't + # be able to sign it. + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "signature before root hash", + None, mw0.put_signature, self.signature)) + + d.addCallback(lambda ignored: + self.failUnlessRaises(LayoutInvalid, mw0.get_signable)) + + # ..and, since that fails, we also shouldn't be able to put the + # verification key. + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "key before signature", + None, mw0.put_verification_key, + self.verification_key)) + + # Now write the share hashes and verify that it works. + d.addCallback(lambda ignored: + mw0.put_sharehashes(self.share_hash_chain)) + + # We should still be unable to sign the header + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "signature before hashes", + None, + mw0.put_signature, self.signature)) + + # We should be able to write the root hash now too + d.addCallback(lambda ignored: + mw0.put_root_and_salt_hashes(self.root_hash, self.salt_hash)) + + # We should still be unable to put the verification key + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "key before signature", + None, mw0.put_verification_key, + self.verification_key)) + + d.addCallback(lambda ignored: + mw0.put_signature(self.signature)) + + # We shouldn't be able to write the offsets to the remote server + # until the offset table is finished; IOW, until we have written + # the verification key. + d.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "offsets before verification key", + None, + mw0.finish_publishing)) + + d.addCallback(lambda ignored: + mw0.put_verification_key(self.verification_key)) + return d + + + def test_end_to_end(self): + mw = self._make_new_mw("si1", 0) + # Write a share using the mutable writer, and make sure that the + # reader knows how to read everything back to us. + d = defer.succeed(None) + for i in xrange(6): + d.addCallback(lambda ignored, i=i: + mw.put_block(self.block, i, self.salt)) + d.addCallback(lambda ignored: + mw.put_encprivkey(self.encprivkey)) + d.addCallback(lambda ignored: + mw.put_blockhashes(self.block_hash_tree)) + d.addCallback(lambda ignored: + mw.put_sharehashes(self.share_hash_chain)) + d.addCallback(lambda ignored: + mw.put_root_and_salt_hashes(self.root_hash, self.salt_hash)) + d.addCallback(lambda ignored: + mw.put_signature(self.signature)) + d.addCallback(lambda ignored: + mw.put_verification_key(self.verification_key)) + d.addCallback(lambda ignored: + mw.finish_publishing()) + + mr = MDMFSlotReadProxy(self.rref, self.secrets, "si1", 0) + def _check_block_and_salt((block, salt)): + self.failUnlessEqual(block, self.block) + self.failUnlessEqual(salt, self.salt) + + for i in xrange(6): + d.addCallback(lambda ignored, i=i: + mr.get_block_and_salt(i)) + d.addCallback(_check_block_and_salt) + + d.addCallback(lambda ignored: + mr.get_encprivkey()) + d.addCallback(lambda encprivkey: + self.failUnlessEqual(self.encprivkey, encprivkey)) + + d.addCallback(lambda ignored: + mr.get_blockhashes()) + d.addCallback(lambda blockhashes: + self.failUnlessEqual(self.block_hash_tree, blockhashes)) + + d.addCallback(lambda ignored: + mr.get_sharehashes()) + d.addCallback(lambda sharehashes: + self.failUnlessEqual(self.share_hash_chain, sharehashes)) + + d.addCallback(lambda ignored: + mr.get_signature()) + d.addCallback(lambda signature: + self.failUnlessEqual(signature, self.signature)) + + d.addCallback(lambda ignored: + mr.get_verification_key()) + d.addCallback(lambda verification_key: + self.failUnlessEqual(verification_key, self.verification_key)) + + d.addCallback(lambda ignored: + mr.get_seqnum()) + d.addCallback(lambda seqnum: + self.failUnlessEqual(seqnum, 0)) + + d.addCallback(lambda ignored: + mr.get_root_hash()) + d.addCallback(lambda root_hash: + self.failUnlessEqual(self.root_hash, root_hash)) + + d.addCallback(lambda ignored: + mr.get_salt_hash()) + d.addCallback(lambda salt_hash: + self.failUnlessEqual(self.salt_hash, salt_hash)) + + d.addCallback(lambda ignored: + mr.get_encoding_parameters()) + def _check_encoding_parameters((k, n, segsize, datalen)): + self.failUnlessEqual(k, 3) + self.failUnlessEqual(n, 10) + self.failUnlessEqual(segsize, 6) + self.failUnlessEqual(datalen, 36) + d.addCallback(_check_encoding_parameters) + + d.addCallback(lambda ignored: + mr.get_checkstring()) + d.addCallback(lambda checkstring: + self.failUnlessEqual(checkstring, mw.get_checkstring())) + return d + + class Stats(unittest.TestCase): def setUp(self): } [Add objects for MDMF shares in support of a new segmented uploader Kevan Carstensen **20100531231009 Ignore-this: 7e811139c2cd20df744f0ed0c3cb4ce8 ] { hunk ./src/allmydata/interfaces.py 7 ChoiceOf, IntegerConstraint, Any, RemoteInterface, Referenceable HASH_SIZE=32 +SALT_SIZE=16 Hash = StringConstraint(maxLength=HASH_SIZE, minLength=HASH_SIZE)# binary format 32-byte SHA256 hash hunk ./src/allmydata/mutable/layout.py 4 import struct from common import NeedMoreDataError, UnknownVersionError +from twisted.python import failure +from allmydata.interfaces import HASH_SIZE, SALT_SIZE +from allmydata.util import mathutil +from twisted.internet import defer + +# These strings describe the format of the packed structs they help process +# Here's what they mean: +# +# PREFIX: +# >: Big-endian byte order; the most significant byte is first (leftmost). +# B: The version information; an 8 bit version identifier. Stored as +# an unsigned char. This is currently 00 00 00 00; our modifications +# will turn it into 00 00 00 01. +# Q: The sequence number; this is sort of like a revision history for +# mutable files; they start at 1 and increase as they are changed after +# being uploaded. Stored as an unsigned long long, which is 8 bytes in +# length. +# 32s: The root hash of the share hash tree. We use sha-256d, so we use 32 +# characters = 32 bytes to store the value. +# 16s: The salt for the readkey. This is a 16-byte random value, stored as +# 16 characters. +# +# SIGNED_PREFIX additions, things that are covered by the signature: +# B: The "k" encoding parameter. We store this as an 8-bit character, +# which is convenient because our erasure coding scheme cannot +# encode if you ask for more than 255 pieces. +# B: The "N" encoding parameter. Stored as an 8-bit character for the +# same reasons as above. +# Q: The segment size of the uploaded file. This will essentially be the +# length of the file in SDMF. An unsigned long long, so we can store +# files of quite large size. +# Q: The data length of the uploaded file. Modulo padding, this will be +# the same of the data length field. Like the data length field, it is +# an unsigned long long and can be quite large. +# +# HEADER additions: +# L: The offset of the signature of this. An unsigned long. +# L: The offset of the share hash chain. An unsigned long. +# L: The offset of the block hash tree. An unsigned long. +# L: The offset of the share data. An unsigned long. +# Q: The offset of the encrypted private key. An unsigned long long, to +# account for the possibility of a lot of share data. +# Q: The offset of the EOF. An unsigned long long, to account for the +# possibility of a lot of share data. +# +# After all of these, we have the following: +# - The verification key: Occupies the space between the end of the header +# and the start of the signature (i.e.: data[HEADER_LENGTH:o['signature']]. +# - The signature, which goes from the signature offset to the share hash +# chain offset. +# - The share hash chain, which goes from the share hash chain offset to +# the block hash tree offset. +# - The share data, which goes from the share data offset to the encrypted +# private key offset. +# - The encrypted private key offset, which goes until the end of the file. +# +# The block hash tree in this encoding has only one share, so the offset of +# the share data will be 32 bits more than the offset of the block hash tree. +# Given this, we may need to check to see how many bytes a reasonably sized +# block hash tree will take up. PREFIX = ">BQ32s16s" # each version has a different prefix SIGNED_PREFIX = ">BQ32s16s BBQQ" # this is covered by the signature hunk ./src/allmydata/mutable/layout.py 264 encprivkey]) return final_share +def pack_prefix(seqnum, root_hash, IV, + required_shares, total_shares, + segment_size, data_length): + prefix = struct.pack(SIGNED_PREFIX, + 0, # version, + seqnum, + root_hash, + IV, + required_shares, + total_shares, + segment_size, + data_length, + ) + return prefix + + +MDMFHEADER = ">BQ32s32sBBQQ LQQQQQQ" +MDMFHEADERSIZE = struct.calcsize(MDMFHEADER) +MDMFCHECKSTRING = ">BQ32s32s" +MDMFSIGNABLEHEADER = ">BQ32s32sBBQQ" +MDMFOFFSETS = ">LQQQQQQ" + +class MDMFSlotWriteProxy: + #implements(IMutableSlotWriter) TODO + + """ + I represent a remote write slot for an MDMF mutable file. + + I abstract away from my caller the details of block and salt + management, and the implementation of the on-disk format for MDMF + shares. + """ + + # Expected layout, MDMF: + # offset: size: name: + #-- signed part -- + # 0 1 version number (01) + # 1 8 sequence number + # 9 32 share tree root hash + # 41 32 concatenated salts hash + # 73 1 The "k" encoding parameter + # 74 1 The "N" encoding parameter + # 75 8 The segment size of the uploaded file + # 83 8 The data length of the uploaded file + #-- end signed part -- + # 91 4 The offset of the share data + # 95 8 The offset of the encrypted private key + # 103 8 The offset of the block hash tree + # 111 8 The offset of the signature hash chain + # 119 8 The offset of the signature + # 127 8 The offset of the verification key + # 135 8 offset of the EOF + # + # followed by salts, share data, the encrypted private key, the + # block hash tree, the share hash chain, a signature over the first + # eight fields, and a verification key. + # + # The checkstring is the first four fields -- the version number, + # sequence number, root hash and salt hash. This is consistent in + # meaning to what we have with SDMF files, except now instead of + # using the literal salt, we use a value derived from all of the + # salts. + # + # The ordering of the offsets is different to reflect the dependencies + # that we'll run into with an MDMF file. The expected write flow is + # something like this: + # + # 0: Initialize with the sequence number, encoding + # parameters and data length. From this, we can deduce the + # number of segments, and from that we can deduce the size of + # the AES salt field, telling us where to write AES salts, and + # where to write share data. We can also figure out where the + # encrypted private key should go, because we can figure out + # how big the share data will be. + # + # 1: Encrypt, encode, and upload the file in chunks. Do something + # like + # + # put_block(data, segnum, salt) + # + # to write a block and a salt to the disk. We can do both of + # these operations now because we have enough of the offsets to + # know where to put them. + # + # 2: Put the encrypted private key. Use: + # + # put_encprivkey(encprivkey) + # + # Now that we know the length of the private key, we can fill + # in the offset for the block hash tree. + # + # 3: We're now in a position to upload the block hash tree for + # a share. Put that using something like: + # + # put_blockhashes(block_hash_tree) + # + # Note that block_hash_tree is a list of hashes -- we'll take + # care of the details of serializing that appropriately. When + # we get the block hash tree, we are also in a position to + # calculate the offset for the share hash chain, and fill that + # into the offsets table. + # + # 4: We're now in a position to upload the share hash chain for + # a share. Do that with something like: + # + # put_sharehashes(share_hash_chain) + # + # share_hash_chain should be a dictionary mapping shnums to + # 32-byte hashes -- the wrapper handles serialization. + # We'll know where to put the signature at this point, also, + # but, for various reasons, will not allow clients to do that + # until after they've put the flat salt hash and the root hash + # in the next step. + # + # 5: Put the root hash and the flat salt hash. Use: + # + # put_root_and_salt_hashes(root_hash, salt_hash) + # + # These must both be 32-byte values. Since they have fixed + # offsets in the header, we could conceivably put them whenever + # we want to, but it makes sense enough to put them only after + # putting the share hash chain, since having a root hash + # implies that we have a share hash chain. + # + # After this step, callers can call my get_signable method, + # which returns a packed representation of the data that they + # need to sign for the signature field, which is the next one + # to be placed. + # + # 5: With the root hash put, we can now sign the header. Use: + # + # put_signature(signature) + # + # 6: Add the verification key, and finish. Do: + # + # put_verification_key(key) + # + # and + # + # finish_publish() + # + # Checkstring management: + # + # To write to a mutable slot, we have to provide test vectors to ensure + # that we are writing to the same data that we think we are. These + # vectors allow us to detect uncoordinated writes; that is, writes + # where both we and some other shareholder are writing to the + # mutable slot, and to report those back to the parts of the program + # doing the writing. + # + # With SDMF, this was easy -- all of the share data was written in + # one go, so it was easy to detect uncoordinated writes, and we only + # had to do it once. With MDMF, not all of the file is written at + # once. + # + # If a share is new, we write out as much of the header as we can + # before writing out anything else. This gives other writers a + # canary that they can use to detect uncoordinated writes, and, if + # they do the same thing, gives us the same canary. We them update + # the share. We won't be able to write out two fields of the header + # -- the share tree hash and the salt hash -- until we finish + # writing out the share. We only require the writer to provide the + # initial checkstring, and keep track of what it should be after + # updates ourselves. + # + # If we haven't written anything yet, then on the first write (which + # will probably be a block + salt of a share), we'll also write out + # the header. On subsequent passes, we'll expect to see the header. + # This changes in two places: + # + # - When we write out the salt hash + # - When we write out the root of the share hash tree + # + # since these values will change the header. It is possible that we + # can just make those be written in one operation to minimize + # disruption. + def __init__(self, + shnum, + rref, # a remote reference to a storage server + storage_index, + secrets, # (write_enabler, renew_secret, cancel_secret) + seqnum, # the sequence number of the mutable file + required_shares, + total_shares, + segment_size, + data_length): # the length of the original file + self._shnum = shnum + self._rref = rref + self._storage_index = storage_index + self._seqnum = seqnum + self._required_shares = required_shares + assert self._shnum >= 0 and self._shnum < total_shares + self._total_shares = total_shares + # We build up the offset table as we write things. It is the + # last thing we write to the remote server. + self._offsets = {} + self._testvs = [] + self._secrets = secrets + # The segment size needs to be a multiple of the k parameter -- + # any padding should have been carried out by the publisher + # already. + assert segment_size % required_shares == 0 + self._segment_size = segment_size + self._data_length = data_length + + # These are set later -- we define them here so that we can + # check for their existence easily + self._root_hash = None + self._salt_hash = None + + # We haven't yet written anything to the remote bucket. By + # setting this, we tell the _write method as much. The write + # method will then know that it also needs to add a write vector + # for the checkstring (or what we have of it) to the first write + # request. We'll then record that value for future use. If + # we're expecting something to be there already, we need to call + # set_checkstring before we write anything to tell the first + # write about that. + self._written = False + + # When writing data to the storage servers, we get a read vector + # for free. We'll read the checkstring, which will help us + # figure out what's gone wrong if a write fails. + self._readv = [(0, struct.calcsize(MDMFCHECKSTRING))] + + # We calculate the number of segments because it tells us + # where the salt part of the file ends/share segment begins, + # and also because it provides a useful amount of bounds checking. + self._num_segments = mathutil.div_ceil(self._data_length, + self._segment_size) + self._block_size = self._segment_size / self._required_shares + # We also calculate the share size, to help us with block + # constraints later. + tail_size = self._data_length % self._segment_size + if not tail_size: + self._tail_block_size = self._block_size + else: + self._tail_block_size = mathutil.next_multiple(tail_size, + self._required_shares) + self._tail_block_size /= self._required_shares + + # We already know where the AES salts start; right after the end + # of the header (which is defined as the signable part + the offsets) + # We need to calculate where the share data starts, since we're + # responsible (after this method) for being able to write it. + self._offsets['share-data'] = MDMFHEADERSIZE + self._offsets['share-data'] += self._num_segments * SALT_SIZE + # We can also calculate where the encrypted private key begins + # from what we know know. + self._offsets['enc_privkey'] = self._offsets['share-data'] + self._offsets['enc_privkey'] += self._block_size * self._num_segments + # We'll wait for the rest. Callers can now call my "put_block" and + # "set_checkstring" methods. + + + def set_checkstring(self, checkstring): + """ + Set checkstring checkstring for the given shnum. + + By default, I assume that I am writing new shares to the grid. + If you don't explcitly set your own checkstring, I will use + one that requires that the remote share not exist. + """ + # You're allowed to overwrite checkstrings with this method; + # I assume that users know what they are doing when they call + # it. + if checkstring == "": + # We special-case this, since len("") = 0, but we need + # length of 1 for the case of an empty share to work on the + # storage server, which is what a checkstring that is the + # empty string means. + self._testvs = [] + else: + self._testvs = [] + self._testvs.append((0, len(checkstring), "eq", checkstring)) + + + def get_checkstring(self): + """ + Given a share number, I return a representation of what the + checkstring for that share on the server will look like. + """ + if self._root_hash: + roothash = self._root_hash + else: + roothash = "\x00" * 32 + if self._salt_hash: + salthash = self._salt_hash + else: + salthash = "\x00" * 32 + checkstring = struct.pack(MDMFCHECKSTRING, + 1, + self._seqnum, + roothash, + salthash) + return checkstring + + + def put_block(self, data, segnum, salt): + """ + Put the encrypted-and-encoded data segment in the slot, along + with the salt. + """ + if segnum >= self._num_segments: + raise LayoutInvalid("I won't overwrite the private key") + if len(salt) != SALT_SIZE: + raise LayoutInvalid("I was given a salt of size %d, but " + "I wanted a salt of size %d") + if segnum + 1 == self._num_segments: + if len(data) != self._tail_block_size: + raise LayoutInvalid("I was given the wrong size block to write") + elif len(data) != self._block_size: + raise LayoutInvalid("I was given the wrong size block to write") + + # We want to write at offsets['share-data'] + segnum * block_size. + assert self._offsets + assert self._offsets['share-data'] + + offset = self._offsets['share-data'] + segnum * self._block_size + datavs = [tuple([offset, data])] + # We also have to write the salt. This is at: + salt_offset = MDMFHEADERSIZE + SALT_SIZE * segnum + datavs.append(tuple([salt_offset, salt])) + return self._write(datavs) + + + def put_encprivkey(self, encprivkey): + """ + Put the encrypted private key in the remote slot. + """ + assert self._offsets + assert self._offsets['enc_privkey'] + # You shouldn't re-write the encprivkey after the block hash + # tree is written, since that could cause the private key to run + # into the block hash tree. Before it writes the block hash + # tree, the block hash tree writing method writes the offset of + # the signature hash chain. So that's a good indicator of + # whether or not the block hash tree has been written. + if "share_hash_chain" in self._offsets: + raise LayoutInvalid("You must write this before the block hash tree") + + self._offsets['block_hash_tree'] = self._offsets['enc_privkey'] + len(encprivkey) + datavs = [(tuple([self._offsets['enc_privkey'], encprivkey]))] + def _on_failure(): + del(self._offsets['block_hash_tree']) + return self._write(datavs, on_failure=_on_failure) + + + def put_blockhashes(self, blockhashes): + """ + Put the block hash tree in the remote slot. + + The encrypted private key must be put before the block hash + tree, since we need to know how large it is to know where the + block hash tree should go. The block hash tree must be put + before the share hash chain, since its size determines the + offset of the share hash chain. + """ + assert self._offsets + assert isinstance(blockhashes, list) + if "block_hash_tree" not in self._offsets: + raise LayoutInvalid("You must put the encrypted private key " + "before you put the block hash tree") + # If written, the share hash chain causes the signature offset + # to be defined. + if "signature" in self._offsets: + raise LayoutInvalid("You must put the block hash tree before " + "you put the share hash chain") + blockhashes_s = "".join(blockhashes) + self._offsets['share_hash_chain'] = self._offsets['block_hash_tree'] + len(blockhashes_s) + datavs = [] + datavs.append(tuple([self._offsets['block_hash_tree'], blockhashes_s])) + def _on_failure(): + del(self._offsets['share_hash_chain']) + return self._write(datavs, on_failure=_on_failure) + + + def put_sharehashes(self, sharehashes): + """ + Put the share hash chain in the remote slot. + + The block hash tree must be put before the share hash chain, + since we need to know where the block hash tree ends before we + can know where the share hash chain starts. The share hash chain + must be put before the signature, since the length of the packed + share hash chain determines the offset of the signature. + """ + assert isinstance(sharehashes, dict) + if "share_hash_chain" not in self._offsets: + raise LayoutInvalid("You need to put the block hashes before " + "you can put the share hash chain") + # The signature comes after the share hash chain. If the + # signature has already been written, we must not write another + # share hash chain. The signature writes the verification key + # offset when it gets sent to the remote server, so we look for + # that. + if "verification_key" in self._offsets: + raise LayoutInvalid("You must write the share hash chain " + "before you write the signature") + datavs = [] + sharehashes_s = "".join([struct.pack(">H32s", i, sharehashes[i]) + for i in sorted(sharehashes.keys())]) + self._offsets['signature'] = self._offsets['share_hash_chain'] + len(sharehashes_s) + datavs.append(tuple([self._offsets['share_hash_chain'], sharehashes_s])) + def _on_failure(): + del(self._offsets['signature']) + return self._write(datavs, on_failure=_on_failure) + + + def put_root_and_salt_hashes(self, roothash, salthash): + """ + Put the root hash (the root of the share hash tree) in the + remote slot. + """ + # It does not make sense to be able to put the root and salt + # hashes without first putting the share hashes, since you need + # the share hashes to generate the root hash. + # + # Signature is defined by the routine that places the share hash + # chain, so it's a good thing to look for in finding out whether + # or not the share hash chain exists on the remote server. + if "signature" not in self._offsets: + raise LayoutInvalid("You need to put the share hash chain " + "before you can put the root share hash") + if len(roothash) != HASH_SIZE or len(salthash) != HASH_SIZE: + raise LayoutInvalid("hashes and salts must be exactly %d bytes" + % HASH_SIZE) + datavs = [] + self._root_hash = roothash + self._salt_hash = salthash + checkstring = self.get_checkstring() + datavs.append(tuple([0, checkstring])) + # This write, if successful, changes the checkstring, so we need + # to update our internal checkstring to be consistent with the + # one on the server. + def _on_success(): + self._testvs = [(0, len(checkstring), "eq", checkstring)] + def _on_failure(): + self._root_hash = None + self._salt_hash = None + return self._write(datavs, + on_success=_on_success, + on_failure=_on_failure) + + + def get_signable(self): + """ + Get the first eight fields of the mutable file; the parts that + are signed. + """ + if not self._root_hash or not self._salt_hash: + raise LayoutInvalid("You need to set the root hash and the " + "salt hash before getting something to " + "sign") + return struct.pack(MDMFSIGNABLEHEADER, + 1, + self._seqnum, + self._root_hash, + self._salt_hash, + self._required_shares, + self._total_shares, + self._segment_size, + self._data_length) + + + def put_signature(self, signature): + """ + Put the signature field to the remote slot. + + I require that the root hash and share hash chain have been put + to the grid before I will write the signature to the grid. + """ + if "signature" not in self._offsets: + raise LayoutInvalid("You must put the share hash chain " + # It does not make sense to put a signature without first + # putting the root hash and the salt hash (since otherwise + # the signature would be incomplete), so we don't allow that. + "before putting the signature") + if not self._root_hash: + raise LayoutInvalid("You must complete the signed prefix " + "before computing a signature") + # If we put the signature after we put the verification key, we + # could end up running into the verification key, and will + # probably screw up the offsets as well. So we don't allow that. + # The method that writes the verification key defines the EOF + # offset before writing the verification key, so look for that. + if "EOF" in self._offsets: + raise LayoutInvalid("You must write the signature before the verification key") + + self._offsets['verification_key'] = self._offsets['signature'] + len(signature) + datavs = [] + datavs.append(tuple([self._offsets['signature'], signature])) + def _on_failure(): + del(self._offsets['verification_key']) + return self._write(datavs, on_failure=_on_failure) + + + def put_verification_key(self, verification_key): + """ + Put the verification key into the remote slot. + + I require that the signature have been written to the storage + server before I allow the verification key to be written to the + remote server. + """ + if "verification_key" not in self._offsets: + raise LayoutInvalid("You must put the signature before you " + "can put the verification key") + self._offsets['EOF'] = self._offsets['verification_key'] + len(verification_key) + datavs = [] + datavs.append(tuple([self._offsets['verification_key'], verification_key])) + def _on_failure(): + del(self._offsets['EOF']) + return self._write(datavs, on_failure=_on_failure) + + + def finish_publishing(self): + """ + Write the offset table and encoding parameters to the remote + slot, since that's the only thing we have yet to publish at this + point. + """ + if "EOF" not in self._offsets: + raise LayoutInvalid("You must put the verification key before " + "you can publish the offsets") + offsets_offset = struct.calcsize(MDMFSIGNABLEHEADER) + offsets = struct.pack(MDMFOFFSETS, + self._offsets['share-data'], + self._offsets['enc_privkey'], + self._offsets['block_hash_tree'], + self._offsets['share_hash_chain'], + self._offsets['signature'], + self._offsets['verification_key'], + self._offsets['EOF']) + datavs = [] + datavs.append(tuple([offsets_offset, offsets])) + encoding_parameters_offset = struct.calcsize(MDMFCHECKSTRING) + params = struct.pack(">BBQQ", + self._required_shares, + self._total_shares, + self._num_segments, + self._data_length) + datavs.append(tuple([encoding_parameters_offset, params])) + return self._write(datavs) + + + def _write(self, datavs, on_failure=None, on_success=None): + """I write the data vectors in datavs to the remote slot.""" + tw_vectors = {} + new_share = False + if not self._testvs: + self._testvs = [] + self._testvs.append(tuple([0, 1, "eq", ""])) + new_share = True + if not self._written: + # Write a new checkstring to the share when we write it, so + # that we have something to check later. + new_checkstring = self.get_checkstring() + datavs.append((0, new_checkstring)) + def _first_write(): + self._written = True + self._testvs = [(0, len(new_checkstring), "eq", new_checkstring)] + on_success = _first_write + tw_vectors[self._shnum] = (self._testvs, datavs, None) + d = self._rref.callRemote("slot_testv_and_readv_and_writev", + self._storage_index, + self._secrets, + tw_vectors, + self._readv) + def _result(results): + if isinstance(results, failure.Failure) or not results[0]: + # Do nothing; the write was unsuccessful. + if on_failure: + on_failure() + else: + if on_success: + on_success() + return results + d.addCallback(_result) + return d + + +class MDMFSlotReadProxy: + """ + I read from a mutable slot filled with data written in the MDMF data + format (which is described above). + """ + def __init__(self, + rref, + secrets, + storage_index, + shnum): + # Start the initialization process. + self._rref = rref + self._storage_index = storage_index + self._shnum = shnum + self._secrets = secrets + + # Before doing anything, the reader is probably going to want to + # verify that the signature is correct. To do that, they'll need + # the verification key, and the signature. To get those, we'll + # need the offset table. So fetch the offset table on the + # assumption that that will be the first thing that a reader is + # going to do. + d = self._fetch_offsets() + + # The fact that these encoding parameters are None tells us + # that we haven't yet fetched them from the remote share, so we + # should. We could just not set them, but the checks will be + # easier to read if we don't have to use hasattr. + self._version_number = None + self._sequence_number = None + self._root_hash = None + self._salt_hash = None + self._required_shares = None + self._total_shares = None + self._segment_size = None + self._data_length = None + + + def _fetch_offsets(self): + """ + I fetch the offset table from the remote slot. + """ + # The offset table starts at 91 + readv = (91, struct.calcsize(MDMFOFFSETS)) + readvs = [readv] + d = self._read(readvs) + def _set_offsets(data): + assert self._shnum in data + offsets = data[self._shnum][0] + + (share_data, + encprivkey, + blockhashes, + sharehashes, + signature, + verification_key, + eof) = struct.unpack(MDMFOFFSETS, offsets) + self._offsets = {} + self._offsets['share_data'] = share_data + self._offsets['enc_privkey'] = encprivkey + self._offsets['block_hash_tree'] = blockhashes + self._offsets['share_hash_chain'] = sharehashes + self._offsets['signature'] = signature + self._offsets['verification_key'] = verification_key + self._offsets['EOF'] = eof + d.addCallback(_set_offsets) + return d + + + def _fetch_header_without_offsets(self): + """ + I fetch the part of the header that isn't the offsets. + + I am called after the reader has verified that the signature is + correct -- at that point, the reader will want to retrieve the + rest of the file, and will need those parameters. + """ + readvs = [(0, 91)] + d = self._read(readvs) + def _set_encoding_parameters(data): + assert self._shnum in data + encoding_parameters = data[self._shnum][0] + (verno, + seqnum, + root_hash, + salt_hash, + k, + n, + segsize, + datalen) = struct.unpack(MDMFSIGNABLEHEADER, + encoding_parameters) + + self._version_number = verno + self._sequence_number = seqnum + self._root_hash = root_hash + self._salt_hash = salt_hash + self._required_shares = k + self._total_shares = n + self._segment_size = segsize + self._data_length = datalen + + self._num_segments = mathutil.div_ceil(self._data_length, + self._segment_size) + self._block_size = self._segment_size / self._required_shares + tail_size = self._data_length % self._segment_size + if not tail_size: + self._tail_block_size = self._block_size + else: + self._tail_block_size = mathutil.next_multiple(tail_size, + self._required_shares) + self._tail_block_size /= self._required_shares + + d.addCallback(_set_encoding_parameters) + return d + + + def get_block_and_salt(self, segnum): + """ + I return (block, salt), where block is the block data and + salt is the salt used to encrypt that segment. + """ + assert self._offsets, "I need offsets to get share data" + + if not self._segment_size: + d = self._fetch_header_without_offsets() + else: + d = defer.succeed(None) + base_salt_offset = struct.calcsize(MDMFHEADER) + base_share_offset = self._offsets['share_data'] + salt_offset = base_salt_offset + SALT_SIZE * segnum + + def _calculate_share_offset(ignored): + if segnum + 1 > self._num_segments: + raise LayoutInvalid("Not a valid segment number") + + share_offset = base_share_offset + self._block_size * segnum + if segnum + 1 == self._num_segments: + data = self._tail_block_size + else: + data = self._block_size + readvs = [(salt_offset, SALT_SIZE), (share_offset, data)] + return readvs + + d.addCallback(_calculate_share_offset) + d.addCallback(lambda readvs: + self._read(readvs)) + def _process_results(results): + assert self._shnum in results + salt, data = results[self._shnum] + return data, salt + d.addCallback(_process_results) + return d + + + def get_blockhashes(self): + """ + I return the block hash tree + """ + # TODO: Return only the parts of the block hash tree necessary + # to validate the blocknum provided? + assert self._offsets + + blockhashes_offset = self._offsets['block_hash_tree'] + blockhashes_length = self._offsets['share_hash_chain'] - blockhashes_offset + readvs = [(blockhashes_offset, blockhashes_length)] + d = self._read(readvs) + def _build_block_hash_tree(results): + assert self._shnum in results + + rawhashes = results[self._shnum][0] + results = [rawhashes[i:i+HASH_SIZE] + for i in range(0, len(rawhashes), HASH_SIZE)] + return results + d.addCallback(_build_block_hash_tree) + return d + + + def get_sharehashes(self): + """ + I return the part of the share hash chain placed to validate + this share. + """ + assert self._offsets + + sharehashes_offset = self._offsets['share_hash_chain'] + sharehashes_length = self._offsets['signature'] - sharehashes_offset + + readvs = [(sharehashes_offset, sharehashes_length)] + d = self._read(readvs) + def _build_share_hash_chain(results): + assert self._shnum in results + + sharehashes = results[self._shnum][0] + results = [sharehashes[i:i+(HASH_SIZE + 2)] + for i in range(0, len(sharehashes), HASH_SIZE + 2)] + results = dict([struct.unpack(">H32s", data) + for data in results]) + return results + d.addCallback(_build_share_hash_chain) + return d + + + def get_encprivkey(self): + """ + I return the encrypted private key. + """ + assert self._offsets + + privkey_offset = self._offsets['enc_privkey'] + privkey_length = self._offsets['block_hash_tree'] - privkey_offset + readvs = [(privkey_offset, privkey_length)] + d = self._read(readvs) + def _process_results(results): + assert self._shnum in results + privkey = results[self._shnum][0] + return privkey + d.addCallback(_process_results) + return d + + + def get_signature(self): + """ + I return the signature of my share. + """ + assert self._offsets + + signature_offset = self._offsets['signature'] + signature_length = self._offsets['verification_key'] - signature_offset + readvs = [(signature_offset, signature_length)] + d = self._read(readvs) + def _process_results(results): + assert self._shnum in results + signature = results[self._shnum][0] + return signature + d.addCallback(_process_results) + return d + + + def get_verification_key(self): + """ + I return the verification key. + """ + assert self._offsets + vk_offset = self._offsets['verification_key'] + vk_length = self._offsets['EOF'] - vk_offset + readvs = [(vk_offset, vk_length)] + d = self._read(readvs) + def _process_results(results): + assert self._shnum in results + verification_key = results[self._shnum][0] + return verification_key + d.addCallback(_process_results) + return d + + + def get_encoding_parameters(self): + """ + I return (k, n, segsize, datalen) + """ + if not self._required_shares: + d = self._fetch_header_without_offsets() + else: + d = defer.succeed(None) + d.addCallback(lambda ignored: + (self._required_shares, + self._total_shares, + self._segment_size, + self._data_length)) + return d + + + def get_seqnum(self): + """ + I return the sequence number for this share. + """ + if self._sequence_number == None: + d = self._fetch_header_without_offsets() + else: + d = defer.succeed(None) + d.addCallback(lambda ignored: + self._sequence_number) + return d + + + def get_root_hash(self): + """ + I return the root of the block hash tree + """ + if not self._root_hash: + d = self._fetch_header_without_offsets() + else: + d = defer.succeed(None) + d.addCallback(lambda ignored: self._root_hash) + return d + + + def get_salt_hash(self): + """ + I return the flat salt hash + """ + if not self._salt_hash: + d = self._fetch_header_without_offsets() + else: + d = defer.succeed(None) + d.addCallback(lambda ignored: self._salt_hash) + return d + + + def get_checkstring(self): + """ + I return the packed representation of the following: + + - version number + - sequence number + - root hash + - salt hash + + which my users use as a checkstring to detect other writers. + """ + if self._version_number == None: + d = self._fetch_header_without_offsets() + else: + d = defer.succeed(None) + def _build_checkstring(ignored): + checkstring = struct.pack(MDMFCHECKSTRING, + self._version_number, + self._sequence_number, + self._root_hash, + self._salt_hash) + return checkstring + d.addCallback(_build_checkstring) + return d + + + def _read(self, readvs): + d = self._rref.callRemote("slot_readv", + self._storage_index, + [self._shnum], + readvs) + return d + + +class LayoutInvalid(Exception): + """ + This isn't a valid MDMF mutable file + """ } Context: [Suppress deprecation warning for twisted.web.error.NoResource when using Twisted >= 9.0.0. david-sarah@jacaranda.org**20100516205625 Ignore-this: 2361a3023cd3db86bde5e1af759ed01 ] [docs: CREDITS for Jeremy Visser zooko@zooko.com**20100524081829 Ignore-this: d7c1465fd8d4e25b8d46d38a1793465b ] [test: show stdout and stderr in case of non-zero exit code from "tahoe" command zooko@zooko.com**20100524073348 Ignore-this: 695e81cd6683f4520229d108846cd551 ] [setup: upgrade bundled zetuptoolz to zetuptoolz-0.6c15dev and make it unpacked and directly loaded by setup.py zooko@zooko.com**20100523205228 Ignore-this: 24fb32aaee3904115a93d1762f132c7 Also fix the relevant "make clean" target behavior. ] [setup: remove bundled zipfile egg of setuptools zooko@zooko.com**20100523205120 Ignore-this: c68b5f2635bb93d1c1fa7b613a026f9e We're about to replace it with bundled unpacked source code of setuptools, which is much nicer for debugging and evolving under revision control. ] [setup: remove bundled copy of setuptools_trial-0.5.2.tar zooko@zooko.com**20100522221539 Ignore-this: 140f90eb8fb751a509029c4b24afe647 Hopefully it will get installed automatically as needed and we won't bundle it anymore. ] [setup: remove bundled setuptools_darcs-1.2.8.tar zooko@zooko.com**20100522015333 Ignore-this: 378b1964b513ae7fe22bae2d3478285d This version of setuptools_darcs had a bug when used on Windows which has been fixed in setuptools_darcs-1.2.9. Hopefully we will not need to bundle a copy of setuptools_darcs-1.2.9 in with Tahoe-LAFS and can instead rely on it to be downloaded from PyPI or bundled in the "tahoe deps" separate tarball. ] [tests: fix pyflakes warnings in bench_dirnode.py zooko@zooko.com**20100521202511 Ignore-this: f23d55b4ed05e52865032c65a15753c4 ] [setup: if the string '--reporter=bwverbose-coverage' appears on sys.argv then you need trialcoverage zooko@zooko.com**20100521122226 Ignore-this: e760c45dcfb5a43c1dc1e8a27346bdc2 ] [tests: don't let bench_dirnode.py do stuff and have side-effects at import time (unless __name__ == '__main__') zooko@zooko.com**20100521122052 Ignore-this: 96144a412250d9bbb5fccbf83b8753b8 ] [tests: increase timeout to give François's ARM buildslave a chance to complete the tests zooko@zooko.com**20100520134526 Ignore-this: 3dd399fdc8b91149c82b52f955b50833 ] [run_trial.darcspath freestorm77@gmail.com**20100510232829 Ignore-this: 5ebb4df74e9ea8a4bdb22b65373d1ff2 ] [docs: line-wrap README.txt zooko@zooko.com**20100518174240 Ignore-this: 670a02d360df7de51ebdcf4fae752577 ] [Hush pyflakes warnings Kevan Carstensen **20100515184344 Ignore-this: fd602c3bba115057770715c36a87b400 ] [setup: new improved misc/show-tool-versions.py zooko@zooko.com**20100516050122 Ignore-this: ce9b1de1b35b07d733e6cf823b66335a ] [Improve code coverage of the Tahoe2PeerSelector tests. Kevan Carstensen **20100515032913 Ignore-this: 793151b63ffa65fdae6915db22d9924a ] [Remove a comment that no longer makes sense. Kevan Carstensen **20100514203516 Ignore-this: 956983c7e7c7e4477215494dfce8f058 ] [docs: update docs/architecture.txt to more fully and correctly explain the upload procedure zooko@zooko.com**20100514043458 Ignore-this: 538b6ea256a49fed837500342092efa3 ] [Fix up the behavior of #778, per reviewers' comments Kevan Carstensen **20100514004917 Ignore-this: 9c20b60716125278b5456e8feb396bff - Make some important utility functions clearer and more thoroughly documented. - Assert in upload.servers_of_happiness that the buckets attributes of PeerTrackers passed to it are mutually disjoint. - Get rid of some silly non-Pythonisms that I didn't see when I first wrote these patches. - Make sure that should_add_server returns true when queried about a shnum that it doesn't know about yet. - Change Tahoe2PeerSelector.preexisting_shares to map a shareid to a set of peerids, alter dependencies to deal with that. - Remove upload.should_add_servers, because it is no longer necessary - Move upload.shares_of_happiness and upload.shares_by_server to a utility file. - Change some points in Tahoe2PeerSelector. - Compute servers_of_happiness using a bipartite matching algorithm that we know is optimal instead of an ad-hoc greedy algorithm that isn't. - Change servers_of_happiness to just take a sharemap as an argument, change its callers to merge existing_shares and used_peers before calling it. - Change an error message in the encoder to be more appropriate for servers of happiness. - Clarify the wording of an error message in immutable/upload.py - Refactor a happiness failure message to happinessutil.py, and make immutable/upload.py and immutable/encode.py use it. - Move the word "only" as far to the right as possible in failure messages. - Use a better definition of progress during peer selection. - Do read-only peer share detection queries in parallel, not sequentially. - Clean up logging semantics; print the query statistics whenever an upload is unsuccessful, not just in one case. ] [Alter the error message when an upload fails, per some comments in #778. Kevan Carstensen **20091230210344 Ignore-this: ba97422b2f9737c46abeb828727beb1 When I first implemented #778, I just altered the error messages to refer to servers where they referred to shares. The resulting error messages weren't very good. These are a bit better. ] [Change "UploadHappinessError" to "UploadUnhappinessError" Kevan Carstensen **20091205043037 Ignore-this: 236b64ab19836854af4993bb5c1b221a ] [Alter the error message returned when peer selection fails Kevan Carstensen **20091123002405 Ignore-this: b2a7dc163edcab8d9613bfd6907e5166 The Tahoe2PeerSelector returned either NoSharesError or NotEnoughSharesError for a variety of error conditions that weren't informatively described by them. This patch creates a new error, UploadHappinessError, replaces uses of NoSharesError and NotEnoughSharesError with it, and alters the error message raised with the errors to be more in line with the new servers_of_happiness behavior. See ticket #834 for more information. ] [Eliminate overcounting iof servers_of_happiness in Tahoe2PeerSelector; also reorganize some things. Kevan Carstensen **20091118014542 Ignore-this: a6cb032cbff74f4f9d4238faebd99868 ] [Change stray "shares_of_happiness" to "servers_of_happiness" Kevan Carstensen **20091116212459 Ignore-this: 1c971ba8c3c4d2e7ba9f020577b28b73 ] [Alter Tahoe2PeerSelector to make sure that it recognizes existing shares on readonly servers, fixing an issue in #778 Kevan Carstensen **20091116192805 Ignore-this: 15289f4d709e03851ed0587b286fd955 ] [Alter 'immutable/encode.py' and 'immutable/upload.py' to use servers_of_happiness instead of shares_of_happiness. Kevan Carstensen **20091104111222 Ignore-this: abb3283314820a8bbf9b5d0cbfbb57c8 ] [Alter the signature of set_shareholders in IEncoder to add a 'servermap' parameter, which gives IEncoders enough information to perform a sane check for servers_of_happiness. Kevan Carstensen **20091104033241 Ignore-this: b3a6649a8ac66431beca1026a31fed94 ] [Alter CiphertextDownloader to work with servers_of_happiness Kevan Carstensen **20090924041932 Ignore-this: e81edccf0308c2d3bedbc4cf217da197 ] [Revisions of the #778 tests, per reviewers' comments Kevan Carstensen **20100514012542 Ignore-this: 735bbc7f663dce633caeb3b66a53cf6e - Fix comments and confusing naming. - Add tests for the new error messages suggested by David-Sarah and Zooko. - Alter existing tests for new error messages. - Make sure that the tests continue to work with the trunk. - Add a test for a mutual disjointedness assertion that I added to upload.servers_of_happiness. - Fix the comments to correctly reflect read-onlyness - Add a test for an edge case in should_add_server - Add an assertion to make sure that share redistribution works as it should - Alter tests to work with revised servers_of_happiness semantics - Remove tests for should_add_server, since that function no longer exists. - Alter tests to know about merge_peers, and to use it before calling servers_of_happiness. - Add tests for merge_peers. - Add Zooko's puzzles to the tests. - Edit encoding tests to expect the new kind of failure message. - Edit tests to expect error messages with the word "only" moved as far to the right as possible. - Extended and cleaned up some helper functions. - Changed some tests to call more appropriate helper functions. - Added a test for the failing redistribution algorithm - Added a test for the progress message - Added a test for the upper bound on readonly peer share discovery. ] [Alter various unit tests to work with the new happy behavior Kevan Carstensen **20100107181325 Ignore-this: 132032bbf865e63a079f869b663be34a ] [Replace "UploadHappinessError" with "UploadUnhappinessError" in tests. Kevan Carstensen **20091205043453 Ignore-this: 83f4bc50c697d21b5f4e2a4cd91862ca ] [Add tests for the behavior described in #834. Kevan Carstensen **20091123012008 Ignore-this: d8e0aa0f3f7965ce9b5cea843c6d6f9f ] [Re-work 'test_upload.py' to be more readable; add more tests for #778 Kevan Carstensen **20091116192334 Ignore-this: 7e8565f92fe51dece5ae28daf442d659 ] [Test Tahoe2PeerSelector to make sure that it recognizeses existing shares on readonly servers Kevan Carstensen **20091109003735 Ignore-this: 12f9b4cff5752fca7ed32a6ebcff6446 ] [Add more tests for comment:53 in ticket #778 Kevan Carstensen **20091104112849 Ignore-this: 3bb2edd299a944cc9586e14d5d83ec8c ] [Add a test for upload.shares_by_server Kevan Carstensen **20091104111324 Ignore-this: f9802e82d6982a93e00f92e0b276f018 ] [Minor tweak to an existing test -- make the first server read-write, instead of read-only Kevan Carstensen **20091104034232 Ignore-this: a951a46c93f7f58dd44d93d8623b2aee ] [Alter tests to use the new form of set_shareholders Kevan Carstensen **20091104033602 Ignore-this: 3deac11fc831618d11441317463ef830 ] [Refactor some behavior into a mixin, and add tests for the behavior described in #778 "Kevan Carstensen" **20091030091908 Ignore-this: a6f9797057ca135579b249af3b2b66ac ] [Alter NoNetworkGrid to allow the creation of readonly servers for testing purposes. Kevan Carstensen **20091018013013 Ignore-this: e12cd7c4ddeb65305c5a7e08df57c754 ] [Update 'docs/architecture.txt' to reflect readonly share discovery kevan@isnotajoke.com**20100514003852 Ignore-this: 7ead71b34df3b1ecfdcfd3cb2882e4f9 ] [Alter the wording in docs/architecture.txt to more accurately describe the servers_of_happiness behavior. Kevan Carstensen **20100428002455 Ignore-this: 6eff7fa756858a1c6f73728d989544cc ] [Alter wording in 'interfaces.py' to be correct wrt #778 "Kevan Carstensen" **20091205034005 Ignore-this: c9913c700ac14e7a63569458b06980e0 ] [Update 'docs/configuration.txt' to reflect the servers_of_happiness behavior. Kevan Carstensen **20091205033813 Ignore-this: 5e1cb171f8239bfb5b565d73c75ac2b8 ] [Clarify quickstart instructions for installing pywin32 david-sarah@jacaranda.org**20100511180300 Ignore-this: d4668359673600d2acbc7cd8dd44b93c ] [web: add a simple test that you can load directory.xhtml zooko@zooko.com**20100510063729 Ignore-this: e49b25fa3c67b3c7a56c8b1ae01bb463 ] [setup: fix typos in misc/show-tool-versions.py zooko@zooko.com**20100510063615 Ignore-this: 2181b1303a0e288e7a9ebd4c4855628 ] [setup: show code-coverage tool versions in show-tools-versions.py zooko@zooko.com**20100510062955 Ignore-this: 4b4c68eb3780b762c8dbbd22b39df7cf ] [docs: update README, mv it to README.txt, update setup.py zooko@zooko.com**20100504094340 Ignore-this: 40e28ca36c299ea1fd12d3b91e5b421c ] [Dependency on Windmill test framework is not needed yet. david-sarah@jacaranda.org**20100504161043 Ignore-this: be088712bec650d4ef24766c0026ebc8 ] [tests: pass z to tar so that BSD tar will know to ungzip zooko@zooko.com**20100504090628 Ignore-this: 1339e493f255e8fc0b01b70478f23a09 ] [setup: update comments and URLs in setup.cfg zooko@zooko.com**20100504061653 Ignore-this: f97692807c74bcab56d33100c899f829 ] [setup: reorder and extend the show-tool-versions script, the better to glean information about our new buildslaves zooko@zooko.com**20100504045643 Ignore-this: 836084b56b8d4ee8f1de1f4efb706d36 ] [CLI: Support for https url in option --node-url Francois Deppierraz **20100430185609 Ignore-this: 1717176b4d27c877e6bc67a944d9bf34 This patch modifies the regular expression used for verifying of '--node-url' parameter. Support for accessing a Tahoe gateway over HTTPS was already present, thanks to Python's urllib. ] [backupdb.did_create_directory: use REPLACE INTO, not INSERT INTO + ignore error Brian Warner **20100428050803 Ignore-this: 1fca7b8f364a21ae413be8767161e32f This handles the case where we upload a new tahoe directory for a previously-processed local directory, possibly creating a new dircap (if the metadata had changed). Now we replace the old dirhash->dircap record. The previous behavior left the old record in place (with the old dircap and timestamps), so we'd never stop creating new directories and never converge on a null backup. ] ["tahoe webopen": add --info flag, to get ?t=info Brian Warner **20100424233003 Ignore-this: 126b0bb6db340fabacb623d295eb45fa Also fix some trailing whitespace. ] [docs: install.html http-equiv refresh to quickstart.html zooko@zooko.com**20100421165708 Ignore-this: 52b4b619f9dde5886ae2cd7f1f3b734b ] [docs: install.html -> quickstart.html zooko@zooko.com**20100421155757 Ignore-this: 6084e203909306bed93efb09d0e6181d It is not called "installing" because that implies that it is going to change the configuration of your operating system. It is not called "building" because that implies that you need developer tools like a compiler. Also I added a stern warning against looking at the "InstallDetails" wiki page, which I have renamed to "AdvancedInstall". ] [Fix another typo in tahoe_storagespace munin plugin david-sarah@jacaranda.org**20100416220935 Ignore-this: ad1f7aa66b554174f91dfb2b7a3ea5f3 ] [Add dependency on windmill >= 1.3 david-sarah@jacaranda.org**20100416190404 Ignore-this: 4437a7a464e92d6c9012926b18676211 ] [licensing: phrase the OpenSSL-exemption in the vocabulary of copyright instead of computer technology, and replicate the exemption from the GPL to the TGPPL zooko@zooko.com**20100414232521 Ignore-this: a5494b2f582a295544c6cad3f245e91 ] [munin-tahoe_storagespace freestorm77@gmail.com**20100221203626 Ignore-this: 14d6d6a587afe1f8883152bf2e46b4aa Plugin configuration rename ] [setup: add licensing declaration for setuptools (noticed by the FSF compliance folks) zooko@zooko.com**20100309184415 Ignore-this: 2dfa7d812d65fec7c72ddbf0de609ccb ] [setup: fix error in licensing declaration from Shawn Willden, as noted by the FSF compliance division zooko@zooko.com**20100309163736 Ignore-this: c0623d27e469799d86cabf67921a13f8 ] [CREDITS to Jacob Appelbaum zooko@zooko.com**20100304015616 Ignore-this: 70db493abbc23968fcc8db93f386ea54 ] [desert-island-build-with-proper-versions jacob@appelbaum.net**20100304013858] [docs: a few small edits to try to guide newcomers through the docs zooko@zooko.com**20100303231902 Ignore-this: a6aab44f5bf5ad97ea73e6976bc4042d These edits were suggested by my watching over Jake Appelbaum's shoulder as he completely ignored/skipped/missed install.html and also as he decided that debian.txt wouldn't help him with basic installation. Then I threw in a few docs edits that have been sitting around in my sandbox asking to be committed for months. ] [TAG allmydata-tahoe-1.6.1 david-sarah@jacaranda.org**20100228062314 Ignore-this: eb5f03ada8ea953ee7780e7fe068539 ] Patch bundle hash: ab650c46d18c7e17e294bf10dd0a86677010cd12