Sun Aug 21 11:49:45 PDT 2011 Kevan Carstensen * interfaces: fix typo Sun Aug 21 11:50:04 PDT 2011 Kevan Carstensen * retrieve: code cleanup, use .callback and .errback appropriately Fri Aug 26 14:39:37 PDT 2011 Kevan Carstensen * mutable/publish: fix buggy get_size implementation in TransformingUploadable, clarify self.datalength calculation Fri Aug 26 14:44:19 PDT 2011 Kevan Carstensen * mutable/filenode: clean up stale and misleading comments New patches: [interfaces: fix typo Kevan Carstensen **20110821184945 Ignore-this: 10a587147aefc63738469bf2e855351b ] hunk ./src/allmydata/interfaces.py 452 Add a block and salt to the share. """ - def put_encprivey(encprivkey): + def put_encprivkey(encprivkey): """ Add the encrypted private key to the share. """ [retrieve: code cleanup, use .callback and .errback appropriately Kevan Carstensen **20110821185004 Ignore-this: e9228a6562e8450b2df0b27d169a9fd5 ] { hunk ./src/allmydata/mutable/retrieve.py 901 # hash tree, and is a leaf in the share hash tree. This is fine, # since any share corruption will be detected in the share hash # tree. - #needed.discard(0) self.log("getting blockhashes for segment %d, share %d: %s" % \ (segnum, reader.shnum, str(needed))) d1 = reader.get_blockhashes(needed, queue=True, force_remote=True) hunk ./src/allmydata/mutable/retrieve.py 1108 if self._verify: ret = list(self._bad_shares) + eventually(self.done_deferred.callback, ret) else: format = ("ran out of peers: " "have %(have)d of %(total)d segments " hunk ./src/allmydata/mutable/retrieve.py 1124 (format % args, str(self._last_failure))) f = failure.Failure(e) ret = f - eventually(self._done_deferred.callback, ret) + eventually(self._done_deferred.errback, ret) } [mutable/publish: fix buggy get_size implementation in TransformingUploadable, clarify self.datalength calculation Kevan Carstensen **20110826213937 Ignore-this: f0b9077417a28a5333e161b6d527e3b9 ] { hunk ./src/allmydata/mutable/publish.py 158 self.data = data # XXX: Use the MutableFileVersion instead. - self.datalength = self._node.get_size() - if data.get_size() > self.datalength: - self.datalength = data.get_size() + self.new_length = max(self._node.get_size(), offset + data.get_size()) self.log("starting update") self.log("adding new data of length %d at offset %d" % \ hunk ./src/allmydata/mutable/publish.py 163 (data.get_size(), offset)) - self.log("new data length is %d" % self.datalength) - self._status.set_size(self.datalength) + self.log("new data length is %d" % self.new_length) + self._status.set_size(self.new_length) self._status.set_status("Started") self._started = time.time() hunk ./src/allmydata/mutable/publish.py 275 self.required_shares, self.total_shares, self.segment_size, - self.datalength) + self.new_length) self.writers[shnum].peerid = peerid assert (peerid, shnum) in self._servermap.servermap old_versionid, old_timestamp = self._servermap.servermap[key] hunk ./src/allmydata/mutable/publish.py 349 assert IMutableUploadable.providedBy(newdata) self.data = newdata - self.datalength = newdata.get_size() + self.new_length = newdata.get_size() #if self.datalength >= DEFAULT_MAX_SEGMENT_SIZE: # self._version = MDMF_VERSION #else: hunk ./src/allmydata/mutable/publish.py 355 # self._version = SDMF_VERSION - self.log("starting publish, datalen is %s" % self.datalength) - self._status.set_size(self.datalength) + self.log("starting publish, datalen is %s" % self.new_length) + self._status.set_size(self.new_length) self._status.set_status("Started") self._started = time.time() hunk ./src/allmydata/mutable/publish.py 480 self.required_shares, self.total_shares, self.segment_size, - self.datalength) + self.new_length) self.writers[shnum].peerid = peerid if (peerid, shnum) in self._servermap.servermap: old_versionid, old_timestamp = self._servermap.servermap[key] hunk ./src/allmydata/mutable/publish.py 538 if self._version == MDMF_VERSION: segment_size = DEFAULT_MAX_SEGMENT_SIZE # 128 KiB by default else: - segment_size = self.datalength # SDMF is only one segment + segment_size = self.new_length # SDMF is only one segment # this must be a multiple of self.required_shares segment_size = mathutil.next_multiple(segment_size, self.required_shares) hunk ./src/allmydata/mutable/publish.py 548 if segment_size: # We use div_ceil instead of integer division here because # it is semantically correct. - # If datalength isn't an even multiple of segment_size, but - # is larger than segment_size, datalength // segment_size - # will be the largest number such that num <= datalength and + # If new_length isn't an even multiple of segment_size, but + # is larger than segment_size, new_length // segment_size + # will be the largest number such that num <= new_length and # num % segment_size == 0. But that's not what we want, # because it ignores the extra data. div_ceil will give us # the right number of segments for the data that we're hunk ./src/allmydata/mutable/publish.py 555 # given. - self.num_segments = mathutil.div_ceil(self.datalength, + self.num_segments = mathutil.div_ceil(self.new_length, segment_size) self.starting_segment = offset // segment_size hunk ./src/allmydata/mutable/publish.py 573 assert self.num_segments in (0, 1) # SDMF # calculate the tail segment size. - if segment_size and self.datalength: - self.tail_segment_size = self.datalength % segment_size + if segment_size and self.new_length: + self.tail_segment_size = self.new_length % segment_size self.log("got tail segment size %d" % self.tail_segment_size) else: self.tail_segment_size = 0 hunk ./src/allmydata/mutable/publish.py 602 self._current_segment = self.starting_segment self.end_segment = self.num_segments - 1 # Now figure out where the last segment should be. - if self.data.get_size() != self.datalength: + if offset + self.data.get_size() != self.new_length: # We're updating a few segments in the middle of a mutable # file, so we don't want to republish the whole thing. # (we don't have enough data to do that even if we wanted hunk ./src/allmydata/mutable/publish.py 607 # to) - end = self.data.get_size() + end = offset + self.data.get_size() self.end_segment = end // segment_size if end % segment_size == 0: self.end_segment -= 1 hunk ./src/allmydata/mutable/publish.py 1295 def get_size(self): - return self._offset + self._newdata.get_size() + return self._newdata.get_size() def read(self, length): } [mutable/filenode: clean up stale and misleading comments Kevan Carstensen **20110826214419 Ignore-this: 9d79831b814b4ddfd46023f61c5f4930 ] { hunk ./src/allmydata/mutable/filenode.py 549 new_contents as an argument. I return a Deferred that eventually fires with the results of my replacement process. """ - # TODO: Update downloader hints. return self._do_serialized(self._overwrite, new_contents) hunk ./src/allmydata/mutable/filenode.py 569 creating/updating our own servermap. I return a Deferred that fires with the results of my upload. """ - # TODO: Update downloader hints return self._do_serialized(self._upload, new_contents, servermap) hunk ./src/allmydata/mutable/filenode.py 580 Deferred that eventually fires with an UploadResults instance describing this process. """ - # TODO: Update downloader hints. return self._do_serialized(self._modify, modifier, backoffer) hunk ./src/allmydata/mutable/filenode.py 647 return u.update() - #def set_version(self, version): - # I can be set in two ways: - # 1. When the node is created. - # 2. (for an existing share) when the Servermap is updated - # before I am read. - # assert version in (MDMF_VERSION, SDMF_VERSION) - # self._protocol_version = version - - def get_version(self): return self._protocol_version hunk ./src/allmydata/mutable/filenode.py 671 def _upload(self, new_contents, servermap): - """ - A MutableFileNode still has to have some way of getting - published initially, which is what I am here for. After that, - all publishing, updating, modifying and so on happens through - MutableFileVersions. - """ assert self._pubkey, "update_servermap must be called before publish" # Define IPublishInvoker with a set_downloader_hints method? } Context: [test_mutable.Update: increase timeout from 120s to 400s, slaves are failing Brian Warner **20110825230140 Ignore-this: 101b1924a30cdbda9b2e419e95ca15ec ] [tests: fix check_memory test zooko@zooko.com**20110825201116 Ignore-this: 4d66299fa8cb61d2ca04b3f45344d835 fixes #1503 ] [TAG allmydata-tahoe-1.9.0a1 warner@lothar.com**20110825161122 Ignore-this: 3cbf49f00dbda58189f893c427f65605 ] Patch bundle hash: 7add88d8b5da41b74f86c083972ed743f5757083