Thu Jul 15 16:17:14 PDT 2010 Kevan Carstensen * immutable/upload.py: abort buckets if peer selection fails Thu Jul 15 16:18:20 PDT 2010 Kevan Carstensen * test/test_upload.py: changes to test plumbing for #1117 tests - Add a callRemoteOnly method to FakeBucketWriter. - Change the abort method in FakeBucketWriter to not return a RuntimeError. Thu Jul 15 16:21:05 PDT 2010 Kevan Carstensen * storage/immutable.py: make remote_abort btell the storage server about aborted buckets. Thu Jul 15 16:21:48 PDT 2010 Kevan Carstensen * test/test_storage.py: test for the new remote_abort semantics. Thu Jul 15 17:10:46 PDT 2010 Kevan Carstensen * test/test_upload.py: test to see that aborted buckets are ignored by the storage server New patches: [immutable/upload.py: abort buckets if peer selection fails Kevan Carstensen **20100715231714 Ignore-this: 2a0b643a22284df292d8ed9d91b1fd37 ] { hunk ./src/allmydata/immutable/upload.py 138 return (alreadygot, set(b.keys())) + def abort(self): + """ + I abort the remote bucket writers for the share numbers in + sharenums. This is a good idea to conserve space on the storage + server. + """ + for writer in self.buckets.itervalues(): writer.abort() + + class Tahoe2PeerSelector: def __init__(self, upload_id, logparent=None, upload_status=None): hunk ./src/allmydata/immutable/upload.py 367 self.needed_shares, self.servers_of_happiness, effective_happiness) - raise UploadUnhappinessError("%s (%s)" % (msg, - self._get_progress_message())) + return self._failed("%s (%s)" % (msg, self._get_progress_message())) if self.uncontacted_peers: peer = self.uncontacted_peers.pop(0) hunk ./src/allmydata/immutable/upload.py 428 if self.last_failure_msg: msg += " (%s)" % (self.last_failure_msg,) log.msg(msg, level=log.UNUSUAL, parent=self._log_parent) - raise UploadUnhappinessError(msg) + return self._failed(msg) else: # we placed enough to be happy, so we're done if self._status: hunk ./src/allmydata/immutable/upload.py 517 return self._loop() + def _failed(self, msg): + """ + I am called when peer selection fails. I first abort all of the + remote buckets that I allocated during my unsuccessful attempt to + place shares for this file. I then raise an + UploadUnhappinessError with my msg argument. + """ + for peer in self.use_peers: + assert isinstance(peer, PeerTracker) + + peer.abort() + + raise UploadUnhappinessError(msg) + + class EncryptAnUploadable: """This is a wrapper that takes an IUploadable and provides IEncryptedUploadable.""" } [test/test_upload.py: changes to test plumbing for #1117 tests Kevan Carstensen **20100715231820 Ignore-this: 78a6d359d7bf8529d283e2815bf1e2de - Add a callRemoteOnly method to FakeBucketWriter. - Change the abort method in FakeBucketWriter to not return a RuntimeError. ] { hunk ./src/allmydata/test/test_upload.py 162 d.addCallback(lambda res: _call()) return d + + def callRemoteOnly(self, methname, *args, **kwargs): + d = self.callRemote(methname, *args, **kwargs) + del d # callRemoteOnly ignores this + return None + + def remote_write(self, offset, data): precondition(not self.closed) precondition(offset >= 0) hunk ./src/allmydata/test/test_upload.py 183 self.closed = True def remote_abort(self): - log.err(RuntimeError("uh oh, I was asked to abort")) + pass class FakeClient: DEFAULT_ENCODING_PARAMETERS = {"k":25, } [storage/immutable.py: make remote_abort btell the storage server about aborted buckets. Kevan Carstensen **20100715232105 Ignore-this: 16ab0090676355abdd5600ed44ff19c9 ] { hunk ./src/allmydata/storage/immutable.py 282 def _abort(self): if self.closed: return + os.remove(self.incominghome) # if we were the last share to be moved, remove the incoming/ # directory that was our parent hunk ./src/allmydata/storage/immutable.py 289 parentdir = os.path.split(self.incominghome)[0] if not os.listdir(parentdir): os.rmdir(parentdir) + self._sharefile = None hunk ./src/allmydata/storage/immutable.py 291 + # We are now considered closed for further writing. We must tell + # the storage server about this so that it stops expecting us to + # use the space it allocated for us earlier. + self.closed = True + self.ss.bucket_writer_closed(self, 0) class BucketReader(Referenceable): } [test/test_storage.py: test for the new remote_abort semantics. Kevan Carstensen **20100715232148 Ignore-this: d3d6491f17bf670e770ca4b385007515 ] hunk ./src/allmydata/test/test_storage.py 324 self.failIf(os.path.exists(incoming_prefix_dir), incoming_prefix_dir) self.failUnless(os.path.exists(incoming_dir), incoming_dir) + def test_abort(self): + # remote_abort, when called on a writer, should make sure that + # the allocated size of the bucket is not counted by the storage + # server when accounting for space. + ss = self.create("test_abort") + already, writers = self.allocate(ss, "allocate", [0, 1, 2], 150) + self.failIfEqual(ss.allocated_size(), 0) + + # Now abort the writers. + for writer in writers.itervalues(): + writer.remote_abort() + self.failUnlessEqual(ss.allocated_size(), 0) + + def test_allocate(self): ss = self.create("test_allocate") [test/test_upload.py: test to see that aborted buckets are ignored by the storage server Kevan Carstensen **20100716001046 Ignore-this: cc075c24b1c86d737f3199af894cc780 ] hunk ./src/allmydata/test/test_upload.py 1809 return d + def test_peer_selector_bucket_abort(self): + # If peer selection for an upload fails due to an unhappy + # layout, the peer selection process should abort the buckets it + # allocates before failing, so that the space can be re-used. + self.basedir = self.mktemp() + self.set_up_grid(num_servers=5) + + # Try to upload a file with happy=7, which is unsatisfiable with + # the current grid. This will fail, but should not take up any + # space on the storage servers after it fails. + client = self.g.clients[0] + client.DEFAULT_ENCODING_PARAMETERS['happy'] = 7 + d = defer.succeed(None) + d.addCallback(lambda ignored: + self.shouldFail(UploadUnhappinessError, + "test_peer_selection_bucket_abort", + "", + client.upload, upload.Data("data" * 10000, + convergence=""))) + # wait for the abort messages to get there. + def _turn_barrier(res): + return fireEventually(res) + d.addCallback(_turn_barrier) + def _then(ignored): + for server in self.g.servers_by_number.values(): + self.failUnlessEqual(server.allocated_size(), 0) + d.addCallback(_then) + return d + + + def test_encoder_bucket_abort(self): + # If enough servers die in the process of encoding and uploading + # a file to make the layout unhappy, we should cancel the + # newly-allocated buckets before dying. + self.basedir = self.mktemp() + self.set_up_grid(num_servers=4) + + client = self.g.clients[0] + client.DEFAULT_ENCODING_PARAMETERS['happy'] = 7 + + d = defer.succeed(None) + d.addCallback(lambda ignored: + self.shouldFail(UploadUnhappinessError, + "test_encoder_bucket_abort", + "", + self._do_upload_with_broken_servers, 1)) + def _turn_barrier(res): + return fireEventually(res) + d.addCallback(_turn_barrier) + def _then(ignored): + for server in self.g.servers_by_number.values(): + self.failUnlessEqual(server.allocated_size(), 0) + d.addCallback(_then) + return d + + def _set_up_nodes_extra_config(self, clientdir): cfgfn = os.path.join(clientdir, "tahoe.cfg") oldcfg = open(cfgfn, "r").read() Context: [SFTP: address some of the comments in zooko's review (#1106). david-sarah@jacaranda.org**20100712025537 Ignore-this: c3921638a2d4f1de2a776ae78e4dc37e ] [docs/logging.txt: note that setting flogging vars might affect tests with race conditions. david-sarah@jacaranda.org**20100712050721 Ignore-this: fc1609d215fcd5561a57fd1226206f27 ] [test_storage.py: potential fix for failures when logging is enabled. david-sarah@jacaranda.org**19700713040546 Ignore-this: 5815693a0df3e64c52c3c6b7be2846c7 ] [upcase_since_on_welcome terrellrussell@gmail.com**20100708193903] [server_version_on_welcome_page.dpatch.txt freestorm77@gmail.com**20100605191721 Ignore-this: b450c76dc875f5ac8cca229a666cbd0a - The storage server version is 0 for all storage nodes in the Welcome Page ] [NEWS: add NEWS snippets about two recent patches zooko@zooko.com**20100708162058 Ignore-this: 6c9da6a0ad7351a960bdd60f81532899 ] [directory_html_top_banner.dpatch freestorm77@gmail.com**20100622205301 Ignore-this: 1d770d975e0c414c996564774f049bca The div tag with the link "Return to Welcome page" on the directory.xhtml page is not correct ] [tahoe_css_toolbar.dpatch freestorm77@gmail.com**20100622210046 Ignore-this: 5b3ebb2e0f52bbba718a932f80c246c0 CSS modification to be correctly diplayed with Internet Explorer 8 The links on the top of page directory.xhtml are not diplayed in the same line as display with Firefox. ] [runnin_test_tahoe_css.dpatch freestorm77@gmail.com**20100622214714 Ignore-this: e0db73d68740aad09a7b9ae60a08c05c Runnin test for changes in tahoe.css file ] [runnin_test_directory_xhtml.dpatch freestorm77@gmail.com**20100622201403 Ignore-this: f8962463fce50b9466405cb59fe11d43 Runnin test for diretory.xhtml top banner ] [stringutils.py: tolerate sys.stdout having no 'encoding' attribute. david-sarah@jacaranda.org**20100626040817 Ignore-this: f42cad81cef645ee38ac1df4660cc850 ] [quickstart.html: python 2.5 -> 2.6 as recommended version david-sarah@jacaranda.org**20100705175858 Ignore-this: bc3a14645ea1d5435002966ae903199f ] [SFTP: don't call .stopProducing on the producer registered with OverwriteableFileConsumer (which breaks with warner's new downloader). david-sarah@jacaranda.org**20100628231926 Ignore-this: 131b7a5787bc85a9a356b5740d9d996f ] [docs/how_to_make_a_tahoe-lafs_release.txt: trivial correction, install.html should now be quickstart.html. david-sarah@jacaranda.org**20100625223929 Ignore-this: 99a5459cac51bd867cc11ad06927ff30 ] [setup: in the Makefile, refuse to upload tarballs unless someone has passed the environment variable "BB_BRANCH" with value "trunk" zooko@zooko.com**20100619034928 Ignore-this: 276ddf9b6ad7ec79e27474862e0f7d6 ] [trivial: tiny update to in-line comment zooko@zooko.com**20100614045715 Ignore-this: 10851b0ed2abfed542c97749e5d280bc (I'm actually committing this patch as a test of the new eager-annotation-computation of trac-darcs.) ] [docs: about.html link to home page early on, and be decentralized storage instead of cloud storage this time around zooko@zooko.com**20100619065318 Ignore-this: dc6db03f696e5b6d2848699e754d8053 ] [docs: update about.html, especially to have a non-broken link to quickstart.html, and also to comment out the broken links to "for Paranoids" and "for Corporates" zooko@zooko.com**20100619065124 Ignore-this: e292c7f51c337a84ebfeb366fbd24d6c ] [TAG allmydata-tahoe-1.7.0 zooko@zooko.com**20100619052631 Ignore-this: d21e27afe6d85e2e3ba6a3292ba2be1 ] Patch bundle hash: 00056b67380e13933aba18bc9d0dca86cfcf6566