5 patches for repository /home/davidsarah/tahoe/newtrunk: Mon Dec 5 02:40:24 GMT 2011 david-sarah@jacaranda.org * interfaces.py: 'which -> that' grammar cleanup. Mon Dec 5 04:35:06 GMT 2011 david-sarah@jacaranda.org * Pluggable backends: documentation. refs #999 Mon Dec 5 04:36:53 GMT 2011 david-sarah@jacaranda.org * Pluggable backends: new and moved code files. refs #999 Fri Dec 16 18:12:07 GMT 2011 david-sarah@jacaranda.org * Cosmetic changes in pluggable backends branch. refs #999, #1569 Fri Dec 16 18:36:51 GMT 2011 david-sarah@jacaranda.org * All other changes in pluggable backends branch. refs #999, #1569 New patches: [interfaces.py: 'which -> that' grammar cleanup. david-sarah@jacaranda.org**20111205024024 Ignore-this: 2de8d902dd7b473fb5c15701fcd8e0f7 ] { merger 0.0 ( hunk ./src/allmydata/interfaces.py 33 -class RIStubClient(RemoteInterface): - """Each client publishes a service announcement for a dummy object called - the StubClient. This object doesn't actually offer any services, but the - announcement helps the Introducer keep track of which clients are - subscribed (so the grid admin can keep track of things like the size of - the grid and the client versions in use. This is the (empty) - RemoteInterface for the StubClient.""" - hunk ./src/allmydata/interfaces.py 38 - the grid and the client versions in use. This is the (empty) + the grid and the client versions in use). This is the (empty) ) hunk ./src/allmydata/interfaces.py 276 (binary) storage index string, and 'shnum' is the integer share number. 'reason' is a human-readable explanation of the problem, probably including some expected hash values and the computed ones - which did not match. Corruption advisories for mutable shares should + that did not match. Corruption advisories for mutable shares should include a hash of the public key (the same value that appears in the mutable-file verify-cap), since the current share format does not store that on disk. hunk ./src/allmydata/interfaces.py 413 remote_host: the IAddress, if connected, otherwise None This method is intended for monitoring interfaces, such as a web page - which describes connecting and connected peers. + that describes connecting and connected peers. """ def get_all_peerids(): hunk ./src/allmydata/interfaces.py 515 # TODO: rename to get_read_cap() def get_readonly(): - """Return another IURI instance, which represents a read-only form of + """Return another IURI instance that represents a read-only form of this one. If is_readonly() is True, this returns self.""" def get_verify_cap(): hunk ./src/allmydata/interfaces.py 542 passing into init_from_string.""" class IDirnodeURI(Interface): - """I am a URI which represents a dirnode.""" + """I am a URI that represents a dirnode.""" class IFileURI(Interface): hunk ./src/allmydata/interfaces.py 545 - """I am a URI which represents a filenode.""" + """I am a URI that represents a filenode.""" def get_size(): """Return the length (in bytes) of the file that I represent.""" hunk ./src/allmydata/interfaces.py 851 """ class IFileNode(IFilesystemNode): - """I am a node which represents a file: a sequence of bytes. I am not a + """I am a node that represents a file: a sequence of bytes. I am not a container, like IDirectoryNode.""" def get_best_readable_version(): """Return a Deferred that fires with an IReadable for the 'best' hunk ./src/allmydata/interfaces.py 900 multiple versions of a file present in the grid, some of which might be unrecoverable (i.e. have fewer than 'k' shares). These versions are loosely ordered: each has a sequence number and a hash, and any version - with seqnum=N was uploaded by a node which has seen at least one version + with seqnum=N was uploaded by a node that has seen at least one version with seqnum=N-1. The 'servermap' (an instance of IMutableFileServerMap) is used to hunk ./src/allmydata/interfaces.py 1009 as a guide to where the shares are located. I return a Deferred that fires with the requested contents, or - errbacks with UnrecoverableFileError. Note that a servermap which was + errbacks with UnrecoverableFileError. Note that a servermap that was updated with MODE_ANYTHING or MODE_READ may not know about shares for all versions (those modes stop querying servers as soon as they can fulfil their goals), so you may want to use MODE_CHECK (which checks hunk ./src/allmydata/interfaces.py 1068 """Upload was unable to satisfy 'servers_of_happiness'""" class UnableToFetchCriticalDownloadDataError(Exception): - """I was unable to fetch some piece of critical data which is supposed to + """I was unable to fetch some piece of critical data that is supposed to be identically present in all shares.""" class NoServersError(Exception): hunk ./src/allmydata/interfaces.py 1080 exists, and overwrite= was set to False.""" class NoSuchChildError(Exception): - """A directory node was asked to fetch a child which does not exist.""" + """A directory node was asked to fetch a child that does not exist.""" class ChildOfWrongTypeError(Exception): """An operation was attempted on a child of the wrong type (file or directory).""" hunk ./src/allmydata/interfaces.py 1398 if you initially thought you were going to use 10 peers, started encoding, and then two of the peers dropped out: you could use desired_share_ids= to skip the work (both memory and CPU) of - producing shares for the peers which are no longer available. + producing shares for the peers that are no longer available. """ hunk ./src/allmydata/interfaces.py 1473 if you initially thought you were going to use 10 peers, started encoding, and then two of the peers dropped out: you could use desired_share_ids= to skip the work (both memory and CPU) of - producing shares for the peers which are no longer available. + producing shares for the peers that are no longer available. For each call, encode() will return a Deferred that fires with two lists, one containing shares and the other containing the shareids. hunk ./src/allmydata/interfaces.py 1530 required to be of the same length. The i'th element of their_shareids is required to be the shareid of the i'th buffer in some_shares. - This returns a Deferred which fires with a sequence of buffers. This + This returns a Deferred that fires with a sequence of buffers. This sequence will contain all of the segments of the original data, in order. The sum of the lengths of all of the buffers will be the 'data_size' value passed into the original ICodecEncode.set_params() hunk ./src/allmydata/interfaces.py 1577 Encoding parameters can be set in three ways. 1: The Encoder class provides defaults (3/7/10). 2: the Encoder can be constructed with an 'options' dictionary, in which the - needed_and_happy_and_total_shares' key can be a (k,d,n) tuple. 3: + 'needed_and_happy_and_total_shares' key can be a (k,d,n) tuple. 3: set_params((k,d,n)) can be called. If you intend to use set_params(), you must call it before hunk ./src/allmydata/interfaces.py 1775 produced, so that the segment hashes can be generated with only a single pass. - This returns a Deferred which fires with a sequence of hashes, using: + This returns a Deferred that fires with a sequence of hashes, using: tuple(segment_hashes[first:last]) hunk ./src/allmydata/interfaces.py 1791 def get_plaintext_hash(): """OBSOLETE; Get the hash of the whole plaintext. - This returns a Deferred which fires with a tagged SHA-256 hash of the + This returns a Deferred that fires with a tagged SHA-256 hash of the whole plaintext, obtained from hashutil.plaintext_hash(data). """ hunk ./src/allmydata/interfaces.py 1851 be used to encrypt the data. The key will also be hashed to derive the StorageIndex. - Uploadables which want to achieve convergence should hash their file + Uploadables that want to achieve convergence should hash their file contents and the serialized_encoding_parameters to form the key (which of course requires a full pass over the data). Uploadables can use the upload.ConvergentUploadMixin class to achieve this hunk ./src/allmydata/interfaces.py 1857 automatically. - Uploadables which do not care about convergence (or do not wish to + Uploadables that do not care about convergence (or do not wish to make multiple passes over the data) can simply return a strongly-random 16 byte string. hunk ./src/allmydata/interfaces.py 1867 def read(length): """Return a Deferred that fires with a list of strings (perhaps with - only a single element) which, when concatenated together, contain the + only a single element) that, when concatenated together, contain the next 'length' bytes of data. If EOF is near, this may provide fewer than 'length' bytes. The total number of bytes provided by read() before it signals EOF must equal the size provided by get_size(). hunk ./src/allmydata/interfaces.py 1914 def read(length): """ - Returns a list of strings which, when concatenated, are the next + Returns a list of strings that, when concatenated, are the next length bytes of the file, or fewer if there are fewer bytes between the current location and the end of the file. """ hunk ./src/allmydata/interfaces.py 1927 class IUploadResults(Interface): """I am returned by upload() methods. I contain a number of public - attributes which can be read to determine the results of the upload. Some + attributes that can be read to determine the results of the upload. Some of these are functional, some are timing information. All of these may be None. hunk ./src/allmydata/interfaces.py 1960 class IDownloadResults(Interface): """I am created internally by download() methods. I contain a number of - public attributes which contain details about the download process.:: + public attributes that contain details about the download process.:: .file_size : the size of the file, in bytes .servers_used : set of server peerids that were used during download hunk ./src/allmydata/interfaces.py 1986 class IUploader(Interface): def upload(uploadable): """Upload the file. 'uploadable' must impement IUploadable. This - returns a Deferred which fires with an IUploadResults instance, from + returns a Deferred that fires with an IUploadResults instance, from which the URI of the file can be obtained as results.uri .""" def upload_ssk(write_capability, new_version, uploadable): hunk ./src/allmydata/interfaces.py 2036 kind of lease that is obtained (which account number to claim, etc). TODO: any problems seen during checking will be reported to the - health-manager.furl, a centralized object which is responsible for + health-manager.furl, a centralized object that is responsible for figuring out why files are unhealthy so corrective action can be taken. """ hunk ./src/allmydata/interfaces.py 2051 will be put in the check-and-repair results. The Deferred will not fire until the repair is complete. - This returns a Deferred which fires with an instance of + This returns a Deferred that fires with an instance of ICheckAndRepairResults.""" class IDeepCheckable(Interface): hunk ./src/allmydata/interfaces.py 2136 that was found to be corrupt. Each share locator is a list of (serverid, storage_index, sharenum). - count-incompatible-shares: the number of shares which are of a share + count-incompatible-shares: the number of shares that are of a share format unknown to this checker list-incompatible-shares: a list of 'share locators', one for each share that was found to be of an unknown hunk ./src/allmydata/interfaces.py 2143 format. Each share locator is a list of (serverid, storage_index, sharenum). servers-responding: list of (binary) storage server identifiers, - one for each server which responded to the share + one for each server that responded to the share query (even if they said they didn't have shares, and even if they said they did have shares but then didn't send them when asked, or hunk ./src/allmydata/interfaces.py 2340 will use the data in the checker results to guide the repair process, such as which servers provided bad data and should therefore be avoided. The ICheckResults object is inside the - ICheckAndRepairResults object, which is returned by the + ICheckAndRepairResults object that is returned by the ICheckable.check() method:: d = filenode.check(repair=False) hunk ./src/allmydata/interfaces.py 2431 methods to create new objects. I return synchronously.""" def create_mutable_file(contents=None, keysize=None): - """I create a new mutable file, and return a Deferred which will fire + """I create a new mutable file, and return a Deferred that will fire with the IMutableFileNode instance when it is ready. If contents= is provided (a bytestring), it will be used as the initial contents of the new file, otherwise the file will contain zero bytes. keysize= is hunk ./src/allmydata/interfaces.py 2439 usual.""" def create_new_mutable_directory(initial_children={}): - """I create a new mutable directory, and return a Deferred which will + """I create a new mutable directory, and return a Deferred that will fire with the IDirectoryNode instance when it is ready. If initial_children= is provided (a dict mapping unicode child name to (childnode, metadata_dict) tuples), the directory will be populated hunk ./src/allmydata/interfaces.py 2447 class IClientStatus(Interface): def list_all_uploads(): - """Return a list of uploader objects, one for each upload which + """Return a list of uploader objects, one for each upload that currently has an object available (tracked with weakrefs). This is intended for debugging purposes.""" def list_active_uploads(): hunk ./src/allmydata/interfaces.py 2457 started uploads.""" def list_all_downloads(): - """Return a list of downloader objects, one for each download which + """Return a list of downloader objects, one for each download that currently has an object available (tracked with weakrefs). This is intended for debugging purposes.""" def list_active_downloads(): hunk ./src/allmydata/interfaces.py 2684 def provide(provider=RIStatsProvider, nickname=str): """ - @param provider: a stats collector instance which should be polled + @param provider: a stats collector instance that should be polled periodically by the gatherer to collect stats. @param nickname: a name useful to identify the provided client """ hunk ./src/allmydata/interfaces.py 2717 class IValidatedThingProxy(Interface): def start(): - """ Acquire a thing and validate it. Return a deferred which is + """ Acquire a thing and validate it. Return a deferred that is eventually fired with self if the thing is valid or errbacked if it can't be acquired or validated.""" } [Pluggable backends: documentation. refs #999 david-sarah@jacaranda.org**20111205043506 Ignore-this: 5290f8788c2b710243572ec04ee032ee ] { adddir ./docs/backends addfile ./docs/backends/S3.rst hunk ./docs/backends/S3.rst 1 +==================================================== +Storing Shares in Amazon Simple Storage Service (S3) +==================================================== hunk ./docs/backends/S3.rst 5 +S3 is a commercial storage service provided by Amazon, described at +``_. + +The Tahoe-LAFS storage server can be configured to store its shares in +an S3 bucket, rather than on local filesystem. To enable this, add the +following keys to the server's ``tahoe.cfg`` file: + +``[storage]`` + +``backend = s3`` + + This turns off the local filesystem backend and enables use of S3. + +``s3.access_key_id = (string, required)`` + + This identifies your Amazon Web Services access key. The access key id is + not secret, but there is a secret key associated with it. The secret key + is stored in a separate file named ``private/s3secret``. + +``s3.bucket = (string, required)`` + + This controls which bucket will be used to hold shares. The Tahoe-LAFS + storage server will only modify and access objects in the configured S3 + bucket. Multiple storage servers cannot share the same bucket. + +``s3.url = (URL string, optional)`` + + This URL tells the storage server how to access the S3 service. It + defaults to ``http://s3.amazonaws.com``, but by setting it to something + else, you may be able to use some other S3-like service if it is + sufficiently compatible. + +The system time of the storage server must be correct to within 15 minutes +in order for S3 to accept the authentication provided with requests. + + +DevPay +====== + +Optionally, Amazon `DevPay`_ may be used to delegate billing for a service +based on Tahoe-LAFS and S3 to Amazon Payments. + +If DevPay is to be used, the user token and product token (in base64 form) +must be stored in the files ``private/s3usertoken`` and ``private/s3producttoken`` +respectively. DevPay-related request headers will be sent only if these files +are present when the server is started. It is currently assumed that only one +user and product token pair is needed by a given storage server. + +.. _DevPay: http://docs.amazonwebservices.com/AmazonDevPay/latest/DevPayGettingStartedGuide/ addfile ./docs/backends/disk.rst hunk ./docs/backends/disk.rst 1 +==================================== +Storing Shares on a Local Filesystem +==================================== + +The "disk" backend stores shares on the local filesystem. Versions of +Tahoe-LAFS <= 1.9.0 always stored shares in this way. + +``[storage]`` + +``backend = disk`` + + This enables use of the disk backend, and is the default. + +``readonly = (boolean, optional)`` + + If ``True``, the node will run a storage server but will not accept any + shares, making it effectively read-only. Use this for storage servers + that are being decommissioned: the ``storage/`` directory could be + mounted read-only, while shares are moved to other servers. Note that + this currently only affects immutable shares. Mutable shares (used for + directories) will be written and modified anyway. See ticket `#390 + `_ for the current + status of this bug. The default value is ``False``. + +``reserved_space = (quantity of space, optional)`` + + If provided, this value defines how much disk space is reserved: the + storage server will not accept any share that causes the amount of free + disk space to drop below this value. (The free space is measured by a + call to statvfs(2) on Unix, or GetDiskFreeSpaceEx on Windows, and is the + space available to the user account under which the storage server runs.) + + This string contains a number, with an optional case-insensitive scale + suffix like "K" or "M" or "G", and an optional "B" or "iB" suffix. So + "100MB", "100M", "100000000B", "100000000", and "100000kb" all mean the + same thing. Likewise, "1MiB", "1024KiB", and "1048576B" all mean the same + thing. + + "``tahoe create-node``" generates a tahoe.cfg with + "``reserved_space=1G``", but you may wish to raise, lower, or remove the + reservation to suit your needs. + +``expire.enabled =`` + +``expire.mode =`` + +``expire.override_lease_duration =`` + +``expire.cutoff_date =`` + +``expire.immutable =`` + +``expire.mutable =`` + + These settings control garbage collection, causing the server to + delete shares that no longer have an up-to-date lease on them. Please + see ``_ for full details. hunk ./docs/configuration.rst 433 mounted read-only, while shares are moved to other servers. Note that this currently only affects immutable shares. Mutable shares (used for directories) will be written and modified anyway. See ticket `#390 - `_ for the current + `_ for the current status of this bug. The default value is ``False``. hunk ./docs/configuration.rst 436 -``reserved_space = (str, optional)`` +``backend = (string, optional)`` hunk ./docs/configuration.rst 438 - If provided, this value defines how much disk space is reserved: the - storage server will not accept any share that causes the amount of free - disk space to drop below this value. (The free space is measured by a - call to statvfs(2) on Unix, or GetDiskFreeSpaceEx on Windows, and is the - space available to the user account under which the storage server runs.) + Storage servers can store the data into different "backends". Clients + need not be aware of which backend is used by a server. The default + value is ``disk``. hunk ./docs/configuration.rst 442 - This string contains a number, with an optional case-insensitive scale - suffix like "K" or "M" or "G", and an optional "B" or "iB" suffix. So - "100MB", "100M", "100000000B", "100000000", and "100000kb" all mean the - same thing. Likewise, "1MiB", "1024KiB", and "1048576B" all mean the same - thing. +``backend = disk`` hunk ./docs/configuration.rst 444 - "``tahoe create-node``" generates a tahoe.cfg with - "``reserved_space=1G``", but you may wish to raise, lower, or remove the - reservation to suit your needs. + The storage server stores shares on the local filesystem (in + BASEDIR/storage/shares/). For configuration details (including how to + reserve a minimum amount of free space), see ``_. hunk ./docs/configuration.rst 448 -``expire.enabled =`` +``backend = s3`` hunk ./docs/configuration.rst 450 -``expire.mode =`` + The storage server stores all shares to an Amazon Simple Storage Service + (S3) bucket. For configuration details, see ``_. hunk ./docs/configuration.rst 453 -``expire.override_lease_duration =`` +``backend = debug_discard`` hunk ./docs/configuration.rst 455 -``expire.cutoff_date =`` - -``expire.immutable =`` - -``expire.mutable =`` - - These settings control garbage collection, in which the server will - delete shares that no longer have an up-to-date lease on them. Please see - ``_ for full details. + The storage server stores all shares in /dev/null. This is actually used, + for testing. It is not recommended for storage of data that you might + want to retrieve in the future. Running A Helper } [Pluggable backends: new and moved code files. refs #999 david-sarah@jacaranda.org**20111205043653 Ignore-this: 107135f528e48ad51f97dd9616bbc542 ] { adddir ./src/allmydata/storage/backends adddir ./src/allmydata/storage/backends/disk move ./src/allmydata/storage/immutable.py ./src/allmydata/storage/backends/disk/immutable.py move ./src/allmydata/storage/mutable.py ./src/allmydata/storage/backends/disk/mutable.py adddir ./src/allmydata/storage/backends/null adddir ./src/allmydata/storage/backends/s3 addfile ./src/allmydata/storage/backends/__init__.py addfile ./src/allmydata/storage/backends/base.py hunk ./src/allmydata/storage/backends/base.py 1 + +from twisted.application import service +from twisted.internet import defer + +from allmydata.util import fileutil, log, time_format +from allmydata.util.deferredutil import async_iterate, gatherResults +from allmydata.storage.common import si_b2a +from allmydata.storage.lease import LeaseInfo +from allmydata.storage.bucket import BucketReader + + +class Backend(service.MultiService): + def __init__(self): + service.MultiService.__init__(self) + self._corruption_advisory_dir = None + + def supports_crawlers(self): + return False + + def advise_corrupt_share(self, sharetype, storageindex, shnum, reason): + si_s = si_b2a(storageindex) + if self._corruption_advisory_dir is not None: + fileutil.fp_make_dirs(self._corruption_advisory_dir) + now = time_format.iso_utc(sep="T") + + # Windows can't handle colons in the filename. + name = ("%s--%s-%d" % (now, si_s, shnum)).replace(":", "") + f = self._corruption_advisory_dir.child(name).open("w") + try: + f.write("report: Share Corruption\n") + f.write("type: %s\n" % sharetype) + f.write("storage_index: %s\n" % si_s) + f.write("share_number: %d\n" % shnum) + f.write("\n") + f.write(reason) + f.write("\n") + finally: + f.close() + + log.msg(format=("client claims corruption in (%(share_type)s) " + + "%(si)s-%(shnum)d: %(reason)s"), + share_type=sharetype, si=si_s, shnum=shnum, reason=reason, + level=log.SCARY, umid="2fASGx") + + +class ShareSet(object): + """ + This class implements shareset logic that could work for all backends, but + might be useful to override for efficiency. + """ + + def __init__(self, storageindex): + self.storageindex = storageindex + + def get_shares_synchronous(self): + raise NotImplementedError + + def get_storage_index(self): + return self.storageindex + + def get_storage_index_string(self): + return si_b2a(self.storageindex) + + def make_bucket_reader(self, storageserver, share): + return BucketReader(storageserver, share) + + def testv_and_readv_and_writev(self, storageserver, secrets, + test_and_write_vectors, read_vector, + expiration_time): + # The implementation here depends on the following helper methods, + # which must be provided by subclasses: + # + # def _clean_up_after_unlink(self): + # """clean up resources associated with the shareset after some + # shares might have been deleted""" + # + # def _create_mutable_share(self, storageserver, shnum, write_enabler): + # """create a mutable share with the given shnum and write_enabler""" + + (write_enabler, renew_secret, cancel_secret) = secrets + + sharemap = {} + d = self.get_shares() + def _got_shares( (shares, corrupted) ): + d2 = defer.succeed(None) + for share in shares: + assert not isinstance(share, defer.Deferred), share + # XXX is it correct to ignore immutable shares? Maybe get_shares should + # have a parameter saying what type it's expecting. + if share.sharetype == "mutable": + d2.addCallback(lambda ign, share=share: share.check_write_enabler(write_enabler)) + sharemap[share.get_shnum()] = share + + shnums = sorted(sharemap.keys()) + + # if d2 does not fail, write_enabler is good for all existing shares + + # now evaluate test vectors + def _check_testv(shnum): + (testv, datav, new_length) = test_and_write_vectors[shnum] + if shnum in sharemap: + d3 = sharemap[shnum].check_testv(testv) + elif shnum in corrupted: + # a corrupted share does not match any test vector + d3 = defer.succeed(False) + else: + # compare the vectors against an empty share, in which all + # reads return empty strings + d3 = defer.succeed(empty_check_testv(testv)) + + def _check_result(res): + if not res: + storageserver.log("testv failed: [%d] %r" % (shnum, testv)) + return res + d3.addCallback(_check_result) + return d3 + + d2.addCallback(lambda ign: async_iterate(_check_testv, test_and_write_vectors)) + + def _gather(testv_is_good): + # Gather the read vectors, before we do any writes. This ignores any + # corrupted shares. + d3 = gatherResults([sharemap[shnum].readv(read_vector) for shnum in shnums]) + + def _do_writes(reads): + read_data = {} + for i in range(len(shnums)): + read_data[shnums[i]] = reads[i] + + ownerid = 1 # TODO + lease_info = LeaseInfo(ownerid, renew_secret, cancel_secret, + expiration_time, storageserver.get_serverid()) + + d4 = defer.succeed(None) + if testv_is_good: + if len(set(test_and_write_vectors.keys()) & corrupted) > 0: + # XXX think of a better exception to raise + raise AssertionError("You asked to write share numbers %r of storage index %r, " + "but one or more of those is corrupt (numbers %r)" + % (list(sorted(test_and_write_vectors.keys())), + self.get_storage_index_string(), + list(sorted(corrupted))) ) + + # now apply the write vectors + for shnum in test_and_write_vectors: + (testv, datav, new_length) = test_and_write_vectors[shnum] + if new_length == 0: + if shnum in sharemap: + d4.addCallback(lambda ign, shnum=shnum: sharemap[shnum].unlink()) + else: + if shnum not in sharemap: + # allocate a new share + d4.addCallback(lambda ign, shnum=shnum: + self._create_mutable_share(storageserver, shnum, + write_enabler)) + def _record_share(share, shnum=shnum): + sharemap[shnum] = share + d4.addCallback(_record_share) + d4.addCallback(lambda ign, shnum=shnum, datav=datav, new_length=new_length: + sharemap[shnum].writev(datav, new_length)) + # and update the lease + d4.addCallback(lambda ign, shnum=shnum: + sharemap[shnum].add_or_renew_lease(lease_info)) + if new_length == 0: + d4.addCallback(lambda ign: self._clean_up_after_unlink()) + + d4.addCallback(lambda ign: (testv_is_good, read_data)) + return d4 + d3.addCallback(_do_writes) + return d3 + d2.addCallback(_gather) + return d2 + d.addCallback(_got_shares) + return d + + def readv(self, wanted_shnums, read_vector): + """ + Read a vector from the numbered shares in this shareset. An empty + shares list means to return data from all known shares. + + @param wanted_shnums=ListOf(int) + @param read_vector=ReadVector + @return DictOf(int, ReadData): shnum -> results, with one key per share + """ + shnums = [] + dreads = [] + d = self.get_shares() + def _got_shares( (shares, corrupted) ): + # We ignore corrupted shares. + for share in shares: + assert not isinstance(share, defer.Deferred), share + shnum = share.get_shnum() + if not wanted_shnums or shnum in wanted_shnums: + shnums.append(share.get_shnum()) + dreads.append(share.readv(read_vector)) + return gatherResults(dreads) + d.addCallback(_got_shares) + + def _got_reads(reads): + datavs = {} + for i in range(len(shnums)): + datavs[shnums[i]] = reads[i] + return datavs + d.addCallback(_got_reads) + return d + + +def testv_compare(a, op, b): + assert op in ("lt", "le", "eq", "ne", "ge", "gt") + if op == "lt": + return a < b + if op == "le": + return a <= b + if op == "eq": + return a == b + if op == "ne": + return a != b + if op == "ge": + return a >= b + if op == "gt": + return a > b + # never reached + + +def empty_check_testv(testv): + test_good = True + for (offset, length, operator, specimen) in testv: + data = "" + if not testv_compare(data, operator, specimen): + test_good = False + break + return test_good + addfile ./src/allmydata/storage/backends/disk/__init__.py addfile ./src/allmydata/storage/backends/disk/disk_backend.py hunk ./src/allmydata/storage/backends/disk/disk_backend.py 1 + +import re, struct + +from twisted.internet import defer + +from zope.interface import implements +from allmydata.interfaces import IStorageBackend, IShareSet +from allmydata.util import fileutil, log +from allmydata.storage.common import si_b2a, si_a2b, UnknownMutableContainerVersionError, \ + UnknownImmutableContainerVersionError +from allmydata.storage.bucket import BucketWriter +from allmydata.storage.backends.base import Backend, ShareSet +from allmydata.storage.backends.disk.immutable import load_immutable_disk_share, create_immutable_disk_share +from allmydata.storage.backends.disk.mutable import load_mutable_disk_share, create_mutable_disk_share +from allmydata.mutable.layout import MUTABLE_MAGIC + + +# storage/ +# storage/shares/incoming +# incoming/ holds temp dirs named $PREFIX/$STORAGEINDEX/$SHNUM which will +# be moved to storage/shares/$PREFIX/$STORAGEINDEX/$SHNUM upon success +# storage/shares/$PREFIX/$STORAGEINDEX +# storage/shares/$PREFIX/$STORAGEINDEX/$SHNUM + +# where "$PREFIX" denotes the first 10 bits worth of $STORAGEINDEX (that's 2 +# base-32 chars). +# $SHNUM matches this regex: +NUM_RE=re.compile("^[0-9]+$") + + +def si_si2dir(startfp, storageindex): + sia = si_b2a(storageindex) + newfp = startfp.child(sia[:2]) + return newfp.child(sia) + +def get_disk_share(home, storageindex=None, shnum=None): + f = home.open('rb') + try: + prefix = f.read(len(MUTABLE_MAGIC)) + finally: + f.close() + + if prefix == MUTABLE_MAGIC: + return load_mutable_disk_share(home, storageindex, shnum) + else: + # assume it's immutable + return load_immutable_disk_share(home, storageindex, shnum) + + +def configure_disk_backend(storedir, config): + readonly = config.get_config("storage", "readonly", False, boolean=True) + reserved_space = config.get_config_size("storage", "reserved_space", "0") + + return DiskBackend(storedir, readonly, reserved_space) + + +class DiskBackend(Backend): + implements(IStorageBackend) + + def __init__(self, storedir, readonly=False, reserved_space=0): + Backend.__init__(self) + self._setup_storage(storedir, readonly, reserved_space) + self._setup_corruption_advisory() + + def _setup_storage(self, storedir, readonly, reserved_space): + self._storedir = storedir + self._readonly = readonly + self._reserved_space = int(reserved_space) + self._sharedir = self._storedir.child("shares") + fileutil.fp_make_dirs(self._sharedir) + self._incomingdir = self._sharedir.child('incoming') + self._clean_incomplete() + if self._reserved_space and (self.get_available_space() is None): + log.msg("warning: [storage]reserved_space= is set, but this platform does not support an API to get disk statistics (statvfs(2) or GetDiskFreeSpaceEx), so this reservation cannot be honored", + umid="0wZ27w", level=log.UNUSUAL) + + def _clean_incomplete(self): + fileutil.fp_remove(self._incomingdir) + fileutil.fp_make_dirs(self._incomingdir) + + def _setup_corruption_advisory(self): + # we don't actually create the corruption-advisory dir until necessary + self._corruption_advisory_dir = self._storedir.child("corruption-advisories") + + def _make_shareset(self, sharehomedir): + return self.get_shareset(si_a2b(str(sharehomedir.basename()))) + + def supports_crawlers(self): + return True + + def get_sharesets_for_prefix(self, prefix): + prefixfp = self._sharedir.child(prefix) + sharesets = map(self._make_shareset, fileutil.fp_list(prefixfp)) + def _by_base32si(b): + return b.get_storage_index_string() + sharesets.sort(key=_by_base32si) + return sharesets + + def get_shareset(self, storageindex): + sharehomedir = si_si2dir(self._sharedir, storageindex) + incominghomedir = si_si2dir(self._incomingdir, storageindex) + return DiskShareSet(storageindex, sharehomedir, incominghomedir) + + def fill_in_space_stats(self, stats): + stats['storage_server.reserved_space'] = self._reserved_space + try: + disk = fileutil.get_disk_stats(self._sharedir, self._reserved_space) + writeable = disk['avail'] > 0 + + # spacetime predictors should use disk_avail / (d(disk_used)/dt) + stats['storage_server.disk_total'] = disk['total'] + stats['storage_server.disk_used'] = disk['used'] + stats['storage_server.disk_free_for_root'] = disk['free_for_root'] + stats['storage_server.disk_free_for_nonroot'] = disk['free_for_nonroot'] + stats['storage_server.disk_avail'] = disk['avail'] + except AttributeError: + writeable = True + except EnvironmentError: + log.msg("OS call to get disk statistics failed", level=log.UNUSUAL) + writeable = False + + if self._readonly: + stats['storage_server.disk_avail'] = 0 + writeable = False + + stats['storage_server.accepting_immutable_shares'] = int(writeable) + + def get_available_space(self): + if self._readonly: + return 0 + try: + return fileutil.get_available_space(self._sharedir, self._reserved_space) + except EnvironmentError: + return 0 + + +class DiskShareSet(ShareSet): + implements(IShareSet) + + def __init__(self, storageindex, sharehomedir, incominghomedir=None): + ShareSet.__init__(self, storageindex) + self._sharehomedir = sharehomedir + self._incominghomedir = incominghomedir + + def get_overhead(self): + return (fileutil.get_used_space(self._sharehomedir) + + fileutil.get_used_space(self._incominghomedir)) + + def get_shares_synchronous(self): + children = fileutil.fp_list(self._sharehomedir) + si = self.get_storage_index() + shares = {} + corrupted = set() + for fp in children: + shnumstr = str(fp.basename()) + if NUM_RE.match(shnumstr): + shnum = int(shnumstr) + try: + shares[shnum] = get_disk_share(fp, si, shnum) + except (UnknownMutableContainerVersionError, + UnknownImmutableContainerVersionError, + struct.error): + corrupted.add(shnum) + + return ([shares[shnum] for shnum in sorted(shares.keys())], corrupted) + + def get_shares(self): + return defer.succeed(self.get_shares_synchronous()) + + def get_share(self, shnum): + return get_disk_share(self._sharehomedir.child(str(shnum)), self.get_storage_index(), shnum) + + def has_incoming(self, shnum): + if self._incominghomedir is None: + return False + return self._incominghomedir.child(str(shnum)).exists() + + def renew_lease(self, renew_secret, new_expiration_time): + found_shares = False + (shares, corrupted) = self.get_shares_synchronous() + for share in shares: + found_shares = True + share.renew_lease(renew_secret, new_expiration_time) + + if not found_shares: + raise IndexError("no such lease to renew") + + def get_leases(self): + # Since all shares get the same lease data, we just grab the leases + # from the first share. + (shares, corrupted) = self.get_shares_synchronous() + if len(shares) > 0: + return shares[0].get_leases() + else: + return iter([]) + + def add_or_renew_lease(self, lease_info): + # This implementation assumes that lease data is duplicated in + # all shares of a shareset, which might not be true for all backends. + (shares, corrupted) = self.get_shares_synchronous() + for share in shares: + share.add_or_renew_lease(lease_info) + + def make_bucket_writer(self, storageserver, shnum, max_space_per_bucket, lease_info, canary): + finalhome = self._sharehomedir.child(str(shnum)) + incominghome = self._incominghomedir.child(str(shnum)) + immsh = create_immutable_disk_share(incominghome, finalhome, max_space_per_bucket, + self.get_storage_index(), shnum) + bw = BucketWriter(storageserver, immsh, lease_info, canary) + return bw + + def _create_mutable_share(self, storageserver, shnum, write_enabler): + fileutil.fp_make_dirs(self._sharehomedir) + sharehome = self._sharehomedir.child(str(shnum)) + serverid = storageserver.get_serverid() + return create_mutable_disk_share(sharehome, serverid, write_enabler, + self.get_storage_index(), shnum, parent=storageserver) + + def _clean_up_after_unlink(self): + fileutil.fp_rmdir_if_empty(self._sharehomedir) + + def _get_sharedir(self): + return self._sharehomedir addfile ./src/allmydata/storage/backends/null/__init__.py addfile ./src/allmydata/storage/backends/null/null_backend.py hunk ./src/allmydata/storage/backends/null/null_backend.py 1 + +from twisted.internet import defer + +from zope.interface import implements +from allmydata.interfaces import IStorageBackend, IShareSet, IShareBase, \ + IShareForReading, IShareForWriting, IMutableShare + +from allmydata.util.assertutil import precondition +from allmydata.storage.backends.base import Backend, ShareSet, empty_check_testv +from allmydata.storage.bucket import BucketWriter, BucketReader +from allmydata.storage.common import si_b2a + + +def configure_null_backend(storedir, config): + return NullBackend() + + +class NullBackend(Backend): + implements(IStorageBackend) + """ + I am a test backend that records (in memory) which shares exist, but not their contents, leases, + or write-enablers. + """ + + def __init__(self): + Backend.__init__(self) + # mapping from storageindex to NullShareSet + self._sharesets = {} + + def get_available_space(self): + return None + + def get_sharesets_for_prefix(self, prefix): + sharesets = [] + for (si, shareset) in self._sharesets.iteritems(): + if si_b2a(si).startswith(prefix): + sharesets.append(shareset) + + def _by_base32si(b): + return b.get_storage_index_string() + sharesets.sort(key=_by_base32si) + return sharesets + + def get_shareset(self, storageindex): + shareset = self._sharesets.get(storageindex, None) + if shareset is None: + shareset = NullShareSet(storageindex) + self._sharesets[storageindex] = shareset + return shareset + + def fill_in_space_stats(self, stats): + pass + + +class NullShareSet(ShareSet): + implements(IShareSet) + + def __init__(self, storageindex): + self.storageindex = storageindex + self._incoming_shnums = set() + self._immutable_shnums = set() + self._mutable_shnums = set() + + def close_shnum(self, shnum): + self._incoming_shnums.remove(shnum) + self._immutable_shnums.add(shnum) + return defer.succeed(None) + + def unlink_shnum(self, shnum): + if shnum in self._incoming_shnums: + self._incoming_shnums.remove(shnum) + if shnum in self._immutable_shnums: + self._immutable_shnums.remove(shnum) + if shnum in self._mutable_shnums: + self._mutable_shnums.remove(shnum) + return defer.succeed(None) + + def get_overhead(self): + return 0 + + def get_shares(self): + shares = {} + for shnum in self._immutable_shnums: + shares[shnum] = ImmutableNullShare(self, shnum) + for shnum in self._mutable_shnums: + shares[shnum] = MutableNullShare(self, shnum) + # This backend never has any corrupt shares. + return defer.succeed( ([shares[shnum] for shnum in sorted(shares.keys())], set()) ) + + def get_share(self, shnum): + if shnum in self._immutable_shnums: + return defer.succeed(ImmutableNullShare(self, shnum)) + elif shnum in self._mutable_shnums: + return defer.succeed(MutableNullShare(self, shnum)) + else: + def _not_found(): raise IndexError("no such share %d" % (shnum,)) + return defer.execute(_not_found) + + def renew_lease(self, renew_secret, new_expiration_time): + raise IndexError("no such lease to renew") + + def get_leases(self): + pass + + def add_or_renew_lease(self, lease_info): + pass + + def has_incoming(self, shnum): + return shnum in self._incoming_shnums + + def get_storage_index(self): + return self.storageindex + + def get_storage_index_string(self): + return si_b2a(self.storageindex) + + def make_bucket_writer(self, storageserver, shnum, max_space_per_bucket, lease_info, canary): + self._incoming_shnums.add(shnum) + immutableshare = ImmutableNullShare(self, shnum) + bw = BucketWriter(storageserver, immutableshare, lease_info, canary) + bw.throw_out_all_data = True + return bw + + def make_bucket_reader(self, storageserver, share): + return BucketReader(storageserver, share) + + +class NullShareBase(object): + implements(IShareBase) + + def __init__(self, shareset, shnum): + self.shareset = shareset + self.shnum = shnum + + def get_storage_index(self): + return self.shareset.get_storage_index() + + def get_storage_index_string(self): + return self.shareset.get_storage_index_string() + + def get_shnum(self): + return self.shnum + + def get_data_length(self): + return 0 + + def get_size(self): + return 0 + + def get_used_space(self): + return 0 + + def unlink(self): + return self.shareset.unlink_shnum(self.shnum) + + def readv(self, readv): + datav = [] + for (offset, length) in readv: + datav.append("") + return defer.succeed(datav) + + def get_leases(self): + pass + + def add_lease(self, lease): + pass + + def renew_lease(self, renew_secret, new_expire_time): + raise IndexError("unable to renew non-existent lease") + + def add_or_renew_lease(self, lease_info): + pass + + +class ImmutableNullShare(NullShareBase): + implements(IShareForReading, IShareForWriting) + sharetype = "immutable" + + def read_share_data(self, offset, length): + precondition(offset >= 0) + return defer.succeed("") + + def get_allocated_size(self): + return 0 + + def write_share_data(self, offset, data): + return defer.succeed(None) + + def close(self): + return self.shareset.close_shnum(self.shnum) + + +class MutableNullShare(NullShareBase): + implements(IMutableShare) + sharetype = "mutable" + + def create(self, serverid, write_enabler): + return defer.succeed(self) + + def check_write_enabler(self, write_enabler): + # Null backend doesn't check write enablers. + return defer.succeed(None) + + def check_testv(self, testv): + return defer.succeed(empty_check_testv(testv)) + + def writev(self, datav, new_length): + return defer.succeed(None) addfile ./src/allmydata/storage/backends/s3/immutable.py hunk ./src/allmydata/storage/backends/s3/immutable.py 1 + +import struct +from cStringIO import StringIO + +from twisted.internet import defer + +from zope.interface import implements +from allmydata.interfaces import IShareBase, IShareForReading, IShareForWriting + +from allmydata.util.assertutil import precondition +from allmydata.storage.common import si_b2a, UnknownImmutableContainerVersionError, DataTooLargeError +from allmydata.storage.backends.s3.s3_common import get_s3_share_key + + +# Each share file (with key 'shares/$PREFIX/$STORAGEINDEX/$SHNUM') contains +# lease information [currently inaccessible] and share data. The share data is +# accessed by RIBucketWriter.write and RIBucketReader.read . + +# The share file has the following layout: +# 0x00: share file version number, four bytes, current version is 1 +# 0x04: always zero (was share data length prior to Tahoe-LAFS v1.3.0) +# 0x08: number of leases, four bytes big-endian +# 0x0c: beginning of share data (see immutable.layout.WriteBucketProxy) +# data_length+0x0c: first lease. Each lease record is 72 bytes. + + +class ImmutableS3ShareBase(object): + implements(IShareBase) + + sharetype = "immutable" + LEASE_SIZE = struct.calcsize(">L32s32sL") # for compatibility + HEADER = ">LLL" + HEADER_SIZE = struct.calcsize(HEADER) + + def __init__(self, s3bucket, storageindex, shnum): + self._s3bucket = s3bucket + self._storageindex = storageindex + self._shnum = shnum + self._key = get_s3_share_key(storageindex, shnum) + + def __repr__(self): + return ("<%s at %r key %r>" % (self.__class__.__name__, self._s3bucket, self._key,)) + + def get_storage_index(self): + return self._storageindex + + def get_storage_index_string(self): + return si_b2a(self._storageindex) + + def get_shnum(self): + return self._shnum + + def get_data_length(self): + return self.get_size() - self.HEADER_SIZE + + def get_used_space(self): + return self.get_size() + + def unlink(self): + self._discard() + try: + return self._s3bucket.delete_object(self._key) + except self._s3bucket.S3Error, e: + if e.get_error_code() != 404: + raise + return defer.succeed(None) + + def get_size(self): + # subclasses should implement + raise NotImplementedError + + # XXX should these lease methods be necessary? + + def get_leases(self): + pass + + def add_lease(self, lease_info): + pass + + def renew_lease(self, renew_secret, new_expire_time): + pass + + def add_or_renew_lease(self, lease_info): + pass + + def cancel_lease(self, cancel_secret): + pass + + def _get_filepath(self): + # For use by tests, only with the mock S3 backend. + # It is OK that _get_filepath doesn't exist on a real S3Bucket object. + return self._s3bucket._get_filepath(self._key) + + +class ImmutableS3ShareForWriting(ImmutableS3ShareBase): + implements(IShareForWriting) + + def __init__(self, s3bucket, storageindex, shnum, max_size, incomingset): + """ + I won't allow more than max_size to be written to me. + """ + precondition(isinstance(max_size, (int, long)), max_size) + ImmutableS3ShareBase.__init__(self, s3bucket, storageindex, shnum) + self._max_size = max_size + + self._buf = StringIO() + # The second field, which was the four-byte share data length in + # Tahoe-LAFS versions prior to 1.3.0, is not used; we always write 0. + # We also write 0 for the number of leases. + self._buf.write(struct.pack(self.HEADER, 1, 0, 0) ) + self._size = self._buf.tell() + + self._incomingset = incomingset + self._incomingset.add( (storageindex, shnum) ) + + def get_size(self): + return self._size + + def get_allocated_size(self): + return self._max_size + + def write_share_data(self, offset, data): + precondition(offset >= 0, offset) + if offset+len(data) > self._max_size: + raise DataTooLargeError(self._max_size, offset, len(data)) + self._buf.seek(self.HEADER_SIZE+offset) + self._buf.write(data) + self._size = self._buf.tell() + return defer.succeed(None) + + def close(self): + # We really want to stream writes to S3, but txaws doesn't support + # that yet (and neither does IS3Bucket, since that's a thin wrapper + # over the txaws S3 API). See + # https://bugs.launchpad.net/txaws/+bug/767205 and + # https://bugs.launchpad.net/txaws/+bug/783801 + data = self._buf.getvalue() + self._discard() + return self._s3bucket.put_object(self._key, data) + + def _discard(self): + self._buf = None + self._incomingset.discard( (self.get_storage_index(), self.get_shnum()) ) + + +class ImmutableS3ShareForReading(ImmutableS3ShareBase): + implements(IShareForReading) + + def __init__(self, s3bucket, storageindex, shnum, data): + ImmutableS3ShareBase.__init__(self, s3bucket, storageindex, shnum) + self._data = data + + header = self._data[:self.HEADER_SIZE] + (version, unused, num_leases) = struct.unpack(self.HEADER, header) + + if version != 1: + msg = "%r had version %d but we wanted 1" % (self, version) + raise UnknownImmutableContainerVersionError(msg) + + # We cannot write leases in share files, but allow them to be present + # in case a share file is copied from a disk backend, or in case we + # need them in future. + self._end_offset = len(self._data) - (num_leases * self.LEASE_SIZE) + + def get_size(self): + return len(self._data) + + def readv(self, readv): + datav = [] + for (offset, length) in readv: + datav.append(self.read_share_data(offset, length)) + return defer.succeed(datav) + + def read_share_data(self, offset, length): + precondition(offset >= 0) + + # Reads beyond the end of the data are truncated. Reads that start + # beyond the end of the data return an empty string. + seekpos = self.HEADER_SIZE+offset + actuallength = max(0, min(length, self._end_offset-seekpos)) + if actuallength == 0: + return defer.succeed("") + return defer.succeed(self._data[seekpos:seekpos+actuallength]) + + def _discard(self): + pass addfile ./src/allmydata/storage/backends/s3/mock_s3.py hunk ./src/allmydata/storage/backends/s3/mock_s3.py 1 + +from twisted.internet import defer + +from zope.interface import implements +from allmydata.storage.backends.s3.s3_common import IS3Bucket +from allmydata.util.time_format import iso_utc +from allmydata.util import fileutil +from allmydata.util.deferredutil import async_iterate + + +def configure_mock_s3_backend(storedir, config): + from allmydata.storage.backends.s3.s3_backend import S3Backend + + corruption_advisory_dir = storedir.child("corruption-advisories") + + s3bucket = MockS3Bucket(storedir) + return S3Backend(s3bucket, corruption_advisory_dir) + + +MAX_KEYS = 1000 + +class MockS3Bucket(object): + implements(IS3Bucket) + """ + I represent a mock S3 bucket that stores its data in the local filesystem, + using a directory structure compatible with the disk backend. + """ + + def __init__(self, storagedir): + self._storagedir = storagedir + self.bucketname = "bucket" + self.model = MockS3Bucket.__module__ + self.S3Error = MockS3Error + + def __repr__(self): + return ("<%s at %r>" % (self.__class__.__name__, self._storagedir,)) + + def create(self): + return defer.execute(self._not_implemented) + + def delete(self): + return defer.execute(self._not_implemented) + + def _iterate_dirs(self): + for prefixdir in fileutil.fp_list(self._storagedir.child("shares")): + prefixstr = prefixdir.basename() + prefixkey = "shares/%s" % (prefixstr,) + for sidir in fileutil.fp_list(prefixdir): + sistr = sidir.basename() + sikey = "%s/%s" % (prefixkey, sistr) + for sharefp in fileutil.fp_list(sidir): + shnumstr = sharefp.basename() + yield (sharefp, "%s/%s" % (sikey, shnumstr)) + + def list_all_objects(self): + contents = [] + def _next_share(res): + if res is None: + return + (sharefp, sharekey) = res + mtime_utc = iso_utc(sharefp.getmtime(), sep=' ')+'+00:00' + item = BucketItem(key=sharekey, modification_date=mtime_utc, etag="", + size=sharefp.getsize(), storage_class="STANDARD") + contents.append(item) + return len(contents) < MAX_KEYS + + d = async_iterate(_next_share, self._iterate_dirs()) + d.addCallback(lambda completed: + BucketListing(self.bucketname, '', '/', MAX_KEYS, + is_truncated=not completed, contents=contents)) + return d + + def _get_filepath(self, object_name, must_exist=False): + # This method is also called by tests. + sharefp = self._storagedir.preauthChild(object_name) + if must_exist and not sharefp.exists(): + raise MockS3Error(404, "not found") + return sharefp + + def put_object(self, object_name, data, content_type=None, metadata={}): + assert content_type is None, content_type + assert metadata == {}, metadata + sharefp = self._get_filepath(object_name) + fileutil.fp_make_dirs(sharefp.parent()) + sharefp.setContent(data) + return defer.succeed(None) + + def get_object(self, object_name): + return defer.succeed(self._get_filepath(object_name, must_exist=True).getContent()) + + def head_object(self, object_name): + return defer.execute(self._not_implemented) + + def delete_object(self, object_name): + self._get_filepath(object_name, must_exist=True).remove() + return defer.succeed(None) + + def _not_implemented(self): + raise NotImplementedError + + +class MockS3Error(Exception): + """ + A error class providing custom methods on S3 errors. + """ + def __init__(self, error_code, error_message, request_id="", host_id=""): + Exception.__init__(self, "%r: %r" % (error_code, error_message)) + self.error_code = error_code + self.error_message = error_message + self.request_id = request_id + self.host_id = host_id + + def get_error_code(self): + return self.error_code + + def get_error_message(self): + return self.error_message + + def parse(self, xml_bytes=""): + raise NotImplementedError + + def has_error(self, errorString): + raise NotImplementedError + + def get_error_codes(self): + raise NotImplementedError + + def get_error_messages(self): + raise NotImplementedError + + +# Copied from txaws.s3.model. This ensures that we can test without needing to import txaws. +# This code was under the MIT / Expat licence. + +class BucketItem(object): + """ + The contents of an Amazon S3 bucket. + """ + def __init__(self, key, modification_date, etag, size, storage_class, + owner=None): + self.key = key + self.modification_date = modification_date + self.etag = etag + self.size = size + self.storage_class = storage_class + self.owner = owner + + +class BucketListing(object): + def __init__(self, name, prefix, marker, max_keys, is_truncated, + contents=None, common_prefixes=None): + self.name = name + self.prefix = prefix + self.marker = marker + self.max_keys = max_keys + self.is_truncated = is_truncated + self.contents = contents + self.common_prefixes = common_prefixes addfile ./src/allmydata/storage/backends/s3/mutable.py hunk ./src/allmydata/storage/backends/s3/mutable.py 1 + +import struct +from cStringIO import StringIO +from types import NoneType + +from twisted.internet import defer + +from zope.interface import implements + +from allmydata.interfaces import IMutableShare, BadWriteEnablerError +from allmydata.util import idlib, log +from allmydata.util.assertutil import precondition +from allmydata.util.hashutil import constant_time_compare +from allmydata.storage.common import si_b2a, UnknownMutableContainerVersionError, \ + DataTooLargeError +from allmydata.storage.backends.base import testv_compare +from allmydata.mutable.layout import MUTABLE_MAGIC +from allmydata.storage.backends.s3.s3_common import get_s3_share_key + + +# The MutableS3Share is like the ImmutableS3Share, but used for mutable data. +# It has a different layout. See docs/mutable.rst for more details. + +# # offset size name +# 1 0 32 magic verstr "tahoe mutable container v1" plus binary +# 2 32 20 write enabler's nodeid +# 3 52 32 write enabler +# 4 84 8 data size (actual share data present) (a) +# 5 92 8 offset of (8) count of extra leases (after data) +# 6 100 368 four leases, 92 bytes each, unused +# 7 468 (a) data +# 8 ?? 4 count of extra leases +# 9 ?? n*92 extra leases + + +# The struct module doc says that L's are 4 bytes in size, and that Q's are +# 8 bytes in size. Since compatibility depends upon this, double-check it. +assert struct.calcsize(">L") == 4, struct.calcsize(">L") +assert struct.calcsize(">Q") == 8, struct.calcsize(">Q") + + +class MutableS3Share(object): + implements(IMutableShare) + + sharetype = "mutable" + DATA_LENGTH_OFFSET = struct.calcsize(">32s20s32s") + EXTRA_LEASE_OFFSET = DATA_LENGTH_OFFSET + 8 + HEADER_SIZE = struct.calcsize(">32s20s32sQQ") # doesn't include leases + LEASE_SIZE = struct.calcsize(">LL32s32s20s") + assert LEASE_SIZE == 92 + DATA_OFFSET = HEADER_SIZE + 4*LEASE_SIZE + assert DATA_OFFSET == 468, DATA_OFFSET + NUM_EXTRA_LEASES_SIZE = struct.calcsize(">L") + + MAGIC = MUTABLE_MAGIC + assert len(MAGIC) == 32 + MAX_SIZE = 2*1000*1000*1000 # 2GB, kind of arbitrary + # TODO: decide upon a policy for max share size + + def __init__(self, s3bucket, data, storageindex, shnum, parent=None): + """ + Clients should use the load_mutable_s3_share and create_mutable_s3_share + factory functions rather than creating instances directly. + """ + precondition(isinstance(data, (str, NoneType)), type(data)) + precondition(isinstance(storageindex, str), storageindex=storageindex) + precondition(isinstance(shnum, int), shnum=shnum) + + self._s3bucket = s3bucket + self._storageindex = storageindex + self._shnum = shnum + self._key = get_s3_share_key(storageindex, shnum) + + # _buf is a file object containing a local copy of the share contents. + self._buf = StringIO() + if data is not None: + (magic, write_enabler_nodeid, write_enabler, + data_length, extra_lease_offset) = struct.unpack(">32s20s32sQQ", data[:self.HEADER_SIZE]) + + if magic != self.MAGIC: + msg = "%r had magic %r but we wanted %r" % (self, magic, self.MAGIC) + raise UnknownMutableContainerVersionError(msg) + + self._buf.write(data) + self._data_length = data_length + + self.parent = parent # for logging + + def __repr__(self): + return ("<%s at %r key %r>" % (self.__class__.__name__, self._s3bucket, self._key,)) + + def log(self, *args, **kwargs): + if self.parent: + return self.parent.log(*args, **kwargs) + + def create(self, serverid, write_enabler): + # Unlike the disk backend, we don't check that the S3 object does not exist; + # we assume that it does not because create was used, and no-one else should be + # writing to the bucket. + + # There are no extra leases, but for compatibility, the offset they would have + # still needs to be stored in the header. + self._data_length = 0 + num_extra_leases = 0 + extra_lease_offset = self.DATA_OFFSET + self._data_length + header = struct.pack(">32s20s32sQQ", + self.MAGIC, serverid, write_enabler, + self._data_length, extra_lease_offset, + ) + leases = ("\x00"*self.LEASE_SIZE) * 4 + self._buf.write(header + leases) + # data goes here, empty at creation + self._buf.write(struct.pack(">L", num_extra_leases)) + + # As an optimization, we don't create empty share objects, we only write the + # data when writev is called. Note that this depends on there being a call + # to writev for shares of empty files, which needs a test. + return self + + def get_storage_index(self): + return self._storageindex + + def get_storage_index_string(self): + return si_b2a(self._storageindex) + + def get_shnum(self): + return self._shnum + + def get_size(self): + self._buf.seek(0, 2) # 2 == os.SEEK_END in Python 2.5+ + return self._buf.tell() + + def get_data_length(self): + return self._data_length + + def get_used_space(self): + # We're not charged for any per-object overheads in S3, so object data sizes are + # what we're interested in for statistics and accounting. + return self.get_size() + + def unlink(self): + self._discard() + try: + return self._s3bucket.delete_object(self._key) + except self._s3bucket.S3Error, e: + if e.get_error_code() != 404: + raise + return defer.succeed(None) + + def _discard(self): + self._buf = None + + def _read_share_data(self, offset, length): + precondition(offset >= 0, offset=offset) + if offset+length > self._data_length: + # reads beyond the end of the data are truncated. Reads that + # start beyond the end of the data return an empty string. + length = max(0, self._data_length-offset) + if length == 0: + return "" + precondition(offset+length <= self._data_length) + self._buf.seek(self.DATA_OFFSET+offset) + data = self._buf.read(length) + return data + + def _write_share_data(self, offset, data): + length = len(data) + precondition(offset >= 0, offset=offset) + precondition(offset+length < self.MAX_SIZE, offset=offset, length=length) + + self._buf.seek(self.DATA_OFFSET + offset) + self._buf.write(data) + if offset+length >= self._data_length: + self._data_length = offset+length + self._change_container_size() + return defer.succeed(None) + + def _change_container_size(self): + new_container_size = self.DATA_OFFSET + self._data_length + self.NUM_EXTRA_LEASES_SIZE + + self._buf.seek(self.DATA_LENGTH_OFFSET) + self._buf.write(struct.pack(">Q", self._data_length)) + + extra_lease_offset = self.DATA_OFFSET + self._data_length + self._buf.seek(self.EXTRA_LEASE_OFFSET) + self._buf.write(struct.pack(">Q", extra_lease_offset)) + + # Just discard any extra leases. + self._buf.seek(extra_lease_offset) + self._buf.write(struct.pack(">L", 0)) + assert self._buf.tell() == new_container_size + self._buf.truncate(new_container_size) + + def readv(self, readv): + datav = [] + for (offset, length) in readv: + datav.append(self._read_share_data(offset, length)) + return defer.succeed(datav) + + def check_write_enabler(self, write_enabler): + self._buf.seek(0) + data = self._buf.read(self.HEADER_SIZE) + (magic, write_enabler_nodeid, real_write_enabler, + data_length, extra_least_offset) = struct.unpack(">32s20s32sQQ", data) + assert magic == self.MAGIC + + # avoid a timing attack + if not constant_time_compare(write_enabler, real_write_enabler): + # accomodate share migration by reporting the nodeid used for the + # old write enabler. + self.log(format="bad write enabler on SI %(si)s," + " recorded by nodeid %(nodeid)s", + facility="tahoe.storage", + level=log.WEIRD, umid="DF2fCR", + si=self.get_storage_index_string(), + nodeid=idlib.nodeid_b2a(write_enabler_nodeid)) + msg = "The write enabler was recorded by nodeid '%s'." % \ + (idlib.nodeid_b2a(write_enabler_nodeid),) + raise BadWriteEnablerError(msg) + return defer.succeed(None) + + def check_testv(self, testv): + test_good = True + for (offset, length, operator, specimen) in testv: + data = self._read_share_data(offset, length) + if not testv_compare(data, operator, specimen): + test_good = False + break + return defer.succeed(test_good) + + def writev(self, datav, new_length): + precondition(new_length is None or new_length >= 0, new_length=new_length) + for (offset, data) in datav: + precondition(offset >= 0, offset=offset) + if offset+len(data) > self.MAX_SIZE: + raise DataTooLargeError() + + for (offset, data) in datav: + self._write_share_data(offset, data) + + # new_length can only be used to truncate, not extend. + if new_length is not None and new_length < self._data_length: + self._data_length = new_length + self._change_container_size() + + # We really want to stream writes to S3, but txaws doesn't support + # that yet (and neither does IS3Bucket, since that's a thin wrapper + # over the txaws S3 API). See + # https://bugs.launchpad.net/txaws/+bug/767205 and + # https://bugs.launchpad.net/txaws/+bug/783801 + data = self._buf.getvalue() + return self._s3bucket.put_object(self._key, data) + + def close(self): + self._discard() + return defer.succeed(None) + + # XXX should these lease methods be necessary? + + def get_leases(self): + pass + + def add_lease(self, lease_info): + pass + + def renew_lease(self, renew_secret, new_expire_time): + pass + + def add_or_renew_lease(self, lease_info): + pass + + def cancel_lease(self, cancel_secret): + pass + + def _get_filepath(self): + # For use by tests, only with the mock S3 backend. + # It is OK that _get_filepath doesn't exist on a real S3Bucket object. + return self._s3bucket._get_filepath(self._key) + + +def load_mutable_s3_share(s3bucket, data, storageindex=None, shnum=None, parent=None): + return MutableS3Share(s3bucket, data, storageindex=storageindex, shnum=shnum, + parent=parent) + +def create_mutable_s3_share(s3bucket, serverid, write_enabler, storageindex=None, shnum=None, parent=None): + return MutableS3Share(s3bucket, None, storageindex=storageindex, shnum=shnum, + parent=parent).create(serverid, write_enabler) addfile ./src/allmydata/storage/backends/s3/s3_backend.py hunk ./src/allmydata/storage/backends/s3/s3_backend.py 1 + +from zope.interface import implements +from allmydata.interfaces import IStorageBackend, IShareSet + +from allmydata.node import InvalidValueError +from allmydata.util.deferredutil import gatherResults +from allmydata.storage.common import si_a2b +from allmydata.storage.bucket import BucketWriter +from allmydata.storage.backends.base import Backend, ShareSet +from allmydata.storage.backends.s3.immutable import ImmutableS3ShareForReading, ImmutableS3ShareForWriting +from allmydata.storage.backends.s3.mutable import load_mutable_s3_share, create_mutable_s3_share +from allmydata.storage.backends.s3.s3_common import get_s3_share_key, list_objects, NUM_RE +from allmydata.mutable.layout import MUTABLE_MAGIC + + +def get_s3_share(s3bucket, storageindex, shnum): + key = get_s3_share_key(storageindex, shnum) + d = s3bucket.get_object(key) + def _make_share(data): + if data.startswith(MUTABLE_MAGIC): + return load_mutable_s3_share(s3bucket, data, storageindex, shnum) + else: + # assume it's immutable + return ImmutableS3ShareForReading(s3bucket, storageindex, shnum, data=data) + d.addCallback(_make_share) + return d + + +def configure_s3_backend(storedir, config): + from allmydata.storage.backends.s3.s3_bucket import S3Bucket + + if config.get_config("storage", "readonly", False, boolean=True): + raise InvalidValueError("[storage]readonly is not supported by the S3 backend; " + "make the S3 bucket read-only instead.") + + corruption_advisory_dir = storedir.child("corruption-advisories") + + accesskeyid = config.get_config("storage", "s3.access_key_id") + secretkey = config.get_or_create_private_config("s3secret") + usertoken = config.get_optional_private_config("s3usertoken") + producttoken = config.get_optional_private_config("s3producttoken") + if producttoken and not usertoken: + raise InvalidValueError("If private/s3producttoken is present, private/s3usertoken must also be present.") + url = config.get_config("storage", "s3.url", "http://s3.amazonaws.com") + bucketname = config.get_config("storage", "s3.bucket") + + s3bucket = S3Bucket(accesskeyid, secretkey, url, bucketname, usertoken, producttoken) + return S3Backend(s3bucket, corruption_advisory_dir) + + +class S3Backend(Backend): + implements(IStorageBackend) + + def __init__(self, s3bucket, corruption_advisory_dir=None): + Backend.__init__(self) + self._s3bucket = s3bucket + + # we don't actually create the corruption-advisory dir until necessary + self._corruption_advisory_dir = corruption_advisory_dir + + # set of (storageindex, shnum) of incoming shares + self._incomingset = set() + + def get_sharesets_for_prefix(self, prefix): + d = list_objects(self._s3bucket, 'shares/%s/' % (prefix,)) + def _get_sharesets(res): + # XXX this enumerates all shares to get the set of SIs. + # Is there a way to enumerate SIs more efficiently? + si_strings = set() + for item in res.contents: + # XXX better error handling + path = item.key.split('/') + assert path[0:2] == ["shares", prefix] + si_strings.add(path[2]) + + # XXX we want this to be deterministic, so we return the sharesets sorted + # by their si_strings, but we shouldn't need to explicitly re-sort them + # because list_objects returns a sorted list. + return [S3ShareSet(si_a2b(s), self._s3bucket, self._incomingset) for s in sorted(si_strings)] + d.addCallback(_get_sharesets) + return d + + def get_shareset(self, storageindex): + return S3ShareSet(storageindex, self._s3bucket, self._incomingset) + + def fill_in_space_stats(self, stats): + # TODO: query space usage of S3 bucket + # TODO: query whether the bucket is read-only and set + # accepting_immutable_shares accordingly. + stats['storage_server.accepting_immutable_shares'] = 1 + + def get_available_space(self): + # TODO: query space usage of S3 bucket + return 2**64 + + +class S3ShareSet(ShareSet): + implements(IShareSet) + + def __init__(self, storageindex, s3bucket, incomingset): + ShareSet.__init__(self, storageindex) + self._s3bucket = s3bucket + self._incomingset = incomingset + self._key = get_s3_share_key(storageindex) + + def get_overhead(self): + return 0 + + def get_shares(self): + d = list_objects(self._s3bucket, self._key) + def _get_shares(res): + si = self.get_storage_index() + shnums = [] + for item in res.contents: + assert item.key.startswith(self._key), item.key + path = item.key.split('/') + if len(path) == 4: + shnumstr = path[3] + if NUM_RE.match(shnumstr): + shnums.append(int(shnumstr)) + + return gatherResults([get_s3_share(self._s3bucket, si, shnum) + for shnum in sorted(shnums)]) + d.addCallback(_get_shares) + # TODO: return information about corrupt shares. + d.addCallback(lambda shares: (shares, set()) ) + return d + + def get_share(self, shnum): + return get_s3_share(self._s3bucket, self.get_storage_index(), shnum) + + def has_incoming(self, shnum): + return (self.get_storage_index(), shnum) in self._incomingset + + def make_bucket_writer(self, storageserver, shnum, max_space_per_bucket, lease_info, canary): + immsh = ImmutableS3ShareForWriting(self._s3bucket, self.get_storage_index(), shnum, + max_space_per_bucket, self._incomingset) + return BucketWriter(storageserver, immsh, lease_info, canary) + + def _create_mutable_share(self, storageserver, shnum, write_enabler): + serverid = storageserver.get_serverid() + return create_mutable_s3_share(self._s3bucket, serverid, write_enabler, + self.get_storage_index(), shnum, parent=storageserver) + + def _clean_up_after_unlink(self): + pass + + def _get_sharedir(self): + # For use by tests, only with the mock S3 backend. + # It is OK that _get_filepath doesn't exist on a real S3Bucket object. + return self._s3bucket._get_filepath(self._key) + + def get_leases(self): + raise NotImplementedError + + def add_or_renew_lease(self, lease_info): + raise NotImplementedError + + def renew_lease(self, renew_secret, new_expiration_time): + raise NotImplementedError addfile ./src/allmydata/storage/backends/s3/s3_bucket.py hunk ./src/allmydata/storage/backends/s3/s3_bucket.py 1 + +from twisted.internet.defer import maybeDeferred + +from zope.interface import implements +from allmydata.storage.backends.s3.s3_common import IS3Bucket + + +class S3Bucket(object): + implements(IS3Bucket) + """ + I represent a real S3 bucket, accessed using the txaws library. + """ + + def __init__(self, access_key, secret_key, url, bucketname, usertoken=None, producttoken=None): + # We only depend on txaws when this class is actually instantiated. + from txaws.credentials import AWSCredentials + from txaws.service import AWSServiceEndpoint + from txaws.s3.client import S3Client, Query + from txaws.s3 import model + from txaws.s3.exception import S3Error + + creds = AWSCredentials(access_key=access_key, secret_key=secret_key) + endpoint = AWSServiceEndpoint(uri=url) + + query_factory = None + if usertoken is not None: + def make_query(*args, **kwargs): + amz_headers = kwargs.get("amz_headers", {}) + if producttoken is not None: + amz_headers["security-token"] = (usertoken, producttoken) + else: + amz_headers["security-token"] = usertoken + kwargs["amz_headers"] = amz_headers + + return Query(*args, **kwargs) + query_factory = make_query + + self.client = S3Client(creds=creds, endpoint=endpoint, query_factory=query_factory) + self.bucketname = bucketname + self.model = model + self.S3Error = S3Error + + def __repr__(self): + return ("<%s %r>" % (self.__class__.__name__, self.bucketname,)) + + def create(self): + return maybeDeferred(self.client.create, self.bucketname) + + def delete(self): + return maybeDeferred(self.client.delete, self.bucketname) + + # We want to be able to do prefix queries, but txaws 0.2 doesn't implement that. + def list_all_objects(self): + return maybeDeferred(self.client.get_bucket, self.bucketname) + + def put_object(self, object_name, data, content_type=None, metadata={}): + return maybeDeferred(self.client.put_object, self.bucketname, + object_name, data, content_type, metadata) + + def get_object(self, object_name): + return maybeDeferred(self.client.get_object, self.bucketname, object_name) + + def head_object(self, object_name): + return maybeDeferred(self.client.head_object, self.bucketname, object_name) + + def delete_object(self, object_name): + return maybeDeferred(self.client.delete_object, self.bucketname, object_name) + + def put_policy(self, policy): + """ + Set access control policy on a bucket. + """ + query = self.client.query_factory( + action='PUT', creds=self.client.creds, endpoint=self.client.endpoint, + bucket=self.bucketname, object_name='?policy', data=policy) + return maybeDeferred(query.submit) + + def get_policy(self): + query = self.client.query_factory( + action='GET', creds=self.client.creds, endpoint=self.client.endpoint, + bucket=self.bucketname, object_name='?policy') + return maybeDeferred(query.submit) + + def delete_policy(self): + query = self.client.query_factory( + action='DELETE', creds=self.client.creds, endpoint=self.client.endpoint, + bucket=self.bucketname, object_name='?policy') + return maybeDeferred(query.submit) addfile ./src/allmydata/storage/backends/s3/s3_common.py hunk ./src/allmydata/storage/backends/s3/s3_common.py 1 + +import re + +from zope.interface import Interface + +from allmydata.storage.common import si_b2a + + +# The S3 bucket has keys of the form shares/$PREFIX/$STORAGEINDEX/$SHNUM . + +def get_s3_share_key(si, shnum=None): + sistr = si_b2a(si) + if shnum is None: + return "shares/%s/%s/" % (sistr[:2], sistr) + else: + return "shares/%s/%s/%d" % (sistr[:2], sistr, shnum) + +def list_objects(s3bucket, prefix, marker='/'): + # XXX we want to be able to implement this in terms of a prefix query. Fake it for now. + #d = self._s3bucket.list_objects('shares/%s/' % (prefix,), marker) + d = s3bucket.list_all_objects() + def _filter(res): + res.contents = [item for item in res.contents if item.key.startswith(prefix)] + return res + d.addCallback(_filter) + return d + +NUM_RE=re.compile("^[0-9]+$") + + +class IS3Bucket(Interface): + """ + I represent an S3 bucket. + """ + def create(): + """ + Create this bucket. + """ + + def delete(): + """ + Delete this bucket. + The bucket must be empty before it can be deleted. + """ + + def list_all_objects(): + """ + Get a BucketListing that lists all the objects in this bucket. + """ + + def put_object(object_name, data, content_type=None, metadata={}): + """ + Put an object in this bucket. + Any existing object of the same name will be replaced. + """ + + def get_object(object_name): + """ + Get an object from this bucket. + """ + + def head_object(object_name): + """ + Retrieve object metadata only. + """ + + def delete_object(object_name): + """ + Delete an object from this bucket. + Once deleted, there is no method to restore or undelete an object. + """ addfile ./src/allmydata/storage/bucket.py hunk ./src/allmydata/storage/bucket.py 1 + +import time + +from foolscap.api import Referenceable +from twisted.internet import defer + +from zope.interface import implements +from allmydata.interfaces import RIBucketWriter, RIBucketReader + +from allmydata.util import base32, log +from allmydata.util.assertutil import precondition + + +class BucketWriter(Referenceable): + implements(RIBucketWriter) + + def __init__(self, ss, immutableshare, lease_info, canary): + self.ss = ss + self._canary = canary + self._disconnect_marker = canary.notifyOnDisconnect(self._disconnected) + self.closed = False + self.throw_out_all_data = False + self._share = immutableshare + # also, add our lease to the file now, so that other ones can be + # added by simultaneous uploaders + self._share.add_lease(lease_info) + + def allocated_size(self): + return self._share.get_allocated_size() + + def _add_latency(self, res, name, start): + self.ss.add_latency(name, time.time() - start) + self.ss.count(name) + return res + + def remote_write(self, offset, data): + start = time.time() + precondition(not self.closed) + if self.throw_out_all_data: + return defer.succeed(None) + d = self._share.write_share_data(offset, data) + d.addBoth(self._add_latency, "write", start) + return d + + def remote_close(self): + precondition(not self.closed) + start = time.time() + + d = defer.succeed(None) + d.addCallback(lambda ign: self._share.close()) + # XXX should this be self._share.get_used_space() ? + d.addCallback(lambda ign: self._share.get_size()) + def _got_size(consumed_size): + self._share = None + self.closed = True + self._canary.dontNotifyOnDisconnect(self._disconnect_marker) + + self.ss.bucket_writer_closed(self, consumed_size) + d.addCallback(_got_size) + d.addBoth(self._add_latency, "close", start) + return d + + def _disconnected(self): + if not self.closed: + return self._abort() + return defer.succeed(None) + + def remote_abort(self): + log.msg("storage: aborting write to share %r" % self._share, + facility="tahoe.storage", level=log.UNUSUAL) + if not self.closed: + self._canary.dontNotifyOnDisconnect(self._disconnect_marker) + d = self._abort() + def _count(ign): + self.ss.count("abort") + d.addBoth(_count) + return d + + def _abort(self): + d = defer.succeed(None) + if self.closed: + return d + d.addCallback(lambda ign: self._share.unlink()) + def _unlinked(ign): + self._share = None + + # We are now considered closed for further writing. We must tell + # the storage server about this so that it stops expecting us to + # use the space it allocated for us earlier. + self.closed = True + self.ss.bucket_writer_closed(self, 0) + d.addCallback(_unlinked) + return d + + +class BucketReader(Referenceable): + implements(RIBucketReader) + + def __init__(self, ss, share): + self.ss = ss + self._share = share + self.storageindex = share.get_storage_index() + self.shnum = share.get_shnum() + + def __repr__(self): + return "<%s %s %s>" % (self.__class__.__name__, + base32.b2a_l(self.storageindex[:8], 60), + self.shnum) + + def _add_latency(self, res, name, start): + self.ss.add_latency(name, time.time() - start) + self.ss.count(name) + return res + + def remote_read(self, offset, length): + start = time.time() + d = self._share.read_share_data(offset, length) + d.addBoth(self._add_latency, "read", start) + return d + + def remote_advise_corrupt_share(self, reason): + return self.ss.remote_advise_corrupt_share("immutable", + self.storageindex, + self.shnum, + reason) hunk ./src/allmydata/storage/shares.py 1 -#! /usr/bin/python - -from allmydata.storage.mutable import MutableShareFile -from allmydata.storage.immutable import ShareFile - -def get_share_file(filename): - f = open(filename, "rb") - prefix = f.read(32) - f.close() - if prefix == MutableShareFile.MAGIC: - return MutableShareFile(filename) - # otherwise assume it's immutable - return ShareFile(filename) - rmfile ./src/allmydata/storage/shares.py } [Cosmetic changes in pluggable backends branch. refs #999, #1569 david-sarah@jacaranda.org**20111216181207 Ignore-this: ea6cf274dd733abba20032197ed17beb ] { hunk ./src/allmydata/client.py 225 # searches. seed = base32.b2a(self.nodeid) else: - # otherwise, we're free to use the more natural seed of our - # pubkey-based serverid + # Otherwise, we're free to use the more natural seed of our + # pubkey-based serverid. vk = self._server_key.get_verifying_key() seed = vk.to_ascii(encoding="base32") self.write_config("permutation-seed", seed+"\n") hunk ./src/allmydata/client.py 233 return seed.strip() def init_storage(self): - # should we run a storage server (and publish it for others to use)? + # Should we run a storage server (and publish it for others to use)? if not self.get_config("storage", "enabled", True, boolean=True): return readonly = self.get_config("storage", "readonly", False, boolean=True) hunk ./src/allmydata/immutable/offloaded.py 306 if os.path.exists(self._encoding_file): self.log("ciphertext already present, bypassing fetch", level=log.UNUSUAL) + # XXX the following comment is probably stale, since + # LocalCiphertextReader.get_plaintext_hashtree_leaves does not exist. + # # we'll still need the plaintext hashes (when # LocalCiphertextReader.get_plaintext_hashtree_leaves() is # called), and currently the easiest way to get them is to ask hunk ./src/allmydata/immutable/upload.py 765 self._status.set_progress(1, progress) return cryptdata - def get_plaintext_hashtree_leaves(self, first, last, num_segments): hunk ./src/allmydata/immutable/upload.py 766 + """OBSOLETE; Get the leaf nodes of a merkle hash tree over the + plaintext segments, i.e. get the tagged hashes of the given segments. + The segment size is expected to be generated by the + IEncryptedUploadable before any plaintext is read or ciphertext + produced, so that the segment hashes can be generated with only a + single pass. + + This returns a Deferred that fires with a sequence of hashes, using: + + tuple(segment_hashes[first:last]) + + 'num_segments' is used to assert that the number of segments that the + IEncryptedUploadable handled matches the number of segments that the + encoder was expecting. + + This method must not be called until the final byte has been read + from read_encrypted(). Once this method is called, read_encrypted() + can never be called again. + """ # this is currently unused, but will live again when we fix #453 if len(self._plaintext_segment_hashes) < num_segments: # close out the last one hunk ./src/allmydata/immutable/upload.py 803 return defer.succeed(tuple(self._plaintext_segment_hashes[first:last])) def get_plaintext_hash(self): + """OBSOLETE; Get the hash of the whole plaintext. + + This returns a Deferred that fires with a tagged SHA-256 hash of the + whole plaintext, obtained from hashutil.plaintext_hash(data). + """ + # this is currently unused, but will live again when we fix #453 h = self._plaintext_hasher.digest() return defer.succeed(h) hunk ./src/allmydata/interfaces.py 29 Number = IntegerConstraint(8) # 2**(8*8) == 16EiB ~= 18e18 ~= 18 exabytes Offset = Number ReadSize = int # the 'int' constraint is 2**31 == 2Gib -- large files are processed in not-so-large increments -WriteEnablerSecret = Hash # used to protect mutable bucket modifications -LeaseRenewSecret = Hash # used to protect bucket lease renewal requests -LeaseCancelSecret = Hash # used to protect bucket lease cancellation requests +WriteEnablerSecret = Hash # used to protect mutable share modifications +LeaseRenewSecret = Hash # used to protect lease renewal requests +LeaseCancelSecret = Hash # formerly used to protect lease cancellation requests class RIStubClient(RemoteInterface): """Each client publishes a service announcement for a dummy object called hunk ./src/allmydata/interfaces.py 59 """ return None + class RIBucketReader(RemoteInterface): def read(offset=Offset, length=ReadSize): return ShareData hunk ./src/allmydata/interfaces.py 76 documentation. """ + TestVector = ListOf(TupleOf(Offset, ReadSize, str, str)) # elements are (offset, length, operator, specimen) # operator is one of "lt, le, eq, ne, ge, gt" hunk ./src/allmydata/interfaces.py 93 ReadData = ListOf(ShareData) # returns data[offset:offset+length] for each element of TestVector + class RIStorageServer(RemoteInterface): __remote_name__ = "RIStorageServer.tahoe.allmydata.com" hunk ./src/allmydata/interfaces.py 109 sharenums=SetOf(int, maxLength=MAX_BUCKETS), allocated_size=Offset, canary=Referenceable): """ - @param storage_index: the index of the bucket to be created or + @param storage_index: the index of the shareset to be created or increfed. @param sharenums: these are the share numbers (probably between 0 and 99) that the sender is proposing to store on this hunk ./src/allmydata/interfaces.py 114 server. - @param renew_secret: This is the secret used to protect bucket refresh + @param renew_secret: This is the secret used to protect lease renewal. This secret is generated by the client and stored for later comparison by the server. Each server is given a different secret. hunk ./src/allmydata/interfaces.py 118 - @param cancel_secret: Like renew_secret, but protects bucket decref. - @param canary: If the canary is lost before close(), the bucket is + @param cancel_secret: ignored + @param canary: If the canary is lost before close(), the allocation is deleted. @return: tuple of (alreadygot, allocated), where alreadygot is what we already have and allocated is what we hereby agree to accept. hunk ./src/allmydata/interfaces.py 132 renew_secret=LeaseRenewSecret, cancel_secret=LeaseCancelSecret): """ - Add a new lease on the given bucket. If the renew_secret matches an + Add a new lease on the given shareset. If the renew_secret matches an existing lease, that lease will be renewed instead. If there is no hunk ./src/allmydata/interfaces.py 134 - bucket for the given storage_index, return silently. (note that in + shareset for the given storage_index, return silently. (Note that in tahoe-1.3.0 and earlier, IndexError was raised if there was no hunk ./src/allmydata/interfaces.py 136 - bucket) + shareset.) """ return Any() # returns None now, but future versions might change hunk ./src/allmydata/interfaces.py 142 def renew_lease(storage_index=StorageIndex, renew_secret=LeaseRenewSecret): """ - Renew the lease on a given bucket, resetting the timer to 31 days. - Some networks will use this, some will not. If there is no bucket for + Renew the lease on a given shareset, resetting the timer to 31 days. + Some networks will use this, some will not. If there is no shareset for the given storage_index, IndexError will be raised. For mutable shares, if the given renew_secret does not match an hunk ./src/allmydata/interfaces.py 149 existing lease, IndexError will be raised with a note listing the server-nodeids on the existing leases, so leases on migrated shares - can be renewed or cancelled. For immutable shares, IndexError - (without the note) will be raised. + can be renewed. For immutable shares, IndexError (without the note) + will be raised. """ return Any() hunk ./src/allmydata/interfaces.py 157 def get_buckets(storage_index=StorageIndex): return DictOf(int, RIBucketReader, maxKeys=MAX_BUCKETS) - - def slot_readv(storage_index=StorageIndex, shares=ListOf(int), readv=ReadVector): """Read a vector from the numbered shares associated with the given hunk ./src/allmydata/interfaces.py 171 tw_vectors=TestAndWriteVectorsForShares, r_vector=ReadVector, ): - """General-purpose test-and-set operation for mutable slots. Perform - a bunch of comparisons against the existing shares. If they all pass, - then apply a bunch of write vectors to those shares. Then use the - read vectors to extract data from all the shares and return the data. + """ + General-purpose atomic test-read-and-set operation for mutable slots. + Perform a bunch of comparisons against the existing shares. If they + all pass: use the read vectors to extract data from all the shares, + then apply a bunch of write vectors to those shares. Return the read + data, which does not include any modifications made by the writes. This method is, um, large. The goal is to allow clients to update all the shares associated with a mutable file in a single round trip. hunk ./src/allmydata/interfaces.py 181 - @param storage_index: the index of the bucket to be created or + @param storage_index: the index of the shareset to be created or increfed. @param write_enabler: a secret that is stored along with the slot. Writes are accepted from any caller who can hunk ./src/allmydata/interfaces.py 187 present the matching secret. A different secret should be used for each slot*server pair. - @param renew_secret: This is the secret used to protect bucket refresh + @param renew_secret: This is the secret used to protect lease renewal. This secret is generated by the client and stored for later comparison by the server. Each server is given a different secret. hunk ./src/allmydata/interfaces.py 300 @return: a Deferred that fires (with None) when the operation completes """ - def put_crypttext_hashes(hashes=ListOf(Hash)): + def put_crypttext_hashes(hashes): """ hunk ./src/allmydata/interfaces.py 302 + @param hashes=ListOf(Hash) @return: a Deferred that fires (with None) when the operation completes """ hunk ./src/allmydata/interfaces.py 306 - def put_block_hashes(blockhashes=ListOf(Hash)): + def put_block_hashes(blockhashes): """ hunk ./src/allmydata/interfaces.py 308 + @param blockhashes=ListOf(Hash) @return: a Deferred that fires (with None) when the operation completes """ hunk ./src/allmydata/interfaces.py 312 - def put_share_hashes(sharehashes=ListOf(TupleOf(int, Hash))): + def put_share_hashes(sharehashes): """ hunk ./src/allmydata/interfaces.py 314 + @param sharehashes=ListOf(TupleOf(int, Hash)) @return: a Deferred that fires (with None) when the operation completes """ hunk ./src/allmydata/interfaces.py 318 - def put_uri_extension(data=URIExtensionData): + def put_uri_extension(data): """This block of data contains integrity-checking information (hashes of plaintext, crypttext, and shares), as well as encoding parameters that are necessary to recover the data. This is a serialized dict hunk ./src/allmydata/interfaces.py 323 mapping strings to other strings. The hash of this data is kept in - the URI and verified before any of the data is used. All buckets for - a given file contain identical copies of this data. + the URI and verified before any of the data is used. All share + containers for a given file contain identical copies of this data. The serialization format is specified with the following pseudocode: for k in sorted(dict.keys()): hunk ./src/allmydata/interfaces.py 331 assert re.match(r'^[a-zA-Z_\-]+$', k) write(k + ':' + netstring(dict[k])) + @param data=URIExtensionData @return: a Deferred that fires (with None) when the operation completes """ hunk ./src/allmydata/interfaces.py 346 class IStorageBucketReader(Interface): - def get_block_data(blocknum=int, blocksize=int, size=int): + def get_block_data(blocknum, blocksize, size): """Most blocks will be the same size. The last block might be shorter than the others. hunk ./src/allmydata/interfaces.py 350 + @param blocknum=int + @param blocksize=int + @param size=int @return: ShareData """ hunk ./src/allmydata/interfaces.py 361 @return: ListOf(Hash) """ - def get_block_hashes(at_least_these=SetOf(int)): + def get_block_hashes(at_least_these=()): """ hunk ./src/allmydata/interfaces.py 363 + @param at_least_these=SetOf(int) @return: ListOf(Hash) """ hunk ./src/allmydata/interfaces.py 476 Add the encrypted private key to the share. """ - def put_blockhashes(blockhashes=list): + def put_blockhashes(blockhashes): """ hunk ./src/allmydata/interfaces.py 478 + @param blockhashes=list Add the block hash tree to the share. """ hunk ./src/allmydata/interfaces.py 482 - def put_sharehashes(sharehashes=dict): + def put_sharehashes(sharehashes): """ hunk ./src/allmydata/interfaces.py 484 + @param sharehashes=dict Add the share hash chain to the share. """ hunk ./src/allmydata/interfaces.py 1572 Block Hash, and the encoding parameters, both of which must be included in the URI. - I do not choose shareholders, that is left to the IUploader. I must be - given a dict of RemoteReferences to storage buckets that are ready and - willing to receive data. + I do not choose shareholders, that is left to the IUploader. """ def set_size(size): hunk ./src/allmydata/interfaces.py 1579 """Specify the number of bytes that will be encoded. This must be peformed before get_serialized_params() can be called. """ + def set_params(params): """Override the default encoding parameters. 'params' is a tuple of (k,d,n), where 'k' is the number of required shares, 'd' is the hunk ./src/allmydata/interfaces.py 1675 download, validate, decode, and decrypt data from them, writing the results to an output file. - I do not locate the shareholders, that is left to the IDownloader. I must - be given a dict of RemoteReferences to storage buckets that are ready to - send data. + I do not locate the shareholders, that is left to the IDownloader. """ def setup(outfile): hunk ./src/allmydata/interfaces.py 2103 def get_storage_index(): """Return a string with the (binary) storage index.""" + def get_storage_index_string(): """Return a string with the (printable) abbreviated storage index.""" hunk ./src/allmydata/interfaces.py 2106 + def get_uri(): """Return the (string) URI of the object that was checked.""" hunk ./src/allmydata/interfaces.py 2201 def get_report(): """Return a list of strings with more detailed results.""" + class ICheckAndRepairResults(Interface): """I contain the detailed results of a check/verify/repair operation. hunk ./src/allmydata/interfaces.py 2211 def get_storage_index(): """Return a string with the (binary) storage index.""" + def get_storage_index_string(): """Return a string with the (printable) abbreviated storage index.""" hunk ./src/allmydata/interfaces.py 2214 + def get_repair_attempted(): """Return a boolean, True if a repair was attempted. We might not attempt to repair the file because it was healthy, or healthy enough hunk ./src/allmydata/interfaces.py 2220 (i.e. some shares were missing but not enough to exceed some threshold), or because we don't know how to repair this object.""" + def get_repair_successful(): """Return a boolean, True if repair was attempted and the file/dir was fully healthy afterwards. False if no repair was attempted or if hunk ./src/allmydata/interfaces.py 2225 a repair attempt failed.""" + def get_pre_repair_results(): """Return an ICheckResults instance that describes the state of the file/dir before any repair was attempted.""" hunk ./src/allmydata/interfaces.py 2229 + def get_post_repair_results(): """Return an ICheckResults instance that describes the state of the file/dir after any repair was attempted. If no repair was attempted, hunk ./src/allmydata/interfaces.py 2463 (childnode, metadata_dict) tuples), the directory will be populated with those children, otherwise it will be empty.""" + class IClientStatus(Interface): def list_all_uploads(): """Return a list of uploader objects, one for each upload that hunk ./src/allmydata/interfaces.py 2469 currently has an object available (tracked with weakrefs). This is intended for debugging purposes.""" + def list_active_uploads(): """Return a list of active IUploadStatus objects.""" hunk ./src/allmydata/interfaces.py 2472 + def list_recent_uploads(): """Return a list of IUploadStatus objects for the most recently started uploads.""" hunk ./src/allmydata/interfaces.py 2481 """Return a list of downloader objects, one for each download that currently has an object available (tracked with weakrefs). This is intended for debugging purposes.""" + def list_active_downloads(): """Return a list of active IDownloadStatus objects.""" hunk ./src/allmydata/interfaces.py 2484 + def list_recent_downloads(): """Return a list of IDownloadStatus objects for the most recently started downloads.""" hunk ./src/allmydata/interfaces.py 2489 + class IUploadStatus(Interface): def get_started(): """Return a timestamp (float with seconds since epoch) indicating hunk ./src/allmydata/interfaces.py 2494 when the operation was started.""" + def get_storage_index(): """Return a string with the (binary) storage index in use on this upload. Returns None if the storage index has not yet been hunk ./src/allmydata/interfaces.py 2499 calculated.""" + def get_size(): """Return an integer with the number of bytes that will eventually be uploaded for this file. Returns None if the size is not yet known. hunk ./src/allmydata/interfaces.py 2504 """ + def using_helper(): """Return True if this upload is using a Helper, False if not.""" hunk ./src/allmydata/interfaces.py 2507 + def get_status(): """Return a string describing the current state of the upload process.""" hunk ./src/allmydata/interfaces.py 2511 + def get_progress(): """Returns a tuple of floats, (chk, ciphertext, encode_and_push), each from 0.0 to 1.0 . 'chk' describes how much progress has been hunk ./src/allmydata/interfaces.py 2523 process has finished: for helper uploads this is dependent upon the helper providing progress reports. It might be reasonable to add all three numbers and report the sum to the user.""" + def get_active(): """Return True if the upload is currently active, False if not.""" hunk ./src/allmydata/interfaces.py 2526 + def get_results(): """Return an instance of UploadResults (which contains timing and sharemap information). Might return None if the upload is not yet hunk ./src/allmydata/interfaces.py 2531 finished.""" + def get_counter(): """Each upload status gets a unique number: this method returns that number. This provides a handle to this particular upload, so a web hunk ./src/allmydata/interfaces.py 2537 page can generate a suitable hyperlink.""" + class IDownloadStatus(Interface): def get_started(): """Return a timestamp (float with seconds since epoch) indicating hunk ./src/allmydata/interfaces.py 2542 when the operation was started.""" + def get_storage_index(): """Return a string with the (binary) storage index in use on this download. This may be None if there is no storage index (i.e. LIT hunk ./src/allmydata/interfaces.py 2547 files).""" + def get_size(): """Return an integer with the number of bytes that will eventually be retrieved for this file. Returns None if the size is not yet known. hunk ./src/allmydata/interfaces.py 2552 """ + def using_helper(): """Return True if this download is using a Helper, False if not.""" hunk ./src/allmydata/interfaces.py 2555 + def get_status(): """Return a string describing the current state of the download process.""" hunk ./src/allmydata/interfaces.py 2559 + def get_progress(): """Returns a float (from 0.0 to 1.0) describing the amount of the download that has completed. This value will remain at 0.0 until the hunk ./src/allmydata/interfaces.py 2564 first byte of plaintext is pushed to the download target.""" + def get_active(): """Return True if the download is currently active, False if not.""" hunk ./src/allmydata/interfaces.py 2567 + def get_counter(): """Each download status gets a unique number: this method returns that number. This provides a handle to this particular download, so a hunk ./src/allmydata/interfaces.py 2573 web page can generate a suitable hyperlink.""" + class IServermapUpdaterStatus(Interface): pass hunk ./src/allmydata/interfaces.py 2576 + + class IPublishStatus(Interface): pass hunk ./src/allmydata/interfaces.py 2580 + + class IRetrieveStatus(Interface): pass hunk ./src/allmydata/interfaces.py 2585 + class NotCapableError(Exception): """You have tried to write to a read-only node.""" hunk ./src/allmydata/interfaces.py 2589 + class BadWriteEnablerError(Exception): pass hunk ./src/allmydata/interfaces.py 2649 return DictOf(str, float) + UploadResults = Any() #DictOf(str, str) hunk ./src/allmydata/interfaces.py 2652 + class RIEncryptedUploadable(RemoteInterface): __remote_name__ = "RIEncryptedUploadable.tahoe.allmydata.com" hunk ./src/allmydata/interfaces.py 2725 """ return DictOf(str, DictOf(str, ChoiceOf(float, int, long, None))) + class RIStatsGatherer(RemoteInterface): __remote_name__ = "RIStatsGatherer.tahoe.allmydata.com" """ hunk ./src/allmydata/interfaces.py 2765 class FileTooLargeError(Exception): pass + class IValidatedThingProxy(Interface): def start(): """ Acquire a thing and validate it. Return a deferred that is hunk ./src/allmydata/interfaces.py 2772 eventually fired with self if the thing is valid or errbacked if it can't be acquired or validated.""" + class InsufficientVersionError(Exception): def __init__(self, needed, got): self.needed = needed hunk ./src/allmydata/interfaces.py 2781 return "InsufficientVersionError(need '%s', got %s)" % (self.needed, self.got) + class EmptyPathnameComponentError(Exception): """The webapi disallows empty pathname components.""" hunk ./src/allmydata/mutable/filenode.py 742 self._writekey = writekey self._serializer = defer.succeed(None) - def get_sequence_number(self): """ Get the sequence number of the mutable version that I represent. hunk ./src/allmydata/mutable/filenode.py 749 return self._version[0] # verinfo[0] == the sequence number - # TODO: Terminology? def get_writekey(self): """ I return a writekey or None if I don't have a writekey. hunk ./src/allmydata/mutable/filenode.py 755 """ return self._writekey - def set_downloader_hints(self, hints): """ I set the downloader hints. hunk ./src/allmydata/mutable/filenode.py 763 self._downloader_hints = hints - def get_downloader_hints(self): """ I return the downloader hints. hunk ./src/allmydata/mutable/filenode.py 769 """ return self._downloader_hints - def overwrite(self, new_contents): """ I overwrite the contents of this mutable file version with the hunk ./src/allmydata/mutable/filenode.py 778 return self._do_serialized(self._overwrite, new_contents) - def _overwrite(self, new_contents): assert IMutableUploadable.providedBy(new_contents) assert self._servermap.get_last_update()[0] == MODE_WRITE hunk ./src/allmydata/mutable/filenode.py 784 return self._upload(new_contents) - def modify(self, modifier, backoffer=None): """I use a modifier callback to apply a change to the mutable file. I implement the following pseudocode:: hunk ./src/allmydata/mutable/filenode.py 828 return self._do_serialized(self._modify, modifier, backoffer) - def _modify(self, modifier, backoffer): if backoffer is None: backoffer = BackoffAgent().delay hunk ./src/allmydata/mutable/filenode.py 833 return self._modify_and_retry(modifier, backoffer, True) - def _modify_and_retry(self, modifier, backoffer, first_time): """ I try to apply modifier to the contents of this version of the hunk ./src/allmydata/mutable/filenode.py 865 d.addErrback(_retry) return d - def _modify_once(self, modifier, first_time): """ I attempt to apply a modifier to the contents of the mutable hunk ./src/allmydata/mutable/filenode.py 900 d.addCallback(_apply) return d - def is_readonly(self): """ I return True if this MutableFileVersion provides no write hunk ./src/allmydata/mutable/filenode.py 908 """ return self._writekey is None - def is_mutable(self): """ I return True, since mutable files are always mutable by hunk ./src/allmydata/mutable/filenode.py 915 """ return True - def get_storage_index(self): """ I return the storage index of the reference that I encapsulate. hunk ./src/allmydata/mutable/filenode.py 921 """ return self._storage_index - def get_size(self): """ I return the length, in bytes, of this readable object. hunk ./src/allmydata/mutable/filenode.py 927 """ return self._servermap.size_of_version(self._version) - def download_to_data(self, fetch_privkey=False): """ I return a Deferred that fires with the contents of this hunk ./src/allmydata/mutable/filenode.py 938 d.addCallback(lambda mc: "".join(mc.chunks)) return d - def _try_to_download_data(self): """ I am an unserialized cousin of download_to_data; I am called hunk ./src/allmydata/mutable/filenode.py 950 d.addCallback(lambda mc: "".join(mc.chunks)) return d - def read(self, consumer, offset=0, size=None, fetch_privkey=False): """ I read a portion (possibly all) of the mutable file that I hunk ./src/allmydata/mutable/filenode.py 958 return self._do_serialized(self._read, consumer, offset, size, fetch_privkey) - def _read(self, consumer, offset=0, size=None, fetch_privkey=False): """ I am the serialized companion of read. hunk ./src/allmydata/mutable/filenode.py 969 d = r.download(consumer, offset, size) return d - def _do_serialized(self, cb, *args, **kwargs): # note: to avoid deadlock, this callable is *not* allowed to invoke # other serialized methods within this (or any other) hunk ./src/allmydata/mutable/filenode.py 987 self._serializer.addErrback(log.err) return d - def _upload(self, new_contents): #assert self._pubkey, "update_servermap must be called before publish" p = Publish(self._node, self._storage_broker, self._servermap) hunk ./src/allmydata/mutable/filenode.py 997 d.addCallback(self._did_upload, new_contents.get_size()) return d - def _did_upload(self, res, size): self._most_recent_size = size return res hunk ./src/allmydata/mutable/filenode.py 1017 """ return self._do_serialized(self._update, data, offset) - def _update(self, data, offset): """ I update the mutable file version represented by this particular hunk ./src/allmydata/mutable/filenode.py 1046 d.addCallback(self._build_uploadable_and_finish, data, offset) return d - def _do_modify_update(self, data, offset): """ I perform a file update by modifying the contents of the file hunk ./src/allmydata/mutable/filenode.py 1061 return new return self._modify(m, None) - def _do_update_update(self, data, offset): """ I start the Servermap update that gets us the data we need to hunk ./src/allmydata/mutable/filenode.py 1096 return self._update_servermap(update_range=(start_segment, end_segment)) - def _decode_and_decrypt_segments(self, ignored, data, offset): """ After the servermap update, I take the encrypted and encoded hunk ./src/allmydata/mutable/filenode.py 1137 d3 = defer.succeed(blockhashes) return deferredutil.gatherResults([d1, d2, d3]) - def _build_uploadable_and_finish(self, segments_and_bht, data, offset): """ After the process has the plaintext segments, I build the hunk ./src/allmydata/mutable/filenode.py 1152 p = Publish(self._node, self._storage_broker, self._servermap) return p.update(u, offset, segments_and_bht[2], self._version) - def _update_servermap(self, mode=MODE_WRITE, update_range=None): """ I update the servermap. I return a Deferred that fires when the hunk ./src/allmydata/node.py 1 + import datetime, os.path, re, types, ConfigParser, tempfile from base64 import b32decode, b32encode hunk ./src/allmydata/scripts/debug.py 791 elif struct.unpack(">L", prefix[:4]) == (1,): # immutable - class ImmediateReadBucketProxy(ReadBucketProxy): def __init__(self, sf): self.sf = sf hunk ./src/allmydata/scripts/debug.py 850 sharedirs = listdir_unicode(abbrevdir) for si_s in sorted(sharedirs): si_dir = os.path.join(abbrevdir, si_s) - catalog_shares_one_abbrevdir(si_s, si_dir, now, out,err) + catalog_shares_one_abbrevdir(si_s, si_dir, now, out, err) except: print >>err, "Error processing %s" % quote_output(abbrevdir) failure.Failure().printTraceback(err) hunk ./src/allmydata/scripts/debug.py 871 abs_sharefile = os.path.join(si_dir, shnum_s) assert os.path.isfile(abs_sharefile) try: - describe_share(abs_sharefile, si_s, shnum_s, now, - out) + describe_share(abs_sharefile, si_s, shnum_s, now, out) except: print >>err, "Error processing %s" % quote_output(abs_sharefile) failure.Failure().printTraceback(err) hunk ./src/allmydata/scripts/debug.py 879 print >>err, "Error processing %s" % quote_output(si_dir) failure.Failure().printTraceback(err) + class CorruptShareOptions(usage.Options): def getSynopsis(self): return "Usage: tahoe debug corrupt-share SHARE_FILENAME" hunk ./src/allmydata/scripts/debug.py 903 Obviously, this command should not be used in normal operation. """ return t + def parseArgs(self, filename): self['filename'] = filename hunk ./src/allmydata/storage/backends/disk/immutable.py 14 from allmydata.storage.common import UnknownImmutableContainerVersionError, \ DataTooLargeError -# each share file (in storage/shares/$SI/$SHNUM) contains lease information -# and share data. The share data is accessed by RIBucketWriter.write and -# RIBucketReader.read . The lease information is not accessible through these -# interfaces. + +# Each share file (in storage/shares/$PREFIX/$STORAGEINDEX/$SHNUM) contains +# lease information and share data. The share data is accessed by +# RIBucketWriter.write and RIBucketReader.read . The lease information is not +# accessible through these remote interfaces. # The share file has the following layout: # 0x00: share file version number, four bytes, current version is 1 hunk ./src/allmydata/storage/backends/disk/immutable.py 86 def read_share_data(self, offset, length): precondition(offset >= 0) - # reads beyond the end of the data are truncated. Reads that start + + # Reads beyond the end of the data are truncated. Reads that start # beyond the end of the data return an empty string. seekpos = self._data_offset+offset actuallength = max(0, min(length, self._lease_offset-seekpos)) hunk ./src/allmydata/storage/backends/disk/immutable.py 164 except IndexError: self.add_lease(lease_info) - def cancel_lease(self, cancel_secret): """Remove a lease with the given cancel_secret. If the last lease is cancelled, the file will be removed. Return the number of bytes that hunk ./src/allmydata/storage/backends/disk/immutable.py 168 were freed (by truncating the list of leases, and possibly by - deleting the file. Raise IndexError if there was no lease with the + deleting the file). Raise IndexError if there was no lease with the given cancel_secret. """ hunk ./src/allmydata/storage/backends/disk/immutable.py 174 leases = list(self.get_leases()) num_leases_removed = 0 - for i,lease in enumerate(leases): + for i, lease in enumerate(leases): if constant_time_compare(lease.cancel_secret, cancel_secret): leases[i] = None num_leases_removed += 1 hunk ./src/allmydata/storage/backends/disk/mutable.py 25 # 4 4 expiration timestamp # 8 32 renewal token # 40 32 cancel token -# 72 20 nodeid which accepted the tokens +# 72 20 nodeid that accepted the tokens # 7 468 (a) data # 8 ?? 4 count of extra leases # 9 ?? n*92 extra leases hunk ./src/allmydata/storage/backends/disk/mutable.py 31 -# The struct module doc says that L's are 4 bytes in size., and that Q's are +# The struct module doc says that L's are 4 bytes in size, and that Q's are # 8 bytes in size. Since compatibility depends upon this, double-check it. assert struct.calcsize(">L") == 4, struct.calcsize(">L") assert struct.calcsize(">Q") == 8, struct.calcsize(">Q") hunk ./src/allmydata/storage/backends/disk/mutable.py 272 try: data = self._read_lease_record(f, i) if data is not None: - yield i,data + yield i, data except IndexError: return hunk ./src/allmydata/storage/backends/disk/mutable.py 302 accepting_nodeids.add(lease.nodeid) f.close() # Return the accepting_nodeids set, to give the client a chance to - # update the leases on a share which has been migrated from its + # update the leases on a share that has been migrated from its # original server to a new one. msg = ("Unable to renew non-existent lease. I have leases accepted by" " nodeids: ") hunk ./src/allmydata/storage/backends/disk/mutable.py 323 """Remove any leases with the given cancel_secret. If the last lease is cancelled, the file will be removed. Return the number of bytes that were freed (by truncating the list of leases, and possibly by - deleting the file. Raise IndexError if there was no lease with the + deleting the file). Raise IndexError if there was no lease with the given cancel_secret.""" accepting_nodeids = set() hunk ./src/allmydata/storage/crawler.py 12 class TimeSliceExceeded(Exception): pass + class ShareCrawler(service.MultiService): """A ShareCrawler subclass is attached to a StorageServer, and periodically walks all of its shares, processing each one in some hunk ./src/allmydata/storage/crawler.py 29 long enough to ensure that 'minimum_cycle_time' elapses between the start of two consecutive cycles. - We assume that the normal upload/download/get_buckets traffic of a tahoe + We assume that the normal upload/download/DYHB traffic of a Tahoe-LAFS grid will cause the prefixdir contents to be mostly cached in the kernel, hunk ./src/allmydata/storage/crawler.py 31 - or that the number of buckets in each prefixdir will be small enough to - load quickly. A 1TB allmydata.com server was measured to have 2.56M - buckets, spread into the 1024 prefixdirs, with about 2500 buckets per + or that the number of sharesets in each prefixdir will be small enough to + load quickly. A 1TB allmydata.com server was measured to have 2.56 million + sharesets, spread into the 1024 prefixdirs, with about 2500 sharesets per prefix. On this server, each prefixdir took 130ms-200ms to list the first time, and 17ms to list the second time. hunk ./src/allmydata/storage/crawler.py 143 left = len(self.prefixes) - self.last_complete_prefix_index remaining = left * self.last_prefix_elapsed_time # TODO: remainder of this prefix: we need to estimate the - # per-bucket time, probably by measuring the time spent on - # this prefix so far, divided by the number of buckets we've + # per-shareset time, probably by measuring the time spent on + # this prefix so far, divided by the number of sharesets we've # processed. d["estimated-cycle-complete-time-left"] = remaining # it's possible to call get_progress() from inside a crawler's hunk ./src/allmydata/storage/crawler.py 177 def load_state(self): # we use this to store state for both the crawler's internals and # anything the subclass-specific code needs. The state is stored - # after each bucket is processed, after each prefixdir is processed, + # after each shareset is processed, after each prefixdir is processed, # and after a cycle is complete. The internal keys we use are: # ["version"]: int, always 1 # ["last-cycle-finished"]: int, or None if we have not yet finished hunk ./src/allmydata/storage/crawler.py 191 # are sleeping between cycles, or if we # have not yet finished any prefixdir since # a cycle was started - # ["last-complete-bucket"]: str, base32 storage index bucket name - # of the last bucket to be processed, or - # None if we are sleeping between cycles + # ["last-complete-bucket"]: str, base32 storage index of the last + # shareset to be processed, or None if we + # are sleeping between cycles try: f = open(self.statefile, "rb") state = pickle.load(f) hunk ./src/allmydata/storage/crawler.py 279 sleep_time = (this_slice / self.allowed_cpu_percentage) - this_slice # if the math gets weird, or a timequake happens, don't sleep # forever. Note that this means that, while a cycle is running, we - # will process at least one bucket every 5 minutes, no matter how - # long that bucket takes. + # will process at least one shareset every 5 minutes, no matter how + # long that shareset takes. sleep_time = max(0.0, min(sleep_time, 299)) if finished_cycle: # how long should we sleep between cycles? Don't run faster than hunk ./src/allmydata/storage/crawler.py 352 You can override this if your crawler doesn't care about the actual shares, for example a crawler which merely keeps track of how many - buckets are being managed by this server. + sharesets are being managed by this server. Subclasses which *do* care about actual bucket should leave this method along, and implement process_bucket() instead. hunk ./src/allmydata/storage/crawler.py 369 # the remaining methods are explictly for subclasses to implement. def started_cycle(self, cycle): - """Notify a subclass that the crawler is about to start a cycle. + """ + Notify a subclass that the crawler is about to start a cycle. This method is for subclasses to override. No upcall is necessary. """ hunk ./src/allmydata/storage/crawler.py 381 to do to the shares therein, then update self.state as necessary. If the crawler is never interrupted by SIGKILL, this method will be - called exactly once per share (per cycle). If it *is* interrupted, + called exactly once per shareset (per cycle). If it *is* interrupted, then the next time the node is started, some amount of work will be duplicated, according to when self.save_state() was last called. By default, save_state() is called at the end of each timeslice, and hunk ./src/allmydata/storage/crawler.py 400 pass def finished_prefix(self, cycle, prefix): - """Notify a subclass that the crawler has just finished processing a - prefix directory (all buckets with the same two-character/10bit + """ + Notify a subclass that the crawler has just finished processing a + prefix directory (all sharesets with the same two-character/10-bit prefix). To impose a limit on how much work might be duplicated by a SIGKILL that occurs during a timeslice, you can call self.save_state() here, but be aware that it may represent a hunk ./src/allmydata/storage/crawler.py 413 pass def finished_cycle(self, cycle): - """Notify subclass that a cycle (one complete traversal of all + """ + Notify subclass that a cycle (one complete traversal of all prefixdirs) has just finished. 'cycle' is the number of the cycle that just finished. This method should perform summary work and update self.state to publish information to status displays. hunk ./src/allmydata/storage/crawler.py 431 pass def yielding(self, sleep_time): - """The crawler is about to sleep for 'sleep_time' seconds. This + """ + The crawler is about to sleep for 'sleep_time' seconds. This method is mostly for the convenience of unit tests. This method is for subclasses to override. No upcall is necessary. hunk ./src/allmydata/storage/crawler.py 469 def process_prefixdir(self, cycle, prefix, prefixdir, buckets, start_slice): # we override process_prefixdir() because we don't want to look at - # the individual buckets. We'll save state after each one. On my + # the individual sharesets. We'll save state after each one. On my # laptop, a mostly-empty storage server can process about 70 # prefixdirs in a 1.0s slice. if cycle not in self.state["bucket-counts"]: hunk ./src/allmydata/storage/crawler.py 493 old_cycle,buckets = self.state["storage-index-samples"][prefix] if old_cycle != cycle: del self.state["storage-index-samples"][prefix] - hunk ./src/allmydata/storage/expirer.py 15 removed. I collect statistics on the leases and make these available to a web - status page, including:: + status page, including: Space recovered during this cycle-so-far: actual (only if expiration_enabled=True): hunk ./src/allmydata/storage/expirer.py 19 - num-buckets, num-shares, sum of share sizes, real disk usage + num-storage-indices, num-shares, sum of share sizes, real disk usage ('real disk usage' means we use stat(fn).st_blocks*512 and include any space used by the directory) what it would have been with the original lease expiration time hunk ./src/allmydata/storage/expirer.py 30 Space recovered during the last 10 cycles <-- saved in separate pickle - Shares/buckets examined: + Shares/storage-indices examined: this cycle-so-far prediction of rest of cycle during last 10 cycles <-- separate pickle hunk ./src/allmydata/storage/expirer.py 40 Histogram of leases-per-share: this-cycle-to-date last 10 cycles <-- separate pickle - Histogram of lease ages, buckets = 1day + Histogram of lease ages, storage-indices over 1 day cycle-to-date last 10 cycles <-- separate pickle hunk ./src/allmydata/storage/server.py 37 class StorageServer(service.MultiService, Referenceable): implements(RIStorageServer, IStatsProducer) + name = 'storage' LeaseCheckerClass = LeaseCheckingCrawler hunk ./src/allmydata/storage/server.py 268 remaining_space = self.get_available_space() limited = remaining_space is not None if limited: - # this is a bit conservative, since some of this allocated_size() - # has already been written to disk, where it will show up in + # This is a bit conservative, since some of this allocated_size() + # has already been written to the backend, where it will show up in # get_available_space. remaining_space -= self.allocated_size() # self.readonly_storage causes remaining_space <= 0 hunk ./src/allmydata/storage/server.py 274 - # fill alreadygot with all shares that we have, not just the ones + # Fill alreadygot with all shares that we have, not just the ones # they asked about: this will save them a lot of work. Add or update # leases for all of them: if they want us to hold shares for this # file, they'll want us to hold leases for this file. hunk ./src/allmydata/test/no_network.py 23 from twisted.python.failure import Failure from foolscap.api import Referenceable, fireEventually, RemoteException from base64 import b32encode + from allmydata import uri as tahoe_uri from allmydata.client import Client from allmydata.storage.server import StorageServer, storage_index_to_dir hunk ./src/allmydata/test/no_network.py 89 return Failure(RemoteException(f)) d.addErrback(_wrap_exception) def _return_membrane(res): - # rather than complete the difficult task of building a + # Rather than complete the difficult task of building a # fully-general Membrane (which would locate all Referenceable # objects that cross the simulated wire and replace them with # wrappers), we special-case certain methods that we happen to hunk ./src/allmydata/test/no_network.py 156 seed = server.get_permutation_seed() return sha1(peer_selection_index + seed).digest() return sorted(self.get_connected_servers(), key=_permuted) + def get_connected_servers(self): return self.client._servers hunk ./src/allmydata/test/no_network.py 159 + def get_nickname_for_serverid(self, serverid): return None hunk ./src/allmydata/test/test_client.py 71 def test_secrets(self): basedir = "test_client.Basic.test_secrets" os.mkdir(basedir) - fileutil.write(os.path.join(basedir, "tahoe.cfg"), \ - BASECONFIG) + fileutil.write(os.path.join(basedir, "tahoe.cfg"), + BASECONFIG) c = client.Client(basedir) secret_fname = os.path.join(basedir, "private", "secret") self.failUnless(os.path.exists(secret_fname), secret_fname) hunk ./src/allmydata/test/test_client.py 84 def test_reserved_1(self): basedir = "client.Basic.test_reserved_1" os.mkdir(basedir) - fileutil.write(os.path.join(basedir, "tahoe.cfg"), \ - BASECONFIG + \ - "[storage]\n" + \ - "enabled = true\n" + \ - "reserved_space = 1000\n") + fileutil.write(os.path.join(basedir, "tahoe.cfg"), + BASECONFIG + + "[storage]\n" + + "enabled = true\n" + + "reserved_space = 1000\n") c = client.Client(basedir) self.failUnlessEqual(c.getServiceNamed("storage").reserved_space, 1000) hunk ./src/allmydata/test/test_client.py 95 def test_reserved_2(self): basedir = "client.Basic.test_reserved_2" os.mkdir(basedir) - fileutil.write(os.path.join(basedir, "tahoe.cfg"), \ - BASECONFIG + \ - "[storage]\n" + \ - "enabled = true\n" + \ - "reserved_space = 10K\n") + fileutil.write(os.path.join(basedir, "tahoe.cfg"), + BASECONFIG + + "[storage]\n" + + "enabled = true\n" + + "reserved_space = 10K\n") c = client.Client(basedir) self.failUnlessEqual(c.getServiceNamed("storage").reserved_space, 10*1000) hunk ./src/allmydata/test/test_client.py 106 def test_reserved_3(self): basedir = "client.Basic.test_reserved_3" os.mkdir(basedir) - fileutil.write(os.path.join(basedir, "tahoe.cfg"), \ - BASECONFIG + \ - "[storage]\n" + \ - "enabled = true\n" + \ - "reserved_space = 5mB\n") + fileutil.write(os.path.join(basedir, "tahoe.cfg"), + BASECONFIG + + "[storage]\n" + + "enabled = true\n" + + "reserved_space = 5mB\n") c = client.Client(basedir) self.failUnlessEqual(c.getServiceNamed("storage").reserved_space, 5*1000*1000) hunk ./src/allmydata/test/test_crawler.py 19 class BucketEnumeratingCrawler(ShareCrawler): cpu_slice = 500 # make sure it can complete in a single slice slow_start = 0 + def __init__(self, *args, **kwargs): ShareCrawler.__init__(self, *args, **kwargs) self.all_buckets = [] hunk ./src/allmydata/test/test_crawler.py 29 def finished_cycle(self, cycle): eventually(self.finished_d.callback, None) + class PacedCrawler(ShareCrawler): cpu_slice = 500 # make sure it can complete in a single slice slow_start = 0 hunk ./src/allmydata/test/test_crawler.py 33 + def __init__(self, *args, **kwargs): ShareCrawler.__init__(self, *args, **kwargs) self.countdown = 6 hunk ./src/allmydata/test/test_crawler.py 46 if self.countdown == 0: # force a timeout. We restore it in yielding() self.cpu_slice = -1.0 + def yielding(self, sleep_time): self.cpu_slice = 500 if self.yield_cb: hunk ./src/allmydata/test/test_crawler.py 51 self.yield_cb() + def finished_cycle(self, cycle): eventually(self.finished_d.callback, None) hunk ./src/allmydata/test/test_crawler.py 55 + class ConsumingCrawler(ShareCrawler): cpu_slice = 0.5 allowed_cpu_percentage = 0.5 hunk ./src/allmydata/test/test_crawler.py 73 elapsed = time.time() - start self.accumulated += elapsed self.last_yield += elapsed + def finished_cycle(self, cycle): self.cycles += 1 hunk ./src/allmydata/test/test_crawler.py 76 + def yielding(self, sleep_time): self.last_yield = 0.0 hunk ./src/allmydata/test/test_crawler.py 80 + class OneShotCrawler(ShareCrawler): cpu_slice = 500 # make sure it can complete in a single slice slow_start = 0 hunk ./src/allmydata/test/test_crawler.py 84 + def __init__(self, *args, **kwargs): ShareCrawler.__init__(self, *args, **kwargs) self.counter = 0 hunk ./src/allmydata/test/test_crawler.py 91 self.finished_d = defer.Deferred() def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32): self.counter += 1 + def finished_cycle(self, cycle): self.finished_d.callback(None) self.disownServiceParent() hunk ./src/allmydata/test/test_crawler.py 96 + class Basic(unittest.TestCase, StallMixin, pollmixin.PollMixin): def setUp(self): self.s = service.MultiService() hunk ./src/allmydata/test/test_crawler.py 107 def si(self, i): return hashutil.storage_index_hash(str(i)) + def rs(self, i, serverid): return hashutil.bucket_renewal_secret_hash(str(i), serverid) hunk ./src/allmydata/test/test_crawler.py 110 + def cs(self, i, serverid): return hashutil.bucket_cancel_secret_hash(str(i), serverid) hunk ./src/allmydata/test/test_crawler.py 423 d.addCallback(_done) return d - def test_oneshot(self): self.basedir = "crawler/Basic/oneshot" fileutil.make_dirs(self.basedir) hunk ./src/allmydata/test/test_crawler.py 452 self.failUnlessEqual(s["current-cycle"], None) d.addCallback(_check) return d - hunk ./src/allmydata/test/test_deepcheck.py 903 d.addErrback(self.explain_error) return d - - def set_up_damaged_tree(self): # 6.4s hunk ./src/allmydata/test/test_deepcheck.py 1083 d.addCallback(lambda ign: _checkv("mutable-good", self.check_is_healthy)) d.addCallback(lambda ign: _checkv("mutable-missing-shares", - self.check_is_missing_shares)) + self.check_is_missing_shares)) d.addCallback(lambda ign: _checkv("mutable-corrupt-shares", hunk ./src/allmydata/test/test_deepcheck.py 1085 - self.check_has_corrupt_shares)) + self.check_has_corrupt_shares)) d.addCallback(lambda ign: _checkv("mutable-unrecoverable", hunk ./src/allmydata/test/test_deepcheck.py 1087 - self.check_is_unrecoverable)) + self.check_is_unrecoverable)) d.addCallback(lambda ign: _checkv("large-good", self.check_is_healthy)) d.addCallback(lambda ign: _checkv("large-missing-shares", self.check_is_missing_shares)) d.addCallback(lambda ign: _checkv("large-corrupt-shares", self.check_has_corrupt_shares)) hunk ./src/allmydata/test/test_deepcheck.py 1092 d.addCallback(lambda ign: _checkv("large-unrecoverable", - self.check_is_unrecoverable)) + self.check_is_unrecoverable)) return d hunk ./src/allmydata/test/test_deepcheck.py 1202 d.addCallback(lambda ign: _checkv("mutable-good", self.json_is_healthy)) d.addCallback(lambda ign: _checkv("mutable-missing-shares", - self.json_is_missing_shares)) + self.json_is_missing_shares)) d.addCallback(lambda ign: _checkv("mutable-corrupt-shares", hunk ./src/allmydata/test/test_deepcheck.py 1204 - self.json_has_corrupt_shares)) + self.json_has_corrupt_shares)) d.addCallback(lambda ign: _checkv("mutable-unrecoverable", hunk ./src/allmydata/test/test_deepcheck.py 1206 - self.json_is_unrecoverable)) + self.json_is_unrecoverable)) d.addCallback(lambda ign: _checkv("large-good", self.json_is_healthy)) d.addCallback(lambda ign: _checkv("large-missing-shares", self.json_is_missing_shares)) hunk ./src/allmydata/test/test_deepcheck.py 1212 d.addCallback(lambda ign: _checkv("large-corrupt-shares", self.json_has_corrupt_shares)) d.addCallback(lambda ign: _checkv("large-unrecoverable", - self.json_is_unrecoverable)) + self.json_is_unrecoverable)) return d hunk ./src/allmydata/test/test_download.py 1002 d.addCallback(_got_data) return d - d = self.c0.upload(u) def _uploaded(ur): imm_uri = ur.uri hunk ./src/allmydata/test/test_download.py 1067 d.addCallback(fireEventually) return d d.addCallback(_uploaded) + def _show_results(ign): print print ("of [0:%d], corruption ignored in %s" % hunk ./src/allmydata/test/test_hung_server.py 20 immutable_plaintext = "data" * 10000 mutable_plaintext = "muta" * 10000 + class HungServerDownloadTest(GridTestMixin, ShouldFailMixin, PollMixin, unittest.TestCase): # Many of these tests take around 60 seconds on François's ARM buildslave: hunk ./src/allmydata/test/test_hung_server.py 159 self._download_and_check) else: return self.shouldFail(NotEnoughSharesError, self.basedir, - "ran out of shares", + "ran out of shares", self._download_and_check) hunk ./src/allmydata/test/test_hung_server.py 268 # stuck-but-not-overdue, and 4 live requests. All 4 live requests # will retire before the download is complete and the ShareFinder # is shut off. That will leave 4 OVERDUE and 1 - # stuck-but-not-overdue, for a total of 5 requests in in + # stuck-but-not-overdue, for a total of 5 requests in # _sf.pending_requests for t in self._sf.overdue_timers.values()[:4]: t.reset(-1.0) hunk ./src/allmydata/test/test_mutable.py 3542 sdmf_old_shares[9] = "VGFob2UgbXV0YWJsZSBjb250YWluZXIgdjEKdQlEA47ESLbTdKdpLJXCpBxd5OH239tl5hvAiz1dvGdE5rIOpf8cbfxbPcwNF+Y5dM92uBVbmV6KAAAAAAAAB/wAAAAAAAAJ0AAAAAFOWSw7jSx7WXzaMpdleJYXwYsRCV82jNA5oex9m2YhXSnb2POh+vvC1LE1NAfRc9GOb2zQG84Xdsx1Jub2brEeKkyt0sRIttN0p2kslcKkHF3k4fbf22XmAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABamJprL6ecrsOoFKdrXUmWveLq8nzEGDOjFnyK9detI3noX3uyK2MwSnFdAfyN0tuAwoAAAAAAAAAFQAAAAAAAAAVAAABjwAAAo8AAAMXAAADNwAAAAAAAAM+AAAAAAAAB/wwggEgMA0GCSqGSIb3DQEBAQUAA4IBDQAwggEIAoIBAQC1IkainlJF12IBXBQdpRK1zXB7a26vuEYqRmQM09YjC6sQjCs0F2ICk8n9m/2Kw4l16eIEboB2Au9pODCE+u/dEAakEFh4qidTMn61rbGUbsLK8xzuWNW22ezzz9/nPia0HDrulXt51/FYtfnnAuD1RJGXJv/8tDllE9FL/18TzlH4WuB6Fp8FTgv7QdbZAfWJHDGFIpVCJr1XxOCsSZNFJIqGwZnD2lsChiWw5OJDbKd8otqN1hIbfHyMyfMOJ/BzRzvZXaUt4Dv5nf93EmQDWClxShRwpuX/NkZ5B2K9OFonFTbOCexm/MjMAdCBqebKKaiHFkiknUCn9eJQpZ5bAgERgV50VKj+AVTDfgTpqfO2vfo4wrufi6ZBb8QV7hllhUFBjYogQ9C96dnS7skv0s+cqFuUjwMILr5/rsbEmEMGvl0T0ytyAbtlXuowEFVj/YORNknM4yjY72YUtEPTlMpk0Cis7aIgTvu5qWMPER26PMApZuRqiwRsGIkaJIvOVOTHHjFYe3/YzdMkc7OZtqRMfQLtwVl2/zKQQV8b/a9vaT6q3mRLRd4P3esaAFe/+7sR/t+9tmB+a8kxtKM6kmaVQJMbXJZ4aoHGfeLX0m35Rcvu2Bmph7QfSDjk/eaE3q55zYSoGWShmlhlw4Kwg84sMuhmcVhLvo0LovR8bKmbdgABUSzNKiMx0E91q51/WH6ASL0fDEOLef9oxuyBX5F5cpoABojmWkDX3k3FKfgNHIeptE3lxB8HHzxDfSD250psyfNCAAwGsKbMxbmI2NpdTozZ3SICrySwgGkatA1gsDOJmOnTzgAXVnLiODzHiLFAI/MsXcR71fmvb7UghLA1b8pq66KAyl+aopjsD29AKG5hrXt9hLIp6shvfrzaPGIid5C8IxYIrjgBj1YohGgDE0Wua7Lx6Bnad5n91qmHAnwSEJE5YIhQM634omd6cq9Wk4seJCUIn+ucoknrpxp0IR9QMxpKSMRHRUg2K8ZegnY3YqFunRZKCfsq9ufQEKgjZN12AFqi551KPBdn4/3V5HK6xTv0P4robSsE/BvuIfByvRf/W7ZrDx+CFC4EEcsBOACOZCrkhhqd5TkYKbe9RA+vs56+9N5qZGurkxcoKviiyEncxvTuShD65DK/6x6kMDMgQv/EdZDI3x9GtHTnRBYXwDGnPJ19w+q2zC3e2XarbxTGYQIPEC5mYx0gAA0sbjf018NGfwBhl6SB54iGsa8uLvR3jHv6OSRJgwxL6j7P0Ts4Hv2EtO12P0Lv21pwi3JC1O/WviSrKCvrQD5lMHL9Uym3hwFi2zu0mqwZvxOAbGy7kfOPXkLYKOHTZLthzKj3PsdjeceWBfYIvPGKYcd6wDr36d1aXSYS4IWeApTS2AQ2lu0DUcgSefAvsA8NkgOklvJY1cjTMSg6j6cxQo48Bvl8RAWGLbr4h2S/8KwDGxwLsSv0Gop/gnFc3GzCsmL0EkEyHHWkCA8YRXCghfW80KLDV495ff7yF5oiwK56GniqowZ3RG9Jxp5MXoJQgsLV1VMQFMAmsY69yz8eoxRH3wl9L0dMyndLulhWWzNwPMQ2I0yAWdzA/pksVmwTJTFenB3MHCiWc5rEwJ3yofe6NZZnZQrYyL9r1TNnVwfTwRUiykPiLSk4x9Mi6DX7RamDAxc8u3gDVfjPsTOTagBOEGUWlGAL54KE/E6sgCQ5DEAt12chk8AxbjBFLPgV+/idrzS0lZHOL+IVBI9D0i3Bq1yZcSIqcjZB0M3IbxbPm4gLAYOWEiTUN2ecsEHHg9nt6rhgffVoqSbCCFPbpC0xf7WOC3+BQORIZECOCC7cUAciXq3xn+GuxpFE40RWRJeKAK7bBQ21X89ABIXlQFkFddZ9kRvlZ2Pnl0oeF+2pjnZu0Yc2czNfZEQF2P7BKIdLrgMgxG89snxAY8qAYTCKyQw6xTG87wkjDcpy1wzsZLP3WsOuO7cAm7b27xU0jRKq8Cw4d1hDoyRG+RdS53F8RFJzVMaNNYgxU2tfRwUvXpTRXiOheeRVvh25+YGVnjakUXjx/dSDnOw4ETHGHD+7styDkeSfc3BdSZxswzc6OehgMI+xsCxeeRym15QUm9hxvg8X7Bfz/0WulgFwgzrm11TVynZYOmvyHpiZKoqQyQyKahIrfhwuchCr7lMsZ4a+umIkNkKxCLZnI+T7jd+eGFMgKItjz3kTTxRl3IhaJG3LbPmwRUJynMxQKdMi4Uf0qy0U7+i8hIJ9m50QXc+3tw2bwDSbx22XYJ9Wf14gxx5G5SPTb1JVCbhe4fxNt91xIxCow2zk62tzbYfRe6dfmDmgYHkv2PIEtMJZK8iKLDjFfu2ZUxsKT2A5g1q17og6o9MeXeuFS3mzJXJYFQZd+3UzlFR9qwkFkby9mg5y4XSeMvRLOHPt/H/r5SpEqBE6a9MadZYt61FBV152CUEzd43ihXtrAa0XH9HdsiySBcWI1SpM3mv9rRP0DiLjMUzHw/K1D8TE2f07zW4t/9kvE11tFj/NpICixQAAAAA=" sdmf_old_cap = "URI:SSK:gmjgofw6gan57gwpsow6gtrz3e:5adm6fayxmu3e4lkmfvt6lkkfix34ai2wop2ioqr4bgvvhiol3kq" sdmf_old_contents = "This is a test file.\n" + def copy_sdmf_shares(self): # We'll basically be short-circuiting the upload process. servernums = self.g.servers_by_number.keys() hunk ./src/allmydata/test/test_mutable.py 3576 d.addCallback(self.failUnlessEqual, self.sdmf_old_contents) return d + class DifferentEncoding(unittest.TestCase): def setUp(self): self._storage = s = FakeStorage() hunk ./src/allmydata/test/test_no_network.py 10 from allmydata.immutable.upload import Data from allmydata.util.consumer import download_to_data + class Harness(unittest.TestCase): def setUp(self): self.s = service.MultiService() hunk ./src/allmydata/test/test_storage.py 6 import mock from twisted.trial import unittest - from twisted.internet import defer from twisted.application import service from foolscap.api import fireEventually hunk ./src/allmydata/test/test_storage.py 36 from allmydata.test.no_network import NoNetworkServer from allmydata.web.storage import StorageStatus, remove_prefix + class Marker: pass hunk ./src/allmydata/test/test_storage.py 39 + + class FakeCanary: def __init__(self, ignore_disconnectors=False): self.ignore = ignore_disconnectors hunk ./src/allmydata/test/test_storage.py 56 return del self.disconnectors[marker] + class FakeStatsProvider: def count(self, name, delta=1): pass hunk ./src/allmydata/test/test_storage.py 63 def register_producer(self, producer): pass + class Bucket(unittest.TestCase): def make_workdir(self, name): basedir = os.path.join("storage", "Bucket", name) hunk ./src/allmydata/test/test_storage.py 292 return d1 d.addCallback(_start_reading) - return d def test_readwrite_v1(self): hunk ./src/allmydata/test/test_storage.py 1388 # header. self.salt_hash_tree_s = self.serialize_blockhashes(self.salt_hash_tree[1:]) - def tearDown(self): self.sparent.stopService() shutil.rmtree(self.workdir("MDMFProxies storage test server")) hunk ./src/allmydata/test/test_storage.py 1396 def write_enabler(self, we_tag): return hashutil.tagged_hash("we_blah", we_tag) - def renew_secret(self, tag): return hashutil.tagged_hash("renew_blah", str(tag)) hunk ./src/allmydata/test/test_storage.py 1399 - def cancel_secret(self, tag): return hashutil.tagged_hash("cancel_blah", str(tag)) hunk ./src/allmydata/test/test_storage.py 1402 - def workdir(self, name): basedir = os.path.join("storage", "MutableServer", name) return basedir hunk ./src/allmydata/test/test_storage.py 1413 ss.setServiceParent(self.sparent) return ss - def build_test_mdmf_share(self, tail_segment=False, empty=False): # Start with the checkstring data = struct.pack(">BQ32s", hunk ./src/allmydata/test/test_storage.py 1510 data += self.block_hash_tree_s return data - def write_test_share_to_server(self, storage_index, tail_segment=False, hunk ./src/allmydata/test/test_storage.py 1582 self.offsets['EOF'] = eof_offset return final_share - def write_sdmf_share_to_server(self, storage_index, empty=False): hunk ./src/allmydata/test/test_storage.py 1667 self.failUnlessEqual(checkstring, checkstring)) return d - def test_read_with_different_tail_segment_size(self): self.write_test_share_to_server("si1", tail_segment=True) mr = MDMFSlotReadProxy(self.rref, "si1", 0) hunk ./src/allmydata/test/test_storage.py 1678 d.addCallback(_check_tail_segment) return d - def test_get_block_with_invalid_segnum(self): self.write_test_share_to_server("si1") mr = MDMFSlotReadProxy(self.rref, "si1", 0) hunk ./src/allmydata/test/test_storage.py 1688 mr.get_block_and_salt, 7)) return d - def test_get_encoding_parameters_first(self): self.write_test_share_to_server("si1") mr = MDMFSlotReadProxy(self.rref, "si1", 0) hunk ./src/allmydata/test/test_storage.py 1700 d.addCallback(_check_encoding_parameters) return d - def test_get_seqnum_first(self): self.write_test_share_to_server("si1") mr = MDMFSlotReadProxy(self.rref, "si1", 0) hunk ./src/allmydata/test/test_storage.py 1708 self.failUnlessEqual(seqnum, 0)) return d - def test_get_root_hash_first(self): self.write_test_share_to_server("si1") mr = MDMFSlotReadProxy(self.rref, "si1", 0) hunk ./src/allmydata/test/test_storage.py 1716 self.failUnlessEqual(root_hash, self.root_hash)) return d - def test_get_checkstring_first(self): self.write_test_share_to_server("si1") mr = MDMFSlotReadProxy(self.rref, "si1", 0) hunk ./src/allmydata/test/test_storage.py 1724 self.failUnlessEqual(checkstring, self.checkstring)) return d - def test_write_read_vectors(self): # When writing for us, the storage server will return to us a # read vector, along with its result. If a write fails because hunk ./src/allmydata/test/test_storage.py 1740 mw.put_root_hash(self.root_hash) mw.put_signature(self.signature) mw.put_verification_key(self.verification_key) + d = mw.finish_publishing() def _then(results): self.failUnless(len(results), 2) hunk ./src/allmydata/test/test_storage.py 1763 # The checkstring remains the same for the rest of the process. return d - def test_private_key_after_share_hash_chain(self): mw = self._make_new_mw("si1", 0) d = defer.succeed(None) hunk ./src/allmydata/test/test_storage.py 1781 mw.put_encprivkey, self.encprivkey)) return d - def test_signature_after_verification_key(self): mw = self._make_new_mw("si1", 0) d = defer.succeed(None) hunk ./src/allmydata/test/test_storage.py 1807 mw.put_signature, self.signature)) return d - def test_uncoordinated_write(self): # Make two mutable writers, both pointing to the same storage # server, both at the same storage index, and try writing to the hunk ./src/allmydata/test/test_storage.py 1839 d.addCallback(_check_failure) return d - def test_invalid_salt_size(self): # Salts need to be 16 bytes in size. Writes that attempt to # write more or less than this should be rejected. hunk ./src/allmydata/test/test_storage.py 1857 another_invalid_salt)) return d - def test_write_test_vectors(self): # If we give the write proxy a bogus test vector at # any point during the process, it should fail to write when we hunk ./src/allmydata/test/test_storage.py 1881 mw.put_root_hash(self.root_hash) mw.put_signature(self.signature) mw.put_verification_key(self.verification_key) + d = mw.finish_publishing() d.addCallback(_check_failure) d.addCallback(lambda ignored: hunk ./src/allmydata/test/test_storage.py 1891 d.addCallback(_check_success) return d - def serialize_blockhashes(self, blockhashes): return "".join(blockhashes) hunk ./src/allmydata/test/test_storage.py 1894 - def serialize_sharehashes(self, sharehashes): ret = "".join([struct.pack(">H32s", i, sharehashes[i]) for i in sorted(sharehashes.keys())]) hunk ./src/allmydata/test/test_storage.py 1899 return ret - def test_write(self): # This translates to a file with 6 6-byte segments, and with 2-byte # blocks. hunk ./src/allmydata/test/test_storage.py 1922 mw.put_root_hash(self.root_hash) mw.put_signature(self.signature) mw.put_verification_key(self.verification_key) + d = mw.finish_publishing() def _check_publish(results): self.failUnlessEqual(len(results), 2) hunk ./src/allmydata/test/test_storage.py 2031 6, datalength) return mw - def test_write_rejected_with_too_many_blocks(self): mw = self._make_new_mw("si0", 0) hunk ./src/allmydata/test/test_storage.py 2035 # Try writing too many blocks. We should not be able to write - # more than 6 - # blocks into each share. + # more than 6 blocks into each share. d = defer.succeed(None) for i in xrange(6): d.addCallback(lambda ignored, i=i: hunk ./src/allmydata/test/test_storage.py 2046 mw.put_block, self.block, 7, self.salt)) return d - def test_write_rejected_with_invalid_salt(self): # Try writing an invalid salt. Salts are 16 bytes -- any more or # less should cause an error. hunk ./src/allmydata/test/test_storage.py 2057 None, mw.put_block, self.block, 7, bad_salt)) return d - def test_write_rejected_with_invalid_root_hash(self): # Try writing an invalid root hash. This should be SHA256d, and # 32 bytes long as a result. hunk ./src/allmydata/test/test_storage.py 2082 None, mw.put_root_hash, invalid_root_hash)) return d - def test_write_rejected_with_invalid_blocksize(self): # The blocksize implied by the writer that we get from # _make_new_mw is 2bytes -- any more or any less than this hunk ./src/allmydata/test/test_storage.py 2115 mw.put_block(valid_block, 5, self.salt)) return d - def test_write_enforces_order_constraints(self): # We require that the MDMFSlotWriteProxy be interacted with in a # specific way. hunk ./src/allmydata/test/test_storage.py 2200 mw0.put_verification_key(self.verification_key)) return d - def test_end_to_end(self): mw = self._make_new_mw("si1", 0) # Write a share using the mutable writer, and make sure that the hunk ./src/allmydata/test/test_storage.py 2283 self.failUnlessEqual(checkstring, mw.get_checkstring())) return d - def test_is_sdmf(self): # The MDMFSlotReadProxy should also know how to read SDMF files, # since it will encounter them on the grid. Callers use the hunk ./src/allmydata/test/test_storage.py 2294 self.failUnless(issdmf)) return d - def test_reads_sdmf(self): # The slot read proxy should, naturally, know how to tell us # about data in the SDMF format hunk ./src/allmydata/test/test_storage.py 2381 mr.get_block_and_salt, 1)) return d - def test_read_with_prefetched_mdmf_data(self): # The MDMFSlotReadProxy will prefill certain fields if you pass # it data that you have already fetched. This is useful for hunk ./src/allmydata/test/test_storage.py 2429 self.failUnlessEqual(expected_prefix, prefix) self.failUnlessEqual(self.rref.read_count, 0) d.addCallback(_check_verinfo) + # This is not enough data to read a block and a share, so the # wrapper should attempt to read this from the remote server. d.addCallback(_make_mr, 123) hunk ./src/allmydata/test/test_storage.py 2439 self.failUnlessEqual(block, self.block) self.failUnlessEqual(salt, self.salt) self.failUnlessEqual(self.rref.read_count, 1) + # This should be enough data to read one block. d.addCallback(_make_mr, 123 + PRIVATE_KEY_SIZE + SIGNATURE_SIZE + VERIFICATION_KEY_SIZE + SHARE_HASH_CHAIN_SIZE + 140) d.addCallback(lambda mr: hunk ./src/allmydata/test/test_storage.py 2447 d.addCallback(_check_block_and_salt) return d - def test_read_with_prefetched_sdmf_data(self): sdmf_data = self.build_test_sdmf_share() self.write_sdmf_share_to_server("si1") hunk ./src/allmydata/test/test_storage.py 2510 d.addCallback(_check_block_and_salt) return d - def test_read_with_empty_mdmf_file(self): # Some tests upload a file with no contents to test things # unrelated to the actual handling of the content of the file. hunk ./src/allmydata/test/test_storage.py 2564 mr.get_block_and_salt, 0)) return d - def test_verinfo_with_sdmf_file(self): self.write_sdmf_share_to_server("si1") mr = MDMFSlotReadProxy(self.rref, "si1", 0) hunk ./src/allmydata/test/test_storage.py 2604 d.addCallback(_check_verinfo) return d - def test_verinfo_with_mdmf_file(self): self.write_test_share_to_server("si1") mr = MDMFSlotReadProxy(self.rref, "si1", 0) hunk ./src/allmydata/test/test_storage.py 2642 d.addCallback(_check_verinfo) return d - def test_sdmf_writer(self): # Go through the motions of writing an SDMF share to the storage # server. Then read the storage server to see that the share got hunk ./src/allmydata/test/test_storage.py 2685 d.addCallback(_then) return d - def test_sdmf_writer_preexisting_share(self): data = self.build_test_sdmf_share() self.write_sdmf_share_to_server("si1") hunk ./src/allmydata/test/test_storage.py 2828 self.failUnless(output["get"]["99_0_percentile"] is None, output) self.failUnless(output["get"]["99_9_percentile"] is None, output) + def remove_tags(s): s = re.sub(r'<[^>]*>', ' ', s) s = re.sub(r'\s+', ' ', s) hunk ./src/allmydata/test/test_storage.py 2834 return s + class MyBucketCountingCrawler(BucketCountingCrawler): def finished_prefix(self, cycle, prefix): BucketCountingCrawler.finished_prefix(self, cycle, prefix) hunk ./src/allmydata/test/test_storage.py 2960 fileutil.make_dirs(basedir) ss = MyStorageServer(basedir, "\x00" * 20) ss.bucket_counter.slow_start = 0 + # these will be fired inside finished_prefix() hooks = ss.bucket_counter.hook_ds = [defer.Deferred() for i in range(3)] w = StorageStatus(ss) hunk ./src/allmydata/test/test_storage.py 2994 ss.setServiceParent(self.s) return d + class InstrumentedLeaseCheckingCrawler(LeaseCheckingCrawler): stop_after_first_bucket = False def process_bucket(self, *args, **kwargs): hunk ./src/allmydata/test/test_storage.py 3742 # expirer.py . This will have to change if/when the # progress-measurer gets smart enough to count buckets (we'll # have to interrupt it even earlier, before it's finished the - # first bucket). + # first shareset). s = lc.get_state() if "cycle-to-date" not in s: d2 = fireEventually() } [All other changes in pluggable backends branch. refs #999, #1569 david-sarah@jacaranda.org**20111216183651 Ignore-this: c3957b0d5efc42dc2555ebd5e1567e16 ] { hunk ./src/allmydata/client.py 8 from twisted.internet import reactor, defer from twisted.application import service from twisted.application.internet import TimerService +from twisted.python.filepath import FilePath from pycryptopp.publickey import rsa import allmydata hunk ./src/allmydata/client.py 12 +from allmydata.node import InvalidValueError from allmydata.storage.server import StorageServer hunk ./src/allmydata/client.py 14 +from allmydata.storage.backends.null.null_backend import configure_null_backend +from allmydata.storage.backends.disk.disk_backend import configure_disk_backend +from allmydata.storage.backends.s3.s3_backend import configure_s3_backend +from allmydata.storage.backends.s3.mock_s3 import configure_mock_s3_backend from allmydata import storage_client from allmydata.immutable.upload import Uploader from allmydata.immutable.offloaded import Helper hunk ./src/allmydata/client.py 23 from allmydata.control import ControlServer from allmydata.introducer.client import IntroducerClient -from allmydata.util import hashutil, base32, pollmixin, log, keyutil -from allmydata.util.encodingutil import get_filesystem_encoding -from allmydata.util.abbreviate import parse_abbreviated_size +from allmydata.util import hashutil, fileutil, keyutil, base32, pollmixin, log +from allmydata.util.encodingutil import get_filesystem_encoding, quote_output from allmydata.util.time_format import parse_duration, parse_date from allmydata.stats import StatsProvider from allmydata.history import History hunk ./src/allmydata/client.py 217 self._server_key = sk def _init_permutation_seed(self, ss): - seed = self.get_config_from_file("permutation-seed") + seed = self.get_optional_private_config("permutation-seed") if not seed: hunk ./src/allmydata/client.py 219 - have_shares = ss.have_shares() - if have_shares: - # if the server has shares but not a recorded + # quick test to decide if we need to commit to an implicit + # permutation-seed or if we should use a new one + if self._have_disk_shares: + # If the server has shares but not a recorded # permutation-seed, then it has been around since pre-#466 # days, and the clients who uploaded those shares used our # TubID as a permutation-seed. We should keep using that same hunk ./src/allmydata/client.py 242 # Should we run a storage server (and publish it for others to use)? if not self.get_config("storage", "enabled", True, boolean=True): return - readonly = self.get_config("storage", "readonly", False, boolean=True) self._maybe_create_server_key() hunk ./src/allmydata/client.py 245 - storedir = os.path.join(self.basedir, self.STOREDIR) + storedir = FilePath(self.basedir).child(self.STOREDIR) hunk ./src/allmydata/client.py 247 - data = self.get_config("storage", "reserved_space", None) - reserved = None - try: - reserved = parse_abbreviated_size(data) - except ValueError: - log.msg("[storage]reserved_space= contains unparseable value %s" - % data) - if reserved is None: - reserved = 0 - discard = self.get_config("storage", "debug_discard", False, - boolean=True) + # Record whether we have stored shares as a disk backend. + # (This should be true if there are shares on disk even if we're not currently + # configured as a disk backend.) + self._have_disk_shares = bool([True for child in fileutil.fp_list(storedir.child("shares")) + if child.basename() != "incoming"]) + + # What sort of backend? + backendtype = self.get_config("storage", "backend", "disk") + + backend_configurators = { + 'disk': configure_disk_backend, + 's3': configure_s3_backend, + 'mock_s3': configure_mock_s3_backend, + 'debug_discard': configure_null_backend, + } + + if backendtype not in backend_configurators: + raise InvalidValueError("[storage]backend= is required to be one of %s, but was %s" + % (backend_configurators.keys(), quote_output(backendtype)) ) + + backend = backend_configurators[backendtype](storedir, self) expire = self.get_config("storage", "expire.enabled", False, boolean=True) if expire: hunk ./src/allmydata/client.py 289 sharetypes.append("immutable") if self.get_config("storage", "expire.mutable", True, boolean=True): sharetypes.append("mutable") - expiration_sharetypes = tuple(sharetypes) hunk ./src/allmydata/client.py 290 - ss = StorageServer(storedir, self.nodeid, - reserved_space=reserved, - discard_storage=discard, - readonly_storage=readonly, + expiration_policy = { + 'enabled': expire, + 'mode': mode, + 'override_lease_duration': o_l_d, + 'cutoff_date': cutoff_date, + 'sharetypes': tuple(sharetypes), + } + + statedir = storedir + ss = StorageServer(self.nodeid, backend, statedir, stats_provider=self.stats_provider, hunk ./src/allmydata/client.py 301 - expiration_enabled=expire, - expiration_mode=mode, - expiration_override_lease_duration=o_l_d, - expiration_cutoff_date=cutoff_date, - expiration_sharetypes=expiration_sharetypes) + expiration_policy=expiration_policy) self.add_service(ss) d = self.when_tub_ready() hunk ./src/allmydata/immutable/layout.py 340 return self._read(0, 0x44) def _parse_offsets(self, data): - precondition(len(data) >= 0x4) + precondition(len(data) >= 0x4, len(data)) self._offsets = {} (version,) = struct.unpack(">L", data[0:4]) if version != 1 and version != 2: hunk ./src/allmydata/interfaces.py 33 LeaseRenewSecret = Hash # used to protect lease renewal requests LeaseCancelSecret = Hash # formerly used to protect lease cancellation requests -class RIStubClient(RemoteInterface): - """Each client publishes a service announcement for a dummy object called - the StubClient. This object doesn't actually offer any services, but the - announcement helps the Introducer keep track of which clients are - subscribed (so the grid admin can keep track of things like the size of - the grid and the client versions in use. This is the (empty) - RemoteInterface for the StubClient.""" class RIBucketWriter(RemoteInterface): """ Objects of this kind live on the server side. """ hunk ./src/allmydata/interfaces.py 184 This secret is generated by the client and stored for later comparison by the server. Each server is given a different secret. - @param cancel_secret: Like renew_secret, but protects bucket decref. + @param cancel_secret: This no longer allows lease cancellation, but + must still be a unique value identifying the + lease. XXX stop relying on it to be unique. hunk ./src/allmydata/interfaces.py 188 - The 'secrets' argument is a tuple of (write_enabler, renew_secret, - cancel_secret). The first is required to perform any write. The - latter two are used when allocating new shares. To simply acquire a - new lease on existing shares, use an empty testv and an empty writev. + The 'secrets' argument is a tuple with (write_enabler, renew_secret). + The write_enabler is required to perform any write. The renew_secret + is used when allocating new shares. Each share can have a separate test vector (i.e. a list of comparisons to perform). If all vectors for all shares pass, then all hunk ./src/allmydata/interfaces.py 279 store that on disk. """ -class IStorageBucketWriter(Interface): + +class IStorageBackend(Interface): """ hunk ./src/allmydata/interfaces.py 282 - Objects of this kind live on the client side. + Objects of this kind live on the server side and are used by the + storage server object. """ hunk ./src/allmydata/interfaces.py 285 - def put_block(segmentnum=int, data=ShareData): - """@param data: For most segments, this data will be 'blocksize' - bytes in length. The last segment might be shorter. - @return: a Deferred that fires (with None) when the operation completes + def supports_crawlers(): + """ + Returns True if this backend is able to support crawlers, + otherwise False. + """ + + def get_available_space(): + """ + Returns available space for share storage in bytes, or + None if this information is not available or if the available + space is unlimited. + + If the backend is configured for read-only mode then this will + return 0. + """ + + def get_sharesets_for_prefix(prefix): + """ + Return an iterable containing IShareSet objects for all storage + indices matching the given base-32 prefix, for which this backend + holds shares. + XXX This will probably need to return a Deferred, but for now it + is synchronous. + """ + + def get_shareset(storageindex): + """ + Get an IShareSet object for the given storage index. + This method is synchronous. + """ + + def fill_in_space_stats(stats): + """ + Fill in the 'stats' dict with space statistics for this backend, in + 'storage_server.*' keys. + """ + + def advise_corrupt_share(storageindex, sharetype, shnum, reason): + """ + Clients who discover hash failures in shares that they have + downloaded from me will use this method to inform me about the + failures. I will record their concern so that my operator can + manually inspect the shares in question. This method is synchronous. + + 'sharetype' is either 'mutable' or 'immutable'. 'shnum' is the integer + share number. 'reason' is a human-readable explanation of the problem, + probably including some expected hash values and the computed ones + that did not match. Corruption advisories for mutable shares should + include a hash of the public key (the same value that appears in the + mutable-file verify-cap), since the current share format does not + store that on disk. + + @param storageindex=str + @param sharetype=str + @param shnum=int + @param reason=str + """ + + +class IShareSet(Interface): + def get_storage_index(): + """ + Returns the storage index for this shareset. + """ + + def get_storage_index_string(): + """ + Returns the base32-encoded storage index for this shareset. + """ + + def get_overhead(): + """ + Returns an estimate of the storage overhead, in bytes, of this shareset + (exclusive of the space used by its shares). + """ + + def get_shares(): + """ + Returns a Deferred that fires with a pair + (list of IShareBase objects, set of corrupted shnums). + The share objects include only completed shares in this shareset. + """ + # XXX rename to get_shares_and_corrupted? + + def get_shares_synchronous(): + """ + A synchronous version of get_shares() that returns a pair + (list of IShareBase objects, set of corrupted shnums). + This is only available on sharesets from a backend that supports_crawlers(). + """ + + def get_share(shnum): + """ + Returns a Deferred that fires with an IShareBase object if the given + share exists, or fails with IndexError otherwise. + """ + + def has_incoming(shnum): + """ + Returns True if this shareset has an incoming (partial) share with this + number, otherwise False. + """ + + def make_bucket_writer(storageserver, shnum, max_space_per_bucket, lease_info, canary): + """ + Create a bucket writer that can be used to write data to a given share. + + @param storageserver=RIStorageServer + @param shnum=int: A share number in this shareset + @param max_space_per_bucket=int: The maximum space allocated for the + share, in bytes + @param lease_info=LeaseInfo: The initial lease information + @param canary=Referenceable: If the canary is lost before close(), the + bucket is deleted. + @return an IStorageBucketWriter for the given share """ hunk ./src/allmydata/interfaces.py 402 - def put_plaintext_hashes(hashes=ListOf(Hash)): + def make_bucket_reader(storageserver, share): + """ + Create a bucket reader that can be used to read data from a given share. + + @param storageserver=RIStorageServer + @param share=IStoredShare + @return an IStorageBucketReader for the given share + """ + + def readv(wanted_shnums, read_vector): + """ + Read a vector from the numbered shares in this shareset. An empty + wanted_shnums list means to return data from all known shares. + Return a Deferred that fires with a dict mapping the share number + to the corresponding ReadData. + + @param wanted_shnums=ListOf(int) + @param read_vector=ReadVector + @return DeferredOf(DictOf(int, ReadData)): shnum -> results, with one key per share + """ + + def testv_and_readv_and_writev(storageserver, secrets, test_and_write_vectors, read_vector, expiration_time): + """ + General-purpose atomic test-read-and-set operation for mutable slots. + Perform a bunch of comparisons against the existing shares in this + shareset. If they all pass: use the read vectors to extract data from + all the shares, then apply a bunch of write vectors to those shares. + Return a Deferred that fires with a pair consisting of a boolean that is + True iff the test vectors passed, and a dict mapping the share number + to the corresponding ReadData. Reads do not include any modifications + made by the writes. + + See the similar method in RIStorageServer for more detail. + + @param storageserver=RIStorageServer + @param secrets=TupleOf(WriteEnablerSecret, LeaseRenewSecret[, ...]) + @param test_and_write_vectors=TestAndWriteVectorsForShares + @param read_vector=ReadVector + @param expiration_time=int + @return DeferredOf(TupleOf(bool, DictOf(int, ReadData))) """ hunk ./src/allmydata/interfaces.py 443 + + def get_leases(): + """ + Yield a LeaseInfo instance for each lease on this shareset. + """ + + def add_or_renew_lease(lease_info): + """ + Add a new lease on the shares in this shareset. If the renew_secret + matches an existing lease, that lease will be renewed instead. If + there are no shares in this shareset, return silently. + + @param lease_info=LeaseInfo + """ + + def renew_lease(renew_secret, new_expiration_time): + """ + Renew a lease on the shares in this shareset, resetting the timer + to 31 days. Some grids will use this, some will not. If there are no + shares in this shareset, IndexError will be raised. + + For mutable shares, if the given renew_secret does not match an + existing lease, IndexError will be raised with a note listing the + server-nodeids on the existing leases, so leases on migrated shares + can be renewed. For immutable shares, IndexError (without the note) + will be raised. + + @param renew_secret=LeaseRenewSecret + """ + + +class IShareBase(Interface): + """ + I represent an immutable or mutable share stored by a particular backend. + I may hold some, all, or none of the share data in memory. + + XXX should this interface also include lease operations? + """ + def get_storage_index(): + """ + Returns the storage index. + """ + + def get_storage_index_string(): + """ + Returns the base32-encoded storage index. + """ + + def get_shnum(): + """ + Returns the share number. + """ + + def get_data_length(): + """ + Returns the data length in bytes. + """ + + def get_size(): + """ + Returns the size of the share in bytes. + """ + + def get_used_space(): + """ + Returns the amount of backend storage including overhead (which may + have to be estimated), in bytes, used by this share. + """ + + def unlink(): + """ + Signal that this share can be removed from the backend storage. This does + not guarantee that the share data will be immediately inaccessible, or + that it will be securely erased. + Returns a Deferred that fires after the share has been removed. + + This may be called on a share that is being written and is not closed. + """ + + +class IShareForReading(IShareBase): + """ + I represent an immutable share that can be read from. + """ + def read_share_data(offset, length): + """ + Return a Deferred that fires with the read result. + """ + + def readv(read_vector): + """ + Given a list of (offset, length) pairs, return a Deferred that fires with + a list of read results. + """ + + +class IShareForWriting(IShareBase): + """ + I represent an immutable share that is being written. + """ + def get_allocated_size(): + """ + Returns the allocated size of the share (not including header) in bytes. + This is the maximum amount of data that can be written. + """ + + def write_share_data(offset, data): + """ + Write data at the given offset. Return a Deferred that fires when we + are ready to accept the next write. + + XXX should we require that data is written with no backtracking (i.e. that + offset must not be before the previous end-of-data)? + """ + + def close(): + """ + Complete writing to this share. + """ + + +class IMutableShare(IShareBase): + """ + I represent a mutable share. + """ + def create(serverid, write_enabler): + """ + Create an empty mutable share with the given serverid and write enabler. + Return a Deferred that fires when the share has been created. + """ + + def check_write_enabler(write_enabler): + """ + XXX + """ + + def check_testv(test_vector): + """ + XXX + """ + + def writev(datav, new_length): + """ + XXX + """ + + +class IStorageBucketWriter(Interface): + """ + Objects of this kind live on the client side. + """ + def put_block(segmentnum, data): + """ + @param segmentnum=int + @param data=ShareData: For most segments, this data will be 'blocksize' + bytes in length. The last segment might be shorter. @return: a Deferred that fires (with None) when the operation completes """ hunk ./src/allmydata/interfaces.py 669 @return: ListOf(Hash) """ - def get_share_hashes(at_least_these=SetOf(int)): + def get_share_hashes(): """ @return: ListOf(TupleOf(int, Hash)) """ hunk ./src/allmydata/interfaces.py 701 @return: unicode nickname, or None """ - # methods moved from IntroducerClient, need review - def get_all_connections(): - """Return a frozenset of (nodeid, service_name, rref) tuples, one for - each active connection we've established to a remote service. This is - mostly useful for unit tests that need to wait until a certain number - of connections have been made.""" - - def get_all_connectors(): - """Return a dict that maps from (nodeid, service_name) to a - RemoteServiceConnector instance for all services that we are actively - trying to connect to. Each RemoteServiceConnector has the following - public attributes:: - - service_name: the type of service provided, like 'storage' - announcement_time: when we first heard about this service - last_connect_time: when we last established a connection - last_loss_time: when we last lost a connection - - version: the peer's version, from the most recent connection - oldest_supported: the peer's oldest supported version, same - - rref: the RemoteReference, if connected, otherwise None - remote_host: the IAddress, if connected, otherwise None - - This method is intended for monitoring interfaces, such as a web page - that describes connecting and connected peers. - """ - - def get_all_peerids(): - """Return a frozenset of all peerids to whom we have a connection (to - one or more services) established. Mostly useful for unit tests.""" - - def get_all_connections_for(service_name): - """Return a frozenset of (nodeid, service_name, rref) tuples, one - for each active connection that provides the given SERVICE_NAME.""" - - def get_permuted_peers(service_name, key): - """Returns an ordered list of (peerid, rref) tuples, selecting from - the connections that provide SERVICE_NAME, using a hash-based - permutation keyed by KEY. This randomizes the service list in a - repeatable way, to distribute load over many peers. - """ - class IMutableSlotWriter(Interface): """ hunk ./src/allmydata/interfaces.py 706 The interface for a writer around a mutable slot on a remote server. """ - def set_checkstring(checkstring, *args): + def set_checkstring(seqnum_or_checkstring, root_hash=None, salt=None): """ Set the checkstring that I will pass to the remote server when writing. hunk ./src/allmydata/interfaces.py 730 Add a block and salt to the share. """ - def put_encprivkey(encprivkey): + def put_encprivkey(encrypted_privkey): """ Add the encrypted private key to the share. """ hunk ./src/allmydata/interfaces.py 964 writer-visible data using this writekey. """ - # TODO: Can this be overwrite instead of replace? - def replace(new_contents): - """Replace the contents of the mutable file, provided that no other + def overwrite(new_contents): + """Overwrite the contents of the mutable file, provided that no other node has published (or is attempting to publish, concurrently) a newer version of the file than this one. hunk ./src/allmydata/interfaces.py 1431 is empty, the metadata will be an empty dictionary. """ - def set_uri(name, writecap, readcap=None, metadata=None, overwrite=True): + def set_uri(name, writecap, readcap, metadata=None, overwrite=True): """I add a child (by writecap+readcap) at the specific name. I return a Deferred that fires when the operation finishes. If overwrite= is True, I will replace any existing child of the same name, otherwise hunk ./src/allmydata/interfaces.py 2035 resuming an interrupted upload (where we need to compute the plaintext hashes, but don't need the redundant encrypted data).""" - def get_plaintext_hashtree_leaves(first, last, num_segments): - """OBSOLETE; Get the leaf nodes of a merkle hash tree over the - plaintext segments, i.e. get the tagged hashes of the given segments. - The segment size is expected to be generated by the - IEncryptedUploadable before any plaintext is read or ciphertext - produced, so that the segment hashes can be generated with only a - single pass. - - This returns a Deferred that fires with a sequence of hashes, using: - - tuple(segment_hashes[first:last]) - - 'num_segments' is used to assert that the number of segments that the - IEncryptedUploadable handled matches the number of segments that the - encoder was expecting. - - This method must not be called until the final byte has been read - from read_encrypted(). Once this method is called, read_encrypted() - can never be called again. - """ - - def get_plaintext_hash(): - """OBSOLETE; Get the hash of the whole plaintext. - - This returns a Deferred that fires with a tagged SHA-256 hash of the - whole plaintext, obtained from hashutil.plaintext_hash(data). - """ - def close(): """Just like IUploadable.close().""" hunk ./src/allmydata/interfaces.py 2229 returns a Deferred that fires with an IUploadResults instance, from which the URI of the file can be obtained as results.uri .""" - def upload_ssk(write_capability, new_version, uploadable): - """TODO: how should this work?""" - class ICheckable(Interface): def check(monitor, verify=False, add_lease=False): """Check up on my health, optionally repairing any problems. hunk ./src/allmydata/interfaces.py 2598 class IRepairResults(Interface): """I contain the results of a repair operation.""" - def get_successful(self): + def get_successful(): """Returns a boolean: True if the repair made the file healthy, False if not. Repair failure generally indicates a file that has been damaged beyond repair.""" hunk ./src/allmydata/interfaces.py 2670 Tahoe process will typically have a single NodeMaker, but unit tests may create simplified/mocked forms for testing purposes. """ - def create_from_cap(writecap, readcap=None, **kwargs): + def create_from_cap(writecap, readcap=None, deep_immutable=False, name=u""): """I create an IFilesystemNode from the given writecap/readcap. I can only provide nodes for existing file/directory objects: use my other methods to create new objects. I return synchronously.""" hunk ./src/allmydata/interfaces.py 2820 class BadWriteEnablerError(Exception): pass -class RIControlClient(RemoteInterface): hunk ./src/allmydata/interfaces.py 2821 +class RIControlClient(RemoteInterface): def wait_for_client_connections(num_clients=int): """Do not return until we have connections to at least NUM_CLIENTS storage servers. hunk ./src/allmydata/monitor.py 30 # the following methods are provided for the operation code - def is_cancelled(self): + def is_cancelled(): """Returns True if the operation has been cancelled. If True, operation code should stop creating new work, and attempt to stop any work already in progress.""" hunk ./src/allmydata/monitor.py 35 - def raise_if_cancelled(self): + def raise_if_cancelled(): """Raise OperationCancelledError if the operation has been cancelled. Operation code that has a robust error-handling path can simply call this periodically.""" hunk ./src/allmydata/monitor.py 40 - def set_status(self, status): + def set_status(status): """Sets the Monitor's 'status' object to an arbitrary value. Different operations will store different sorts of status information here. Operation code should use get+modify+set sequences to update hunk ./src/allmydata/monitor.py 46 this.""" - def get_status(self): + def get_status(): """Return the status object. If the operation failed, this will be a Failure instance.""" hunk ./src/allmydata/monitor.py 50 - def finish(self, status): + def finish(status): """Call this when the operation is done, successful or not. The Monitor's lifetime is influenced by the completion of the operation it is monitoring. The Monitor's 'status' value will be set with the hunk ./src/allmydata/monitor.py 63 # the following methods are provided for the initiator of the operation - def is_finished(self): + def is_finished(): """Return a boolean, True if the operation is done (whether successful or failed), False if it is still running.""" hunk ./src/allmydata/monitor.py 67 - def when_done(self): + def when_done(): """Return a Deferred that fires when the operation is complete. It will fire with the operation status, the same value as returned by get_status().""" hunk ./src/allmydata/monitor.py 72 - def cancel(self): + def cancel(): """Cancel the operation as soon as possible. is_cancelled() will start returning True after this is called.""" hunk ./src/allmydata/mutable/filenode.py 748 """ return self._version[0] # verinfo[0] == the sequence number + def get_servermap(self): + return self._servermap def get_writekey(self): """ hunk ./src/allmydata/mutable/layout.py 76 OFFSETS = ">LLLLQQ" OFFSETS_LENGTH = struct.calcsize(OFFSETS) +# our sharefiles share with a recognizable string, plus some random +# binary data to reduce the chance that a regular text file will look +# like a sharefile. +MUTABLE_MAGIC = "Tahoe mutable container v1\n" + "\x75\x09\x44\x03\x8e" + # These are still used for some tests. def unpack_header(data): o = {} hunk ./src/allmydata/mutable/layout.py 1250 def _process_encoding_parameters(self, encoding_parameters): - assert self.shnum in encoding_parameters + assert self.shnum in encoding_parameters, (self.shnum, encoding_parameters) encoding_parameters = encoding_parameters[self.shnum][0] # The first byte is the version number. It will tell us what # to do next. hunk ./src/allmydata/mutable/layout.py 1395 d.addCallback(_then) d.addCallback(lambda readvs: self._read(readvs)) def _process_results(results): - assert self.shnum in results + assert self.shnum in results, (self.shnum, results) if self._version_number == 0: # We only read the share data, but we know the salt from # when we fetched the header hunk ./src/allmydata/mutable/publish.py 872 def _record_verinfo(self): - self.versioninfo = self.writers.values()[0].get_verinfo() + writers = self.writers.values() + if len(writers) > 0: + self.versioninfo = writers[0].get_verinfo() def _connection_problem(self, f, writer): hunk ./src/allmydata/node.py 6 from base64 import b32decode, b32encode from twisted.python import log as twlog +from twisted.python.filepath import FilePath from twisted.application import service from twisted.internet import defer, reactor from foolscap.api import Tub, eventually, app_versions hunk ./src/allmydata/node.py 16 from allmydata.util import fileutil, iputil, observer from allmydata.util.assertutil import precondition, _assert from allmydata.util.fileutil import abspath_expanduser_unicode -from allmydata.util.encodingutil import get_filesystem_encoding, quote_output +from allmydata.util.encodingutil import get_filesystem_encoding, quote_output, quote_filepath +from allmydata.util.abbreviate import parse_abbreviated_size + # Add our application versions to the data that Foolscap's LogPublisher # reports. hunk ./src/allmydata/node.py 44 are set to disallow users other than its owner from reading the contents of the files. See the 'configuration.rst' documentation file for details.""" -class _None: # used as a marker in get_config() +class _None: # used as a marker in get_config() and get_or_create_private_config() pass hunk ./src/allmydata/node.py 47 +class InvalidValueError(Exception): + """ The configured value was not valid. """ + class MissingConfigEntry(Exception): """ A required config entry was not found. """ hunk ./src/allmydata/node.py 120 % (quote_output(fn), section, option)) return default + def get_config_size(self, section, option, default=_None): + data = self.get_config(section, option, default) + if data is None: + return None + try: + return parse_abbreviated_size(data) + except ValueError: + raise InvalidValueError("[%s]%s= contains unparseable size value %s" + % (section, option, quote_output(data)) ) + def set_config(self, section, option, value): if not self.config.has_section(section): self.config.add_section(section) hunk ./src/allmydata/node.py 212 # TODO: merge this with allmydata.get_package_versions return dict(app_versions.versions) - def get_config_from_file(self, name, required=False): - """Get the (string) contents of a config file, or None if the file - did not exist. If required=True, raise an exception rather than - returning None. Any leading or trailing whitespace will be stripped - from the data.""" - fn = os.path.join(self.basedir, name) + def _get_private_config_filepath(self, name): + return FilePath(self.basedir).child("private").child(name) + + def get_optional_private_config(self, name): + """Try to get the (string) contents of a private config file (which + is a config file that resides within the subdirectory named + 'private'), and return it. Any leading or trailing whitespace will be + stripped from the data. If the file does not exist, return None. + """ + priv_fp = self._get_private_config_filepath(name) try: hunk ./src/allmydata/node.py 223 - return fileutil.read(fn).strip() + value = priv_fp.getContent() except EnvironmentError: hunk ./src/allmydata/node.py 225 - if not required: - return None - raise + if priv_fp.exists(): + raise + return None + return value.strip() def write_private_config(self, name, value): """Write the (string) contents of a private config file (which is a hunk ./src/allmydata/node.py 236 return it. Any leading or trailing whitespace will be stripped from the data. """ - privname = os.path.join(self.basedir, "private", name) - open(privname, "w").write(value.strip()) + self._get_private_config_filepath(name).setContent(value.strip()) hunk ./src/allmydata/node.py 238 - def get_or_create_private_config(self, name, default): + def get_or_create_private_config(self, name, default=_None): """Try to get the (string) contents of a private config file (which is a config file that resides within the subdirectory named 'private'), and return it. Any leading or trailing whitespace will be hunk ./src/allmydata/node.py 244 stripped from the data. - If the file does not exist, try to create it using default, and - then return the value that was written. If 'default' is a string, - use it as a default value. If not, treat it as a 0-argument callable - which is expected to return a string. + If the file does not exist, and default is not given, report an error. + If the file does not exist and a default is specified, try to create + it using that default, and then return the value that was written. + If 'default' is a string, use it as a default value. If not, treat it + as a zero-argument callable that is expected to return a string. """ hunk ./src/allmydata/node.py 250 - privname = os.path.join(self.basedir, "private", name) - try: - value = fileutil.read(privname) - except EnvironmentError: - if isinstance(default, basestring): - value = default + value = self.get_optional_private_config(name) + if value is None: + priv_fp = self._get_private_config_filepath(name) + if default is _None: + raise MissingConfigEntry("The required configuration file %s is missing." + % (quote_filepath(priv_fp),)) + elif isinstance(default, basestring): + value = default.strip() else: hunk ./src/allmydata/node.py 259 - value = default() - fileutil.write(privname, value) - return value.strip() + value = default().strip() + priv_fp.setContent(value) + return value def write_config(self, name, value, mode="w"): """Write a string to a config file.""" hunk ./src/allmydata/scripts/debug.py 8 from twisted.python import usage, failure from twisted.internet import defer from twisted.scripts import trial as twisted_trial +from twisted.python.filepath import FilePath class DumpOptions(usage.Options): hunk ./src/allmydata/scripts/debug.py 38 self['filename'] = argv_to_abspath(filename) def dump_share(options): - from allmydata.storage.mutable import MutableShareFile + from allmydata.storage.backends.disk.disk_backend import get_disk_share from allmydata.util.encodingutil import quote_output out = options.stdout hunk ./src/allmydata/scripts/debug.py 42 + filename = options['filename'] # check the version, to see if we have a mutable or immutable share hunk ./src/allmydata/scripts/debug.py 45 - print >>out, "share filename: %s" % quote_output(options['filename']) + print >>out, "share filename: %s" % quote_output(filename) hunk ./src/allmydata/scripts/debug.py 47 - f = open(options['filename'], "rb") - prefix = f.read(32) - f.close() - if prefix == MutableShareFile.MAGIC: - return dump_mutable_share(options) - # otherwise assume it's immutable - return dump_immutable_share(options) + share = get_disk_share(FilePath(filename)) hunk ./src/allmydata/scripts/debug.py 49 -def dump_immutable_share(options): - from allmydata.storage.immutable import ShareFile + if share.sharetype == "mutable": + return dump_mutable_share(options, share) + else: + assert share.sharetype == "immutable", share.sharetype + return dump_immutable_share(options, share) hunk ./src/allmydata/scripts/debug.py 55 +def dump_immutable_share(options, share): out = options.stdout hunk ./src/allmydata/scripts/debug.py 57 - f = ShareFile(options['filename']) if not options["leases-only"]: hunk ./src/allmydata/scripts/debug.py 58 - dump_immutable_chk_share(f, out, options) - dump_immutable_lease_info(f, out) + dump_immutable_chk_share(share, out, options) + dump_immutable_lease_info(share, out) print >>out return 0 hunk ./src/allmydata/scripts/debug.py 63 -def dump_immutable_chk_share(f, out, options): +def dump_immutable_chk_share(share, out, options): from allmydata import uri from allmydata.util import base32 from allmydata.immutable.layout import ReadBucketProxy hunk ./src/allmydata/scripts/debug.py 71 # use a ReadBucketProxy to parse the bucket and find the uri extension bp = ReadBucketProxy(None, None, '') - offsets = bp._parse_offsets(f.read_share_data(0, 0x44)) + f = share._get_filepath().open("rb") + # XXX yuck, private API + def read_share_data(offset, length): + return share._read_share_data(f, offset, length) + + offsets = bp._parse_offsets(read_share_data(0, 0x44)) print >>out, "%20s: %d" % ("version", bp._version) seek = offsets['uri_extension'] length = struct.unpack(bp._fieldstruct, hunk ./src/allmydata/scripts/debug.py 80 - f.read_share_data(seek, bp._fieldsize))[0] + read_share_data(seek, bp._fieldsize))[0] seek += bp._fieldsize hunk ./src/allmydata/scripts/debug.py 82 - UEB_data = f.read_share_data(seek, length) + UEB_data = read_share_data(seek, length) unpacked = uri.unpack_extension_readable(UEB_data) keys1 = ("size", "num_segments", "segment_size", hunk ./src/allmydata/scripts/debug.py 142 if options['offsets']: print >>out print >>out, " Section Offsets:" - print >>out, "%20s: %s" % ("share data", f._data_offset) + print >>out, "%20s: %s" % ("share data", share._data_offset) for k in ["data", "plaintext_hash_tree", "crypttext_hash_tree", "block_hashes", "share_hashes", "uri_extension"]: name = {"data": "block data"}.get(k,k) hunk ./src/allmydata/scripts/debug.py 146 - offset = f._data_offset + offsets[k] + offset = share._data_offset + offsets[k] print >>out, " %20s: %s (0x%x)" % (name, offset, offset) hunk ./src/allmydata/scripts/debug.py 148 - print >>out, "%20s: %s" % ("leases", f._lease_offset) + print >>out, "%20s: %s" % ("leases", share._lease_offset) def dump_immutable_lease_info(f, out): # display lease information too hunk ./src/allmydata/scripts/debug.py 173 return when -def dump_mutable_share(options): - from allmydata.storage.mutable import MutableShareFile +def dump_mutable_share(options, m): from allmydata.util import base32, idlib out = options.stdout hunk ./src/allmydata/scripts/debug.py 176 - m = MutableShareFile(options['filename']) f = open(options['filename'], "rb") WE, nodeid = m._read_write_enabler_and_nodeid(f) num_extra_leases = m._read_num_extra_leases(f) hunk ./src/allmydata/scripts/debug.py 286 if options['offsets']: # NOTE: this offset-calculation code is fragile, and needs to be - # merged with MutableShareFile's internals. + # merged with MutableDiskShare's internals. print >>out print >>out, " Section Offsets:" def printoffset(name, value, shift=0): hunk ./src/allmydata/scripts/debug.py 380 if options['offsets']: # NOTE: this offset-calculation code is fragile, and needs to be - # merged with MutableShareFile's internals. + # merged with MutableDiskShare's internals. print >>out print >>out, " Section Offsets:" hunk ./src/allmydata/scripts/debug.py 647 /home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/9 /home/warner/testnet/node-2/storage/shares/44k/44kai1tui348689nrw8fjegc8c/2 """ - from allmydata.storage.server import si_a2b, storage_index_to_dir - from allmydata.util.encodingutil import listdir_unicode + from allmydata.storage.server import si_a2b + from allmydata.storage.backends.disk.disk_backend import si_si2dir + from allmydata.util.encodingutil import quote_filepath out = options.stdout hunk ./src/allmydata/scripts/debug.py 652 - sharedir = storage_index_to_dir(si_a2b(options.si_s)) - for d in options.nodedirs: - d = os.path.join(d, "storage/shares", sharedir) - if os.path.exists(d): - for shnum in listdir_unicode(d): - print >>out, os.path.join(d, shnum) + si = si_a2b(options.si_s) + for nodedir in options.nodedirs: + sharedir = si_si2dir(FilePath(nodedir).child("storage").child("shares"), si) + if sharedir.exists(): + for sharefp in sharedir.children(): + print >>out, quote_filepath(sharefp, quotemarks=False) return 0 hunk ./src/allmydata/scripts/debug.py 710 def describe_share(abs_sharefile, si_s, shnum_s, now, out): from allmydata import uri - from allmydata.storage.mutable import MutableShareFile - from allmydata.storage.immutable import ShareFile + from allmydata.storage.backends.disk.disk_backend import get_disk_share + from allmydata.storage.common import UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError from allmydata.mutable.layout import unpack_share from allmydata.mutable.common import NeedMoreDataError from allmydata.immutable.layout import ReadBucketProxy hunk ./src/allmydata/scripts/debug.py 717 from allmydata.util import base32 from allmydata.util.encodingutil import quote_output - import struct hunk ./src/allmydata/scripts/debug.py 718 - f = open(abs_sharefile, "rb") - prefix = f.read(32) + sharefp = FilePath(abs_sharefile) + try: + share = get_disk_share(sharefp) + except UnknownMutableContainerVersionError: + print >>out, "UNKNOWN mutable %s" % quote_output(abs_sharefile) + return + except UnknownImmutableContainerVersionError: + print >>out, "UNKNOWN really-unknown %s" % quote_output(abs_sharefile) + return + + f = sharefp.open("rb") hunk ./src/allmydata/scripts/debug.py 730 - if prefix == MutableShareFile.MAGIC: - # mutable share - m = MutableShareFile(abs_sharefile) - WE, nodeid = m._read_write_enabler_and_nodeid(f) - data_length = m._read_data_length(f) - expiration_time = min( [lease.expiration_time - for (i,lease) in m._enumerate_leases(f)] ) + leases = list(share.get_leases()) + if len(leases) > 0: + expiration_time = min( [lease.expiration_time for lease in leases] ) expiration = max(0, expiration_time - now) hunk ./src/allmydata/scripts/debug.py 734 + else: + expiration = None + + if share.sharetype == "mutable": + WE, nodeid = share._read_write_enabler_and_nodeid(f) + data_length = share._read_data_length(f) share_type = "unknown" hunk ./src/allmydata/scripts/debug.py 742 - f.seek(m.DATA_OFFSET) + f.seek(share.DATA_OFFSET) version = f.read(1) if version == "\x00": # this slot contains an SMDF share hunk ./src/allmydata/scripts/debug.py 751 share_type = "MDMF" if share_type == "SDMF": - f.seek(m.DATA_OFFSET) + f.seek(share.DATA_OFFSET) data = f.read(min(data_length, 2000)) try: hunk ./src/allmydata/scripts/debug.py 759 except NeedMoreDataError, e: # retry once with the larger size size = e.needed_bytes - f.seek(m.DATA_OFFSET) + f.seek(share.DATA_OFFSET) data = f.read(min(data_length, size)) pieces = unpack_share(data) (seqnum, root_hash, IV, k, N, segsize, datalen, hunk ./src/allmydata/scripts/debug.py 766 pubkey, signature, share_hash_chain, block_hash_tree, share_data, enc_privkey) = pieces - print >>out, "SDMF %s %d/%d %d #%d:%s %d %s" % \ + print >>out, "SDMF %s %d/%d %d #%d:%s %r %s" % \ (si_s, k, N, datalen, seqnum, base32.b2a(root_hash), expiration, quote_output(abs_sharefile)) hunk ./src/allmydata/scripts/debug.py 778 def _read(self, readvs, force_remote=False, queue=False): data = [] for (where,length) in readvs: - f.seek(m.DATA_OFFSET+where) + f.seek(share.DATA_OFFSET+where) data.append(f.read(length)) return defer.succeed({fake_shnum: data}) hunk ./src/allmydata/scripts/debug.py 795 verinfo = extract(p.get_verinfo) (seqnum, root_hash, salt_to_use, segsize, datalen, k, N, prefix, offsets) = verinfo - print >>out, "MDMF %s %d/%d %d #%d:%s %d %s" % \ + print >>out, "MDMF %s %d/%d %d #%d:%s %r %s" % \ (si_s, k, N, datalen, seqnum, base32.b2a(root_hash), expiration, quote_output(abs_sharefile)) hunk ./src/allmydata/scripts/debug.py 802 else: print >>out, "UNKNOWN mutable %s" % quote_output(abs_sharefile) - elif struct.unpack(">L", prefix[:4]) == (1,): + else: # immutable class ImmediateReadBucketProxy(ReadBucketProxy): hunk ./src/allmydata/scripts/debug.py 805 - def __init__(self, sf): - self.sf = sf + def __init__(self, share): + self.share = share ReadBucketProxy.__init__(self, None, None, "") def __repr__(self): return "" hunk ./src/allmydata/scripts/debug.py 811 def _read(self, offset, size): - return defer.succeed(sf.read_share_data(offset, size)) + return defer.maybeDeferred(self.share.read_share_data, offset, size) # use a ReadBucketProxy to parse the bucket and find the uri extension hunk ./src/allmydata/scripts/debug.py 814 - sf = ShareFile(abs_sharefile) - bp = ImmediateReadBucketProxy(sf) - - expiration_time = min( [lease.expiration_time - for lease in sf.get_leases()] ) - expiration = max(0, expiration_time - now) + bp = ImmediateReadBucketProxy(share) UEB_data = call(bp.get_uri_extension) unpacked = uri.unpack_extension_readable(UEB_data) hunk ./src/allmydata/scripts/debug.py 824 filesize = unpacked["size"] ueb_hash = unpacked["UEB_hash"] - print >>out, "CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize, + print >>out, "CHK %s %d/%d %d %s %r %s" % (si_s, k, N, filesize, ueb_hash, expiration, quote_output(abs_sharefile)) hunk ./src/allmydata/scripts/debug.py 828 - else: - print >>out, "UNKNOWN really-unknown %s" % quote_output(abs_sharefile) - f.close() def catalog_shares(options): hunk ./src/allmydata/scripts/debug.py 913 self['filename'] = filename def corrupt_share(options): + do_corrupt_share(options.stdout, FilePath(options['filename']), options['offset']) + +def do_corrupt_share(out, fp, offset="block-random"): import random hunk ./src/allmydata/scripts/debug.py 917 - from allmydata.storage.mutable import MutableShareFile - from allmydata.storage.immutable import ShareFile + from allmydata.storage.backends.disk.disk_backend import get_disk_share from allmydata.mutable.layout import unpack_header from allmydata.immutable.layout import ReadBucketProxy hunk ./src/allmydata/scripts/debug.py 920 - out = options.stdout - fn = options['filename'] - assert options["offset"] == "block-random", "other offsets not implemented" - # first, what kind of share is it? + + assert offset == "block-random", "other offsets not implemented" def flip_bit(start, end): offset = random.randrange(start, end) hunk ./src/allmydata/scripts/debug.py 927 bit = random.randrange(0, 8) print >>out, "[%d..%d): %d.b%d" % (start, end, offset, bit) - f = open(fn, "rb+") - f.seek(offset) - d = f.read(1) - d = chr(ord(d) ^ 0x01) - f.seek(offset) - f.write(d) - f.close() + f = fp.open("rb+") + try: + f.seek(offset) + d = f.read(1) + d = chr(ord(d) ^ 0x01) + f.seek(offset) + f.write(d) + finally: + f.close() hunk ./src/allmydata/scripts/debug.py 937 - f = open(fn, "rb") - prefix = f.read(32) - f.close() - if prefix == MutableShareFile.MAGIC: - # mutable - m = MutableShareFile(fn) - f = open(fn, "rb") - f.seek(m.DATA_OFFSET) - data = f.read(2000) - # make sure this slot contains an SMDF share - assert data[0] == "\x00", "non-SDMF mutable shares not supported" - f.close() + # what kind of share is it? + + share = get_disk_share(fp) + if share.sharetype == "mutable": + f = fp.open("rb") + try: + f.seek(share.DATA_OFFSET) + data = f.read(2000) + # make sure this slot contains an SMDF share + assert data[0] == "\x00", "non-SDMF mutable shares not supported" + finally: + f.close() (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize, ig_datalen, offsets) = unpack_header(data) hunk ./src/allmydata/scripts/debug.py 954 assert version == 0, "we only handle v0 SDMF files" - start = m.DATA_OFFSET + offsets["share_data"] - end = m.DATA_OFFSET + offsets["enc_privkey"] + start = share.DATA_OFFSET + offsets["share_data"] + end = share.DATA_OFFSET + offsets["enc_privkey"] flip_bit(start, end) else: # otherwise assume it's immutable hunk ./src/allmydata/scripts/debug.py 959 - f = ShareFile(fn) bp = ReadBucketProxy(None, None, '') hunk ./src/allmydata/scripts/debug.py 960 - offsets = bp._parse_offsets(f.read_share_data(0, 0x24)) - start = f._data_offset + offsets["data"] - end = f._data_offset + offsets["plaintext_hash_tree"] + f = fp.open("rb") + try: + # XXX yuck, private API + header = share._read_share_data(f, 0, 0x24) + finally: + f.close() + offsets = bp._parse_offsets(header) + start = share._data_offset + offsets["data"] + end = share._data_offset + offsets["plaintext_hash_tree"] flip_bit(start, end) hunk ./src/allmydata/storage/backends/disk/immutable.py 1 -import os, stat, struct, time hunk ./src/allmydata/storage/backends/disk/immutable.py 2 -from foolscap.api import Referenceable +import struct + +from twisted.internet import defer from zope.interface import implements hunk ./src/allmydata/storage/backends/disk/immutable.py 7 -from allmydata.interfaces import RIBucketWriter, RIBucketReader -from allmydata.util import base32, fileutil, log +from allmydata.interfaces import IShareForReading, IShareForWriting + +from allmydata.util import fileutil from allmydata.util.assertutil import precondition hunk ./src/allmydata/storage/backends/disk/immutable.py 11 +from allmydata.util.fileutil import fp_make_dirs from allmydata.util.hashutil import constant_time_compare hunk ./src/allmydata/storage/backends/disk/immutable.py 13 +from allmydata.util.encodingutil import quote_filepath +from allmydata.storage.common import si_b2a, UnknownImmutableContainerVersionError, DataTooLargeError from allmydata.storage.lease import LeaseInfo hunk ./src/allmydata/storage/backends/disk/immutable.py 16 -from allmydata.storage.common import UnknownImmutableContainerVersionError, \ - DataTooLargeError # Each share file (in storage/shares/$PREFIX/$STORAGEINDEX/$SHNUM) contains hunk ./src/allmydata/storage/backends/disk/immutable.py 43 # then the value stored in this field will be the actual share data length # modulo 2**32. -class ShareFile: - LEASE_SIZE = struct.calcsize(">L32s32sL") +class ImmutableDiskShare(object): + implements(IShareForReading, IShareForWriting) + sharetype = "immutable" hunk ./src/allmydata/storage/backends/disk/immutable.py 47 + LEASE_SIZE = struct.calcsize(">L32s32sL") + HEADER = ">LLL" + HEADER_SIZE = struct.calcsize(HEADER) hunk ./src/allmydata/storage/backends/disk/immutable.py 51 - def __init__(self, filename, max_size=None, create=False): - """ If max_size is not None then I won't allow more than max_size to be written to me. If create=True and max_size must not be None. """ - precondition((max_size is not None) or (not create), max_size, create) - self.home = filename + def __init__(self, home, storageindex, shnum, finalhome=None, max_size=None): + """ + If max_size is not None then I won't allow more than max_size to be written to me. + If finalhome is not None (meaning that we are creating the share) then max_size + must not be None. + + Clients should use the load_immutable_disk_share and create_immutable_disk_share + factory functions rather than creating instances directly. + """ + precondition((max_size is not None) or (finalhome is None), max_size, finalhome) + self._storageindex = storageindex self._max_size = max_size hunk ./src/allmydata/storage/backends/disk/immutable.py 63 - if create: - # touch the file, so later callers will see that we're working on + + # If we are creating the share, _finalhome refers to the final path and + # _home to the incoming path. Otherwise, _finalhome is None. + self._finalhome = finalhome + self._home = home + self._shnum = shnum + + if self._finalhome is not None: + # Touch the file, so later callers will see that we're working on # it. Also construct the metadata. hunk ./src/allmydata/storage/backends/disk/immutable.py 73 - assert not os.path.exists(self.home) - fileutil.make_dirs(os.path.dirname(self.home)) - f = open(self.home, 'wb') + assert not self._finalhome.exists() + fp_make_dirs(self._home.parent()) # The second field -- the four-byte share data length -- is no # longer used as of Tahoe v1.3.0, but we continue to write it in # there in case someone downgrades a storage server from >= hunk ./src/allmydata/storage/backends/disk/immutable.py 84 # the largest length that can fit into the field. That way, even # if this does happen, the old < v1.3.0 server will still allow # clients to read the first part of the share. - f.write(struct.pack(">LLL", 1, min(2**32-1, max_size), 0)) - f.close() - self._lease_offset = max_size + 0x0c + self._home.setContent(struct.pack(self.HEADER, 1, min(2**32-1, max_size), 0) ) + self._lease_offset = self.HEADER_SIZE + max_size self._num_leases = 0 else: hunk ./src/allmydata/storage/backends/disk/immutable.py 88 - f = open(self.home, 'rb') - filesize = os.path.getsize(self.home) - (version, unused, num_leases) = struct.unpack(">LLL", f.read(0xc)) - f.close() + f = self._home.open(mode='rb') + try: + (version, unused, num_leases) = struct.unpack(self.HEADER, f.read(self.HEADER_SIZE)) + finally: + f.close() if version != 1: msg = "sharefile %s had version %d but we wanted 1" % \ hunk ./src/allmydata/storage/backends/disk/immutable.py 95 - (filename, version) + (self._home, version) raise UnknownImmutableContainerVersionError(msg) hunk ./src/allmydata/storage/backends/disk/immutable.py 97 + + filesize = self._home.getsize() self._num_leases = num_leases self._lease_offset = filesize - (num_leases * self.LEASE_SIZE) hunk ./src/allmydata/storage/backends/disk/immutable.py 101 - self._data_offset = 0xc + self._data_offset = self.HEADER_SIZE + + def __repr__(self): + return ("" + % (si_b2a(self._storageindex or ""), self._shnum, quote_filepath(self._home))) + + def close(self): + fileutil.fp_make_dirs(self._finalhome.parent()) + self._home.moveTo(self._finalhome) + + # self._home is like storage/shares/incoming/ab/abcde/4 . + # We try to delete the parent (.../ab/abcde) to avoid leaving + # these directories lying around forever, but the delete might + # fail if we're working on another share for the same storage + # index (like ab/abcde/5). The alternative approach would be to + # use a hierarchy of objects (PrefixHolder, BucketHolder, + # ShareWriter), each of which is responsible for a single + # directory on disk, and have them use reference counting of + # their children to know when they should do the rmdir. This + # approach is simpler, but relies on os.rmdir (used by + # fp_rmdir_if_empty) refusing to delete a non-empty directory. + # Do *not* use fileutil.fp_remove() here! + parent = self._home.parent() + fileutil.fp_rmdir_if_empty(parent) + + # we also delete the grandparent (prefix) directory, .../ab , + # again to avoid leaving directories lying around. This might + # fail if there is another bucket open that shares a prefix (like + # ab/abfff). + fileutil.fp_rmdir_if_empty(parent.parent()) + + # we leave the great-grandparent (incoming/) directory in place. + + # allow lease changes after closing. + self._home = self._finalhome + self._finalhome = None + return defer.succeed(None) + + def get_used_space(self): + return (fileutil.get_used_space(self._finalhome) + + fileutil.get_used_space(self._home)) + + def get_storage_index(self): + return self._storageindex + + def get_storage_index_string(self): + return si_b2a(self._storageindex) + + def get_shnum(self): + return self._shnum def unlink(self): hunk ./src/allmydata/storage/backends/disk/immutable.py 153 - os.unlink(self.home) + fileutil.fp_remove(self._home) + return defer.succeed(None) hunk ./src/allmydata/storage/backends/disk/immutable.py 156 - def read_share_data(self, offset, length): + def get_allocated_size(self): + return self._max_size + + def get_size(self): + return self._home.getsize() + + def get_data_length(self): + return self._lease_offset - self._data_offset + + def readv(self, readv): + datav = [] + f = self._home.open('rb') + try: + for (offset, length) in readv: + datav.append(self._read_share_data(f, offset, length)) + finally: + f.close() + return defer.succeed(datav) + + def _get_filepath(self): + return self._home + + def _read_share_data(self, f, offset, length): precondition(offset >= 0) # Reads beyond the end of the data are truncated. Reads that start hunk ./src/allmydata/storage/backends/disk/immutable.py 187 actuallength = max(0, min(length, self._lease_offset-seekpos)) if actuallength == 0: return "" - f = open(self.home, 'rb') f.seek(seekpos) return f.read(actuallength) hunk ./src/allmydata/storage/backends/disk/immutable.py 190 + def read_share_data(self, offset, length): + f = self._home.open(mode='rb') + try: + return defer.succeed(self._read_share_data(f, offset, length)) + finally: + f.close() + def write_share_data(self, offset, data): length = len(data) precondition(offset >= 0, offset) hunk ./src/allmydata/storage/backends/disk/immutable.py 202 if self._max_size is not None and offset+length > self._max_size: raise DataTooLargeError(self._max_size, offset, length) - f = open(self.home, 'rb+') - real_offset = self._data_offset+offset - f.seek(real_offset) - assert f.tell() == real_offset - f.write(data) - f.close() + f = self._home.open(mode='rb+') + try: + real_offset = self._data_offset+offset + f.seek(real_offset) + assert f.tell() == real_offset + f.write(data) + return defer.succeed(None) + finally: + f.close() def _write_lease_record(self, f, lease_number, lease_info): offset = self._lease_offset + lease_number * self.LEASE_SIZE hunk ./src/allmydata/storage/backends/disk/immutable.py 220 def _read_num_leases(self, f): f.seek(0x08) - (num_leases,) = struct.unpack(">L", f.read(4)) + ro = f.read(4) + (num_leases,) = struct.unpack(">L", ro) return num_leases def _write_num_leases(self, f, num_leases): hunk ./src/allmydata/storage/backends/disk/immutable.py 231 def _truncate_leases(self, f, num_leases): f.truncate(self._lease_offset + num_leases * self.LEASE_SIZE) + # These lease operations are intended for use by disk_backend.py. + # Other clients should not depend on the fact that the disk backend + # stores leases in share files. + # XXX BucketWriter in bucket.py also relies on add_lease. + def get_leases(self): """Yields a LeaseInfo instance for all leases.""" hunk ./src/allmydata/storage/backends/disk/immutable.py 238 - f = open(self.home, 'rb') - (version, unused, num_leases) = struct.unpack(">LLL", f.read(0xc)) - f.seek(self._lease_offset) - for i in range(num_leases): - data = f.read(self.LEASE_SIZE) - if data: - yield LeaseInfo().from_immutable_data(data) + f = self._home.open(mode='rb') + try: + (version, unused, num_leases) = struct.unpack(self.HEADER, f.read(self.HEADER_SIZE)) + f.seek(self._lease_offset) + for i in range(num_leases): + data = f.read(self.LEASE_SIZE) + if data: + yield LeaseInfo().from_immutable_data(data) + finally: + f.close() def add_lease(self, lease_info): hunk ./src/allmydata/storage/backends/disk/immutable.py 250 - f = open(self.home, 'rb+') - num_leases = self._read_num_leases(f) - self._write_lease_record(f, num_leases, lease_info) - self._write_num_leases(f, num_leases+1) - f.close() + f = self._home.open(mode='rb+') + try: + num_leases = self._read_num_leases(f) + self._write_lease_record(f, num_leases, lease_info) + self._write_num_leases(f, num_leases+1) + finally: + f.close() def renew_lease(self, renew_secret, new_expire_time): hunk ./src/allmydata/storage/backends/disk/immutable.py 259 - for i,lease in enumerate(self.get_leases()): - if constant_time_compare(lease.renew_secret, renew_secret): - # yup. See if we need to update the owner time. - if new_expire_time > lease.expiration_time: - # yes - lease.expiration_time = new_expire_time - f = open(self.home, 'rb+') - self._write_lease_record(f, i, lease) - f.close() - return + try: + for i, lease in enumerate(self.get_leases()): + if constant_time_compare(lease.renew_secret, renew_secret): + # yup. See if we need to update the owner time. + if new_expire_time > lease.expiration_time: + # yes + lease.expiration_time = new_expire_time + f = self._home.open('rb+') + try: + self._write_lease_record(f, i, lease) + finally: + f.close() + return + except IndexError, e: + raise Exception("IndexError: %s" % (e,)) raise IndexError("unable to renew non-existent lease") def add_or_renew_lease(self, lease_info): hunk ./src/allmydata/storage/backends/disk/immutable.py 299 num_leases_removed += 1 if not num_leases_removed: raise IndexError("unable to find matching lease to cancel") + + space_freed = 0 if num_leases_removed: # pack and write out the remaining leases. We write these out in # the same order as they were added, so that if we crash while hunk ./src/allmydata/storage/backends/disk/immutable.py 306 # doing this, we won't lose any non-cancelled leases. leases = [l for l in leases if l] # remove the cancelled leases - f = open(self.home, 'rb+') - for i,lease in enumerate(leases): - self._write_lease_record(f, i, lease) - self._write_num_leases(f, len(leases)) - self._truncate_leases(f, len(leases)) - f.close() - space_freed = self.LEASE_SIZE * num_leases_removed - if not len(leases): - space_freed += os.stat(self.home)[stat.ST_SIZE] - self.unlink() + if len(leases) > 0: + f = self._home.open('rb+') + try: + for i, lease in enumerate(leases): + self._write_lease_record(f, i, lease) + self._write_num_leases(f, len(leases)) + self._truncate_leases(f, len(leases)) + finally: + f.close() + space_freed = self.LEASE_SIZE * num_leases_removed + else: + space_freed = fileutil.get_used_space(self._home) + self.unlink() return space_freed hunk ./src/allmydata/storage/backends/disk/immutable.py 322 -class BucketWriter(Referenceable): - implements(RIBucketWriter) - - def __init__(self, ss, incominghome, finalhome, max_size, lease_info, canary): - self.ss = ss - self.incominghome = incominghome - self.finalhome = finalhome - self._max_size = max_size # don't allow the client to write more than this - self._canary = canary - self._disconnect_marker = canary.notifyOnDisconnect(self._disconnected) - self.closed = False - self.throw_out_all_data = False - self._sharefile = ShareFile(incominghome, create=True, max_size=max_size) - # also, add our lease to the file now, so that other ones can be - # added by simultaneous uploaders - self._sharefile.add_lease(lease_info) - - def allocated_size(self): - return self._max_size - - def remote_write(self, offset, data): - start = time.time() - precondition(not self.closed) - if self.throw_out_all_data: - return - self._sharefile.write_share_data(offset, data) - self.ss.add_latency("write", time.time() - start) - self.ss.count("write") - - def remote_close(self): - precondition(not self.closed) - start = time.time() - - fileutil.make_dirs(os.path.dirname(self.finalhome)) - fileutil.rename(self.incominghome, self.finalhome) - try: - # self.incominghome is like storage/shares/incoming/ab/abcde/4 . - # We try to delete the parent (.../ab/abcde) to avoid leaving - # these directories lying around forever, but the delete might - # fail if we're working on another share for the same storage - # index (like ab/abcde/5). The alternative approach would be to - # use a hierarchy of objects (PrefixHolder, BucketHolder, - # ShareWriter), each of which is responsible for a single - # directory on disk, and have them use reference counting of - # their children to know when they should do the rmdir. This - # approach is simpler, but relies on os.rmdir refusing to delete - # a non-empty directory. Do *not* use fileutil.rm_dir() here! - os.rmdir(os.path.dirname(self.incominghome)) - # we also delete the grandparent (prefix) directory, .../ab , - # again to avoid leaving directories lying around. This might - # fail if there is another bucket open that shares a prefix (like - # ab/abfff). - os.rmdir(os.path.dirname(os.path.dirname(self.incominghome))) - # we leave the great-grandparent (incoming/) directory in place. - except EnvironmentError: - # ignore the "can't rmdir because the directory is not empty" - # exceptions, those are normal consequences of the - # above-mentioned conditions. - pass - self._sharefile = None - self.closed = True - self._canary.dontNotifyOnDisconnect(self._disconnect_marker) - - filelen = os.stat(self.finalhome)[stat.ST_SIZE] - self.ss.bucket_writer_closed(self, filelen) - self.ss.add_latency("close", time.time() - start) - self.ss.count("close") - - def _disconnected(self): - if not self.closed: - self._abort() - - def remote_abort(self): - log.msg("storage: aborting sharefile %s" % self.incominghome, - facility="tahoe.storage", level=log.UNUSUAL) - if not self.closed: - self._canary.dontNotifyOnDisconnect(self._disconnect_marker) - self._abort() - self.ss.count("abort") - - def _abort(self): - if self.closed: - return - - os.remove(self.incominghome) - # if we were the last share to be moved, remove the incoming/ - # directory that was our parent - parentdir = os.path.split(self.incominghome)[0] - if not os.listdir(parentdir): - os.rmdir(parentdir) - self._sharefile = None - - # We are now considered closed for further writing. We must tell - # the storage server about this so that it stops expecting us to - # use the space it allocated for us earlier. - self.closed = True - self.ss.bucket_writer_closed(self, 0) - - -class BucketReader(Referenceable): - implements(RIBucketReader) - - def __init__(self, ss, sharefname, storage_index=None, shnum=None): - self.ss = ss - self._share_file = ShareFile(sharefname) - self.storage_index = storage_index - self.shnum = shnum - - def __repr__(self): - return "<%s %s %s>" % (self.__class__.__name__, - base32.b2a_l(self.storage_index[:8], 60), - self.shnum) - - def remote_read(self, offset, length): - start = time.time() - data = self._share_file.read_share_data(offset, length) - self.ss.add_latency("read", time.time() - start) - self.ss.count("read") - return data +def load_immutable_disk_share(home, storageindex=None, shnum=None): + return ImmutableDiskShare(home, storageindex=storageindex, shnum=shnum) hunk ./src/allmydata/storage/backends/disk/immutable.py 325 - def remote_advise_corrupt_share(self, reason): - return self.ss.remote_advise_corrupt_share("immutable", - self.storage_index, - self.shnum, - reason) +def create_immutable_disk_share(home, finalhome, max_size, storageindex=None, shnum=None): + return ImmutableDiskShare(home, finalhome=finalhome, max_size=max_size, + storageindex=storageindex, shnum=shnum) hunk ./src/allmydata/storage/backends/disk/mutable.py 1 -import os, stat, struct hunk ./src/allmydata/storage/backends/disk/mutable.py 2 -from allmydata.interfaces import BadWriteEnablerError -from allmydata.util import idlib, log +import struct + +from twisted.internet import defer + +from zope.interface import implements +from allmydata.interfaces import IMutableShare, BadWriteEnablerError + +from allmydata.util import fileutil, idlib, log from allmydata.util.assertutil import precondition from allmydata.util.hashutil import constant_time_compare hunk ./src/allmydata/storage/backends/disk/mutable.py 12 -from allmydata.storage.lease import LeaseInfo -from allmydata.storage.common import UnknownMutableContainerVersionError, \ +from allmydata.util.encodingutil import quote_filepath +from allmydata.storage.common import si_b2a, UnknownMutableContainerVersionError, \ DataTooLargeError hunk ./src/allmydata/storage/backends/disk/mutable.py 15 +from allmydata.storage.lease import LeaseInfo +from allmydata.storage.backends.base import testv_compare +from allmydata.mutable.layout import MUTABLE_MAGIC + hunk ./src/allmydata/storage/backends/disk/mutable.py 20 -# the MutableShareFile is like the ShareFile, but used for mutable data. It -# has a different layout. See docs/mutable.txt for more details. +# The MutableDiskShare is like the ImmutableDiskShare, but used for mutable data. +# It has a different layout. See docs/mutable.rst for more details. # # offset size name # 1 0 32 magic verstr "tahoe mutable container v1" plus binary hunk ./src/allmydata/storage/backends/disk/mutable.py 45 assert struct.calcsize(">L") == 4, struct.calcsize(">L") assert struct.calcsize(">Q") == 8, struct.calcsize(">Q") -class MutableShareFile: + +class MutableDiskShare(object): + implements(IMutableShare) sharetype = "mutable" DATA_LENGTH_OFFSET = struct.calcsize(">32s20s32s") hunk ./src/allmydata/storage/backends/disk/mutable.py 57 assert LEASE_SIZE == 92 DATA_OFFSET = HEADER_SIZE + 4*LEASE_SIZE assert DATA_OFFSET == 468, DATA_OFFSET - # our sharefiles share with a recognizable string, plus some random - # binary data to reduce the chance that a regular text file will look - # like a sharefile. - MAGIC = "Tahoe mutable container v1\n" + "\x75\x09\x44\x03\x8e" + + MAGIC = MUTABLE_MAGIC assert len(MAGIC) == 32 MAX_SIZE = 2*1000*1000*1000 # 2GB, kind of arbitrary # TODO: decide upon a policy for max share size hunk ./src/allmydata/storage/backends/disk/mutable.py 63 - def __init__(self, filename, parent=None): - self.home = filename - if os.path.exists(self.home): + def __init__(self, home, storageindex, shnum, parent=None): + """ + Clients should use the load_mutable_disk_share and create_mutable_disk_share + factory functions rather than creating instances directly. + """ + self._storageindex = storageindex + self._shnum = shnum + self._home = home + if self._home.exists(): # we don't cache anything, just check the magic hunk ./src/allmydata/storage/backends/disk/mutable.py 73 - f = open(self.home, 'rb') - data = f.read(self.HEADER_SIZE) - (magic, - write_enabler_nodeid, write_enabler, - data_length, extra_least_offset) = \ - struct.unpack(">32s20s32sQQ", data) - if magic != self.MAGIC: - msg = "sharefile %s had magic '%r' but we wanted '%r'" % \ - (filename, magic, self.MAGIC) - raise UnknownMutableContainerVersionError(msg) + f = self._home.open('rb') + try: + data = f.read(self.HEADER_SIZE) + (magic, + write_enabler_nodeid, write_enabler, + data_length, extra_lease_offset) = \ + struct.unpack(">32s20s32sQQ", data) + if magic != self.MAGIC: + msg = "sharefile %s had magic '%r' but we wanted '%r'" % \ + (quote_filepath(self._home), magic, self.MAGIC) + raise UnknownMutableContainerVersionError(msg) + finally: + f.close() self.parent = parent # for logging def log(self, *args, **kwargs): hunk ./src/allmydata/storage/backends/disk/mutable.py 89 - return self.parent.log(*args, **kwargs) + if self.parent: + return self.parent.log(*args, **kwargs) hunk ./src/allmydata/storage/backends/disk/mutable.py 92 - def create(self, my_nodeid, write_enabler): - assert not os.path.exists(self.home) + def create(self, serverid, write_enabler): + assert not self._home.exists(), "%r already exists and should not" % (self._home,) data_length = 0 extra_lease_offset = (self.HEADER_SIZE + 4 * self.LEASE_SIZE hunk ./src/allmydata/storage/backends/disk/mutable.py 100 + data_length) assert extra_lease_offset == self.DATA_OFFSET # true at creation num_extra_leases = 0 - f = open(self.home, 'wb') - header = struct.pack(">32s20s32sQQ", - self.MAGIC, my_nodeid, write_enabler, - data_length, extra_lease_offset, - ) - leases = ("\x00"*self.LEASE_SIZE) * 4 - f.write(header + leases) - # data goes here, empty after creation - f.write(struct.pack(">L", num_extra_leases)) - # extra leases go here, none at creation - f.close() + f = self._home.open('wb') + try: + header = struct.pack(">32s20s32sQQ", + self.MAGIC, serverid, write_enabler, + data_length, extra_lease_offset, + ) + leases = ("\x00"*self.LEASE_SIZE) * 4 + f.write(header + leases) + # data goes here, empty after creation + f.write(struct.pack(">L", num_extra_leases)) + # extra leases go here, none at creation + finally: + f.close() + return self + + def __repr__(self): + return ("" + % (si_b2a(self._storageindex or ""), self._shnum, quote_filepath(self._home))) + + def get_used_space(self): + return fileutil.get_used_space(self._home) + + def get_storage_index(self): + return self._storageindex + + def get_storage_index_string(self): + return si_b2a(self._storageindex) + + def get_shnum(self): + return self._shnum def unlink(self): hunk ./src/allmydata/storage/backends/disk/mutable.py 132 - os.unlink(self.home) + fileutil.fp_remove(self._home) + return defer.succeed(None) + + def _get_filepath(self): + return self._home def _read_data_length(self, f): f.seek(self.DATA_LENGTH_OFFSET) hunk ./src/allmydata/storage/backends/disk/mutable.py 148 f.write(struct.pack(">Q", data_length)) def _read_share_data(self, f, offset, length): - precondition(offset >= 0) + precondition(offset >= 0, offset=offset) data_length = self._read_data_length(f) if offset+length > data_length: # reads beyond the end of the data are truncated. Reads that hunk ./src/allmydata/storage/backends/disk/mutable.py 181 f.seek(extra_lease_offset) f.write(struct.pack(">L", num_leases)) - def _change_container_size(self, f, new_container_size): - if new_container_size > self.MAX_SIZE: + def _change_container_size(self, f, new_data_length): + if new_data_length > self.MAX_SIZE: raise DataTooLargeError() old_extra_lease_offset = self._read_extra_lease_offset(f) hunk ./src/allmydata/storage/backends/disk/mutable.py 185 - new_extra_lease_offset = self.DATA_OFFSET + new_container_size + new_extra_lease_offset = self.DATA_OFFSET + new_data_length if new_extra_lease_offset < old_extra_lease_offset: # TODO: allow containers to shrink. For now they remain large. return hunk ./src/allmydata/storage/backends/disk/mutable.py 208 def _write_share_data(self, f, offset, data): length = len(data) - precondition(offset >= 0) + precondition(offset >= 0, offset=offset) + precondition(offset+length < self.MAX_SIZE, offset=offset, length=length) + data_length = self._read_data_length(f) extra_lease_offset = self._read_extra_lease_offset(f) hunk ./src/allmydata/storage/backends/disk/mutable.py 306 def get_leases(self): """Yields a LeaseInfo instance for all leases.""" - f = open(self.home, 'rb') - for i, lease in self._enumerate_leases(f): - yield lease - f.close() + f = self._home.open('rb') + try: + for i, lease in self._enumerate_leases(f): + yield lease + finally: + f.close() def _enumerate_leases(self, f): for i in range(self._get_num_lease_slots(f)): hunk ./src/allmydata/storage/backends/disk/mutable.py 322 except IndexError: return + # These lease operations are intended for use by disk_backend.py. + # Other non-test clients should not depend on the fact that the disk + # backend stores leases in share files. + def add_lease(self, lease_info): precondition(lease_info.owner_num != 0) # 0 means "no lease here" hunk ./src/allmydata/storage/backends/disk/mutable.py 328 - f = open(self.home, 'rb+') - num_lease_slots = self._get_num_lease_slots(f) - empty_slot = self._get_first_empty_lease_slot(f) - if empty_slot is not None: - self._write_lease_record(f, empty_slot, lease_info) - else: - self._write_lease_record(f, num_lease_slots, lease_info) - f.close() + f = self._home.open('rb+') + try: + num_lease_slots = self._get_num_lease_slots(f) + empty_slot = self._get_first_empty_lease_slot(f) + if empty_slot is not None: + self._write_lease_record(f, empty_slot, lease_info) + else: + self._write_lease_record(f, num_lease_slots, lease_info) + finally: + f.close() def renew_lease(self, renew_secret, new_expire_time): accepting_nodeids = set() hunk ./src/allmydata/storage/backends/disk/mutable.py 341 - f = open(self.home, 'rb+') - for (leasenum,lease) in self._enumerate_leases(f): - if constant_time_compare(lease.renew_secret, renew_secret): - # yup. See if we need to update the owner time. - if new_expire_time > lease.expiration_time: - # yes - lease.expiration_time = new_expire_time - self._write_lease_record(f, leasenum, lease) - f.close() - return - accepting_nodeids.add(lease.nodeid) - f.close() + f = self._home.open('rb+') + try: + for (leasenum, lease) in self._enumerate_leases(f): + if constant_time_compare(lease.renew_secret, renew_secret): + # yup. See if we need to update the owner time. + if new_expire_time > lease.expiration_time: + # yes + lease.expiration_time = new_expire_time + self._write_lease_record(f, leasenum, lease) + return + accepting_nodeids.add(lease.nodeid) + finally: + f.close() # Return the accepting_nodeids set, to give the client a chance to # update the leases on a share that has been migrated from its # original server to a new one. hunk ./src/allmydata/storage/backends/disk/mutable.py 379 deleting the file). Raise IndexError if there was no lease with the given cancel_secret.""" + # XXX can this be more like ImmutableDiskShare.cancel_lease? + accepting_nodeids = set() modified = 0 remaining = 0 hunk ./src/allmydata/storage/backends/disk/mutable.py 389 cancel_secret="\x00"*32, expiration_time=0, nodeid="\x00"*20) - f = open(self.home, 'rb+') - for (leasenum,lease) in self._enumerate_leases(f): - accepting_nodeids.add(lease.nodeid) - if constant_time_compare(lease.cancel_secret, cancel_secret): - self._write_lease_record(f, leasenum, blank_lease) - modified += 1 - else: - remaining += 1 - if modified: - freed_space = self._pack_leases(f) + f = self._home.open('rb+') + try: + for (leasenum, lease) in self._enumerate_leases(f): + accepting_nodeids.add(lease.nodeid) + if constant_time_compare(lease.cancel_secret, cancel_secret): + self._write_lease_record(f, leasenum, blank_lease) + modified += 1 + else: + remaining += 1 + if modified: + freed_space = self._pack_leases(f) + finally: f.close() hunk ./src/allmydata/storage/backends/disk/mutable.py 402 - if not remaining: - freed_space += os.stat(self.home)[stat.ST_SIZE] + + if modified > 0: + if remaining == 0: + freed_space = fileutil.get_used_space(self._home) self.unlink() return freed_space hunk ./src/allmydata/storage/backends/disk/mutable.py 432 def readv(self, readv): datav = [] - f = open(self.home, 'rb') - for (offset, length) in readv: - datav.append(self._read_share_data(f, offset, length)) - f.close() - return datav + f = self._home.open('rb') + try: + for (offset, length) in readv: + datav.append(self._read_share_data(f, offset, length)) + finally: + f.close() + return defer.succeed(datav) hunk ./src/allmydata/storage/backends/disk/mutable.py 440 -# def remote_get_length(self): -# f = open(self.home, 'rb') -# data_length = self._read_data_length(f) -# f.close() -# return data_length + def get_size(self): + return self._home.getsize() hunk ./src/allmydata/storage/backends/disk/mutable.py 443 - def check_write_enabler(self, write_enabler, si_s): - f = open(self.home, 'rb+') - (real_write_enabler, write_enabler_nodeid) = \ - self._read_write_enabler_and_nodeid(f) - f.close() + def get_data_length(self): + f = self._home.open('rb') + try: + data_length = self._read_data_length(f) + finally: + f.close() + return data_length + + def check_write_enabler(self, write_enabler): + f = self._home.open('rb+') + try: + (real_write_enabler, write_enabler_nodeid) = self._read_write_enabler_and_nodeid(f) + finally: + f.close() # avoid a timing attack hunk ./src/allmydata/storage/backends/disk/mutable.py 458 - #if write_enabler != real_write_enabler: if not constant_time_compare(write_enabler, real_write_enabler): # accomodate share migration by reporting the nodeid used for the # old write enabler. hunk ./src/allmydata/storage/backends/disk/mutable.py 465 " recorded by nodeid %(nodeid)s", facility="tahoe.storage", level=log.WEIRD, umid="cE1eBQ", - si=si_s, nodeid=idlib.nodeid_b2a(write_enabler_nodeid)) + si=self.get_storage_index_string(), + nodeid=idlib.nodeid_b2a(write_enabler_nodeid)) msg = "The write enabler was recorded by nodeid '%s'." % \ (idlib.nodeid_b2a(write_enabler_nodeid),) raise BadWriteEnablerError(msg) hunk ./src/allmydata/storage/backends/disk/mutable.py 470 + return defer.succeed(None) def check_testv(self, testv): test_good = True hunk ./src/allmydata/storage/backends/disk/mutable.py 474 - f = open(self.home, 'rb+') - for (offset, length, operator, specimen) in testv: - data = self._read_share_data(f, offset, length) - if not testv_compare(data, operator, specimen): - test_good = False - break - f.close() - return test_good + f = self._home.open('rb+') + try: + for (offset, length, operator, specimen) in testv: + data = self._read_share_data(f, offset, length) + if not testv_compare(data, operator, specimen): + test_good = False + break + finally: + f.close() + return defer.succeed(test_good) def writev(self, datav, new_length): hunk ./src/allmydata/storage/backends/disk/mutable.py 486 - f = open(self.home, 'rb+') + precondition(new_length is None or new_length >= 0, new_length=new_length) for (offset, data) in datav: hunk ./src/allmydata/storage/backends/disk/mutable.py 488 - self._write_share_data(f, offset, data) - if new_length is not None: - cur_length = self._read_data_length(f) - if new_length < cur_length: - self._write_data_length(f, new_length) - # TODO: if we're going to shrink the share file when the - # share data has shrunk, then call - # self._change_container_size() here. - f.close() + precondition(offset >= 0, offset=offset) + if offset+len(data) > self.MAX_SIZE: + raise DataTooLargeError() hunk ./src/allmydata/storage/backends/disk/mutable.py 492 -def testv_compare(a, op, b): - assert op in ("lt", "le", "eq", "ne", "ge", "gt") - if op == "lt": - return a < b - if op == "le": - return a <= b - if op == "eq": - return a == b - if op == "ne": - return a != b - if op == "ge": - return a >= b - if op == "gt": - return a > b - # never reached + f = self._home.open('rb+') + try: + for (offset, data) in datav: + self._write_share_data(f, offset, data) + if new_length is not None: + cur_length = self._read_data_length(f) + if new_length < cur_length: + self._write_data_length(f, new_length) + # TODO: if we're going to shrink the share file when the + # share data has shrunk, then call + # self._change_container_size() here. + finally: + f.close() + return defer.succeed(None) hunk ./src/allmydata/storage/backends/disk/mutable.py 507 -class EmptyShare: + def close(self): + return defer.succeed(None) hunk ./src/allmydata/storage/backends/disk/mutable.py 510 - def check_testv(self, testv): - test_good = True - for (offset, length, operator, specimen) in testv: - data = "" - if not testv_compare(data, operator, specimen): - test_good = False - break - return test_good hunk ./src/allmydata/storage/backends/disk/mutable.py 511 -def create_mutable_sharefile(filename, my_nodeid, write_enabler, parent): - ms = MutableShareFile(filename, parent) - ms.create(my_nodeid, write_enabler) - del ms - return MutableShareFile(filename, parent) +def load_mutable_disk_share(home, storageindex=None, shnum=None, parent=None): + return MutableDiskShare(home, storageindex, shnum, parent) hunk ./src/allmydata/storage/backends/disk/mutable.py 514 +def create_mutable_disk_share(home, serverid, write_enabler, storageindex=None, shnum=None, parent=None): + ms = MutableDiskShare(home, storageindex, shnum, parent) + return ms.create(serverid, write_enabler) hunk ./src/allmydata/storage/common.py 1 - -import os.path from allmydata.util import base32 class DataTooLargeError(Exception): hunk ./src/allmydata/storage/common.py 5 pass -class UnknownMutableContainerVersionError(Exception): + +class UnknownContainerVersionError(Exception): + pass + +class UnknownMutableContainerVersionError(UnknownContainerVersionError): pass hunk ./src/allmydata/storage/common.py 11 -class UnknownImmutableContainerVersionError(Exception): + +class UnknownImmutableContainerVersionError(UnknownContainerVersionError): pass hunk ./src/allmydata/storage/common.py 21 def si_a2b(ascii_storageindex): return base32.a2b(ascii_storageindex) - -def storage_index_to_dir(storageindex): - sia = si_b2a(storageindex) - return os.path.join(sia[:2], sia) hunk ./src/allmydata/storage/crawler.py 2 -import os, time, struct -import cPickle as pickle +import pickle, struct from twisted.internet import reactor from twisted.application import service hunk ./src/allmydata/storage/crawler.py 5 + +from allmydata.util.assertutil import precondition +from allmydata.interfaces import IStorageBackend from allmydata.storage.common import si_b2a hunk ./src/allmydata/storage/crawler.py 9 -from allmydata.util import fileutil + class TimeSliceExceeded(Exception): pass hunk ./src/allmydata/storage/crawler.py 16 class ShareCrawler(service.MultiService): - """A ShareCrawler subclass is attached to a StorageServer, and - periodically walks all of its shares, processing each one in some - fashion. This crawl is rate-limited, to reduce the IO burden on the host, - since large servers can easily have a terabyte of shares, in several - million files, which can take hours or days to read. + """ + An instance of a subclass of ShareCrawler is attached to a storage + backend, and periodically walks the backend's shares, processing them + in some fashion. This crawl is rate-limited to reduce the I/O burden on + the host, since large servers can easily have a terabyte of shares in + several million files, which can take hours or days to read. Once the crawler starts a cycle, it will proceed at a rate limited by the allowed_cpu_percentage= and cpu_slice= parameters: yielding the reactor hunk ./src/allmydata/storage/crawler.py 40 prefix. On this server, each prefixdir took 130ms-200ms to list the first time, and 17ms to list the second time. - To use a crawler, create a subclass which implements the process_bucket() - method. It will be called with a prefixdir and a base32 storage index - string. process_bucket() must run synchronously. Any keys added to - self.state will be preserved. Override add_initial_state() to set up - initial state keys. Override finished_cycle() to perform additional - processing when the cycle is complete. Any status that the crawler - produces should be put in the self.state dictionary. Status renderers - (like a web page which describes the accomplishments of your crawler) - will use crawler.get_state() to retrieve this dictionary; they can - present the contents as they see fit. + To implement a crawler, create a subclass that implements the + process_shareset() method. It will be called with a prefixdir and an + object providing the IShareSet interface. process_shareset() must run + synchronously. Any keys added to self.state will be preserved. Override + add_initial_state() to set up initial state keys. Override + finished_cycle() to perform additional processing when the cycle is + complete. Any status that the crawler produces should be put in the + self.state dictionary. Status renderers (like a web page describing the + accomplishments of your crawler) will use crawler.get_state() to retrieve + this dictionary; they can present the contents as they see fit. hunk ./src/allmydata/storage/crawler.py 51 - Then create an instance, with a reference to a StorageServer and a - filename where it can store persistent state. The statefile is used to - keep track of how far around the ring the process has travelled, as well - as timing history to allow the pace to be predicted and controlled. The - statefile will be updated and written to disk after each time slice (just - before the crawler yields to the reactor), and also after each cycle is - finished, and also when stopService() is called. Note that this means - that a crawler which is interrupted with SIGKILL while it is in the - middle of a time slice will lose progress: the next time the node is - started, the crawler will repeat some unknown amount of work. + Then create an instance, with a reference to a backend object providing + the IStorageBackend interface, and a filename where it can store + persistent state. The statefile is used to keep track of how far around + the ring the process has travelled, as well as timing history to allow + the pace to be predicted and controlled. The statefile will be updated + and written to disk after each time slice (just before the crawler yields + to the reactor), and also after each cycle is finished, and also when + stopService() is called. Note that this means that a crawler that is + interrupted with SIGKILL while it is in the middle of a time slice will + lose progress: the next time the node is started, the crawler will repeat + some unknown amount of work. The crawler instance must be started with startService() before it will hunk ./src/allmydata/storage/crawler.py 64 - do any work. To make it stop doing work, call stopService(). + do any work. To make it stop doing work, call stopService(). A crawler + is usually a child service of a StorageServer, although it should not + depend on that. + + For historical reasons, some dictionary key names use the term "bucket" + for what is now preferably called a "shareset" (the set of shares that a + server holds under a given storage index). + + Subclasses should measure time using self.clock.seconds(), rather than + time.time(), in order to make themselves deterministically testable. """ slow_start = 300 # don't start crawling for 5 minutes after startup hunk ./src/allmydata/storage/crawler.py 82 cpu_slice = 1.0 # use up to 1.0 seconds before yielding minimum_cycle_time = 300 # don't run a cycle faster than this - def __init__(self, server, statefile, allowed_cpu_percentage=None): + def __init__(self, backend, statefp, allowed_cpu_percentage=None, clock=None): + precondition(IStorageBackend.providedBy(backend), backend) + assert backend.supports_crawlers(), backend service.MultiService.__init__(self) hunk ./src/allmydata/storage/crawler.py 86 + self.backend = backend + self.statefp = statefp if allowed_cpu_percentage is not None: self.allowed_cpu_percentage = allowed_cpu_percentage hunk ./src/allmydata/storage/crawler.py 90 - self.server = server - self.sharedir = server.sharedir - self.statefile = statefile + self.clock = clock or reactor self.prefixes = [si_b2a(struct.pack(">H", i << (16-10)))[:2] for i in range(2**10)] self.prefixes.sort() hunk ./src/allmydata/storage/crawler.py 95 self.timer = None - self.bucket_cache = (None, []) + self.shareset_cache = (None, []) self.current_sleep_time = None self.next_wake_time = None self.last_prefix_finished_time = None hunk ./src/allmydata/storage/crawler.py 148 d["cycle-in-progress"] = False d["next-crawl-time"] = self.next_wake_time d["remaining-wait-time"] = self.minus_or_none(self.next_wake_time, - time.time()) + self.clock.seconds()) else: d["cycle-in-progress"] = True pct = 100.0 * self.last_complete_prefix_index / len(self.prefixes) hunk ./src/allmydata/storage/crawler.py 165 # it's possible to call get_progress() from inside a crawler's # finished_prefix() function d["remaining-sleep-time"] = self.minus_or_none(self.next_wake_time, - time.time()) + self.clock.seconds()) per_cycle = None if self.last_cycle_elapsed_time is not None: per_cycle = self.last_cycle_elapsed_time hunk ./src/allmydata/storage/crawler.py 179 state dictionary. If we are not currently sleeping (i.e. get_state() was called from - inside the process_prefixdir, process_bucket, or finished_cycle() + inside the process_prefixdir, process_shareset, or finished_cycle() methods, or if startService has not yet been called on this crawler), these two keys will be None. hunk ./src/allmydata/storage/crawler.py 210 # shareset to be processed, or None if we # are sleeping between cycles try: - f = open(self.statefile, "rb") - state = pickle.load(f) - f.close() + pickled = self.statefp.getContent() except EnvironmentError: hunk ./src/allmydata/storage/crawler.py 212 + if self.statefp.exists(): + raise state = {"version": 1, "last-cycle-finished": None, "current-cycle": None, hunk ./src/allmydata/storage/crawler.py 220 "last-complete-prefix": None, "last-complete-bucket": None, } - state.setdefault("current-cycle-start-time", time.time()) # approximate + else: + state = pickle.loads(pickled) + + state.setdefault("current-cycle-start-time", self.clock.seconds()) # approximate self.state = state lcp = state["last-complete-prefix"] if lcp == None: hunk ./src/allmydata/storage/crawler.py 251 else: last_complete_prefix = self.prefixes[lcpi] self.state["last-complete-prefix"] = last_complete_prefix - tmpfile = self.statefile + ".tmp" - f = open(tmpfile, "wb") - pickle.dump(self.state, f) - f.close() - fileutil.move_into_place(tmpfile, self.statefile) + pickled = pickle.dumps(self.state) + self.statefp.setContent(pickled) def startService(self): # arrange things to look like we were just sleeping, so hunk ./src/allmydata/storage/crawler.py 259 # status/progress values work correctly self.sleeping_between_cycles = True self.current_sleep_time = self.slow_start - self.next_wake_time = time.time() + self.slow_start - self.timer = reactor.callLater(self.slow_start, self.start_slice) + self.next_wake_time = self.clock.seconds() + self.slow_start + self.timer = self.clock.callLater(self.slow_start, self.start_slice) service.MultiService.startService(self) def stopService(self): hunk ./src/allmydata/storage/crawler.py 271 return service.MultiService.stopService(self) def start_slice(self): - start_slice = time.time() + start_slice = self.clock.seconds() self.timer = None self.sleeping_between_cycles = False self.current_sleep_time = None hunk ./src/allmydata/storage/crawler.py 286 # someone might have used stopService() to shut us down return # either we finished a whole cycle, or we ran out of time - now = time.time() + now = self.clock.seconds() this_slice = now - start_slice # this_slice/(this_slice+sleep_time) = percentage # this_slice/percentage = this_slice+sleep_time hunk ./src/allmydata/storage/crawler.py 308 self.current_sleep_time = sleep_time # for status page self.next_wake_time = now + sleep_time self.yielding(sleep_time) - self.timer = reactor.callLater(sleep_time, self.start_slice) + self.timer = self.clock.callLater(sleep_time, self.start_slice) def start_current_prefix(self, start_slice): state = self.state hunk ./src/allmydata/storage/crawler.py 313 if state["current-cycle"] is None: - self.last_cycle_started_time = time.time() + self.last_cycle_started_time = self.clock.seconds() state["current-cycle-start-time"] = self.last_cycle_started_time if state["last-cycle-finished"] is None: state["current-cycle"] = 0 hunk ./src/allmydata/storage/crawler.py 325 for i in range(self.last_complete_prefix_index+1, len(self.prefixes)): # if we want to yield earlier, just raise TimeSliceExceeded() prefix = self.prefixes[i] - prefixdir = os.path.join(self.sharedir, prefix) - if i == self.bucket_cache[0]: - buckets = self.bucket_cache[1] + if i == self.shareset_cache[0]: + sharesets = self.shareset_cache[1] else: hunk ./src/allmydata/storage/crawler.py 328 - try: - buckets = os.listdir(prefixdir) - buckets.sort() - except EnvironmentError: - buckets = [] - self.bucket_cache = (i, buckets) - self.process_prefixdir(cycle, prefix, prefixdir, - buckets, start_slice) + sharesets = self.backend.get_sharesets_for_prefix(prefix) + self.shareset_cache = (i, sharesets) + self.process_prefixdir(cycle, prefix, sharesets, start_slice) self.last_complete_prefix_index = i hunk ./src/allmydata/storage/crawler.py 333 - now = time.time() + now = self.clock.seconds() if self.last_prefix_finished_time is not None: elapsed = now - self.last_prefix_finished_time self.last_prefix_elapsed_time = elapsed hunk ./src/allmydata/storage/crawler.py 340 self.last_prefix_finished_time = now self.finished_prefix(cycle, prefix) - if time.time() >= start_slice + self.cpu_slice: + if self.clock.seconds() >= start_slice + self.cpu_slice: raise TimeSliceExceeded() # yay! we finished the whole cycle hunk ./src/allmydata/storage/crawler.py 346 self.last_complete_prefix_index = -1 self.last_prefix_finished_time = None # don't include the sleep - now = time.time() + now = self.clock.seconds() if self.last_cycle_started_time is not None: self.last_cycle_elapsed_time = now - self.last_cycle_started_time state["last-complete-bucket"] = None hunk ./src/allmydata/storage/crawler.py 355 self.finished_cycle(cycle) self.save_state() - def process_prefixdir(self, cycle, prefix, prefixdir, buckets, start_slice): - """This gets a list of bucket names (i.e. storage index strings, + def process_prefixdir(self, cycle, prefix, sharesets, start_slice): + """ + This gets a list of shareset names (i.e. storage index strings, base32-encoded) in sorted order. You can override this if your crawler doesn't care about the actual hunk ./src/allmydata/storage/crawler.py 364 shares, for example a crawler which merely keeps track of how many sharesets are being managed by this server. - Subclasses which *do* care about actual bucket should leave this - method along, and implement process_bucket() instead. + Subclasses which *do* care about actual shareset should leave this + method alone, and implement process_shareset() instead. """ hunk ./src/allmydata/storage/crawler.py 367 - - for bucket in buckets: - if bucket <= self.state["last-complete-bucket"]: + for shareset in sharesets: + base32si = shareset.get_storage_index_string() + if base32si <= self.state["last-complete-bucket"]: continue hunk ./src/allmydata/storage/crawler.py 371 - self.process_bucket(cycle, prefix, prefixdir, bucket) - self.state["last-complete-bucket"] = bucket - if time.time() >= start_slice + self.cpu_slice: + self.process_shareset(cycle, prefix, shareset) + self.state["last-complete-bucket"] = base32si + if self.clock.seconds() >= start_slice + self.cpu_slice: raise TimeSliceExceeded() # the remaining methods are explictly for subclasses to implement. hunk ./src/allmydata/storage/crawler.py 386 """ pass - def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32): - """Examine a single bucket. Subclasses should do whatever they want + def process_shareset(self, cycle, prefix, shareset): + """ + Examine a single shareset. Subclasses should do whatever they want to do to the shares therein, then update self.state as necessary. If the crawler is never interrupted by SIGKILL, this method will be hunk ./src/allmydata/storage/crawler.py 400 To reduce the chance of duplicate work (i.e. to avoid adding multiple records to a database), you can call save_state() at the end of your - process_bucket() method. This will reduce the maximum duplicated work - to one bucket per SIGKILL. It will also add overhead, probably 1-20ms - per bucket (and some disk writes), which will count against your - allowed_cpu_percentage, and which may be considerable if - process_bucket() runs quickly. + process_shareset() method. This will reduce the maximum duplicated + work to one shareset per SIGKILL. It will also add overhead, probably + 1-20ms per shareset (and some disk writes), which will count against + your allowed_cpu_percentage, and which may be considerable if + process_shareset() runs quickly. This method is for subclasses to override. No upcall is necessary. """ hunk ./src/allmydata/storage/crawler.py 452 class BucketCountingCrawler(ShareCrawler): - """I keep track of how many buckets are being managed by this server. - This is equivalent to the number of distributed files and directories for - which I am providing storage. The actual number of files+directories in - the full grid is probably higher (especially when there are more servers - than 'N', the number of generated shares), because some files+directories - will have shares on other servers instead of me. Also note that the - number of buckets will differ from the number of shares in small grids, - when more than one share is placed on a single server. + """ + I keep track of how many sharesets, each corresponding to a storage index, + are being managed by this server. This is equivalent to the number of + distributed files and directories for which I am providing storage. The + actual number of files and directories in the full grid is probably higher + (especially when there are more servers than 'N', the number of generated + shares), because some files and directories will have shares on other + servers instead of me. Also note that the number of sharesets will differ + from the number of shares in small grids, when more than one share is + placed on a single server. """ minimum_cycle_time = 60*60 # we don't need this more than once an hour hunk ./src/allmydata/storage/crawler.py 466 - def __init__(self, server, statefile, num_sample_prefixes=1): - ShareCrawler.__init__(self, server, statefile) + def __init__(self, backend, statefp, num_sample_prefixes=1, **kwargs): + ShareCrawler.__init__(self, backend, statefp, **kwargs) self.num_sample_prefixes = num_sample_prefixes def add_initial_state(self): hunk ./src/allmydata/storage/crawler.py 480 self.state.setdefault("last-complete-bucket-count", None) self.state.setdefault("storage-index-samples", {}) - def process_prefixdir(self, cycle, prefix, prefixdir, buckets, start_slice): + def process_prefixdir(self, cycle, prefix, sharesets, start_slice): # we override process_prefixdir() because we don't want to look at # the individual sharesets. We'll save state after each one. On my # laptop, a mostly-empty storage server can process about 70 hunk ./src/allmydata/storage/crawler.py 487 # prefixdirs in a 1.0s slice. if cycle not in self.state["bucket-counts"]: self.state["bucket-counts"][cycle] = {} - self.state["bucket-counts"][cycle][prefix] = len(buckets) + self.state["bucket-counts"][cycle][prefix] = len(sharesets) if prefix in self.prefixes[:self.num_sample_prefixes]: hunk ./src/allmydata/storage/crawler.py 489 - self.state["storage-index-samples"][prefix] = (cycle, buckets) + si_strings = [shareset.get_storage_index_string() for shareset in sharesets] + self.state["storage-index-samples"][prefix] = (cycle, si_strings) def finished_cycle(self, cycle): last_counts = self.state["bucket-counts"].get(cycle, []) hunk ./src/allmydata/storage/crawler.py 496 if len(last_counts) == len(self.prefixes): # great, we have a whole cycle. - num_buckets = sum(last_counts.values()) - self.state["last-complete-bucket-count"] = num_buckets + num_sharesets = sum(last_counts.values()) + self.state["last-complete-bucket-count"] = num_sharesets # get rid of old counts for old_cycle in list(self.state["bucket-counts"].keys()): if old_cycle != cycle: hunk ./src/allmydata/storage/crawler.py 504 del self.state["bucket-counts"][old_cycle] # get rid of old samples too for prefix in list(self.state["storage-index-samples"].keys()): - old_cycle,buckets = self.state["storage-index-samples"][prefix] + old_cycle, storage_indices = self.state["storage-index-samples"][prefix] if old_cycle != cycle: del self.state["storage-index-samples"][prefix] hunk ./src/allmydata/storage/expirer.py 1 -import time, os, pickle, struct + +import pickle, struct +from twisted.python import log as twlog + from allmydata.storage.crawler import ShareCrawler hunk ./src/allmydata/storage/expirer.py 6 -from allmydata.storage.shares import get_share_file from allmydata.storage.common import UnknownMutableContainerVersionError, \ UnknownImmutableContainerVersionError hunk ./src/allmydata/storage/expirer.py 8 -from twisted.python import log as twlog + class LeaseCheckingCrawler(ShareCrawler): """I examine the leases on all shares, determining which are still valid hunk ./src/allmydata/storage/expirer.py 53 slow_start = 360 # wait 6 minutes after startup minimum_cycle_time = 12*60*60 # not more than twice per day - def __init__(self, server, statefile, historyfile, - expiration_enabled, mode, - override_lease_duration, # used if expiration_mode=="age" - cutoff_date, # used if expiration_mode=="cutoff-date" - sharetypes): - self.historyfile = historyfile - self.expiration_enabled = expiration_enabled - self.mode = mode + def __init__(self, backend, statefp, historyfp, expiration_policy, clock=None): + # ShareCrawler.__init__ will call add_initial_state, so self.historyfp has to be set first. + self.historyfp = historyfp + ShareCrawler.__init__(self, backend, statefp, clock=clock) + + self.expiration_enabled = expiration_policy['enabled'] + self.mode = expiration_policy['mode'] self.override_lease_duration = None self.cutoff_date = None if self.mode == "age": hunk ./src/allmydata/storage/expirer.py 63 - assert isinstance(override_lease_duration, (int, type(None))) - self.override_lease_duration = override_lease_duration # seconds + assert isinstance(expiration_policy['override_lease_duration'], (int, type(None))) + self.override_lease_duration = expiration_policy['override_lease_duration'] # seconds elif self.mode == "cutoff-date": hunk ./src/allmydata/storage/expirer.py 66 - assert isinstance(cutoff_date, int) # seconds-since-epoch - assert cutoff_date is not None - self.cutoff_date = cutoff_date + assert isinstance(expiration_policy['cutoff_date'], int) # seconds-since-epoch + self.cutoff_date = expiration_policy['cutoff_date'] else: hunk ./src/allmydata/storage/expirer.py 69 - raise ValueError("GC mode '%s' must be 'age' or 'cutoff-date'" % mode) - self.sharetypes_to_expire = sharetypes - ShareCrawler.__init__(self, server, statefile) + raise ValueError("GC mode '%s' must be 'age' or 'cutoff-date'" % expiration_policy['mode']) + self.sharetypes_to_expire = expiration_policy['sharetypes'] def add_initial_state(self): # we fill ["cycle-to-date"] here (even though they will be reset in hunk ./src/allmydata/storage/expirer.py 84 self.state["cycle-to-date"].setdefault(k, so_far[k]) # initialize history - if not os.path.exists(self.historyfile): + if not self.historyfp.exists(): history = {} # cyclenum -> dict hunk ./src/allmydata/storage/expirer.py 86 - f = open(self.historyfile, "wb") - pickle.dump(history, f) - f.close() + pickled = pickle.dumps(history) + self.historyfp.setContent(pickled) def create_empty_cycle_dict(self): recovered = self.create_empty_recovered_dict() hunk ./src/allmydata/storage/expirer.py 100 def create_empty_recovered_dict(self): recovered = {} + # "buckets" is ambiguous; here it means the number of sharesets (one per storage index per server) for a in ("actual", "original", "configured", "examined"): for b in ("buckets", "shares", "sharebytes", "diskbytes"): recovered[a+"-"+b] = 0 hunk ./src/allmydata/storage/expirer.py 111 def started_cycle(self, cycle): self.state["cycle-to-date"] = self.create_empty_cycle_dict() - def stat(self, fn): - return os.stat(fn) + def _process_corrupt_share(self, si, shnum): + twlog.msg("lease-checker error processing share %r:%r" % (si, shnum)) + self.state["cycle-to-date"]["corrupt-shares"].append( (si, shnum) ) + return (1, 1, 1, "unknown") hunk ./src/allmydata/storage/expirer.py 116 - def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32): - bucketdir = os.path.join(prefixdir, storage_index_b32) - s = self.stat(bucketdir) + def process_shareset(self, cycle, prefix, shareset): would_keep_shares = [] wks = None hunk ./src/allmydata/storage/expirer.py 119 + si = shareset.get_storage_index_string() hunk ./src/allmydata/storage/expirer.py 121 - for fn in os.listdir(bucketdir): - try: - shnum = int(fn) - except ValueError: - continue # non-numeric means not a sharefile - sharefile = os.path.join(bucketdir, fn) + (shares, corrupted) = shareset.get_shares_synchronous() + for share in shares: try: hunk ./src/allmydata/storage/expirer.py 124 - wks = self.process_share(sharefile) + wks = self.process_share(share) except (UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError, struct.error): hunk ./src/allmydata/storage/expirer.py 128 - twlog.msg("lease-checker error processing %s" % sharefile) + wks = self._process_corrupt_share(si, share.get_shnum()) twlog.err() hunk ./src/allmydata/storage/expirer.py 130 - which = (storage_index_b32, shnum) - self.state["cycle-to-date"]["corrupt-shares"].append(which) - wks = (1, 1, 1, "unknown") would_keep_shares.append(wks) hunk ./src/allmydata/storage/expirer.py 132 - sharetype = None + for shnum in sorted(corrupted): + wks = self._process_corrupt_share(si, shnum) + would_keep_shares.append(wks) + + shareset_type = None if wks: hunk ./src/allmydata/storage/expirer.py 138 - # use the last share's sharetype as the buckettype - sharetype = wks[3] + # use the last share's type as the shareset type + shareset_type = wks[3] rec = self.state["cycle-to-date"]["space-recovered"] self.increment(rec, "examined-buckets", 1) hunk ./src/allmydata/storage/expirer.py 142 - if sharetype: - self.increment(rec, "examined-buckets-"+sharetype, 1) + if shareset_type: + self.increment(rec, "examined-buckets-"+shareset_type, 1) + + shareset_diskbytes = shareset.get_overhead() hunk ./src/allmydata/storage/expirer.py 147 - try: - bucket_diskbytes = s.st_blocks * 512 - except AttributeError: - bucket_diskbytes = 0 # no stat().st_blocks on windows if sum([wks[0] for wks in would_keep_shares]) == 0: hunk ./src/allmydata/storage/expirer.py 148 - self.increment_bucketspace("original", bucket_diskbytes, sharetype) + self.increment_shareset_space("original", shareset_diskbytes, shareset_type) if sum([wks[1] for wks in would_keep_shares]) == 0: hunk ./src/allmydata/storage/expirer.py 150 - self.increment_bucketspace("configured", bucket_diskbytes, sharetype) + self.increment_shareset_space("configured", shareset_diskbytes, shareset_type) if sum([wks[2] for wks in would_keep_shares]) == 0: hunk ./src/allmydata/storage/expirer.py 152 - self.increment_bucketspace("actual", bucket_diskbytes, sharetype) + self.increment_shareset_space("actual", shareset_diskbytes, shareset_type) hunk ./src/allmydata/storage/expirer.py 154 - def process_share(self, sharefilename): - # first, find out what kind of a share it is - sf = get_share_file(sharefilename) - sharetype = sf.sharetype - now = time.time() - s = self.stat(sharefilename) + def process_share(self, share): + sharetype = share.sharetype + now = self.clock.seconds() + sharebytes = share.get_size() + diskbytes = share.get_used_space() num_leases = 0 num_valid_leases_original = 0 hunk ./src/allmydata/storage/expirer.py 165 num_valid_leases_configured = 0 expired_leases_configured = [] - for li in sf.get_leases(): + for li in share.get_leases(): num_leases += 1 original_expiration_time = li.get_expiration_time() grant_renew_time = li.get_grant_renew_time_time() hunk ./src/allmydata/storage/expirer.py 178 # expired-or-not according to our configured age limit expired = False - if self.mode == "age": - age_limit = original_expiration_time - if self.override_lease_duration is not None: - age_limit = self.override_lease_duration - if age > age_limit: - expired = True - else: - assert self.mode == "cutoff-date" - if grant_renew_time < self.cutoff_date: - expired = True - if sharetype not in self.sharetypes_to_expire: - expired = False + if sharetype in self.sharetypes_to_expire: + if self.mode == "age": + age_limit = original_expiration_time + if self.override_lease_duration is not None: + age_limit = self.override_lease_duration + if age > age_limit: + expired = True + else: + assert self.mode == "cutoff-date" + if grant_renew_time < self.cutoff_date: + expired = True if expired: expired_leases_configured.append(li) hunk ./src/allmydata/storage/expirer.py 197 so_far = self.state["cycle-to-date"] self.increment(so_far["leases-per-share-histogram"], num_leases, 1) - self.increment_space("examined", s, sharetype) + self.increment_space("examined", sharebytes, diskbytes, sharetype) would_keep_share = [1, 1, 1, sharetype] hunk ./src/allmydata/storage/expirer.py 203 if self.expiration_enabled: for li in expired_leases_configured: - sf.cancel_lease(li.cancel_secret) + share.cancel_lease(li.cancel_secret) if num_valid_leases_original == 0: would_keep_share[0] = 0 hunk ./src/allmydata/storage/expirer.py 207 - self.increment_space("original", s, sharetype) + self.increment_space("original", sharebytes, diskbytes, sharetype) if num_valid_leases_configured == 0: would_keep_share[1] = 0 hunk ./src/allmydata/storage/expirer.py 211 - self.increment_space("configured", s, sharetype) + self.increment_space("configured", sharebytes, diskbytes, sharetype) if self.expiration_enabled: would_keep_share[2] = 0 hunk ./src/allmydata/storage/expirer.py 214 - self.increment_space("actual", s, sharetype) + self.increment_space("actual", sharebytes, diskbytes, sharetype) return would_keep_share hunk ./src/allmydata/storage/expirer.py 218 - def increment_space(self, a, s, sharetype): - sharebytes = s.st_size - try: - # note that stat(2) says that st_blocks is 512 bytes, and that - # st_blksize is "optimal file sys I/O ops blocksize", which is - # independent of the block-size that st_blocks uses. - diskbytes = s.st_blocks * 512 - except AttributeError: - # the docs say that st_blocks is only on linux. I also see it on - # MacOS. But it isn't available on windows. - diskbytes = sharebytes + def increment_space(self, a, sharebytes, diskbytes, sharetype): so_far_sr = self.state["cycle-to-date"]["space-recovered"] self.increment(so_far_sr, a+"-shares", 1) self.increment(so_far_sr, a+"-sharebytes", sharebytes) hunk ./src/allmydata/storage/expirer.py 228 self.increment(so_far_sr, a+"-sharebytes-"+sharetype, sharebytes) self.increment(so_far_sr, a+"-diskbytes-"+sharetype, diskbytes) - def increment_bucketspace(self, a, bucket_diskbytes, sharetype): + def increment_shareset_space(self, a, shareset_diskbytes, shareset_type): rec = self.state["cycle-to-date"]["space-recovered"] hunk ./src/allmydata/storage/expirer.py 230 - self.increment(rec, a+"-diskbytes", bucket_diskbytes) + self.increment(rec, a+"-diskbytes", shareset_diskbytes) self.increment(rec, a+"-buckets", 1) hunk ./src/allmydata/storage/expirer.py 232 - if sharetype: - self.increment(rec, a+"-diskbytes-"+sharetype, bucket_diskbytes) - self.increment(rec, a+"-buckets-"+sharetype, 1) + if shareset_type: + self.increment(rec, a+"-diskbytes-"+shareset_type, shareset_diskbytes) + self.increment(rec, a+"-buckets-"+shareset_type, 1) def increment(self, d, k, delta=1): if k not in d: hunk ./src/allmydata/storage/expirer.py 264 h = {} start = self.state["current-cycle-start-time"] - now = time.time() + now = self.clock.seconds() h["cycle-start-finish-times"] = (start, now) h["expiration-enabled"] = self.expiration_enabled h["configured-expiration-mode"] = (self.mode, hunk ./src/allmydata/storage/expirer.py 288 # copy() needs to become a deepcopy h["space-recovered"] = s["space-recovered"].copy() - history = pickle.load(open(self.historyfile, "rb")) + pickled = self.historyfp.getContent() + history = pickle.loads(pickled) history[cycle] = h while len(history) > 10: oldcycles = sorted(history.keys()) hunk ./src/allmydata/storage/expirer.py 294 del history[oldcycles[0]] - f = open(self.historyfile, "wb") - pickle.dump(history, f) - f.close() + repickled = pickle.dumps(history) + self.historyfp.setContent(repickled) def get_state(self): """In addition to the crawler state described in hunk ./src/allmydata/storage/expirer.py 364 progress = self.get_progress() state = ShareCrawler.get_state(self) # does a shallow copy - history = pickle.load(open(self.historyfile, "rb")) + pickled = self.historyfp.getContent() + history = pickle.loads(pickled) state["history"] = history if not progress["cycle-in-progress"]: hunk ./src/allmydata/storage/server.py 1 -import os, re, weakref, struct, time +import weakref from foolscap.api import Referenceable from twisted.application import service hunk ./src/allmydata/storage/server.py 5 +from twisted.internet import defer, reactor from zope.interface import implements hunk ./src/allmydata/storage/server.py 8 -from allmydata.interfaces import RIStorageServer, IStatsProducer -from allmydata.util import fileutil, idlib, log, time_format +from allmydata.interfaces import RIStorageServer, IStatsProducer, IStorageBackend +from allmydata.util.assertutil import precondition +from allmydata.util import idlib, log, fileutil import allmydata # for __full_version__ hunk ./src/allmydata/storage/server.py 13 -from allmydata.storage.common import si_b2a, si_a2b, storage_index_to_dir -_pyflakes_hush = [si_b2a, si_a2b, storage_index_to_dir] # re-exported +from allmydata.storage.common import si_a2b, si_b2a +[si_a2b] # hush pyflakes from allmydata.storage.lease import LeaseInfo hunk ./src/allmydata/storage/server.py 16 -from allmydata.storage.mutable import MutableShareFile, EmptyShare, \ - create_mutable_sharefile -from allmydata.storage.immutable import ShareFile, BucketWriter, BucketReader -from allmydata.storage.crawler import BucketCountingCrawler from allmydata.storage.expirer import LeaseCheckingCrawler hunk ./src/allmydata/storage/server.py 17 - -# storage/ -# storage/shares/incoming -# incoming/ holds temp dirs named $START/$STORAGEINDEX/$SHARENUM which will -# be moved to storage/shares/$START/$STORAGEINDEX/$SHARENUM upon success -# storage/shares/$START/$STORAGEINDEX -# storage/shares/$START/$STORAGEINDEX/$SHARENUM - -# Where "$START" denotes the first 10 bits worth of $STORAGEINDEX (that's 2 -# base-32 chars). - -# $SHARENUM matches this regex: -NUM_RE=re.compile("^[0-9]+$") - +from allmydata.storage.crawler import BucketCountingCrawler class StorageServer(service.MultiService, Referenceable): hunk ./src/allmydata/storage/server.py 25 name = 'storage' LeaseCheckerClass = LeaseCheckingCrawler + BucketCounterClass = BucketCountingCrawler + DEFAULT_EXPIRATION_POLICY = { + 'enabled': False, + 'mode': 'age', + 'override_lease_duration': None, + 'cutoff_date': None, + 'sharetypes': ('mutable', 'immutable'), + } hunk ./src/allmydata/storage/server.py 34 - def __init__(self, storedir, nodeid, reserved_space=0, - discard_storage=False, readonly_storage=False, + def __init__(self, serverid, backend, statedir, stats_provider=None, hunk ./src/allmydata/storage/server.py 36 - expiration_enabled=False, - expiration_mode="age", - expiration_override_lease_duration=None, - expiration_cutoff_date=None, - expiration_sharetypes=("mutable", "immutable")): + expiration_policy=None, + clock=None): service.MultiService.__init__(self) hunk ./src/allmydata/storage/server.py 39 - assert isinstance(nodeid, str) - assert len(nodeid) == 20 - self.my_nodeid = nodeid - self.storedir = storedir - sharedir = os.path.join(storedir, "shares") - fileutil.make_dirs(sharedir) - self.sharedir = sharedir - # we don't actually create the corruption-advisory dir until necessary - self.corruption_advisory_dir = os.path.join(storedir, - "corruption-advisories") - self.reserved_space = int(reserved_space) - self.no_storage = discard_storage - self.readonly_storage = readonly_storage + precondition(IStorageBackend.providedBy(backend), backend) + precondition(isinstance(serverid, str), serverid) + precondition(len(serverid) == 20, serverid) + + self._serverid = serverid + self.clock = clock or reactor self.stats_provider = stats_provider if self.stats_provider: self.stats_provider.register_producer(self) hunk ./src/allmydata/storage/server.py 48 - self.incomingdir = os.path.join(sharedir, 'incoming') - self._clean_incomplete() - fileutil.make_dirs(self.incomingdir) self._active_writers = weakref.WeakKeyDictionary() hunk ./src/allmydata/storage/server.py 49 + self.backend = backend + self.backend.setServiceParent(self) + self._statedir = statedir + fileutil.fp_make_dirs(self._statedir) log.msg("StorageServer created", facility="tahoe.storage") hunk ./src/allmydata/storage/server.py 55 - if reserved_space: - if self.get_available_space() is None: - log.msg("warning: [storage]reserved_space= is set, but this platform does not support an API to get disk statistics (statvfs(2) or GetDiskFreeSpaceEx), so this reservation cannot be honored", - umin="0wZ27w", level=log.UNUSUAL) - self.latencies = {"allocate": [], # immutable "write": [], "close": [], hunk ./src/allmydata/storage/server.py 66 "renew": [], "cancel": [], } - self.add_bucket_counter() hunk ./src/allmydata/storage/server.py 67 - statefile = os.path.join(self.storedir, "lease_checker.state") - historyfile = os.path.join(self.storedir, "lease_checker.history") - klass = self.LeaseCheckerClass - self.lease_checker = klass(self, statefile, historyfile, - expiration_enabled, expiration_mode, - expiration_override_lease_duration, - expiration_cutoff_date, - expiration_sharetypes) - self.lease_checker.setServiceParent(self) + self.bucket_counter = None + self.lease_checker = None + if backend.supports_crawlers(): + self._setup_bucket_counter() + self._setup_lease_checker(expiration_policy or self.DEFAULT_EXPIRATION_POLICY) def __repr__(self): hunk ./src/allmydata/storage/server.py 74 - return "" % (idlib.shortnodeid_b2a(self.my_nodeid),) + return "" % (idlib.shortnodeid_b2a(self._serverid),) hunk ./src/allmydata/storage/server.py 76 - def have_shares(self): - # quick test to decide if we need to commit to an implicit - # permutation-seed or if we should use a new one - return bool(set(os.listdir(self.sharedir)) - set(["incoming"])) - - def add_bucket_counter(self): - statefile = os.path.join(self.storedir, "bucket_counter.state") - self.bucket_counter = BucketCountingCrawler(self, statefile) + def _setup_bucket_counter(self): + statefp = self._statedir.child("bucket_counter.state") + self.bucket_counter = self.BucketCounterClass(self.backend, statefp, + clock=self.clock) self.bucket_counter.setServiceParent(self) hunk ./src/allmydata/storage/server.py 82 + def _setup_lease_checker(self, expiration_policy): + statefp = self._statedir.child("lease_checker.state") + historyfp = self._statedir.child("lease_checker.history") + self.lease_checker = self.LeaseCheckerClass(self.backend, statefp, historyfp, expiration_policy, + clock=self.clock) + self.lease_checker.setServiceParent(self) + def count(self, name, delta=1): if self.stats_provider: self.stats_provider.count("storage_server." + name, delta) hunk ./src/allmydata/storage/server.py 103 """Return a dict, indexed by category, that contains a dict of latency numbers for each category. If there are sufficient samples for unambiguous interpretation, each dict will contain the - following keys: mean, 01_0_percentile, 10_0_percentile, + following keys: samplesize, mean, 01_0_percentile, 10_0_percentile, 50_0_percentile (median), 90_0_percentile, 95_0_percentile, 99_0_percentile, 99_9_percentile. If there are insufficient samples for a given percentile to be interpreted unambiguously hunk ./src/allmydata/storage/server.py 125 else: stats["mean"] = None - orderstatlist = [(0.01, "01_0_percentile", 100), (0.1, "10_0_percentile", 10),\ - (0.50, "50_0_percentile", 10), (0.90, "90_0_percentile", 10),\ - (0.95, "95_0_percentile", 20), (0.99, "99_0_percentile", 100),\ + orderstatlist = [(0.1, "10_0_percentile", 10), (0.5, "50_0_percentile", 10), \ + (0.9, "90_0_percentile", 10), (0.95, "95_0_percentile", 20), \ + (0.01, "01_0_percentile", 100), (0.99, "99_0_percentile", 100),\ (0.999, "99_9_percentile", 1000)] for percentile, percentilestring, minnumtoobserve in orderstatlist: hunk ./src/allmydata/storage/server.py 144 kwargs["facility"] = "tahoe.storage" return log.msg(*args, **kwargs) - def _clean_incomplete(self): - fileutil.rm_dir(self.incomingdir) + def get_serverid(self): + return self._serverid def get_stats(self): # remember: RIStatsProvider requires that our return dict hunk ./src/allmydata/storage/server.py 149 - # contains numeric values. + # contains numeric, or None values. stats = { 'storage_server.allocated': self.allocated_size(), } hunk ./src/allmydata/storage/server.py 151 - stats['storage_server.reserved_space'] = self.reserved_space for category,ld in self.get_latencies().items(): for name,v in ld.items(): stats['storage_server.latencies.%s.%s' % (category, name)] = v hunk ./src/allmydata/storage/server.py 155 - try: - disk = fileutil.get_disk_stats(self.sharedir, self.reserved_space) - writeable = disk['avail'] > 0 - - # spacetime predictors should use disk_avail / (d(disk_used)/dt) - stats['storage_server.disk_total'] = disk['total'] - stats['storage_server.disk_used'] = disk['used'] - stats['storage_server.disk_free_for_root'] = disk['free_for_root'] - stats['storage_server.disk_free_for_nonroot'] = disk['free_for_nonroot'] - stats['storage_server.disk_avail'] = disk['avail'] - except AttributeError: - writeable = True - except EnvironmentError: - log.msg("OS call to get disk statistics failed", level=log.UNUSUAL) - writeable = False + self.backend.fill_in_space_stats(stats) hunk ./src/allmydata/storage/server.py 157 - if self.readonly_storage: - stats['storage_server.disk_avail'] = 0 - writeable = False - - stats['storage_server.accepting_immutable_shares'] = int(writeable) - s = self.bucket_counter.get_state() - bucket_count = s.get("last-complete-bucket-count") - if bucket_count: - stats['storage_server.total_bucket_count'] = bucket_count + if self.bucket_counter: + s = self.bucket_counter.get_state() + bucket_count = s.get("last-complete-bucket-count") + if bucket_count: + stats['storage_server.total_bucket_count'] = bucket_count return stats def get_available_space(self): hunk ./src/allmydata/storage/server.py 165 - """Returns available space for share storage in bytes, or None if no - API to get this information is available.""" - - if self.readonly_storage: - return 0 - return fileutil.get_available_space(self.sharedir, self.reserved_space) + return self.backend.get_available_space() def allocated_size(self): space = 0 hunk ./src/allmydata/storage/server.py 174 return space def remote_get_version(self): - remaining_space = self.get_available_space() + remaining_space = self.backend.get_available_space() if remaining_space is None: # We're on a platform that has no API to get disk stats. remaining_space = 2**64 hunk ./src/allmydata/storage/server.py 185 "delete-mutable-shares-with-zero-length-writev": True, "fills-holes-with-zero-bytes": True, "prevents-read-past-end-of-share-data": True, + "has-immutable-readv": True, }, "application-version": str(allmydata.__full_version__), } hunk ./src/allmydata/storage/server.py 191 return version - def remote_allocate_buckets(self, storage_index, + def _add_latency(self, res, name, start): + self.add_latency(name, self.clock.seconds() - start) + return res + + def remote_allocate_buckets(self, storageindex, renew_secret, cancel_secret, sharenums, allocated_size, canary, owner_num=0): hunk ./src/allmydata/storage/server.py 200 # owner_num is not for clients to set, but rather it should be - # curried into the PersonalStorageServer instance that is dedicated - # to a particular owner. - start = time.time() + # curried into a StorageServer instance dedicated to a particular + # owner. + start = self.clock.seconds() self.count("allocate") hunk ./src/allmydata/storage/server.py 204 - alreadygot = set() bucketwriters = {} # k: shnum, v: BucketWriter hunk ./src/allmydata/storage/server.py 205 - si_dir = storage_index_to_dir(storage_index) - si_s = si_b2a(storage_index) hunk ./src/allmydata/storage/server.py 206 + si_s = si_b2a(storageindex) log.msg("storage: allocate_buckets %s" % si_s) hunk ./src/allmydata/storage/server.py 209 - # in this implementation, the lease information (including secrets) - # goes into the share files themselves. It could also be put into a - # separate database. Note that the lease should not be added until - # the BucketWriter has been closed. - expire_time = time.time() + 31*24*60*60 - lease_info = LeaseInfo(owner_num, - renew_secret, cancel_secret, - expire_time, self.my_nodeid) + # Note that the lease should not be added until the BucketWriter + # has been closed. + expire_time = start + 31*24*60*60 + lease_info = LeaseInfo(owner_num, renew_secret, cancel_secret, + expire_time, self._serverid) max_space_per_bucket = allocated_size hunk ./src/allmydata/storage/server.py 217 - remaining_space = self.get_available_space() + remaining_space = self.backend.get_available_space() limited = remaining_space is not None if limited: # This is a bit conservative, since some of this allocated_size() hunk ./src/allmydata/storage/server.py 224 # has already been written to the backend, where it will show up in # get_available_space. remaining_space -= self.allocated_size() - # self.readonly_storage causes remaining_space <= 0 + # If the backend is read-only, remaining_space will be <= 0. + + shareset = self.backend.get_shareset(storageindex) # Fill alreadygot with all shares that we have, not just the ones # they asked about: this will save them a lot of work. Add or update hunk ./src/allmydata/storage/server.py 231 # leases for all of them: if they want us to hold shares for this - # file, they'll want us to hold leases for this file. - for (shnum, fn) in self._get_bucket_shares(storage_index): - alreadygot.add(shnum) - sf = ShareFile(fn) - sf.add_or_renew_lease(lease_info) + # file, they'll want us to hold leases for all the shares of it. + # + # XXX should we be making the assumption here that lease info is + # duplicated in all shares? + alreadygot = set() + d = shareset.get_shares() + def _got_shares( (shares, corrupted) ): + remaining = remaining_space + for share in shares: + share.add_or_renew_lease(lease_info) + alreadygot.add(share.get_shnum()) hunk ./src/allmydata/storage/server.py 243 - for shnum in sharenums: - incominghome = os.path.join(self.incomingdir, si_dir, "%d" % shnum) - finalhome = os.path.join(self.sharedir, si_dir, "%d" % shnum) - if os.path.exists(finalhome): - # great! we already have it. easy. - pass - elif os.path.exists(incominghome): - # Note that we don't create BucketWriters for shnums that - # have a partial share (in incoming/), so if a second upload - # occurs while the first is still in progress, the second - # uploader will use different storage servers. - pass - elif (not limited) or (remaining_space >= max_space_per_bucket): - # ok! we need to create the new share file. - bw = BucketWriter(self, incominghome, finalhome, - max_space_per_bucket, lease_info, canary) - if self.no_storage: - bw.throw_out_all_data = True - bucketwriters[shnum] = bw - self._active_writers[bw] = 1 - if limited: - remaining_space -= max_space_per_bucket - else: - # bummer! not enough space to accept this bucket - pass + d2 = defer.succeed(None) hunk ./src/allmydata/storage/server.py 245 - if bucketwriters: - fileutil.make_dirs(os.path.join(self.sharedir, si_dir)) + # We only want BucketWriters for the shares we're going to write. hunk ./src/allmydata/storage/server.py 247 - self.add_latency("allocate", time.time() - start) - return alreadygot, bucketwriters + # Currently we don't create BucketWriters for shnums where we have a + # share that is corrupted. Is that right, or should we allow the corrupted + # share to be clobbered? Note that currently the disk share classes + # have assertions that prevent them from clobbering existing files. + for shnum in set(sharenums) - alreadygot - corrupted: + if shareset.has_incoming(shnum): + # Note that we don't create BucketWriters for shnums that + # have an incoming share, so if a second upload occurs while + # the first is still in progress, the second uploader will + # use different storage servers. + pass + elif (not limited) or (remaining >= max_space_per_bucket): + if limited: + remaining -= max_space_per_bucket hunk ./src/allmydata/storage/server.py 262 - def _iter_share_files(self, storage_index): - for shnum, filename in self._get_bucket_shares(storage_index): - f = open(filename, 'rb') - header = f.read(32) - f.close() - if header[:32] == MutableShareFile.MAGIC: - sf = MutableShareFile(filename, self) - # note: if the share has been migrated, the renew_lease() - # call will throw an exception, with information to help the - # client update the lease. - elif header[:4] == struct.pack(">L", 1): - sf = ShareFile(filename) - else: - continue # non-sharefile - yield sf + d2.addCallback(lambda ign, shnum=shnum: + shareset.make_bucket_writer(self, shnum, max_space_per_bucket, + lease_info, canary)) + def _record_writer(bw, shnum=shnum): + bucketwriters[shnum] = bw + self._active_writers[bw] = 1 + d2.addCallback(_record_writer) + else: + # Bummer not enough space to accept this share. + pass + + d2.addCallback(lambda ign: (alreadygot, bucketwriters)) + return d2 + d.addCallback(_got_shares) + d.addBoth(self._add_latency, "allocate", start) + return d hunk ./src/allmydata/storage/server.py 279 - def remote_add_lease(self, storage_index, renew_secret, cancel_secret, + def remote_add_lease(self, storageindex, renew_secret, cancel_secret, owner_num=1): hunk ./src/allmydata/storage/server.py 281 - start = time.time() + start = self.clock.seconds() self.count("add-lease") hunk ./src/allmydata/storage/server.py 283 - new_expire_time = time.time() + 31*24*60*60 - lease_info = LeaseInfo(owner_num, - renew_secret, cancel_secret, - new_expire_time, self.my_nodeid) - for sf in self._iter_share_files(storage_index): - sf.add_or_renew_lease(lease_info) - self.add_latency("add-lease", time.time() - start) - return None + new_expire_time = start + 31*24*60*60 + lease_info = LeaseInfo(owner_num, renew_secret, cancel_secret, + new_expire_time, self._serverid) + + try: + shareset = self.backend.get_shareset(storageindex) + shareset.add_or_renew_lease(lease_info) + finally: + self.add_latency("add-lease", self.clock.seconds() - start) hunk ./src/allmydata/storage/server.py 293 - def remote_renew_lease(self, storage_index, renew_secret): - start = time.time() + def remote_renew_lease(self, storageindex, renew_secret): + start = self.clock.seconds() self.count("renew") hunk ./src/allmydata/storage/server.py 296 - new_expire_time = time.time() + 31*24*60*60 - found_buckets = False - for sf in self._iter_share_files(storage_index): - found_buckets = True - sf.renew_lease(renew_secret, new_expire_time) - self.add_latency("renew", time.time() - start) - if not found_buckets: - raise IndexError("no such lease to renew") + + try: + shareset = self.backend.get_shareset(storageindex) + new_expiration_time = start + 31*24*60*60 # one month from now + shareset.renew_lease(renew_secret, new_expiration_time) + finally: + self.add_latency("renew", self.clock.seconds() - start) def bucket_writer_closed(self, bw, consumed_size): if self.stats_provider: hunk ./src/allmydata/storage/server.py 309 self.stats_provider.count('storage_server.bytes_added', consumed_size) del self._active_writers[bw] - def _get_bucket_shares(self, storage_index): - """Return a list of (shnum, pathname) tuples for files that hold - shares for this storage_index. In each tuple, 'shnum' will always be - the integer form of the last component of 'pathname'.""" - storagedir = os.path.join(self.sharedir, storage_index_to_dir(storage_index)) - try: - for f in os.listdir(storagedir): - if NUM_RE.match(f): - filename = os.path.join(storagedir, f) - yield (int(f), filename) - except OSError: - # Commonly caused by there being no buckets at all. - pass - - def remote_get_buckets(self, storage_index): - start = time.time() + def remote_get_buckets(self, storageindex): + start = self.clock.seconds() self.count("get") hunk ./src/allmydata/storage/server.py 312 - si_s = si_b2a(storage_index) + si_s = si_b2a(storageindex) log.msg("storage: get_buckets %s" % si_s) bucketreaders = {} # k: sharenum, v: BucketReader hunk ./src/allmydata/storage/server.py 315 - for shnum, filename in self._get_bucket_shares(storage_index): - bucketreaders[shnum] = BucketReader(self, filename, - storage_index, shnum) - self.add_latency("get", time.time() - start) - return bucketreaders hunk ./src/allmydata/storage/server.py 316 - def get_leases(self, storage_index): - """Provide an iterator that yields all of the leases attached to this - bucket. Each lease is returned as a LeaseInfo instance. + shareset = self.backend.get_shareset(storageindex) + d = shareset.get_shares() + def _make_readers( (shares, corrupted) ): + # We don't create BucketReaders for corrupted shares. + for share in shares: + assert not isinstance(share, defer.Deferred), share + bucketreaders[share.get_shnum()] = shareset.make_bucket_reader(self, share) + return bucketreaders + d.addCallback(_make_readers) + d.addBoth(self._add_latency, "get", start) + return d hunk ./src/allmydata/storage/server.py 328 - This method is not for client use. + def get_leases(self, storageindex): """ hunk ./src/allmydata/storage/server.py 330 + Provide an iterator that yields all of the leases attached to this + bucket. Each lease is returned as a LeaseInfo instance. hunk ./src/allmydata/storage/server.py 333 - # since all shares get the same lease data, we just grab the leases - # from the first share - try: - shnum, filename = self._get_bucket_shares(storage_index).next() - sf = ShareFile(filename) - return sf.get_leases() - except StopIteration: - return iter([]) + This method is not for client use. XXX do we need it at all? + For the time being this is synchronous. + """ + return self.backend.get_shareset(storageindex).get_leases() hunk ./src/allmydata/storage/server.py 338 - def remote_slot_testv_and_readv_and_writev(self, storage_index, + def remote_slot_testv_and_readv_and_writev(self, storageindex, secrets, test_and_write_vectors, read_vector): hunk ./src/allmydata/storage/server.py 342 - start = time.time() + start = self.clock.seconds() self.count("writev") hunk ./src/allmydata/storage/server.py 344 - si_s = si_b2a(storage_index) + si_s = si_b2a(storageindex) log.msg("storage: slot_writev %s" % si_s) hunk ./src/allmydata/storage/server.py 346 - si_dir = storage_index_to_dir(storage_index) - (write_enabler, renew_secret, cancel_secret) = secrets - # shares exist if there is a file for them - bucketdir = os.path.join(self.sharedir, si_dir) - shares = {} - if os.path.isdir(bucketdir): - for sharenum_s in os.listdir(bucketdir): - try: - sharenum = int(sharenum_s) - except ValueError: - continue - filename = os.path.join(bucketdir, sharenum_s) - msf = MutableShareFile(filename, self) - msf.check_write_enabler(write_enabler, si_s) - shares[sharenum] = msf - # write_enabler is good for all existing shares. - - # Now evaluate test vectors. - testv_is_good = True - for sharenum in test_and_write_vectors: - (testv, datav, new_length) = test_and_write_vectors[sharenum] - if sharenum in shares: - if not shares[sharenum].check_testv(testv): - self.log("testv failed: [%d]: %r" % (sharenum, testv)) - testv_is_good = False - break - else: - # compare the vectors against an empty share, in which all - # reads return empty strings. - if not EmptyShare().check_testv(testv): - self.log("testv failed (empty): [%d] %r" % (sharenum, - testv)) - testv_is_good = False - break hunk ./src/allmydata/storage/server.py 347 - # now gather the read vectors, before we do any writes - read_data = {} - for sharenum, share in shares.items(): - read_data[sharenum] = share.readv(read_vector) - - ownerid = 1 # TODO - expire_time = time.time() + 31*24*60*60 # one month - lease_info = LeaseInfo(ownerid, - renew_secret, cancel_secret, - expire_time, self.my_nodeid) - - if testv_is_good: - # now apply the write vectors - for sharenum in test_and_write_vectors: - (testv, datav, new_length) = test_and_write_vectors[sharenum] - if new_length == 0: - if sharenum in shares: - shares[sharenum].unlink() - else: - if sharenum not in shares: - # allocate a new share - allocated_size = 2000 # arbitrary, really - share = self._allocate_slot_share(bucketdir, secrets, - sharenum, - allocated_size, - owner_num=0) - shares[sharenum] = share - shares[sharenum].writev(datav, new_length) - # and update the lease - shares[sharenum].add_or_renew_lease(lease_info) + shareset = self.backend.get_shareset(storageindex) + expiration_time = start + 31*24*60*60 # one month from now hunk ./src/allmydata/storage/server.py 350 - if new_length == 0: - # delete empty bucket directories - if not os.listdir(bucketdir): - os.rmdir(bucketdir) + d = shareset.testv_and_readv_and_writev(self, secrets, test_and_write_vectors, + read_vector, expiration_time) + d.addBoth(self._add_latency, "writev", start) + return d hunk ./src/allmydata/storage/server.py 355 - - # all done - self.add_latency("writev", time.time() - start) - return (testv_is_good, read_data) - - def _allocate_slot_share(self, bucketdir, secrets, sharenum, - allocated_size, owner_num=0): - (write_enabler, renew_secret, cancel_secret) = secrets - my_nodeid = self.my_nodeid - fileutil.make_dirs(bucketdir) - filename = os.path.join(bucketdir, "%d" % sharenum) - share = create_mutable_sharefile(filename, my_nodeid, write_enabler, - self) - return share - - def remote_slot_readv(self, storage_index, shares, readv): - start = time.time() + def remote_slot_readv(self, storageindex, shares, readv): + start = self.clock.seconds() self.count("readv") hunk ./src/allmydata/storage/server.py 358 - si_s = si_b2a(storage_index) - lp = log.msg("storage: slot_readv %s %s" % (si_s, shares), - facility="tahoe.storage", level=log.OPERATIONAL) - si_dir = storage_index_to_dir(storage_index) - # shares exist if there is a file for them - bucketdir = os.path.join(self.sharedir, si_dir) - if not os.path.isdir(bucketdir): - self.add_latency("readv", time.time() - start) - return {} - datavs = {} - for sharenum_s in os.listdir(bucketdir): - try: - sharenum = int(sharenum_s) - except ValueError: - continue - if sharenum in shares or not shares: - filename = os.path.join(bucketdir, sharenum_s) - msf = MutableShareFile(filename, self) - datavs[sharenum] = msf.readv(readv) - log.msg("returning shares %s" % (datavs.keys(),), - facility="tahoe.storage", level=log.NOISY, parent=lp) - self.add_latency("readv", time.time() - start) - return datavs + si_s = si_b2a(storageindex) + log.msg("storage: slot_readv %s %s" % (si_s, shares), + facility="tahoe.storage", level=log.OPERATIONAL) + + shareset = self.backend.get_shareset(storageindex) + d = shareset.readv(shares, readv) + d.addBoth(self._add_latency, "readv", start) + return d hunk ./src/allmydata/storage/server.py 367 - def remote_advise_corrupt_share(self, share_type, storage_index, shnum, - reason): - fileutil.make_dirs(self.corruption_advisory_dir) - now = time_format.iso_utc(sep="T") - si_s = si_b2a(storage_index) - # windows can't handle colons in the filename - fn = os.path.join(self.corruption_advisory_dir, - "%s--%s-%d" % (now, si_s, shnum)).replace(":","") - f = open(fn, "w") - f.write("report: Share Corruption\n") - f.write("type: %s\n" % share_type) - f.write("storage_index: %s\n" % si_s) - f.write("share_number: %d\n" % shnum) - f.write("\n") - f.write(reason) - f.write("\n") - f.close() - log.msg(format=("client claims corruption in (%(share_type)s) " + - "%(si)s-%(shnum)d: %(reason)s"), - share_type=share_type, si=si_s, shnum=shnum, reason=reason, - level=log.SCARY, umid="SGx2fA") - return None + def remote_advise_corrupt_share(self, share_type, storage_index, shnum, reason): + self.backend.advise_corrupt_share(share_type, storage_index, shnum, reason) hunk ./src/allmydata/test/common.py 19 DeepCheckResults, DeepCheckAndRepairResults from allmydata.mutable.layout import unpack_header from allmydata.mutable.publish import MutableData -from allmydata.storage.mutable import MutableShareFile +from allmydata.storage.backends.disk.mutable import MutableDiskShare from allmydata.util import hashutil, log, fileutil, pollmixin from allmydata.util.assertutil import precondition from allmydata.util.consumer import download_to_data hunk ./src/allmydata/test/common.py 481 d.addBoth(flush_but_dont_ignore) return d + def workdir(self, name): + return os.path.join("system", self.__class__.__name__, name) + def getdir(self, subdir): return os.path.join(self.basedir, subdir) hunk ./src/allmydata/test/common.py 596 config += "web.port = tcp:0:interface=127.0.0.1\n" config += "timeout.disconnect = 1800\n" - fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config) + # give subclasses a chance to append lines to the nodes' tahoe.cfg files. + config += self._get_extra_config(i) hunk ./src/allmydata/test/common.py 599 - # give subclasses a chance to append lines to the node's tahoe.cfg - # files before they are launched. - self._set_up_nodes_extra_config() + fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config) # start clients[0], wait for it's tub to be ready (at which point it # will have registered the helper furl). hunk ./src/allmydata/test/common.py 637 d.addCallback(_connected) return d - def _set_up_nodes_extra_config(self): + def _get_extra_config(self, i): # for overriding by subclasses hunk ./src/allmydata/test/common.py 639 - pass + return "" def _grab_stats(self, res): d = self.stats_gatherer.poll() hunk ./src/allmydata/test/common.py 1299 def _corrupt_mutable_share_data(data, debug=False): prefix = data[:32] - assert prefix == MutableShareFile.MAGIC, "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableShareFile.MAGIC) - data_offset = MutableShareFile.DATA_OFFSET + assert prefix == MutableDiskShare.MAGIC, "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableDiskShare.MAGIC) + data_offset = MutableDiskShare.DATA_OFFSET sharetype = data[data_offset:data_offset+1] assert sharetype == "\x00", "non-SDMF mutable shares not supported" (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize, hunk ./src/allmydata/test/no_network.py 21 from twisted.application import service from twisted.internet import defer, reactor from twisted.python.failure import Failure +from twisted.python.filepath import FilePath from foolscap.api import Referenceable, fireEventually, RemoteException from base64 import b32encode hunk ./src/allmydata/test/no_network.py 27 from allmydata import uri as tahoe_uri from allmydata.client import Client -from allmydata.storage.server import StorageServer, storage_index_to_dir +from allmydata.storage.server import StorageServer +from allmydata.storage.backends.disk.disk_backend import DiskBackend from allmydata.util import fileutil, idlib, hashutil from allmydata.util.hashutil import sha1 from allmydata.test.common_web import HTTPClientGETFactory hunk ./src/allmydata/test/no_network.py 36 from allmydata.test.common import TEST_RSA_KEY_SIZE +PRINT_TRACEBACKS = False + class IntentionalError(Exception): pass hunk ./src/allmydata/test/no_network.py 87 return d2 return _really_call() + if PRINT_TRACEBACKS: + import traceback + tb = traceback.extract_stack() d = fireEventually() d.addCallback(lambda res: _call()) def _wrap_exception(f): hunk ./src/allmydata/test/no_network.py 93 + if PRINT_TRACEBACKS and not f.check(NameError): + print ">>>" + ">>>".join(traceback.format_list(tb)) + print "+++ %s%r %r: %s" % (methname, args, kwargs, f) + #f.printDetailedTraceback() return Failure(RemoteException(f)) d.addErrback(_wrap_exception) def _return_membrane(res): hunk ./src/allmydata/test/no_network.py 105 # objects that cross the simulated wire and replace them with # wrappers), we special-case certain methods that we happen to # know will return Referenceables. + # The outer return value of such a method may be Deferred, but + # its components must not be. if methname == "allocate_buckets": (alreadygot, allocated) = res for shnum in allocated: hunk ./src/allmydata/test/no_network.py 110 + assert not isinstance(allocated[shnum], defer.Deferred), (methname, allocated) allocated[shnum] = LocalWrapper(allocated[shnum]) if methname == "get_buckets": for shnum in res: hunk ./src/allmydata/test/no_network.py 114 + assert not isinstance(res[shnum], defer.Deferred), (methname, res) res[shnum] = LocalWrapper(res[shnum]) return res d.addCallback(_return_membrane) hunk ./src/allmydata/test/no_network.py 178 def get_nickname_for_serverid(self, serverid): return None + def get_known_servers(self): + return self.get_connected_servers() + + def get_all_serverids(self): + return self.client.get_all_serverids() + + class NoNetworkClient(Client): def create_tub(self): pass hunk ./src/allmydata/test/no_network.py 278 def make_server(self, i, readonly=False): serverid = hashutil.tagged_hash("serverid", str(i))[:20] - serverdir = os.path.join(self.basedir, "servers", - idlib.shortnodeid_b2a(serverid), "storage") - fileutil.make_dirs(serverdir) - ss = StorageServer(serverdir, serverid, stats_provider=SimpleStats(), - readonly_storage=readonly) + storagedir = FilePath(self.basedir).child("servers").child(idlib.shortnodeid_b2a(serverid)).child("storage") + + # The backend will make the storage directory and any necessary parents. + backend = DiskBackend(storagedir, readonly=readonly) + ss = StorageServer(serverid, backend, storagedir, stats_provider=SimpleStats()) ss._no_network_server_number = i return ss hunk ./src/allmydata/test/no_network.py 292 middleman = service.MultiService() middleman.setServiceParent(self) ss.setServiceParent(middleman) - serverid = ss.my_nodeid + serverid = ss.get_serverid() self.servers_by_number[i] = ss wrapper = wrap_storage_server(ss) self.wrappers_by_id[serverid] = wrapper hunk ./src/allmydata/test/no_network.py 311 # it's enough to remove the server from c._servers (we don't actually # have to detach and stopService it) for i,ss in self.servers_by_number.items(): - if ss.my_nodeid == serverid: + if ss.get_serverid() == serverid: del self.servers_by_number[i] break del self.wrappers_by_id[serverid] hunk ./src/allmydata/test/no_network.py 361 def get_clientdir(self, i=0): return self.g.clients[i].basedir + def get_server(self, i): + return self.g.servers_by_number[i] + def get_serverdir(self, i): hunk ./src/allmydata/test/no_network.py 365 - return self.g.servers_by_number[i].storedir + return self.g.servers_by_number[i].backend._storedir + + def remove_server(self, i): + self.g.remove_server(self.g.servers_by_number[i].get_serverid()) def iterate_servers(self): for i in sorted(self.g.servers_by_number.keys()): hunk ./src/allmydata/test/no_network.py 373 ss = self.g.servers_by_number[i] - yield (i, ss, ss.storedir) + yield (i, ss, ss.backend._storedir) def find_uri_shares(self, uri): si = tahoe_uri.from_string(uri).get_storage_index() hunk ./src/allmydata/test/no_network.py 377 - prefixdir = storage_index_to_dir(si) - shares = [] - for i,ss in self.g.servers_by_number.items(): - serverid = ss.my_nodeid - basedir = os.path.join(ss.sharedir, prefixdir) - if not os.path.exists(basedir): - continue - for f in os.listdir(basedir): - try: - shnum = int(f) - shares.append((shnum, serverid, os.path.join(basedir, f))) - except ValueError: - pass - return sorted(shares) + sharelist = [] + d = defer.succeed(None) + for i, ss in self.g.servers_by_number.items(): + d.addCallback(lambda ign, ss=ss: ss.backend.get_shareset(si).get_shares()) + def _append_shares( (shares_for_server, corrupted), ss=ss): + assert len(corrupted) == 0, (shares_for_server, corrupted) + for share in shares_for_server: + assert not isinstance(share, defer.Deferred), share + sharelist.append( (share.get_shnum(), ss.get_serverid(), share._get_filepath()) ) + d.addCallback(_append_shares) + + d.addCallback(lambda ign: sorted(sharelist)) + return d + + def count_leases(self, uri): + """Return (filename, leasecount) pairs in arbitrary order.""" + si = tahoe_uri.from_string(uri).get_storage_index() + lease_counts = [] + d = defer.succeed(None) + for i, ss in self.g.servers_by_number.items(): + d.addCallback(lambda ign, ss=ss: ss.backend.get_shareset(si).get_shares()) + def _append_counts( (shares_for_server, corrupted) ): + assert len(corrupted) == 0, (shares_for_server, corrupted) + for share in shares_for_server: + num_leases = len(list(share.get_leases())) + lease_counts.append( (share._get_filepath().path, num_leases) ) + d.addCallback(_append_counts) + + d.addCallback(lambda ign: lease_counts) + return d def copy_shares(self, uri): shares = {} hunk ./src/allmydata/test/no_network.py 410 - for (shnum, serverid, sharefile) in self.find_uri_shares(uri): - shares[sharefile] = open(sharefile, "rb").read() - return shares + d = self.find_uri_shares(uri) + def _got_shares(sharelist): + for (shnum, serverid, sharefp) in sharelist: + shares[sharefp.path] = sharefp.getContent() + + return shares + d.addCallback(_got_shares) + return d + + def copy_share(self, from_share, uri, to_server): + si = tahoe_uri.from_string(uri).get_storage_index() + (i_shnum, i_serverid, i_sharefp) = from_share + shares_dir = to_server.backend.get_shareset(si)._get_sharedir() + fileutil.fp_make_dirs(shares_dir) + i_sharefp.copyTo(shares_dir.child(str(i_shnum))) def restore_all_shares(self, shares): hunk ./src/allmydata/test/no_network.py 427 - for sharefile, data in shares.items(): - open(sharefile, "wb").write(data) + for sharepath, data in shares.items(): + FilePath(sharepath).setContent(data) hunk ./src/allmydata/test/no_network.py 430 - def delete_share(self, (shnum, serverid, sharefile)): - os.unlink(sharefile) + def delete_share(self, (shnum, serverid, sharefp)): + sharefp.remove() def delete_shares_numbered(self, uri, shnums): hunk ./src/allmydata/test/no_network.py 434 - for (i_shnum, i_serverid, i_sharefile) in self.find_uri_shares(uri): - if i_shnum in shnums: - os.unlink(i_sharefile) + d = self.find_uri_shares(uri) + def _got_shares(sharelist): + for (i_shnum, i_serverid, i_sharefp) in sharelist: + if i_shnum in shnums: + i_sharefp.remove() + d.addCallback(_got_shares) + return d + + def delete_all_shares(self, uri): + d = self.find_uri_shares(uri) + def _got_shares(shares): + for sh in shares: + self.delete_share(sh) + d.addCallback(_got_shares) + return d hunk ./src/allmydata/test/no_network.py 450 - def corrupt_share(self, (shnum, serverid, sharefile), corruptor_function): - sharedata = open(sharefile, "rb").read() - corruptdata = corruptor_function(sharedata) - open(sharefile, "wb").write(corruptdata) + def corrupt_share(self, (shnum, serverid, sharefp), corruptor_function, debug=False): + sharedata = sharefp.getContent() + corruptdata = corruptor_function(sharedata, debug=debug) + sharefp.setContent(corruptdata) def corrupt_shares_numbered(self, uri, shnums, corruptor, debug=False): hunk ./src/allmydata/test/no_network.py 456 - for (i_shnum, i_serverid, i_sharefile) in self.find_uri_shares(uri): - if i_shnum in shnums: - sharedata = open(i_sharefile, "rb").read() - corruptdata = corruptor(sharedata, debug=debug) - open(i_sharefile, "wb").write(corruptdata) + d = self.find_uri_shares(uri) + def _got_shares(sharelist): + for (i_shnum, i_serverid, i_sharefp) in sharelist: + if i_shnum in shnums: + self.corrupt_share((i_shnum, i_serverid, i_sharefp), corruptor, debug=debug) + d.addCallback(_got_shares) + return d def corrupt_all_shares(self, uri, corruptor, debug=False): hunk ./src/allmydata/test/no_network.py 465 - for (i_shnum, i_serverid, i_sharefile) in self.find_uri_shares(uri): - sharedata = open(i_sharefile, "rb").read() - corruptdata = corruptor(sharedata, debug=debug) - open(i_sharefile, "wb").write(corruptdata) + d = self.find_uri_shares(uri) + def _got_shares(sharelist): + for (i_shnum, i_serverid, i_sharefp) in sharelist: + self.corrupt_share((i_shnum, i_serverid, i_sharefp), corruptor, debug=debug) + d.addCallback(_got_shares) + return d def GET(self, urlpath, followRedirect=False, return_response=False, method="GET", clientnum=0, **kwargs): hunk ./src/allmydata/test/test_cli.py 2901 self.failUnlessReallyEqual(to_str(data["summary"]), "Healthy") d.addCallback(_check2) - def _clobber_shares(ignored): + d.addCallback(lambda ign: self.find_uri_shares(self.uri)) + def _clobber_shares(shares): # delete one, corrupt a second hunk ./src/allmydata/test/test_cli.py 2904 - shares = self.find_uri_shares(self.uri) self.failUnlessReallyEqual(len(shares), 10) hunk ./src/allmydata/test/test_cli.py 2905 - os.unlink(shares[0][2]) - cso = debug.CorruptShareOptions() - cso.stdout = StringIO() - cso.parseOptions([shares[1][2]]) + shares[0][2].remove() + stdout = StringIO() + sharefile = shares[1][2] storage_index = uri.from_string(self.uri).get_storage_index() self._corrupt_share_line = " server %s, SI %s, shnum %d" % \ (base32.b2a(shares[1][1]), hunk ./src/allmydata/test/test_cli.py 2913 base32.b2a(storage_index), shares[1][0]) - debug.corrupt_share(cso) + debug.do_corrupt_share(stdout, sharefile) d.addCallback(_clobber_shares) d.addCallback(lambda ign: self.do_cli("check", "--verify", self.uri)) hunk ./src/allmydata/test/test_cli.py 3027 self.failUnlessIn(" 317-1000 : 1 (1000 B, 1000 B)", lines) d.addCallback(_check_stats) - def _clobber_shares(ignored): - shares = self.find_uri_shares(self.uris[u"g\u00F6\u00F6d"]) + d.addCallback(lambda ign: self.find_uri_shares(self.uris[u"g\u00F6\u00F6d"])) + def _clobber_shares(shares): self.failUnlessReallyEqual(len(shares), 10) hunk ./src/allmydata/test/test_cli.py 3030 - os.unlink(shares[0][2]) + shares[0][2].remove() + d.addCallback(_clobber_shares) hunk ./src/allmydata/test/test_cli.py 3033 - shares = self.find_uri_shares(self.uris["mutable"]) - cso = debug.CorruptShareOptions() - cso.stdout = StringIO() - cso.parseOptions([shares[1][2]]) + d.addCallback(lambda ign: self.find_uri_shares(self.uris["mutable"])) + def _clobber_mutable_shares(shares): + stdout = StringIO() + sharefile = shares[1][2] storage_index = uri.from_string(self.uris["mutable"]).get_storage_index() self._corrupt_share_line = " corrupt: server %s, SI %s, shnum %d" % \ (base32.b2a(shares[1][1]), hunk ./src/allmydata/test/test_cli.py 3042 base32.b2a(storage_index), shares[1][0]) - debug.corrupt_share(cso) - d.addCallback(_clobber_shares) + debug.do_corrupt_share(stdout, sharefile) + d.addCallback(_clobber_mutable_shares) # root # root/g\u00F6\u00F6d [9 shares] hunk ./src/allmydata/test/test_client.py 6 from twisted.application import service import allmydata -from allmydata.node import OldConfigError +from allmydata.node import OldConfigError, InvalidValueError, MissingConfigEntry from allmydata import client from allmydata.storage_client import StorageFarmBroker hunk ./src/allmydata/test/test_client.py 9 +from allmydata.storage.backends.disk.disk_backend import DiskBackend +from allmydata.storage.backends.s3.s3_backend import S3Backend from allmydata.util import base32, fileutil from allmydata.interfaces import IFilesystemNode, IFileNode, \ IImmutableFileNode, IMutableFileNode, IDirectoryNode hunk ./src/allmydata/test/test_client.py 31 def test_loadable(self): basedir = "test_client.Basic.test_loadable" os.mkdir(basedir) - fileutil.write(os.path.join(basedir, "tahoe.cfg"), \ - BASECONFIG) - client.Client(basedir) + fileutil.write(os.path.join(basedir, "tahoe.cfg"), + BASECONFIG) + c = client.Client(basedir) + server = c.getServiceNamed("storage") + self.failUnless(isinstance(server.backend, DiskBackend), server.backend) @mock.patch('twisted.python.log.msg') def test_error_on_old_config_files(self, mock_log_msg): hunk ./src/allmydata/test/test_client.py 94 "enabled = true\n" + "reserved_space = 1000\n") c = client.Client(basedir) - self.failUnlessEqual(c.getServiceNamed("storage").reserved_space, 1000) + server = c.getServiceNamed("storage") + self.failUnlessReallyEqual(server.backend._reserved_space, 1000) def test_reserved_2(self): basedir = "client.Basic.test_reserved_2" hunk ./src/allmydata/test/test_client.py 106 "enabled = true\n" + "reserved_space = 10K\n") c = client.Client(basedir) - self.failUnlessEqual(c.getServiceNamed("storage").reserved_space, 10*1000) + server = c.getServiceNamed("storage") + self.failUnlessReallyEqual(server.backend._reserved_space, 10*1000) def test_reserved_3(self): basedir = "client.Basic.test_reserved_3" hunk ./src/allmydata/test/test_client.py 118 "enabled = true\n" + "reserved_space = 5mB\n") c = client.Client(basedir) - self.failUnlessEqual(c.getServiceNamed("storage").reserved_space, - 5*1000*1000) + server = c.getServiceNamed("storage") + self.failUnlessReallyEqual(server.backend._reserved_space, 5*1000*1000) def test_reserved_4(self): basedir = "client.Basic.test_reserved_4" hunk ./src/allmydata/test/test_client.py 124 os.mkdir(basedir) - fileutil.write(os.path.join(basedir, "tahoe.cfg"), \ - BASECONFIG + \ - "[storage]\n" + \ - "enabled = true\n" + \ - "reserved_space = 78Gb\n") + fileutil.write(os.path.join(basedir, "tahoe.cfg"), + BASECONFIG + + "[storage]\n" + + "enabled = true\n" + + "reserved_space = 78Gb\n") + c = client.Client(basedir) + server = c.getServiceNamed("storage") + self.failUnlessReallyEqual(server.backend._reserved_space, 78*1000*1000*1000) + + def test_reserved_default(self): + # This is testing the default when 'reserved_space' is not present, not + # the default for a newly created node. + basedir = "client.Basic.test_reserved_default" + os.mkdir(basedir) + fileutil.write(os.path.join(basedir, "tahoe.cfg"), + BASECONFIG + + "[storage]\n" + + "enabled = true\n") c = client.Client(basedir) hunk ./src/allmydata/test/test_client.py 143 - self.failUnlessEqual(c.getServiceNamed("storage").reserved_space, - 78*1000*1000*1000) + server = c.getServiceNamed("storage") + self.failUnlessReallyEqual(server.backend._reserved_space, 0) def test_reserved_bad(self): basedir = "client.Basic.test_reserved_bad" hunk ./src/allmydata/test/test_client.py 149 os.mkdir(basedir) - fileutil.write(os.path.join(basedir, "tahoe.cfg"), \ - BASECONFIG + \ - "[storage]\n" + \ - "enabled = true\n" + \ - "reserved_space = bogus\n") + fileutil.write(os.path.join(basedir, "tahoe.cfg"), + BASECONFIG + + "[storage]\n" + + "enabled = true\n" + + "reserved_space = bogus\n") + self.failUnlessRaises(InvalidValueError, client.Client, basedir) + + def _write_s3secret(self, basedir, secret="dummy"): + os.mkdir(os.path.join(basedir, "private")) + fileutil.write(os.path.join(basedir, "private", "s3secret"), secret) + + @mock.patch('allmydata.storage.backends.s3.s3_bucket.S3Bucket') + def test_s3_config_good_defaults(self, mock_S3Bucket): + basedir = "client.Basic.test_s3_config_good_defaults" + os.mkdir(basedir) + self._write_s3secret(basedir) + config = (BASECONFIG + + "[storage]\n" + + "enabled = true\n" + + "backend = s3\n" + + "s3.access_key_id = keyid\n" + + "s3.bucket = test\n") + fileutil.write(os.path.join(basedir, "tahoe.cfg"), config) + + c = client.Client(basedir) + mock_S3Bucket.assert_called_with("keyid", "dummy", "http://s3.amazonaws.com", "test", None, None) + server = c.getServiceNamed("storage") + self.failUnless(isinstance(server.backend, S3Backend), server.backend) + + mock_S3Bucket.reset_mock() + fileutil.write(os.path.join(basedir, "private", "s3producttoken"), "{ProductToken}") + self.failUnlessRaises(InvalidValueError, client.Client, basedir) + + mock_S3Bucket.reset_mock() + fileutil.write(os.path.join(basedir, "private", "s3usertoken"), "{UserToken}") + fileutil.write(os.path.join(basedir, "tahoe.cfg"), config + "s3.url = http://s3.example.com\n") + c = client.Client(basedir) hunk ./src/allmydata/test/test_client.py 187 - self.failUnlessEqual(c.getServiceNamed("storage").reserved_space, 0) + mock_S3Bucket.assert_called_with("keyid", "dummy", "http://s3.example.com", "test", + "{UserToken}", "{ProductToken}") + + def test_s3_readonly_bad(self): + basedir = "client.Basic.test_s3_readonly_bad" + os.mkdir(basedir) + self._write_s3secret(basedir) + fileutil.write(os.path.join(basedir, "tahoe.cfg"), + BASECONFIG + + "[storage]\n" + + "enabled = true\n" + + "readonly = true\n" + + "backend = s3\n" + + "s3.access_key_id = keyid\n" + + "s3.bucket = test\n") + self.failUnlessRaises(InvalidValueError, client.Client, basedir) + + def test_s3_config_no_access_key_id(self): + basedir = "client.Basic.test_s3_config_no_access_key_id" + os.mkdir(basedir) + self._write_s3secret(basedir) + fileutil.write(os.path.join(basedir, "tahoe.cfg"), + BASECONFIG + + "[storage]\n" + + "enabled = true\n" + + "backend = s3\n" + + "s3.bucket = test\n") + self.failUnlessRaises(MissingConfigEntry, client.Client, basedir) + + def test_s3_config_no_bucket(self): + basedir = "client.Basic.test_s3_config_no_bucket" + os.mkdir(basedir) + self._write_s3secret(basedir) + fileutil.write(os.path.join(basedir, "tahoe.cfg"), + BASECONFIG + + "[storage]\n" + + "enabled = true\n" + + "backend = s3\n" + + "s3.access_key_id = keyid\n") + self.failUnlessRaises(MissingConfigEntry, client.Client, basedir) + + def test_s3_config_no_s3secret(self): + basedir = "client.Basic.test_s3_config_no_s3secret" + os.mkdir(basedir) + fileutil.write(os.path.join(basedir, "tahoe.cfg"), + BASECONFIG + + "[storage]\n" + + "enabled = true\n" + + "backend = s3\n" + + "s3.access_key_id = keyid\n" + + "s3.bucket = test\n") + self.failUnlessRaises(MissingConfigEntry, client.Client, basedir) def _permute(self, sb, key): return [ s.get_longname() for s in sb.get_servers_for_psi(key) ] hunk ./src/allmydata/test/test_crawler.py 3 import time -import os.path + from twisted.trial import unittest from twisted.application import service from twisted.internet import defer hunk ./src/allmydata/test/test_crawler.py 7 +from twisted.python.filepath import FilePath from foolscap.api import eventually, fireEventually hunk ./src/allmydata/test/test_crawler.py 10 -from allmydata.util import fileutil, hashutil, pollmixin +from allmydata.util import hashutil, pollmixin from allmydata.storage.server import StorageServer, si_b2a from allmydata.storage.crawler import ShareCrawler, TimeSliceExceeded hunk ./src/allmydata/test/test_crawler.py 13 +from allmydata.storage.backends.disk.disk_backend import DiskBackend from allmydata.test.test_storage import FakeCanary from allmydata.test.common_util import StallMixin hunk ./src/allmydata/test/test_crawler.py 26 ShareCrawler.__init__(self, *args, **kwargs) self.all_buckets = [] self.finished_d = defer.Deferred() - def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32): - self.all_buckets.append(storage_index_b32) + + def process_shareset(self, cycle, prefix, shareset, *args, **kwargs): + self.all_buckets.append(shareset.get_storage_index_string()) + def finished_cycle(self, cycle): eventually(self.finished_d.callback, None) hunk ./src/allmydata/test/test_crawler.py 44 self.all_buckets = [] self.finished_d = defer.Deferred() self.yield_cb = None - def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32): - self.all_buckets.append(storage_index_b32) + + def process_shareset(self, cycle, prefix, shareset, *args, **kwargs): + self.all_buckets.append(shareset.get_storage_index_string()) self.countdown -= 1 if self.countdown == 0: # force a timeout. We restore it in yielding() hunk ./src/allmydata/test/test_crawler.py 72 self.accumulated = 0.0 self.cycles = 0 self.last_yield = 0.0 - def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32): + + def process_shareset(self, *args, **kwargs): start = time.time() time.sleep(0.05) elapsed = time.time() - start hunk ./src/allmydata/test/test_crawler.py 95 ShareCrawler.__init__(self, *args, **kwargs) self.counter = 0 self.finished_d = defer.Deferred() - def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32): + + def process_shareset(self, *args, **kwargs): self.counter += 1 def finished_cycle(self, cycle): hunk ./src/allmydata/test/test_crawler.py 124 def write(self, i, ss, serverid, tail=0): si = self.si(i) si = si[:-1] + chr(tail) - had,made = ss.remote_allocate_buckets(si, - self.rs(i, serverid), - self.cs(i, serverid), - set([0]), 99, FakeCanary()) - made[0].remote_write(0, "data") - made[0].remote_close() - return si_b2a(si) + d = defer.succeed(None) + d.addCallback(lambda ign: ss.remote_allocate_buckets(si, + self.rs(i, serverid), + self.cs(i, serverid), + set([0]), 99, FakeCanary())) + def _allocated( (had, made) ): + d2 = defer.succeed(None) + d2.addCallback(lambda ign: made[0].remote_write(0, "data")) + d2.addCallback(lambda ign: made[0].remote_close()) + d2.addCallback(lambda ign: si_b2a(si)) + return d2 + d.addCallback(_allocated) + return d def test_immediate(self): self.basedir = "crawler/Basic/immediate" hunk ./src/allmydata/test/test_crawler.py 140 - fileutil.make_dirs(self.basedir) serverid = "\x00" * 20 hunk ./src/allmydata/test/test_crawler.py 141 - ss = StorageServer(self.basedir, serverid) + fp = FilePath(self.basedir) + backend = DiskBackend(fp) + ss = StorageServer(serverid, backend, fp) ss.setServiceParent(self.s) hunk ./src/allmydata/test/test_crawler.py 146 - sis = [self.write(i, ss, serverid) for i in range(10)] - statefile = os.path.join(self.basedir, "statefile") + d = defer.gatherResults([self.write(i, ss, serverid) for i in range(10)]) + def _done_writes(sis): + statefp = fp.child("statefile") hunk ./src/allmydata/test/test_crawler.py 150 - c = BucketEnumeratingCrawler(ss, statefile, allowed_cpu_percentage=.1) - c.load_state() + c = BucketEnumeratingCrawler(backend, statefp, allowed_cpu_percentage=.1) + c.load_state() hunk ./src/allmydata/test/test_crawler.py 153 - c.start_current_prefix(time.time()) - self.failUnlessEqual(sorted(sis), sorted(c.all_buckets)) + c.start_current_prefix(time.time()) + self.failUnlessEqual(sorted(sis), sorted(c.all_buckets)) hunk ./src/allmydata/test/test_crawler.py 156 - # make sure the statefile has been returned to the starting point - c.finished_d = defer.Deferred() - c.all_buckets = [] - c.start_current_prefix(time.time()) - self.failUnlessEqual(sorted(sis), sorted(c.all_buckets)) + # make sure the statefile has been returned to the starting point + c.finished_d = defer.Deferred() + c.all_buckets = [] + c.start_current_prefix(time.time()) + self.failUnlessEqual(sorted(sis), sorted(c.all_buckets)) hunk ./src/allmydata/test/test_crawler.py 162 - # check that a new crawler picks up on the state file properly - c2 = BucketEnumeratingCrawler(ss, statefile) - c2.load_state() + # check that a new crawler picks up on the state file properly + c2 = BucketEnumeratingCrawler(backend, statefp) + c2.load_state() hunk ./src/allmydata/test/test_crawler.py 166 - c2.start_current_prefix(time.time()) - self.failUnlessEqual(sorted(sis), sorted(c2.all_buckets)) + c2.start_current_prefix(time.time()) + self.failUnlessEqual(sorted(sis), sorted(c2.all_buckets)) + d.addCallback(_done_writes) + return d def test_service(self): self.basedir = "crawler/Basic/service" hunk ./src/allmydata/test/test_crawler.py 173 - fileutil.make_dirs(self.basedir) serverid = "\x00" * 20 hunk ./src/allmydata/test/test_crawler.py 174 - ss = StorageServer(self.basedir, serverid) + fp = FilePath(self.basedir) + backend = DiskBackend(fp) + ss = StorageServer(serverid, backend, fp) ss.setServiceParent(self.s) hunk ./src/allmydata/test/test_crawler.py 179 - sis = [self.write(i, ss, serverid) for i in range(10)] + d = defer.gatherResults([self.write(i, ss, serverid) for i in range(10)]) + def _done_writes(sis): + statefp = fp.child("statefile") + c = BucketEnumeratingCrawler(backend, statefp) + c.setServiceParent(self.s) hunk ./src/allmydata/test/test_crawler.py 185 - statefile = os.path.join(self.basedir, "statefile") - c = BucketEnumeratingCrawler(ss, statefile) - c.setServiceParent(self.s) - - # it should be legal to call get_state() and get_progress() right - # away, even before the first tick is performed. No work should have - # been done yet. - s = c.get_state() - p = c.get_progress() - self.failUnlessEqual(s["last-complete-prefix"], None) - self.failUnlessEqual(s["current-cycle"], None) - self.failUnlessEqual(p["cycle-in-progress"], False) + # it should be legal to call get_state() and get_progress() right + # away, even before the first tick is performed. No work should have + # been done yet. + s = c.get_state() + p = c.get_progress() + self.failUnlessEqual(s["last-complete-prefix"], None) + self.failUnlessEqual(s["current-cycle"], None) + self.failUnlessEqual(p["cycle-in-progress"], False) hunk ./src/allmydata/test/test_crawler.py 194 - d = c.finished_d - def _check(ignored): - self.failUnlessEqual(sorted(sis), sorted(c.all_buckets)) - d.addCallback(_check) + d2 = c.finished_d + def _check(ignored): + self.failUnlessEqual(sorted(sis), sorted(c.all_buckets)) + d2.addCallback(_check) + return d2 + d.addCallback(_done_writes) return d def test_paced(self): hunk ./src/allmydata/test/test_crawler.py 204 self.basedir = "crawler/Basic/paced" - fileutil.make_dirs(self.basedir) serverid = "\x00" * 20 hunk ./src/allmydata/test/test_crawler.py 205 - ss = StorageServer(self.basedir, serverid) + fp = FilePath(self.basedir) + backend = DiskBackend(fp) + ss = StorageServer(serverid, backend, fp) ss.setServiceParent(self.s) hunk ./src/allmydata/test/test_crawler.py 210 - # put four buckets in each prefixdir - sis = [] + # put four sharesets in each prefixdir + d_sis = [] for i in range(10): for tail in range(4): hunk ./src/allmydata/test/test_crawler.py 214 - sis.append(self.write(i, ss, serverid, tail)) + d_sis.append(self.write(i, ss, serverid, tail)) + d = defer.gatherResults(d_sis) + def _done_writes(sis): + statefp = fp.child("statefile") hunk ./src/allmydata/test/test_crawler.py 219 - statefile = os.path.join(self.basedir, "statefile") - - c = PacedCrawler(ss, statefile) - c.load_state() - try: - c.start_current_prefix(time.time()) - except TimeSliceExceeded: - pass - # that should stop in the middle of one of the buckets. Since we - # aren't using its normal scheduler, we have to save its state - # manually. - c.save_state() - c.cpu_slice = PacedCrawler.cpu_slice - self.failUnlessEqual(len(c.all_buckets), 6) - - c.start_current_prefix(time.time()) # finish it - self.failUnlessEqual(len(sis), len(c.all_buckets)) - self.failUnlessEqual(sorted(sis), sorted(c.all_buckets)) + c = PacedCrawler(backend, statefp) + c.load_state() + try: + c.start_current_prefix(time.time()) + except TimeSliceExceeded: + pass + # that should stop in the middle of one of the sharesets. Since we + # aren't using its normal scheduler, we have to save its state + # manually. + c.save_state() + c.cpu_slice = PacedCrawler.cpu_slice + self.failUnlessEqual(len(c.all_buckets), 6) hunk ./src/allmydata/test/test_crawler.py 232 - # make sure the statefile has been returned to the starting point - c.finished_d = defer.Deferred() - c.all_buckets = [] - c.start_current_prefix(time.time()) - self.failUnlessEqual(sorted(sis), sorted(c.all_buckets)) - del c + c.start_current_prefix(time.time()) # finish it + self.failUnlessEqual(len(sis), len(c.all_buckets)) + self.failUnlessEqual(sorted(sis), sorted(c.all_buckets)) hunk ./src/allmydata/test/test_crawler.py 236 - # start a new crawler, it should start from the beginning - c = PacedCrawler(ss, statefile) - c.load_state() - try: + # make sure the statefile has been returned to the starting point + c.finished_d = defer.Deferred() + c.all_buckets = [] c.start_current_prefix(time.time()) hunk ./src/allmydata/test/test_crawler.py 240 - except TimeSliceExceeded: - pass - # that should stop in the middle of one of the buckets. Since we - # aren't using its normal scheduler, we have to save its state - # manually. - c.save_state() - c.cpu_slice = PacedCrawler.cpu_slice + self.failUnlessEqual(sorted(sis), sorted(c.all_buckets)) hunk ./src/allmydata/test/test_crawler.py 242 - # a third crawler should pick up from where it left off - c2 = PacedCrawler(ss, statefile) - c2.all_buckets = c.all_buckets[:] - c2.load_state() - c2.countdown = -1 - c2.start_current_prefix(time.time()) - self.failUnlessEqual(len(sis), len(c2.all_buckets)) - self.failUnlessEqual(sorted(sis), sorted(c2.all_buckets)) - del c, c2 + # start a new crawler, it should start from the beginning + c = PacedCrawler(backend, statefp) + c.load_state() + try: + c.start_current_prefix(time.time()) + except TimeSliceExceeded: + pass + # that should stop in the middle of one of the sharesets. Since we + # aren't using its normal scheduler, we have to save its state + # manually. + c.save_state() + c.cpu_slice = PacedCrawler.cpu_slice hunk ./src/allmydata/test/test_crawler.py 255 - # now stop it at the end of a bucket (countdown=4), to exercise a - # different place that checks the time - c = PacedCrawler(ss, statefile) - c.load_state() - c.countdown = 4 - try: - c.start_current_prefix(time.time()) - except TimeSliceExceeded: - pass - # that should stop at the end of one of the buckets. Again we must - # save state manually. - c.save_state() - c.cpu_slice = PacedCrawler.cpu_slice - self.failUnlessEqual(len(c.all_buckets), 4) - c.start_current_prefix(time.time()) # finish it - self.failUnlessEqual(len(sis), len(c.all_buckets)) - self.failUnlessEqual(sorted(sis), sorted(c.all_buckets)) - del c + # a third crawler should pick up from where it left off + c2 = PacedCrawler(backend, statefp) + c2.all_buckets = c.all_buckets[:] + c2.load_state() + c2.countdown = -1 + c2.start_current_prefix(time.time()) + self.failUnlessEqual(len(sis), len(c2.all_buckets)) + self.failUnlessEqual(sorted(sis), sorted(c2.all_buckets)) + del c2 hunk ./src/allmydata/test/test_crawler.py 265 - # stop it again at the end of the bucket, check that a new checker - # picks up correctly - c = PacedCrawler(ss, statefile) - c.load_state() - c.countdown = 4 - try: - c.start_current_prefix(time.time()) - except TimeSliceExceeded: - pass - # that should stop at the end of one of the buckets. - c.save_state() + # now stop it at the end of a shareset (countdown=4), to exercise a + # different place that checks the time + c = PacedCrawler(backend, statefp) + c.load_state() + c.countdown = 4 + try: + c.start_current_prefix(time.time()) + except TimeSliceExceeded: + pass + # that should stop at the end of one of the sharesets. Again we must + # save state manually. + c.save_state() + c.cpu_slice = PacedCrawler.cpu_slice + self.failUnlessEqual(len(c.all_buckets), 4) + c.start_current_prefix(time.time()) # finish it + self.failUnlessEqual(len(sis), len(c.all_buckets)) + self.failUnlessEqual(sorted(sis), sorted(c.all_buckets)) + + # stop it again at the end of the shareset, check that a new checker + # picks up correctly + c = PacedCrawler(backend, statefp) + c.load_state() + c.countdown = 4 + try: + c.start_current_prefix(time.time()) + except TimeSliceExceeded: + pass + # that should stop at the end of one of the sharesets. + c.save_state() hunk ./src/allmydata/test/test_crawler.py 295 - c2 = PacedCrawler(ss, statefile) - c2.all_buckets = c.all_buckets[:] - c2.load_state() - c2.countdown = -1 - c2.start_current_prefix(time.time()) - self.failUnlessEqual(len(sis), len(c2.all_buckets)) - self.failUnlessEqual(sorted(sis), sorted(c2.all_buckets)) - del c, c2 + c2 = PacedCrawler(backend, statefp) + c2.all_buckets = c.all_buckets[:] + c2.load_state() + c2.countdown = -1 + c2.start_current_prefix(time.time()) + self.failUnlessEqual(len(sis), len(c2.all_buckets)) + self.failUnlessEqual(sorted(sis), sorted(c2.all_buckets)) + d.addCallback(_done_writes) + return d def test_paced_service(self): self.basedir = "crawler/Basic/paced_service" hunk ./src/allmydata/test/test_crawler.py 307 - fileutil.make_dirs(self.basedir) serverid = "\x00" * 20 hunk ./src/allmydata/test/test_crawler.py 308 - ss = StorageServer(self.basedir, serverid) + fp = FilePath(self.basedir) + backend = DiskBackend(fp) + ss = StorageServer(serverid, backend, fp) ss.setServiceParent(self.s) hunk ./src/allmydata/test/test_crawler.py 313 - sis = [self.write(i, ss, serverid) for i in range(10)] - - statefile = os.path.join(self.basedir, "statefile") - c = PacedCrawler(ss, statefile) + d = defer.gatherResults([self.write(i, ss, serverid) for i in range(10)]) + def _done_writes(sis): + statefp = fp.child("statefile") + c = PacedCrawler(backend, statefp) hunk ./src/allmydata/test/test_crawler.py 318 - did_check_progress = [False] - def check_progress(): - c.yield_cb = None - try: - p = c.get_progress() - self.failUnlessEqual(p["cycle-in-progress"], True) - pct = p["cycle-complete-percentage"] - # after 6 buckets, we happen to be at 76.17% complete. As - # long as we create shares in deterministic order, this will - # continue to be true. - self.failUnlessEqual(int(pct), 76) - left = p["remaining-sleep-time"] - self.failUnless(isinstance(left, float), left) - self.failUnless(left > 0.0, left) - except Exception, e: - did_check_progress[0] = e - else: - did_check_progress[0] = True - c.yield_cb = check_progress + did_check_progress = [False] + def check_progress(): + c.yield_cb = None + try: + p = c.get_progress() + self.failUnlessEqual(p["cycle-in-progress"], True) + pct = p["cycle-complete-percentage"] + # after 6 sharesets, we happen to be at 76.17% complete. As + # long as we create shares in deterministic order, this will + # continue to be true. + self.failUnlessEqual(int(pct), 76) + left = p["remaining-sleep-time"] + self.failUnless(isinstance(left, float), left) + self.failUnless(left > 0.0, left) + except Exception, e: + did_check_progress[0] = e + else: + did_check_progress[0] = True + c.yield_cb = check_progress hunk ./src/allmydata/test/test_crawler.py 338 - c.setServiceParent(self.s) - # that should get through 6 buckets, pause for a little while (and - # run check_progress()), then resume + c.setServiceParent(self.s) + # that should get through 6 sharesets, pause for a little while (and + # run check_progress()), then resume hunk ./src/allmydata/test/test_crawler.py 342 - d = c.finished_d - def _check(ignored): - if did_check_progress[0] is not True: - raise did_check_progress[0] - self.failUnless(did_check_progress[0]) - self.failUnlessEqual(sorted(sis), sorted(c.all_buckets)) - # at this point, the crawler should be sitting in the inter-cycle - # timer, which should be pegged at the minumum cycle time - self.failUnless(c.timer) - self.failUnless(c.sleeping_between_cycles) - self.failUnlessEqual(c.current_sleep_time, c.minimum_cycle_time) + d2 = c.finished_d + def _check(ignored): + if did_check_progress[0] is not True: + raise did_check_progress[0] + self.failUnless(did_check_progress[0]) + self.failUnlessEqual(sorted(sis), sorted(c.all_buckets)) + # at this point, the crawler should be sitting in the inter-cycle + # timer, which should be pegged at the minumum cycle time + self.failUnless(c.timer) + self.failUnless(c.sleeping_between_cycles) + self.failUnlessEqual(c.current_sleep_time, c.minimum_cycle_time) hunk ./src/allmydata/test/test_crawler.py 354 - p = c.get_progress() - self.failUnlessEqual(p["cycle-in-progress"], False) - naptime = p["remaining-wait-time"] - self.failUnless(isinstance(naptime, float), naptime) - # min-cycle-time is 300, so this is basically testing that it took - # less than 290s to crawl - self.failUnless(naptime > 10.0, naptime) - soon = p["next-crawl-time"] - time.time() - self.failUnless(soon > 10.0, soon) + p = c.get_progress() + self.failUnlessEqual(p["cycle-in-progress"], False) + naptime = p["remaining-wait-time"] + self.failUnless(isinstance(naptime, float), naptime) + # min-cycle-time is 300, so this is basically testing that it took + # less than 290s to crawl + self.failUnless(naptime > 10.0, naptime) + soon = p["next-crawl-time"] - time.time() + self.failUnless(soon > 10.0, soon) hunk ./src/allmydata/test/test_crawler.py 364 - d.addCallback(_check) + d2.addCallback(_check) + return d2 + d.addCallback(_done_writes) return d def OFF_test_cpu_usage(self): hunk ./src/allmydata/test/test_crawler.py 377 # and read the stdout when it runs. self.basedir = "crawler/Basic/cpu_usage" - fileutil.make_dirs(self.basedir) serverid = "\x00" * 20 hunk ./src/allmydata/test/test_crawler.py 378 - ss = StorageServer(self.basedir, serverid) + fp = FilePath(self.basedir) + backend = DiskBackend(fp) + ss = StorageServer(serverid, backend, fp) ss.setServiceParent(self.s) hunk ./src/allmydata/test/test_crawler.py 383 - for i in range(10): - self.write(i, ss, serverid) - - statefile = os.path.join(self.basedir, "statefile") - c = ConsumingCrawler(ss, statefile) - c.setServiceParent(self.s) + d = defer.gatherResults([self.write(i, ss, serverid) for i in range(10)]) + def _done_writes(sis): + statefp = fp.child("statefile") + c = ConsumingCrawler(backend, statefp) + c.setServiceParent(self.s) hunk ./src/allmydata/test/test_crawler.py 389 - # this will run as fast as it can, consuming about 50ms per call to - # process_bucket(), limited by the Crawler to about 50% cpu. We let - # it run for a few seconds, then compare how much time - # process_bucket() got vs wallclock time. It should get between 10% - # and 70% CPU. This is dicey, there's about 100ms of overhead per - # 300ms slice (saving the state file takes about 150-200us, but we do - # it 1024 times per cycle, one for each [empty] prefixdir), leaving - # 200ms for actual processing, which is enough to get through 4 - # buckets each slice, then the crawler sleeps for 300ms/0.5 = 600ms, - # giving us 900ms wallclock per slice. In 4.0 seconds we can do 4.4 - # slices, giving us about 17 shares, so we merely assert that we've - # finished at least one cycle in that time. + # this will run as fast as it can, consuming about 50ms per call to + # process_shareset(), limited by the Crawler to about 50% cpu. We let + # it run for a few seconds, then compare how much time + # process_shareset() got vs wallclock time. It should get between 10% + # and 70% CPU. This is dicey, there's about 100ms of overhead per + # 300ms slice (saving the state file takes about 150-200us, but we do + # it 1024 times per cycle, one for each [empty] prefixdir), leaving + # 200ms for actual processing, which is enough to get through 4 + # sharesets each slice, then the crawler sleeps for 300ms/0.5 = 600ms, + # giving us 900ms wallclock per slice. In 4.0 seconds we can do 4.4 + # slices, giving us about 17 shares, so we merely assert that we've + # finished at least one cycle in that time. hunk ./src/allmydata/test/test_crawler.py 402 - # with a short cpu_slice (so we can keep this test down to 4 - # seconds), the overhead is enough to make a nominal 50% usage more - # like 30%. Forcing sleep_time to 0 only gets us 67% usage. + # with a short cpu_slice (so we can keep this test down to 4 + # seconds), the overhead is enough to make a nominal 50% usage more + # like 30%. Forcing sleep_time to 0 only gets us 67% usage. hunk ./src/allmydata/test/test_crawler.py 406 - start = time.time() - d = self.stall(delay=4.0) - def _done(res): - elapsed = time.time() - start - percent = 100.0 * c.accumulated / elapsed - # our buildslaves vary too much in their speeds and load levels, - # and many of them only manage to hit 7% usage when our target is - # 50%. So don't assert anything about the results, just log them. - print - print "crawler: got %d%% percent when trying for 50%%" % percent - print "crawler: got %d full cycles" % c.cycles - d.addCallback(_done) + start = time.time() + d2 = self.stall(delay=4.0) + def _done(res): + elapsed = time.time() - start + percent = 100.0 * c.accumulated / elapsed + # our buildslaves vary too much in their speeds and load levels, + # and many of them only manage to hit 7% usage when our target is + # 50%. So don't assert anything about the results, just log them. + print + print "crawler: got %d%% percent when trying for 50%%" % percent + print "crawler: got %d full cycles" % c.cycles + d2.addCallback(_done) + return d2 + d.addCallback(_done_writes) return d def test_empty_subclass(self): hunk ./src/allmydata/test/test_crawler.py 424 self.basedir = "crawler/Basic/empty_subclass" - fileutil.make_dirs(self.basedir) serverid = "\x00" * 20 hunk ./src/allmydata/test/test_crawler.py 425 - ss = StorageServer(self.basedir, serverid) + fp = FilePath(self.basedir) + backend = DiskBackend(fp) + ss = StorageServer(serverid, backend, fp) ss.setServiceParent(self.s) hunk ./src/allmydata/test/test_crawler.py 430 - for i in range(10): - self.write(i, ss, serverid) - - statefile = os.path.join(self.basedir, "statefile") - c = ShareCrawler(ss, statefile) - c.slow_start = 0 - c.setServiceParent(self.s) + d = defer.gatherResults([self.write(i, ss, serverid) for i in range(10)]) + def _done_writes(sis): + statefp = fp.child("statefile") + c = ShareCrawler(backend, statefp) + c.slow_start = 0 + c.setServiceParent(self.s) hunk ./src/allmydata/test/test_crawler.py 437 - # we just let it run for a while, to get figleaf coverage of the - # empty methods in the base class + # we just let it run for a while, to get figleaf coverage of the + # empty methods in the base class hunk ./src/allmydata/test/test_crawler.py 440 - def _check(): - return bool(c.state["last-cycle-finished"] is not None) - d = self.poll(_check) - def _done(ignored): - state = c.get_state() - self.failUnless(state["last-cycle-finished"] is not None) - d.addCallback(_done) + def _check(): + return bool(c.state["last-cycle-finished"] is not None) + d2 = self.poll(_check) + def _done(ignored): + state = c.get_state() + self.failUnless(state["last-cycle-finished"] is not None) + d2.addCallback(_done) + return d2 + d.addCallback(_done_writes) return d def test_oneshot(self): hunk ./src/allmydata/test/test_crawler.py 453 self.basedir = "crawler/Basic/oneshot" - fileutil.make_dirs(self.basedir) serverid = "\x00" * 20 hunk ./src/allmydata/test/test_crawler.py 454 - ss = StorageServer(self.basedir, serverid) + fp = FilePath(self.basedir) + backend = DiskBackend(fp) + ss = StorageServer(serverid, backend, fp) ss.setServiceParent(self.s) hunk ./src/allmydata/test/test_crawler.py 459 - for i in range(30): - self.write(i, ss, serverid) - - statefile = os.path.join(self.basedir, "statefile") - c = OneShotCrawler(ss, statefile) - c.setServiceParent(self.s) + d = defer.gatherResults([self.write(i, ss, serverid) for i in range(30)]) + def _done_writes(sis): + statefp = fp.child("statefile") + c = OneShotCrawler(backend, statefp) + c.setServiceParent(self.s) hunk ./src/allmydata/test/test_crawler.py 465 - d = c.finished_d - def _finished_first_cycle(ignored): - return fireEventually(c.counter) - d.addCallback(_finished_first_cycle) - def _check(old_counter): - # the crawler should do any work after it's been stopped - self.failUnlessEqual(old_counter, c.counter) - self.failIf(c.running) - self.failIf(c.timer) - self.failIf(c.current_sleep_time) - s = c.get_state() - self.failUnlessEqual(s["last-cycle-finished"], 0) - self.failUnlessEqual(s["current-cycle"], None) - d.addCallback(_check) + d2 = c.finished_d + def _finished_first_cycle(ignored): + return fireEventually(c.counter) + d2.addCallback(_finished_first_cycle) + def _check(old_counter): + # the crawler should do any work after it's been stopped + self.failUnlessEqual(old_counter, c.counter) + self.failIf(c.running) + self.failIf(c.timer) + self.failIf(c.current_sleep_time) + s = c.get_state() + self.failUnlessEqual(s["last-cycle-finished"], 0) + self.failUnlessEqual(s["current-cycle"], None) + d2.addCallback(_check) + return d2 + d.addCallback(_done_writes) return d hunk ./src/allmydata/test/test_deepcheck.py 23 ShouldFailMixin from allmydata.test.common_util import StallMixin from allmydata.test.no_network import GridTestMixin +from allmydata.scripts import debug + timeout = 2400 # One of these took 1046.091s on Zandr's ARM box. hunk ./src/allmydata/test/test_deepcheck.py 68 def _stash_and_corrupt(node): self.node = node self.fileurl = "uri/" + urllib.quote(node.get_uri()) - self.corrupt_shares_numbered(node.get_uri(), [0], - _corrupt_mutable_share_data) + return self.corrupt_shares_numbered(node.get_uri(), [0], + _corrupt_mutable_share_data) d.addCallback(_stash_and_corrupt) # now make sure the webapi verifier notices it d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true", hunk ./src/allmydata/test/test_deepcheck.py 989 return d - def _run_cli(self, argv): - stdout, stderr = StringIO(), StringIO() - # this can only do synchronous operations - assert argv[0] == "debug" - runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr) - return stdout.getvalue() - def _delete_some_shares(self, node): hunk ./src/allmydata/test/test_deepcheck.py 990 - self.delete_shares_numbered(node.get_uri(), [0,1]) + return self.delete_shares_numbered(node.get_uri(), [0,1]) def _corrupt_some_shares(self, node): hunk ./src/allmydata/test/test_deepcheck.py 993 - for (shnum, serverid, sharefile) in self.find_uri_shares(node.get_uri()): - if shnum in (0,1): - self._run_cli(["debug", "corrupt-share", sharefile]) + d = self.find_uri_shares(node.get_uri()) + def _got_shares(sharelist): + for (shnum, serverid, sharefile) in sharelist: + if shnum in (0,1): + debug.do_corrupt_share(StringIO(), sharefile) + d.addCallback(_got_shares) + return d def _delete_most_shares(self, node): hunk ./src/allmydata/test/test_deepcheck.py 1002 - self.delete_shares_numbered(node.get_uri(), range(1,10)) - + return self.delete_shares_numbered(node.get_uri(), range(1,10)) def check_is_healthy(self, cr, where): try: hunk ./src/allmydata/test/test_download.py 6 # a previous run. This asserts that the current code is capable of decoding # shares from a previous version. -import os from twisted.trial import unittest from twisted.internet import defer, reactor from allmydata import uri hunk ./src/allmydata/test/test_download.py 9 -from allmydata.storage.server import storage_index_to_dir from allmydata.util import base32, fileutil, spans, log, hashutil from allmydata.util.consumer import download_to_data, MemoryConsumer from allmydata.immutable import upload, layout hunk ./src/allmydata/test/test_download.py 85 u = upload.Data(plaintext, None) d = self.c0.upload(u) f = open("stored_shares.py", "w") - def _created_immutable(ur): - # write the generated shares and URI to a file, which can then be - # incorporated into this one next time. - f.write('immutable_uri = "%s"\n' % ur.uri) - f.write('immutable_shares = {\n') - si = uri.from_string(ur.uri).get_storage_index() - si_dir = storage_index_to_dir(si) + + def _write_py(uri): + si = uri.from_string(uri).get_storage_index() for (i,ss,ssdir) in self.iterate_servers(): hunk ./src/allmydata/test/test_download.py 89 - sharedir = os.path.join(ssdir, "shares", si_dir) - shares = {} - for fn in os.listdir(sharedir): - shnum = int(fn) - sharedata = open(os.path.join(sharedir, fn), "rb").read() - shares[shnum] = sharedata - fileutil.rm_dir(sharedir) - if shares: + sharemap = {} + shareset = ss.backend.get_shareset(si) + (shares, corrupted) = shareset.get_shares_synchronous() + assert len(corrupted) == 0, (shares, corrupted) + for share in shares: + sharedata = share._get_filepath().getContent() + sharemap[share.get_shnum()] = sharedata + + fileutil.fp_remove(shareset._get_sharedir()) + if sharemap: f.write(' %d: { # client[%d]\n' % (i, i)) hunk ./src/allmydata/test/test_download.py 100 - for shnum in sorted(shares.keys()): + for shnum in sorted(sharemap.keys()): f.write(' %d: base32.a2b("%s"),\n' % hunk ./src/allmydata/test/test_download.py 102 - (shnum, base32.b2a(shares[shnum]))) + (shnum, base32.b2a(sharemap[shnum]))) f.write(' },\n') f.write('}\n') hunk ./src/allmydata/test/test_download.py 105 - f.write('\n') hunk ./src/allmydata/test/test_download.py 106 + def _created_immutable(ur): + # write the generated shares and URI to a file, which can then be + # incorporated into this one next time. + f.write('immutable_uri = "%s"\n' % ur.uri) + f.write('immutable_shares = {\n') + _write_py(ur.uri) + f.write('\n') d.addCallback(_created_immutable) d.addCallback(lambda ignored: hunk ./src/allmydata/test/test_download.py 120 def _created_mutable(n): f.write('mutable_uri = "%s"\n' % n.get_uri()) f.write('mutable_shares = {\n') - si = uri.from_string(n.get_uri()).get_storage_index() - si_dir = storage_index_to_dir(si) - for (i,ss,ssdir) in self.iterate_servers(): - sharedir = os.path.join(ssdir, "shares", si_dir) - shares = {} - for fn in os.listdir(sharedir): - shnum = int(fn) - sharedata = open(os.path.join(sharedir, fn), "rb").read() - shares[shnum] = sharedata - fileutil.rm_dir(sharedir) - if shares: - f.write(' %d: { # client[%d]\n' % (i, i)) - for shnum in sorted(shares.keys()): - f.write(' %d: base32.a2b("%s"),\n' % - (shnum, base32.b2a(shares[shnum]))) - f.write(' },\n') - f.write('}\n') - - f.close() + _write_py(n.get_uri()) d.addCallback(_created_mutable) def _done(ignored): hunk ./src/allmydata/test/test_download.py 125 f.close() - d.addCallback(_done) + d.addBoth(_done) return d hunk ./src/allmydata/test/test_download.py 129 + def _write_shares(self, fileuri, shares): + si = uri.from_string(fileuri).get_storage_index() + for i in shares: + shares_for_server = shares[i] + for shnum in shares_for_server: + share_dir = self.get_server(i).backend.get_shareset(si)._get_sharedir() + fileutil.fp_make_dirs(share_dir) + share_dir.child(str(shnum)).setContent(shares_for_server[shnum]) + def load_shares(self, ignored=None): # this uses the data generated by create_shares() to populate the # storage servers with pre-generated shares hunk ./src/allmydata/test/test_download.py 141 - si = uri.from_string(immutable_uri).get_storage_index() - si_dir = storage_index_to_dir(si) - for i in immutable_shares: - shares = immutable_shares[i] - for shnum in shares: - dn = os.path.join(self.get_serverdir(i), "shares", si_dir) - fileutil.make_dirs(dn) - fn = os.path.join(dn, str(shnum)) - f = open(fn, "wb") - f.write(shares[shnum]) - f.close() - - si = uri.from_string(mutable_uri).get_storage_index() - si_dir = storage_index_to_dir(si) - for i in mutable_shares: - shares = mutable_shares[i] - for shnum in shares: - dn = os.path.join(self.get_serverdir(i), "shares", si_dir) - fileutil.make_dirs(dn) - fn = os.path.join(dn, str(shnum)) - f = open(fn, "wb") - f.write(shares[shnum]) - f.close() + self._write_shares(immutable_uri, immutable_shares) + self._write_shares(mutable_uri, mutable_shares) def download_immutable(self, ignored=None): n = self.c0.create_node_from_uri(immutable_uri) hunk ./src/allmydata/test/test_download.py 185 self.load_shares() si = uri.from_string(immutable_uri).get_storage_index() - si_dir = storage_index_to_dir(si) n = self.c0.create_node_from_uri(immutable_uri) d = download_to_data(n) hunk ./src/allmydata/test/test_download.py 196 # find the three shares that were used, and delete them. Then # download again, forcing the downloader to fail over to other # shares + d2 = defer.succeed(None) for s in n._cnode._node._shares: for clientnum in immutable_shares: for shnum in immutable_shares[clientnum]: hunk ./src/allmydata/test/test_download.py 201 if s._shnum == shnum: - fn = os.path.join(self.get_serverdir(clientnum), - "shares", si_dir, str(shnum)) - os.unlink(fn) + d2.addCallback(lambda ign, clientnum=clientnum, shnum=shnum: + self.get_server(clientnum).backend.get_shareset(si).get_share(shnum)) + d2.addCallback(lambda share: share.unlink()) + return d2 d.addCallback(_clobber_some_shares) d.addCallback(lambda ign: download_to_data(n)) d.addCallback(_got_data) hunk ./src/allmydata/test/test_download.py 213 # delete all but one of the shares that are still alive live_shares = [s for s in n._cnode._node._shares if s.is_alive()] save_me = live_shares[0]._shnum + d2 = defer.succeed(None) for clientnum in immutable_shares: for shnum in immutable_shares[clientnum]: if shnum == save_me: hunk ./src/allmydata/test/test_download.py 218 continue - fn = os.path.join(self.get_serverdir(clientnum), - "shares", si_dir, str(shnum)) - if os.path.exists(fn): - os.unlink(fn) + d2.addCallback(lambda ign, clientnum=clientnum, shnum=shnum: + self.get_server(clientnum).backend.get_shareset(si).get_share(shnum)) + def _eb(f): + f.trap(EnvironmentError) + d2.addCallbacks(lambda share: share.unlink(), _eb) + # now the download should fail with NotEnoughSharesError hunk ./src/allmydata/test/test_download.py 225 - return self.shouldFail(NotEnoughSharesError, "1shares", None, - download_to_data, n) + d2.addCallback(lambda ign: self.shouldFail(NotEnoughSharesError, "1shares", None, + download_to_data, n)) + return d2 d.addCallback(_clobber_most_shares) def _clobber_all_shares(ign): hunk ./src/allmydata/test/test_download.py 234 # delete the last remaining share for clientnum in immutable_shares: for shnum in immutable_shares[clientnum]: - fn = os.path.join(self.get_serverdir(clientnum), - "shares", si_dir, str(shnum)) - if os.path.exists(fn): - os.unlink(fn) + share_dir = self.get_server(clientnum).backend.get_shareset(si)._get_sharedir() + fileutil.fp_remove(share_dir.child(str(shnum))) # now a new download should fail with NoSharesError. We want a # new ImmutableFileNode so it will forget about the old shares. # If we merely called create_node_from_uri() without first hunk ./src/allmydata/test/test_download.py 812 # will report two shares, and the ShareFinder will handle the # duplicate by attaching both to the same CommonShare instance. si = uri.from_string(immutable_uri).get_storage_index() - si_dir = storage_index_to_dir(si) - sh0_file = [sharefile - for (shnum, serverid, sharefile) - in self.find_uri_shares(immutable_uri) - if shnum == 0][0] - sh0_data = open(sh0_file, "rb").read() - for clientnum in immutable_shares: - if 0 in immutable_shares[clientnum]: - continue - cdir = self.get_serverdir(clientnum) - target = os.path.join(cdir, "shares", si_dir, "0") - outf = open(target, "wb") - outf.write(sh0_data) - outf.close() hunk ./src/allmydata/test/test_download.py 813 - d = self.download_immutable() + d = defer.succeed(None) + d.addCallback(lambda ign: self.find_uri_shares(immutable_uri)) + def _duplicate(sharelist): + sh0_fp = [sharefp for (shnum, serverid, sharefp) in sharelist + if shnum == 0][0] + sh0_data = sh0_fp.getContent() + for clientnum in immutable_shares: + if 0 in immutable_shares[clientnum]: + continue + cdir = self.get_server(clientnum).backend.get_shareset(si)._get_sharedir() + fileutil.fp_make_dirs(cdir) + cdir.child(str(shnum)).setContent(sh0_data) + d.addCallback(_duplicate) + + d.addCallback(lambda ign: self.download_immutable()) return d def test_verifycap(self): hunk ./src/allmydata/test/test_download.py 912 log.msg("corrupt %d" % which) def _corruptor(s, debug=False): return s[:which] + chr(ord(s[which])^0x01) + s[which+1:] - self.corrupt_shares_numbered(imm_uri, [0], _corruptor) + return self.corrupt_shares_numbered(imm_uri, [0], _corruptor) def _corrupt_set(self, ign, imm_uri, which, newvalue): log.msg("corrupt %d" % which) hunk ./src/allmydata/test/test_download.py 918 def _corruptor(s, debug=False): return s[:which] + chr(newvalue) + s[which+1:] - self.corrupt_shares_numbered(imm_uri, [0], _corruptor) + return self.corrupt_shares_numbered(imm_uri, [0], _corruptor) def test_each_byte(self): # Setting catalog_detection=True performs an exhaustive test of the hunk ./src/allmydata/test/test_download.py 929 # (since we don't need every byte of the share). That takes 50s to # run on my laptop and doesn't have any actual asserts, so we don't # normally do that. + # XXX this has bitrotted (before v1.8.2) and gives an AttributeError. self.catalog_detection = False self.basedir = "download/Corruption/each_byte" hunk ./src/allmydata/test/test_download.py 981 d = self.c0.upload(u) def _uploaded(ur): imm_uri = ur.uri - self.shares = self.copy_shares(imm_uri) - d = defer.succeed(None) + # 'victims' is a list of corruption tests to run. Each one flips # the low-order bit of the specified offset in the share file (so # offset=0 is the MSB of the container version, offset=15 is the hunk ./src/allmydata/test/test_download.py 1025 [(i, "need-4th") for i in need_4th_victims]) if self.catalog_detection: corrupt_me = [(i, "") for i in range(len(self.sh0_orig))] - for i,expected in corrupt_me: - # All these tests result in a successful download. What we're - # measuring is how many shares the downloader had to use. - d.addCallback(self._corrupt_flip, imm_uri, i) - d.addCallback(_download, imm_uri, i, expected) - d.addCallback(lambda ign: self.restore_all_shares(self.shares)) - d.addCallback(fireEventually) - corrupt_values = [(3, 2, "no-sh0"), - (15, 2, "need-4th"), # share looks v2 - ] - for i,newvalue,expected in corrupt_values: - d.addCallback(self._corrupt_set, imm_uri, i, newvalue) - d.addCallback(_download, imm_uri, i, expected) - d.addCallback(lambda ign: self.restore_all_shares(self.shares)) - d.addCallback(fireEventually) - return d + + d2 = defer.succeed(None) + d2.addCallback(lambda ign: self.copy_shares(imm_uri)) + def _copied(copied_shares): + d3 = defer.succeed(None) + + for i, expected in corrupt_me: + # All these tests result in a successful download. What we're + # measuring is how many shares the downloader had to use. + d3.addCallback(self._corrupt_flip, imm_uri, i) + d3.addCallback(_download, imm_uri, i, expected) + d3.addCallback(lambda ign: self.restore_all_shares(copied_shares)) + d3.addCallback(fireEventually) + corrupt_values = [(3, 2, "no-sh0"), + (15, 2, "need-4th"), # share looks v2 + ] + for i, newvalue, expected in corrupt_values: + d3.addCallback(self._corrupt_set, imm_uri, i, newvalue) + d3.addCallback(_download, imm_uri, i, expected) + d3.addCallback(lambda ign: self.restore_all_shares(copied_shares)) + d3.addCallback(fireEventually) + return d3 + d2.addCallback(_copied) + return d2 d.addCallback(_uploaded) def _show_results(ign): hunk ./src/allmydata/test/test_download.py 1086 d = self.c0.upload(u) def _uploaded(ur): imm_uri = ur.uri - self.shares = self.copy_shares(imm_uri) - corrupt_me = [(48, "block data", "Last failure: None"), (600+2*32, "block_hashes[2]", "BadHashError"), (376+2*32, "crypttext_hash_tree[2]", "BadHashError"), hunk ./src/allmydata/test/test_download.py 1099 assert not n._cnode._node._shares return download_to_data(n) - d = defer.succeed(None) - for i,which,substring in corrupt_me: - # All these tests result in a failed download. - d.addCallback(self._corrupt_flip_all, imm_uri, i) - d.addCallback(lambda ign, which=which, substring=substring: - self.shouldFail(NoSharesError, which, - substring, - _download, imm_uri)) - d.addCallback(lambda ign: self.restore_all_shares(self.shares)) - d.addCallback(fireEventually) - return d - d.addCallback(_uploaded) + d2 = defer.succeed(None) + d2.addCallback(lambda ign: self.copy_shares(imm_uri)) + def _copied(copied_shares): + d3 = defer.succeed(None) hunk ./src/allmydata/test/test_download.py 1104 + for i, which, substring in corrupt_me: + # All these tests result in a failed download. + d3.addCallback(self._corrupt_flip_all, imm_uri, i) + d3.addCallback(lambda ign, which=which, substring=substring: + self.shouldFail(NoSharesError, which, + substring, + _download, imm_uri)) + d3.addCallback(lambda ign: self.restore_all_shares(copied_shares)) + d3.addCallback(fireEventually) + return d3 + d2.addCallback(_copied) + return d2 + d.addCallback(_uploaded) return d def _corrupt_flip_all(self, ign, imm_uri, which): hunk ./src/allmydata/test/test_download.py 1122 def _corruptor(s, debug=False): return s[:which] + chr(ord(s[which])^0x01) + s[which+1:] - self.corrupt_all_shares(imm_uri, _corruptor) + return self.corrupt_all_shares(imm_uri, _corruptor) + class DownloadV2(_Base, unittest.TestCase): # tests which exercise v2-share code. They first upload a file with hunk ./src/allmydata/test/test_download.py 1193 d = self.c0.upload(u) def _uploaded(ur): imm_uri = ur.uri - def _do_corrupt(which, newvalue): - def _corruptor(s, debug=False): - return s[:which] + chr(newvalue) + s[which+1:] - self.corrupt_shares_numbered(imm_uri, [0], _corruptor) - _do_corrupt(12+3, 0x00) - n = self.c0.create_node_from_uri(imm_uri) - d = download_to_data(n) - def _got_data(data): - self.failUnlessEqual(data, plaintext) - d.addCallback(_got_data) - return d + which = 12+3 + newvalue = 0x00 + def _corruptor(s, debug=False): + return s[:which] + chr(newvalue) + s[which+1:] + + d2 = defer.succeed(None) + d2.addCallback(lambda ign: self.corrupt_shares_numbered(imm_uri, [0], _corruptor)) + d2.addCallback(lambda ign: self.c0.create_node_from_uri(imm_uri)) + d2.addCallback(lambda n: download_to_data(n)) + d2.addCallback(lambda data: self.failUnlessEqual(data, plaintext)) + return d2 d.addCallback(_uploaded) return d hunk ./src/allmydata/test/test_encode.py 134 d.addCallback(_try) return d - def get_share_hashes(self, at_least_these=()): + def get_share_hashes(self): d = self._start() def _try(unused=None): if self.mode == "bad sharehash": hunk ./src/allmydata/test/test_hung_server.py 3 # -*- coding: utf-8 -*- -import os, shutil from twisted.trial import unittest from twisted.internet import defer hunk ./src/allmydata/test/test_hung_server.py 5 -from allmydata import uri + from allmydata.util.consumer import download_to_data from allmydata.immutable import upload from allmydata.mutable.common import UnrecoverableFileError hunk ./src/allmydata/test/test_hung_server.py 10 from allmydata.mutable.publish import MutableData -from allmydata.storage.common import storage_index_to_dir from allmydata.test.no_network import GridTestMixin from allmydata.test.common import ShouldFailMixin from allmydata.util.pollmixin import PollMixin hunk ./src/allmydata/test/test_hung_server.py 31 timeout = 240 def _break(self, servers): - for (id, ss) in servers: - self.g.break_server(id) + for ss in servers: + self.g.break_server(ss.original.get_serverid()) def _hang(self, servers, **kwargs): hunk ./src/allmydata/test/test_hung_server.py 35 - for (id, ss) in servers: - self.g.hang_server(id, **kwargs) + for ss in servers: + self.g.hang_server(ss.original.get_serverid(), **kwargs) def _unhang(self, servers, **kwargs): hunk ./src/allmydata/test/test_hung_server.py 39 - for (id, ss) in servers: - self.g.unhang_server(id, **kwargs) + for ss in servers: + self.g.unhang_server(ss.original.get_serverid(), **kwargs) def _hang_shares(self, shnums, **kwargs): # hang all servers who are holding the given shares hunk ./src/allmydata/test/test_hung_server.py 52 hung_serverids.add(i_serverid) def _delete_all_shares_from(self, servers): - serverids = [id for (id, ss) in servers] - for (i_shnum, i_serverid, i_sharefile) in self.shares: + serverids = [ss.original.get_serverid() for ss in servers] + for (i_shnum, i_serverid, i_sharefp) in self.shares: if i_serverid in serverids: hunk ./src/allmydata/test/test_hung_server.py 55 - os.unlink(i_sharefile) + i_sharefp.remove() def _corrupt_all_shares_in(self, servers, corruptor_func): hunk ./src/allmydata/test/test_hung_server.py 58 - serverids = [id for (id, ss) in servers] - for (i_shnum, i_serverid, i_sharefile) in self.shares: + serverids = [ss.original.get_serverid() for ss in servers] + for (i_shnum, i_serverid, i_sharefp) in self.shares: if i_serverid in serverids: hunk ./src/allmydata/test/test_hung_server.py 61 - self._corrupt_share((i_shnum, i_sharefile), corruptor_func) + self.corrupt_share((i_shnum, i_serverid, i_sharefp), corruptor_func) def _copy_all_shares_from(self, from_servers, to_server): hunk ./src/allmydata/test/test_hung_server.py 64 - serverids = [id for (id, ss) in from_servers] - for (i_shnum, i_serverid, i_sharefile) in self.shares: + serverids = [ss.original.get_serverid() for ss in from_servers] + for (i_shnum, i_serverid, i_sharefp) in self.shares: if i_serverid in serverids: hunk ./src/allmydata/test/test_hung_server.py 67 - self._copy_share((i_shnum, i_sharefile), to_server) - - def _copy_share(self, share, to_server): - (sharenum, sharefile) = share - (id, ss) = to_server - shares_dir = os.path.join(ss.original.storedir, "shares") - si = uri.from_string(self.uri).get_storage_index() - si_dir = os.path.join(shares_dir, storage_index_to_dir(si)) - if not os.path.exists(si_dir): - os.makedirs(si_dir) - new_sharefile = os.path.join(si_dir, str(sharenum)) - shutil.copy(sharefile, new_sharefile) - self.shares = self.find_uri_shares(self.uri) - # Make sure that the storage server has the share. - self.failUnless((sharenum, ss.original.my_nodeid, new_sharefile) - in self.shares) + self.copy_share((i_shnum, i_serverid, i_sharefp), self.uri, to_server.original) hunk ./src/allmydata/test/test_hung_server.py 69 - def _corrupt_share(self, share, corruptor_func): - (sharenum, sharefile) = share - data = open(sharefile, "rb").read() - newdata = corruptor_func(data) - os.unlink(sharefile) - wf = open(sharefile, "wb") - wf.write(newdata) - wf.close() + d = self.find_uri_shares(self.uri) + def _got_shares(shares): + self.shares = shares + d.addCallback(_got_shares) + return d def _set_up(self, mutable, testdir, num_clients=1, num_servers=10): self.mutable = mutable hunk ./src/allmydata/test/test_hung_server.py 86 self.c0 = self.g.clients[0] nm = self.c0.nodemaker - self.servers = sorted([(s.get_serverid(), s.get_rref()) - for s in nm.storage_broker.get_connected_servers()]) + unsorted = [(s.get_serverid(), s.get_rref()) for s in nm.storage_broker.get_connected_servers()] + self.servers = [ss for (id, ss) in sorted(unsorted)] self.servers = self.servers[5:] + self.servers[:5] if mutable: hunk ./src/allmydata/test/test_hung_server.py 95 d = nm.create_mutable_file(uploadable) def _uploaded_mutable(node): self.uri = node.get_uri() - self.shares = self.find_uri_shares(self.uri) d.addCallback(_uploaded_mutable) else: data = upload.Data(immutable_plaintext, convergence="") hunk ./src/allmydata/test/test_hung_server.py 101 d = self.c0.upload(data) def _uploaded_immutable(upload_res): self.uri = upload_res.uri - self.shares = self.find_uri_shares(self.uri) d.addCallback(_uploaded_immutable) hunk ./src/allmydata/test/test_hung_server.py 102 + + d.addCallback(lambda ign: self.find_uri_shares(self.uri)) + def _got_shares(shares): + self.shares = shares + d.addCallback(_got_shares) return d def _start_download(self): hunk ./src/allmydata/test/test_immutable.py 240 d = self.startup("download_from_only_3_shares_with_good_crypttext_hash") def _corrupt_7(ign): c = common._corrupt_offset_of_block_hashes_to_truncate_crypttext_hashes - self.corrupt_shares_numbered(self.uri, self._shuffled(7), c) + return self.corrupt_shares_numbered(self.uri, self._shuffled(7), c) d.addCallback(_corrupt_7) d.addCallback(self._download_and_check_plaintext) return d hunk ./src/allmydata/test/test_immutable.py 267 d = self.startup("download_abort_if_too_many_corrupted_shares") def _corrupt_8(ign): c = common._corrupt_sharedata_version_number - self.corrupt_shares_numbered(self.uri, self._shuffled(8), c) + return self.corrupt_shares_numbered(self.uri, self._shuffled(8), c) d.addCallback(_corrupt_8) def _try_download(ign): start_reads = self._count_reads() hunk ./src/allmydata/test/test_mutable.py 21 from foolscap.api import eventually, fireEventually from foolscap.logging import log from allmydata.storage_client import StorageFarmBroker -from allmydata.storage.common import storage_index_to_dir from allmydata.scripts import debug from allmydata.mutable.filenode import MutableFileNode, BackoffAgent hunk ./src/allmydata/test/test_mutable.py 1865 class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): - def get_shares(self, s): + def get_all_shares(self, s): all_shares = {} # maps (peerid, shnum) to share data for peerid in s._peers: shares = s._peers[peerid] hunk ./src/allmydata/test/test_mutable.py 1875 return all_shares def copy_shares(self, ignored=None): - self.old_shares.append(self.get_shares(self._storage)) + self.old_shares.append(self.get_all_shares(self._storage)) def test_repair_nop(self): self.old_shares = [] hunk ./src/allmydata/test/test_mutable.py 2922 fso = debug.FindSharesOptions() storage_index = base32.b2a(n.get_storage_index()) fso.si_s = storage_index - fso.nodedirs = [unicode(os.path.dirname(os.path.abspath(storedir))) + fso.nodedirs = [unicode(storedir.parent().path) for (i,ss,storedir) in self.iterate_servers()] fso.stdout = StringIO() hunk ./src/allmydata/test/test_mutable.py 2956 cso.stderr = StringIO() debug.catalog_shares(cso) shares = cso.stdout.getvalue().splitlines() + self.failIf(len(shares) < 1, shares) oneshare = shares[0] # all shares should be MDMF self.failIf(oneshare.startswith("UNKNOWN"), oneshare) self.failUnless(oneshare.startswith("MDMF"), oneshare) hunk ./src/allmydata/test/test_mutable.py 3556 # Now execute each assignment by writing the storage. for (share, servernum) in assignments: sharedata = base64.b64decode(self.sdmf_old_shares[share]) - storedir = self.get_serverdir(servernum) - storage_path = os.path.join(storedir, "shares", - storage_index_to_dir(si)) - fileutil.make_dirs(storage_path) - fileutil.write(os.path.join(storage_path, "%d" % share), - sharedata) + # This must be a disk backend. + storage_dir = self.get_server(servernum).backend.get_shareset(si)._get_sharedir() + fileutil.fp_make_dirs(storage_dir) + storage_dir.child("%d" % share).setContent(sharedata) # ...and verify that the shares are there. hunk ./src/allmydata/test/test_mutable.py 3561 - shares = self.find_uri_shares(self.sdmf_old_cap) - assert len(shares) == 10 + d = self.find_uri_shares(self.sdmf_old_cap) + def _got_shares(shares): + assert len(shares) == 10 + d.addCallback(_got_shares) + return d def test_new_downloader_can_read_old_shares(self): self.basedir = "mutable/Interoperability/new_downloader_can_read_old_shares" hunk ./src/allmydata/test/test_mutable.py 3570 self.set_up_grid() - self.copy_sdmf_shares() - nm = self.g.clients[0].nodemaker - n = nm.create_from_cap(self.sdmf_old_cap) - d = n.download_best_version() - d.addCallback(self.failUnlessEqual, self.sdmf_old_contents) + d = self.copy_sdmf_shares() + def _create_node(ign): + nm = self.g.clients[0].nodemaker + return nm.create_from_cap(self.sdmf_old_cap) + d.addCallback(_create_node) + d.addCallback(lambda n: n.download_best_version()) + d.addCallback(lambda res: self.failUnlessEqual(res, self.sdmf_old_contents)) return d hunk ./src/allmydata/test/test_provisioning.py 13 from nevow import inevow from zope.interface import implements -class MyRequest: +class MockRequest: implements(inevow.IRequest) pass hunk ./src/allmydata/test/test_provisioning.py 26 def test_load(self): pt = provisioning.ProvisioningTool() self.fields = {} - #r = MyRequest() + #r = MockRequest() #r.fields = self.fields #ctx = RequestContext() #unfilled = pt.renderSynchronously(ctx) hunk ./src/allmydata/test/test_repairer.py 89 self.failIfBigger(delta_reads, 0) d.addCallback(_check) - def _remove_all(ignored): - for sh in self.find_uri_shares(self.uri): - self.delete_share(sh) - d.addCallback(_remove_all) + d.addCallback(lambda ign: self.delete_all_shares(self.uri)) d.addCallback(lambda ignored: self._stash_counts()) d.addCallback(lambda ignored: hunk ./src/allmydata/test/test_repairer.py 409 Monitor(), verify=False)) # test share corruption - def _test_corrupt(ignored): + d.addCallback(lambda ign: self.find_uri_shares(self.uri)) + def _test_corrupt(shares): olddata = {} hunk ./src/allmydata/test/test_repairer.py 412 - shares = self.find_uri_shares(self.uri) - for (shnum, serverid, sharefile) in shares: - olddata[ (shnum, serverid) ] = open(sharefile, "rb").read() + for (shnum, serverid, sharefp) in shares: + olddata[ (shnum, serverid) ] = sharefp.getContent() for sh in shares: self.corrupt_share(sh, common._corrupt_uri_extension) hunk ./src/allmydata/test/test_repairer.py 416 - for (shnum, serverid, sharefile) in shares: - newdata = open(sharefile, "rb").read() + for (shnum, serverid, sharefp) in shares: + newdata = sharefp.getContent() self.failIfEqual(olddata[ (shnum, serverid) ], newdata) d.addCallback(_test_corrupt) hunk ./src/allmydata/test/test_repairer.py 421 - def _remove_all(ignored): - for sh in self.find_uri_shares(self.uri): - self.delete_share(sh) - d.addCallback(_remove_all) - d.addCallback(lambda ignored: self.find_uri_shares(self.uri)) - d.addCallback(lambda shares: self.failUnlessEqual(shares, [])) + d.addCallback(lambda ign: self.delete_all_shares(self.uri)) hunk ./src/allmydata/test/test_repairer.py 423 + d.addCallback(lambda ign: self.find_uri_shares(self.uri)) + d.addCallback(lambda shares: self.failUnlessEqual(shares, [])) return d def test_repair_from_deletion_of_1(self): hunk ./src/allmydata/test/test_repairer.py 450 self.failIfBigger(delta_allocates, DELTA_WRITES_PER_SHARE) self.failIf(pre.is_healthy()) self.failUnless(post.is_healthy()) - - # Now we inspect the filesystem to make sure that it has 10 - # shares. - shares = self.find_uri_shares(self.uri) - self.failIf(len(shares) < 10) d.addCallback(_check_results) hunk ./src/allmydata/test/test_repairer.py 452 + # Now we inspect the filesystem to make sure that it has 10 shares. + d.addCallback(lambda ign: self.find_uri_shares(self.uri)) + d.addCallback(lambda shares: self.failIf(len(shares) < 10)) + d.addCallback(lambda ignored: self.c0_filenode.check(Monitor(), verify=True)) d.addCallback(lambda vr: self.failUnless(vr.is_healthy())) hunk ./src/allmydata/test/test_repairer.py 495 self.failIfBigger(delta_allocates, (DELTA_WRITES_PER_SHARE * 7)) self.failIf(pre.is_healthy()) self.failUnless(post.is_healthy(), post.data) - - # Make sure we really have 10 shares. - shares = self.find_uri_shares(self.uri) - self.failIf(len(shares) < 10) d.addCallback(_check_results) hunk ./src/allmydata/test/test_repairer.py 497 + # Now we inspect the filesystem to make sure that it has 10 shares. + d.addCallback(lambda ign: self.find_uri_shares(self.uri)) + d.addCallback(lambda shares: self.failIf(len(shares) < 10)) + d.addCallback(lambda ignored: self.c0_filenode.check(Monitor(), verify=True)) d.addCallback(lambda vr: self.failUnless(vr.is_healthy())) hunk ./src/allmydata/test/test_repairer.py 530 # happiness setting. def _delete_some_servers(ignored): for i in xrange(7): - self.g.remove_server(self.g.servers_by_number[i].my_nodeid) + self.remove_server(i) assert len(self.g.servers_by_number) == 3 hunk ./src/allmydata/test/test_storage.py 1 -import time, os.path, platform, stat, re, simplejson, struct, shutil +import time, os.path, stat, platform, re, simplejson, struct, itertools import mock hunk ./src/allmydata/test/test_storage.py 8 from twisted.trial import unittest from twisted.internet import defer from twisted.application import service +from twisted.python.filepath import FilePath from foolscap.api import fireEventually hunk ./src/allmydata/test/test_storage.py 10 -import itertools + from allmydata import interfaces from allmydata.util import fileutil, hashutil, base32, pollmixin, time_format hunk ./src/allmydata/test/test_storage.py 13 +from allmydata.util.deferredutil import for_items from allmydata.storage.server import StorageServer hunk ./src/allmydata/test/test_storage.py 15 -from allmydata.storage.mutable import MutableShareFile -from allmydata.storage.immutable import BucketWriter, BucketReader -from allmydata.storage.common import DataTooLargeError, storage_index_to_dir, \ +from allmydata.storage.backends.null.null_backend import NullBackend +from allmydata.storage.backends.disk.disk_backend import DiskBackend +from allmydata.storage.backends.disk.immutable import load_immutable_disk_share, create_immutable_disk_share +from allmydata.storage.backends.disk.mutable import MutableDiskShare +from allmydata.storage.backends.s3.s3_backend import S3Backend +from allmydata.storage.backends.s3.mock_s3 import MockS3Bucket +from allmydata.storage.bucket import BucketWriter, BucketReader +from allmydata.storage.common import DataTooLargeError, UnknownContainerVersionError, \ UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError from allmydata.storage.lease import LeaseInfo from allmydata.storage.crawler import BucketCountingCrawler hunk ./src/allmydata/test/test_storage.py 73 class Bucket(unittest.TestCase): def make_workdir(self, name): - basedir = os.path.join("storage", "Bucket", name) - incoming = os.path.join(basedir, "tmp", "bucket") - final = os.path.join(basedir, "bucket") - fileutil.make_dirs(basedir) - fileutil.make_dirs(os.path.join(basedir, "tmp")) + basedir = FilePath("storage").child("Bucket").child(name) + tmpdir = basedir.child("tmp") + tmpdir.makedirs() + incoming = tmpdir.child("bucket") + final = basedir.child("bucket") return incoming, final def bucket_writer_closed(self, bw, consumed): hunk ./src/allmydata/test/test_storage.py 97 def test_create(self): incoming, final = self.make_workdir("test_create") - bw = BucketWriter(self, incoming, final, 200, self.make_lease(), - FakeCanary()) - bw.remote_write(0, "a"*25) - bw.remote_write(25, "b"*25) - bw.remote_write(50, "c"*25) - bw.remote_write(75, "d"*7) - bw.remote_close() + d = defer.succeed(None) + d.addCallback(lambda ign: create_immutable_disk_share(incoming, final, max_size=200)) + def _got_share(share): + bw = BucketWriter(self, share, self.make_lease(), FakeCanary()) + d2 = defer.succeed(None) + d2.addCallback(lambda ign: bw.remote_write(0, "a"*25)) + d2.addCallback(lambda ign: bw.remote_write(25, "b"*25)) + d2.addCallback(lambda ign: bw.remote_write(50, "c"*25)) + d2.addCallback(lambda ign: bw.remote_write(75, "d"*7)) + d2.addCallback(lambda ign: bw.remote_close()) + return d2 + d.addCallback(_got_share) + return d def test_readwrite(self): incoming, final = self.make_workdir("test_readwrite") hunk ./src/allmydata/test/test_storage.py 113 - bw = BucketWriter(self, incoming, final, 200, self.make_lease(), - FakeCanary()) - bw.remote_write(0, "a"*25) - bw.remote_write(25, "b"*25) - bw.remote_write(50, "c"*7) # last block may be short - bw.remote_close() + d = defer.succeed(None) + d.addCallback(lambda ign: create_immutable_disk_share(incoming, final, max_size=200)) + def _got_share(share): + bw = BucketWriter(self, share, self.make_lease(), FakeCanary()) + d2 = defer.succeed(None) + d2.addCallback(lambda ign: bw.remote_write(0, "a"*25)) + d2.addCallback(lambda ign: bw.remote_write(25, "b"*25)) + d2.addCallback(lambda ign: bw.remote_write(50, "c"*7)) # last block may be short + d2.addCallback(lambda ign: bw.remote_close()) hunk ./src/allmydata/test/test_storage.py 123 - # now read from it - br = BucketReader(self, bw.finalhome) - self.failUnlessEqual(br.remote_read(0, 25), "a"*25) - self.failUnlessEqual(br.remote_read(25, 25), "b"*25) - self.failUnlessEqual(br.remote_read(50, 7), "c"*7) + # now read from it + def _read(ign): + br = BucketReader(self, share) + d3 = defer.succeed(None) + d3.addCallback(lambda ign: br.remote_read(0, 25)) + d3.addCallback(lambda res: self.failUnlessEqual(res, "a"*25)) + d3.addCallback(lambda ign: br.remote_read(25, 25)) + d3.addCallback(lambda res: self.failUnlessEqual(res, "b"*25)) + d3.addCallback(lambda ign: br.remote_read(50, 7)) + d3.addCallback(lambda res: self.failUnlessEqual(res, "c"*7)) + return d3 + d2.addCallback(_read) + return d2 + d.addCallback(_got_share) + return d def test_read_past_end_of_share_data(self): # test vector for immutable files (hard-coded contents of an immutable share hunk ./src/allmydata/test/test_storage.py 159 ownernumber = struct.pack('>L', 0) renewsecret = 'THIS LETS ME RENEW YOUR FILE....' assert len(renewsecret) == 32 - cancelsecret = 'THIS LETS ME KILL YOUR FILE HAHA' + cancelsecret = 'THIS USED TO LET ME KILL YR FILE' assert len(cancelsecret) == 32 expirationtime = struct.pack('>L', 60*60*24*31) # 31 days in seconds hunk ./src/allmydata/test/test_storage.py 169 incoming, final = self.make_workdir("test_read_past_end_of_share_data") - fileutil.write(final, share_file_data) + final.setContent(share_file_data) + d = defer.succeed(None) + d.addCallback(lambda ign: load_immutable_disk_share(final)) + def _got_share(share): + mockstorageserver = mock.Mock() hunk ./src/allmydata/test/test_storage.py 175 - mockstorageserver = mock.Mock() + # Now read from it. + br = BucketReader(mockstorageserver, share) hunk ./src/allmydata/test/test_storage.py 178 - # Now read from it. - br = BucketReader(mockstorageserver, final) + d2 = br.remote_read(0, len(share_data)) + d2.addCallback(lambda res: self.failUnlessEqual(res, share_data)) hunk ./src/allmydata/test/test_storage.py 181 - self.failUnlessEqual(br.remote_read(0, len(share_data)), share_data) + # Read past the end of share data to get the cancel secret. + read_length = len(share_data) + len(ownernumber) + len(renewsecret) + len(cancelsecret) hunk ./src/allmydata/test/test_storage.py 184 - # Read past the end of share data to get the cancel secret. - read_length = len(share_data) + len(ownernumber) + len(renewsecret) + len(cancelsecret) + d2.addCallback(lambda ign: br.remote_read(0, read_length)) + d2.addCallback(lambda res: self.failUnlessEqual(res, share_data)) hunk ./src/allmydata/test/test_storage.py 187 - result_of_read = br.remote_read(0, read_length) - self.failUnlessEqual(result_of_read, share_data) + d2.addCallback(lambda ign: br.remote_read(0, len(share_data)+1)) + d2.addCallback(lambda res: self.failUnlessEqual(res, share_data)) + return d2 + d.addCallback(_got_share) + return d hunk ./src/allmydata/test/test_storage.py 193 - result_of_read = br.remote_read(0, len(share_data)+1) - self.failUnlessEqual(result_of_read, share_data) class RemoteBucket: hunk ./src/allmydata/test/test_storage.py 215 class BucketProxy(unittest.TestCase): def make_bucket(self, name, size): - basedir = os.path.join("storage", "BucketProxy", name) - incoming = os.path.join(basedir, "tmp", "bucket") - final = os.path.join(basedir, "bucket") - fileutil.make_dirs(basedir) - fileutil.make_dirs(os.path.join(basedir, "tmp")) - bw = BucketWriter(self, incoming, final, size, self.make_lease(), - FakeCanary()) - rb = RemoteBucket() - rb.target = bw - return bw, rb, final + basedir = FilePath("storage").child("BucketProxy").child(name) + tmpdir = basedir.child("tmp") + tmpdir.makedirs() + incoming = tmpdir.child("bucket") + final = basedir.child("bucket") + + d = defer.succeed(None) + d.addCallback(lambda ign: create_immutable_disk_share(incoming, final, size)) + def _got_share(share): + bw = BucketWriter(self, share, self.make_lease(), FakeCanary()) + rb = RemoteBucket() + rb.target = bw + return bw, rb, final + d.addCallback(_got_share) + return d def make_lease(self): owner_num = 0 hunk ./src/allmydata/test/test_storage.py 247 pass def test_create(self): - bw, rb, sharefname = self.make_bucket("test_create", 500) - bp = WriteBucketProxy(rb, None, - data_size=300, - block_size=10, - num_segments=5, - num_share_hashes=3, - uri_extension_size_max=500) - self.failUnless(interfaces.IStorageBucketWriter.providedBy(bp), bp) + d = self.make_bucket("test_create", 500) + def _made_bucket( (bw, rb, sharefp) ): + bp = WriteBucketProxy(rb, None, + data_size=300, + block_size=10, + num_segments=5, + num_share_hashes=3, + uri_extension_size_max=500) + self.failUnless(interfaces.IStorageBucketWriter.providedBy(bp), bp) + d.addCallback(_made_bucket) + return d def _do_test_readwrite(self, name, header_size, wbp_class, rbp_class): # Let's pretend each share has 100 bytes of data, and that there are hunk ./src/allmydata/test/test_storage.py 281 for i in (1,9,13)] uri_extension = "s" + "E"*498 + "e" - bw, rb, sharefname = self.make_bucket(name, sharesize) - bp = wbp_class(rb, None, - data_size=95, - block_size=25, - num_segments=4, - num_share_hashes=3, - uri_extension_size_max=len(uri_extension)) + d = self.make_bucket(name, sharesize) + def _made_bucket( (bw, rb, sharefp) ): + bp = wbp_class(rb, None, + data_size=95, + block_size=25, + num_segments=4, + num_share_hashes=3, + uri_extension_size_max=len(uri_extension)) + + d2 = bp.put_header() + d2.addCallback(lambda ign: bp.put_block(0, "a"*25)) + d2.addCallback(lambda ign: bp.put_block(1, "b"*25)) + d2.addCallback(lambda ign: bp.put_block(2, "c"*25)) + d2.addCallback(lambda ign: bp.put_block(3, "d"*20)) + d2.addCallback(lambda ign: bp.put_crypttext_hashes(crypttext_hashes)) + d2.addCallback(lambda ign: bp.put_block_hashes(block_hashes)) + d2.addCallback(lambda ign: bp.put_share_hashes(share_hashes)) + d2.addCallback(lambda ign: bp.put_uri_extension(uri_extension)) + d2.addCallback(lambda ign: bp.close()) hunk ./src/allmydata/test/test_storage.py 301 - d = bp.put_header() - d.addCallback(lambda res: bp.put_block(0, "a"*25)) - d.addCallback(lambda res: bp.put_block(1, "b"*25)) - d.addCallback(lambda res: bp.put_block(2, "c"*25)) - d.addCallback(lambda res: bp.put_block(3, "d"*20)) - d.addCallback(lambda res: bp.put_crypttext_hashes(crypttext_hashes)) - d.addCallback(lambda res: bp.put_block_hashes(block_hashes)) - d.addCallback(lambda res: bp.put_share_hashes(share_hashes)) - d.addCallback(lambda res: bp.put_uri_extension(uri_extension)) - d.addCallback(lambda res: bp.close()) + d2.addCallback(lambda ign: load_immutable_disk_share(sharefp)) + return d2 + d.addCallback(_made_bucket) # now read everything back hunk ./src/allmydata/test/test_storage.py 306 - def _start_reading(res): - br = BucketReader(self, sharefname) + def _start_reading(share): + br = BucketReader(self, share) rb = RemoteBucket() rb.target = br server = NoNetworkServer("abc", None) hunk ./src/allmydata/test/test_storage.py 315 self.failUnlessIn("to peer", repr(rbp)) self.failUnless(interfaces.IStorageBucketReader.providedBy(rbp), rbp) - d1 = rbp.get_block_data(0, 25, 25) - d1.addCallback(lambda res: self.failUnlessEqual(res, "a"*25)) - d1.addCallback(lambda res: rbp.get_block_data(1, 25, 25)) - d1.addCallback(lambda res: self.failUnlessEqual(res, "b"*25)) - d1.addCallback(lambda res: rbp.get_block_data(2, 25, 25)) - d1.addCallback(lambda res: self.failUnlessEqual(res, "c"*25)) - d1.addCallback(lambda res: rbp.get_block_data(3, 25, 20)) - d1.addCallback(lambda res: self.failUnlessEqual(res, "d"*20)) - - d1.addCallback(lambda res: rbp.get_crypttext_hashes()) - d1.addCallback(lambda res: - self.failUnlessEqual(res, crypttext_hashes)) - d1.addCallback(lambda res: rbp.get_block_hashes(set(range(4)))) - d1.addCallback(lambda res: self.failUnlessEqual(res, block_hashes)) - d1.addCallback(lambda res: rbp.get_share_hashes()) - d1.addCallback(lambda res: self.failUnlessEqual(res, share_hashes)) - d1.addCallback(lambda res: rbp.get_uri_extension()) - d1.addCallback(lambda res: - self.failUnlessEqual(res, uri_extension)) - - return d1 + d2 = defer.succeed(None) + d2.addCallback(lambda ign: rbp.get_block_data(0, 25, 25)) + d2.addCallback(lambda res: self.failUnlessEqual(res, "a"*25)) + d2.addCallback(lambda ign: rbp.get_block_data(1, 25, 25)) + d2.addCallback(lambda res: self.failUnlessEqual(res, "b"*25)) + d2.addCallback(lambda ign: rbp.get_block_data(2, 25, 25)) + d2.addCallback(lambda res: self.failUnlessEqual(res, "c"*25)) + d2.addCallback(lambda ign: rbp.get_block_data(3, 25, 20)) + d2.addCallback(lambda res: self.failUnlessEqual(res, "d"*20)) hunk ./src/allmydata/test/test_storage.py 325 + d2.addCallback(lambda ign: rbp.get_crypttext_hashes()) + d2.addCallback(lambda res: self.failUnlessEqual(res, crypttext_hashes)) + d2.addCallback(lambda ign: rbp.get_block_hashes(set(range(4)))) + d2.addCallback(lambda res: self.failUnlessEqual(res, block_hashes)) + d2.addCallback(lambda ign: rbp.get_share_hashes()) + d2.addCallback(lambda res: self.failUnlessEqual(res, share_hashes)) + d2.addCallback(lambda ign: rbp.get_uri_extension()) + d2.addCallback(lambda res: self.failUnlessEqual(res, uri_extension)) + return d2 d.addCallback(_start_reading) return d hunk ./src/allmydata/test/test_storage.py 345 return self._do_test_readwrite("test_readwrite_v2", 0x44, WriteBucketProxy_v2, ReadBucketProxy) -class Server(unittest.TestCase): hunk ./src/allmydata/test/test_storage.py 346 +class Seek(unittest.TestCase): + def workdir(self, name): + return FilePath("storage").child(self.__class__.__name__).child(name) + + def test_seek(self): + basedir = self.workdir("test_seek") + basedir.makedirs() + fp = basedir.child("testfile") + fp.setContent("start") + + # mode="w" allows seeking-to-create-holes, but truncates pre-existing + # files. mode="a" preserves previous contents but does not allow + # seeking-to-create-holes. mode="r+" allows both. + f = fp.open("rb+") + try: + f.seek(100) + f.write("100") + finally: + f.close() + fp.restat() + filelen = fp.getsize() + self.failUnlessEqual(filelen, 100+3) + f2 = fp.open("rb") + try: + self.failUnlessEqual(f2.read(5), "start") + finally: + f2.close() + + +class ServerMixin: def setUp(self): self.sparent = LoggingServiceParent() self.sparent.startService() hunk ./src/allmydata/test/test_storage.py 384 return self.sparent.stopService() def workdir(self, name): - basedir = os.path.join("storage", "Server", name) - return basedir + return FilePath("storage").child(self.__class__.__name__).child(name) hunk ./src/allmydata/test/test_storage.py 386 - def create(self, name, reserved_space=0, klass=StorageServer): - workdir = self.workdir(name) - ss = klass(workdir, "\x00" * 20, reserved_space=reserved_space, - stats_provider=FakeStatsProvider()) - ss.setServiceParent(self.sparent) - return ss + def allocate(self, ss, storage_index, sharenums, size, canary=None): + renew_secret = hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()) + cancel_secret = hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()) + if not canary: + canary = FakeCanary() + return defer.maybeDeferred(ss.remote_allocate_buckets, + storage_index, renew_secret, cancel_secret, + sharenums, size, canary) + + def _write_and_close(self, ign, i, bw): + d = defer.succeed(None) + d.addCallback(lambda ign: bw.remote_write(0, "%25d" % i)) + d.addCallback(lambda ign: bw.remote_close()) + return d hunk ./src/allmydata/test/test_storage.py 401 + def _close_writer(self, ign, i, bw): + return bw.remote_close() + + def _abort_writer(self, ign, i, bw): + return bw.remote_abort() + + +class ServerTest(ServerMixin, ShouldFailMixin): def test_create(self): self.create("test_create") hunk ./src/allmydata/test/test_storage.py 418 sv1 = ver['http://allmydata.org/tahoe/protocols/storage/v1'] self.failUnless(sv1.get('prevents-read-past-end-of-share-data'), sv1) - def allocate(self, ss, storage_index, sharenums, size, canary=None): - renew_secret = hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()) - cancel_secret = hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()) - if not canary: - canary = FakeCanary() - return ss.remote_allocate_buckets(storage_index, - renew_secret, cancel_secret, - sharenums, size, canary) + def test_has_immutable_readv(self): + ss = self.create("test_has_immutable_readv") + ver = ss.remote_get_version() + sv1 = ver['http://allmydata.org/tahoe/protocols/storage/v1'] + self.failUnless(sv1.get('has-immutable-readv'), sv1) hunk ./src/allmydata/test/test_storage.py 424 - def test_large_share(self): - syslow = platform.system().lower() - if 'cygwin' in syslow or 'windows' in syslow or 'darwin' in syslow: - raise unittest.SkipTest("If your filesystem doesn't support efficient sparse files then it is very expensive (Mac OS X and Windows don't support efficient sparse files).") + # TODO: test that we actually support it hunk ./src/allmydata/test/test_storage.py 426 - avail = fileutil.get_available_space('.', 512*2**20) - if avail <= 4*2**30: - raise unittest.SkipTest("This test will spuriously fail if you have less than 4 GiB free on your filesystem.") + def test_create_share(self): + ss = self.create("test_create_share") hunk ./src/allmydata/test/test_storage.py 429 - ss = self.create("test_large_share") + d = self.allocate(ss, "si1", [0], 75) + def _allocated( (already, writers) ): + self.failUnlessEqual(already, set()) + self.failUnlessEqual(set(writers.keys()), set([0])) hunk ./src/allmydata/test/test_storage.py 434 - already,writers = self.allocate(ss, "allocate", [0], 2**32+2) - self.failUnlessEqual(already, set()) - self.failUnlessEqual(set(writers.keys()), set([0])) + d2 = defer.succeed(None) + d2.addCallback(lambda ign: writers[0].remote_write(0, "data")) + d2.addCallback(lambda ign: writers[0].remote_close()) hunk ./src/allmydata/test/test_storage.py 438 - shnum, bucket = writers.items()[0] - # This test is going to hammer your filesystem if it doesn't make a sparse file for this. :-( - bucket.remote_write(2**32, "ab") - bucket.remote_close() + d2.addCallback(lambda ign: ss.backend.get_shareset("si1").get_share(0)) + d2.addCallback(lambda share: self.failUnless(interfaces.IShareForReading.providedBy(share))) hunk ./src/allmydata/test/test_storage.py 441 - readers = ss.remote_get_buckets("allocate") - reader = readers[shnum] - self.failUnlessEqual(reader.remote_read(2**32, 2), "ab") + d2.addCallback(lambda ign: ss.backend.get_shareset("si1").get_shares()) + def _check( (shares, corrupted) ): + self.failUnlessEqual(len(shares), 1, str(shares)) + self.failUnlessEqual(len(corrupted), 0, str(corrupted)) + d2.addCallback(_check) + return d2 + d.addCallback(_allocated) + return d def test_dont_overfill_dirs(self): """ hunk ./src/allmydata/test/test_storage.py 457 same storage index), this won't add an entry to the share directory. """ ss = self.create("test_dont_overfill_dirs") - already, writers = self.allocate(ss, "storageindex", [0], 10) - for i, wb in writers.items(): - wb.remote_write(0, "%10d" % i) - wb.remote_close() - storedir = os.path.join(self.workdir("test_dont_overfill_dirs"), - "shares") - children_of_storedir = set(os.listdir(storedir)) + storedir = self.workdir("test_dont_overfill_dirs").child("shares") hunk ./src/allmydata/test/test_storage.py 459 - # Now store another one under another storageindex that has leading - # chars the same as the first storageindex. - already, writers = self.allocate(ss, "storageindey", [0], 10) - for i, wb in writers.items(): - wb.remote_write(0, "%10d" % i) - wb.remote_close() - storedir = os.path.join(self.workdir("test_dont_overfill_dirs"), - "shares") - new_children_of_storedir = set(os.listdir(storedir)) - self.failUnlessEqual(children_of_storedir, new_children_of_storedir) + def _write_and_get_children( (already, writers) ): + d = for_items(self._write_and_close, writers) + d.addCallback(lambda ign: sorted([str(child.basename()) for child in storedir.children()])) + return d hunk ./src/allmydata/test/test_storage.py 464 - def test_remove_incoming(self): - ss = self.create("test_remove_incoming") - already, writers = self.allocate(ss, "vid", range(3), 10) - for i,wb in writers.items(): - wb.remote_write(0, "%10d" % i) - wb.remote_close() - incoming_share_dir = wb.incominghome - incoming_bucket_dir = os.path.dirname(incoming_share_dir) - incoming_prefix_dir = os.path.dirname(incoming_bucket_dir) - incoming_dir = os.path.dirname(incoming_prefix_dir) - self.failIf(os.path.exists(incoming_bucket_dir), incoming_bucket_dir) - self.failIf(os.path.exists(incoming_prefix_dir), incoming_prefix_dir) - self.failUnless(os.path.exists(incoming_dir), incoming_dir) + d = self.allocate(ss, "storageindex", [0], 25) + d.addCallback(_write_and_get_children) + + def _got_children(children_of_storedir): + # Now store another one under another storageindex that has leading + # chars the same as the first storageindex. + d2 = self.allocate(ss, "storageindey", [0], 25) + d2.addCallback(_write_and_get_children) + d2.addCallback(lambda res: self.failUnlessEqual(res, children_of_storedir)) + return d2 + d.addCallback(_got_children) + return d def test_abort(self): # remote_abort, when called on a writer, should make sure that hunk ./src/allmydata/test/test_storage.py 482 # the allocated size of the bucket is not counted by the storage # server when accounting for space. ss = self.create("test_abort") - already, writers = self.allocate(ss, "allocate", [0, 1, 2], 150) - self.failIfEqual(ss.allocated_size(), 0) hunk ./src/allmydata/test/test_storage.py 483 - # Now abort the writers. - for writer in writers.itervalues(): - writer.remote_abort() - self.failUnlessEqual(ss.allocated_size(), 0) + d = self.allocate(ss, "allocate", [0, 1, 2], 150) + def _allocated( (already, writers) ): + self.failIfEqual(ss.allocated_size(), 0) hunk ./src/allmydata/test/test_storage.py 487 + # Now abort the writers. + d2 = for_items(self._abort_writer, writers) + d2.addCallback(lambda ign: self.failUnlessEqual(ss.allocated_size(), 0)) + return d2 + d.addCallback(_allocated) + return d def test_allocate(self): ss = self.create("test_allocate") hunk ./src/allmydata/test/test_storage.py 497 - self.failUnlessEqual(ss.remote_get_buckets("allocate"), {}) + d = defer.succeed(None) + d.addCallback(lambda ign: ss.remote_get_buckets("allocate")) + d.addCallback(lambda res: self.failUnlessEqual(res, {})) hunk ./src/allmydata/test/test_storage.py 501 - already,writers = self.allocate(ss, "allocate", [0,1,2], 75) - self.failUnlessEqual(already, set()) - self.failUnlessEqual(set(writers.keys()), set([0,1,2])) + d.addCallback(lambda ign: self.allocate(ss, "allocate", [0,1,2], 75)) + def _allocated( (already, writers) ): + self.failUnlessEqual(already, set()) + self.failUnlessEqual(set(writers.keys()), set([0,1,2])) hunk ./src/allmydata/test/test_storage.py 506 - # while the buckets are open, they should not count as readable - self.failUnlessEqual(ss.remote_get_buckets("allocate"), {}) + # while the buckets are open, they should not count as readable + d2 = defer.succeed(None) + d2.addCallback(lambda ign: ss.remote_get_buckets("allocate")) + d2.addCallback(lambda res: self.failUnlessEqual(res, {})) hunk ./src/allmydata/test/test_storage.py 511 - # close the buckets - for i,wb in writers.items(): - wb.remote_write(0, "%25d" % i) - wb.remote_close() - # aborting a bucket that was already closed is a no-op - wb.remote_abort() + # close the buckets + for i, bw in writers.items(): + d2.addCallback(self._write_and_close, i, bw) + # aborting a bucket that was already closed is a no-op + d2.addCallback(self._abort_writer, i, bw) hunk ./src/allmydata/test/test_storage.py 517 - # now they should be readable - b = ss.remote_get_buckets("allocate") - self.failUnlessEqual(set(b.keys()), set([0,1,2])) - self.failUnlessEqual(b[0].remote_read(0, 25), "%25d" % 0) - b_str = str(b[0]) - self.failUnlessIn("BucketReader", b_str) - self.failUnlessIn("mfwgy33dmf2g 0", b_str) + # now they should be readable + d2.addCallback(lambda ign: ss.remote_get_buckets("allocate")) + def _got_buckets(b): + self.failUnlessEqual(set(b.keys()), set([0,1,2])) + b_str = str(b[0]) + self.failUnlessIn("BucketReader", b_str) + self.failUnlessIn("mfwgy33dmf2g 0", b_str) + + d3 = defer.succeed(None) + d3.addCallback(lambda ign: b[0].remote_read(0, 25)) + d3.addCallback(lambda res: self.failUnlessEqual(res, "%25d" % 0)) + return d3 + d2.addCallback(_got_buckets) + d.addCallback(_allocated) # now if we ask about writing again, the server should offer those # three buckets as already present. It should offer them even if we hunk ./src/allmydata/test/test_storage.py 535 # don't ask about those specific ones. - already,writers = self.allocate(ss, "allocate", [2,3,4], 75) - self.failUnlessEqual(already, set([0,1,2])) - self.failUnlessEqual(set(writers.keys()), set([3,4])) hunk ./src/allmydata/test/test_storage.py 536 - # while those two buckets are open for writing, the server should - # refuse to offer them to uploaders + d.addCallback(lambda ign: self.allocate(ss, "allocate", [2,3,4], 75)) + def _allocated_again( (already, writers) ): + self.failUnlessEqual(already, set([0,1,2])) + self.failUnlessEqual(set(writers.keys()), set([3,4])) hunk ./src/allmydata/test/test_storage.py 541 - already2,writers2 = self.allocate(ss, "allocate", [2,3,4,5], 75) - self.failUnlessEqual(already2, set([0,1,2])) - self.failUnlessEqual(set(writers2.keys()), set([5])) + # while those two buckets are open for writing, the server should + # refuse to offer them to uploaders hunk ./src/allmydata/test/test_storage.py 544 - # aborting the writes should remove the tempfiles - for i,wb in writers2.items(): - wb.remote_abort() - already2,writers2 = self.allocate(ss, "allocate", [2,3,4,5], 75) - self.failUnlessEqual(already2, set([0,1,2])) - self.failUnlessEqual(set(writers2.keys()), set([5])) + d2 = self.allocate(ss, "allocate", [2,3,4,5], 75) + def _allocated_again2( (already2, writers2) ): + self.failUnlessEqual(already2, set([0,1,2])) + self.failUnlessEqual(set(writers2.keys()), set([5])) hunk ./src/allmydata/test/test_storage.py 549 - for i,wb in writers2.items(): - wb.remote_abort() - for i,wb in writers.items(): - wb.remote_abort() + # aborting the writes should remove the tempfiles + return for_items(self._abort_writer, writers2) + d2.addCallback(_allocated_again2) hunk ./src/allmydata/test/test_storage.py 553 - def test_bad_container_version(self): - ss = self.create("test_bad_container_version") - a,w = self.allocate(ss, "si1", [0], 10) - w[0].remote_write(0, "\xff"*10) - w[0].remote_close() - - fn = os.path.join(ss.sharedir, storage_index_to_dir("si1"), "0") - f = open(fn, "rb+") - f.seek(0) - f.write(struct.pack(">L", 0)) # this is invalid: minimum used is v1 - f.close() + d2.addCallback(lambda ign: self.allocate(ss, "allocate", [2,3,4,5], 75)) + d2.addCallback(_allocated_again2) hunk ./src/allmydata/test/test_storage.py 556 - ss.remote_get_buckets("allocate") + d2.addCallback(lambda ign: for_items(self._abort_writer, writers)) + return d2 + d.addCallback(_allocated_again) + return d hunk ./src/allmydata/test/test_storage.py 561 - e = self.failUnlessRaises(UnknownImmutableContainerVersionError, - ss.remote_get_buckets, "si1") - self.failUnlessIn(" had version 0 but we wanted 1", str(e)) + # The following share file content was generated with + # storage.immutable.ShareFile from Tahoe-LAFS v1.8.2 + # with share data == 'a'. The total size of this input + # is 85 bytes. + shareversionnumber = '\x00\x00\x00\x01' + sharedatalength = '\x00\x00\x00\x01' + numberofleases = '\x00\x00\x00\x01' + shareinputdata = 'a' + ownernumber = '\x00\x00\x00\x00' + renewsecret = 'x'*32 + cancelsecret = 'y'*32 + expirationtime = '\x00(\xde\x80' + nextlease = '' + containerdata = shareversionnumber + sharedatalength + numberofleases + client_data = (shareinputdata + ownernumber + renewsecret + + cancelsecret + expirationtime + nextlease) + share_data = containerdata + client_data + testnodeid = 'testnodeidxxxxxxxxxx' hunk ./src/allmydata/test/test_storage.py 580 - def test_disconnect(self): - # simulate a disconnection - ss = self.create("test_disconnect") + def test_write_and_read_share(self): + """ + Write a new share, read it, and test the server's (and disk backend's) + handling of simultaneous and successive attempts to write the same + share. + """ + ss = self.create("test_write_and_read_share") canary = FakeCanary() hunk ./src/allmydata/test/test_storage.py 588 - already,writers = self.allocate(ss, "disconnect", [0,1,2], 75, canary) - self.failUnlessEqual(already, set()) - self.failUnlessEqual(set(writers.keys()), set([0,1,2])) - for (f,args,kwargs) in canary.disconnectors.values(): - f(*args, **kwargs) - del already - del writers hunk ./src/allmydata/test/test_storage.py 589 - # that ought to delete the incoming shares - already,writers = self.allocate(ss, "disconnect", [0,1,2], 75) - self.failUnlessEqual(already, set()) - self.failUnlessEqual(set(writers.keys()), set([0,1,2])) + shareset = ss.backend.get_shareset('teststorage_index') + self.failIf(shareset.has_incoming(0)) hunk ./src/allmydata/test/test_storage.py 592 - @mock.patch('allmydata.util.fileutil.get_disk_stats') - def test_reserved_space(self, mock_get_disk_stats): - reserved_space=10000 - mock_get_disk_stats.return_value = { - 'free_for_nonroot': 15000, - 'avail': max(15000 - reserved_space, 0), - } + # Populate incoming with the sharenum: 0. + d = ss.remote_allocate_buckets('teststorage_index', 'x'*32, 'y'*32, frozenset((0,)), 1, canary) + def _allocated( (already, writers) ): + # This is a white-box test: Inspect incoming and fail unless the sharenum: 0 is listed there. + self.failUnless(shareset.has_incoming(0)) hunk ./src/allmydata/test/test_storage.py 598 - ss = self.create("test_reserved_space", reserved_space=reserved_space) - # 15k available, 10k reserved, leaves 5k for shares + # Attempt to create a second share writer with the same sharenum. + d2 = ss.remote_allocate_buckets('teststorage_index', 'x'*32, 'y'*32, frozenset((0,)), 1, canary) hunk ./src/allmydata/test/test_storage.py 601 - # a newly created and filled share incurs this much overhead, beyond - # the size we request. - OVERHEAD = 3*4 - LEASE_SIZE = 4+32+32+4 - canary = FakeCanary(True) - already,writers = self.allocate(ss, "vid1", [0,1,2], 1000, canary) - self.failUnlessEqual(len(writers), 3) - # now the StorageServer should have 3000 bytes provisionally - # allocated, allowing only 2000 more to be claimed - self.failUnlessEqual(len(ss._active_writers), 3) - - # allocating 1001-byte shares only leaves room for one - already2,writers2 = self.allocate(ss, "vid2", [0,1,2], 1001, canary) - self.failUnlessEqual(len(writers2), 1) - self.failUnlessEqual(len(ss._active_writers), 4) - - # we abandon the first set, so their provisional allocation should be - # returned - del already - del writers - self.failUnlessEqual(len(ss._active_writers), 1) - # now we have a provisional allocation of 1001 bytes - - # and we close the second set, so their provisional allocation should - # become real, long-term allocation, and grows to include the - # overhead. - for bw in writers2.values(): - bw.remote_write(0, "a"*25) - bw.remote_close() - del already2 - del writers2 - del bw - self.failUnlessEqual(len(ss._active_writers), 0) - - allocated = 1001 + OVERHEAD + LEASE_SIZE - - # we have to manually increase available, since we're not doing real - # disk measurements - mock_get_disk_stats.return_value = { - 'free_for_nonroot': 15000 - allocated, - 'avail': max(15000 - allocated - reserved_space, 0), - } - - # now there should be ALLOCATED=1001+12+72=1085 bytes allocated, and - # 5000-1085=3915 free, therefore we can fit 39 100byte shares - already3,writers3 = self.allocate(ss,"vid3", range(100), 100, canary) - self.failUnlessEqual(len(writers3), 39) - self.failUnlessEqual(len(ss._active_writers), 39) - - del already3 - del writers3 - self.failUnlessEqual(len(ss._active_writers), 0) - ss.disownServiceParent() - del ss + # Show that no sharewriter results from a remote_allocate_buckets + # with the same si and sharenum, until BucketWriter.remote_close() + # has been called. + d2.addCallback(lambda (already2, writers2): self.failIf(writers2)) hunk ./src/allmydata/test/test_storage.py 606 - def test_seek(self): - basedir = self.workdir("test_seek_behavior") - fileutil.make_dirs(basedir) - filename = os.path.join(basedir, "testfile") - f = open(filename, "wb") - f.write("start") - f.close() - # mode="w" allows seeking-to-create-holes, but truncates pre-existing - # files. mode="a" preserves previous contents but does not allow - # seeking-to-create-holes. mode="r+" allows both. - f = open(filename, "rb+") - f.seek(100) - f.write("100") - f.close() - filelen = os.stat(filename)[stat.ST_SIZE] - self.failUnlessEqual(filelen, 100+3) - f2 = open(filename, "rb") - self.failUnlessEqual(f2.read(5), "start") - - - def test_leases(self): - ss = self.create("test_leases") - canary = FakeCanary() - sharenums = range(5) - size = 100 + # Test allocated size. + d2.addCallback(lambda ign: ss.allocated_size()) + d2.addCallback(lambda space: self.failUnlessEqual(space, 1)) hunk ./src/allmydata/test/test_storage.py 610 - rs0,cs0 = (hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()), - hashutil.tagged_hash("blah", "%d" % self._lease_secret.next())) - already,writers = ss.remote_allocate_buckets("si0", rs0, cs0, - sharenums, size, canary) - self.failUnlessEqual(len(already), 0) - self.failUnlessEqual(len(writers), 5) - for wb in writers.values(): - wb.remote_close() + # Write 'a' to shnum 0. Only tested together with close and read. + d2.addCallback(lambda ign: writers[0].remote_write(0, 'a')) hunk ./src/allmydata/test/test_storage.py 613 - leases = list(ss.get_leases("si0")) - self.failUnlessEqual(len(leases), 1) - self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs0])) + # Preclose: Inspect final, failUnless nothing there. + d2.addCallback(lambda ign: ss.backend.get_shareset('teststorage_index').get_shares()) + def _check( (shares, corrupted) ): + self.failUnlessEqual(len(shares), 0, str(shares)) + self.failUnlessEqual(len(corrupted), 0, str(corrupted)) + d2.addCallback(_check) hunk ./src/allmydata/test/test_storage.py 620 - rs1,cs1 = (hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()), - hashutil.tagged_hash("blah", "%d" % self._lease_secret.next())) - already,writers = ss.remote_allocate_buckets("si1", rs1, cs1, - sharenums, size, canary) - for wb in writers.values(): - wb.remote_close() + d2.addCallback(lambda ign: writers[0].remote_close()) hunk ./src/allmydata/test/test_storage.py 622 - # take out a second lease on si1 - rs2,cs2 = (hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()), - hashutil.tagged_hash("blah", "%d" % self._lease_secret.next())) - already,writers = ss.remote_allocate_buckets("si1", rs2, cs2, - sharenums, size, canary) - self.failUnlessEqual(len(already), 5) - self.failUnlessEqual(len(writers), 0) + # Postclose: fail unless written data is in final. + d2.addCallback(lambda ign: ss.backend.get_shareset('teststorage_index').get_shares()) + def _got_shares( (sharesinfinal, corrupted) ): + self.failUnlessEqual(len(sharesinfinal), 1, str(sharesinfinal)) + self.failUnlessEqual(len(corrupted), 0, str(corrupted)) hunk ./src/allmydata/test/test_storage.py 628 - leases = list(ss.get_leases("si1")) - self.failUnlessEqual(len(leases), 2) - self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs1, rs2])) + d3 = defer.succeed(None) + d3.addCallback(lambda ign: sharesinfinal[0].read_share_data(0, 73)) + d3.addCallback(lambda contents: self.failUnlessEqual(contents, self.shareinputdata)) + return d3 + d2.addCallback(_got_shares) hunk ./src/allmydata/test/test_storage.py 634 - # and a third lease, using add-lease - rs2a,cs2a = (hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()), - hashutil.tagged_hash("blah", "%d" % self._lease_secret.next())) - ss.remote_add_lease("si1", rs2a, cs2a) - leases = list(ss.get_leases("si1")) - self.failUnlessEqual(len(leases), 3) - self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs1, rs2, rs2a])) + # Exercise the case that the share we're asking to allocate is + # already (completely) uploaded. + d2.addCallback(lambda ign: ss.remote_allocate_buckets('teststorage_index', + 'x'*32, 'y'*32, set((0,)), 1, canary)) + return d2 + d.addCallback(_allocated) + return d hunk ./src/allmydata/test/test_storage.py 642 - # add-lease on a missing storage index is silently ignored - self.failUnlessEqual(ss.remote_add_lease("si18", "", ""), None) + def test_read_old_share(self): + """ + This tests whether the code correctly finds and reads shares written out by + pre-pluggable-backends (Tahoe-LAFS <= v1.8.2) servers. There is a similar test + in test_download, but that one is from the perspective of the client and exercises + a deeper stack of code. This one is for exercising just the StorageServer and backend. + """ + ss = self.create("test_read_old_share") hunk ./src/allmydata/test/test_storage.py 651 - # check that si0 is readable - readers = ss.remote_get_buckets("si0") - self.failUnlessEqual(len(readers), 5) + # Contruct a file with the appropriate contents. + datalen = len(self.share_data) + sharedir = ss.backend.get_shareset('teststorage_index')._get_sharedir() + fileutil.fp_make_dirs(sharedir) + sharedir.child("0").setContent(self.share_data) hunk ./src/allmydata/test/test_storage.py 657 - # renew the first lease. Only the proper renew_secret should work - ss.remote_renew_lease("si0", rs0) - self.failUnlessRaises(IndexError, ss.remote_renew_lease, "si0", cs0) - self.failUnlessRaises(IndexError, ss.remote_renew_lease, "si0", rs1) + # Now begin the test. + d = ss.remote_get_buckets('teststorage_index') + def _got_buckets(bs): + self.failUnlessEqual(len(bs), 1) + self.failUnlessIn(0, bs) + b = bs[0] hunk ./src/allmydata/test/test_storage.py 664 - # check that si0 is still readable - readers = ss.remote_get_buckets("si0") - self.failUnlessEqual(len(readers), 5) + d2 = defer.succeed(None) + d2.addCallback(lambda ign: b.remote_read(0, datalen)) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.shareinputdata)) hunk ./src/allmydata/test/test_storage.py 668 - # There is no such method as remote_cancel_lease for now -- see - # ticket #1528. - self.failIf(hasattr(ss, 'remote_cancel_lease'), \ - "ss should not have a 'remote_cancel_lease' method/attribute") + # If you try to read past the end you get as much input data as is there. + d2.addCallback(lambda ign: b.remote_read(0, datalen+20)) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.shareinputdata)) hunk ./src/allmydata/test/test_storage.py 672 - # test overlapping uploads - rs3,cs3 = (hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()), - hashutil.tagged_hash("blah", "%d" % self._lease_secret.next())) - rs4,cs4 = (hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()), - hashutil.tagged_hash("blah", "%d" % self._lease_secret.next())) - already,writers = ss.remote_allocate_buckets("si3", rs3, cs3, - sharenums, size, canary) - self.failUnlessEqual(len(already), 0) - self.failUnlessEqual(len(writers), 5) - already2,writers2 = ss.remote_allocate_buckets("si3", rs4, cs4, - sharenums, size, canary) - self.failUnlessEqual(len(already2), 0) - self.failUnlessEqual(len(writers2), 0) - for wb in writers.values(): - wb.remote_close() + # If you start reading past the end of the file you get the empty string. + d2.addCallback(lambda ign: b.remote_read(datalen+1, 3)) + d2.addCallback(lambda res: self.failUnlessEqual(res, '')) + return d2 + d.addCallback(_got_buckets) + return d hunk ./src/allmydata/test/test_storage.py 679 - leases = list(ss.get_leases("si3")) - self.failUnlessEqual(len(leases), 1) + def test_bad_container_version(self): + ss = self.create("test_bad_container_version") hunk ./src/allmydata/test/test_storage.py 682 - already3,writers3 = ss.remote_allocate_buckets("si3", rs4, cs4, - sharenums, size, canary) - self.failUnlessEqual(len(already3), 5) - self.failUnlessEqual(len(writers3), 0) + d = self.allocate(ss, "si1", [0,1], 25) + d.addCallback(lambda (already, writers): for_items(self._write_and_close, writers)) hunk ./src/allmydata/test/test_storage.py 685 - leases = list(ss.get_leases("si3")) - self.failUnlessEqual(len(leases), 2) + d.addCallback(lambda ign: ss.backend.get_shareset("si1").get_share(0)) + def _write_invalid_version(share0): + f = share0._get_filepath().open("rb+") + try: + f.seek(0) + f.write(struct.pack(">L", 0)) # this is invalid: minimum used is v1 + finally: + f.close() + d.addCallback(_write_invalid_version) hunk ./src/allmydata/test/test_storage.py 695 - def test_readonly(self): - workdir = self.workdir("test_readonly") - ss = StorageServer(workdir, "\x00" * 20, readonly_storage=True) - ss.setServiceParent(self.sparent) + # The corrupt shnum 0 should be ignored... + d.addCallback(lambda ign: ss.remote_get_buckets("si1")) + d.addCallback(lambda b: self.failUnlessEqual(set(b.keys()), set([1]))) hunk ./src/allmydata/test/test_storage.py 699 - already,writers = self.allocate(ss, "vid", [0,1,2], 75) - self.failUnlessEqual(already, set()) - self.failUnlessEqual(writers, {}) + # but the error should still be reported if we specifically ask for shnum 0. + d.addCallback(lambda ign: self.shouldFail(UnknownImmutableContainerVersionError, + 'bad_container_version', " had version 0 but we wanted 1", + lambda: ss.backend.get_shareset("si1").get_share(0) )) + return d hunk ./src/allmydata/test/test_storage.py 705 - stats = ss.get_stats() - self.failUnlessEqual(stats["storage_server.accepting_immutable_shares"], 0) - if "storage_server.disk_avail" in stats: - # Some platforms may not have an API to get disk stats. - # But if there are stats, readonly_storage means disk_avail=0 - self.failUnlessEqual(stats["storage_server.disk_avail"], 0) + def test_disconnect(self): + # simulate a disconnection + ss = self.create("test_disconnect") + canary = FakeCanary() hunk ./src/allmydata/test/test_storage.py 710 - def test_discard(self): - # discard is really only used for other tests, but we test it anyways - workdir = self.workdir("test_discard") - ss = StorageServer(workdir, "\x00" * 20, discard_storage=True) - ss.setServiceParent(self.sparent) + d = self.allocate(ss, "disconnect", [0,1,2], 75, canary) + def _allocated( (already, writers) ): + self.failUnlessEqual(already, set()) + self.failUnlessEqual(set(writers.keys()), set([0,1,2])) + for (f,args,kwargs) in canary.disconnectors.values(): + f(*args, **kwargs) + d.addCallback(_allocated) hunk ./src/allmydata/test/test_storage.py 718 - already,writers = self.allocate(ss, "vid", [0,1,2], 75) - self.failUnlessEqual(already, set()) - self.failUnlessEqual(set(writers.keys()), set([0,1,2])) - for i,wb in writers.items(): - wb.remote_write(0, "%25d" % i) - wb.remote_close() - # since we discard the data, the shares should be present but sparse. - # Since we write with some seeks, the data we read back will be all - # zeros. - b = ss.remote_get_buckets("vid") - self.failUnlessEqual(set(b.keys()), set([0,1,2])) - self.failUnlessEqual(b[0].remote_read(0, 25), "\x00" * 25) + # returning from _allocated ought to delete the incoming shares + d.addCallback(lambda ign: self.allocate(ss, "disconnect", [0,1,2], 75)) + def _allocated2( (already, writers) ): + self.failUnlessEqual(already, set()) + self.failUnlessEqual(set(writers.keys()), set([0,1,2])) + d.addCallback(_allocated2) + return d def test_advise_corruption(self): hunk ./src/allmydata/test/test_storage.py 727 - workdir = self.workdir("test_advise_corruption") - ss = StorageServer(workdir, "\x00" * 20, discard_storage=True) - ss.setServiceParent(self.sparent) + ss = self.create("test_advise_corruption") si0_s = base32.b2a("si0") ss.remote_advise_corrupt_share("immutable", "si0", 0, hunk ./src/allmydata/test/test_storage.py 732 "This share smells funny.\n") - reportdir = os.path.join(workdir, "corruption-advisories") - reports = os.listdir(reportdir) + reportdir = ss._statedir.child("corruption-advisories") + self.failUnless(reportdir.exists(), reportdir) + reports = [child.basename() for child in reportdir.children()] self.failUnlessEqual(len(reports), 1) report_si0 = reports[0] hunk ./src/allmydata/test/test_storage.py 737 - self.failUnlessIn(si0_s, report_si0) - f = open(os.path.join(reportdir, report_si0), "r") - report = f.read() - f.close() + self.failUnlessIn(si0_s, str(report_si0)) + report = reportdir.child(report_si0).getContent() + self.failUnlessIn("type: immutable", report) self.failUnlessIn("storage_index: %s" % si0_s, report) self.failUnlessIn("share_number: 0", report) hunk ./src/allmydata/test/test_storage.py 747 # test the RIBucketWriter version too si1_s = base32.b2a("si1") - already,writers = self.allocate(ss, "si1", [1], 75) - self.failUnlessEqual(already, set()) - self.failUnlessEqual(set(writers.keys()), set([1])) - writers[1].remote_write(0, "data") - writers[1].remote_close() + d = self.allocate(ss, "si1", [1], 75) + def _allocated( (already, writers) ): + self.failUnlessEqual(already, set()) + self.failUnlessEqual(set(writers.keys()), set([1])) hunk ./src/allmydata/test/test_storage.py 752 - b = ss.remote_get_buckets("si1") - self.failUnlessEqual(set(b.keys()), set([1])) - b[1].remote_advise_corrupt_share("This share tastes like dust.\n") + d2 = defer.succeed(None) + d2.addCallback(lambda ign: writers[1].remote_write(0, "data")) + d2.addCallback(lambda ign: writers[1].remote_close()) hunk ./src/allmydata/test/test_storage.py 756 - reports = os.listdir(reportdir) - self.failUnlessEqual(len(reports), 2) - report_si1 = [r for r in reports if si1_s in r][0] - f = open(os.path.join(reportdir, report_si1), "r") - report = f.read() - f.close() - self.failUnlessIn("type: immutable", report) - self.failUnlessIn("storage_index: %s" % si1_s, report) - self.failUnlessIn("share_number: 1", report) - self.failUnlessIn("This share tastes like dust.", report) + d2.addCallback(lambda ign: ss.remote_get_buckets("si1")) + def _got_buckets(b): + self.failUnlessEqual(set(b.keys()), set([1])) + b[1].remote_advise_corrupt_share("This share tastes like dust.\n") hunk ./src/allmydata/test/test_storage.py 761 + reports = [child.basename() for child in reportdir.children()] + self.failUnlessEqual(len(reports), 2) + report_si1 = [r for r in reports if si1_s in str(r)][0] + report = reportdir.child(report_si1).getContent() hunk ./src/allmydata/test/test_storage.py 766 + self.failUnlessIn("type: immutable", report) + self.failUnlessIn("storage_index: %s" % si1_s, report) + self.failUnlessIn("share_number: 1", report) + self.failUnlessIn("This share tastes like dust.", report) + d2.addCallback(_got_buckets) + return d2 + d.addCallback(_allocated) + return d hunk ./src/allmydata/test/test_storage.py 775 -class MutableServer(unittest.TestCase): hunk ./src/allmydata/test/test_storage.py 776 +class MutableServerMixin: def setUp(self): self.sparent = LoggingServiceParent() hunk ./src/allmydata/test/test_storage.py 779 + self.sparent.startService() self._lease_secret = itertools.count() def tearDown(self): return self.sparent.stopService() hunk ./src/allmydata/test/test_storage.py 785 def workdir(self, name): - basedir = os.path.join("storage", "MutableServer", name) - return basedir - - def create(self, name): - workdir = self.workdir(name) - ss = StorageServer(workdir, "\x00" * 20) - ss.setServiceParent(self.sparent) - return ss - - def test_create(self): - self.create("test_create") + return FilePath("storage").child(self.__class__.__name__).child(name) def write_enabler(self, we_tag): return hashutil.tagged_hash("we_blah", we_tag) hunk ./src/allmydata/test/test_storage.py 802 cancel_secret = self.cancel_secret(lease_tag) rstaraw = ss.remote_slot_testv_and_readv_and_writev testandwritev = dict( [ (shnum, ([], [], None) ) - for shnum in sharenums ] ) + for shnum in sharenums ] ) readv = [] hunk ./src/allmydata/test/test_storage.py 804 - rc = rstaraw(storage_index, - (write_enabler, renew_secret, cancel_secret), - testandwritev, - readv) - (did_write, readv_data) = rc - self.failUnless(did_write) - self.failUnless(isinstance(readv_data, dict)) - self.failUnlessEqual(len(readv_data), 0) hunk ./src/allmydata/test/test_storage.py 805 + d = defer.succeed(None) + d.addCallback(lambda ign: rstaraw(storage_index, + (write_enabler, renew_secret, cancel_secret), + testandwritev, + readv)) + def _check( (did_write, readv_data) ): + self.failUnless(did_write) + self.failUnless(isinstance(readv_data, dict)) + self.failUnlessEqual(len(readv_data), 0) + d.addCallback(_check) + return d + + +class MutableServerTest(MutableServerMixin, ShouldFailMixin): def test_bad_magic(self): ss = self.create("test_bad_magic") hunk ./src/allmydata/test/test_storage.py 821 - self.allocate(ss, "si1", "we1", self._lease_secret.next(), set([0]), 10) - fn = os.path.join(ss.sharedir, storage_index_to_dir("si1"), "0") - f = open(fn, "rb+") - f.seek(0) - f.write("BAD MAGIC") - f.close() read = ss.remote_slot_readv hunk ./src/allmydata/test/test_storage.py 822 - e = self.failUnlessRaises(UnknownMutableContainerVersionError, - read, "si1", [0], [(0,10)]) - self.failUnlessIn(" had magic ", str(e)) - self.failUnlessIn(" but we wanted ", str(e)) + + d = self.allocate(ss, "si1", "we1", self._lease_secret.next(), set([0,1]), 25) + d.addCallback(lambda ign: ss.backend.get_shareset("si1").get_share(0)) + def _write_bad_magic(share0): + f = share0._get_filepath().open("rb+") + try: + f.seek(0) + f.write("BAD MAGIC") + finally: + f.close() + d.addCallback(_write_bad_magic) + + # The corrupt shnum 0 should be ignored when we read shnum 1... + d.addCallback(lambda ign: read("si1", [1], [(0,25)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {1: [""]})) + + # but the error should still be reported if we specifically ask for shnum 0. + # This used to test for UnknownMutableContainerVersionError, + # but the current code raises UnknownImmutableContainerVersionError. + # (It changed because remote_slot_readv now works with either + # mutable or immutable shares.) Since the share file doesn't have + # the mutable magic, it's not clear that this is wrong. + # For now, accept either exception. + d.addCallback(lambda ign: + self.shouldFail(UnknownContainerVersionError, "bad_magic", + " but we wanted ", + lambda: read("si1", [0], [(0,25)]) )) + return d + test_bad_magic.todo = "Error reporting for corrupt shares doesn't currently work." def test_container_size(self): ss = self.create("test_container_size") hunk ./src/allmydata/test/test_storage.py 854 - self.allocate(ss, "si1", "we1", self._lease_secret.next(), - set([0,1,2]), 100) read = ss.remote_slot_readv rstaraw = ss.remote_slot_testv_and_readv_and_writev secrets = ( self.write_enabler("we1"), hunk ./src/allmydata/test/test_storage.py 860 self.renew_secret("we1"), self.cancel_secret("we1") ) data = "".join([ ("%d" % i) * 10 for i in range(10) ]) - answer = rstaraw("si1", secrets, - {0: ([], [(0,data)], len(data)+12)}, - []) - self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) ) + + d = self.allocate(ss, "si1", "we1", self._lease_secret.next(), + set([0,1,2]), 100) + d.addCallback(lambda ign: rstaraw("si1", secrets, + {0: ([], [(0,data)], len(data)+12)}, + [])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {0:[],1:[],2:[]}) )) # Trying to make the container too large (by sending a write vector # whose offset is too high) will raise an exception. hunk ./src/allmydata/test/test_storage.py 870 - TOOBIG = MutableShareFile.MAX_SIZE + 10 - self.failUnlessRaises(DataTooLargeError, - rstaraw, "si1", secrets, - {0: ([], [(TOOBIG,data)], None)}, - []) + TOOBIG = MutableDiskShare.MAX_SIZE + 10 + d.addCallback(lambda ign: self.shouldFail(DataTooLargeError, + 'make container too large', None, + lambda: rstaraw("si1", secrets, + {0: ([], [(TOOBIG,data)], None)}, + []) )) hunk ./src/allmydata/test/test_storage.py 877 - answer = rstaraw("si1", secrets, - {0: ([], [(0,data)], None)}, - []) - self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) ) + d.addCallback(lambda ign: rstaraw("si1", secrets, + {0: ([], [(0,data)], None)}, + [])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {0:[],1:[],2:[]}) )) hunk ./src/allmydata/test/test_storage.py 882 - read_answer = read("si1", [0], [(0,10)]) - self.failUnlessEqual(read_answer, {0: [data[:10]]}) + d.addCallback(lambda ign: read("si1", [0], [(0,10)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [data[:10]]})) # Sending a new_length shorter than the current length truncates the # data. hunk ./src/allmydata/test/test_storage.py 887 - answer = rstaraw("si1", secrets, - {0: ([], [], 9)}, - []) - read_answer = read("si1", [0], [(0,10)]) - self.failUnlessEqual(read_answer, {0: [data[:9]]}) + d.addCallback(lambda ign: rstaraw("si1", secrets, + {0: ([], [], 9)}, + [])) + d.addCallback(lambda ign: read("si1", [0], [(0,10)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [data[:9]]})) # Sending a new_length longer than the current length doesn't change # the data. hunk ./src/allmydata/test/test_storage.py 895 - answer = rstaraw("si1", secrets, - {0: ([], [], 20)}, - []) - assert answer == (True, {0:[],1:[],2:[]}) - read_answer = read("si1", [0], [(0, 20)]) - self.failUnlessEqual(read_answer, {0: [data[:9]]}) + d.addCallback(lambda ign: rstaraw("si1", secrets, + {0: ([], [], 20)}, + [])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {0:[],1:[],2:[]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0, 20)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [data[:9]]})) # Sending a write vector whose start is after the end of the current # data doesn't reveal "whatever was there last time" (palimpsest), hunk ./src/allmydata/test/test_storage.py 908 # To test this, we fill the data area with a recognizable pattern. pattern = ''.join([chr(i) for i in range(100)]) - answer = rstaraw("si1", secrets, - {0: ([], [(0, pattern)], None)}, - []) - assert answer == (True, {0:[],1:[],2:[]}) + d.addCallback(lambda ign: rstaraw("si1", secrets, + {0: ([], [(0, pattern)], None)}, + [])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {0:[],1:[],2:[]}) )) # Then truncate the data... hunk ./src/allmydata/test/test_storage.py 913 - answer = rstaraw("si1", secrets, - {0: ([], [], 20)}, - []) - assert answer == (True, {0:[],1:[],2:[]}) + d.addCallback(lambda ign: rstaraw("si1", secrets, + {0: ([], [], 20)}, + [])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {0:[],1:[],2:[]}) )) # Just confirm that you get an empty string if you try to read from # past the (new) endpoint now. hunk ./src/allmydata/test/test_storage.py 919 - answer = rstaraw("si1", secrets, - {0: ([], [], None)}, - [(20, 1980)]) - self.failUnlessEqual(answer, (True, {0:[''],1:[''],2:['']})) + d.addCallback(lambda ign: rstaraw("si1", secrets, + {0: ([], [], None)}, + [(20, 1980)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {0:[''],1:[''],2:['']}) )) # Then the extend the file by writing a vector which starts out past # the end... hunk ./src/allmydata/test/test_storage.py 926 - answer = rstaraw("si1", secrets, - {0: ([], [(50, 'hellothere')], None)}, - []) - assert answer == (True, {0:[],1:[],2:[]}) + d.addCallback(lambda ign: rstaraw("si1", secrets, + {0: ([], [(50, 'hellothere')], None)}, + [])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {0:[],1:[],2:[]}) )) # Now if you read the stuff between 20 (where we earlier truncated) # and 50, it had better be all zeroes. hunk ./src/allmydata/test/test_storage.py 932 - answer = rstaraw("si1", secrets, - {0: ([], [], None)}, - [(20, 30)]) - self.failUnlessEqual(answer, (True, {0:['\x00'*30],1:[''],2:['']})) + d.addCallback(lambda ign: rstaraw("si1", secrets, + {0: ([], [], None)}, + [(20, 30)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {0:['\x00'*30],1:[''],2:['']}) )) # Also see if the server explicitly declares that it supports this # feature. hunk ./src/allmydata/test/test_storage.py 939 - ver = ss.remote_get_version() - storage_v1_ver = ver["http://allmydata.org/tahoe/protocols/storage/v1"] - self.failUnless(storage_v1_ver.get("fills-holes-with-zero-bytes")) + d.addCallback(lambda ign: ss.remote_get_version()) + def _check_declaration(ver): + storage_v1_ver = ver["http://allmydata.org/tahoe/protocols/storage/v1"] + self.failUnless(storage_v1_ver.get("fills-holes-with-zero-bytes")) + d.addCallback(_check_declaration) # If the size is dropped to zero the share is deleted. hunk ./src/allmydata/test/test_storage.py 946 - answer = rstaraw("si1", secrets, - {0: ([], [(0,data)], 0)}, - []) - self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) ) + d.addCallback(lambda ign: rstaraw("si1", secrets, + {0: ([], [(0,data)], 0)}, + [])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {0:[],1:[],2:[]}) )) hunk ./src/allmydata/test/test_storage.py 951 - read_answer = read("si1", [0], [(0,10)]) - self.failUnlessEqual(read_answer, {}) + d.addCallback(lambda ign: read("si1", [0], [(0,10)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {})) + return d def test_allocate(self): ss = self.create("test_allocate") hunk ./src/allmydata/test/test_storage.py 957 - self.allocate(ss, "si1", "we1", self._lease_secret.next(), - set([0,1,2]), 100) - read = ss.remote_slot_readv hunk ./src/allmydata/test/test_storage.py 958 - self.failUnlessEqual(read("si1", [0], [(0, 10)]), - {0: [""]}) - self.failUnlessEqual(read("si1", [], [(0, 10)]), - {0: [""], 1: [""], 2: [""]}) - self.failUnlessEqual(read("si1", [0], [(100, 10)]), - {0: [""]}) + write = ss.remote_slot_testv_and_readv_and_writev + + d = self.allocate(ss, "si1", "we1", self._lease_secret.next(), + set([0,1,2]), 100) + + d.addCallback(lambda ign: read("si1", [0], [(0, 10)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [""]})) + d.addCallback(lambda ign: read("si1", [], [(0, 10)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [""], 1: [""], 2: [""]})) + d.addCallback(lambda ign: read("si1", [0], [(100, 10)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [""]})) # try writing to one secrets = ( self.write_enabler("we1"), hunk ./src/allmydata/test/test_storage.py 975 self.renew_secret("we1"), self.cancel_secret("we1") ) data = "".join([ ("%d" % i) * 10 for i in range(10) ]) - write = ss.remote_slot_testv_and_readv_and_writev - answer = write("si1", secrets, - {0: ([], [(0,data)], None)}, - []) - self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) ) hunk ./src/allmydata/test/test_storage.py 976 - self.failUnlessEqual(read("si1", [0], [(0,20)]), - {0: ["00000000001111111111"]}) - self.failUnlessEqual(read("si1", [0], [(95,10)]), - {0: ["99999"]}) - #self.failUnlessEqual(s0.remote_get_length(), 100) + d.addCallback(lambda ign: write("si1", secrets, + {0: ([], [(0,data)], None)}, + [])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {0:[],1:[],2:[]}) )) + + d.addCallback(lambda ign: read("si1", [0], [(0,20)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: ["00000000001111111111"]})) + d.addCallback(lambda ign: read("si1", [0], [(95,10)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: ["99999"]})) + #d.addCallback(lambda ign: s0.remote_get_length()) + #d.addCallback(lambda res: self.failUnlessEqual(res, 100)) bad_secrets = ("bad write enabler", secrets[1], secrets[2]) hunk ./src/allmydata/test/test_storage.py 989 - f = self.failUnlessRaises(BadWriteEnablerError, - write, "si1", bad_secrets, - {}, []) - self.failUnlessIn("The write enabler was recorded by nodeid 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'.", f) + d.addCallback(lambda ign: self.shouldFail(BadWriteEnablerError, 'bad write enabler', + "The write enabler was recorded by nodeid " + "'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'.", + lambda: write("si1", bad_secrets, {}, []) )) # this testv should fail hunk ./src/allmydata/test/test_storage.py 995 - answer = write("si1", secrets, - {0: ([(0, 12, "eq", "444444444444"), - (20, 5, "eq", "22222"), - ], - [(0, "x"*100)], - None), - }, - [(0,12), (20,5)], - ) - self.failUnlessEqual(answer, (False, - {0: ["000000000011", "22222"], - 1: ["", ""], - 2: ["", ""], - })) - self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]}) + d.addCallback(lambda ign: write("si1", secrets, + {0: ([(0, 12, "eq", "444444444444"), + (20, 5, "eq", "22222"),], + [(0, "x"*100)], + None)}, + [(0,12), (20,5)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (False, + {0: ["000000000011", "22222"], + 1: ["", ""], + 2: ["", ""]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [data]})) # as should this one hunk ./src/allmydata/test/test_storage.py 1009 - answer = write("si1", secrets, - {0: ([(10, 5, "lt", "11111"), - ], - [(0, "x"*100)], - None), - }, - [(10,5)], - ) - self.failUnlessEqual(answer, (False, - {0: ["11111"], - 1: [""], - 2: [""]}, - )) - self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]}) - + d.addCallback(lambda ign: write("si1", secrets, + {0: ([(10, 5, "lt", "11111"),], + [(0, "x"*100)], + None)}, + [(10,5)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (False, + {0: ["11111"], + 1: [""], + 2: [""]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [data]})) + return d def test_operators(self): # test operators, the data we're comparing is '11111' in all cases. hunk ./src/allmydata/test/test_storage.py 1034 write = ss.remote_slot_testv_and_readv_and_writev read = ss.remote_slot_readv - def reset(): - write("si1", secrets, - {0: ([], [(0,data)], None)}, - []) + def _reset(ign): + return write("si1", secrets, + {0: ([], [(0,data)], None)}, + []) hunk ./src/allmydata/test/test_storage.py 1039 - reset() + d = defer.succeed(None) + d.addCallback(_reset) # lt hunk ./src/allmydata/test/test_storage.py 1043 - answer = write("si1", secrets, {0: ([(10, 5, "lt", "11110"), - ], - [(0, "x"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (False, {0: ["11111"]})) - self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]}) - self.failUnlessEqual(read("si1", [], [(0,100)]), {0: [data]}) - reset() + d.addCallback(lambda ign: write("si1", secrets, {0: ([(10, 5, "lt", "11110"),], + [(0, "x"*100)], + None, + )}, [(10,5)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (False, {0: ["11111"]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [data]})) + d.addCallback(lambda ign: read("si1", [], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [data]})) + d.addCallback(_reset) hunk ./src/allmydata/test/test_storage.py 1054 - answer = write("si1", secrets, {0: ([(10, 5, "lt", "11111"), - ], - [(0, "x"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (False, {0: ["11111"]})) - self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]}) - reset() + d.addCallback(lambda ign: write("si1", secrets, {0: ([(10, 5, "lt", "11111"),], + [(0, "x"*100)], + None, + )}, [(10,5)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (False, {0: ["11111"]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [data]})) + d.addCallback(_reset) hunk ./src/allmydata/test/test_storage.py 1063 - answer = write("si1", secrets, {0: ([(10, 5, "lt", "11112"), - ], - [(0, "y"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (True, {0: ["11111"]})) - self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]}) - reset() + d.addCallback(lambda ign: write("si1", secrets, {0: ([(10, 5, "lt", "11112"),], + [(0, "y"*100)], + None, + )}, [(10,5)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {0: ["11111"]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: ["y"*100]})) + d.addCallback(_reset) # le hunk ./src/allmydata/test/test_storage.py 1073 - answer = write("si1", secrets, {0: ([(10, 5, "le", "11110"), - ], - [(0, "x"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (False, {0: ["11111"]})) - self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]}) - reset() + d.addCallback(lambda ign: write("si1", secrets, {0: ([(10, 5, "le", "11110"),], + [(0, "x"*100)], + None, + )}, [(10,5)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (False, {0: ["11111"]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [data]})) + d.addCallback(_reset) hunk ./src/allmydata/test/test_storage.py 1082 - answer = write("si1", secrets, {0: ([(10, 5, "le", "11111"), - ], - [(0, "y"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (True, {0: ["11111"]})) - self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]}) - reset() + d.addCallback(lambda ign: write("si1", secrets, {0: ([(10, 5, "le", "11111"),], + [(0, "y"*100)], + None, + )}, [(10,5)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {0: ["11111"]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: ["y"*100]})) + d.addCallback(_reset) hunk ./src/allmydata/test/test_storage.py 1091 - answer = write("si1", secrets, {0: ([(10, 5, "le", "11112"), - ], - [(0, "y"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (True, {0: ["11111"]})) - self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]}) - reset() + d.addCallback(lambda ign: write("si1", secrets, {0: ([(10, 5, "le", "11112"),], + [(0, "y"*100)], + None, + )}, [(10,5)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {0: ["11111"]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: ["y"*100]})) + d.addCallback(_reset) # eq hunk ./src/allmydata/test/test_storage.py 1101 - answer = write("si1", secrets, {0: ([(10, 5, "eq", "11112"), - ], - [(0, "x"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (False, {0: ["11111"]})) - self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]}) - reset() + d.addCallback(lambda ign: write("si1", secrets, {0: ([(10, 5, "eq", "11112"),], + [(0, "x"*100)], + None, + )}, [(10,5)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (False, {0: ["11111"]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [data]})) + d.addCallback(_reset) hunk ./src/allmydata/test/test_storage.py 1110 - answer = write("si1", secrets, {0: ([(10, 5, "eq", "11111"), - ], - [(0, "y"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (True, {0: ["11111"]})) - self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]}) - reset() + d.addCallback(lambda ign: write("si1", secrets, {0: ([(10, 5, "eq", "11111"),], + [(0, "y"*100)], + None, + )}, [(10,5)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {0: ["11111"]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: ["y"*100]})) + d.addCallback(_reset) # ne hunk ./src/allmydata/test/test_storage.py 1120 - answer = write("si1", secrets, {0: ([(10, 5, "ne", "11111"), - ], - [(0, "x"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (False, {0: ["11111"]})) - self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]}) - reset() + d.addCallback(lambda ign: write("si1", secrets, {0: ([(10, 5, "ne", "11111"),], + [(0, "x"*100)], + None, + )}, [(10,5)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (False, {0: ["11111"]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [data]})) + d.addCallback(_reset) hunk ./src/allmydata/test/test_storage.py 1129 - answer = write("si1", secrets, {0: ([(10, 5, "ne", "11112"), - ], - [(0, "y"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (True, {0: ["11111"]})) - self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]}) - reset() + d.addCallback(lambda ign: write("si1", secrets, {0: ([(10, 5, "ne", "11112"),], + [(0, "y"*100)], + None, + )}, [(10,5)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {0: ["11111"]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: ["y"*100]})) + d.addCallback(_reset) # ge hunk ./src/allmydata/test/test_storage.py 1139 - answer = write("si1", secrets, {0: ([(10, 5, "ge", "11110"), - ], - [(0, "y"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (True, {0: ["11111"]})) - self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]}) - reset() + d.addCallback(lambda ign: write("si1", secrets, {0: ([(10, 5, "ge", "11110"),], + [(0, "y"*100)], + None, + )}, [(10,5)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {0: ["11111"]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: ["y"*100]})) + d.addCallback(_reset) hunk ./src/allmydata/test/test_storage.py 1148 - answer = write("si1", secrets, {0: ([(10, 5, "ge", "11111"), - ], - [(0, "y"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (True, {0: ["11111"]})) - self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]}) - reset() + d.addCallback(lambda ign: write("si1", secrets, {0: ([(10, 5, "ge", "11111"),], + [(0, "y"*100)], + None, + )}, [(10,5)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {0: ["11111"]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: ["y"*100]})) + d.addCallback(_reset) hunk ./src/allmydata/test/test_storage.py 1157 - answer = write("si1", secrets, {0: ([(10, 5, "ge", "11112"), - ], - [(0, "y"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (False, {0: ["11111"]})) - self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]}) - reset() + d.addCallback(lambda ign: write("si1", secrets, {0: ([(10, 5, "ge", "11112"),], + [(0, "y"*100)], + None, + )}, [(10,5)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (False, {0: ["11111"]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [data]})) + d.addCallback(_reset) # gt hunk ./src/allmydata/test/test_storage.py 1167 - answer = write("si1", secrets, {0: ([(10, 5, "gt", "11110"), - ], - [(0, "y"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (True, {0: ["11111"]})) - self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]}) - reset() + d.addCallback(lambda ign: write("si1", secrets, {0: ([(10, 5, "gt", "11110"),], + [(0, "y"*100)], + None, + )}, [(10,5)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {0: ["11111"]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: ["y"*100]})) + d.addCallback(_reset) hunk ./src/allmydata/test/test_storage.py 1176 - answer = write("si1", secrets, {0: ([(10, 5, "gt", "11111"), - ], - [(0, "x"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (False, {0: ["11111"]})) - self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]}) - reset() + d.addCallback(lambda ign: write("si1", secrets, {0: ([(10, 5, "gt", "11111"),], + [(0, "x"*100)], + None, + )}, [(10,5)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (False, {0: ["11111"]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [data]})) + d.addCallback(_reset) hunk ./src/allmydata/test/test_storage.py 1185 - answer = write("si1", secrets, {0: ([(10, 5, "gt", "11112"), - ], - [(0, "x"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (False, {0: ["11111"]})) - self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]}) - reset() + d.addCallback(lambda ign: write("si1", secrets, {0: ([(10, 5, "gt", "11112"),], + [(0, "x"*100)], + None, + )}, [(10,5)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (False, {0: ["11111"]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [data]})) + d.addCallback(_reset) # finally, test some operators against empty shares hunk ./src/allmydata/test/test_storage.py 1195 - answer = write("si1", secrets, {1: ([(10, 5, "eq", "11112"), - ], - [(0, "x"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (False, {0: ["11111"]})) - self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]}) - reset() + d.addCallback(lambda ign: write("si1", secrets, {1: ([(10, 5, "eq", "11112"),], + [(0, "x"*100)], + None, + )}, [(10,5)])) + d.addCallback(lambda res: self.failUnlessEqual(res, (False, {0: ["11111"]}) )) + d.addCallback(lambda ign: read("si1", [0], [(0,100)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [data]})) + d.addCallback(_reset) + return d def test_readv(self): ss = self.create("test_readv") hunk ./src/allmydata/test/test_storage.py 1214 write = ss.remote_slot_testv_and_readv_and_writev read = ss.remote_slot_readv data = [("%d" % i) * 100 for i in range(3)] - rc = write("si1", secrets, - {0: ([], [(0,data[0])], None), - 1: ([], [(0,data[1])], None), - 2: ([], [(0,data[2])], None), - }, []) - self.failUnlessEqual(rc, (True, {})) hunk ./src/allmydata/test/test_storage.py 1215 - answer = read("si1", [], [(0, 10)]) - self.failUnlessEqual(answer, {0: ["0"*10], - 1: ["1"*10], - 2: ["2"*10]}) + d = defer.succeed(None) + d.addCallback(lambda ign: write("si1", secrets, + {0: ([], [(0,data[0])], None), + 1: ([], [(0,data[1])], None), + 2: ([], [(0,data[2])], None), + }, [])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {}) )) + + d.addCallback(lambda ign: read("si1", [], [(0, 10)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: ["0"*10], + 1: ["1"*10], + 2: ["2"*10]})) + return d + + def test_remove(self): + ss = self.create("test_remove") + readv = ss.remote_slot_readv + writev = ss.remote_slot_testv_and_readv_and_writev + secrets = ( self.write_enabler("we1"), + self.renew_secret("we1"), + self.cancel_secret("we1") ) + + d = defer.succeed(None) + d.addCallback(lambda ign: self.allocate(ss, "si1", "we1", self._lease_secret.next(), + set([0,1,2]), 100)) + # delete sh0 by setting its size to zero + d.addCallback(lambda ign: writev("si1", secrets, + {0: ([], [], 0)}, + [])) + # the answer should mention all the shares that existed before the + # write + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {0:[],1:[],2:[]}) )) + # but a new read should show only sh1 and sh2 + d.addCallback(lambda ign: readv("si1", [], [(0,10)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {1: [""], 2: [""]})) + + # delete sh1 by setting its size to zero + d.addCallback(lambda ign: writev("si1", secrets, + {1: ([], [], 0)}, + [])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {1:[],2:[]}) )) + d.addCallback(lambda ign: readv("si1", [], [(0,10)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {2: [""]})) + + # delete sh2 by setting its size to zero + d.addCallback(lambda ign: writev("si1", secrets, + {2: ([], [], 0)}, + [])) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {2:[]}) )) + d.addCallback(lambda ign: readv("si1", [], [(0,10)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {})) + + d.addCallback(lambda ign: ss.backend.get_shareset("si1").get_overhead()) + d.addCallback(lambda overhead: self.failUnlessEqual(overhead, 0)) + + # and the shareset directory should now be gone. This check is only + # applicable to the disk backend. + def _check_gone(ign): + si = base32.b2a("si1") + # note: this is a detail of the disk backend, and may change in the future + prefix = si[:2] + prefixdir = self.workdir("test_remove").child("shares").child(prefix) + sidir = prefixdir.child(si) + self.failUnless(prefixdir.exists(), prefixdir) + self.failIf(sidir.exists(), sidir) + + if isinstance(ss.backend, DiskBackend): + d.addCallback(_check_gone) + return d + + +class ServerWithNullBackend(ServerMixin, unittest.TestCase): + def test_null_backend(self): + workdir = self.workdir("test_null_backend") + backend = NullBackend() + ss = StorageServer("\x00" * 20, backend, workdir) + ss.setServiceParent(self.sparent) + + d = self.allocate(ss, "vid", [0,1,2], 75) + def _allocated( (already, writers) ): + self.failUnlessEqual(already, set()) + self.failUnlessEqual(set(writers.keys()), set([0,1,2])) + + d2 = for_items(self._write_and_close, writers) + + # The shares should be present but have no data. + d2.addCallback(lambda ign: ss.remote_get_buckets("vid")) + def _check(buckets): + self.failUnlessEqual(set(buckets.keys()), set([0,1,2])) + d3 = defer.succeed(None) + d3.addCallback(lambda ign: buckets[0].remote_read(0, 25)) + d3.addCallback(lambda res: self.failUnlessEqual(res, "")) + return d3 + d2.addCallback(_check) + return d2 + d.addCallback(_allocated) + return d + + +class CreateS3Backend: + def create(self, name, readonly=False, reserved_space=0, klass=StorageServer): + assert not readonly + workdir = self.workdir(name) + s3bucket = MockS3Bucket(workdir) + corruption_advisory_dir = workdir.child("corruption-advisories") + backend = S3Backend(s3bucket, corruption_advisory_dir=corruption_advisory_dir) + ss = klass("\x00" * 20, backend, workdir, + stats_provider=FakeStatsProvider()) + ss.setServiceParent(self.sparent) + return ss + + +class CreateDiskBackend: + def create(self, name, readonly=False, reserved_space=0, klass=StorageServer): + workdir = self.workdir(name) + backend = DiskBackend(workdir, readonly=readonly, reserved_space=reserved_space) + ss = klass("\x00" * 20, backend, workdir, + stats_provider=FakeStatsProvider()) + ss.setServiceParent(self.sparent) + return ss + + +class ServerWithS3Backend(ServerTest, CreateS3Backend, unittest.TestCase): + def test_bad_container_version(self): + return ServerTest.test_bad_container_version(self) + test_bad_container_version.todo = "The S3 backend doesn't pass this test." + + + +class ServerWithDiskBackend(ServerTest, CreateDiskBackend, unittest.TestCase): + + # The following tests are for behaviour that is only supported by a disk backend. + + def test_readonly(self): + ss = self.create("test_readonly", readonly=True) + + d = self.allocate(ss, "vid", [0,1,2], 75) + def _allocated( (already, writers) ): + self.failUnlessEqual(already, set()) + self.failUnlessEqual(writers, {}) + + stats = ss.get_stats() + self.failUnlessEqual(stats["storage_server.accepting_immutable_shares"], 0) + if "storage_server.disk_avail" in stats: + # Some platforms may not have an API to get disk stats. + # But if there are stats, readonly_storage means disk_avail=0 + self.failUnlessEqual(stats["storage_server.disk_avail"], 0) + d.addCallback(_allocated) + return d + + def test_large_share(self): + syslow = platform.system().lower() + if 'cygwin' in syslow or 'windows' in syslow or 'darwin' in syslow: + raise unittest.SkipTest("If your filesystem doesn't support efficient sparse files then it is very expensive (Mac OS X and Windows don't support efficient sparse files).") + + avail = fileutil.get_available_space(FilePath('.'), 512*2**20) + if avail <= 4*2**30: + raise unittest.SkipTest("This test will spuriously fail if you have less than 4 GiB free on your filesystem.") + + ss = self.create("test_large_share") + + d = self.allocate(ss, "allocate", [0], 2**32+2) + def _allocated( (already, writers) ): + self.failUnlessEqual(already, set()) + self.failUnlessEqual(set(writers.keys()), set([0])) + + shnum, bucket = writers.items()[0] + + # This test is going to hammer your filesystem if it doesn't make a sparse file for this. :-( + d2 = defer.succeed(None) + d2.addCallback(lambda ign: bucket.remote_write(2**32, "ab")) + d2.addCallback(lambda ign: bucket.remote_close()) + + d2.addCallback(lambda ign: ss.remote_get_buckets("allocate")) + d2.addCallback(lambda readers: readers[shnum].remote_read(2**32, 2)) + d2.addCallback(lambda res: self.failUnlessEqual(res, "ab")) + return d2 + d.addCallback(_allocated) + return d + + def test_immutable_leases(self): + ss = self.create("test_immutable_leases") + canary = FakeCanary() + sharenums = range(5) + size = 100 + + rs = [] + cs = [] + for i in range(6): + rs.append(hashutil.tagged_hash("blah", "%d" % self._lease_secret.next())) + cs.append(hashutil.tagged_hash("blah", "%d" % self._lease_secret.next())) + + d = ss.remote_allocate_buckets("si0", rs[0], cs[0], + sharenums, size, canary) + def _allocated( (already, writers) ): + self.failUnlessEqual(len(already), 0) + self.failUnlessEqual(len(writers), 5) + + d2 = for_items(self._close_writer, writers) + + d2.addCallback(lambda ign: list(ss.get_leases("si0"))) + def _check_leases(leases): + self.failUnlessEqual(len(leases), 1) + self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs[0]])) + d2.addCallback(_check_leases) + + d2.addCallback(lambda ign: ss.remote_allocate_buckets("si1", rs[1], cs[1], + sharenums, size, canary)) + return d2 + d.addCallback(_allocated) + + def _allocated2( (already, writers) ): + d2 = for_items(self._close_writer, writers) + + # take out a second lease on si1 + d2.addCallback(lambda ign: ss.remote_allocate_buckets("si1", rs[2], cs[2], + sharenums, size, canary)) + return d2 + d.addCallback(_allocated2) + + def _allocated2a( (already, writers) ): + self.failUnlessEqual(len(already), 5) + self.failUnlessEqual(len(writers), 0) + + d2 = defer.succeed(None) + d2.addCallback(lambda ign: list(ss.get_leases("si1"))) + def _check_leases2(leases): + self.failUnlessEqual(len(leases), 2) + self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs[1], rs[2]])) + d2.addCallback(_check_leases2) + + # and a third lease, using add-lease + d2.addCallback(lambda ign: ss.remote_add_lease("si1", rs[3], cs[3])) + + d2.addCallback(lambda ign: list(ss.get_leases("si1"))) + def _check_leases3(leases): + self.failUnlessEqual(len(leases), 3) + self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs[1], rs[2], rs[3]])) + d2.addCallback(_check_leases3) + + # add-lease on a missing storage index is silently ignored + d2.addCallback(lambda ign: ss.remote_add_lease("si18", "", "")) + d2.addCallback(lambda res: self.failUnlessEqual(res, None)) + + # check that si0 is readable + d2.addCallback(lambda ign: ss.remote_get_buckets("si0")) + d2.addCallback(lambda readers: self.failUnlessEqual(len(readers), 5)) + + # renew the first lease. Only the proper renew_secret should work + d2.addCallback(lambda ign: ss.remote_renew_lease("si0", rs[0])) + d2.addCallback(lambda ign: self.shouldFail(IndexError, 'wrong secret 1', None, + lambda: ss.remote_renew_lease("si0", cs[0]) )) + d2.addCallback(lambda ign: self.shouldFail(IndexError, 'wrong secret 2', None, + lambda: ss.remote_renew_lease("si0", rs[1]) )) + + # check that si0 is still readable + d2.addCallback(lambda ign: ss.remote_get_buckets("si0")) + d2.addCallback(lambda readers: self.failUnlessEqual(len(readers), 5)) + + # There is no such method as remote_cancel_lease for now -- see + # ticket #1528. + d2.addCallback(lambda ign: self.failIf(hasattr(ss, 'remote_cancel_lease'), + "ss should not have a 'remote_cancel_lease' method/attribute")) + + # test overlapping uploads + d2.addCallback(lambda ign: ss.remote_allocate_buckets("si3", rs[4], cs[4], + sharenums, size, canary)) + return d2 + d.addCallback(_allocated2a) + + def _allocated4( (already, writers) ): + self.failUnlessEqual(len(already), 0) + self.failUnlessEqual(len(writers), 5) + + d2 = defer.succeed(None) + d2.addCallback(lambda ign: ss.remote_allocate_buckets("si3", rs[5], cs[5], + sharenums, size, canary)) + def _allocated5( (already2, writers2) ): + self.failUnlessEqual(len(already2), 0) + self.failUnlessEqual(len(writers2), 0) + + d3 = for_items(self._close_writer, writers) + + d3.addCallback(lambda ign: list(ss.get_leases("si3"))) + d3.addCallback(lambda leases: self.failUnlessEqual(len(leases), 1)) + + d3.addCallback(lambda ign: ss.remote_allocate_buckets("si3", rs[5], cs[5], + sharenums, size, canary)) + return d3 + d2.addCallback(_allocated5) + + def _allocated6( (already3, writers3) ): + self.failUnlessEqual(len(already3), 5) + self.failUnlessEqual(len(writers3), 0) + + d3 = defer.succeed(None) + d3.addCallback(lambda ign: list(ss.get_leases("si3"))) + d3.addCallback(lambda leases: self.failUnlessEqual(len(leases), 2)) + return d3 + d2.addCallback(_allocated6) + return d2 + d.addCallback(_allocated4) + return d + + def test_remove_incoming(self): + ss = self.create("test_remove_incoming") + d = self.allocate(ss, "vid", range(3), 25) + def _allocated( (already, writers) ): + d2 = defer.succeed(None) + for i, bw in writers.items(): + incoming_share_home = bw._share._get_filepath() + d2.addCallback(self._write_and_close, i, bw) + + incoming_si_dir = incoming_share_home.parent() + incoming_prefix_dir = incoming_si_dir.parent() + incoming_dir = incoming_prefix_dir.parent() + + def _check_existence(ign): + self.failIf(incoming_si_dir.exists(), incoming_si_dir) + self.failIf(incoming_prefix_dir.exists(), incoming_prefix_dir) + self.failUnless(incoming_dir.exists(), incoming_dir) + d2.addCallback(_check_existence) + return d2 + d.addCallback(_allocated) + return d + + @mock.patch('allmydata.util.fileutil.get_disk_stats') + def test_reserved_space(self, mock_get_disk_stats): + reserved_space=10000 + mock_get_disk_stats.return_value = { + 'free_for_nonroot': 15000, + 'avail': max(15000 - reserved_space, 0), + } + + ss = self.create("test_reserved_space", reserved_space=reserved_space) + # 15k available, 10k reserved, leaves 5k for shares + + # a newly created and filled share incurs this much overhead, beyond + # the size we request. + OVERHEAD = 3*4 + LEASE_SIZE = 4+32+32+4 + canary = FakeCanary(True) + + d = self.allocate(ss, "vid1", [0,1,2], 1000, canary) + def _allocated( (already, writers) ): + self.failUnlessEqual(len(writers), 3) + # now the StorageServer should have 3000 bytes provisionally + # allocated, allowing only 2000 more to be claimed + self.failUnlessEqual(len(ss._active_writers), 3) + self.writers = writers + del already + + # allocating 1001-byte shares only leaves room for one + d2 = self.allocate(ss, "vid2", [0,1,2], 1001, canary) + def _allocated2( (already2, writers2) ): + self.failUnlessEqual(len(writers2), 1) + self.failUnlessEqual(len(ss._active_writers), 4) + + # we abandon the first set, so their provisional allocation should be + # returned + d3 = for_items(self._abort_writer, self.writers) + #def _del_writers(ign): + # del self.writers + #d3.addCallback(_del_writers) + d3.addCallback(lambda ign: self.failUnlessEqual(len(ss._active_writers), 1)) + + # and we close the second set, so their provisional allocation should + # become real, long-term allocation, and grows to include the + # overhead. + d3.addCallback(lambda ign: for_items(self._write_and_close, writers2)) + d3.addCallback(lambda ign: self.failUnlessEqual(len(ss._active_writers), 0)) + return d3 + d2.addCallback(_allocated2) + + allocated = 1001 + OVERHEAD + LEASE_SIZE + + # we have to manually increase available, since we're not doing real + # disk measurements + def _mock(ign): + mock_get_disk_stats.return_value = { + 'free_for_nonroot': 15000 - allocated, + 'avail': max(15000 - allocated - reserved_space, 0), + } + d2.addCallback(_mock) + + # now there should be ALLOCATED=1001+12+72=1085 bytes allocated, and + # 5000-1085=3915 free, therefore we can fit 39 100byte shares + d2.addCallback(lambda ign: self.allocate(ss,"vid3", range(100), 100, canary)) + def _allocated3( (already3, writers3) ): + self.failUnlessEqual(len(writers3), 39) + self.failUnlessEqual(len(ss._active_writers), 39) + + d3 = for_items(self._abort_writer, writers3) + d3.addCallback(lambda ign: self.failUnlessEqual(len(ss._active_writers), 0)) + d3.addCallback(lambda ign: ss.disownServiceParent()) + return d3 + d2.addCallback(_allocated3) + d.addCallback(_allocated) + return d + + +class MutableServerWithS3Backend(MutableServerTest, CreateS3Backend, unittest.TestCase): + def test_bad_magic(self): + return MutableServerTest.test_bad_magic(self) + test_bad_magic.todo = "The S3 backend doesn't pass this test." + + +class MutableServerWithDiskBackend(MutableServerTest, CreateDiskBackend, unittest.TestCase): + + # The following tests are for behaviour that is only supported by a disk backend. def compare_leases_without_timestamps(self, leases_a, leases_b): self.failUnlessEqual(len(leases_a), len(leases_b)) hunk ./src/allmydata/test/test_storage.py 1647 self.failUnlessEqual(a.nodeid, b.nodeid) self.failUnlessEqual(a.expiration_time, b.expiration_time) - def test_leases(self): - ss = self.create("test_leases") + def test_mutable_leases(self): + ss = self.create("test_mutable_leases") def secrets(n): return ( self.write_enabler("we1"), self.renew_secret("we1-%d" % n), hunk ./src/allmydata/test/test_storage.py 1656 data = "".join([ ("%d" % i) * 10 for i in range(10) ]) write = ss.remote_slot_testv_and_readv_and_writev read = ss.remote_slot_readv - rc = write("si1", secrets(0), {0: ([], [(0,data)], None)}, []) - self.failUnlessEqual(rc, (True, {})) hunk ./src/allmydata/test/test_storage.py 1657 - # create a random non-numeric file in the bucket directory, to - # exercise the code that's supposed to ignore those. - bucket_dir = os.path.join(self.workdir("test_leases"), - "shares", storage_index_to_dir("si1")) - f = open(os.path.join(bucket_dir, "ignore_me.txt"), "w") - f.write("you ought to be ignoring me\n") - f.close() + d = write("si1", secrets(0), {0: ([], [(0,data)], None)}, []) + d.addCallback(lambda res: self.failUnlessEqual(res, (True, {}) )) hunk ./src/allmydata/test/test_storage.py 1660 - s0 = MutableShareFile(os.path.join(bucket_dir, "0")) - self.failUnlessEqual(len(list(s0.get_leases())), 1) + def _create_nonsharefile(ign): + # create a random non-numeric file in the shareset directory, to + # exercise the code that's supposed to ignore those. hunk ./src/allmydata/test/test_storage.py 1664 - # add-lease on a missing storage index is silently ignored - self.failUnlessEqual(ss.remote_add_lease("si18", "", ""), None) + shareset = ss.backend.get_shareset("si1") + shareset._get_sharedir().child("ignore_me.txt").setContent("you ought to be ignoring me\n") + return shareset.get_share(0) + d.addCallback(_create_nonsharefile) + def _got_s0(s0): + self.failUnlessEqual(len(list(s0.get_leases())), 1) hunk ./src/allmydata/test/test_storage.py 1671 - # re-allocate the slots and use the same secrets, that should update - # the lease - write("si1", secrets(0), {0: ([], [(0,data)], None)}, []) - self.failUnlessEqual(len(list(s0.get_leases())), 1) + d2 = defer.succeed(None) + d2.addCallback(lambda ign: ss.remote_add_lease("si18", "", "")) + # add-lease on a missing storage index is silently ignored + d2.addCallback(lambda res: self.failUnlessEqual(res, None)) hunk ./src/allmydata/test/test_storage.py 1676 - # renew it directly - ss.remote_renew_lease("si1", secrets(0)[1]) - self.failUnlessEqual(len(list(s0.get_leases())), 1) + # re-allocate the slots and use the same secrets, that should update + # the lease + d2.addCallback(lambda ign: write("si1", secrets(0), {0: ([], [(0,data)], None)}, [])) + d2.addCallback(lambda ign: self.failUnlessEqual(len(list(s0.get_leases())), 1)) hunk ./src/allmydata/test/test_storage.py 1681 - # now allocate them with a bunch of different secrets, to trigger the - # extended lease code. Use add_lease for one of them. - write("si1", secrets(1), {0: ([], [(0,data)], None)}, []) - self.failUnlessEqual(len(list(s0.get_leases())), 2) - secrets2 = secrets(2) - ss.remote_add_lease("si1", secrets2[1], secrets2[2]) - self.failUnlessEqual(len(list(s0.get_leases())), 3) - write("si1", secrets(3), {0: ([], [(0,data)], None)}, []) - write("si1", secrets(4), {0: ([], [(0,data)], None)}, []) - write("si1", secrets(5), {0: ([], [(0,data)], None)}, []) + # renew it directly + d2.addCallback(lambda ign: ss.remote_renew_lease("si1", secrets(0)[1])) + d2.addCallback(lambda ign: self.failUnlessEqual(len(list(s0.get_leases())), 1)) hunk ./src/allmydata/test/test_storage.py 1685 - self.failUnlessEqual(len(list(s0.get_leases())), 6) + # now allocate them with a bunch of different secrets, to trigger the + # extended lease code. Use add_lease for one of them. + d2.addCallback(lambda ign: write("si1", secrets(1), {0: ([], [(0,data)], None)}, [])) + d2.addCallback(lambda ign: self.failUnlessEqual(len(list(s0.get_leases())), 2)) + secrets2 = secrets(2) + d2.addCallback(lambda ign: ss.remote_add_lease("si1", secrets2[1], secrets2[2])) + d2.addCallback(lambda ign: self.failUnlessEqual(len(list(s0.get_leases())), 3)) + d2.addCallback(lambda ign: write("si1", secrets(3), {0: ([], [(0,data)], None)}, [])) + d2.addCallback(lambda ign: write("si1", secrets(4), {0: ([], [(0,data)], None)}, [])) + d2.addCallback(lambda ign: write("si1", secrets(5), {0: ([], [(0,data)], None)}, [])) hunk ./src/allmydata/test/test_storage.py 1696 - all_leases = list(s0.get_leases()) - # and write enough data to expand the container, forcing the server - # to move the leases - write("si1", secrets(0), - {0: ([], [(0,data)], 200), }, - []) + d2.addCallback(lambda ign: self.failUnlessEqual(len(list(s0.get_leases())), 6)) hunk ./src/allmydata/test/test_storage.py 1698 - # read back the leases, make sure they're still intact. - self.compare_leases_without_timestamps(all_leases, list(s0.get_leases())) + def _check_all_leases(ign): + all_leases = list(s0.get_leases()) hunk ./src/allmydata/test/test_storage.py 1701 - ss.remote_renew_lease("si1", secrets(0)[1]) - ss.remote_renew_lease("si1", secrets(1)[1]) - ss.remote_renew_lease("si1", secrets(2)[1]) - ss.remote_renew_lease("si1", secrets(3)[1]) - ss.remote_renew_lease("si1", secrets(4)[1]) - self.compare_leases_without_timestamps(all_leases, list(s0.get_leases())) - # get a new copy of the leases, with the current timestamps. Reading - # data and failing to renew/cancel leases should leave the timestamps - # alone. - all_leases = list(s0.get_leases()) - # renewing with a bogus token should prompt an error message + # and write enough data to expand the container, forcing the server + # to move the leases + d3 = defer.succeed(None) + d3.addCallback(lambda ign: write("si1", secrets(0), + {0: ([], [(0,data)], 200), }, + [])) hunk ./src/allmydata/test/test_storage.py 1708 - # examine the exception thus raised, make sure the old nodeid is - # present, to provide for share migration - e = self.failUnlessRaises(IndexError, - ss.remote_renew_lease, "si1", - secrets(20)[1]) - e_s = str(e) - self.failUnlessIn("Unable to renew non-existent lease", e_s) - self.failUnlessIn("I have leases accepted by nodeids:", e_s) - self.failUnlessIn("nodeids: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' .", e_s) + # read back the leases, make sure they're still intact. + d3.addCallback(lambda ign: self.compare_leases_without_timestamps(all_leases, + list(s0.get_leases()))) hunk ./src/allmydata/test/test_storage.py 1712 - self.compare_leases(all_leases, list(s0.get_leases())) + d3.addCallback(lambda ign: ss.remote_renew_lease("si1", secrets(0)[1])) + d3.addCallback(lambda ign: ss.remote_renew_lease("si1", secrets(1)[1])) + d3.addCallback(lambda ign: ss.remote_renew_lease("si1", secrets(2)[1])) + d3.addCallback(lambda ign: ss.remote_renew_lease("si1", secrets(3)[1])) + d3.addCallback(lambda ign: ss.remote_renew_lease("si1", secrets(4)[1])) + d3.addCallback(lambda ign: self.compare_leases_without_timestamps(all_leases, + list(s0.get_leases()))) + d2.addCallback(_check_all_leases) hunk ./src/allmydata/test/test_storage.py 1721 - # reading shares should not modify the timestamp - read("si1", [], [(0,200)]) - self.compare_leases(all_leases, list(s0.get_leases())) + def _check_all_leases_again(ign): + # get a new copy of the leases, with the current timestamps. Reading + # data and failing to renew/cancel leases should leave the timestamps + # alone. + all_leases = list(s0.get_leases()) + # renewing with a bogus token should prompt an error message hunk ./src/allmydata/test/test_storage.py 1728 - write("si1", secrets(0), - {0: ([], [(200, "make me bigger")], None)}, []) - self.compare_leases_without_timestamps(all_leases, list(s0.get_leases())) + # examine the exception thus raised, make sure the old nodeid is + # present, to provide for share migration + d3 = self.shouldFail(IndexError, 'old nodeid present', + "Unable to renew non-existent lease. " + "I have leases accepted by nodeids: " + "'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' .", + lambda: ss.remote_renew_lease("si1", secrets(20)[1]) ) hunk ./src/allmydata/test/test_storage.py 1736 - write("si1", secrets(0), - {0: ([], [(500, "make me really bigger")], None)}, []) - self.compare_leases_without_timestamps(all_leases, list(s0.get_leases())) + d3.addCallback(lambda ign: self.compare_leases(all_leases, list(s0.get_leases()))) hunk ./src/allmydata/test/test_storage.py 1738 - def test_remove(self): - ss = self.create("test_remove") - self.allocate(ss, "si1", "we1", self._lease_secret.next(), - set([0,1,2]), 100) - readv = ss.remote_slot_readv - writev = ss.remote_slot_testv_and_readv_and_writev - secrets = ( self.write_enabler("we1"), - self.renew_secret("we1"), - self.cancel_secret("we1") ) - # delete sh0 by setting its size to zero - answer = writev("si1", secrets, - {0: ([], [], 0)}, - []) - # the answer should mention all the shares that existed before the - # write - self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) ) - # but a new read should show only sh1 and sh2 - self.failUnlessEqual(readv("si1", [], [(0,10)]), - {1: [""], 2: [""]}) + # reading shares should not modify the timestamp + d3.addCallback(lambda ign: read("si1", [], [(0,200)])) + d3.addCallback(lambda ign: self.compare_leases(all_leases, list(s0.get_leases()))) hunk ./src/allmydata/test/test_storage.py 1742 - # delete sh1 by setting its size to zero - answer = writev("si1", secrets, - {1: ([], [], 0)}, - []) - self.failUnlessEqual(answer, (True, {1:[],2:[]}) ) - self.failUnlessEqual(readv("si1", [], [(0,10)]), - {2: [""]}) + d3.addCallback(lambda ign: write("si1", secrets(0), + {0: ([], [(200, "make me bigger")], None)}, [])) + d3.addCallback(lambda ign: self.compare_leases_without_timestamps(all_leases, list(s0.get_leases()))) hunk ./src/allmydata/test/test_storage.py 1746 - # delete sh2 by setting its size to zero - answer = writev("si1", secrets, - {2: ([], [], 0)}, - []) - self.failUnlessEqual(answer, (True, {2:[]}) ) - self.failUnlessEqual(readv("si1", [], [(0,10)]), - {}) - # and the bucket directory should now be gone - si = base32.b2a("si1") - # note: this is a detail of the storage server implementation, and - # may change in the future - prefix = si[:2] - prefixdir = os.path.join(self.workdir("test_remove"), "shares", prefix) - bucketdir = os.path.join(prefixdir, si) - self.failUnless(os.path.exists(prefixdir), prefixdir) - self.failIf(os.path.exists(bucketdir), bucketdir) + d3.addCallback(lambda ign: write("si1", secrets(0), + {0: ([], [(500, "make me really bigger")], None)}, [])) + d3.addCallback(lambda ign: self.compare_leases_without_timestamps(all_leases, list(s0.get_leases()))) + d2.addCallback(_check_all_leases_again) + return d2 + d.addCallback(_got_s0) + return d class MDMFProxies(unittest.TestCase, ShouldFailMixin): hunk ./src/allmydata/test/test_storage.py 1787 def tearDown(self): self.sparent.stopService() - shutil.rmtree(self.workdir("MDMFProxies storage test server")) - + fileutil.fp_remove(self.workdir("MDMFProxies storage test server")) def write_enabler(self, we_tag): return hashutil.tagged_hash("we_blah", we_tag) hunk ./src/allmydata/test/test_storage.py 1799 return hashutil.tagged_hash("cancel_blah", str(tag)) def workdir(self, name): - basedir = os.path.join("storage", "MutableServer", name) - return basedir - + return FilePath("storage").child(self.__class__.__name__).child(name) def create(self, name): workdir = self.workdir(name) hunk ./src/allmydata/test/test_storage.py 1803 - ss = StorageServer(workdir, "\x00" * 20) + backend = DiskBackend(workdir) + ss = StorageServer("\x00" * 20, backend, workdir) ss.setServiceParent(self.sparent) return ss hunk ./src/allmydata/test/test_storage.py 1923 tws = {} tws[0] = (testvs, [(0, data)], None) readv = [(0, 1)] - results = write(storage_index, self.secrets, tws, readv) - self.failUnless(results[0]) hunk ./src/allmydata/test/test_storage.py 1924 + d = defer.succeed(None) + d.addCallback(lambda ign: write(storage_index, self.secrets, tws, readv)) + d.addCallback(lambda results: self.failUnless(results[0])) + return d def build_test_sdmf_share(self, empty=False): if empty: hunk ./src/allmydata/test/test_storage.py 1991 tws = {} tws[0] = (testvs, [(0, share)], None) readv = [] - results = write(storage_index, self.secrets, tws, readv) - self.failUnless(results[0]) - hunk ./src/allmydata/test/test_storage.py 1992 - def test_read(self): - self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) - # Check that every method equals what we expect it to. d = defer.succeed(None) hunk ./src/allmydata/test/test_storage.py 1993 - def _check_block_and_salt((block, salt)): - self.failUnlessEqual(block, self.block) - self.failUnlessEqual(salt, self.salt) + d.addCallback(lambda ign: write(storage_index, self.secrets, tws, readv)) + d.addCallback(lambda results: self.failUnless(results[0], results)) + return d hunk ./src/allmydata/test/test_storage.py 1997 - for i in xrange(6): - d.addCallback(lambda ignored, i=i: - mr.get_block_and_salt(i)) - d.addCallback(_check_block_and_salt) + def test_read(self): + d = self.write_test_share_to_server("si1") + def _written(ign): + mr = MDMFSlotReadProxy(self.rref, "si1", 0) + # Check that every method returns what we expect it to. + def _check_block_and_salt((block, salt)): + self.failUnlessEqual(block, self.block) + self.failUnlessEqual(salt, self.salt) hunk ./src/allmydata/test/test_storage.py 2006 - d.addCallback(lambda ignored: - mr.get_encprivkey()) - d.addCallback(lambda encprivkey: - self.failUnlessEqual(self.encprivkey, encprivkey)) + d2 = defer.succeed(None) + for i in xrange(6): + d2.addCallback(lambda ign, i=i: mr.get_block_and_salt(i)) + d2.addCallback(_check_block_and_salt) hunk ./src/allmydata/test/test_storage.py 2011 - d.addCallback(lambda ignored: - mr.get_blockhashes()) - d.addCallback(lambda blockhashes: - self.failUnlessEqual(self.block_hash_tree, blockhashes)) + d2.addCallback(lambda ign: mr.get_encprivkey()) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.encprivkey)) hunk ./src/allmydata/test/test_storage.py 2014 - d.addCallback(lambda ignored: - mr.get_sharehashes()) - d.addCallback(lambda sharehashes: - self.failUnlessEqual(self.share_hash_chain, sharehashes)) + d2.addCallback(lambda ign: mr.get_blockhashes()) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.block_hash_tree)) hunk ./src/allmydata/test/test_storage.py 2017 - d.addCallback(lambda ignored: - mr.get_signature()) - d.addCallback(lambda signature: - self.failUnlessEqual(signature, self.signature)) + d2.addCallback(lambda ign: mr.get_sharehashes()) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.share_hash_chain)) hunk ./src/allmydata/test/test_storage.py 2020 - d.addCallback(lambda ignored: - mr.get_verification_key()) - d.addCallback(lambda verification_key: - self.failUnlessEqual(verification_key, self.verification_key)) + d2.addCallback(lambda ign: mr.get_signature()) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.signature)) hunk ./src/allmydata/test/test_storage.py 2023 - d.addCallback(lambda ignored: - mr.get_seqnum()) - d.addCallback(lambda seqnum: - self.failUnlessEqual(seqnum, 0)) + d2.addCallback(lambda ign: mr.get_verification_key()) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.verification_key)) hunk ./src/allmydata/test/test_storage.py 2026 - d.addCallback(lambda ignored: - mr.get_root_hash()) - d.addCallback(lambda root_hash: - self.failUnlessEqual(self.root_hash, root_hash)) + d2.addCallback(lambda ign: mr.get_seqnum()) + d2.addCallback(lambda seqnum: self.failUnlessEqual(seqnum, 0)) hunk ./src/allmydata/test/test_storage.py 2029 - d.addCallback(lambda ignored: - mr.get_seqnum()) - d.addCallback(lambda seqnum: - self.failUnlessEqual(0, seqnum)) + d2.addCallback(lambda ign: mr.get_root_hash()) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.root_hash)) hunk ./src/allmydata/test/test_storage.py 2032 - d.addCallback(lambda ignored: - mr.get_encoding_parameters()) - def _check_encoding_parameters((k, n, segsize, datalen)): - self.failUnlessEqual(k, 3) - self.failUnlessEqual(n, 10) - self.failUnlessEqual(segsize, 6) - self.failUnlessEqual(datalen, 36) - d.addCallback(_check_encoding_parameters) + d2.addCallback(lambda ign: mr.get_encoding_parameters()) + def _check_encoding_parameters((k, n, segsize, datalen)): + self.failUnlessEqual(k, 3) + self.failUnlessEqual(n, 10) + self.failUnlessEqual(segsize, 6) + self.failUnlessEqual(datalen, 36) + d2.addCallback(_check_encoding_parameters) hunk ./src/allmydata/test/test_storage.py 2040 - d.addCallback(lambda ignored: - mr.get_checkstring()) - d.addCallback(lambda checkstring: - self.failUnlessEqual(checkstring, checkstring)) + d2.addCallback(lambda ign: mr.get_checkstring()) + # XXX check against expected value + return d2 + d.addCallback(_written) return d def test_read_with_different_tail_segment_size(self): hunk ./src/allmydata/test/test_storage.py 2047 - self.write_test_share_to_server("si1", tail_segment=True) - mr = MDMFSlotReadProxy(self.rref, "si1", 0) - d = mr.get_block_and_salt(5) + d = self.write_test_share_to_server("si1", tail_segment=True) + d.addCallback(lambda ign: MDMFSlotReadProxy(self.rref, "si1", 0)) + d.addCallback(lambda mr: mr.get_block_and_salt(5)) def _check_tail_segment(results): block, salt = results self.failUnlessEqual(len(block), 1) hunk ./src/allmydata/test/test_storage.py 2058 return d def test_get_block_with_invalid_segnum(self): - self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) - d = defer.succeed(None) - d.addCallback(lambda ignored: + d = self.write_test_share_to_server("si1") + d.addCallback(lambda ign: MDMFSlotReadProxy(self.rref, "si1", 0)) + d.addCallback(lambda mr: self.shouldFail(LayoutInvalid, "test invalid segnum", None, hunk ./src/allmydata/test/test_storage.py 2063 - mr.get_block_and_salt, 7)) + lambda: mr.get_block_and_salt(7) )) return d def test_get_encoding_parameters_first(self): hunk ./src/allmydata/test/test_storage.py 2067 - self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) - d = mr.get_encoding_parameters() + d = self.write_test_share_to_server("si1") + d.addCallback(lambda ign: MDMFSlotReadProxy(self.rref, "si1", 0)) + d.addCallback(lambda mr: mr.get_encoding_parameters()) def _check_encoding_parameters((k, n, segment_size, datalen)): self.failUnlessEqual(k, 3) self.failUnlessEqual(n, 10) hunk ./src/allmydata/test/test_storage.py 2079 return d def test_get_seqnum_first(self): - self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) - d = mr.get_seqnum() + d = self.write_test_share_to_server("si1") + d.addCallback(lambda ign: MDMFSlotReadProxy(self.rref, "si1", 0)) + d.addCallback(lambda mr: mr.get_seqnum()) d.addCallback(lambda seqnum: self.failUnlessEqual(seqnum, 0)) return d hunk ./src/allmydata/test/test_storage.py 2087 def test_get_root_hash_first(self): - self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) - d = mr.get_root_hash() + d = self.write_test_share_to_server("si1") + d.addCallback(lambda ign: MDMFSlotReadProxy(self.rref, "si1", 0)) + d.addCallback(lambda mr: mr.get_root_hash()) d.addCallback(lambda root_hash: self.failUnlessEqual(root_hash, self.root_hash)) return d hunk ./src/allmydata/test/test_storage.py 2095 def test_get_checkstring_first(self): - self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) - d = mr.get_checkstring() + d = self.write_test_share_to_server("si1") + d.addCallback(lambda ign: MDMFSlotReadProxy(self.rref, "si1", 0)) + d.addCallback(lambda mr: mr.get_checkstring()) d.addCallback(lambda checkstring: self.failUnlessEqual(checkstring, self.checkstring)) return d hunk ./src/allmydata/test/test_storage.py 2145 mw = self._make_new_mw("si1", 0) d = defer.succeed(None) for i in xrange(6): - d.addCallback(lambda ignored, i=i: - mw.put_block(self.block, i, self.salt)) - d.addCallback(lambda ignored: - mw.put_encprivkey(self.encprivkey)) - d.addCallback(lambda ignored: - mw.put_sharehashes(self.share_hash_chain)) + d.addCallback(lambda ign, i=i: + mw.put_block(self.block, i, self.salt)) + d.addCallback(lambda ign: mw.put_encprivkey(self.encprivkey)) + d.addCallback(lambda ign: mw.put_sharehashes(self.share_hash_chain)) # Now try to put the private key again. d.addCallback(lambda ignored: hunk ./src/allmydata/test/test_storage.py 2154 self.shouldFail(LayoutInvalid, "test repeat private key", None, - mw.put_encprivkey, self.encprivkey)) + lambda: mw.put_encprivkey(self.encprivkey) )) return d def test_signature_after_verification_key(self): hunk ./src/allmydata/test/test_storage.py 2162 d = defer.succeed(None) # Put everything up to and including the verification key. for i in xrange(6): - d.addCallback(lambda ignored, i=i: - mw.put_block(self.block, i, self.salt)) - d.addCallback(lambda ignored: - mw.put_encprivkey(self.encprivkey)) - d.addCallback(lambda ignored: - mw.put_blockhashes(self.block_hash_tree)) - d.addCallback(lambda ignored: - mw.put_sharehashes(self.share_hash_chain)) - d.addCallback(lambda ignored: - mw.put_root_hash(self.root_hash)) - d.addCallback(lambda ignored: - mw.put_signature(self.signature)) - d.addCallback(lambda ignored: - mw.put_verification_key(self.verification_key)) + d.addCallback(lambda ign, i=i: + mw.put_block(self.block, i, self.salt)) + d.addCallback(lambda ign: mw.put_encprivkey(self.encprivkey)) + d.addCallback(lambda ign: mw.put_blockhashes(self.block_hash_tree)) + d.addCallback(lambda ign: mw.put_sharehashes(self.share_hash_chain)) + d.addCallback(lambda ign: mw.put_root_hash(self.root_hash)) + d.addCallback(lambda ign: mw.put_signature(self.signature)) + d.addCallback(lambda ign: mw.put_verification_key(self.verification_key)) # Now try to put the signature again. This should fail d.addCallback(lambda ignored: self.shouldFail(LayoutInvalid, "signature after verification", hunk ./src/allmydata/test/test_storage.py 2174 None, - mw.put_signature, self.signature)) + lambda: mw.put_signature(self.signature) )) return d def test_uncoordinated_write(self): hunk ./src/allmydata/test/test_storage.py 2219 d.addCallback(lambda ignored: self.shouldFail(LayoutInvalid, "salt too big", None, - mw.put_block, self.block, 0, invalid_salt)) + lambda: mw.put_block(self.block, 0, invalid_salt) )) d.addCallback(lambda ignored: self.shouldFail(LayoutInvalid, "salt too small", None, hunk ./src/allmydata/test/test_storage.py 2223 - mw.put_block, self.block, 0, - another_invalid_salt)) + lambda: mw.put_block(self.block, 0, another_invalid_salt) )) return d def test_write_test_vectors(self): hunk ./src/allmydata/test/test_storage.py 2253 d = mw.finish_publishing() d.addCallback(_check_failure) - d.addCallback(lambda ignored: - mw.set_checkstring("")) - d.addCallback(lambda ignored: - mw.finish_publishing()) + d.addCallback(lambda ign: mw.set_checkstring("")) + d.addCallback(lambda ign: mw.finish_publishing()) d.addCallback(_check_success) return d hunk ./src/allmydata/test/test_storage.py 2291 mw.put_verification_key(self.verification_key) d = mw.finish_publishing() - def _check_publish(results): - self.failUnlessEqual(len(results), 2) - result, ign = results - self.failUnless(result, "publish failed") - for i in xrange(6): - self.failUnlessEqual(read("si1", [0], [(expected_sharedata_offset + (i * written_block_size), written_block_size)]), - {0: [written_block]}) + d.addCallback(lambda (result, ign): self.failUnless(result, "publish failed")) + + for i in xrange(6): + d.addCallback(lambda ign, i=i: read("si1", [0], + [(expected_sharedata_offset + (i * written_block_size), + written_block_size)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [written_block]})) hunk ./src/allmydata/test/test_storage.py 2299 - self.failUnlessEqual(len(self.encprivkey), 7) - self.failUnlessEqual(read("si1", [0], [(expected_private_key_offset, 7)]), - {0: [self.encprivkey]}) + d.addCallback(lambda ign: self.failUnlessEqual(len(self.encprivkey), 7)) + d.addCallback(lambda ign: read("si1", [0], [(expected_private_key_offset, 7)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [self.encprivkey]})) hunk ./src/allmydata/test/test_storage.py 2303 - expected_block_hash_offset = expected_sharedata_offset + \ - (6 * written_block_size) - self.failUnlessEqual(len(self.block_hash_tree_s), 32 * 6) - self.failUnlessEqual(read("si1", [0], [(expected_block_hash_offset, 32 * 6)]), - {0: [self.block_hash_tree_s]}) + expected_block_hash_offset = expected_sharedata_offset + (6 * written_block_size) + d.addCallback(lambda ign: self.failUnlessEqual(len(self.block_hash_tree_s), 32 * 6)) + d.addCallback(lambda ign, ebho=expected_block_hash_offset: + read("si1", [0], [(ebho, 32 * 6)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [self.block_hash_tree_s]})) expected_share_hash_offset = expected_private_key_offset + len(self.encprivkey) hunk ./src/allmydata/test/test_storage.py 2310 - self.failUnlessEqual(read("si1", [0],[(expected_share_hash_offset, (32 + 2) * 6)]), - {0: [self.share_hash_chain_s]}) + d.addCallback(lambda ign, esho=expected_share_hash_offset: + read("si1", [0], [(esho, (32 + 2) * 6)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [self.share_hash_chain_s]})) hunk ./src/allmydata/test/test_storage.py 2314 - self.failUnlessEqual(read("si1", [0], [(9, 32)]), - {0: [self.root_hash]}) - expected_signature_offset = expected_share_hash_offset + \ - len(self.share_hash_chain_s) - self.failUnlessEqual(len(self.signature), 9) - self.failUnlessEqual(read("si1", [0], [(expected_signature_offset, 9)]), - {0: [self.signature]}) + d.addCallback(lambda ign: read("si1", [0], [(9, 32)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [self.root_hash]})) + + expected_signature_offset = expected_share_hash_offset + len(self.share_hash_chain_s) + d.addCallback(lambda ign: self.failUnlessEqual(len(self.signature), 9)) + d.addCallback(lambda ign, esigo=expected_signature_offset: + read("si1", [0], [(esigo, 9)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [self.signature]})) expected_verification_key_offset = expected_signature_offset + len(self.signature) hunk ./src/allmydata/test/test_storage.py 2324 - self.failUnlessEqual(len(self.verification_key), 6) - self.failUnlessEqual(read("si1", [0], [(expected_verification_key_offset, 6)]), - {0: [self.verification_key]}) + d.addCallback(lambda ign: self.failUnlessEqual(len(self.verification_key), 6)) + d.addCallback(lambda ign, evko=expected_verification_key_offset: + read("si1", [0], [(evko, 6)])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [self.verification_key]})) hunk ./src/allmydata/test/test_storage.py 2329 - signable = mw.get_signable() - verno, seq, roothash, k, n, segsize, datalen = \ - struct.unpack(">BQ32sBBQQ", - signable) - self.failUnlessEqual(verno, 1) - self.failUnlessEqual(seq, 0) - self.failUnlessEqual(roothash, self.root_hash) - self.failUnlessEqual(k, 3) - self.failUnlessEqual(n, 10) - self.failUnlessEqual(segsize, 6) - self.failUnlessEqual(datalen, 36) - expected_eof_offset = expected_block_hash_offset + \ - len(self.block_hash_tree_s) + def _check_other_fields(ign, ebho=expected_block_hash_offset, + esho=expected_share_hash_offset, + esigo=expected_signature_offset, + evko=expected_verification_key_offset): + signable = mw.get_signable() + verno, seq, roothash, k, N, segsize, datalen = struct.unpack(">BQ32sBBQQ", + signable) + self.failUnlessEqual(verno, 1) + self.failUnlessEqual(seq, 0) + self.failUnlessEqual(roothash, self.root_hash) + self.failUnlessEqual(k, 3) + self.failUnlessEqual(N, 10) + self.failUnlessEqual(segsize, 6) + self.failUnlessEqual(datalen, 36) + + def _check_field(res, offset, fmt, which, value): + encoded = struct.pack(fmt, value) + d3 = defer.succeed(None) + d3.addCallback(lambda ign: read("si1", [0], [(offset, len(encoded))])) + d3.addCallback(lambda res: self.failUnlessEqual(res, {0: [encoded]}, which)) + return d3 + + d2 = defer.succeed(None) + d2.addCallback(_check_field, 0, ">B", "version number", verno) + d2.addCallback(_check_field, 1, ">Q", "sequence number", seq) + d2.addCallback(_check_field, 41, ">B", "k", k) + d2.addCallback(_check_field, 42, ">B", "N", N) + d2.addCallback(_check_field, 43, ">Q", "segment size", segsize) + d2.addCallback(_check_field, 51, ">Q", "data length", datalen) + d2.addCallback(_check_field, 59, ">Q", "private key offset", + expected_private_key_offset) + d2.addCallback(_check_field, 67, ">Q", "share hash offset", esho) + d2.addCallback(_check_field, 75, ">Q", "signature offset", esigo) + d2.addCallback(_check_field, 83, ">Q", "verification key offset", evko) + d2.addCallback(_check_field, 91, ">Q", "end of verification key", + evko + len(self.verification_key)) + d2.addCallback(_check_field, 99, ">Q", "sharedata offset", + expected_sharedata_offset) + d2.addCallback(_check_field, 107, ">Q", "block hash offset", ebho) + d2.addCallback(_check_field, 115, ">Q", "eof offset", + ebho + len(self.block_hash_tree_s)) + return d2 + d.addCallback(_check_other_fields) hunk ./src/allmydata/test/test_storage.py 2373 - # Check the version number to make sure that it is correct. - expected_version_number = struct.pack(">B", 1) - self.failUnlessEqual(read("si1", [0], [(0, 1)]), - {0: [expected_version_number]}) - # Check the sequence number to make sure that it is correct - expected_sequence_number = struct.pack(">Q", 0) - self.failUnlessEqual(read("si1", [0], [(1, 8)]), - {0: [expected_sequence_number]}) - # Check that the encoding parameters (k, N, segement size, data - # length) are what they should be. These are 3, 10, 6, 36 - expected_k = struct.pack(">B", 3) - self.failUnlessEqual(read("si1", [0], [(41, 1)]), - {0: [expected_k]}) - expected_n = struct.pack(">B", 10) - self.failUnlessEqual(read("si1", [0], [(42, 1)]), - {0: [expected_n]}) - expected_segment_size = struct.pack(">Q", 6) - self.failUnlessEqual(read("si1", [0], [(43, 8)]), - {0: [expected_segment_size]}) - expected_data_length = struct.pack(">Q", 36) - self.failUnlessEqual(read("si1", [0], [(51, 8)]), - {0: [expected_data_length]}) - expected_offset = struct.pack(">Q", expected_private_key_offset) - self.failUnlessEqual(read("si1", [0], [(59, 8)]), - {0: [expected_offset]}) - expected_offset = struct.pack(">Q", expected_share_hash_offset) - self.failUnlessEqual(read("si1", [0], [(67, 8)]), - {0: [expected_offset]}) - expected_offset = struct.pack(">Q", expected_signature_offset) - self.failUnlessEqual(read("si1", [0], [(75, 8)]), - {0: [expected_offset]}) - expected_offset = struct.pack(">Q", expected_verification_key_offset) - self.failUnlessEqual(read("si1", [0], [(83, 8)]), - {0: [expected_offset]}) - expected_offset = struct.pack(">Q", expected_verification_key_offset + len(self.verification_key)) - self.failUnlessEqual(read("si1", [0], [(91, 8)]), - {0: [expected_offset]}) - expected_offset = struct.pack(">Q", expected_sharedata_offset) - self.failUnlessEqual(read("si1", [0], [(99, 8)]), - {0: [expected_offset]}) - expected_offset = struct.pack(">Q", expected_block_hash_offset) - self.failUnlessEqual(read("si1", [0], [(107, 8)]), - {0: [expected_offset]}) - expected_offset = struct.pack(">Q", expected_eof_offset) - self.failUnlessEqual(read("si1", [0], [(115, 8)]), - {0: [expected_offset]}) - d.addCallback(_check_publish) return d def _make_new_mw(self, si, share, datalength=36): hunk ./src/allmydata/test/test_storage.py 2391 # more than 6 blocks into each share. d = defer.succeed(None) for i in xrange(6): - d.addCallback(lambda ignored, i=i: - mw.put_block(self.block, i, self.salt)) + d.addCallback(lambda ign, i=i: + mw.put_block(self.block, i, self.salt)) d.addCallback(lambda ignored: self.shouldFail(LayoutInvalid, "too many blocks", None, hunk ./src/allmydata/test/test_storage.py 2396 - mw.put_block, self.block, 7, self.salt)) + lambda: mw.put_block(self.block, 7, self.salt) )) return d def test_write_rejected_with_invalid_salt(self): hunk ./src/allmydata/test/test_storage.py 2407 d = defer.succeed(None) d.addCallback(lambda ignored: self.shouldFail(LayoutInvalid, "test_invalid_salt", - None, mw.put_block, self.block, 7, bad_salt)) + None, + lambda: mw.put_block(self.block, 7, bad_salt) )) return d def test_write_rejected_with_invalid_root_hash(self): hunk ./src/allmydata/test/test_storage.py 2423 # failures that match what we are looking for, but are caused by # the constraints imposed on operation ordering. for i in xrange(6): - d.addCallback(lambda ignored, i=i: - mw.put_block(self.block, i, self.salt)) - d.addCallback(lambda ignored: - mw.put_encprivkey(self.encprivkey)) - d.addCallback(lambda ignored: - mw.put_blockhashes(self.block_hash_tree)) - d.addCallback(lambda ignored: - mw.put_sharehashes(self.share_hash_chain)) + d.addCallback(lambda ign, i=i: + mw.put_block(self.block, i, self.salt)) + d.addCallback(lambda ign: mw.put_encprivkey(self.encprivkey)) + d.addCallback(lambda ign: mw.put_blockhashes(self.block_hash_tree)) + d.addCallback(lambda ign: mw.put_sharehashes(self.share_hash_chain)) d.addCallback(lambda ignored: self.shouldFail(LayoutInvalid, "invalid root hash", hunk ./src/allmydata/test/test_storage.py 2430 - None, mw.put_root_hash, invalid_root_hash)) + None, + lambda: mw.put_root_hash(invalid_root_hash) )) return d def test_write_rejected_with_invalid_blocksize(self): hunk ./src/allmydata/test/test_storage.py 2446 d = defer.succeed(None) d.addCallback(lambda ignored, invalid_block=invalid_block: self.shouldFail(LayoutInvalid, "test blocksize too small", - None, mw.put_block, invalid_block, 0, - self.salt)) + None, + lambda: mw.put_block(invalid_block, 0, self.salt) )) invalid_block = invalid_block * 3 # 3 bytes != 2 bytes d.addCallback(lambda ignored: hunk ./src/allmydata/test/test_storage.py 2453 self.shouldFail(LayoutInvalid, "test blocksize too large", None, - mw.put_block, invalid_block, 0, self.salt)) + lambda: mw.put_block(invalid_block, 0, self.salt) )) for i in xrange(5): hunk ./src/allmydata/test/test_storage.py 2455 - d.addCallback(lambda ignored, i=i: - mw.put_block(self.block, i, self.salt)) + d.addCallback(lambda ign, i=i: + mw.put_block(self.block, i, self.salt)) # Try to put an invalid tail segment d.addCallback(lambda ignored: self.shouldFail(LayoutInvalid, "test invalid tail segment", hunk ./src/allmydata/test/test_storage.py 2461 None, - mw.put_block, self.block, 5, self.salt)) + lambda: mw.put_block(self.block, 5, self.salt) )) valid_block = "a" hunk ./src/allmydata/test/test_storage.py 2463 - d.addCallback(lambda ignored: - mw.put_block(valid_block, 5, self.salt)) + d.addCallback(lambda ign: mw.put_block(valid_block, 5, self.salt)) return d def test_write_enforces_order_constraints(self): hunk ./src/allmydata/test/test_storage.py 2489 # Write some shares d = defer.succeed(None) for i in xrange(6): - d.addCallback(lambda ignored, i=i: - mw0.put_block(self.block, i, self.salt)) + d.addCallback(lambda ign, i=i: + mw0.put_block(self.block, i, self.salt)) # Try to write the share hash chain without writing the # encrypted private key hunk ./src/allmydata/test/test_storage.py 2498 self.shouldFail(LayoutInvalid, "share hash chain before " "private key", None, - mw0.put_sharehashes, self.share_hash_chain)) + lambda: mw0.put_sharehashes(self.share_hash_chain) )) + # Write the private key. hunk ./src/allmydata/test/test_storage.py 2501 - d.addCallback(lambda ignored: - mw0.put_encprivkey(self.encprivkey)) + d.addCallback(lambda ign: mw0.put_encprivkey(self.encprivkey)) # Now write the block hashes and try again d.addCallback(lambda ignored: hunk ./src/allmydata/test/test_storage.py 2511 # be able to sign it. d.addCallback(lambda ignored: self.shouldFail(LayoutInvalid, "signature before root hash", - None, mw0.put_signature, self.signature)) + None, + lambda: mw0.put_signature(self.signature) )) d.addCallback(lambda ignored: self.failUnlessRaises(LayoutInvalid, mw0.get_signable)) hunk ./src/allmydata/test/test_storage.py 2521 # verification key. d.addCallback(lambda ignored: self.shouldFail(LayoutInvalid, "key before signature", - None, mw0.put_verification_key, - self.verification_key)) + None, + lambda: mw0.put_verification_key(self.verification_key) )) # Now write the share hashes. hunk ./src/allmydata/test/test_storage.py 2525 - d.addCallback(lambda ignored: - mw0.put_sharehashes(self.share_hash_chain)) + d.addCallback(lambda ign: mw0.put_sharehashes(self.share_hash_chain)) + # We should be able to write the root hash now too hunk ./src/allmydata/test/test_storage.py 2528 - d.addCallback(lambda ignored: - mw0.put_root_hash(self.root_hash)) + d.addCallback(lambda ign: mw0.put_root_hash(self.root_hash)) # We should still be unable to put the verification key d.addCallback(lambda ignored: hunk ./src/allmydata/test/test_storage.py 2533 self.shouldFail(LayoutInvalid, "key before signature", - None, mw0.put_verification_key, - self.verification_key)) + None, + lambda: mw0.put_verification_key(self.verification_key) )) hunk ./src/allmydata/test/test_storage.py 2536 - d.addCallback(lambda ignored: - mw0.put_signature(self.signature)) + d.addCallback(lambda ign: mw0.put_signature(self.signature)) # We shouldn't be able to write the offsets to the remote server # until the offset table is finished; IOW, until we have written hunk ./src/allmydata/test/test_storage.py 2556 # reader knows how to read everything back to us. d = defer.succeed(None) for i in xrange(6): - d.addCallback(lambda ignored, i=i: - mw.put_block(self.block, i, self.salt)) - d.addCallback(lambda ignored: - mw.put_encprivkey(self.encprivkey)) - d.addCallback(lambda ignored: - mw.put_blockhashes(self.block_hash_tree)) - d.addCallback(lambda ignored: - mw.put_sharehashes(self.share_hash_chain)) - d.addCallback(lambda ignored: - mw.put_root_hash(self.root_hash)) - d.addCallback(lambda ignored: - mw.put_signature(self.signature)) - d.addCallback(lambda ignored: - mw.put_verification_key(self.verification_key)) - d.addCallback(lambda ignored: - mw.finish_publishing()) + d.addCallback(lambda ign, i=i: + mw.put_block(self.block, i, self.salt)) + d.addCallback(lambda ign: mw.put_encprivkey(self.encprivkey)) + d.addCallback(lambda ign: mw.put_blockhashes(self.block_hash_tree)) + d.addCallback(lambda ign: mw.put_sharehashes(self.share_hash_chain)) + d.addCallback(lambda ign: mw.put_root_hash(self.root_hash)) + d.addCallback(lambda ign: mw.put_signature(self.signature)) + d.addCallback(lambda ign: mw.put_verification_key(self.verification_key)) + d.addCallback(lambda ign: mw.finish_publishing()) hunk ./src/allmydata/test/test_storage.py 2566 - mr = MDMFSlotReadProxy(self.rref, "si1", 0) - def _check_block_and_salt((block, salt)): - self.failUnlessEqual(block, self.block) - self.failUnlessEqual(salt, self.salt) + def _published(ign): + mr = MDMFSlotReadProxy(self.rref, "si1", 0) + def _check_block_and_salt((block, salt)): + self.failUnlessEqual(block, self.block) + self.failUnlessEqual(salt, self.salt) hunk ./src/allmydata/test/test_storage.py 2572 - for i in xrange(6): - d.addCallback(lambda ignored, i=i: - mr.get_block_and_salt(i)) - d.addCallback(_check_block_and_salt) + d2 = defer.succeed(None) + for i in xrange(6): + d2.addCallback(lambda ign, i=i: + mr.get_block_and_salt(i)) + d2.addCallback(_check_block_and_salt) hunk ./src/allmydata/test/test_storage.py 2578 - d.addCallback(lambda ignored: - mr.get_encprivkey()) - d.addCallback(lambda encprivkey: - self.failUnlessEqual(self.encprivkey, encprivkey)) + d2.addCallback(lambda ign: mr.get_encprivkey()) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.encprivkey)) hunk ./src/allmydata/test/test_storage.py 2581 - d.addCallback(lambda ignored: - mr.get_blockhashes()) - d.addCallback(lambda blockhashes: - self.failUnlessEqual(self.block_hash_tree, blockhashes)) + d2.addCallback(lambda ign: mr.get_blockhashes()) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.block_hash_tree)) hunk ./src/allmydata/test/test_storage.py 2584 - d.addCallback(lambda ignored: - mr.get_sharehashes()) - d.addCallback(lambda sharehashes: - self.failUnlessEqual(self.share_hash_chain, sharehashes)) + d2.addCallback(lambda ign: mr.get_sharehashes()) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.share_hash_chain)) hunk ./src/allmydata/test/test_storage.py 2587 - d.addCallback(lambda ignored: - mr.get_signature()) - d.addCallback(lambda signature: - self.failUnlessEqual(signature, self.signature)) + d2.addCallback(lambda ign: mr.get_signature()) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.signature)) hunk ./src/allmydata/test/test_storage.py 2590 - d.addCallback(lambda ignored: - mr.get_verification_key()) - d.addCallback(lambda verification_key: - self.failUnlessEqual(verification_key, self.verification_key)) + d2.addCallback(lambda ign: mr.get_verification_key()) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.verification_key)) hunk ./src/allmydata/test/test_storage.py 2593 - d.addCallback(lambda ignored: - mr.get_seqnum()) - d.addCallback(lambda seqnum: - self.failUnlessEqual(seqnum, 0)) + d2.addCallback(lambda ign: mr.get_seqnum()) + d2.addCallback(lambda seqnum: self.failUnlessEqual(seqnum, 0)) hunk ./src/allmydata/test/test_storage.py 2596 - d.addCallback(lambda ignored: - mr.get_root_hash()) - d.addCallback(lambda root_hash: - self.failUnlessEqual(self.root_hash, root_hash)) + d2.addCallback(lambda ign: mr.get_root_hash()) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.root_hash)) hunk ./src/allmydata/test/test_storage.py 2599 - d.addCallback(lambda ignored: - mr.get_encoding_parameters()) - def _check_encoding_parameters((k, n, segsize, datalen)): - self.failUnlessEqual(k, 3) - self.failUnlessEqual(n, 10) - self.failUnlessEqual(segsize, 6) - self.failUnlessEqual(datalen, 36) - d.addCallback(_check_encoding_parameters) + d2.addCallback(lambda ign: mr.get_encoding_parameters()) + def _check_encoding_parameters((k, n, segsize, datalen)): + self.failUnlessEqual(k, 3) + self.failUnlessEqual(n, 10) + self.failUnlessEqual(segsize, 6) + self.failUnlessEqual(datalen, 36) + d2.addCallback(_check_encoding_parameters) hunk ./src/allmydata/test/test_storage.py 2607 - d.addCallback(lambda ignored: - mr.get_checkstring()) - d.addCallback(lambda checkstring: - self.failUnlessEqual(checkstring, mw.get_checkstring())) + d2.addCallback(lambda ign: mr.get_checkstring()) + d2.addCallback(lambda res: self.failUnlessEqual(res, mw.get_checkstring())) + return d2 + d.addCallback(_published) return d def test_is_sdmf(self): hunk ./src/allmydata/test/test_storage.py 2617 # The MDMFSlotReadProxy should also know how to read SDMF files, # since it will encounter them on the grid. Callers use the # is_sdmf method to test this. - self.write_sdmf_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) - d = mr.is_sdmf() - d.addCallback(lambda issdmf: - self.failUnless(issdmf)) + d = self.write_sdmf_share_to_server("si1") + d.addCallback(lambda ign: MDMFSlotReadProxy(self.rref, "si1", 0)) + d.addCallback(lambda mr: mr.is_sdmf()) + d.addCallback(lambda issdmf: self.failUnless(issdmf)) return d def test_reads_sdmf(self): hunk ./src/allmydata/test/test_storage.py 2626 # The slot read proxy should, naturally, know how to tell us # about data in the SDMF format - self.write_sdmf_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) - d = defer.succeed(None) - d.addCallback(lambda ignored: - mr.is_sdmf()) - d.addCallback(lambda issdmf: - self.failUnless(issdmf)) + d = self.write_sdmf_share_to_server("si1") + def _written(ign): + mr = MDMFSlotReadProxy(self.rref, "si1", 0) + d2 = defer.succeed(None) + d2.addCallback(lambda ign: mr.is_sdmf()) + d2.addCallback(lambda issdmf: self.failUnless(issdmf)) hunk ./src/allmydata/test/test_storage.py 2633 - # What do we need to read? - # - The sharedata - # - The salt - d.addCallback(lambda ignored: - mr.get_block_and_salt(0)) - def _check_block_and_salt(results): - block, salt = results - # Our original file is 36 bytes long. Then each share is 12 - # bytes in size. The share is composed entirely of the - # letter a. self.block contains 2 as, so 6 * self.block is - # what we are looking for. - self.failUnlessEqual(block, self.block * 6) - self.failUnlessEqual(salt, self.salt) - d.addCallback(_check_block_and_salt) + # What do we need to read? + # - The sharedata + # - The salt + d2.addCallback(lambda ign: mr.get_block_and_salt(0)) + def _check_block_and_salt(results): + block, salt = results + # Our original file is 36 bytes long. Then each share is 12 + # bytes in size. The share is composed entirely of the + # letter a. self.block contains 2 as, so 6 * self.block is + # what we are looking for. + self.failUnlessEqual(block, self.block * 6) + self.failUnlessEqual(salt, self.salt) + d2.addCallback(_check_block_and_salt) hunk ./src/allmydata/test/test_storage.py 2647 - # - The blockhashes - d.addCallback(lambda ignored: - mr.get_blockhashes()) - d.addCallback(lambda blockhashes: - self.failUnlessEqual(self.block_hash_tree, - blockhashes, - blockhashes)) - # - The sharehashes - d.addCallback(lambda ignored: - mr.get_sharehashes()) - d.addCallback(lambda sharehashes: - self.failUnlessEqual(self.share_hash_chain, - sharehashes)) - # - The keys - d.addCallback(lambda ignored: - mr.get_encprivkey()) - d.addCallback(lambda encprivkey: - self.failUnlessEqual(encprivkey, self.encprivkey, encprivkey)) - d.addCallback(lambda ignored: - mr.get_verification_key()) - d.addCallback(lambda verification_key: - self.failUnlessEqual(verification_key, - self.verification_key, - verification_key)) - # - The signature - d.addCallback(lambda ignored: - mr.get_signature()) - d.addCallback(lambda signature: - self.failUnlessEqual(signature, self.signature, signature)) + # - The blockhashes + d2.addCallback(lambda ign: mr.get_blockhashes()) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.block_hash_tree)) hunk ./src/allmydata/test/test_storage.py 2651 - # - The sequence number - d.addCallback(lambda ignored: - mr.get_seqnum()) - d.addCallback(lambda seqnum: - self.failUnlessEqual(seqnum, 0, seqnum)) + # - The sharehashes + d2.addCallback(lambda ign: mr.get_sharehashes()) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.share_hash_chain)) hunk ./src/allmydata/test/test_storage.py 2655 - # - The root hash - d.addCallback(lambda ignored: - mr.get_root_hash()) - d.addCallback(lambda root_hash: - self.failUnlessEqual(root_hash, self.root_hash, root_hash)) - return d + # - The keys + d2.addCallback(lambda ign: mr.get_encprivkey()) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.encprivkey)) + d2.addCallback(lambda ign: mr.get_verification_key()) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.verification_key)) + + # - The signature + d2.addCallback(lambda ign: mr.get_signature()) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.signature)) hunk ./src/allmydata/test/test_storage.py 2665 + # - The sequence number + d2.addCallback(lambda ign: mr.get_seqnum()) + d2.addCallback(lambda seqnum: self.failUnlessEqual(seqnum, 0)) + + # - The root hash + d2.addCallback(lambda ign: mr.get_root_hash()) + d2.addCallback(lambda res: self.failUnlessEqual(res, self.root_hash)) + return d2 + d.addCallback(_written) + return d def test_only_reads_one_segment_sdmf(self): # SDMF shares have only one segment, so it doesn't make sense to hunk ./src/allmydata/test/test_storage.py 2680 # read more segments than that. The reader should know this and # complain if we try to do that. - self.write_sdmf_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) - d = defer.succeed(None) - d.addCallback(lambda ignored: - mr.is_sdmf()) - d.addCallback(lambda issdmf: - self.failUnless(issdmf)) - d.addCallback(lambda ignored: - self.shouldFail(LayoutInvalid, "test bad segment", - None, - mr.get_block_and_salt, 1)) + d = self.write_sdmf_share_to_server("si1") + def _written(ign): + mr = MDMFSlotReadProxy(self.rref, "si1", 0) + d2 = defer.succeed(None) + d2.addCallback(lambda ign: mr.is_sdmf()) + d2.addCallback(lambda issdmf: self.failUnless(issdmf)) + d2.addCallback(lambda ign: self.shouldFail(LayoutInvalid, "test bad segment", + None, + lambda: mr.get_block_and_salt(1) )) + return d2 + d.addCallback(_written) return d def test_read_with_prefetched_mdmf_data(self): hunk ./src/allmydata/test/test_storage.py 2700 # finding out which shares are on the remote peer so that it # doesn't waste round trips. mdmf_data = self.build_test_mdmf_share() - self.write_test_share_to_server("si1") def _make_mr(ignored, length): mr = MDMFSlotReadProxy(self.rref, "si1", 0, mdmf_data[:length]) return mr hunk ./src/allmydata/test/test_storage.py 2704 - d = defer.succeed(None) + d = self.write_test_share_to_server("si1") + # This should be enough to fill in both the encoding parameters # and the table of offsets, which will complete the version # information tuple. hunk ./src/allmydata/test/test_storage.py 2761 def test_read_with_prefetched_sdmf_data(self): sdmf_data = self.build_test_sdmf_share() - self.write_sdmf_share_to_server("si1") def _make_mr(ignored, length): mr = MDMFSlotReadProxy(self.rref, "si1", 0, sdmf_data[:length]) return mr hunk ./src/allmydata/test/test_storage.py 2765 - d = defer.succeed(None) + d = self.write_sdmf_share_to_server("si1") + # This should be enough to get us the encoding parameters, # offset table, and everything else we need to build a verinfo # string. hunk ./src/allmydata/test/test_storage.py 2826 # Some tests upload a file with no contents to test things # unrelated to the actual handling of the content of the file. # The reader should behave intelligently in these cases. - self.write_test_share_to_server("si1", empty=True) - mr = MDMFSlotReadProxy(self.rref, "si1", 0) - # We should be able to get the encoding parameters, and they - # should be correct. - d = defer.succeed(None) - d.addCallback(lambda ignored: - mr.get_encoding_parameters()) - def _check_encoding_parameters(params): - self.failUnlessEqual(len(params), 4) - k, n, segsize, datalen = params - self.failUnlessEqual(k, 3) - self.failUnlessEqual(n, 10) - self.failUnlessEqual(segsize, 0) - self.failUnlessEqual(datalen, 0) - d.addCallback(_check_encoding_parameters) + d = self.write_test_share_to_server("si1", empty=True) + def _written(ign): + mr = MDMFSlotReadProxy(self.rref, "si1", 0) hunk ./src/allmydata/test/test_storage.py 2830 - # We should not be able to fetch a block, since there are no - # blocks to fetch - d.addCallback(lambda ignored: - self.shouldFail(LayoutInvalid, "get block on empty file", - None, - mr.get_block_and_salt, 0)) - return d + # We should be able to get the encoding parameters, and they + # should be correct. + d2 = defer.succeed(None) + d2.addCallback(lambda ignored: + mr.get_encoding_parameters()) + def _check_encoding_parameters(params): + self.failUnlessEqual(len(params), 4) + k, n, segsize, datalen = params + self.failUnlessEqual(k, 3) + self.failUnlessEqual(n, 10) + self.failUnlessEqual(segsize, 0) + self.failUnlessEqual(datalen, 0) + d2.addCallback(_check_encoding_parameters) hunk ./src/allmydata/test/test_storage.py 2844 + # We should not be able to fetch a block, since there are no + # blocks to fetch + d2.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "get block on empty file", + None, + lambda: mr.get_block_and_salt(0) )) + return d2 + d.addCallback(_written) + return d def test_read_with_empty_sdmf_file(self): hunk ./src/allmydata/test/test_storage.py 2855 - self.write_sdmf_share_to_server("si1", empty=True) - mr = MDMFSlotReadProxy(self.rref, "si1", 0) - # We should be able to get the encoding parameters, and they - # should be correct - d = defer.succeed(None) - d.addCallback(lambda ignored: - mr.get_encoding_parameters()) - def _check_encoding_parameters(params): - self.failUnlessEqual(len(params), 4) - k, n, segsize, datalen = params - self.failUnlessEqual(k, 3) - self.failUnlessEqual(n, 10) - self.failUnlessEqual(segsize, 0) - self.failUnlessEqual(datalen, 0) - d.addCallback(_check_encoding_parameters) + d = self.write_sdmf_share_to_server("si1", empty=True) + def _written(ign): + mr = MDMFSlotReadProxy(self.rref, "si1", 0) + # We should be able to get the encoding parameters, and they + # should be correct + d2 = defer.succeed(None) + d2.addCallback(lambda ignored: + mr.get_encoding_parameters()) + def _check_encoding_parameters(params): + self.failUnlessEqual(len(params), 4) + k, n, segsize, datalen = params + self.failUnlessEqual(k, 3) + self.failUnlessEqual(n, 10) + self.failUnlessEqual(segsize, 0) + self.failUnlessEqual(datalen, 0) + d2.addCallback(_check_encoding_parameters) hunk ./src/allmydata/test/test_storage.py 2872 - # It does not make sense to get a block in this format, so we - # should not be able to. - d.addCallback(lambda ignored: - self.shouldFail(LayoutInvalid, "get block on an empty file", - None, - mr.get_block_and_salt, 0)) + # It does not make sense to get a block in this format, so we + # should not be able to. + d2.addCallback(lambda ignored: + self.shouldFail(LayoutInvalid, "get block on an empty file", + None, + lambda: mr.get_block_and_salt(0) )) + return d2 + d.addCallback(_written) return d def test_verinfo_with_sdmf_file(self): hunk ./src/allmydata/test/test_storage.py 2883 - self.write_sdmf_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) - # We should be able to get the version information. - d = defer.succeed(None) - d.addCallback(lambda ignored: - mr.get_verinfo()) + d = self.write_sdmf_share_to_server("si1") + d.addCallback(lambda ign: MDMFSlotReadProxy(self.rref, "si1", 0)) + d.addCallback(lambda mr: mr.get_verinfo()) def _check_verinfo(verinfo): self.failUnless(verinfo) self.failUnlessEqual(len(verinfo), 9) hunk ./src/allmydata/test/test_storage.py 2920 return d def test_verinfo_with_mdmf_file(self): - self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) - d = defer.succeed(None) - d.addCallback(lambda ignored: - mr.get_verinfo()) + d = self.write_test_share_to_server("si1") + d.addCallback(lambda ign: MDMFSlotReadProxy(self.rref, "si1", 0)) + d.addCallback(lambda mr: mr.get_verinfo()) def _check_verinfo(verinfo): self.failUnless(verinfo) self.failUnlessEqual(len(verinfo), 9) hunk ./src/allmydata/test/test_storage.py 2990 # Now finish publishing d = sdmfr.finish_publishing() - def _then(ignored): - self.failUnlessEqual(self.rref.write_count, 1) - read = self.ss.remote_slot_readv - self.failUnlessEqual(read("si1", [0], [(0, len(data))]), - {0: [data]}) - d.addCallback(_then) + d.addCallback(lambda ign: self.failUnlessEqual(self.rref.write_count, 1)) + d.addCallback(lambda ign: self.ss.remote_slot_readv("si1", [0], [(0, len(data))])) + d.addCallback(lambda res: self.failUnlessEqual(res, {0: [data]})) return d def test_sdmf_writer_preexisting_share(self): hunk ./src/allmydata/test/test_storage.py 2997 data = self.build_test_sdmf_share() - self.write_sdmf_share_to_server("si1") + d = self.write_sdmf_share_to_server("si1") + def _written(ign): + # Now there is a share on the storage server. To successfully + # write, we need to set the checkstring correctly. When we + # don't, no write should occur. + sdmfw = SDMFSlotWriteProxy(0, + self.rref, + "si1", + self.secrets, + 1, 3, 10, 36, 36) + sdmfw.put_block(self.blockdata, 0, self.salt) hunk ./src/allmydata/test/test_storage.py 3009 - # Now there is a share on the storage server. To successfully - # write, we need to set the checkstring correctly. When we - # don't, no write should occur. - sdmfw = SDMFSlotWriteProxy(0, - self.rref, - "si1", - self.secrets, - 1, 3, 10, 36, 36) - sdmfw.put_block(self.blockdata, 0, self.salt) + # Put the encprivkey + sdmfw.put_encprivkey(self.encprivkey) hunk ./src/allmydata/test/test_storage.py 3012 - # Put the encprivkey - sdmfw.put_encprivkey(self.encprivkey) + # Put the block and share hash chains + sdmfw.put_blockhashes(self.block_hash_tree) + sdmfw.put_sharehashes(self.share_hash_chain) hunk ./src/allmydata/test/test_storage.py 3016 - # Put the block and share hash chains - sdmfw.put_blockhashes(self.block_hash_tree) - sdmfw.put_sharehashes(self.share_hash_chain) + # Put the root hash + sdmfw.put_root_hash(self.root_hash) hunk ./src/allmydata/test/test_storage.py 3019 - # Put the root hash - sdmfw.put_root_hash(self.root_hash) + # Put the signature + sdmfw.put_signature(self.signature) hunk ./src/allmydata/test/test_storage.py 3022 - # Put the signature - sdmfw.put_signature(self.signature) + # Put the verification key + sdmfw.put_verification_key(self.verification_key) hunk ./src/allmydata/test/test_storage.py 3025 - # Put the verification key - sdmfw.put_verification_key(self.verification_key) + # We shouldn't have a checkstring yet + self.failUnlessEqual(sdmfw.get_checkstring(), "") hunk ./src/allmydata/test/test_storage.py 3028 - # We shouldn't have a checkstring yet - self.failUnlessEqual(sdmfw.get_checkstring(), "") + d2 = sdmfw.finish_publishing() + def _then(results): + self.failIf(results[0]) + # this is the correct checkstring + self._expected_checkstring = results[1][0][0] + return self._expected_checkstring hunk ./src/allmydata/test/test_storage.py 3035 - d = sdmfw.finish_publishing() - def _then(results): - self.failIf(results[0]) - # this is the correct checkstring - self._expected_checkstring = results[1][0][0] - return self._expected_checkstring + d2.addCallback(_then) + d2.addCallback(sdmfw.set_checkstring) + d2.addCallback(lambda ign: sdmfw.get_checkstring()) + d2.addCallback(lambda res: self.failUnlessEqual(res, self._expected_checkstring)) hunk ./src/allmydata/test/test_storage.py 3040 - d.addCallback(_then) - d.addCallback(sdmfw.set_checkstring) - d.addCallback(lambda ignored: - sdmfw.get_checkstring()) - d.addCallback(lambda checkstring: - self.failUnlessEqual(checkstring, self._expected_checkstring)) - d.addCallback(lambda ignored: - sdmfw.finish_publishing()) - def _then_again(results): - self.failUnless(results[0]) - read = self.ss.remote_slot_readv - self.failUnlessEqual(read("si1", [0], [(1, 8)]), - {0: [struct.pack(">Q", 1)]}) - self.failUnlessEqual(read("si1", [0], [(9, len(data) - 9)]), - {0: [data[9:]]}) - d.addCallback(_then_again) + d2.addCallback(lambda ign: sdmfw.finish_publishing()) + d2.addCallback(lambda results: self.failUnless(results[0])) + d2.addCallback(lambda ign: self.ss.remote_slot_readv("si1", [0], [(1, 8)])) + d2.addCallback(lambda res: self.failUnlessEqual(res, {0: [struct.pack(">Q", 1)]})) + d2.addCallback(lambda ign: self.ss.remote_slot_readv("si1", [0], [(9, len(data) - 9)])) + d2.addCallback(lambda res: self.failUnlessEqual(res, {0: [data[9:]]})) + return d2 + d.addCallback(_written) return d hunk ./src/allmydata/test/test_storage.py 3060 return self.sparent.stopService() def workdir(self, name): - basedir = os.path.join("storage", "Server", name) - return basedir + return FilePath("storage").child(self.__class__.__name__).child(name) def create(self, name): workdir = self.workdir(name) hunk ./src/allmydata/test/test_storage.py 3064 - ss = StorageServer(workdir, "\x00" * 20) + backend = DiskBackend(workdir) + ss = StorageServer("\x00" * 20, backend, workdir) ss.setServiceParent(self.sparent) return ss hunk ./src/allmydata/test/test_storage.py 3150 d.callback(None) class MyStorageServer(StorageServer): - def add_bucket_counter(self): - statefile = os.path.join(self.storedir, "bucket_counter.state") - self.bucket_counter = MyBucketCountingCrawler(self, statefile) - self.bucket_counter.setServiceParent(self) + BucketCounterClass = MyBucketCountingCrawler + class BucketCounter(unittest.TestCase, pollmixin.PollMixin): hunk ./src/allmydata/test/test_storage.py 3163 def test_bucket_counter(self): basedir = "storage/BucketCounter/bucket_counter" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20) + fp = FilePath(basedir) + backend = DiskBackend(fp) + ss = StorageServer("\x00" * 20, backend, fp) + # to make sure we capture the bucket-counting-crawler in the middle # of a cycle, we reach in and reduce its maximum slice time to 0. We # also make it start sooner than usual. hunk ./src/allmydata/test/test_storage.py 3222 def test_bucket_counter_cleanup(self): basedir = "storage/BucketCounter/bucket_counter_cleanup" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20) + fp = FilePath(basedir) + backend = DiskBackend(fp) + ss = StorageServer("\x00" * 20, backend, fp) + # to make sure we capture the bucket-counting-crawler in the middle # of a cycle, we reach in and reduce its maximum slice time to 0. ss.bucket_counter.slow_start = 0 hunk ./src/allmydata/test/test_storage.py 3266 def test_bucket_counter_eta(self): basedir = "storage/BucketCounter/bucket_counter_eta" - fileutil.make_dirs(basedir) - ss = MyStorageServer(basedir, "\x00" * 20) + fp = FilePath(basedir) + backend = DiskBackend(fp) + ss = MyStorageServer("\x00" * 20, backend, fp) ss.bucket_counter.slow_start = 0 # these will be fired inside finished_prefix() hunk ./src/allmydata/test/test_storage.py 3306 class InstrumentedLeaseCheckingCrawler(LeaseCheckingCrawler): - stop_after_first_bucket = False - def process_bucket(self, *args, **kwargs): - LeaseCheckingCrawler.process_bucket(self, *args, **kwargs) - if self.stop_after_first_bucket: - self.stop_after_first_bucket = False - self.cpu_slice = -1.0 + hook_ds = None + + def process_shareset(self, *args, **kwargs): + try: + LeaseCheckingCrawler.process_shareset(self, *args, **kwargs) + if self.hook_ds: + self.cpu_slice = -1.0 + d = self.hook_ds.pop(0) + d.callback(None) + except Exception, e: + if self.hook_ds: + self.cpu_slice = -1.0 + d = self.hook_ds.pop(0) + d.errback(e) + raise + def yielding(self, sleep_time): hunk ./src/allmydata/test/test_storage.py 3323 - if not self.stop_after_first_bucket: + if self.hook_ds is None: self.cpu_slice = 500 hunk ./src/allmydata/test/test_storage.py 3326 -class BrokenStatResults: - pass -class No_ST_BLOCKS_LeaseCheckingCrawler(LeaseCheckingCrawler): - def stat(self, fn): - s = os.stat(fn) - bsr = BrokenStatResults() - for attrname in dir(s): - if attrname.startswith("_"): - continue - if attrname == "st_blocks": - continue - setattr(bsr, attrname, getattr(s, attrname)) - return bsr - class InstrumentedStorageServer(StorageServer): LeaseCheckerClass = InstrumentedLeaseCheckingCrawler hunk ./src/allmydata/test/test_storage.py 3328 -class No_ST_BLOCKS_StorageServer(StorageServer): - LeaseCheckerClass = No_ST_BLOCKS_LeaseCheckingCrawler + class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): hunk ./src/allmydata/test/test_storage.py 3350 return (hashutil.tagged_hash("renew-%d" % num, si), hashutil.tagged_hash("cancel-%d" % num, si)) + writev = ss.remote_slot_testv_and_readv_and_writev + immutable_si_0, rs0, cs0 = make("\x00" * 16) immutable_si_1, rs1, cs1 = make("\x01" * 16) rs1a, cs1a = make_extra_lease(immutable_si_1, 1) hunk ./src/allmydata/test/test_storage.py 3364 # inner contents are not a valid CHK share data = "\xff" * 1000 - a,w = ss.remote_allocate_buckets(immutable_si_0, rs0, cs0, sharenums, - 1000, canary) - w[0].remote_write(0, data) - w[0].remote_close() - - a,w = ss.remote_allocate_buckets(immutable_si_1, rs1, cs1, sharenums, - 1000, canary) - w[0].remote_write(0, data) - w[0].remote_close() - ss.remote_add_lease(immutable_si_1, rs1a, cs1a) - - writev = ss.remote_slot_testv_and_readv_and_writev - writev(mutable_si_2, (we2, rs2, cs2), - {0: ([], [(0,data)], len(data))}, []) - writev(mutable_si_3, (we3, rs3, cs3), - {0: ([], [(0,data)], len(data))}, []) - ss.remote_add_lease(mutable_si_3, rs3a, cs3a) - self.sis = [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] self.renew_secrets = [rs0, rs1, rs1a, rs2, rs3, rs3a] self.cancel_secrets = [cs0, cs1, cs1a, cs2, cs3, cs3a] hunk ./src/allmydata/test/test_storage.py 3368 + d = defer.succeed(None) + d.addCallback(lambda ign: ss.remote_allocate_buckets(immutable_si_0, rs0, cs0, + sharenums, 1000, canary)) + def _allocated( (a, w) ): + d2 = defer.succeed(None) + d2.addCallback(lambda ign: w[0].remote_write(0, data)) + d2.addCallback(lambda ign: w[0].remote_close()) + d.addCallback(_allocated) + + d.addCallback(lambda ign: ss.remote_allocate_buckets(immutable_si_1, rs1, cs1, + sharenums, 1000, canary)) + d.addCallback(_allocated) + d.addCallback(lambda ign: ss.remote_add_lease(immutable_si_1, rs1a, cs1a)) + + d.addCallback(lambda ign: writev(mutable_si_2, (we2, rs2, cs2), + {0: ([], [(0,data)], len(data))}, [])) + d.addCallback(lambda ign: writev(mutable_si_3, (we3, rs3, cs3), + {0: ([], [(0,data)], len(data))}, [])) + d.addCallback(lambda ign: ss.remote_add_lease(mutable_si_3, rs3a, cs3a)) + return d + def test_basic(self): basedir = "storage/LeaseCrawler/basic" hunk ./src/allmydata/test/test_storage.py 3391 - fileutil.make_dirs(basedir) - ss = InstrumentedStorageServer(basedir, "\x00" * 20) + fp = FilePath(basedir) + backend = DiskBackend(fp) + ss = InstrumentedStorageServer("\x00" * 20, backend, fp) + # make it start sooner than usual. lc = ss.lease_checker lc.slow_start = 0 hunk ./src/allmydata/test/test_storage.py 3399 lc.cpu_slice = 500 - lc.stop_after_first_bucket = True - webstatus = StorageStatus(ss) # create a few shares, with some leases on them hunk ./src/allmydata/test/test_storage.py 3401 - self.make_shares(ss) - [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis + d = self.make_shares(ss) + def _do_test(ign): + d2 = defer.Deferred() + lc.hook_ds = [d2] hunk ./src/allmydata/test/test_storage.py 3406 - # add a non-sharefile to exercise another code path - fn = os.path.join(ss.sharedir, - storage_index_to_dir(immutable_si_0), - "not-a-share") - f = open(fn, "wb") - f.write("I am not a share.\n") - f.close() + DAY = 24*60*60 hunk ./src/allmydata/test/test_storage.py 3408 - # this is before the crawl has started, so we're not in a cycle yet - initial_state = lc.get_state() - self.failIf(lc.get_progress()["cycle-in-progress"]) - self.failIfIn("cycle-to-date", initial_state) - self.failIfIn("estimated-remaining-cycle", initial_state) - self.failIfIn("estimated-current-cycle", initial_state) - self.failUnlessIn("history", initial_state) - self.failUnlessEqual(initial_state["history"], {}) + [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis hunk ./src/allmydata/test/test_storage.py 3410 - ss.setServiceParent(self.s) - - DAY = 24*60*60 + # add a non-sharefile to exercise another code path + fp = ss.backend.get_shareset(immutable_si_0)._get_sharedir().child("not-a-share") + fp.setContent("I am not a share.\n") hunk ./src/allmydata/test/test_storage.py 3414 - d = fireEventually() - - # now examine the state right after the first bucket has been - # processed. - def _after_first_bucket(ignored): + # this is before the crawl has started, so we're not in a cycle yet initial_state = lc.get_state() hunk ./src/allmydata/test/test_storage.py 3416 - if "cycle-to-date" not in initial_state: - d2 = fireEventually() - d2.addCallback(_after_first_bucket) - return d2 - self.failUnlessIn("cycle-to-date", initial_state) - self.failUnlessIn("estimated-remaining-cycle", initial_state) - self.failUnlessIn("estimated-current-cycle", initial_state) + self.failIf(lc.get_progress()["cycle-in-progress"]) + self.failIfIn("cycle-to-date", initial_state) + self.failIfIn("estimated-remaining-cycle", initial_state) + self.failIfIn("estimated-current-cycle", initial_state) self.failUnlessIn("history", initial_state) self.failUnlessEqual(initial_state["history"], {}) hunk ./src/allmydata/test/test_storage.py 3423 - so_far = initial_state["cycle-to-date"] - self.failUnlessEqual(so_far["expiration-enabled"], False) - self.failUnlessIn("configured-expiration-mode", so_far) - self.failUnlessIn("lease-age-histogram", so_far) - lah = so_far["lease-age-histogram"] - self.failUnlessEqual(type(lah), list) - self.failUnlessEqual(len(lah), 1) - self.failUnlessEqual(lah, [ (0.0, DAY, 1) ] ) - self.failUnlessEqual(so_far["leases-per-share-histogram"], {1: 1}) - self.failUnlessEqual(so_far["corrupt-shares"], []) - sr1 = so_far["space-recovered"] - self.failUnlessEqual(sr1["examined-buckets"], 1) - self.failUnlessEqual(sr1["examined-shares"], 1) - self.failUnlessEqual(sr1["actual-shares"], 0) - self.failUnlessEqual(sr1["configured-diskbytes"], 0) - self.failUnlessEqual(sr1["original-sharebytes"], 0) - left = initial_state["estimated-remaining-cycle"] - sr2 = left["space-recovered"] - self.failUnless(sr2["examined-buckets"] > 0, sr2["examined-buckets"]) - self.failUnless(sr2["examined-shares"] > 0, sr2["examined-shares"]) - self.failIfEqual(sr2["actual-shares"], None) - self.failIfEqual(sr2["configured-diskbytes"], None) - self.failIfEqual(sr2["original-sharebytes"], None) - d.addCallback(_after_first_bucket) - d.addCallback(lambda ign: self.render1(webstatus)) - def _check_html_in_cycle(html): - s = remove_tags(html) - self.failUnlessIn("So far, this cycle has examined " - "1 shares in 1 buckets (0 mutable / 1 immutable) ", s) - self.failUnlessIn("and has recovered: " - "0 shares, 0 buckets (0 mutable / 0 immutable), " - "0 B (0 B / 0 B)", s) - self.failUnlessIn("If expiration were enabled, " - "we would have recovered: " - "0 shares, 0 buckets (0 mutable / 0 immutable)," - " 0 B (0 B / 0 B) by now", s) - self.failUnlessIn("and the remainder of this cycle " - "would probably recover: " - "0 shares, 0 buckets (0 mutable / 0 immutable)," - " 0 B (0 B / 0 B)", s) - self.failUnlessIn("and the whole cycle would probably recover: " - "0 shares, 0 buckets (0 mutable / 0 immutable)," - " 0 B (0 B / 0 B)", s) - self.failUnlessIn("if we were strictly using each lease's default " - "31-day lease lifetime", s) - self.failUnlessIn("this cycle would be expected to recover: ", s) - d.addCallback(_check_html_in_cycle) + ss.setServiceParent(self.s) hunk ./src/allmydata/test/test_storage.py 3425 - # wait for the crawler to finish the first cycle. Nothing should have - # been removed. - def _wait(): - return bool(lc.get_state()["last-cycle-finished"] is not None) - d.addCallback(lambda ign: self.poll(_wait)) + # now examine the state right after the first shareset has been + # processed. + def _after_first_shareset(ignored): + initial_state = lc.get_state() + self.failUnlessIn("cycle-to-date", initial_state) + self.failUnlessIn("estimated-remaining-cycle", initial_state) + self.failUnlessIn("estimated-current-cycle", initial_state) + self.failUnlessIn("history", initial_state) + self.failUnlessEqual(initial_state["history"], {}) hunk ./src/allmydata/test/test_storage.py 3435 - def _after_first_cycle(ignored): - s = lc.get_state() - self.failIf("cycle-to-date" in s) - self.failIf("estimated-remaining-cycle" in s) - self.failIf("estimated-current-cycle" in s) - last = s["history"][0] - self.failUnlessIn("cycle-start-finish-times", last) - self.failUnlessEqual(type(last["cycle-start-finish-times"]), tuple) - self.failUnlessEqual(last["expiration-enabled"], False) - self.failUnlessIn("configured-expiration-mode", last) + so_far = initial_state["cycle-to-date"] + self.failUnlessEqual(so_far["expiration-enabled"], False) + self.failUnlessIn("configured-expiration-mode", so_far) + self.failUnlessIn("lease-age-histogram", so_far) + lah = so_far["lease-age-histogram"] + self.failUnlessEqual(type(lah), list) + self.failUnlessEqual(len(lah), 1) + self.failUnlessEqual(lah, [ (0.0, DAY, 1) ] ) + self.failUnlessEqual(so_far["leases-per-share-histogram"], {1: 1}) + self.failUnlessEqual(so_far["corrupt-shares"], []) + sr1 = so_far["space-recovered"] + self.failUnlessEqual(sr1["examined-buckets"], 1) + self.failUnlessEqual(sr1["examined-shares"], 1) + self.failUnlessEqual(sr1["actual-shares"], 0) + self.failUnlessEqual(sr1["configured-diskbytes"], 0) + self.failUnlessEqual(sr1["original-sharebytes"], 0) + left = initial_state["estimated-remaining-cycle"] + sr2 = left["space-recovered"] + self.failUnless(sr2["examined-buckets"] > 0, sr2["examined-buckets"]) + self.failUnless(sr2["examined-shares"] > 0, sr2["examined-shares"]) + self.failIfEqual(sr2["actual-shares"], None) + self.failIfEqual(sr2["configured-diskbytes"], None) + self.failIfEqual(sr2["original-sharebytes"], None) + d2.addCallback(_after_first_shareset) hunk ./src/allmydata/test/test_storage.py 3460 - self.failUnlessIn("lease-age-histogram", last) - lah = last["lease-age-histogram"] - self.failUnlessEqual(type(lah), list) - self.failUnlessEqual(len(lah), 1) - self.failUnlessEqual(lah, [ (0.0, DAY, 6) ] ) + def _render(ign): + webstatus = StorageStatus(ss) + return self.render1(webstatus) + d2.addCallback(_render) + def _check_html_in_cycle(html): + s = remove_tags(html) + self.failUnlessIn("So far, this cycle has examined " + "1 shares in 1 buckets (0 mutable / 1 immutable) ", s) + self.failUnlessIn("and has recovered: " + "0 shares, 0 buckets (0 mutable / 0 immutable), " + "0 B (0 B / 0 B)", s) + self.failUnlessIn("If expiration were enabled, " + "we would have recovered: " + "0 shares, 0 buckets (0 mutable / 0 immutable)," + " 0 B (0 B / 0 B) by now", s) + self.failUnlessIn("and the remainder of this cycle " + "would probably recover: " + "0 shares, 0 buckets (0 mutable / 0 immutable)," + " 0 B (0 B / 0 B)", s) + self.failUnlessIn("and the whole cycle would probably recover: " + "0 shares, 0 buckets (0 mutable / 0 immutable)," + " 0 B (0 B / 0 B)", s) + self.failUnlessIn("if we were strictly using each lease's default " + "31-day lease lifetime", s) + self.failUnlessIn("this cycle would be expected to recover: ", s) + d2.addCallback(_check_html_in_cycle) hunk ./src/allmydata/test/test_storage.py 3487 - self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2}) - self.failUnlessEqual(last["corrupt-shares"], []) + # wait for the crawler to finish the first cycle. Nothing should have + # been removed. + def _wait(): + return lc.get_state()["last-cycle-finished"] is not None + d2.addCallback(lambda ign: self.poll(_wait)) hunk ./src/allmydata/test/test_storage.py 3493 - rec = last["space-recovered"] - self.failUnlessEqual(rec["examined-buckets"], 4) - self.failUnlessEqual(rec["examined-shares"], 4) - self.failUnlessEqual(rec["actual-buckets"], 0) - self.failUnlessEqual(rec["original-buckets"], 0) - self.failUnlessEqual(rec["configured-buckets"], 0) - self.failUnlessEqual(rec["actual-shares"], 0) - self.failUnlessEqual(rec["original-shares"], 0) - self.failUnlessEqual(rec["configured-shares"], 0) - self.failUnlessEqual(rec["actual-diskbytes"], 0) - self.failUnlessEqual(rec["original-diskbytes"], 0) - self.failUnlessEqual(rec["configured-diskbytes"], 0) - self.failUnlessEqual(rec["actual-sharebytes"], 0) - self.failUnlessEqual(rec["original-sharebytes"], 0) - self.failUnlessEqual(rec["configured-sharebytes"], 0) + def _after_first_cycle(ignored): + s = lc.get_state() + self.failIf("cycle-to-date" in s) + self.failIf("estimated-remaining-cycle" in s) + self.failIf("estimated-current-cycle" in s) + last = s["history"][0] + self.failUnlessIn("cycle-start-finish-times", last) + self.failUnlessEqual(type(last["cycle-start-finish-times"]), tuple) + self.failUnlessEqual(last["expiration-enabled"], False) + self.failUnlessIn("configured-expiration-mode", last) hunk ./src/allmydata/test/test_storage.py 3504 - def _get_sharefile(si): - return list(ss._iter_share_files(si))[0] - def count_leases(si): - return len(list(_get_sharefile(si).get_leases())) - self.failUnlessEqual(count_leases(immutable_si_0), 1) - self.failUnlessEqual(count_leases(immutable_si_1), 2) - self.failUnlessEqual(count_leases(mutable_si_2), 1) - self.failUnlessEqual(count_leases(mutable_si_3), 2) - d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) - def _check_html(html): - s = remove_tags(html) - self.failUnlessIn("recovered: 0 shares, 0 buckets " - "(0 mutable / 0 immutable), 0 B (0 B / 0 B) ", s) - self.failUnlessIn("and saw a total of 4 shares, 4 buckets " - "(2 mutable / 2 immutable),", s) - self.failUnlessIn("but expiration was not enabled", s) - d.addCallback(_check_html) - d.addCallback(lambda ign: self.render_json(webstatus)) - def _check_json(json): - data = simplejson.loads(json) - self.failUnlessIn("lease-checker", data) - self.failUnlessIn("lease-checker-progress", data) - d.addCallback(_check_json) + self.failUnlessIn("lease-age-histogram", last) + lah = last["lease-age-histogram"] + self.failUnlessEqual(type(lah), list) + self.failUnlessEqual(len(lah), 1) + self.failUnlessEqual(lah, [ (0.0, DAY, 6) ] ) + + self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2}) + self.failUnlessEqual(last["corrupt-shares"], []) + + rec = last["space-recovered"] + self.failUnlessEqual(rec["examined-buckets"], 4) + self.failUnlessEqual(rec["examined-shares"], 4) + self.failUnlessEqual(rec["actual-buckets"], 0) + self.failUnlessEqual(rec["original-buckets"], 0) + self.failUnlessEqual(rec["configured-buckets"], 0) + self.failUnlessEqual(rec["actual-shares"], 0) + self.failUnlessEqual(rec["original-shares"], 0) + self.failUnlessEqual(rec["configured-shares"], 0) + self.failUnlessEqual(rec["actual-diskbytes"], 0) + self.failUnlessEqual(rec["original-diskbytes"], 0) + self.failUnlessEqual(rec["configured-diskbytes"], 0) + self.failUnlessEqual(rec["actual-sharebytes"], 0) + self.failUnlessEqual(rec["original-sharebytes"], 0) + self.failUnlessEqual(rec["configured-sharebytes"], 0) + d2.addCallback(_after_first_cycle) + + d2.addCallback(lambda ign: self._assert_leasecount(ss, immutable_si_0, 1)) + d2.addCallback(lambda ign: self._assert_leasecount(ss, immutable_si_1, 2)) + d2.addCallback(lambda ign: self._assert_leasecount(ss, mutable_si_2, 1)) + d2.addCallback(lambda ign: self._assert_leasecount(ss, mutable_si_3, 2)) + + d2.addCallback(_render) + def _check_html(html): + s = remove_tags(html) + self.failUnlessIn("recovered: 0 shares, 0 buckets " + "(0 mutable / 0 immutable), 0 B (0 B / 0 B) ", s) + self.failUnlessIn("and saw a total of 4 shares, 4 buckets " + "(2 mutable / 2 immutable),", s) + self.failUnlessIn("but expiration was not enabled", s) + d2.addCallback(_check_html) + + def _render_json(ign): + webstatus = StorageStatus(ss) + return self.render_json(webstatus) + d2.addCallback(_render_json) + def _check_json(json): + data = simplejson.loads(json) + self.failUnlessIn("lease-checker", data) + self.failUnlessIn("lease-checker-progress", data) + d2.addCallback(_check_json) + return d2 + d.addCallback(_do_test) return d hunk ./src/allmydata/test/test_storage.py 3558 - def backdate_lease(self, sf, renew_secret, new_expire_time): - # ShareFile.renew_lease ignores attempts to back-date a lease (i.e. - # "renew" a lease with a new_expire_time that is older than what the - # current lease has), so we have to reach inside it. - for i,lease in enumerate(sf.get_leases()): - if lease.renew_secret == renew_secret: - lease.expiration_time = new_expire_time - f = open(sf.home, 'rb+') - sf._write_lease_record(f, i, lease) + def _backdate_leases(self, ss, si, renew_secrets, new_expire_time): + # The renew_lease method on share files is specified to ignore attempts + # to back-date a lease (i.e. "renew" a lease with a new_expire_time that + # is older than what the current lease has), so we have to reach inside it. + # This works only for shares implemented by the disk backend. + + d = defer.succeed(None) + d.addCallback(lambda ign: ss.backend.get_shareset(si).get_share(0)) + def _got_share(sf): + f = sf._get_filepath().open('rb+') + try: + renewed = 0 + for i, lease in enumerate(sf.get_leases()): + if lease.renew_secret in renew_secrets: + lease.expiration_time = new_expire_time + sf._write_lease_record(f, i, lease) + renewed += 1 + if renewed != len(renew_secrets): + raise IndexError("unable to backdate leases") + finally: f.close() hunk ./src/allmydata/test/test_storage.py 3579 - return - raise IndexError("unable to renew non-existent lease") + d.addCallback(_got_share) + return d + + def _add_share_size(self, accum, ss, si): + d = defer.succeed(None) + d.addCallback(lambda ign: ss.backend.get_shareset(si).get_shares()) + d.addCallback(lambda (shares, corrupted): accum + list(shares)[0].get_size()) + return d + + def _assert_sharecount(self, ss, si, expected): + d = defer.succeed(None) + d.addCallback(lambda ign: ss.backend.get_shareset(si).get_shares()) + def _got_shares( (shares, corrupted) ): + self.failUnlessEqual(len(shares), expected, "share count for %r" % (si,)) + self.failUnlessEqual(len(corrupted), 0, str(corrupted)) + d.addCallback(_got_shares) + return d + + def _assert_leasecount(self, ss, si, expected): + d = defer.succeed(None) + d.addCallback(lambda ign: ss.backend.get_shareset(si).get_shares()) + def _got_shares( (shares, corrupted) ): + share = shares[0] + self.failUnlessEqual(len(list(share.get_leases())), expected, "lease count for %r" % (si,)) + d.addCallback(_got_shares) + return d def test_expire_age(self): basedir = "storage/LeaseCrawler/expire_age" hunk ./src/allmydata/test/test_storage.py 3608 - fileutil.make_dirs(basedir) - # setting expiration_time to 2000 means that any lease which is more - # than 2000s old will be expired. - ss = InstrumentedStorageServer(basedir, "\x00" * 20, - expiration_enabled=True, - expiration_mode="age", - expiration_override_lease_duration=2000) + fp = FilePath(basedir) + backend = DiskBackend(fp) + + # setting 'override_lease_duration' to 2000 means that any lease that + # is more than 2000 seconds old will be expired. + expiration_policy = { + 'enabled': True, + 'mode': 'age', + 'override_lease_duration': 2000, + 'sharetypes': ('mutable', 'immutable'), + } + ss = InstrumentedStorageServer("\x00" * 20, backend, fp, expiration_policy=expiration_policy) + # make it start sooner than usual. lc = ss.lease_checker lc.slow_start = 0 hunk ./src/allmydata/test/test_storage.py 3624 - lc.stop_after_first_bucket = True - webstatus = StorageStatus(ss) # create a few shares, with some leases on them hunk ./src/allmydata/test/test_storage.py 3626 - self.make_shares(ss) - [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis + d = self.make_shares(ss) + def _do_test(ign): + [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis + now = time.time() hunk ./src/allmydata/test/test_storage.py 3631 - def count_shares(si): - return len(list(ss._iter_share_files(si))) - def _get_sharefile(si): - return list(ss._iter_share_files(si))[0] - def count_leases(si): - return len(list(_get_sharefile(si).get_leases())) + d2 = defer.succeed(None) + d2.addCallback(lambda ign: self._assert_sharecount(ss, immutable_si_0, 1)) + d2.addCallback(lambda ign: self._assert_leasecount(ss, immutable_si_0, 1)) + d2.addCallback(lambda ign: self._assert_sharecount(ss, immutable_si_1, 1)) + d2.addCallback(lambda ign: self._assert_leasecount(ss, immutable_si_1, 2)) + d2.addCallback(lambda ign: self._assert_sharecount(ss, mutable_si_2, 1)) + d2.addCallback(lambda ign: self._assert_leasecount(ss, mutable_si_2, 1)) + d2.addCallback(lambda ign: self._assert_sharecount(ss, mutable_si_3, 1)) + d2.addCallback(lambda ign: self._assert_leasecount(ss, mutable_si_3, 2)) hunk ./src/allmydata/test/test_storage.py 3641 - self.failUnlessEqual(count_shares(immutable_si_0), 1) - self.failUnlessEqual(count_leases(immutable_si_0), 1) - self.failUnlessEqual(count_shares(immutable_si_1), 1) - self.failUnlessEqual(count_leases(immutable_si_1), 2) - self.failUnlessEqual(count_shares(mutable_si_2), 1) - self.failUnlessEqual(count_leases(mutable_si_2), 1) - self.failUnlessEqual(count_shares(mutable_si_3), 1) - self.failUnlessEqual(count_leases(mutable_si_3), 2) + # artificially crank back the expiration time on the first lease of + # each share, to make it look like it expired already (age=1000s). + # Some shares have an extra lease which is set to expire at the + # default time in 31 days from now (age=31days). We then run the + # crawler, which will expire the first lease, making some shares get + # deleted and others stay alive (with one remaining lease) hunk ./src/allmydata/test/test_storage.py 3648 - # artificially crank back the expiration time on the first lease of - # each share, to make it look like it expired already (age=1000s). - # Some shares have an extra lease which is set to expire at the - # default time in 31 days from now (age=31days). We then run the - # crawler, which will expire the first lease, making some shares get - # deleted and others stay alive (with one remaining lease) - now = time.time() + d2.addCallback(lambda ign: self._backdate_leases(ss, immutable_si_0, self.renew_secrets[0:1], now - 1000)) + d2.addCallback(lambda ign: self._backdate_leases(ss, immutable_si_1, self.renew_secrets[1:2], now - 1000)) + d2.addCallback(lambda ign: self._backdate_leases(ss, mutable_si_2, self.renew_secrets[3:4], now - 1000)) + d2.addCallback(lambda ign: self._backdate_leases(ss, mutable_si_3, self.renew_secrets[4:5], now - 1000)) hunk ./src/allmydata/test/test_storage.py 3653 - sf0 = _get_sharefile(immutable_si_0) - self.backdate_lease(sf0, self.renew_secrets[0], now - 1000) - sf0_size = os.stat(sf0.home).st_size + d2.addCallback(lambda ign: 0) + d2.addCallback(self._add_share_size, ss, immutable_si_0) + d2.addCallback(self._add_share_size, ss, mutable_si_2) hunk ./src/allmydata/test/test_storage.py 3657 - # immutable_si_1 gets an extra lease - sf1 = _get_sharefile(immutable_si_1) - self.backdate_lease(sf1, self.renew_secrets[1], now - 1000) + def _got_total_size(total_size): + d3 = defer.Deferred() + lc.hook_ds = [d3] + ss.setServiceParent(self.s) hunk ./src/allmydata/test/test_storage.py 3662 - sf2 = _get_sharefile(mutable_si_2) - self.backdate_lease(sf2, self.renew_secrets[3], now - 1000) - sf2_size = os.stat(sf2.home).st_size + # examine the state right after the first shareset has been processed + webstatus = StorageStatus(ss) + d3.addCallback(lambda ign: self.render1(webstatus)) hunk ./src/allmydata/test/test_storage.py 3666 - # mutable_si_3 gets an extra lease - sf3 = _get_sharefile(mutable_si_3) - self.backdate_lease(sf3, self.renew_secrets[4], now - 1000) - - ss.setServiceParent(self.s) + def _check_html_in_cycle(html): + s = remove_tags(html) + # the first shareset encountered gets deleted, and its prefix + # happens to be about 1/5th of the way through the ring, so the + # predictor thinks we'll have 5 shares and that we'll delete them + # all. This part of the test depends upon the SIs landing right + # where they do now. + self.failUnlessIn("The remainder of this cycle is expected to " + "recover: 4 shares, 4 buckets", s) + self.failUnlessIn("The whole cycle is expected to examine " + "5 shares in 5 buckets and to recover: " + "5 shares, 5 buckets", s) + d3.addCallback(_check_html_in_cycle) hunk ./src/allmydata/test/test_storage.py 3680 - d = fireEventually() - # examine the state right after the first bucket has been processed - def _after_first_bucket(ignored): - p = lc.get_progress() - if not p["cycle-in-progress"]: - d2 = fireEventually() - d2.addCallback(_after_first_bucket) - return d2 - d.addCallback(_after_first_bucket) - d.addCallback(lambda ign: self.render1(webstatus)) - def _check_html_in_cycle(html): - s = remove_tags(html) - # the first bucket encountered gets deleted, and its prefix - # happens to be about 1/5th of the way through the ring, so the - # predictor thinks we'll have 5 shares and that we'll delete them - # all. This part of the test depends upon the SIs landing right - # where they do now. - self.failUnlessIn("The remainder of this cycle is expected to " - "recover: 4 shares, 4 buckets", s) - self.failUnlessIn("The whole cycle is expected to examine " - "5 shares in 5 buckets and to recover: " - "5 shares, 5 buckets", s) - d.addCallback(_check_html_in_cycle) + # wait for the crawler to finish the first cycle. Two shares should + # have been removed + def _wait(): + return lc.get_state()["last-cycle-finished"] is not None + d3.addCallback(lambda ign: self.poll(_wait)) hunk ./src/allmydata/test/test_storage.py 3686 - # wait for the crawler to finish the first cycle. Two shares should - # have been removed - def _wait(): - return bool(lc.get_state()["last-cycle-finished"] is not None) - d.addCallback(lambda ign: self.poll(_wait)) + d3.addCallback(lambda ign: self._assert_sharecount(ss, immutable_si_0, 0)) + d3.addCallback(lambda ign: self._assert_sharecount(ss, immutable_si_1, 1)) + d3.addCallback(lambda ign: self._assert_leasecount(ss, immutable_si_1, 1)) + d3.addCallback(lambda ign: self._assert_sharecount(ss, mutable_si_2, 0)) + d3.addCallback(lambda ign: self._assert_sharecount(ss, mutable_si_3, 1)) + d3.addCallback(lambda ign: self._assert_leasecount(ss, mutable_si_3, 1)) hunk ./src/allmydata/test/test_storage.py 3693 - def _after_first_cycle(ignored): - self.failUnlessEqual(count_shares(immutable_si_0), 0) - self.failUnlessEqual(count_shares(immutable_si_1), 1) - self.failUnlessEqual(count_leases(immutable_si_1), 1) - self.failUnlessEqual(count_shares(mutable_si_2), 0) - self.failUnlessEqual(count_shares(mutable_si_3), 1) - self.failUnlessEqual(count_leases(mutable_si_3), 1) + def _after_first_cycle(ignored): + s = lc.get_state() + last = s["history"][0] hunk ./src/allmydata/test/test_storage.py 3697 - s = lc.get_state() - last = s["history"][0] + self.failUnlessEqual(last["expiration-enabled"], True) + self.failUnlessEqual(last["configured-expiration-mode"], + ("age", 2000, None, ("mutable", "immutable"))) + self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2}) hunk ./src/allmydata/test/test_storage.py 3702 - self.failUnlessEqual(last["expiration-enabled"], True) - self.failUnlessEqual(last["configured-expiration-mode"], - ("age", 2000, None, ("mutable", "immutable"))) - self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2}) + rec = last["space-recovered"] + self.failUnlessEqual(rec["examined-buckets"], 4) + self.failUnlessEqual(rec["examined-shares"], 4) + self.failUnlessEqual(rec["actual-buckets"], 2) + self.failUnlessEqual(rec["original-buckets"], 2) + self.failUnlessEqual(rec["configured-buckets"], 2) + self.failUnlessEqual(rec["actual-shares"], 2) + self.failUnlessEqual(rec["original-shares"], 2) + self.failUnlessEqual(rec["configured-shares"], 2) + self.failUnlessEqual(rec["actual-sharebytes"], total_size) + self.failUnlessEqual(rec["original-sharebytes"], total_size) + self.failUnlessEqual(rec["configured-sharebytes"], total_size) hunk ./src/allmydata/test/test_storage.py 3715 - rec = last["space-recovered"] - self.failUnlessEqual(rec["examined-buckets"], 4) - self.failUnlessEqual(rec["examined-shares"], 4) - self.failUnlessEqual(rec["actual-buckets"], 2) - self.failUnlessEqual(rec["original-buckets"], 2) - self.failUnlessEqual(rec["configured-buckets"], 2) - self.failUnlessEqual(rec["actual-shares"], 2) - self.failUnlessEqual(rec["original-shares"], 2) - self.failUnlessEqual(rec["configured-shares"], 2) - size = sf0_size + sf2_size - self.failUnlessEqual(rec["actual-sharebytes"], size) - self.failUnlessEqual(rec["original-sharebytes"], size) - self.failUnlessEqual(rec["configured-sharebytes"], size) - # different platforms have different notions of "blocks used by - # this file", so merely assert that it's a number - self.failUnless(rec["actual-diskbytes"] >= 0, - rec["actual-diskbytes"]) - self.failUnless(rec["original-diskbytes"] >= 0, - rec["original-diskbytes"]) - self.failUnless(rec["configured-diskbytes"] >= 0, - rec["configured-diskbytes"]) - d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) - def _check_html(html): - s = remove_tags(html) - self.failUnlessIn("Expiration Enabled: expired leases will be removed", s) - self.failUnlessIn("Leases created or last renewed more than 33 minutes ago will be considered expired.", s) - self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s) - d.addCallback(_check_html) + # different platforms have different notions of "blocks used by + # this file", so merely assert that it's a number + self.failUnless(rec["actual-diskbytes"] >= 0, + rec["actual-diskbytes"]) + self.failUnless(rec["original-diskbytes"] >= 0, + rec["original-diskbytes"]) + self.failUnless(rec["configured-diskbytes"] >= 0, + rec["configured-diskbytes"]) + d3.addCallback(_after_first_cycle) + d3.addCallback(lambda ign: self.render1(webstatus)) + def _check_html(html): + s = remove_tags(html) + self.failUnlessIn("Expiration Enabled: expired leases will be removed", s) + self.failUnlessIn("Leases created or last renewed more than 33 minutes ago will be considered expired.", s) + self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s) + d3.addCallback(_check_html) + return d3 + d2.addCallback(_got_total_size) + return d2 + d.addCallback(_do_test) return d def test_expire_cutoff_date(self): hunk ./src/allmydata/test/test_storage.py 3738 + # FIXME too much duplicated code between this and test_expire_age + basedir = "storage/LeaseCrawler/expire_cutoff_date" hunk ./src/allmydata/test/test_storage.py 3741 - fileutil.make_dirs(basedir) - # setting cutoff-date to 2000 seconds ago means that any lease which - # is more than 2000s old will be expired. + fp = FilePath(basedir) + backend = DiskBackend(fp) + + # setting 'cutoff_date' to 2000 seconds ago means that any lease that + # is more than 2000 seconds old will be expired. now = time.time() then = int(now - 2000) hunk ./src/allmydata/test/test_storage.py 3748 - ss = InstrumentedStorageServer(basedir, "\x00" * 20, - expiration_enabled=True, - expiration_mode="cutoff-date", - expiration_cutoff_date=then) + expiration_policy = { + 'enabled': True, + 'mode': 'cutoff-date', + 'cutoff_date': then, + 'sharetypes': ('mutable', 'immutable'), + } + ss = InstrumentedStorageServer("\x00" * 20, backend, fp, expiration_policy=expiration_policy) + # make it start sooner than usual. lc = ss.lease_checker lc.slow_start = 0 hunk ./src/allmydata/test/test_storage.py 3759 - lc.stop_after_first_bucket = True - webstatus = StorageStatus(ss) # create a few shares, with some leases on them hunk ./src/allmydata/test/test_storage.py 3761 - self.make_shares(ss) - [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis + d = self.make_shares(ss) + def _do_test(ign): + [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis hunk ./src/allmydata/test/test_storage.py 3765 - def count_shares(si): - return len(list(ss._iter_share_files(si))) - def _get_sharefile(si): - return list(ss._iter_share_files(si))[0] - def count_leases(si): - return len(list(_get_sharefile(si).get_leases())) + d2 = defer.succeed(None) + d2.addCallback(lambda ign: self._assert_sharecount(ss, immutable_si_0, 1)) + d2.addCallback(lambda ign: self._assert_leasecount(ss, immutable_si_0, 1)) + d2.addCallback(lambda ign: self._assert_sharecount(ss, immutable_si_1, 1)) + d2.addCallback(lambda ign: self._assert_leasecount(ss, immutable_si_1, 2)) + d2.addCallback(lambda ign: self._assert_sharecount(ss, mutable_si_2, 1)) + d2.addCallback(lambda ign: self._assert_leasecount(ss, mutable_si_2, 1)) + d2.addCallback(lambda ign: self._assert_sharecount(ss, mutable_si_3, 1)) + d2.addCallback(lambda ign: self._assert_leasecount(ss, mutable_si_3, 2)) hunk ./src/allmydata/test/test_storage.py 3775 - self.failUnlessEqual(count_shares(immutable_si_0), 1) - self.failUnlessEqual(count_leases(immutable_si_0), 1) - self.failUnlessEqual(count_shares(immutable_si_1), 1) - self.failUnlessEqual(count_leases(immutable_si_1), 2) - self.failUnlessEqual(count_shares(mutable_si_2), 1) - self.failUnlessEqual(count_leases(mutable_si_2), 1) - self.failUnlessEqual(count_shares(mutable_si_3), 1) - self.failUnlessEqual(count_leases(mutable_si_3), 2) + # artificially crank back the expiration time on the first lease of + # each share, to make it look like was renewed 3000s ago. To achieve + # this, we need to set the expiration time to now-3000+31days. This + # will change when the lease format is improved to contain both + # create/renew time and duration. + new_expiration_time = now - 3000 + 31*24*60*60 hunk ./src/allmydata/test/test_storage.py 3782 - # artificially crank back the expiration time on the first lease of - # each share, to make it look like was renewed 3000s ago. To achieve - # this, we need to set the expiration time to now-3000+31days. This - # will change when the lease format is improved to contain both - # create/renew time and duration. - new_expiration_time = now - 3000 + 31*24*60*60 + # Some shares have an extra lease which is set to expire at the + # default time in 31 days from now (age=31days). We then run the + # crawler, which will expire the first lease, making some shares get + # deleted and others stay alive (with one remaining lease) hunk ./src/allmydata/test/test_storage.py 3787 - # Some shares have an extra lease which is set to expire at the - # default time in 31 days from now (age=31days). We then run the - # crawler, which will expire the first lease, making some shares get - # deleted and others stay alive (with one remaining lease) + d2.addCallback(lambda ign: self._backdate_leases(ss, immutable_si_0, self.renew_secrets[0:1], + new_expiration_time)) + d2.addCallback(lambda ign: self._backdate_leases(ss, immutable_si_1, self.renew_secrets[1:2], + new_expiration_time)) + d2.addCallback(lambda ign: self._backdate_leases(ss, mutable_si_2, self.renew_secrets[3:4], + new_expiration_time)) + d2.addCallback(lambda ign: self._backdate_leases(ss, mutable_si_3, self.renew_secrets[4:5], + new_expiration_time)) hunk ./src/allmydata/test/test_storage.py 3796 - sf0 = _get_sharefile(immutable_si_0) - self.backdate_lease(sf0, self.renew_secrets[0], new_expiration_time) - sf0_size = os.stat(sf0.home).st_size + d2.addCallback(lambda ign: 0) + d2.addCallback(self._add_share_size, ss, immutable_si_0) + d2.addCallback(self._add_share_size, ss, mutable_si_2) hunk ./src/allmydata/test/test_storage.py 3800 - # immutable_si_1 gets an extra lease - sf1 = _get_sharefile(immutable_si_1) - self.backdate_lease(sf1, self.renew_secrets[1], new_expiration_time) + def _got_total_size(total_size): + d3 = defer.Deferred() + lc.hook_ds = [d3] + ss.setServiceParent(self.s) hunk ./src/allmydata/test/test_storage.py 3805 - sf2 = _get_sharefile(mutable_si_2) - self.backdate_lease(sf2, self.renew_secrets[3], new_expiration_time) - sf2_size = os.stat(sf2.home).st_size + # examine the state right after the first shareset has been processed + webstatus = StorageStatus(ss) + d3.addCallback(lambda ign: self.render1(webstatus)) hunk ./src/allmydata/test/test_storage.py 3809 - # mutable_si_3 gets an extra lease - sf3 = _get_sharefile(mutable_si_3) - self.backdate_lease(sf3, self.renew_secrets[4], new_expiration_time) + def _check_html_in_cycle(html): + s = remove_tags(html) + # the first shareset encountered gets deleted, and its prefix + # happens to be about 1/5th of the way through the ring, so the + # predictor thinks we'll have 5 shares and that we'll delete them + # all. This part of the test depends upon the SIs landing right + # where they do now. + self.failUnlessIn("The remainder of this cycle is expected to " + "recover: 4 shares, 4 buckets", s) + self.failUnlessIn("The whole cycle is expected to examine " + "5 shares in 5 buckets and to recover: " + "5 shares, 5 buckets", s) + d3.addCallback(_check_html_in_cycle) hunk ./src/allmydata/test/test_storage.py 3823 - ss.setServiceParent(self.s) - - d = fireEventually() - # examine the state right after the first bucket has been processed - def _after_first_bucket(ignored): - p = lc.get_progress() - if not p["cycle-in-progress"]: - d2 = fireEventually() - d2.addCallback(_after_first_bucket) - return d2 - d.addCallback(_after_first_bucket) - d.addCallback(lambda ign: self.render1(webstatus)) - def _check_html_in_cycle(html): - s = remove_tags(html) - # the first bucket encountered gets deleted, and its prefix - # happens to be about 1/5th of the way through the ring, so the - # predictor thinks we'll have 5 shares and that we'll delete them - # all. This part of the test depends upon the SIs landing right - # where they do now. - self.failUnlessIn("The remainder of this cycle is expected to " - "recover: 4 shares, 4 buckets", s) - self.failUnlessIn("The whole cycle is expected to examine " - "5 shares in 5 buckets and to recover: " - "5 shares, 5 buckets", s) - d.addCallback(_check_html_in_cycle) + # wait for the crawler to finish the first cycle. Two shares should + # have been removed + def _wait(): + return lc.get_state()["last-cycle-finished"] is not None + d3.addCallback(lambda ign: self.poll(_wait)) hunk ./src/allmydata/test/test_storage.py 3829 - # wait for the crawler to finish the first cycle. Two shares should - # have been removed - def _wait(): - return bool(lc.get_state()["last-cycle-finished"] is not None) - d.addCallback(lambda ign: self.poll(_wait)) + d3.addCallback(lambda ign: self._assert_sharecount(ss, immutable_si_0, 0)) + d3.addCallback(lambda ign: self._assert_sharecount(ss, immutable_si_1, 1)) + d3.addCallback(lambda ign: self._assert_leasecount(ss, immutable_si_1, 1)) + d3.addCallback(lambda ign: self._assert_sharecount(ss, mutable_si_2, 0)) + d3.addCallback(lambda ign: self._assert_sharecount(ss, mutable_si_3, 1)) + d3.addCallback(lambda ign: self._assert_leasecount(ss, mutable_si_3, 1)) hunk ./src/allmydata/test/test_storage.py 3836 - def _after_first_cycle(ignored): - self.failUnlessEqual(count_shares(immutable_si_0), 0) - self.failUnlessEqual(count_shares(immutable_si_1), 1) - self.failUnlessEqual(count_leases(immutable_si_1), 1) - self.failUnlessEqual(count_shares(mutable_si_2), 0) - self.failUnlessEqual(count_shares(mutable_si_3), 1) - self.failUnlessEqual(count_leases(mutable_si_3), 1) + def _after_first_cycle(ignored): + s = lc.get_state() + last = s["history"][0] hunk ./src/allmydata/test/test_storage.py 3840 - s = lc.get_state() - last = s["history"][0] + self.failUnlessEqual(last["expiration-enabled"], True) + self.failUnlessEqual(last["configured-expiration-mode"], + ("cutoff-date", None, then, + ("mutable", "immutable"))) + self.failUnlessEqual(last["leases-per-share-histogram"], + {1: 2, 2: 2}) hunk ./src/allmydata/test/test_storage.py 3847 - self.failUnlessEqual(last["expiration-enabled"], True) - self.failUnlessEqual(last["configured-expiration-mode"], - ("cutoff-date", None, then, - ("mutable", "immutable"))) - self.failUnlessEqual(last["leases-per-share-histogram"], - {1: 2, 2: 2}) + rec = last["space-recovered"] + self.failUnlessEqual(rec["examined-buckets"], 4) + self.failUnlessEqual(rec["examined-shares"], 4) + self.failUnlessEqual(rec["actual-buckets"], 2) + self.failUnlessEqual(rec["original-buckets"], 0) + self.failUnlessEqual(rec["configured-buckets"], 2) + self.failUnlessEqual(rec["actual-shares"], 2) + self.failUnlessEqual(rec["original-shares"], 0) + self.failUnlessEqual(rec["configured-shares"], 2) + self.failUnlessEqual(rec["actual-sharebytes"], total_size) + self.failUnlessEqual(rec["original-sharebytes"], 0) + self.failUnlessEqual(rec["configured-sharebytes"], total_size) hunk ./src/allmydata/test/test_storage.py 3860 - rec = last["space-recovered"] - self.failUnlessEqual(rec["examined-buckets"], 4) - self.failUnlessEqual(rec["examined-shares"], 4) - self.failUnlessEqual(rec["actual-buckets"], 2) - self.failUnlessEqual(rec["original-buckets"], 0) - self.failUnlessEqual(rec["configured-buckets"], 2) - self.failUnlessEqual(rec["actual-shares"], 2) - self.failUnlessEqual(rec["original-shares"], 0) - self.failUnlessEqual(rec["configured-shares"], 2) - size = sf0_size + sf2_size - self.failUnlessEqual(rec["actual-sharebytes"], size) - self.failUnlessEqual(rec["original-sharebytes"], 0) - self.failUnlessEqual(rec["configured-sharebytes"], size) - # different platforms have different notions of "blocks used by - # this file", so merely assert that it's a number - self.failUnless(rec["actual-diskbytes"] >= 0, - rec["actual-diskbytes"]) - self.failUnless(rec["original-diskbytes"] >= 0, - rec["original-diskbytes"]) - self.failUnless(rec["configured-diskbytes"] >= 0, - rec["configured-diskbytes"]) - d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) - def _check_html(html): - s = remove_tags(html) - self.failUnlessIn("Expiration Enabled:" - " expired leases will be removed", s) - date = time.strftime("%Y-%m-%d (%d-%b-%Y) UTC", time.gmtime(then)) - substr = "Leases created or last renewed before %s will be considered expired." % date - self.failUnlessIn(substr, s) - self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s) - d.addCallback(_check_html) + # different platforms have different notions of "blocks used by + # this file", so merely assert that it's a number + self.failUnless(rec["actual-diskbytes"] >= 0, + rec["actual-diskbytes"]) + self.failUnless(rec["original-diskbytes"] >= 0, + rec["original-diskbytes"]) + self.failUnless(rec["configured-diskbytes"] >= 0, + rec["configured-diskbytes"]) + d3.addCallback(_after_first_cycle) + d3.addCallback(lambda ign: self.render1(webstatus)) + def _check_html(html): + s = remove_tags(html) + self.failUnlessIn("Expiration Enabled:" + " expired leases will be removed", s) + date = time.strftime("%Y-%m-%d (%d-%b-%Y) UTC", time.gmtime(then)) + substr = "Leases created or last renewed before %s will be considered expired." % date + self.failUnlessIn(substr, s) + self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s) + d3.addCallback(_check_html) + return d3 + d2.addCallback(_got_total_size) + return d2 + d.addCallback(_do_test) return d def test_only_immutable(self): hunk ./src/allmydata/test/test_storage.py 3887 basedir = "storage/LeaseCrawler/only_immutable" - fileutil.make_dirs(basedir) + fp = FilePath(basedir) + backend = DiskBackend(fp) + + # setting 'cutoff_date' to 2000 seconds ago means that any lease that + # is more than 2000 seconds old will be expired. now = time.time() then = int(now - 2000) hunk ./src/allmydata/test/test_storage.py 3894 - ss = StorageServer(basedir, "\x00" * 20, - expiration_enabled=True, - expiration_mode="cutoff-date", - expiration_cutoff_date=then, - expiration_sharetypes=("immutable",)) + expiration_policy = { + 'enabled': True, + 'mode': 'cutoff-date', + 'cutoff_date': then, + 'sharetypes': ('immutable',), + } + ss = StorageServer("\x00" * 20, backend, fp, expiration_policy=expiration_policy) + + # make it start sooner than usual. lc = ss.lease_checker lc.slow_start = 0 hunk ./src/allmydata/test/test_storage.py 3905 - webstatus = StorageStatus(ss) hunk ./src/allmydata/test/test_storage.py 3906 - self.make_shares(ss) - [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis - # set all leases to be expirable - new_expiration_time = now - 3000 + 31*24*60*60 + # create a few shares, with some leases on them + d = self.make_shares(ss) + def _do_test(ign): + [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis hunk ./src/allmydata/test/test_storage.py 3911 - def count_shares(si): - return len(list(ss._iter_share_files(si))) - def _get_sharefile(si): - return list(ss._iter_share_files(si))[0] - def count_leases(si): - return len(list(_get_sharefile(si).get_leases())) + # set all leases to be expirable + new_expiration_time = now - 3000 + 31*24*60*60 hunk ./src/allmydata/test/test_storage.py 3914 - sf0 = _get_sharefile(immutable_si_0) - self.backdate_lease(sf0, self.renew_secrets[0], new_expiration_time) - sf1 = _get_sharefile(immutable_si_1) - self.backdate_lease(sf1, self.renew_secrets[1], new_expiration_time) - self.backdate_lease(sf1, self.renew_secrets[2], new_expiration_time) - sf2 = _get_sharefile(mutable_si_2) - self.backdate_lease(sf2, self.renew_secrets[3], new_expiration_time) - sf3 = _get_sharefile(mutable_si_3) - self.backdate_lease(sf3, self.renew_secrets[4], new_expiration_time) - self.backdate_lease(sf3, self.renew_secrets[5], new_expiration_time) + d2 = defer.succeed(None) + d2.addCallback(lambda ign: self._backdate_leases(ss, immutable_si_0, self.renew_secrets[0:1], new_expiration_time)) + d2.addCallback(lambda ign: self._backdate_leases(ss, immutable_si_1, self.renew_secrets[1:3], new_expiration_time)) + d2.addCallback(lambda ign: self._backdate_leases(ss, mutable_si_2, self.renew_secrets[3:4], new_expiration_time)) + d2.addCallback(lambda ign: self._backdate_leases(ss, mutable_si_3, self.renew_secrets[4:6], new_expiration_time)) hunk ./src/allmydata/test/test_storage.py 3920 - ss.setServiceParent(self.s) - def _wait(): - return bool(lc.get_state()["last-cycle-finished"] is not None) - d = self.poll(_wait) + d2.addCallback(lambda ign: ss.setServiceParent(self.s)) hunk ./src/allmydata/test/test_storage.py 3922 - def _after_first_cycle(ignored): - self.failUnlessEqual(count_shares(immutable_si_0), 0) - self.failUnlessEqual(count_shares(immutable_si_1), 0) - self.failUnlessEqual(count_shares(mutable_si_2), 1) - self.failUnlessEqual(count_leases(mutable_si_2), 1) - self.failUnlessEqual(count_shares(mutable_si_3), 1) - self.failUnlessEqual(count_leases(mutable_si_3), 2) - d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) - def _check_html(html): - s = remove_tags(html) - self.failUnlessIn("The following sharetypes will be expired: immutable.", s) - d.addCallback(_check_html) + def _wait(): + return lc.get_state()["last-cycle-finished"] is not None + d2.addCallback(lambda ign: self.poll(_wait)) + + d2.addCallback(lambda ign: self._assert_sharecount(ss, immutable_si_0, 0)) + d2.addCallback(lambda ign: self._assert_sharecount(ss, immutable_si_1, 0)) + d2.addCallback(lambda ign: self._assert_sharecount(ss, mutable_si_2, 1)) + d2.addCallback(lambda ign: self._assert_leasecount(ss, mutable_si_2, 1)) + d2.addCallback(lambda ign: self._assert_sharecount(ss, mutable_si_3, 1)) + d2.addCallback(lambda ign: self._assert_leasecount(ss, mutable_si_3, 2)) + + def _render(ign): + webstatus = StorageStatus(ss) + return self.render1(webstatus) + d2.addCallback(_render) + def _check_html(html): + s = remove_tags(html) + self.failUnlessIn("The following sharetypes will be expired: immutable.", s) + d2.addCallback(_check_html) + return d2 + d.addCallback(_do_test) return d def test_only_mutable(self): hunk ./src/allmydata/test/test_storage.py 3947 basedir = "storage/LeaseCrawler/only_mutable" - fileutil.make_dirs(basedir) + fp = FilePath(basedir) + backend = DiskBackend(fp) + + # setting 'cutoff_date' to 2000 seconds ago means that any lease that + # is more than 2000 seconds old will be expired. now = time.time() then = int(now - 2000) hunk ./src/allmydata/test/test_storage.py 3954 - ss = StorageServer(basedir, "\x00" * 20, - expiration_enabled=True, - expiration_mode="cutoff-date", - expiration_cutoff_date=then, - expiration_sharetypes=("mutable",)) + expiration_policy = { + 'enabled': True, + 'mode': 'cutoff-date', + 'cutoff_date': then, + 'sharetypes': ('mutable',), + } + ss = StorageServer("\x00" * 20, backend, fp, expiration_policy=expiration_policy) + + # make it start sooner than usual. lc = ss.lease_checker lc.slow_start = 0 hunk ./src/allmydata/test/test_storage.py 3965 - webstatus = StorageStatus(ss) hunk ./src/allmydata/test/test_storage.py 3966 - self.make_shares(ss) - [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis - # set all leases to be expirable - new_expiration_time = now - 3000 + 31*24*60*60 + # create a few shares, with some leases on them + d = self.make_shares(ss) + def _do_test(ign): + [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis + # set all leases to be expirable + new_expiration_time = now - 3000 + 31*24*60*60 hunk ./src/allmydata/test/test_storage.py 3973 - def count_shares(si): - return len(list(ss._iter_share_files(si))) - def _get_sharefile(si): - return list(ss._iter_share_files(si))[0] - def count_leases(si): - return len(list(_get_sharefile(si).get_leases())) + d2 = defer.succeed(None) + d2.addCallback(lambda ign: self._backdate_leases(ss, immutable_si_0, self.renew_secrets[0:1], new_expiration_time)) + d2.addCallback(lambda ign: self._backdate_leases(ss, immutable_si_1, self.renew_secrets[1:3], new_expiration_time)) + d2.addCallback(lambda ign: self._backdate_leases(ss, mutable_si_2, self.renew_secrets[3:4], new_expiration_time)) + d2.addCallback(lambda ign: self._backdate_leases(ss, mutable_si_3, self.renew_secrets[4:6], new_expiration_time)) hunk ./src/allmydata/test/test_storage.py 3979 - sf0 = _get_sharefile(immutable_si_0) - self.backdate_lease(sf0, self.renew_secrets[0], new_expiration_time) - sf1 = _get_sharefile(immutable_si_1) - self.backdate_lease(sf1, self.renew_secrets[1], new_expiration_time) - self.backdate_lease(sf1, self.renew_secrets[2], new_expiration_time) - sf2 = _get_sharefile(mutable_si_2) - self.backdate_lease(sf2, self.renew_secrets[3], new_expiration_time) - sf3 = _get_sharefile(mutable_si_3) - self.backdate_lease(sf3, self.renew_secrets[4], new_expiration_time) - self.backdate_lease(sf3, self.renew_secrets[5], new_expiration_time) + d2.addCallback(lambda ign: ss.setServiceParent(self.s)) hunk ./src/allmydata/test/test_storage.py 3981 - ss.setServiceParent(self.s) - def _wait(): - return bool(lc.get_state()["last-cycle-finished"] is not None) - d = self.poll(_wait) + def _wait(): + return lc.get_state()["last-cycle-finished"] is not None + d2.addCallback(lambda ign: self.poll(_wait)) hunk ./src/allmydata/test/test_storage.py 3985 - def _after_first_cycle(ignored): - self.failUnlessEqual(count_shares(immutable_si_0), 1) - self.failUnlessEqual(count_leases(immutable_si_0), 1) - self.failUnlessEqual(count_shares(immutable_si_1), 1) - self.failUnlessEqual(count_leases(immutable_si_1), 2) - self.failUnlessEqual(count_shares(mutable_si_2), 0) - self.failUnlessEqual(count_shares(mutable_si_3), 0) - d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) - def _check_html(html): - s = remove_tags(html) - self.failUnlessIn("The following sharetypes will be expired: mutable.", s) - d.addCallback(_check_html) + d2.addCallback(lambda ign: self._assert_sharecount(ss, immutable_si_0, 1)) + d2.addCallback(lambda ign: self._assert_leasecount(ss, immutable_si_0, 1)) + d2.addCallback(lambda ign: self._assert_sharecount(ss, immutable_si_1, 1)) + d2.addCallback(lambda ign: self._assert_leasecount(ss, immutable_si_1, 2)) + d2.addCallback(lambda ign: self._assert_sharecount(ss, mutable_si_2, 0)) + d2.addCallback(lambda ign: self._assert_sharecount(ss, mutable_si_3, 0)) + + def _render(ign): + webstatus = StorageStatus(ss) + return self.render1(webstatus) + d2.addCallback(_render) + def _check_html(html): + s = remove_tags(html) + self.failUnlessIn("The following sharetypes will be expired: mutable.", s) + d2.addCallback(_check_html) + return d2 + d.addCallback(_do_test) return d def test_bad_mode(self): hunk ./src/allmydata/test/test_storage.py 4006 basedir = "storage/LeaseCrawler/bad_mode" - fileutil.make_dirs(basedir) + fp = FilePath(basedir) + backend = DiskBackend(fp) + + expiration_policy = { + 'enabled': True, + 'mode': 'bogus', + 'override_lease_duration': None, + 'cutoff_date': None, + 'sharetypes': ('mutable', 'immutable'), + } e = self.failUnlessRaises(ValueError, hunk ./src/allmydata/test/test_storage.py 4017 - StorageServer, basedir, "\x00" * 20, - expiration_mode="bogus") + StorageServer, "\x00" * 20, backend, fp, + expiration_policy=expiration_policy) self.failUnlessIn("GC mode 'bogus' must be 'age' or 'cutoff-date'", str(e)) def test_parse_duration(self): hunk ./src/allmydata/test/test_storage.py 4042 def test_limited_history(self): basedir = "storage/LeaseCrawler/limited_history" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20) + fp = FilePath(basedir) + backend = DiskBackend(fp) + ss = StorageServer("\x00" * 20, backend, fp) + # make it start sooner than usual. lc = ss.lease_checker lc.slow_start = 0 hunk ./src/allmydata/test/test_storage.py 4052 lc.cpu_slice = 500 # create a few shares, with some leases on them - self.make_shares(ss) - - ss.setServiceParent(self.s) + d = self.make_shares(ss) + d.addCallback(lambda ign: ss.setServiceParent(self.s)) def _wait_until_15_cycles_done(): last = lc.state["last-cycle-finished"] hunk ./src/allmydata/test/test_storage.py 4062 if lc.timer: lc.timer.reset(0) return False - d = self.poll(_wait_until_15_cycles_done) + d.addCallback(lambda ign: self.poll(_wait_until_15_cycles_done)) def _check(ignored): s = lc.get_state() hunk ./src/allmydata/test/test_storage.py 4075 def test_unpredictable_future(self): basedir = "storage/LeaseCrawler/unpredictable_future" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20) + fp = FilePath(basedir) + backend = DiskBackend(fp) + ss = StorageServer("\x00" * 20, backend, fp) + # make it start sooner than usual. lc = ss.lease_checker lc.slow_start = 0 hunk ./src/allmydata/test/test_storage.py 4084 lc.cpu_slice = -1.0 # stop quickly - self.make_shares(ss) - - ss.setServiceParent(self.s) - - d = fireEventually() + # create a few shares, with some leases on them + d = self.make_shares(ss) + d.addCallback(lambda ign: ss.setServiceParent(self.s)) def _check(ignored): hunk ./src/allmydata/test/test_storage.py 4088 - # this should fire after the first bucket is complete, but before + # this should fire after the first shareset is complete, but before # the first prefix is complete, so the progress-measurer won't # think we've gotten far enough to raise our percent-complete # above 0%, triggering the cannot-predict-the-future code in hunk ./src/allmydata/test/test_storage.py 4093 # expirer.py . This will have to change if/when the - # progress-measurer gets smart enough to count buckets (we'll + # progress-measurer gets smart enough to count sharesets (we'll # have to interrupt it even earlier, before it's finished the # first shareset). s = lc.get_state() hunk ./src/allmydata/test/test_storage.py 4105 self.failUnlessIn("estimated-remaining-cycle", s) self.failUnlessIn("estimated-current-cycle", s) - left = s["estimated-remaining-cycle"]["space-recovered"] - self.failUnlessEqual(left["actual-buckets"], None) - self.failUnlessEqual(left["original-buckets"], None) - self.failUnlessEqual(left["configured-buckets"], None) - self.failUnlessEqual(left["actual-shares"], None) - self.failUnlessEqual(left["original-shares"], None) - self.failUnlessEqual(left["configured-shares"], None) - self.failUnlessEqual(left["actual-diskbytes"], None) - self.failUnlessEqual(left["original-diskbytes"], None) - self.failUnlessEqual(left["configured-diskbytes"], None) - self.failUnlessEqual(left["actual-sharebytes"], None) - self.failUnlessEqual(left["original-sharebytes"], None) - self.failUnlessEqual(left["configured-sharebytes"], None) - - full = s["estimated-remaining-cycle"]["space-recovered"] - self.failUnlessEqual(full["actual-buckets"], None) - self.failUnlessEqual(full["original-buckets"], None) - self.failUnlessEqual(full["configured-buckets"], None) - self.failUnlessEqual(full["actual-shares"], None) - self.failUnlessEqual(full["original-shares"], None) - self.failUnlessEqual(full["configured-shares"], None) - self.failUnlessEqual(full["actual-diskbytes"], None) - self.failUnlessEqual(full["original-diskbytes"], None) - self.failUnlessEqual(full["configured-diskbytes"], None) - self.failUnlessEqual(full["actual-sharebytes"], None) - self.failUnlessEqual(full["original-sharebytes"], None) - self.failUnlessEqual(full["configured-sharebytes"], None) + def _do_check(cycle): + self.failUnlessIn("space-recovered", cycle) + rec = cycle["space-recovered"] + self.failUnlessEqual(rec["actual-buckets"], None) + self.failUnlessEqual(rec["original-buckets"], None) + self.failUnlessEqual(rec["configured-buckets"], None) + self.failUnlessEqual(rec["actual-shares"], None) + self.failUnlessEqual(rec["original-shares"], None) + self.failUnlessEqual(rec["configured-shares"], None) + self.failUnlessEqual(rec["actual-diskbytes"], None) + self.failUnlessEqual(rec["original-diskbytes"], None) + self.failUnlessEqual(rec["configured-diskbytes"], None) + self.failUnlessEqual(rec["actual-sharebytes"], None) + self.failUnlessEqual(rec["original-sharebytes"], None) + self.failUnlessEqual(rec["configured-sharebytes"], None) hunk ./src/allmydata/test/test_storage.py 4121 + _do_check(s["estimated-remaining-cycle"]) + _do_check(s["estimated-current-cycle"]) d.addCallback(_check) return d hunk ./src/allmydata/test/test_storage.py 4127 def test_no_st_blocks(self): - basedir = "storage/LeaseCrawler/no_st_blocks" - fileutil.make_dirs(basedir) - ss = No_ST_BLOCKS_StorageServer(basedir, "\x00" * 20, - expiration_mode="age", - expiration_override_lease_duration=-1000) - # a negative expiration_time= means the "configured-" - # space-recovered counts will be non-zero, since all shares will have - # expired by then + # TODO: replace with @patch that supports Deferreds. hunk ./src/allmydata/test/test_storage.py 4129 - # make it start sooner than usual. - lc = ss.lease_checker - lc.slow_start = 0 + class BrokenStatResults: + pass hunk ./src/allmydata/test/test_storage.py 4132 - self.make_shares(ss) - ss.setServiceParent(self.s) - def _wait(): - return bool(lc.get_state()["last-cycle-finished"] is not None) - d = self.poll(_wait) + def call_stat(fn): + s = self.old_os_stat(fn) + bsr = BrokenStatResults() + for attrname in dir(s): + if attrname.startswith("_"): + continue + if attrname == "st_blocks": + continue + setattr(bsr, attrname, getattr(s, attrname)) hunk ./src/allmydata/test/test_storage.py 4142 - def _check(ignored): - s = lc.get_state() - last = s["history"][0] - rec = last["space-recovered"] - self.failUnlessEqual(rec["configured-buckets"], 4) - self.failUnlessEqual(rec["configured-shares"], 4) - self.failUnless(rec["configured-sharebytes"] > 0, - rec["configured-sharebytes"]) - # without the .st_blocks field in os.stat() results, we should be - # reporting diskbytes==sharebytes - self.failUnlessEqual(rec["configured-sharebytes"], - rec["configured-diskbytes"]) - d.addCallback(_check) - return d + # pretend that the directory overhead is zero + if stat.S_ISDIR(bsr.st_mode): + bsr.st_size = 0 + return bsr + + def _cleanup(res): + os.stat = self.old_os_stat + return res + + self.old_os_stat = os.stat + try: + os.stat = call_stat + + basedir = "storage/LeaseCrawler/no_st_blocks" + fp = FilePath(basedir) + backend = DiskBackend(fp) + + # A negative 'override_lease_duration' means that the "configured-" + # space-recovered counts will be non-zero, since all shares will have + # expired by then. + expiration_policy = { + 'enabled': True, + 'mode': 'age', + 'override_lease_duration': -1000, + 'sharetypes': ('mutable', 'immutable'), + } + ss = StorageServer("\x00" * 20, backend, fp, expiration_policy=expiration_policy) + + # make it start sooner than usual. + lc = ss.lease_checker + lc.slow_start = 0 + + d = self.make_shares(ss) + d.addCallback(lambda ign: ss.setServiceParent(self.s)) + def _wait(): + return lc.get_state()["last-cycle-finished"] is not None + d.addCallback(lambda ign: self.poll(_wait)) + + def _check(ignored): + s = lc.get_state() + self.failUnlessIn("history", s) + history = s["history"] + self.failUnlessIn(0, history) + last = history[0] + self.failUnlessIn("space-recovered", last) + rec = last["space-recovered"] + self.failUnlessEqual(rec["configured-buckets"], 4, str(rec)) + self.failUnlessEqual(rec["configured-shares"], 4, str(rec)) + self.failUnless(rec["configured-sharebytes"] > 0, str(rec)) + # without the .st_blocks field in os.stat() results, and with directory + # overhead not counted, we should be reporting diskbytes==sharebytes + self.failUnlessEqual(rec["configured-sharebytes"], + rec["configured-diskbytes"], str(rec)) + d.addCallback(_check) + d.addBoth(_cleanup) + return d + except Exception: + _cleanup(None) + raise def test_share_corruption(self): self._poll_should_ignore_these_errors = [ hunk ./src/allmydata/test/test_storage.py 4208 UnknownImmutableContainerVersionError, ] basedir = "storage/LeaseCrawler/share_corruption" - fileutil.make_dirs(basedir) - ss = InstrumentedStorageServer(basedir, "\x00" * 20) - w = StorageStatus(ss) + fp = FilePath(basedir) + backend = DiskBackend(fp) + ss = InstrumentedStorageServer("\x00" * 20, backend, fp) + # make it start sooner than usual. lc = ss.lease_checker hunk ./src/allmydata/test/test_storage.py 4214 - lc.stop_after_first_bucket = True lc.slow_start = 0 lc.cpu_slice = 500 hunk ./src/allmydata/test/test_storage.py 4218 # create a few shares, with some leases on them - self.make_shares(ss) + d = self.make_shares(ss) + def _do_test(ign): + [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis hunk ./src/allmydata/test/test_storage.py 4222 - # now corrupt one, and make sure the lease-checker keeps going - [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis - first = min(self.sis) - first_b32 = base32.b2a(first) - fn = os.path.join(ss.sharedir, storage_index_to_dir(first), "0") - f = open(fn, "rb+") - f.seek(0) - f.write("BAD MAGIC") - f.close() - # if get_share_file() doesn't see the correct mutable magic, it - # assumes the file is an immutable share, and then - # immutable.ShareFile sees a bad version. So regardless of which kind - # of share we corrupted, this will trigger an - # UnknownImmutableContainerVersionError. + # now corrupt one, and make sure the lease-checker keeps going + first = min(self.sis) + first_b32 = base32.b2a(first) + fp = ss.backend.get_shareset(first)._get_sharedir().child("0") + f = fp.open("rb+") + try: + f.seek(0) + f.write("BAD MAGIC") + finally: + f.close() hunk ./src/allmydata/test/test_storage.py 4233 - # also create an empty bucket - empty_si = base32.b2a("\x04"*16) - empty_bucket_dir = os.path.join(ss.sharedir, - storage_index_to_dir(empty_si)) - fileutil.make_dirs(empty_bucket_dir) + # If the backend doesn't see the correct mutable magic, it + # assumes the file is an immutable share, and then the immutable + # share class will see a bad version. So regardless of which kind + # of share we corrupted, this will trigger an + # UnknownImmutableContainerVersionError. hunk ./src/allmydata/test/test_storage.py 4239 - ss.setServiceParent(self.s) + # also create an empty shareset + empty_si = base32.b2a("\x04"*16) + empty_si_dir = ss.backend.get_shareset(empty_si)._get_sharedir() + fileutil.fp_make_dirs(empty_si_dir) hunk ./src/allmydata/test/test_storage.py 4244 - d = fireEventually() + d2 = defer.Deferred() + lc.hook_ds = [d2] + ss.setServiceParent(self.s) hunk ./src/allmydata/test/test_storage.py 4248 - # now examine the state right after the first bucket has been - # processed. - def _after_first_bucket(ignored): - s = lc.get_state() - if "cycle-to-date" not in s: - d2 = fireEventually() - d2.addCallback(_after_first_bucket) - return d2 - so_far = s["cycle-to-date"] - rec = so_far["space-recovered"] - self.failUnlessEqual(rec["examined-buckets"], 1) - self.failUnlessEqual(rec["examined-shares"], 0) - self.failUnlessEqual(so_far["corrupt-shares"], [(first_b32, 0)]) - d.addCallback(_after_first_bucket) + # now examine the state right after the first shareset has been + # processed. + def _after_first_shareset(ignored): + s = lc.get_state() + self.failUnlessIn("cycle-to-date", s) + so_far = s["cycle-to-date"] + self.failUnlessIn("space-recovered", so_far) + rec = so_far["space-recovered"] + self.failUnlessEqual(rec["examined-buckets"], 1, str(rec)) + self.failUnlessEqual(rec["examined-shares"], 0, str(rec)) + self.failUnlessEqual(so_far["corrupt-shares"], [(first_b32, 0)]) + d2.addCallback(_after_first_shareset) hunk ./src/allmydata/test/test_storage.py 4261 - d.addCallback(lambda ign: self.render_json(w)) - def _check_json(json): - data = simplejson.loads(json) - # grr. json turns all dict keys into strings. - so_far = data["lease-checker"]["cycle-to-date"] - corrupt_shares = so_far["corrupt-shares"] - # it also turns all tuples into lists - self.failUnlessEqual(corrupt_shares, [[first_b32, 0]]) - d.addCallback(_check_json) - d.addCallback(lambda ign: self.render1(w)) - def _check_html(html): - s = remove_tags(html) - self.failUnlessIn("Corrupt shares: SI %s shnum 0" % first_b32, s) - d.addCallback(_check_html) + def _render_json(ign): + webstatus = StorageStatus(ss) + return self.render_json(webstatus) + d2.addCallback(_render_json) + def _check_json(json): + data = simplejson.loads(json) + # grr. json turns all dict keys into strings. + self.failUnlessIn("lease-checker", data) + s = data["lease-checker"] + self.failUnlessIn("cycle-to-date", s) + so_far = s["cycle-to-date"] + corrupt_shares = so_far["corrupt-shares"] + # it also turns all tuples into lists + self.failUnlessEqual(corrupt_shares, [[first_b32, 0]]) + d2.addCallback(_check_json) hunk ./src/allmydata/test/test_storage.py 4277 - def _wait(): - return bool(lc.get_state()["last-cycle-finished"] is not None) - d.addCallback(lambda ign: self.poll(_wait)) + def _render(ign): + webstatus = StorageStatus(ss) + return self.render1(webstatus) + d2.addCallback(_render) + def _check_html(html): + s = remove_tags(html) + self.failUnlessIn("Corrupt shares: SI %s shnum 0" % first_b32, s) + d2.addCallback(_check_html) hunk ./src/allmydata/test/test_storage.py 4286 - def _after_first_cycle(ignored): - s = lc.get_state() - last = s["history"][0] - rec = last["space-recovered"] - self.failUnlessEqual(rec["examined-buckets"], 5) - self.failUnlessEqual(rec["examined-shares"], 3) - self.failUnlessEqual(last["corrupt-shares"], [(first_b32, 0)]) - d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render_json(w)) - def _check_json_history(json): - data = simplejson.loads(json) - last = data["lease-checker"]["history"]["0"] - corrupt_shares = last["corrupt-shares"] - self.failUnlessEqual(corrupt_shares, [[first_b32, 0]]) - d.addCallback(_check_json_history) - d.addCallback(lambda ign: self.render1(w)) - def _check_html_history(html): - s = remove_tags(html) - self.failUnlessIn("Corrupt shares: SI %s shnum 0" % first_b32, s) - d.addCallback(_check_html_history) + def _wait(): + return lc.get_state()["last-cycle-finished"] is not None + d2.addCallback(lambda ign: self.poll(_wait)) hunk ./src/allmydata/test/test_storage.py 4290 - def _cleanup(res): - self.flushLoggedErrors(UnknownMutableContainerVersionError, - UnknownImmutableContainerVersionError) - return res - d.addBoth(_cleanup) + def _after_first_cycle(ignored): + s = lc.get_state() + last = s["history"][0] + rec = last["space-recovered"] + self.failUnlessEqual(rec["examined-buckets"], 5) + self.failUnlessEqual(rec["examined-shares"], 3) + self.failUnlessEqual(last["corrupt-shares"], [(first_b32, 0)]) + d2.addCallback(_after_first_cycle) + + d2.addCallback(_render_json) + def _check_json_history(json): + data = simplejson.loads(json) + last = data["lease-checker"]["history"]["0"] + corrupt_shares = last["corrupt-shares"] + self.failUnlessEqual(corrupt_shares, [[first_b32, 0]]) + d2.addCallback(_check_json_history) + + d2.addCallback(_render) + def _check_html_history(html): + s = remove_tags(html) + self.failUnlessIn("Corrupt shares: SI %s shnum 0" % first_b32, s) + d2.addCallback(_check_html_history) + + def _cleanup(res): + self.flushLoggedErrors(UnknownMutableContainerVersionError, + UnknownImmutableContainerVersionError) + return res + d2.addBoth(_cleanup) + return d2 + d.addCallback(_do_test) return d def render_json(self, page): hunk ./src/allmydata/test/test_storage.py 4326 d = self.render1(page, args={"t": ["json"]}) return d +LeaseCrawler.skip = "takes too long" + + class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): def setUp(self): hunk ./src/allmydata/test/test_storage.py 4344 def test_status(self): basedir = "storage/WebStatus/status" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20) + fp = FilePath(basedir) + backend = DiskBackend(fp) + ss = StorageServer("\x00" * 20, backend, fp) ss.setServiceParent(self.s) w = StorageStatus(ss) d = self.render1(w) hunk ./src/allmydata/test/test_storage.py 4378 # Some platforms may have no disk stats API. Make sure the code can handle that # (test runs on all platforms). basedir = "storage/WebStatus/status_no_disk_stats" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20) + fp = FilePath(basedir) + backend = DiskBackend(fp) + ss = StorageServer("\x00" * 20, backend, fp) ss.setServiceParent(self.s) w = StorageStatus(ss) html = w.renderSynchronously() hunk ./src/allmydata/test/test_storage.py 4398 # If the API to get disk stats exists but a call to it fails, then the status should # show that no shares will be accepted, and get_available_space() should be 0. basedir = "storage/WebStatus/status_bad_disk_stats" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20) + fp = FilePath(basedir) + backend = DiskBackend(fp) + ss = StorageServer("\x00" * 20, backend, fp) ss.setServiceParent(self.s) w = StorageStatus(ss) html = w.renderSynchronously() hunk ./src/allmydata/test/test_storage.py 4429 } basedir = "storage/WebStatus/status_right_disk_stats" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20, reserved_space=reserved_space) - expecteddir = ss.sharedir + fp = FilePath(basedir) + backend = DiskBackend(fp, readonly=False, reserved_space=reserved_space) + ss = StorageServer("\x00" * 20, backend, fp) + expecteddir = backend._sharedir ss.setServiceParent(self.s) w = StorageStatus(ss) html = w.renderSynchronously() hunk ./src/allmydata/test/test_storage.py 4452 def test_readonly(self): basedir = "storage/WebStatus/readonly" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20, readonly_storage=True) + fp = FilePath(basedir) + backend = DiskBackend(fp, readonly=True) + ss = StorageServer("\x00" * 20, backend, fp) ss.setServiceParent(self.s) w = StorageStatus(ss) html = w.renderSynchronously() hunk ./src/allmydata/test/test_storage.py 4464 def test_reserved(self): basedir = "storage/WebStatus/reserved" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20, reserved_space=10e6) - ss.setServiceParent(self.s) - w = StorageStatus(ss) - html = w.renderSynchronously() - self.failUnlessIn("

Storage Server Status

", html) - s = remove_tags(html) - self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s) - - def test_huge_reserved(self): - basedir = "storage/WebStatus/reserved" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20, reserved_space=10e6) + fp = FilePath(basedir) + backend = DiskBackend(fp, readonly=False, reserved_space=10e6) + ss = StorageServer("\x00" * 20, backend, fp) ss.setServiceParent(self.s) w = StorageStatus(ss) html = w.renderSynchronously() hunk ./src/allmydata/test/test_system.py 7 from twisted.trial import unittest from twisted.internet import defer from twisted.internet import threads # CLI tests use deferToThread +from twisted.python.filepath import FilePath import allmydata from allmydata import uri hunk ./src/allmydata/test/test_system.py 11 -from allmydata.storage.mutable import MutableShareFile +from allmydata.storage.backends.disk.disk_backend import DiskBackend +from allmydata.storage.backends.disk.mutable import load_mutable_disk_share from allmydata.storage.server import si_a2b from allmydata.immutable import offloaded, upload from allmydata.immutable.literal import LiteralFileNode hunk ./src/allmydata/test/test_system.py 57 self.interrupt_after_d.callback(self) return upload.Data.read(self, length) -class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): + +class SystemTest(SystemTestMixin, RunBinTahoeMixin): timeout = 3600 # It takes longer than 960 seconds on Zandr's ARM box. def test_connections(self): hunk ./src/allmydata/test/test_system.py 62 - self.basedir = "system/SystemTest/test_connections" + self.basedir = self.workdir("test_connections") d = self.set_up_nodes() self.extra_node = None d.addCallback(lambda res: self.add_extra_node(self.numclients)) hunk ./src/allmydata/test/test_system.py 90 del test_connections def test_upload_and_download_random_key(self): - self.basedir = "system/SystemTest/test_upload_and_download_random_key" + self.basedir = self.workdir("test_upload_and_download_random_key") return self._test_upload_and_download(convergence=None) def test_upload_and_download_convergent(self): hunk ./src/allmydata/test/test_system.py 94 - self.basedir = "system/SystemTest/test_upload_and_download_convergent" + self.basedir = self.workdir("test_upload_and_download_convergent") return self._test_upload_and_download(convergence="some convergence string") def _test_upload_and_download(self, convergence): hunk ./src/allmydata/test/test_system.py 200 facility="tahoe.tests") d1 = download_to_data(badnode) def _baduri_should_fail(res): - log.msg("finished downloading non-existend URI", + log.msg("finished downloading non-existent URI", level=log.UNUSUAL, facility="tahoe.tests") self.failUnless(isinstance(res, Failure)) self.failUnless(res.check(NoSharesError), hunk ./src/allmydata/test/test_system.py 423 self.fail("unable to find any share files in %s" % basedir) return shares - def _corrupt_mutable_share(self, filename, which): - msf = MutableShareFile(filename) - datav = msf.readv([ (0, 1000000) ]) - final_share = datav[0] - assert len(final_share) < 1000000 # ought to be truncated - pieces = mutable_layout.unpack_share(final_share) - (seqnum, root_hash, IV, k, N, segsize, datalen, - verification_key, signature, share_hash_chain, block_hash_tree, - share_data, enc_privkey) = pieces + def _corrupt_mutable_share(self, ign, what, which): + (storageindex, filename, shnum) = what + d = defer.succeed(None) + d.addCallback(lambda ign: load_mutable_disk_share(FilePath(filename), storageindex, shnum)) + def _got_share(msf): + d2 = msf.readv([ (0, 1000000) ]) + def _got_data(datav): + final_share = datav[0] + assert len(final_share) < 1000000 # ought to be truncated + pieces = mutable_layout.unpack_share(final_share) + (seqnum, root_hash, IV, k, N, segsize, datalen, + verification_key, signature, share_hash_chain, block_hash_tree, + share_data, enc_privkey) = pieces hunk ./src/allmydata/test/test_system.py 437 - if which == "seqnum": - seqnum = seqnum + 15 - elif which == "R": - root_hash = self.flip_bit(root_hash) - elif which == "IV": - IV = self.flip_bit(IV) - elif which == "segsize": - segsize = segsize + 15 - elif which == "pubkey": - verification_key = self.flip_bit(verification_key) - elif which == "signature": - signature = self.flip_bit(signature) - elif which == "share_hash_chain": - nodenum = share_hash_chain.keys()[0] - share_hash_chain[nodenum] = self.flip_bit(share_hash_chain[nodenum]) - elif which == "block_hash_tree": - block_hash_tree[-1] = self.flip_bit(block_hash_tree[-1]) - elif which == "share_data": - share_data = self.flip_bit(share_data) - elif which == "encprivkey": - enc_privkey = self.flip_bit(enc_privkey) + if which == "seqnum": + seqnum = seqnum + 15 + elif which == "R": + root_hash = self.flip_bit(root_hash) + elif which == "IV": + IV = self.flip_bit(IV) + elif which == "segsize": + segsize = segsize + 15 + elif which == "pubkey": + verification_key = self.flip_bit(verification_key) + elif which == "signature": + signature = self.flip_bit(signature) + elif which == "share_hash_chain": + nodenum = share_hash_chain.keys()[0] + share_hash_chain[nodenum] = self.flip_bit(share_hash_chain[nodenum]) + elif which == "block_hash_tree": + block_hash_tree[-1] = self.flip_bit(block_hash_tree[-1]) + elif which == "share_data": + share_data = self.flip_bit(share_data) + elif which == "encprivkey": + enc_privkey = self.flip_bit(enc_privkey) hunk ./src/allmydata/test/test_system.py 459 - prefix = mutable_layout.pack_prefix(seqnum, root_hash, IV, k, N, - segsize, datalen) - final_share = mutable_layout.pack_share(prefix, - verification_key, - signature, - share_hash_chain, - block_hash_tree, - share_data, - enc_privkey) - msf.writev( [(0, final_share)], None) + prefix = mutable_layout.pack_prefix(seqnum, root_hash, IV, k, N, + segsize, datalen) + final_share = mutable_layout.pack_share(prefix, + verification_key, + signature, + share_hash_chain, + block_hash_tree, + share_data, + enc_privkey) hunk ./src/allmydata/test/test_system.py 469 + return msf.writev( [(0, final_share)], None) + d2.addCallback(_got_data) + return d2 + d.addCallback(_got_share) + return d def test_mutable(self): hunk ./src/allmydata/test/test_system.py 476 - self.basedir = "system/SystemTest/test_mutable" + self.basedir = self.workdir("test_mutable") DATA = "initial contents go here." # 25 bytes % 3 != 0 DATA_uploadable = MutableData(DATA) NEWDATA = "new contents yay" hunk ./src/allmydata/test/test_system.py 511 filename], stdout=out, stderr=err) output = out.getvalue() + self.failUnlessEqual(err.getvalue(), "") self.failUnlessEqual(rc, 0) try: hunk ./src/allmydata/test/test_system.py 514 - self.failUnless("Mutable slot found:\n" in output) - self.failUnless("share_type: SDMF\n" in output) + self.failUnlessIn("Mutable slot found:\n", output) + self.failUnlessIn("share_type: SDMF\n", output) peerid = idlib.nodeid_b2a(self.clients[client_num].nodeid) hunk ./src/allmydata/test/test_system.py 517 - self.failUnless(" WE for nodeid: %s\n" % peerid in output) - self.failUnless(" num_extra_leases: 0\n" in output) - self.failUnless(" secrets are for nodeid: %s\n" % peerid - in output) - self.failUnless(" SDMF contents:\n" in output) - self.failUnless(" seqnum: 1\n" in output) - self.failUnless(" required_shares: 3\n" in output) - self.failUnless(" total_shares: 10\n" in output) - self.failUnless(" segsize: 27\n" in output, (output, filename)) - self.failUnless(" datalen: 25\n" in output) + self.failUnlessIn(" WE for nodeid: %s\n" % peerid, output) + self.failUnlessIn(" num_extra_leases: 0\n", output) + if isinstance(self.clients[client_num], DiskBackend): + self.failUnlessIn(" secrets are for nodeid: %s\n" % peerid, output) + self.failUnlessIn(" SDMF contents:\n", output) + self.failUnlessIn(" seqnum: 1\n", output) + self.failUnlessIn(" required_shares: 3\n", output) + self.failUnlessIn(" total_shares: 10\n", output) + self.failUnlessIn(" segsize: 27\n", output) + self.failUnlessIn(" datalen: 25\n", output) # the exact share_hash_chain nodes depends upon the sharenum, # and is more of a hassle to compute than I want to deal with # now hunk ./src/allmydata/test/test_system.py 530 - self.failUnless(" share_hash_chain: " in output) - self.failUnless(" block_hash_tree: 1 nodes\n" in output) + self.failUnlessIn(" share_hash_chain: ", output) + self.failUnlessIn(" block_hash_tree: 1 nodes\n", output) expected = (" verify-cap: URI:SSK-Verifier:%s:" % base32.b2a(storage_index)) self.failUnless(expected in output) hunk ./src/allmydata/test/test_system.py 607 shares = self._find_all_shares(self.basedir) ## sort by share number #shares.sort( lambda a,b: cmp(a[3], b[3]) ) - where = dict([ (shnum, filename) - for (client_num, storage_index, filename, shnum) + where = dict([ (shnum, (storageindex, filename, shnum)) + for (client_num, storageindex, filename, shnum) in shares ]) assert len(where) == 10 # this test is designed for 3-of-10 hunk ./src/allmydata/test/test_system.py 611 - for shnum, filename in where.items(): + + d2 = defer.succeed(None) + for shnum, what in where.items(): # shares 7,8,9 are left alone. read will check # (share_hash_chain, block_hash_tree, share_data). New # seqnum+R pairs will trigger a check of (seqnum, R, IV, hunk ./src/allmydata/test/test_system.py 621 if shnum == 0: # read: this will trigger "pubkey doesn't match # fingerprint". - self._corrupt_mutable_share(filename, "pubkey") - self._corrupt_mutable_share(filename, "encprivkey") + d2.addCallback(self._corrupt_mutable_share, what, "pubkey") + d2.addCallback(self._corrupt_mutable_share, what, "encprivkey") elif shnum == 1: # triggers "signature is invalid" hunk ./src/allmydata/test/test_system.py 625 - self._corrupt_mutable_share(filename, "seqnum") + d2.addCallback(self._corrupt_mutable_share, what, "seqnum") elif shnum == 2: # triggers "signature is invalid" hunk ./src/allmydata/test/test_system.py 628 - self._corrupt_mutable_share(filename, "R") + d2.addCallback(self._corrupt_mutable_share, what, "R") elif shnum == 3: # triggers "signature is invalid" hunk ./src/allmydata/test/test_system.py 631 - self._corrupt_mutable_share(filename, "segsize") + d2.addCallback(self._corrupt_mutable_share, what, "segsize") elif shnum == 4: hunk ./src/allmydata/test/test_system.py 633 - self._corrupt_mutable_share(filename, "share_hash_chain") + d2.addCallback(self._corrupt_mutable_share, what, "share_hash_chain") elif shnum == 5: hunk ./src/allmydata/test/test_system.py 635 - self._corrupt_mutable_share(filename, "block_hash_tree") + d2.addCallback(self._corrupt_mutable_share, what, "block_hash_tree") elif shnum == 6: hunk ./src/allmydata/test/test_system.py 637 - self._corrupt_mutable_share(filename, "share_data") + d2.addCallback(self._corrupt_mutable_share, what, "share_data") # other things to correct: IV, signature # 7,8,9 are left alone hunk ./src/allmydata/test/test_system.py 653 # for one failure mode at a time. # when we retrieve this, we should get three signature - # failures (where we've mangled seqnum, R, and segsize). The - # pubkey mangling + # failures (where we've mangled seqnum, R, and segsize). + return d2 d.addCallback(_corrupt_shares) d.addCallback(lambda res: self._newnode3.download_best_version()) hunk ./src/allmydata/test/test_system.py 729 # plaintext_hash check. def test_filesystem(self): - self.basedir = "system/SystemTest/test_filesystem" + self.basedir = self.workdir("test_filesystem") self.data = LARGE_DATA d = self.set_up_nodes(use_stats_gatherer=True) def _new_happy_semantics(ign): hunk ./src/allmydata/test/test_system.py 1342 unicode_to_argv(filename)], stdout=out, stderr=err) output = out.getvalue() + self.failUnlessEqual(err.getvalue(), "") self.failUnlessEqual(rc, 0) # we only upload a single file, so we can assert some things about hunk ./src/allmydata/test/test_system.py 1348 # its size and shares. self.failUnlessIn("share filename: %s" % quote_output(abspath_expanduser_unicode(filename)), output) - self.failUnlessIn("size: %d\n" % len(self.data), output) - self.failUnlessIn("num_segments: 1\n", output) + self.failUnlessIn(" file_size: %d\n" % len(self.data), output) + self.failUnlessIn(" num_segments: 1\n", output) # segment_size is always a multiple of needed_shares hunk ./src/allmydata/test/test_system.py 1351 - self.failUnlessIn("segment_size: %d\n" % mathutil.next_multiple(len(self.data), 3), output) - self.failUnlessIn("total_shares: 10\n", output) + self.failUnlessIn(" segment_size: %d\n" % mathutil.next_multiple(len(self.data), 3), output) + self.failUnlessIn(" total_shares: 10\n", output) # keys which are supposed to be present hunk ./src/allmydata/test/test_system.py 1354 - for key in ("size", "num_segments", "segment_size", + for key in ("file_size", "num_segments", "segment_size", "needed_shares", "total_shares", "codec_name", "codec_params", "tail_codec_params", #"plaintext_hash", "plaintext_root_hash", hunk ./src/allmydata/test/test_system.py 1360 "crypttext_hash", "crypttext_root_hash", "share_root_hash", "UEB_hash"): - self.failUnlessIn("%s: " % key, output) + self.failUnlessIn(" %s: " % key, output) self.failUnlessIn(" verify-cap: URI:CHK-Verifier:", output) # now use its storage index to find the other shares using the hunk ./src/allmydata/test/test_system.py 1372 nodedirs = [self.getdir("client%d" % i) for i in range(self.numclients)] cmd = ["debug", "find-shares", storage_index_s] + nodedirs rc = runner.runner(cmd, stdout=out, stderr=err) + self.failUnlessEqual(err.getvalue(), "") self.failUnlessEqual(rc, 0) out.seek(0) sharefiles = [sfn.strip() for sfn in out.readlines()] hunk ./src/allmydata/test/test_system.py 1383 nodedirs = [self.getdir("client%d" % i) for i in range(self.numclients)] cmd = ["debug", "catalog-shares"] + nodedirs rc = runner.runner(cmd, stdout=out, stderr=err) + self.failUnlessEqual(err.getvalue(), "") self.failUnlessEqual(rc, 0) out.seek(0) descriptions = [sfn.strip() for sfn in out.readlines()] hunk ./src/allmydata/test/test_system.py 1387 - self.failUnlessEqual(len(descriptions), 30) + self.failUnlessEqual(len(descriptions), 30, repr((cmd, descriptions))) matching = [line for line in descriptions if line.startswith("CHK %s " % storage_index_s)] hunk ./src/allmydata/test/test_system.py 1766 def test_filesystem_with_cli_in_subprocess(self): # We do this in a separate test so that test_filesystem doesn't skip if we can't run bin/tahoe. - self.basedir = "system/SystemTest/test_filesystem_with_cli_in_subprocess" + self.basedir = self.workdir("test_filesystem_with_cli_in_subprocess") d = self.set_up_nodes() def _new_happy_semantics(ign): for c in self.clients: hunk ./src/allmydata/test/test_system.py 1892 return d d.addCallback(_got_lit_filenode) return d + + +class SystemWithDiskBackend(SystemTest, unittest.TestCase): + # The disk backend can use default options. + pass + + +class SystemWithS3Backend(SystemTest, unittest.TestCase): + def _get_extra_config(self, i): + # all nodes are storage servers + return ("[storage]\n" + "backend = mock_s3\n") hunk ./src/allmydata/test/test_upload.py 3 # -*- coding: utf-8 -*- -import os, shutil +import os from cStringIO import StringIO from twisted.trial import unittest from twisted.python.failure import Failure hunk ./src/allmydata/test/test_upload.py 14 from allmydata import uri, monitor, client from allmydata.immutable import upload, encode from allmydata.interfaces import FileTooLargeError, UploadUnhappinessError -from allmydata.util import log, base32 +from allmydata.util import log, base32, fileutil from allmydata.util.assertutil import precondition from allmydata.util.deferredutil import DeferredListShouldSucceed from allmydata.test.no_network import GridTestMixin hunk ./src/allmydata/test/test_upload.py 22 from allmydata.util.happinessutil import servers_of_happiness, \ shares_by_server, merge_servers from allmydata.storage_client import StorageFarmBroker -from allmydata.storage.server import storage_index_to_dir MiB = 1024*1024 hunk ./src/allmydata/test/test_upload.py 746 servertoshnums = {} # k: server, v: set(shnum) for i, c in self.g.servers_by_number.iteritems(): - for (dirp, dirns, fns) in os.walk(c.sharedir): + for (dirp, dirns, fns) in os.walk(c.backend._sharedir.path): for fn in fns: try: sharenum = int(fn) hunk ./src/allmydata/test/test_upload.py 820 if share_number is not None: self._copy_share_to_server(share_number, server_number) - def _copy_share_to_server(self, share_number, server_number): ss = self.g.servers_by_number[server_number] hunk ./src/allmydata/test/test_upload.py 822 - # Copy share i from the directory associated with the first - # storage server to the directory associated with this one. - assert self.g, "I tried to find a grid at self.g, but failed" - assert self.shares, "I tried to find shares at self.shares, but failed" - old_share_location = self.shares[share_number][2] - new_share_location = os.path.join(ss.storedir, "shares") - si = uri.from_string(self.uri).get_storage_index() - new_share_location = os.path.join(new_share_location, - storage_index_to_dir(si)) - if not os.path.exists(new_share_location): - os.makedirs(new_share_location) - new_share_location = os.path.join(new_share_location, - str(share_number)) - if old_share_location != new_share_location: - shutil.copy(old_share_location, new_share_location) - shares = self.find_uri_shares(self.uri) - # Make sure that the storage server has the share. - self.failUnless((share_number, ss.my_nodeid, new_share_location) - in shares) + self.copy_share(self.shares[share_number], self.uri, ss) def _setup_grid(self): """ hunk ./src/allmydata/test/test_upload.py 974 readonly=True)) # Remove the first share from server 0. def _remove_share_0_from_server_0(): - share_location = self.shares[0][2] - os.remove(share_location) + self.shares[0][2].remove() d.addCallback(lambda ign: _remove_share_0_from_server_0()) # Set happy = 4 in the client. hunk ./src/allmydata/test/test_upload.py 1103 self._copy_share_to_server(i, 2) d.addCallback(_copy_shares) # Remove the first server, and add a placeholder with share 0 - d.addCallback(lambda ign: - self.g.remove_server(self.g.servers_by_number[0].my_nodeid)) + d.addCallback(lambda ign: self.remove_server(0)) d.addCallback(lambda ign: self._add_server_with_share(server_number=4, share_number=0)) # Now try uploading. hunk ./src/allmydata/test/test_upload.py 1134 d.addCallback(lambda ign: self._add_server(server_number=4)) d.addCallback(_copy_shares) - d.addCallback(lambda ign: - self.g.remove_server(self.g.servers_by_number[0].my_nodeid)) + d.addCallback(lambda ign: self.remove_server(0)) d.addCallback(_reset_encoding_parameters) d.addCallback(lambda client: client.upload(upload.Data("data" * 10000, convergence=""))) hunk ./src/allmydata/test/test_upload.py 1196 self._copy_share_to_server(i, 2) d.addCallback(_copy_shares) # Remove server 0, and add another in its place - d.addCallback(lambda ign: - self.g.remove_server(self.g.servers_by_number[0].my_nodeid)) + d.addCallback(lambda ign: self.remove_server(0)) d.addCallback(lambda ign: self._add_server_with_share(server_number=4, share_number=0, readonly=True)) hunk ./src/allmydata/test/test_upload.py 1237 for i in xrange(1, 10): self._copy_share_to_server(i, 2) d.addCallback(_copy_shares) - d.addCallback(lambda ign: - self.g.remove_server(self.g.servers_by_number[0].my_nodeid)) + d.addCallback(lambda ign: self.remove_server(0)) def _reset_encoding_parameters(ign, happy=4): client = self.g.clients[0] client.DEFAULT_ENCODING_PARAMETERS['happy'] = happy hunk ./src/allmydata/test/test_upload.py 1273 # remove the original server # (necessary to ensure that the Tahoe2ServerSelector will distribute # all the shares) - def _remove_server(ign): - server = self.g.servers_by_number[0] - self.g.remove_server(server.my_nodeid) - d.addCallback(_remove_server) + d.addCallback(lambda ign: self.remove_server(0)) # This should succeed; we still have 4 servers, and the # happiness of the upload is 4. d.addCallback(lambda ign: hunk ./src/allmydata/test/test_upload.py 1285 d.addCallback(lambda ign: self._setup_and_upload()) d.addCallback(_do_server_setup) - d.addCallback(_remove_server) + d.addCallback(lambda ign: self.remove_server(0)) d.addCallback(lambda ign: self.shouldFail(UploadUnhappinessError, "test_dropped_servers_in_encoder", hunk ./src/allmydata/test/test_upload.py 1307 self._add_server_with_share(4, 7, readonly=True) self._add_server_with_share(5, 8, readonly=True) d.addCallback(_do_server_setup_2) - d.addCallback(_remove_server) + d.addCallback(lambda ign: self.remove_server(0)) d.addCallback(lambda ign: self._do_upload_with_broken_servers(1)) d.addCallback(_set_basedir) hunk ./src/allmydata/test/test_upload.py 1314 d.addCallback(lambda ign: self._setup_and_upload()) d.addCallback(_do_server_setup_2) - d.addCallback(_remove_server) + d.addCallback(lambda ign: self.remove_server(0)) d.addCallback(lambda ign: self.shouldFail(UploadUnhappinessError, "test_dropped_servers_in_encoder", hunk ./src/allmydata/test/test_upload.py 1528 for i in xrange(1, 10): self._copy_share_to_server(i, 1) d.addCallback(_copy_shares) - d.addCallback(lambda ign: - self.g.remove_server(self.g.servers_by_number[0].my_nodeid)) + d.addCallback(lambda ign: self.remove_server(0)) def _prepare_client(ign): client = self.g.clients[0] client.DEFAULT_ENCODING_PARAMETERS['happy'] = 4 hunk ./src/allmydata/test/test_upload.py 1550 def _setup(ign): for i in xrange(1, 11): self._add_server(server_number=i) - self.g.remove_server(self.g.servers_by_number[0].my_nodeid) + self.remove_server(0) c = self.g.clients[0] # We set happy to an unsatisfiable value so that we can check the # counting in the exception message. The same progress message hunk ./src/allmydata/test/test_upload.py 1577 self._add_server(server_number=i) self._add_server(server_number=11, readonly=True) self._add_server(server_number=12, readonly=True) - self.g.remove_server(self.g.servers_by_number[0].my_nodeid) + self.remove_server(0) c = self.g.clients[0] c.DEFAULT_ENCODING_PARAMETERS['happy'] = 45 return c hunk ./src/allmydata/test/test_upload.py 1605 # the first one that the selector sees. for i in xrange(10): self._copy_share_to_server(i, 9) - # Remove server 0, and its contents - self.g.remove_server(self.g.servers_by_number[0].my_nodeid) + self.remove_server(0) # Make happiness unsatisfiable c = self.g.clients[0] c.DEFAULT_ENCODING_PARAMETERS['happy'] = 45 hunk ./src/allmydata/test/test_upload.py 1625 def _then(ign): for i in xrange(1, 11): self._add_server(server_number=i, readonly=True) - self.g.remove_server(self.g.servers_by_number[0].my_nodeid) + self.remove_server(0) c = self.g.clients[0] c.DEFAULT_ENCODING_PARAMETERS['k'] = 2 c.DEFAULT_ENCODING_PARAMETERS['happy'] = 4 hunk ./src/allmydata/test/test_upload.py 1661 self._add_server(server_number=4, readonly=True)) d.addCallback(lambda ign: self._add_server(server_number=5, readonly=True)) - d.addCallback(lambda ign: - self.g.remove_server(self.g.servers_by_number[0].my_nodeid)) + d.addCallback(lambda ign: self.remove_server(0)) def _reset_encoding_parameters(ign, happy=4): client = self.g.clients[0] client.DEFAULT_ENCODING_PARAMETERS['happy'] = happy hunk ./src/allmydata/test/test_upload.py 1696 d.addCallback(lambda ign: self._add_server(server_number=2)) def _break_server_2(ign): - serverid = self.g.servers_by_number[2].my_nodeid + serverid = self.get_server(2).get_serverid() self.g.break_server(serverid) d.addCallback(_break_server_2) d.addCallback(lambda ign: hunk ./src/allmydata/test/test_upload.py 1705 self._add_server(server_number=4, readonly=True)) d.addCallback(lambda ign: self._add_server(server_number=5, readonly=True)) - d.addCallback(lambda ign: - self.g.remove_server(self.g.servers_by_number[0].my_nodeid)) + d.addCallback(lambda ign: self.remove_server(0)) d.addCallback(_reset_encoding_parameters) d.addCallback(lambda client: self.shouldFail(UploadUnhappinessError, "test_selection_exceptions", hunk ./src/allmydata/test/test_upload.py 1816 # Copy shares self._copy_share_to_server(1, 1) self._copy_share_to_server(2, 1) - # Remove server 0 - self.g.remove_server(self.g.servers_by_number[0].my_nodeid) + self.remove_server(0) client = self.g.clients[0] client.DEFAULT_ENCODING_PARAMETERS['happy'] = 3 return client hunk ./src/allmydata/test/test_upload.py 1849 self._copy_share_to_server(3, 1) storedir = self.get_serverdir(0) # remove the storedir, wiping out any existing shares - shutil.rmtree(storedir) + fileutil.fp_remove(storedir) # create an empty storedir to replace the one we just removed hunk ./src/allmydata/test/test_upload.py 1851 - os.mkdir(storedir) + storedir.makedirs() client = self.g.clients[0] client.DEFAULT_ENCODING_PARAMETERS['happy'] = 4 return client hunk ./src/allmydata/test/test_upload.py 1890 self._copy_share_to_server(3, 1) storedir = self.get_serverdir(0) # remove the storedir, wiping out any existing shares - shutil.rmtree(storedir) + fileutil.fp_remove(storedir) # create an empty storedir to replace the one we just removed hunk ./src/allmydata/test/test_upload.py 1892 - os.mkdir(storedir) + storedir.makedirs() client = self.g.clients[0] client.DEFAULT_ENCODING_PARAMETERS['happy'] = 4 return client hunk ./src/allmydata/test/test_upload.py 1930 readonly=True) self._add_server_with_share(server_number=4, share_number=3, readonly=True) - # Remove server 0. - self.g.remove_server(self.g.servers_by_number[0].my_nodeid) + self.remove_server(0) # Set the client appropriately c = self.g.clients[0] c.DEFAULT_ENCODING_PARAMETERS['happy'] = 4 hunk ./src/allmydata/test/test_util.py 9 from twisted.trial import unittest from twisted.internet import defer, reactor from twisted.python.failure import Failure +from twisted.python.filepath import FilePath from twisted.python import log from pycryptopp.hash.sha256 import SHA256 as _hash hunk ./src/allmydata/test/test_util.py 508 os.chdir(saved_cwd) def test_disk_stats(self): - avail = fileutil.get_available_space('.', 2**14) + avail = fileutil.get_available_space(FilePath('.'), 2**14) if avail == 0: raise unittest.SkipTest("This test will spuriously fail there is no disk space left.") hunk ./src/allmydata/test/test_util.py 512 - disk = fileutil.get_disk_stats('.', 2**13) + disk = fileutil.get_disk_stats(FilePath('.'), 2**13) self.failUnless(disk['total'] > 0, disk['total']) self.failUnless(disk['used'] > 0, disk['used']) self.failUnless(disk['free_for_root'] > 0, disk['free_for_root']) hunk ./src/allmydata/test/test_util.py 521 def test_disk_stats_avail_nonnegative(self): # This test will spuriously fail if you have more than 2^128 - # bytes of available space on your filesystem. - disk = fileutil.get_disk_stats('.', 2**128) + # bytes of available space on your filesystem (lucky you). + disk = fileutil.get_disk_stats(FilePath('.'), 2**128) self.failUnlessEqual(disk['avail'], 0) class PollMixinTests(unittest.TestCase): hunk ./src/allmydata/test/test_web.py 12 from twisted.python import failure, log from nevow import rend from allmydata import interfaces, uri, webish, dirnode -from allmydata.storage.shares import get_share_file from allmydata.storage_client import StorageFarmBroker from allmydata.immutable import upload from allmydata.immutable.downloader.status import DownloadStatus hunk ./src/allmydata/test/test_web.py 3998 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which]) d.addCallback(_compute_fileurls) - def _clobber_shares(ignored): - good_shares = self.find_uri_shares(self.uris["good"]) - self.failUnlessReallyEqual(len(good_shares), 10) - sick_shares = self.find_uri_shares(self.uris["sick"]) - os.unlink(sick_shares[0][2]) - dead_shares = self.find_uri_shares(self.uris["dead"]) + d.addCallback(lambda ign: self.find_uri_shares(self.uris["good"])) + d.addCallback(lambda good_shares: self.failUnlessReallyEqual(len(good_shares), 10)) + d.addCallback(lambda ign: self.find_uri_shares(self.uris["sick"])) + d.addCallback(lambda sick_shares: sick_shares[0][2].remove()) + d.addCallback(lambda ign: self.find_uri_shares(self.uris["dead"])) + def _remove_dead_shares(dead_shares): for i in range(1, 10): hunk ./src/allmydata/test/test_web.py 4005 - os.unlink(dead_shares[i][2]) - c_shares = self.find_uri_shares(self.uris["corrupt"]) + dead_shares[i][2].remove() + d.addCallback(_remove_dead_shares) + d.addCallback(lambda ign: self.find_uri_shares(self.uris["corrupt"])) + def _corrupt_shares(c_shares): cso = CorruptShareOptions() cso.stdout = StringIO() hunk ./src/allmydata/test/test_web.py 4011 - cso.parseOptions([c_shares[0][2]]) + cso.parseOptions([c_shares[0][2].path]) corrupt_share(cso) hunk ./src/allmydata/test/test_web.py 4013 - d.addCallback(_clobber_shares) + d.addCallback(_corrupt_shares) d.addCallback(self.CHECK, "good", "t=check") def _got_html_good(res): hunk ./src/allmydata/test/test_web.py 4142 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which]) d.addCallback(_compute_fileurls) - def _clobber_shares(ignored): - good_shares = self.find_uri_shares(self.uris["good"]) - self.failUnlessReallyEqual(len(good_shares), 10) - sick_shares = self.find_uri_shares(self.uris["sick"]) - os.unlink(sick_shares[0][2]) - dead_shares = self.find_uri_shares(self.uris["dead"]) + d.addCallback(lambda ign: self.find_uri_shares(self.uris["good"])) + d.addCallback(lambda good_shares: self.failUnlessReallyEqual(len(good_shares), 10)) + d.addCallback(lambda ign: self.find_uri_shares(self.uris["sick"])) + d.addCallback(lambda sick_shares: sick_shares[0][2].remove()) + d.addCallback(lambda ign: self.find_uri_shares(self.uris["dead"])) + def _remove_dead_shares(dead_shares): for i in range(1, 10): hunk ./src/allmydata/test/test_web.py 4149 - os.unlink(dead_shares[i][2]) - c_shares = self.find_uri_shares(self.uris["corrupt"]) + dead_shares[i][2].remove() + d.addCallback(_remove_dead_shares) + d.addCallback(lambda ign: self.find_uri_shares(self.uris["corrupt"])) + def _corrupt_shares(c_shares): cso = CorruptShareOptions() cso.stdout = StringIO() hunk ./src/allmydata/test/test_web.py 4155 - cso.parseOptions([c_shares[0][2]]) + cso.parseOptions([c_shares[0][2].path]) corrupt_share(cso) hunk ./src/allmydata/test/test_web.py 4157 - d.addCallback(_clobber_shares) + d.addCallback(_corrupt_shares) d.addCallback(self.CHECK, "good", "t=check&repair=true") def _got_html_good(res): hunk ./src/allmydata/test/test_web.py 4212 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which]) d.addCallback(_compute_fileurls) - def _clobber_shares(ignored): - sick_shares = self.find_uri_shares(self.uris["sick"]) - os.unlink(sick_shares[0][2]) - d.addCallback(_clobber_shares) + d.addCallback(lambda ign: self.find_uri_shares(self.uris["sick"])) + d.addCallback(lambda sick_shares: sick_shares[0][2].remove()) d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json") def _got_json_sick(res): hunk ./src/allmydata/test/test_web.py 4526 future_node = UnknownNode(unknown_rwcap, unknown_rocap) d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node)) - def _clobber_shares(ignored): - self.delete_shares_numbered(self.uris["sick"], [0,1]) - d.addCallback(_clobber_shares) + d.addCallback(lambda ign: self.delete_shares_numbered(self.uris["sick"], [0,1])) # root # root/good hunk ./src/allmydata/test/test_web.py 4698 #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn)) #d.addCallback(_stash_uri, "corrupt") - def _clobber_shares(ignored): - good_shares = self.find_uri_shares(self.uris["good"]) - self.failUnlessReallyEqual(len(good_shares), 10) - sick_shares = self.find_uri_shares(self.uris["sick"]) - os.unlink(sick_shares[0][2]) - #dead_shares = self.find_uri_shares(self.uris["dead"]) - #for i in range(1, 10): - # os.unlink(dead_shares[i][2]) - - #c_shares = self.find_uri_shares(self.uris["corrupt"]) - #cso = CorruptShareOptions() - #cso.stdout = StringIO() - #cso.parseOptions([c_shares[0][2]]) - #corrupt_share(cso) - d.addCallback(_clobber_shares) + d.addCallback(lambda ign: self.find_uri_shares(self.uris["good"])) + d.addCallback(lambda good_shares: self.failUnlessReallyEqual(len(good_shares), 10)) + d.addCallback(lambda ign: self.find_uri_shares(self.uris["sick"])) + d.addCallback(lambda sick_shares: sick_shares[0][2].remove()) + #d.addCallback(lambda ign: self.find_uri_shares(self.uris["dead"])) + #def _remove_dead_shares(dead_shares): + # for i in range(1, 10): + # dead_shares[i][2].remove() + #d.addCallback(_remove_dead_shares) + #d.addCallback(lambda ign: self.find_uri_shares(self.uris["corrupt"])) + #def _corrupt_shares(c_shares): + # cso = CorruptShareOptions() + # cso.stdout = StringIO() + # cso.parseOptions([c_shares[0][2].path]) + # corrupt_share(cso) + #d.addCallback(_corrupt_shares) # root # root/good CHK, 10 shares hunk ./src/allmydata/test/test_web.py 4762 d.addErrback(self.explain_web_error) return d - def _count_leases(self, ignored, which): - u = self.uris[which] - shares = self.find_uri_shares(u) - lease_counts = [] - for shnum, serverid, fn in shares: - sf = get_share_file(fn) - num_leases = len(list(sf.get_leases())) - lease_counts.append( (fn, num_leases) ) - return lease_counts - - def _assert_leasecount(self, lease_counts, expected): - for (fn, num_leases) in lease_counts: - if num_leases != expected: - self.fail("expected %d leases, have %d, on %s" % - (expected, num_leases, fn)) + def _assert_leasecount(self, which, expected): + d = self.count_leases(self.uris[which]) + def _got_counts(lease_counts): + for (fn, num_leases) in lease_counts: + if num_leases != expected: + self.fail("expected %d leases, have %d, on %s" % + (expected, num_leases, fn)) + d.addCallback(_got_counts) + return d def test_add_lease(self): self.basedir = "web/Grid/add_lease" hunk ./src/allmydata/test/test_web.py 4798 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which]) d.addCallback(_compute_fileurls) - d.addCallback(self._count_leases, "one") - d.addCallback(self._assert_leasecount, 1) - d.addCallback(self._count_leases, "two") - d.addCallback(self._assert_leasecount, 1) - d.addCallback(self._count_leases, "mutable") - d.addCallback(self._assert_leasecount, 1) + d.addCallback(lambda ign: self._assert_leasecount("one", 1)) + d.addCallback(lambda ign: self._assert_leasecount("two", 1)) + d.addCallback(lambda ign: self._assert_leasecount("mutable", 1)) d.addCallback(self.CHECK, "one", "t=check") # no add-lease def _got_html_good(res): hunk ./src/allmydata/test/test_web.py 4808 self.failIf("Not Healthy" in res, res) d.addCallback(_got_html_good) - d.addCallback(self._count_leases, "one") - d.addCallback(self._assert_leasecount, 1) - d.addCallback(self._count_leases, "two") - d.addCallback(self._assert_leasecount, 1) - d.addCallback(self._count_leases, "mutable") - d.addCallback(self._assert_leasecount, 1) + d.addCallback(lambda ign: self._assert_leasecount("one", 1)) + d.addCallback(lambda ign: self._assert_leasecount("two", 1)) + d.addCallback(lambda ign: self._assert_leasecount("mutable", 1)) # this CHECK uses the original client, which uses the same # lease-secrets, so it will just renew the original lease hunk ./src/allmydata/test/test_web.py 4817 d.addCallback(self.CHECK, "one", "t=check&add-lease=true") d.addCallback(_got_html_good) - d.addCallback(self._count_leases, "one") - d.addCallback(self._assert_leasecount, 1) - d.addCallback(self._count_leases, "two") - d.addCallback(self._assert_leasecount, 1) - d.addCallback(self._count_leases, "mutable") - d.addCallback(self._assert_leasecount, 1) + d.addCallback(lambda ign: self._assert_leasecount("one", 1)) + d.addCallback(lambda ign: self._assert_leasecount("two", 1)) + d.addCallback(lambda ign: self._assert_leasecount("mutable", 1)) # this CHECK uses an alternate client, which adds a second lease d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1) hunk ./src/allmydata/test/test_web.py 4825 d.addCallback(_got_html_good) - d.addCallback(self._count_leases, "one") - d.addCallback(self._assert_leasecount, 2) - d.addCallback(self._count_leases, "two") - d.addCallback(self._assert_leasecount, 1) - d.addCallback(self._count_leases, "mutable") - d.addCallback(self._assert_leasecount, 1) + d.addCallback(lambda ign: self._assert_leasecount("one", 2)) + d.addCallback(lambda ign: self._assert_leasecount("two", 1)) + d.addCallback(lambda ign: self._assert_leasecount("mutable", 1)) d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true") d.addCallback(_got_html_good) hunk ./src/allmydata/test/test_web.py 4832 - d.addCallback(self._count_leases, "one") - d.addCallback(self._assert_leasecount, 2) - d.addCallback(self._count_leases, "two") - d.addCallback(self._assert_leasecount, 1) - d.addCallback(self._count_leases, "mutable") - d.addCallback(self._assert_leasecount, 1) + d.addCallback(lambda ign: self._assert_leasecount("one", 2)) + d.addCallback(lambda ign: self._assert_leasecount("two", 1)) + d.addCallback(lambda ign: self._assert_leasecount("mutable", 1)) d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true", clientnum=1) hunk ./src/allmydata/test/test_web.py 4840 d.addCallback(_got_html_good) - d.addCallback(self._count_leases, "one") - d.addCallback(self._assert_leasecount, 2) - d.addCallback(self._count_leases, "two") - d.addCallback(self._assert_leasecount, 1) - d.addCallback(self._count_leases, "mutable") - d.addCallback(self._assert_leasecount, 2) + d.addCallback(lambda ign: self._assert_leasecount("one", 2)) + d.addCallback(lambda ign: self._assert_leasecount("two", 1)) + d.addCallback(lambda ign: self._assert_leasecount("mutable", 2)) d.addErrback(self.explain_web_error) return d hunk ./src/allmydata/test/test_web.py 4884 self.failUnlessReallyEqual(len(units), 4+1) d.addCallback(_done) - d.addCallback(self._count_leases, "root") - d.addCallback(self._assert_leasecount, 1) - d.addCallback(self._count_leases, "one") - d.addCallback(self._assert_leasecount, 1) - d.addCallback(self._count_leases, "mutable") - d.addCallback(self._assert_leasecount, 1) + d.addCallback(lambda ign: self._assert_leasecount("root", 1)) + d.addCallback(lambda ign: self._assert_leasecount("one", 1)) + d.addCallback(lambda ign: self._assert_leasecount("mutable", 1)) d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true") d.addCallback(_done) hunk ./src/allmydata/test/test_web.py 4891 - d.addCallback(self._count_leases, "root") - d.addCallback(self._assert_leasecount, 1) - d.addCallback(self._count_leases, "one") - d.addCallback(self._assert_leasecount, 1) - d.addCallback(self._count_leases, "mutable") - d.addCallback(self._assert_leasecount, 1) + d.addCallback(lambda ign: self._assert_leasecount("root", 1)) + d.addCallback(lambda ign: self._assert_leasecount("one", 1)) + d.addCallback(lambda ign: self._assert_leasecount("mutable", 1)) d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true", clientnum=1) hunk ./src/allmydata/test/test_web.py 4899 d.addCallback(_done) - d.addCallback(self._count_leases, "root") - d.addCallback(self._assert_leasecount, 2) - d.addCallback(self._count_leases, "one") - d.addCallback(self._assert_leasecount, 2) - d.addCallback(self._count_leases, "mutable") - d.addCallback(self._assert_leasecount, 2) + d.addCallback(lambda ign: self._assert_leasecount("root", 2)) + d.addCallback(lambda ign: self._assert_leasecount("one", 2)) + d.addCallback(lambda ign: self._assert_leasecount("mutable", 2)) d.addErrback(self.explain_web_error) return d hunk ./src/allmydata/util/deferredutil.py 1 + +from foolscap.api import fireEventually from twisted.internet import defer # utility wrapper for DeferredList hunk ./src/allmydata/util/deferredutil.py 38 d.addCallbacks(_parseDListResult, _unwrapFirstError) return d + +def async_iterate(process, iterable): + """ + I iterate over the elements of 'iterable' (which may be deferred), eventually + applying 'process' to each one. 'process' should return a (possibly deferred) + boolean: True to continue the iteration, False to stop. + + I return a Deferred that fires with True if all elements of the iterable + were processed (i.e. 'process' only returned True values); with False if + the iteration was stopped by 'process' returning False; or that fails with + the first failure of either 'process' or the iterator. + """ + iterator = iter(iterable) + + d = defer.succeed(None) + def _iterate(ign): + d2 = defer.maybeDeferred(iterator.next) + def _cb(item): + d3 = defer.maybeDeferred(process, item) + def _maybe_iterate(res): + if res: + d4 = fireEventually() + d4.addCallback(_iterate) + return d4 + return False + d3.addCallback(_maybe_iterate) + return d3 + def _eb(f): + if f.trap(StopIteration): + return True + return f + d2.addCallbacks(_cb, _eb) + return d2 + d.addCallback(_iterate) + return d + + +def for_items(cb, mapping): + """ + For each (key, value) pair in a mapping, I add a callback to cb(None, key, value) + to a Deferred that fires immediately. I return that Deferred. + """ + d = defer.succeed(None) + for k, v in mapping.items(): + d.addCallback(lambda ign, k=k, v=v: cb(None, k, v)) + return d hunk ./src/allmydata/util/encodingutil.py 221 def quote_path(path, quotemarks=True): return quote_output("/".join(map(to_str, path)), quotemarks=quotemarks) +def quote_filepath(fp, quotemarks=True, encoding=None): + path = fp.path + if isinstance(path, str): + try: + path = path.decode(filesystem_encoding) + except UnicodeDecodeError: + return 'b"%s"' % (ESCAPABLE_8BIT.sub(_str_escape, path),) + + return quote_output(path, quotemarks=quotemarks, encoding=encoding) + def unicode_platform(): """ hunk ./src/allmydata/util/fileutil.py 5 Futz with files like a pro. """ -import sys, exceptions, os, stat, tempfile, time, binascii +import errno, sys, exceptions, os, stat, tempfile, time, binascii + +from allmydata.util.assertutil import precondition from twisted.python import log hunk ./src/allmydata/util/fileutil.py 10 +from twisted.python.filepath import FilePath, UnlistableError from pycryptopp.cipher.aes import AES hunk ./src/allmydata/util/fileutil.py 189 raise tx raise exceptions.IOError, "unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname # careful not to construct an IOError with a 2-tuple, as that has a special meaning... -def rm_dir(dirname): +def fp_make_dirs(dirfp): + """ + An idempotent version of FilePath.makedirs(). If the dir already + exists, do nothing and return without raising an exception. If this + call creates the dir, return without raising an exception. If there is + an error that prevents creation or if the directory gets deleted after + fp_make_dirs() creates it and before fp_make_dirs() checks that it + exists, raise an exception. + """ + log.msg( "xxx 0 %s" % (dirfp,)) + tx = None + try: + dirfp.makedirs() + except OSError, x: + tx = x + + if not dirfp.isdir(): + if tx: + raise tx + raise exceptions.IOError, "unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirfp # careful not to construct an IOError with a 2-tuple, as that has a special meaning... + +def fp_rmdir_if_empty(dirfp): + """ Remove the directory if it is empty. """ + try: + os.rmdir(dirfp.path) + except OSError, e: + if e.errno != errno.ENOTEMPTY: + raise + else: + dirfp.changed() + +def rmtree(dirname): """ A threadsafe and idempotent version of shutil.rmtree(). If the dir is already gone, do nothing and return without raising an exception. If this hunk ./src/allmydata/util/fileutil.py 239 else: remove(fullname) os.rmdir(dirname) - except Exception, le: - # Ignore "No such file or directory" - if (not isinstance(le, OSError)) or le.args[0] != 2: + except EnvironmentError, le: + # Ignore "No such file or directory", collect any other exception. + if (le.args[0] != 2 and le.args[0] != 3) or (le.args[0] != errno.ENOENT): excs.append(le) hunk ./src/allmydata/util/fileutil.py 243 + except Exception, le: + excs.append(le) # Okay, now we've recursively removed everything, ignoring any "No # such file or directory" errors, and collecting any other errors. hunk ./src/allmydata/util/fileutil.py 256 raise OSError, "Failed to remove dir for unknown reason." raise OSError, excs +def fp_remove(fp): + """ + An idempotent version of shutil.rmtree(). If the file/dir is already + gone, do nothing and return without raising an exception. If this call + removes the file/dir, return without raising an exception. If there is + an error that prevents removal, or if a file or directory at the same + path gets created again by someone else after this deletes it and before + this checks that it is gone, raise an exception. + """ + try: + fp.remove() + except UnlistableError, e: + if e.originalException.errno != errno.ENOENT: + raise + except OSError, e: + if e.errno != errno.ENOENT: + raise + +def rm_dir(dirname): + # Renamed to be like shutil.rmtree and unlike rmdir. + return rmtree(dirname) def remove_if_possible(f): try: hunk ./src/allmydata/util/fileutil.py 284 except: pass +def fp_list(fp): + """ + If fp exists and is a listable directory, return a list of FilePath objects + corresponding to its children. If it does not exist, return an empty list. + If it is not listable for any other reason than not existing (or a parent + directory not existing), raise an exception. + """ + try: + return fp.children() + except UnlistableError, e: + if e.originalException.errno != errno.ENOENT: + raise + return [] + except OSError, e: + if e.errno != errno.ENOENT: + raise + return [] + def open_or_create(fname, binarymode=True): try: return open(fname, binarymode and "r+b" or "r+") hunk ./src/allmydata/util/fileutil.py 405 import traceback traceback.print_exc() -def get_disk_stats(whichdir, reserved_space=0): +def get_disk_stats(whichdirfp, reserved_space=0): """Return disk statistics for the storage disk, in the form of a dict with the following fields. total: total bytes on disk hunk ./src/allmydata/util/fileutil.py 426 you can pass how many bytes you would like to leave unused on this filesystem as reserved_space. """ + precondition(isinstance(whichdirfp, FilePath), whichdirfp) if have_GetDiskFreeSpaceExW: # If this is a Windows system and GetDiskFreeSpaceExW is available, use it. hunk ./src/allmydata/util/fileutil.py 437 n_free_for_nonroot = c_ulonglong(0) n_total = c_ulonglong(0) n_free_for_root = c_ulonglong(0) - retval = GetDiskFreeSpaceExW(whichdir, byref(n_free_for_nonroot), - byref(n_total), - byref(n_free_for_root)) + retval = GetDiskFreeSpaceExW(whichdirfp.path, byref(n_free_for_nonroot), + byref(n_total), + byref(n_free_for_root)) if retval == 0: raise OSError("Windows error %d attempting to get disk statistics for %r" hunk ./src/allmydata/util/fileutil.py 442 - % (GetLastError(), whichdir)) + % (GetLastError(), whichdirfp.path)) free_for_nonroot = n_free_for_nonroot.value total = n_total.value free_for_root = n_free_for_root.value hunk ./src/allmydata/util/fileutil.py 451 # # # - s = os.statvfs(whichdir) + s = os.statvfs(whichdirfp.path) # on my mac laptop: # statvfs(2) is a wrapper around statfs(2). hunk ./src/allmydata/util/fileutil.py 478 'avail': avail, } -def get_available_space(whichdir, reserved_space): + +def get_available_space(whichdirfp, reserved_space): """Returns available space for share storage in bytes, or None if no API to get this information is available. hunk ./src/allmydata/util/fileutil.py 491 you can pass how many bytes you would like to leave unused on this filesystem as reserved_space. """ + precondition(isinstance(whichdirfp, FilePath), whichdirfp) try: hunk ./src/allmydata/util/fileutil.py 493 - return get_disk_stats(whichdir, reserved_space)['avail'] + return get_disk_stats(whichdirfp, reserved_space)['avail'] except AttributeError: return None hunk ./src/allmydata/util/fileutil.py 496 - except EnvironmentError: - log.msg("OS call to get disk statistics failed") + + +def get_used_space(fp): + if fp is None: return 0 hunk ./src/allmydata/util/fileutil.py 501 + try: + s = os.stat(fp.path) + except EnvironmentError: + if not fp.exists(): + return 0 + raise + else: + # POSIX defines st_blocks (originally a BSDism): + # + # but does not require stat() to give it a "meaningful value" + # + # and says: + # "The unit for the st_blocks member of the stat structure is not defined + # within IEEE Std 1003.1-2001. In some implementations it is 512 bytes. + # It may differ on a file system basis. There is no correlation between + # values of the st_blocks and st_blksize, and the f_bsize (from ) + # structure members." + # + # The Linux docs define it as "the number of blocks allocated to the file, + # [in] 512-byte units." It is also defined that way on MacOS X. Python does + # not set the attribute on Windows. + # + # We consider platforms that define st_blocks but give it a wrong value, or + # measure it in a unit other than 512 bytes, to be broken. See also + # . + + if hasattr(s, 'st_blocks'): + return s.st_blocks * 512 + else: + return s.st_size } Context: [setup.py: stop putting pyutil.version_class/etc in _version.py Brian Warner **20111205055049 Ignore-this: 926fa9a8a34a04f24ee6e006423e9c1 allmydata.__version__ can just be a string, it doesn't need to be an instance of some fancy NormalizedVersion class. Everything inside Tahoe uses str(__version__) anyways. Also add .dev0 when a git tree is dirty. Closes #1466 ] [setup.py: get version from git or darcs Brian Warner **20111205044001 Ignore-this: 5a406b33000446d85edc722298391220 This replaces the setup.cfg aliases that run "darcsver" before each major command with the new "update_version". update_version is defined in setup.py, and tries to get a version string from either darcs or git (or leaves the existing _version.py alone if neither VC metadata is available). Also clean up a tiny typo in verlib.py that messed up syntax hilighting. ] [Munge the umids in introducer/{client,server}.py so that check-umids doesn't complain about them being duplicates of the ones in introducer/old.py. refs #466 david-sarah@jacaranda.org**20111129234057 Ignore-this: da053d962ccb32d197ef0f123013acbb ] [new introducer: tests for signed extensible dictionary-based messages. refs #466 david-sarah@jacaranda.org**20111129231920 Ignore-this: 751ed1c993688f838d343423ff68b716 ] [new introducer: signed extensible dictionary-based messages! This patch does not include the tests. refs #466 david-sarah@jacaranda.org**20111129231756 Ignore-this: 18bc491a44f7627202667ef681f2d948 This introduces new client and server halves to the Introducer (renaming the old one with a _V1 suffix). Both have fallbacks to accomodate talking to a different version: the publishing client switches on whether the server's .get_version() advertises V2 support, the server switches on which subscription method was invoked by the subscribing client. The V2 protocol sends a three-tuple of (serialized announcement dictionary, signature, pubkey) for each announcement. The V2 server dispatches messages to subscribers according to the service-name, and throws errors for invalid signatures, but does not otherwise examine the messages. The V2 receiver's subscription callback will receive a (serverid, ann_dict) pair. The 'serverid' will be equal to the pubkey if all of the following are true: the originating client is V2, and was told a privkey to use the announcement went through a V2 server the signature is valid If not, 'serverid' will be equal to the tubid portion of the announced FURL, as was the case for V1 receivers. Servers will create a keypair if one does not exist yet, stored in private/server.privkey . The signed announcement dictionary puts the server FURL in a key named "anonymous-storage-FURL", which anticipates upcoming Accounting-related changes in the server advertisements. It also provides a key named "permutation-seed-base32" to tell clients what permutation seed to use. This is computed at startup, using tubid if there are existing shares, otherwise the pubkey, to retain share-order compatibility for existing servers. ] [docs/known_issues.rst: describe when the unauthorized access attack is known to be possible, and fix a link. david-sarah@jacaranda.org**20111118002013 Ignore-this: d89b1f1040a0a7ee0bde893d23612049 ] [more tiny buildbot-testing whitespace changes warner@lothar.com**20111118002041 Ignore-this: e816e2a5ab939e2f7a89ef12b8a157d8 ] [more tiny buildbot-testing whitespace changes warner@lothar.com**20111118001828 Ignore-this: 57bb52cba83ea9a19728ba0a8ffadb69 ] [tiny change to exercise the buildbot hook warner@lothar.com**20111118001511 Ignore-this: 7220b7790b39f19f9721d9e93b755030 ] [Strengthen description of unauthorized access attack in known_issues.rst. david-sarah@jacaranda.org**20111118000030 Ignore-this: e2f68f621fe666b6201542623aa4d182 ] [remove remaining uses of nevow's "formless" module Brian Warner **20111117225423 Ignore-this: a128dea91a1c63b3bbefa34729344d69 We're slowly moving away from Nevow, and marcusw's previous patch removed uses of the formless CSS file, so now we can stop testing that nevow can find that file, and remove the lingering unused "import formless" call. ] [1585-webui.darcs.patch Marcus Wanner **20111117214923 Ignore-this: 23cf2a06c545be5f821c071d652178ee ] [Remove duplicate tahoe_css links from manifest.xhtml and rename-form.xhtml Brian Warner **20111116224225 Ignore-this: 12024fff17964607799928928b9aadf3 They were probably meant to be links to webform_css, but we aren't really using Nevow's form-generation code anyways, so they can just be removed. Thanks to 'marcusw' for the catch. ] [iputil: handle openbsd5 (just like openbsd4) Brian Warner **20111115220423 Ignore-this: 64b28bd2fd06eb5230ea41d91540dd05 Patch by 'sickness'. Closes #1584 ] [Makefile count-lines: let it work on OS-X (-l not --lines), add XXX Brian Warner **20111109184227 Ignore-this: 204ace1dadc9ed27543c62965b4e6757 OS-X's simple-minded /usr/bin/wc doesn't understand --lines, but everyone understands -l . ] [setup.py: umask=022 for 'sdist', to avoid depending on environment Brian Warner **20111109183632 Ignore-this: acd5db88ba8f1972d618b14f9e5b803c The new tarball-building buildslave had a bogus umask set, causing the 1.9.0 tarballs to be non-other-user-readable (go-rwx), which is a hassle for packaging. (The umask was correct on the old buildslave, but it was moved to a new host shortly before the release). This should make sure tarballs are correct despite the host's setting. Note to others: processes run under twistd get umask=077 unless you arrange otherwise. ] [_auto_deps.py: blacklist PyCrypto 2.4. david-sarah@jacaranda.org**20111105022457 Ignore-this: 876cb24bc71589e735f48bf449cad81e ] [check-miscaptures.py: report the number of files that were not analysed due to syntax errors (and don't count them in the number of suspicious captures). refs #1555 david-sarah@jacaranda.org**20111009050301 Ignore-this: 62ee03f4b8a96c292e75c097ad87d52e ] [check-miscaptures.py: handle corner cases around default arguments correctly. Also make a minor optimization when there are no assigned variables to consider. refs #1555 david-sarah@jacaranda.org**20111009045023 Ignore-this: f49ece515620081da1d745ae6da19d21 ] [check-miscaptures.py: Python doesn't really have declarations; report the topmost assignment. refs #1555 david-sarah@jacaranda.org**20111009044800 Ignore-this: 4905c9dfe7726f433333e216a6760a4b ] [check-miscaptures.py: handle destructuring function arguments correctly. refs #1555 david-sarah@jacaranda.org**20111009044710 Ignore-this: f9de7d95e94446507a206c88d3f98a23 ] [check-miscaptures.py: check while loops and list comprehensions as well as for loops. Also fix a pyflakes warning. refs #1555 david-sarah@jacaranda.org**20111009044022 Ignore-this: 6526e4e315ca6461b1fbc2da5568e444 ] [Add misc/coding_tools/check-miscaptures.py to detect incorrect captures of variables declared in a for loop, and a 'make check-miscaptures' Makefile target to run it. (It is also run by 'make code-checks'.) This is a rewritten version that reports much fewer false positives, by determining captured variables more accurately. fixes #1555 david-sarah@jacaranda.org**20111007074121 Ignore-this: 51318e9678d132c374ea557ab955e79e ] [Fix pyflakes warnings in misc/ directories other than misc/build_helpers. refs #1557 david-sarah@jacaranda.org**20111007033031 Ignore-this: 7daf5862469732d8cabc355266622b74 ] [Makefile: include misc/ directories other than misc/build_helpers in SOURCES. refs #1557 david-sarah@jacaranda.org**20111007032958 Ignore-this: 31376ec01401df7972e83341dc65aa05 ] [show-tool-versions: tolerate missing setuptools Brian Warner **20111101080010 Ignore-this: 72d4e440565273992beb4f010cbca699 ] [show-tool-versions.py: condense output, hide file-not-found exceptions Brian Warner **20111101074532 Ignore-this: a15381a76077ef46a74a4ac40c9ae956 ] [relnotes.txt: fix footnotes Brian Warner **20111101071935 Ignore-this: 668c1bd8618e21beed9bc6b23f048189 ] [Rewrite download-status-timeline visualizer ('viz') with d3.js Brian Warner **20111101061821 Ignore-this: 6149b027bbae52c559ef5a8167240cab * use d3.js v2.4.6 * add a "toggle misc events" button, to get hash/bitmap-checking details * only draw data that's on screen, for speed * add fragment-arg to fetch timeline data.json from somewhere else ] [IServer refactoring: pass IServer instances around, instead of peerids Brian Warner **20111101040319 Ignore-this: 35e4698a0273a0311fe0ccedcc7881b5 refs #1363 This collapses 88 small incremental changes (each of which passes all tests) into one big patch. The development process for the long path started with adding some temporary scaffolding, changing one method at a time, then removing the scaffolding. The individual pieces are as follows, in reverse chronological order (the first patch is at the end of this comment): commit 9bbe4174fd0d98a6cf47a8ef96e85d9ef34b2f9a Author: Brian Warner Date: Tue Oct 4 16:05:00 2011 -0400 immutable/downloader/status.py: correct comment src/allmydata/immutable/downloader/status.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) commit 72146a7c7c91eac2f7c3ceb801eb7a1721376889 Author: Brian Warner Date: Tue Oct 4 15:46:20 2011 -0400 remove temporary ServerMap._storage_broker src/allmydata/mutable/checker.py | 2 +- src/allmydata/mutable/filenode.py | 2 +- src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/servermap.py | 5 ++--- src/allmydata/test/test_mutable.py | 8 ++++---- 5 files changed, 9 insertions(+), 10 deletions(-) commit d703096b41632c47d76414b12672e076a422ff5c Author: Brian Warner Date: Tue Oct 4 15:37:05 2011 -0400 remove temporary storage_broker.get_server_for_id() src/allmydata/storage_client.py | 3 --- src/allmydata/test/no_network.py | 13 ------------- 2 files changed, 0 insertions(+), 16 deletions(-) commit 620cc5d80882ef6f7decfd26af8a6c7c1ddf80d1 Author: Brian Warner Date: Tue Oct 4 12:50:06 2011 -0400 API of Retrieve._try_to_validate_privkey(), trying to remove reader.server src/allmydata/mutable/retrieve.py | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) commit 92f43f856f4a8b36c207d1b190ed8699b5a4ecb4 Author: Brian Warner Date: Tue Oct 4 12:48:08 2011 -0400 API of Retrieve._validate_block(), trying to remove reader.server src/allmydata/mutable/retrieve.py | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) commit 572d5070761861a2190349d1ed8d85dbc25698a5 Author: Brian Warner Date: Tue Oct 4 12:36:58 2011 -0400 API of Retrieve._mark_bad_share(), trying to remove reader.server src/allmydata/mutable/retrieve.py | 21 +++++++++------------ 1 files changed, 9 insertions(+), 12 deletions(-) commit a793ff00c0de1e2eec7b46288fdf388c7a2bec89 Author: Brian Warner Date: Tue Oct 4 12:06:13 2011 -0400 remove now-unused get_rref_for_serverid() src/allmydata/mutable/servermap.py | 3 --- 1 files changed, 0 insertions(+), 3 deletions(-) commit 1b9827cc9366bf90b93297fdd6832f2ad0480ce7 Author: Brian Warner Date: Tue Oct 4 12:03:09 2011 -0400 Retrieve: stop adding .serverid attributes to readers src/allmydata/mutable/retrieve.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) commit 5d4e9d491b19e49d2e443a1dfff2c672842c36ef Author: Brian Warner Date: Tue Oct 4 12:03:34 2011 -0400 return value of Retrieve(verify=True) src/allmydata/mutable/checker.py | 11 ++++++----- src/allmydata/mutable/retrieve.py | 3 +-- 2 files changed, 7 insertions(+), 7 deletions(-) commit e9ab7978c384e1f677cb7779dc449b1044face82 Author: Brian Warner Date: Tue Oct 4 11:54:23 2011 -0400 Retrieve._bad_shares (but not return value, used by Verifier) src/allmydata/mutable/retrieve.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) commit 2d91926de233ec5c881f30e36b4a30ad92ab42a9 Author: Brian Warner Date: Tue Oct 4 11:51:23 2011 -0400 Publish: stop adding .serverid attributes to writers src/allmydata/mutable/publish.py | 9 ++------- 1 files changed, 2 insertions(+), 7 deletions(-) commit 47c7a0105dec7cbf4f7e0a3ce800bbb85b15df4a Author: Brian Warner Date: Tue Oct 4 11:56:33 2011 -0400 API of get_write_enabler() src/allmydata/mutable/filenode.py | 7 ++++--- src/allmydata/mutable/publish.py | 4 ++-- src/allmydata/test/no_network.py | 3 +++ 3 files changed, 9 insertions(+), 5 deletions(-) commit 9196a5c6590fdbfd660325ea8358b345887d3db0 Author: Brian Warner Date: Tue Oct 4 11:46:24 2011 -0400 API of get_(renewal|cancel)_secret() src/allmydata/mutable/filenode.py | 14 ++++++++------ src/allmydata/mutable/publish.py | 8 ++++---- src/allmydata/mutable/servermap.py | 5 ++--- 3 files changed, 14 insertions(+), 13 deletions(-) commit de7c1552f8c163eff5b6d820b5fb3b21c1b47cb5 Author: Brian Warner Date: Tue Oct 4 11:41:52 2011 -0400 API of CorruptShareError. Also comment out some related+unused test_web.py code src/allmydata/mutable/common.py | 13 +++++-------- src/allmydata/mutable/retrieve.py | 10 +++++----- src/allmydata/mutable/servermap.py | 8 +++----- src/allmydata/test/common.py | 13 ++++++++----- 4 files changed, 21 insertions(+), 23 deletions(-) commit 2c1c314046b620c16f1e66d030c150d768b7d01e Author: Brian Warner Date: Tue Oct 4 12:01:46 2011 -0400 API of ServerMap.mark_bad_share() src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/retrieve.py | 6 +++--- src/allmydata/mutable/servermap.py | 6 ++---- src/allmydata/test/test_mutable.py | 3 +-- 4 files changed, 7 insertions(+), 10 deletions(-) commit 1bed349030779fd0c378ae4e821384f953c6f6ff Author: Brian Warner Date: Tue Oct 4 11:11:17 2011 -0400 API+name of ServerMap.shares_on_server() : only for tests, so debug_ prefix src/allmydata/mutable/servermap.py | 7 ++----- src/allmydata/test/test_mutable.py | 6 +++--- 2 files changed, 5 insertions(+), 8 deletions(-) commit 2d32e448677d6b818692e801045d4115b29abf21 Author: Brian Warner Date: Tue Oct 4 11:07:10 2011 -0400 API of ServerMap.all_servers_for_version() src/allmydata/mutable/servermap.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) commit 48f3204d1889c3e7179578125c4bdef515af3d6a Author: Brian Warner Date: Tue Oct 4 11:04:50 2011 -0400 internals of ServerMap methods that use make_versionmap(), remove temp copy src/allmydata/mutable/servermap.py | 28 +++++++++---------------- 1 files changed, 10 insertions(+), 18 deletions(-) commit 5c3da77b6c777a145bd5ddfaa4db849dc9495548 Author: Brian Warner Date: Tue Oct 4 11:01:28 2011 -0400 API of ServerMap.make_versionmap() src/allmydata/mutable/checker.py | 4 ++-- src/allmydata/mutable/retrieve.py | 5 ++--- src/allmydata/mutable/servermap.py | 4 ++-- src/allmydata/test/test_mutable.py | 7 ++++--- 4 files changed, 10 insertions(+), 10 deletions(-) commit b6882ece49afb4c507d118af2db346fa329209dc Author: Brian Warner Date: Tue Oct 4 10:53:38 2011 -0400 make a copy of ServerMap.make_versionmap() (_make_versionmap2) for internal use src/allmydata/mutable/servermap.py | 18 +++++++++++++----- 1 files changed, 13 insertions(+), 5 deletions(-) commit 963f8e63faf32b950eb1b8103cd2ff16fe8f0151 Author: Brian Warner Date: Tue Oct 4 00:45:58 2011 -0400 API of RetrieveStatus.add_problem() src/allmydata/mutable/retrieve.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit 4976d29ffae565a048851601c29013bbae2976d8 Author: Brian Warner Date: Tue Oct 4 00:45:05 2011 -0400 API of RetrieveStatus.add_fetch_timing() src/allmydata/mutable/retrieve.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit d057d3bbba72663ee148a8b916bc2d52be2e3982 Author: Brian Warner Date: Tue Oct 4 00:44:04 2011 -0400 API of Retrieve.notify_server_corruption() src/allmydata/mutable/retrieve.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit 8a2a81e46671c860610e0e96d6add1a57551f22d Author: Brian Warner Date: Tue Oct 4 00:42:32 2011 -0400 remove unused _outstanding_queries src/allmydata/mutable/retrieve.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) commit 56d12cc9968d03ccd53764455c671122c4f391d1 Author: Brian Warner Date: Tue Oct 4 00:40:57 2011 -0400 change Retrieve.remaining_sharemap src/allmydata/mutable/retrieve.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) commit 4f0b7af4821f43290bfc70f2b1fc30149ad81281 Author: Brian Warner Date: Tue Oct 4 10:40:18 2011 -0400 accessor for PublishStatus._problems src/allmydata/mutable/publish.py | 4 +++- src/allmydata/web/status.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) commit 627087cf66d0b8cc519f4d551a967a7bd9b6a741 Author: Brian Warner Date: Tue Oct 4 10:36:39 2011 -0400 accessor for RetrieveStatus._problems src/allmydata/mutable/retrieve.py | 8 ++++++-- src/allmydata/web/status.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) commit ca7dea81f03801b1c7353fc00ecba689268109cf Author: Brian Warner Date: Tue Oct 4 00:35:32 2011 -0400 add .server to "reader", so we can get at it later src/allmydata/mutable/retrieve.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit 6ef516e24908ec195af084a7550d1921a5e983b0 Author: Brian Warner Date: Tue Oct 4 00:32:32 2011 -0400 temporarily give Retrieve a _storage_broker, so it can map serverids to servers src/allmydata/mutable/checker.py | 3 ++- src/allmydata/mutable/filenode.py | 6 ++++-- src/allmydata/mutable/retrieve.py | 5 +++-- src/allmydata/test/test_mutable.py | 4 ++-- 4 files changed, 11 insertions(+), 7 deletions(-) commit afe08e4dd3f4ff9ff7e8a2a8d28b181e3625bcc9 Author: Brian Warner Date: Tue Oct 4 00:21:51 2011 -0400 mutable/retrieve.py: s/peer/server/ src/allmydata/mutable/retrieve.py | 82 +++++++++++++------------- src/allmydata/test/test_mutable.py | 6 +- 2 files changed, 44 insertions(+), 44 deletions(-) commit 910afcb5d7f274880f68dd6cdb5b05f2bbc29adc Author: Brian Warner Date: Tue Oct 4 00:16:01 2011 -0400 web.status.PublishStatusPage: add comment, I think .problems isn't exercised src/allmydata/web/status.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) commit 311466dd8c931bbba40d590ade867704282e7f1a Author: Brian Warner Date: Mon Oct 3 23:48:16 2011 -0400 API of PublishStatus.add_per_server_time() src/allmydata/mutable/publish.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit 2df5faa1b6cbfbaded520d2320305a62fe961118 Author: Brian Warner Date: Mon Oct 3 23:46:37 2011 -0400 more simplifications src/allmydata/mutable/publish.py | 4 +--- 1 files changed, 1 insertions(+), 3 deletions(-) commit 6ac4544a3da385f2aad9392f906b90192f4f919a Author: Brian Warner Date: Mon Oct 3 23:44:08 2011 -0400 API of ServerMap.version_on_server() src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/servermap.py | 4 ++-- src/allmydata/test/test_mutable.py | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) commit 3e187e322511072e4683329df6b2c6c733a66dba Author: Brian Warner Date: Tue Oct 4 00:16:32 2011 -0400 API of ServerMap.make_sharemap() src/allmydata/mutable/servermap.py | 4 ++-- src/allmydata/test/test_mutable.py | 7 ++++--- src/allmydata/web/status.py | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) commit 318feed8437bdd8d4943c6569d38f7b54b6313cc Author: Brian Warner Date: Mon Oct 3 23:36:19 2011 -0400 small cleanups src/allmydata/mutable/publish.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) commit bd459ed5714e1db5a7163935c54b7b0b56db8349 Author: Brian Warner Date: Mon Oct 3 23:33:39 2011 -0400 API of ServerMap.add_new_share() src/allmydata/mutable/publish.py | 4 ++-- src/allmydata/mutable/servermap.py | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) commit f2804fb6ed11d80088e0da8ed48e6c2922f2ffef Author: Brian Warner Date: Mon Oct 3 23:30:26 2011 -0400 API of ServerMap.get_bad_shares() src/allmydata/mutable/publish.py | 3 +-- src/allmydata/mutable/servermap.py | 9 ++++----- 2 files changed, 5 insertions(+), 7 deletions(-) commit 965074a47b3ce1431cb46d9a233840afcf9105f5 Author: Brian Warner Date: Mon Oct 3 23:26:58 2011 -0400 more small cleanups src/allmydata/mutable/publish.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit 38020da34f034f8889947dd3dc05e087ffff7106 Author: Brian Warner Date: Mon Oct 3 23:18:47 2011 -0400 change Publish.bad_share_checkstrings src/allmydata/mutable/publish.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit 5efebcbd2ee0c2f299ea86f7591d856c0f265304 Author: Brian Warner Date: Mon Oct 3 23:16:31 2011 -0400 change internals of Publish.update_goal() src/allmydata/mutable/publish.py | 8 +++----- 1 files changed, 3 insertions(+), 5 deletions(-) commit e91b55ff4c2a69165b71f2c7b217ac319ff4c527 Author: Brian Warner Date: Mon Oct 3 23:11:42 2011 -0400 get rid of Publish.connections src/allmydata/mutable/publish.py | 27 +++++---------------------- 1 files changed, 5 insertions(+), 22 deletions(-) commit 64e9a53b3229ebe2f9ebf7ed502d539311d0e037 Author: Brian Warner Date: Mon Oct 3 23:05:32 2011 -0400 change Publish.bad_servers src/allmydata/mutable/publish.py | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) commit b85a934bef315a06bcfe00c9c12a3627fed2b918 Author: Brian Warner Date: Mon Oct 3 23:03:07 2011 -0400 Publish.bad_servers: fix bug, this should be a set of serverids, not writers src/allmydata/mutable/publish.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) commit 605ea15ec15ed671513819003ccd211cdb9761e0 Author: Brian Warner Date: Mon Oct 3 23:00:21 2011 -0400 change .placed src/allmydata/mutable/publish.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit f7aba37b1b345d5b6d5cb16e3b3f6f3c1afb658e Author: Brian Warner Date: Mon Oct 3 22:59:22 2011 -0400 temporarily stash IServer as .server on the "writer" object src/allmydata/mutable/publish.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) commit f9b551d788e7db1f187fce5ab98ab5d5fe4e1c36 Author: Brian Warner Date: Mon Oct 3 22:48:18 2011 -0400 change Publish.goal and API of log_goal() to use IServer, not serverid src/allmydata/mutable/publish.py | 48 ++++++++++++++-------------- 1 files changed, 24 insertions(+), 24 deletions(-) commit 75f20616558e4900b8b1f685dd99aa838de6d452 Author: Brian Warner Date: Mon Oct 3 15:27:02 2011 -0400 API of ServerMap.get_known_shares() src/allmydata/mutable/publish.py | 16 ++++++++++------ src/allmydata/mutable/servermap.py | 7 ++----- 2 files changed, 12 insertions(+), 11 deletions(-) commit 1c38c9d37bb08221b4418762234b1a62397b3b4b Author: Brian Warner Date: Mon Oct 3 15:20:29 2011 -0400 Publish.full_serverlist src/allmydata/mutable/publish.py | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) commit b6cbd215a04b9cde31a7d92a97a7f048622b16f1 Author: Brian Warner Date: Mon Oct 3 15:12:31 2011 -0400 API of ServerMap.all_servers() src/allmydata/mutable/servermap.py | 19 ++++++------------- 1 files changed, 6 insertions(+), 13 deletions(-) commit e63cd0315fae65357b1727ec6d5ff3c6e0d27c98 Author: Brian Warner Date: Mon Oct 3 15:10:18 2011 -0400 remove ServerMap.connections, set_rref_for_serverid() src/allmydata/mutable/servermap.py | 11 +---------- 1 files changed, 1 insertions(+), 10 deletions(-) commit 4df52db2f80eb12eefa5d57103c24893cde89553 Author: Brian Warner Date: Mon Oct 3 15:04:06 2011 -0400 API of ServerMap.mark_server_reachable() src/allmydata/mutable/servermap.py | 7 ++----- 1 files changed, 2 insertions(+), 5 deletions(-) commit 69c715bde77944dc25181b3dbbeb042c816f9a1b Author: Brian Warner Date: Mon Oct 3 15:03:21 2011 -0400 API of ServerMap.mark_server_unreachable() src/allmydata/mutable/servermap.py | 9 +++------ 1 files changed, 3 insertions(+), 6 deletions(-) commit 3d784d60eec1c508858e3a617e4411ffbcc3c1fa Author: Brian Warner Date: Mon Oct 3 15:02:03 2011 -0400 API of status.set_privkey_from() src/allmydata/mutable/servermap.py | 7 +++---- 1 files changed, 3 insertions(+), 4 deletions(-) commit 544ed3ea29bed7e66da7fd29ca3f6f076f27a9e6 Author: Brian Warner Date: Mon Oct 3 15:01:15 2011 -0400 API of status.add_per_server_time() src/allmydata/mutable/servermap.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) commit fffe5008b6320bd1e04c3c68389a2bf2ee383fa8 Author: Brian Warner Date: Mon Oct 3 14:59:02 2011 -0400 remove unused .versionmap src/allmydata/mutable/servermap.py | 7 ------- 1 files changed, 0 insertions(+), 7 deletions(-) commit 2816562e090d2294179db3588dafcca18de1bc2b Author: Brian Warner Date: Mon Oct 3 14:57:51 2011 -0400 remove serverid from all log messages. Also one unused lambda. src/allmydata/mutable/servermap.py | 30 +++++++++++++------------- 1 files changed, 15 insertions(+), 15 deletions(-) commit 28fa6b1a2738fa98c1f1dbd3d0e01ae98912d11f Author: Brian Warner Date: Mon Oct 3 14:54:30 2011 -0400 removed unused _readers src/allmydata/mutable/servermap.py | 3 --- 1 files changed, 0 insertions(+), 3 deletions(-) commit a8e4ed3d645ab592d1add6a1e69b6d1ebfb77817 Author: Brian Warner Date: Mon Oct 3 14:54:16 2011 -0400 remove unused _sharemap src/allmydata/mutable/servermap.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) commit 3f072e55cf1d0700f9fffe23f8f3a475725df588 Author: Brian Warner Date: Mon Oct 3 14:49:03 2011 -0400 _must_query src/allmydata/mutable/servermap.py | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) commit c599a059b8df3f5785e4bf89fb6ecc6d8dcd708b Author: Brian Warner Date: Mon Oct 3 14:48:05 2011 -0400 _queries_outstanding src/allmydata/mutable/servermap.py | 16 +++++++--------- 1 files changed, 7 insertions(+), 9 deletions(-) commit 7743759f98ac2c07926b2fdbd80bf52dfab33085 Author: Brian Warner Date: Mon Oct 3 14:46:17 2011 -0400 _empty_servers src/allmydata/mutable/servermap.py | 5 ++--- 1 files changed, 2 insertions(+), 3 deletions(-) commit 6bb1825916828a713a32cdf7f7411fa3ea2e1e5d Author: Brian Warner Date: Mon Oct 3 14:45:39 2011 -0400 _good_servers src/allmydata/mutable/servermap.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) commit 1768fab1b51d8dd93ecabbaaabfadfa20cf6c3d4 Author: Brian Warner Date: Mon Oct 3 14:44:59 2011 -0400 _bad_servers src/allmydata/mutable/servermap.py | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) commit dccbaef30f0ba714c746bf6d4a1a803c36e17b65 Author: Brian Warner Date: Mon Oct 3 14:41:54 2011 -0400 API of _try_to_set_pubkey() src/allmydata/mutable/servermap.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) commit 0481ea70042ba3575f15eac7fd0780f8ece580cc Author: Brian Warner Date: Mon Oct 3 14:35:02 2011 -0400 API of notify_server_corruption() src/allmydata/mutable/servermap.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit bea9cba18fb3b9c11bb22f18356a263ecec7351e Author: Brian Warner Date: Mon Oct 3 14:34:09 2011 -0400 API of _got_signature_one_share() src/allmydata/mutable/servermap.py | 9 +++++---- 1 files changed, 5 insertions(+), 4 deletions(-) commit 1520123583cf78650706e114b15bb5b0ac1f4a14 Author: Brian Warner Date: Mon Oct 3 14:32:33 2011 -0400 API of _try_to_validate_privkey() src/allmydata/mutable/servermap.py | 9 +++++---- 1 files changed, 5 insertions(+), 4 deletions(-) commit 938852c9c8519c7a078f58a9b1f4dd8ec8b6715e Author: Brian Warner Date: Mon Oct 3 14:31:48 2011 -0400 API and internals of _add_lease_failed() src/allmydata/mutable/servermap.py | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) commit 3843dba367e3c19e176a622ab853cb51d2472ddf Author: Brian Warner Date: Mon Oct 3 14:30:37 2011 -0400 API of _privkey_query_failed() src/allmydata/mutable/servermap.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit 2219a710e1633cd57d0ca0786490de87b3e19ba7 Author: Brian Warner Date: Mon Oct 3 14:29:43 2011 -0400 fix bug in call to _privkey_query_failed, unrelated to refactoring src/allmydata/mutable/servermap.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) commit ae615bec7d0d1b269710b6902797b12f9592ad62 Author: Brian Warner Date: Mon Oct 3 14:27:17 2011 -0400 API of _got_corrupt_share() src/allmydata/mutable/servermap.py | 17 +++++++++-------- 1 files changed, 9 insertions(+), 8 deletions(-) commit cb51c95a6f4e077278157a77dab060c8c1ad7a81 Author: Brian Warner Date: Mon Oct 3 14:23:16 2011 -0400 API of _got_results() src/allmydata/mutable/servermap.py | 9 +++++---- 1 files changed, 5 insertions(+), 4 deletions(-) commit bac9154fe0af18f226999a58ffc2362d8cf4b802 Author: Brian Warner Date: Mon Oct 3 14:19:19 2011 -0400 API of _query_failed() src/allmydata/mutable/servermap.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit fdc29a8ca95d4b5c503e5382b9e5d4d02141ba12 Author: Brian Warner Date: Mon Oct 3 14:17:20 2011 -0400 API of _do_read() src/allmydata/mutable/servermap.py | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) commit e7e9e338f28d004aa4d423d11c65f1e271ac7322 Author: Brian Warner Date: Mon Oct 3 14:20:21 2011 -0400 API of _do_query() src/allmydata/mutable/servermap.py | 15 +++++++-------- 1 files changed, 7 insertions(+), 8 deletions(-) commit 330625b9dac4cdbe72a11464a893065b9aeed453 Author: Brian Warner Date: Mon Oct 3 14:43:05 2011 -0400 next step: first batch of updates to ServermapUpdater updates: most method-local variables in update() API of _build_initial_querylist() API of _send_initial_requests() .full_serverlist .extra_servers src/allmydata/mutable/servermap.py | 39 ++++++++++++++------------ 1 files changed, 21 insertions(+), 18 deletions(-) commit 4aadc584fa7dcb2daa86b048c81dee0049ba26d9 Author: Brian Warner Date: Mon Oct 3 15:07:00 2011 -0400 internal change: index _bad_shares with IServer src/allmydata/mutable/servermap.py | 20 ++++++++++---------- 1 files changed, 10 insertions(+), 10 deletions(-) commit 16d4e6fa82a9907dbdc92094213387c6a4164e41 Author: Brian Warner Date: Mon Oct 3 18:20:47 2011 +0100 internal change: index _known_shares with IServer instead of serverid callers are unchanged src/allmydata/mutable/servermap.py | 42 +++++++++++++++---------- 1 files changed, 25 insertions(+), 17 deletions(-) commit ceeb5f4938cc814a0c75d1b8f4018aed965c2176 Author: Brian Warner Date: Mon Oct 3 18:11:43 2011 +0100 accessors and name cleanup for servermap.Servermap.last_update_mode/time src/allmydata/mutable/filenode.py | 6 +++--- src/allmydata/mutable/publish.py | 4 ++-- src/allmydata/mutable/servermap.py | 17 +++++++++++------ 3 files changed, 16 insertions(+), 11 deletions(-) commit 8d3cbda82661c0a7e5c3d3b65cf7a5d5ab7e32c0 Author: Brian Warner Date: Mon Oct 3 18:11:14 2011 +0100 accessors and name cleanup for servermap.Servermap.problems src/allmydata/mutable/servermap.py | 21 +++++++++++++-------- src/allmydata/test/test_mutable.py | 6 +++--- 2 files changed, 16 insertions(+), 11 deletions(-) commit 348f57988f79389db0aab7672e6eaa9a6d8e3219 Author: Brian Warner Date: Mon Oct 3 18:10:41 2011 +0100 accessors and name cleanup for servermap.Servermap.bad_shares src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/servermap.py | 30 ++++++++++++++----------- 2 files changed, 18 insertions(+), 14 deletions(-) commit 520c9368134673cdf76c653c5e1bb91c2ab5d51e Author: Brian Warner Date: Mon Oct 3 18:10:05 2011 +0100 accessors and name cleanup for servermap.Servermap.servermap . src/allmydata/mutable/publish.py | 14 +++++---- src/allmydata/mutable/servermap.py | 38 ++++++++++++++----------- 2 files changed, 29 insertions(+), 23 deletions(-) commit b8b8dc38287a91dbdf494426ac801d9381ce5841 Author: Brian Warner Date: Mon Oct 3 18:08:02 2011 +0100 fix reachable_servers src/allmydata/mutable/checker.py | 3 ++- src/allmydata/mutable/publish.py | 4 +++- src/allmydata/mutable/servermap.py | 12 ++++++++++-- 3 files changed, 15 insertions(+), 4 deletions(-) commit cb0cfd1adfefad357c187aaaf690c3df68b622bc Author: Brian Warner Date: Mon Oct 3 18:06:03 2011 +0100 fix Servermap.unreachable_servers src/allmydata/mutable/servermap.py | 11 ++++++++--- 1 files changed, 8 insertions(+), 3 deletions(-) commit 2d9ea79b94bd4db674d40386fda90825785ac495 Author: Brian Warner Date: Mon Oct 3 18:03:48 2011 +0100 give ServerMap a StorageFarmBroker, temporary this makes it possible for the ServerMap to accept bare serverids and still build data structures with IServers src/allmydata/mutable/checker.py | 2 +- src/allmydata/mutable/filenode.py | 2 +- src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/servermap.py | 5 +++-- src/allmydata/test/test_mutable.py | 8 ++++---- 5 files changed, 10 insertions(+), 9 deletions(-) commit 718d1aeff6fded893f65397806d22ece928b0dd4 Author: Brian Warner Date: Mon Oct 3 13:43:30 2011 -0400 add StorageFarmBroker.get_server_for_id(), temporary helper This will go away once we're passing IServers everywhere. src/allmydata/storage_client.py | 2 ++ src/allmydata/test/no_network.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 0 deletions(-) commit ece20231d7fda0d503704842a4aa068dfbc2e54e Author: Brian Warner Date: Sun Oct 2 01:11:50 2011 +0100 add proper accessors for Servermap.connections, to make refactoring easier src/allmydata/mutable/publish.py | 6 +++--- src/allmydata/mutable/retrieve.py | 10 +++++----- src/allmydata/mutable/servermap.py | 17 +++++++++++------ 3 files changed, 19 insertions(+), 14 deletions(-) commit 3b943d6bf302ff702668081a612fc4fe2604cf9c Author: Brian Warner Date: Fri Sep 23 10:34:30 2011 -0700 mutable/servermap.py and neighbors: s/peer/server/ src/allmydata/mutable/checker.py | 22 +- src/allmydata/mutable/publish.py | 204 +++++++------- src/allmydata/mutable/servermap.py | 402 +++++++++++++------------- src/allmydata/test/test_mutable.py | 18 +- 4 files changed, 323 insertions(+), 323 deletions(-) IServer refactoring: pass IServer instances around, instead of peerids refs #1363 This collapses 88 small incremental changes (each of which passes all tests) into one big patch. The development process for the long path started with adding some temporary scaffolding, changing one method at a time, then removing the scaffolding. The individual pieces are as follows, in reverse chronological order (the first patch is at the end of this comment): commit 9bbe4174fd0d98a6cf47a8ef96e85d9ef34b2f9a Author: Brian Warner Date: Tue Oct 4 16:05:00 2011 -0400 immutable/downloader/status.py: correct comment src/allmydata/immutable/downloader/status.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) commit 72146a7c7c91eac2f7c3ceb801eb7a1721376889 Author: Brian Warner Date: Tue Oct 4 15:46:20 2011 -0400 remove temporary ServerMap._storage_broker src/allmydata/mutable/checker.py | 2 +- src/allmydata/mutable/filenode.py | 2 +- src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/servermap.py | 5 ++--- src/allmydata/test/test_mutable.py | 8 ++++---- 5 files changed, 9 insertions(+), 10 deletions(-) commit d703096b41632c47d76414b12672e076a422ff5c Author: Brian Warner Date: Tue Oct 4 15:37:05 2011 -0400 remove temporary storage_broker.get_server_for_id() src/allmydata/storage_client.py | 3 --- src/allmydata/test/no_network.py | 13 ------------- 2 files changed, 0 insertions(+), 16 deletions(-) commit 620cc5d80882ef6f7decfd26af8a6c7c1ddf80d1 Author: Brian Warner Date: Tue Oct 4 12:50:06 2011 -0400 API of Retrieve._try_to_validate_privkey(), trying to remove reader.server src/allmydata/mutable/retrieve.py | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) commit 92f43f856f4a8b36c207d1b190ed8699b5a4ecb4 Author: Brian Warner Date: Tue Oct 4 12:48:08 2011 -0400 API of Retrieve._validate_block(), trying to remove reader.server src/allmydata/mutable/retrieve.py | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) commit 572d5070761861a2190349d1ed8d85dbc25698a5 Author: Brian Warner Date: Tue Oct 4 12:36:58 2011 -0400 API of Retrieve._mark_bad_share(), trying to remove reader.server src/allmydata/mutable/retrieve.py | 21 +++++++++------------ 1 files changed, 9 insertions(+), 12 deletions(-) commit a793ff00c0de1e2eec7b46288fdf388c7a2bec89 Author: Brian Warner Date: Tue Oct 4 12:06:13 2011 -0400 remove now-unused get_rref_for_serverid() src/allmydata/mutable/servermap.py | 3 --- 1 files changed, 0 insertions(+), 3 deletions(-) commit 1b9827cc9366bf90b93297fdd6832f2ad0480ce7 Author: Brian Warner Date: Tue Oct 4 12:03:09 2011 -0400 Retrieve: stop adding .serverid attributes to readers src/allmydata/mutable/retrieve.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) commit 5d4e9d491b19e49d2e443a1dfff2c672842c36ef Author: Brian Warner Date: Tue Oct 4 12:03:34 2011 -0400 return value of Retrieve(verify=True) src/allmydata/mutable/checker.py | 11 ++++++----- src/allmydata/mutable/retrieve.py | 3 +-- 2 files changed, 7 insertions(+), 7 deletions(-) commit e9ab7978c384e1f677cb7779dc449b1044face82 Author: Brian Warner Date: Tue Oct 4 11:54:23 2011 -0400 Retrieve._bad_shares (but not return value, used by Verifier) src/allmydata/mutable/retrieve.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) commit 2d91926de233ec5c881f30e36b4a30ad92ab42a9 Author: Brian Warner Date: Tue Oct 4 11:51:23 2011 -0400 Publish: stop adding .serverid attributes to writers src/allmydata/mutable/publish.py | 9 ++------- 1 files changed, 2 insertions(+), 7 deletions(-) commit 47c7a0105dec7cbf4f7e0a3ce800bbb85b15df4a Author: Brian Warner Date: Tue Oct 4 11:56:33 2011 -0400 API of get_write_enabler() src/allmydata/mutable/filenode.py | 7 ++++--- src/allmydata/mutable/publish.py | 4 ++-- src/allmydata/test/no_network.py | 3 +++ 3 files changed, 9 insertions(+), 5 deletions(-) commit 9196a5c6590fdbfd660325ea8358b345887d3db0 Author: Brian Warner Date: Tue Oct 4 11:46:24 2011 -0400 API of get_(renewal|cancel)_secret() src/allmydata/mutable/filenode.py | 14 ++++++++------ src/allmydata/mutable/publish.py | 8 ++++---- src/allmydata/mutable/servermap.py | 5 ++--- 3 files changed, 14 insertions(+), 13 deletions(-) commit de7c1552f8c163eff5b6d820b5fb3b21c1b47cb5 Author: Brian Warner Date: Tue Oct 4 11:41:52 2011 -0400 API of CorruptShareError. Also comment out some related+unused test_web.py code src/allmydata/mutable/common.py | 13 +++++-------- src/allmydata/mutable/retrieve.py | 10 +++++----- src/allmydata/mutable/servermap.py | 8 +++----- src/allmydata/test/common.py | 13 ++++++++----- 4 files changed, 21 insertions(+), 23 deletions(-) commit 2c1c314046b620c16f1e66d030c150d768b7d01e Author: Brian Warner Date: Tue Oct 4 12:01:46 2011 -0400 API of ServerMap.mark_bad_share() src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/retrieve.py | 6 +++--- src/allmydata/mutable/servermap.py | 6 ++---- src/allmydata/test/test_mutable.py | 3 +-- 4 files changed, 7 insertions(+), 10 deletions(-) commit 1bed349030779fd0c378ae4e821384f953c6f6ff Author: Brian Warner Date: Tue Oct 4 11:11:17 2011 -0400 API+name of ServerMap.shares_on_server() : only for tests, so debug_ prefix src/allmydata/mutable/servermap.py | 7 ++----- src/allmydata/test/test_mutable.py | 6 +++--- 2 files changed, 5 insertions(+), 8 deletions(-) commit 2d32e448677d6b818692e801045d4115b29abf21 Author: Brian Warner Date: Tue Oct 4 11:07:10 2011 -0400 API of ServerMap.all_servers_for_version() src/allmydata/mutable/servermap.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) commit 48f3204d1889c3e7179578125c4bdef515af3d6a Author: Brian Warner Date: Tue Oct 4 11:04:50 2011 -0400 internals of ServerMap methods that use make_versionmap(), remove temp copy src/allmydata/mutable/servermap.py | 28 +++++++++---------------- 1 files changed, 10 insertions(+), 18 deletions(-) commit 5c3da77b6c777a145bd5ddfaa4db849dc9495548 Author: Brian Warner Date: Tue Oct 4 11:01:28 2011 -0400 API of ServerMap.make_versionmap() src/allmydata/mutable/checker.py | 4 ++-- src/allmydata/mutable/retrieve.py | 5 ++--- src/allmydata/mutable/servermap.py | 4 ++-- src/allmydata/test/test_mutable.py | 7 ++++--- 4 files changed, 10 insertions(+), 10 deletions(-) commit b6882ece49afb4c507d118af2db346fa329209dc Author: Brian Warner Date: Tue Oct 4 10:53:38 2011 -0400 make a copy of ServerMap.make_versionmap() (_make_versionmap2) for internal use src/allmydata/mutable/servermap.py | 18 +++++++++++++----- 1 files changed, 13 insertions(+), 5 deletions(-) commit 963f8e63faf32b950eb1b8103cd2ff16fe8f0151 Author: Brian Warner Date: Tue Oct 4 00:45:58 2011 -0400 API of RetrieveStatus.add_problem() src/allmydata/mutable/retrieve.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit 4976d29ffae565a048851601c29013bbae2976d8 Author: Brian Warner Date: Tue Oct 4 00:45:05 2011 -0400 API of RetrieveStatus.add_fetch_timing() src/allmydata/mutable/retrieve.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit d057d3bbba72663ee148a8b916bc2d52be2e3982 Author: Brian Warner Date: Tue Oct 4 00:44:04 2011 -0400 API of Retrieve.notify_server_corruption() src/allmydata/mutable/retrieve.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit 8a2a81e46671c860610e0e96d6add1a57551f22d Author: Brian Warner Date: Tue Oct 4 00:42:32 2011 -0400 remove unused _outstanding_queries src/allmydata/mutable/retrieve.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) commit 56d12cc9968d03ccd53764455c671122c4f391d1 Author: Brian Warner Date: Tue Oct 4 00:40:57 2011 -0400 change Retrieve.remaining_sharemap src/allmydata/mutable/retrieve.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) commit 4f0b7af4821f43290bfc70f2b1fc30149ad81281 Author: Brian Warner Date: Tue Oct 4 10:40:18 2011 -0400 accessor for PublishStatus._problems src/allmydata/mutable/publish.py | 4 +++- src/allmydata/web/status.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) commit 627087cf66d0b8cc519f4d551a967a7bd9b6a741 Author: Brian Warner Date: Tue Oct 4 10:36:39 2011 -0400 accessor for RetrieveStatus._problems src/allmydata/mutable/retrieve.py | 8 ++++++-- src/allmydata/web/status.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) commit ca7dea81f03801b1c7353fc00ecba689268109cf Author: Brian Warner Date: Tue Oct 4 00:35:32 2011 -0400 add .server to "reader", so we can get at it later src/allmydata/mutable/retrieve.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit 6ef516e24908ec195af084a7550d1921a5e983b0 Author: Brian Warner Date: Tue Oct 4 00:32:32 2011 -0400 temporarily give Retrieve a _storage_broker, so it can map serverids to servers src/allmydata/mutable/checker.py | 3 ++- src/allmydata/mutable/filenode.py | 6 ++++-- src/allmydata/mutable/retrieve.py | 5 +++-- src/allmydata/test/test_mutable.py | 4 ++-- 4 files changed, 11 insertions(+), 7 deletions(-) commit afe08e4dd3f4ff9ff7e8a2a8d28b181e3625bcc9 Author: Brian Warner Date: Tue Oct 4 00:21:51 2011 -0400 mutable/retrieve.py: s/peer/server/ src/allmydata/mutable/retrieve.py | 82 +++++++++++++------------- src/allmydata/test/test_mutable.py | 6 +- 2 files changed, 44 insertions(+), 44 deletions(-) commit 910afcb5d7f274880f68dd6cdb5b05f2bbc29adc Author: Brian Warner Date: Tue Oct 4 00:16:01 2011 -0400 web.status.PublishStatusPage: add comment, I think .problems isn't exercised src/allmydata/web/status.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) commit 311466dd8c931bbba40d590ade867704282e7f1a Author: Brian Warner Date: Mon Oct 3 23:48:16 2011 -0400 API of PublishStatus.add_per_server_time() src/allmydata/mutable/publish.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit 2df5faa1b6cbfbaded520d2320305a62fe961118 Author: Brian Warner Date: Mon Oct 3 23:46:37 2011 -0400 more simplifications src/allmydata/mutable/publish.py | 4 +--- 1 files changed, 1 insertions(+), 3 deletions(-) commit 6ac4544a3da385f2aad9392f906b90192f4f919a Author: Brian Warner Date: Mon Oct 3 23:44:08 2011 -0400 API of ServerMap.version_on_server() src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/servermap.py | 4 ++-- src/allmydata/test/test_mutable.py | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) commit 3e187e322511072e4683329df6b2c6c733a66dba Author: Brian Warner Date: Tue Oct 4 00:16:32 2011 -0400 API of ServerMap.make_sharemap() src/allmydata/mutable/servermap.py | 4 ++-- src/allmydata/test/test_mutable.py | 7 ++++--- src/allmydata/web/status.py | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) commit 318feed8437bdd8d4943c6569d38f7b54b6313cc Author: Brian Warner Date: Mon Oct 3 23:36:19 2011 -0400 small cleanups src/allmydata/mutable/publish.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) commit bd459ed5714e1db5a7163935c54b7b0b56db8349 Author: Brian Warner Date: Mon Oct 3 23:33:39 2011 -0400 API of ServerMap.add_new_share() src/allmydata/mutable/publish.py | 4 ++-- src/allmydata/mutable/servermap.py | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) commit f2804fb6ed11d80088e0da8ed48e6c2922f2ffef Author: Brian Warner Date: Mon Oct 3 23:30:26 2011 -0400 API of ServerMap.get_bad_shares() src/allmydata/mutable/publish.py | 3 +-- src/allmydata/mutable/servermap.py | 9 ++++----- 2 files changed, 5 insertions(+), 7 deletions(-) commit 965074a47b3ce1431cb46d9a233840afcf9105f5 Author: Brian Warner Date: Mon Oct 3 23:26:58 2011 -0400 more small cleanups src/allmydata/mutable/publish.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit 38020da34f034f8889947dd3dc05e087ffff7106 Author: Brian Warner Date: Mon Oct 3 23:18:47 2011 -0400 change Publish.bad_share_checkstrings src/allmydata/mutable/publish.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit 5efebcbd2ee0c2f299ea86f7591d856c0f265304 Author: Brian Warner Date: Mon Oct 3 23:16:31 2011 -0400 change internals of Publish.update_goal() src/allmydata/mutable/publish.py | 8 +++----- 1 files changed, 3 insertions(+), 5 deletions(-) commit e91b55ff4c2a69165b71f2c7b217ac319ff4c527 Author: Brian Warner Date: Mon Oct 3 23:11:42 2011 -0400 get rid of Publish.connections src/allmydata/mutable/publish.py | 27 +++++---------------------- 1 files changed, 5 insertions(+), 22 deletions(-) commit 64e9a53b3229ebe2f9ebf7ed502d539311d0e037 Author: Brian Warner Date: Mon Oct 3 23:05:32 2011 -0400 change Publish.bad_servers src/allmydata/mutable/publish.py | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) commit b85a934bef315a06bcfe00c9c12a3627fed2b918 Author: Brian Warner Date: Mon Oct 3 23:03:07 2011 -0400 Publish.bad_servers: fix bug, this should be a set of serverids, not writers src/allmydata/mutable/publish.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) commit 605ea15ec15ed671513819003ccd211cdb9761e0 Author: Brian Warner Date: Mon Oct 3 23:00:21 2011 -0400 change .placed src/allmydata/mutable/publish.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit f7aba37b1b345d5b6d5cb16e3b3f6f3c1afb658e Author: Brian Warner Date: Mon Oct 3 22:59:22 2011 -0400 temporarily stash IServer as .server on the "writer" object src/allmydata/mutable/publish.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) commit f9b551d788e7db1f187fce5ab98ab5d5fe4e1c36 Author: Brian Warner Date: Mon Oct 3 22:48:18 2011 -0400 change Publish.goal and API of log_goal() to use IServer, not serverid src/allmydata/mutable/publish.py | 48 ++++++++++++++-------------- 1 files changed, 24 insertions(+), 24 deletions(-) commit 75f20616558e4900b8b1f685dd99aa838de6d452 Author: Brian Warner Date: Mon Oct 3 15:27:02 2011 -0400 API of ServerMap.get_known_shares() src/allmydata/mutable/publish.py | 16 ++++++++++------ src/allmydata/mutable/servermap.py | 7 ++----- 2 files changed, 12 insertions(+), 11 deletions(-) commit 1c38c9d37bb08221b4418762234b1a62397b3b4b Author: Brian Warner Date: Mon Oct 3 15:20:29 2011 -0400 Publish.full_serverlist src/allmydata/mutable/publish.py | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) commit b6cbd215a04b9cde31a7d92a97a7f048622b16f1 Author: Brian Warner Date: Mon Oct 3 15:12:31 2011 -0400 API of ServerMap.all_servers() src/allmydata/mutable/servermap.py | 19 ++++++------------- 1 files changed, 6 insertions(+), 13 deletions(-) commit e63cd0315fae65357b1727ec6d5ff3c6e0d27c98 Author: Brian Warner Date: Mon Oct 3 15:10:18 2011 -0400 remove ServerMap.connections, set_rref_for_serverid() src/allmydata/mutable/servermap.py | 11 +---------- 1 files changed, 1 insertions(+), 10 deletions(-) commit 4df52db2f80eb12eefa5d57103c24893cde89553 Author: Brian Warner Date: Mon Oct 3 15:04:06 2011 -0400 API of ServerMap.mark_server_reachable() src/allmydata/mutable/servermap.py | 7 ++----- 1 files changed, 2 insertions(+), 5 deletions(-) commit 69c715bde77944dc25181b3dbbeb042c816f9a1b Author: Brian Warner Date: Mon Oct 3 15:03:21 2011 -0400 API of ServerMap.mark_server_unreachable() src/allmydata/mutable/servermap.py | 9 +++------ 1 files changed, 3 insertions(+), 6 deletions(-) commit 3d784d60eec1c508858e3a617e4411ffbcc3c1fa Author: Brian Warner Date: Mon Oct 3 15:02:03 2011 -0400 API of status.set_privkey_from() src/allmydata/mutable/servermap.py | 7 +++---- 1 files changed, 3 insertions(+), 4 deletions(-) commit 544ed3ea29bed7e66da7fd29ca3f6f076f27a9e6 Author: Brian Warner Date: Mon Oct 3 15:01:15 2011 -0400 API of status.add_per_server_time() src/allmydata/mutable/servermap.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) commit fffe5008b6320bd1e04c3c68389a2bf2ee383fa8 Author: Brian Warner Date: Mon Oct 3 14:59:02 2011 -0400 remove unused .versionmap src/allmydata/mutable/servermap.py | 7 ------- 1 files changed, 0 insertions(+), 7 deletions(-) commit 2816562e090d2294179db3588dafcca18de1bc2b Author: Brian Warner Date: Mon Oct 3 14:57:51 2011 -0400 remove serverid from all log messages. Also one unused lambda. src/allmydata/mutable/servermap.py | 30 +++++++++++++------------- 1 files changed, 15 insertions(+), 15 deletions(-) commit 28fa6b1a2738fa98c1f1dbd3d0e01ae98912d11f Author: Brian Warner Date: Mon Oct 3 14:54:30 2011 -0400 removed unused _readers src/allmydata/mutable/servermap.py | 3 --- 1 files changed, 0 insertions(+), 3 deletions(-) commit a8e4ed3d645ab592d1add6a1e69b6d1ebfb77817 Author: Brian Warner Date: Mon Oct 3 14:54:16 2011 -0400 remove unused _sharemap src/allmydata/mutable/servermap.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) commit 3f072e55cf1d0700f9fffe23f8f3a475725df588 Author: Brian Warner Date: Mon Oct 3 14:49:03 2011 -0400 _must_query src/allmydata/mutable/servermap.py | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) commit c599a059b8df3f5785e4bf89fb6ecc6d8dcd708b Author: Brian Warner Date: Mon Oct 3 14:48:05 2011 -0400 _queries_outstanding src/allmydata/mutable/servermap.py | 16 +++++++--------- 1 files changed, 7 insertions(+), 9 deletions(-) commit 7743759f98ac2c07926b2fdbd80bf52dfab33085 Author: Brian Warner Date: Mon Oct 3 14:46:17 2011 -0400 _empty_servers src/allmydata/mutable/servermap.py | 5 ++--- 1 files changed, 2 insertions(+), 3 deletions(-) commit 6bb1825916828a713a32cdf7f7411fa3ea2e1e5d Author: Brian Warner Date: Mon Oct 3 14:45:39 2011 -0400 _good_servers src/allmydata/mutable/servermap.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) commit 1768fab1b51d8dd93ecabbaaabfadfa20cf6c3d4 Author: Brian Warner Date: Mon Oct 3 14:44:59 2011 -0400 _bad_servers src/allmydata/mutable/servermap.py | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) commit dccbaef30f0ba714c746bf6d4a1a803c36e17b65 Author: Brian Warner Date: Mon Oct 3 14:41:54 2011 -0400 API of _try_to_set_pubkey() src/allmydata/mutable/servermap.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) commit 0481ea70042ba3575f15eac7fd0780f8ece580cc Author: Brian Warner Date: Mon Oct 3 14:35:02 2011 -0400 API of notify_server_corruption() src/allmydata/mutable/servermap.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit bea9cba18fb3b9c11bb22f18356a263ecec7351e Author: Brian Warner Date: Mon Oct 3 14:34:09 2011 -0400 API of _got_signature_one_share() src/allmydata/mutable/servermap.py | 9 +++++---- 1 files changed, 5 insertions(+), 4 deletions(-) commit 1520123583cf78650706e114b15bb5b0ac1f4a14 Author: Brian Warner Date: Mon Oct 3 14:32:33 2011 -0400 API of _try_to_validate_privkey() src/allmydata/mutable/servermap.py | 9 +++++---- 1 files changed, 5 insertions(+), 4 deletions(-) commit 938852c9c8519c7a078f58a9b1f4dd8ec8b6715e Author: Brian Warner Date: Mon Oct 3 14:31:48 2011 -0400 API and internals of _add_lease_failed() src/allmydata/mutable/servermap.py | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) commit 3843dba367e3c19e176a622ab853cb51d2472ddf Author: Brian Warner Date: Mon Oct 3 14:30:37 2011 -0400 API of _privkey_query_failed() src/allmydata/mutable/servermap.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit 2219a710e1633cd57d0ca0786490de87b3e19ba7 Author: Brian Warner Date: Mon Oct 3 14:29:43 2011 -0400 fix bug in call to _privkey_query_failed, unrelated to refactoring src/allmydata/mutable/servermap.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) commit ae615bec7d0d1b269710b6902797b12f9592ad62 Author: Brian Warner Date: Mon Oct 3 14:27:17 2011 -0400 API of _got_corrupt_share() src/allmydata/mutable/servermap.py | 17 +++++++++-------- 1 files changed, 9 insertions(+), 8 deletions(-) commit cb51c95a6f4e077278157a77dab060c8c1ad7a81 Author: Brian Warner Date: Mon Oct 3 14:23:16 2011 -0400 API of _got_results() src/allmydata/mutable/servermap.py | 9 +++++---- 1 files changed, 5 insertions(+), 4 deletions(-) commit bac9154fe0af18f226999a58ffc2362d8cf4b802 Author: Brian Warner Date: Mon Oct 3 14:19:19 2011 -0400 API of _query_failed() src/allmydata/mutable/servermap.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit fdc29a8ca95d4b5c503e5382b9e5d4d02141ba12 Author: Brian Warner Date: Mon Oct 3 14:17:20 2011 -0400 API of _do_read() src/allmydata/mutable/servermap.py | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) commit e7e9e338f28d004aa4d423d11c65f1e271ac7322 Author: Brian Warner Date: Mon Oct 3 14:20:21 2011 -0400 API of _do_query() src/allmydata/mutable/servermap.py | 15 +++++++-------- 1 files changed, 7 insertions(+), 8 deletions(-) commit 330625b9dac4cdbe72a11464a893065b9aeed453 Author: Brian Warner Date: Mon Oct 3 14:43:05 2011 -0400 next step: first batch of updates to ServermapUpdater updates: most method-local variables in update() API of _build_initial_querylist() API of _send_initial_requests() .full_serverlist .extra_servers src/allmydata/mutable/servermap.py | 39 ++++++++++++++------------ 1 files changed, 21 insertions(+), 18 deletions(-) commit 4aadc584fa7dcb2daa86b048c81dee0049ba26d9 Author: Brian Warner Date: Mon Oct 3 15:07:00 2011 -0400 internal change: index _bad_shares with IServer src/allmydata/mutable/servermap.py | 20 ++++++++++---------- 1 files changed, 10 insertions(+), 10 deletions(-) commit 16d4e6fa82a9907dbdc92094213387c6a4164e41 Author: Brian Warner Date: Mon Oct 3 18:20:47 2011 +0100 internal change: index _known_shares with IServer instead of serverid callers are unchanged src/allmydata/mutable/servermap.py | 42 +++++++++++++++---------- 1 files changed, 25 insertions(+), 17 deletions(-) commit ceeb5f4938cc814a0c75d1b8f4018aed965c2176 Author: Brian Warner Date: Mon Oct 3 18:11:43 2011 +0100 accessors and name cleanup for servermap.Servermap.last_update_mode/time src/allmydata/mutable/filenode.py | 6 +++--- src/allmydata/mutable/publish.py | 4 ++-- src/allmydata/mutable/servermap.py | 17 +++++++++++------ 3 files changed, 16 insertions(+), 11 deletions(-) commit 8d3cbda82661c0a7e5c3d3b65cf7a5d5ab7e32c0 Author: Brian Warner Date: Mon Oct 3 18:11:14 2011 +0100 accessors and name cleanup for servermap.Servermap.problems src/allmydata/mutable/servermap.py | 21 +++++++++++++-------- src/allmydata/test/test_mutable.py | 6 +++--- 2 files changed, 16 insertions(+), 11 deletions(-) commit 348f57988f79389db0aab7672e6eaa9a6d8e3219 Author: Brian Warner Date: Mon Oct 3 18:10:41 2011 +0100 accessors and name cleanup for servermap.Servermap.bad_shares src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/servermap.py | 30 ++++++++++++++----------- 2 files changed, 18 insertions(+), 14 deletions(-) commit 520c9368134673cdf76c653c5e1bb91c2ab5d51e Author: Brian Warner Date: Mon Oct 3 18:10:05 2011 +0100 accessors and name cleanup for servermap.Servermap.servermap . src/allmydata/mutable/publish.py | 14 +++++---- src/allmydata/mutable/servermap.py | 38 ++++++++++++++----------- 2 files changed, 29 insertions(+), 23 deletions(-) commit b8b8dc38287a91dbdf494426ac801d9381ce5841 Author: Brian Warner Date: Mon Oct 3 18:08:02 2011 +0100 fix reachable_servers src/allmydata/mutable/checker.py | 3 ++- src/allmydata/mutable/publish.py | 4 +++- src/allmydata/mutable/servermap.py | 12 ++++++++++-- 3 files changed, 15 insertions(+), 4 deletions(-) commit cb0cfd1adfefad357c187aaaf690c3df68b622bc Author: Brian Warner Date: Mon Oct 3 18:06:03 2011 +0100 fix Servermap.unreachable_servers src/allmydata/mutable/servermap.py | 11 ++++++++--- 1 files changed, 8 insertions(+), 3 deletions(-) commit 2d9ea79b94bd4db674d40386fda90825785ac495 Author: Brian Warner Date: Mon Oct 3 18:03:48 2011 +0100 give ServerMap a StorageFarmBroker, temporary this makes it possible for the ServerMap to accept bare serverids and still build data structures with IServers src/allmydata/mutable/checker.py | 2 +- src/allmydata/mutable/filenode.py | 2 +- src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/servermap.py | 5 +++-- src/allmydata/test/test_mutable.py | 8 ++++---- 5 files changed, 10 insertions(+), 9 deletions(-) commit 718d1aeff6fded893f65397806d22ece928b0dd4 Author: Brian Warner Date: Mon Oct 3 13:43:30 2011 -0400 add StorageFarmBroker.get_server_for_id(), temporary helper This will go away once we're passing IServers everywhere. src/allmydata/storage_client.py | 2 ++ src/allmydata/test/no_network.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 0 deletions(-) commit ece20231d7fda0d503704842a4aa068dfbc2e54e Author: Brian Warner Date: Sun Oct 2 01:11:50 2011 +0100 add proper accessors for Servermap.connections, to make refactoring easier src/allmydata/mutable/publish.py | 6 +++--- src/allmydata/mutable/retrieve.py | 10 +++++----- src/allmydata/mutable/servermap.py | 17 +++++++++++------ 3 files changed, 19 insertions(+), 14 deletions(-) commit 3b943d6bf302ff702668081a612fc4fe2604cf9c Author: Brian Warner Date: Fri Sep 23 10:34:30 2011 -0700 mutable/servermap.py and neighbors: s/peer/server/ src/allmydata/mutable/checker.py | 22 +- src/allmydata/mutable/publish.py | 204 +++++++------- src/allmydata/mutable/servermap.py | 402 +++++++++++++------------- src/allmydata/test/test_mutable.py | 18 +- 4 files changed, 323 insertions(+), 323 deletions(-) ] [TAG allmydata-tahoe-1.9.0 warner@lothar.com**20111031052301 Ignore-this: cf598210dd1f314a1a121bf29a3d5918 ] Patch bundle hash: bc4f1843530a92623bd1c70ed4fa82b103d252a2