Updated to Commit version 9.

ArqRepo.m

@@ -22,6 +22,8 @@
 #import "CryptoKey.h"
 #import "BlobKey.h"
 #import "Encryption.h"
+#import "NSData-GZip.h"
+
 
 @implementation ArqRepo
 + (NSString *)errorDomain {
@@ -92,7 +94,7 @@
         stretch = [sha1 characterAtIndex:40] == 'Y';
         sha1 = [sha1 substringToIndex:40];
     }
-    return [[[BlobKey alloc] initWithSHA1:sha1 stretchEncryptionKey:stretch] autorelease];
+    return [[[BlobKey alloc] initWithSHA1:sha1 storageType:StorageTypeS3 stretchEncryptionKey:stretch compressed:NO] autorelease];
 }
 - (Commit *)commitForBlobKey:(BlobKey *)commitBlobKey error:(NSError **)error {
     NSError *myError = nil;
@@ -152,7 +154,9 @@
     if (data == nil) {
         return nil;
     }
-
+    if ([blobKey compressed]) {
+        data = [data gzipInflate];
+    }
     DataInputStream *dis = [[DataInputStream alloc] initWithData:data];
     BufferedInputStream *bis = [[BufferedInputStream alloc] initWithUnderlyingStream:dis];
     Tree *tree = [[[Tree alloc] initWithBufferedInputStream:bis error:error] autorelease];

ArqRestoreCommand.m

@@ -49,6 +49,9 @@
 #import "CryptoKey.h"
 
 
+#define BUCKET_PLIST_SALT "BucketPL"
+
+
 @interface ArqRestoreCommand (internal)
 - (BOOL)printArqFolders:(NSError **)error;
 - (BOOL)processPath:(NSError **)error;
@@ -169,12 +172,7 @@
             }
             if (length >= 9 && !strncmp([data bytes], "encrypted", length)) {
                 NSData *encryptedData = [data subdataWithRange:NSMakeRange(9, [data length] - 9)];
-                ArqSalt *arqSalt = [[[ArqSalt alloc] initWithAccessKeyID:accessKey secretAccessKey:secretKey s3BucketName:s3BucketName computerUUID:computerUUID] autorelease];
+                CryptoKey *cryptoKey = [[[CryptoKey alloc] initWithPassword:encryptionPassword salt:[NSData dataWithBytes:BUCKET_PLIST_SALT length:8] error:error] autorelease];
-                NSData *salt = [arqSalt salt:error];
-                if (salt == nil) {
-                    return NO;
-                }
-                CryptoKey *cryptoKey = [[[CryptoKey alloc] initWithPassword:encryptionPassword salt:salt error:error] autorelease];
                 if (cryptoKey == nil) {
                     return NO;
                 }
@@ -217,24 +215,12 @@
     NSString *computerUUID = [path substringWithRange:computerUUIDRange];
     NSString *bucketUUID = [path substringWithRange:bucketUUIDRange];
 
-    NSError *saltError = nil;
-    ArqSalt *arqSalt = [[[ArqSalt alloc] initWithAccessKeyID:accessKey secretAccessKey:secretKey s3BucketName:s3BucketName computerUUID:computerUUID] autorelease];
-    NSData *salt = [arqSalt salt:&saltError];
-    if (salt == nil) {
-        if ([saltError code] != ERROR_NOT_FOUND) {
-            if (error != NULL) {
-                *error = saltError;
-            }
-            return NO;
-        }
-    }
-
     NSString *bucketName = @"(unknown)";
     NSData *data = [s3 dataAtPath:path error:NULL];
     if (data != nil) {
         if (!strncmp([data bytes], "encrypted", 9)) {
             data = [data subdataWithRange:NSMakeRange(9, [data length] - 9)];
-            CryptoKey *cryptoKey = [[[CryptoKey alloc] initWithPassword:encryptionPassword salt:salt error:error] autorelease];
+            CryptoKey *cryptoKey = [[[CryptoKey alloc] initWithPassword:encryptionPassword salt:[NSData dataWithBytes:BUCKET_PLIST_SALT length:8] error:error] autorelease];
             if (cryptoKey == nil) {
                 return NO;
             }
@@ -257,6 +243,18 @@
         uac = [[[UserAndComputer alloc] initWithXMLData:uacData error:&uacError] autorelease];
     }
     
+    NSError *saltError = nil;
+    ArqSalt *arqSalt = [[[ArqSalt alloc] initWithAccessKeyID:accessKey secretAccessKey:secretKey s3BucketName:s3BucketName computerUUID:computerUUID] autorelease];
+    NSData *salt = [arqSalt salt:&saltError];
+    if (salt == nil) {
+        if ([saltError code] != ERROR_NOT_FOUND) {
+            if (error != NULL) {
+                *error = saltError;
+            }
+            return NO;
+        }
+    }
+    
     ArqRepo *repo = [[[ArqRepo alloc] initWithS3Service:s3 s3BucketName:s3BucketName computerUUID:computerUUID bucketUUID:bucketUUID encryptionPassword:encryptionPassword salt:salt error:error] autorelease];
     if (repo == nil) {
         return NO;

BlobKey.h

@@ -1,19 +1,33 @@
 //
 //  BlobKey.h
-//  Arq
 //
 //  Created by Stefan Reitshamer on 6/27/11.
-//  Copyright 2011 __MyCompanyName__. All rights reserved.
+//  Copyright 2011 Haystack Software. All rights reserved.
 //
 
-
+#import "StorageType.h"
+@class BufferedInputStream;
 
 
 @interface BlobKey : NSObject <NSCopying> {
+    StorageType storageType;
+    NSString *archiveId;
+    uint64_t archiveSize;
+    NSDate *archiveUploadedDate;
     NSString *sha1;
     BOOL stretchEncryptionKey;
+    BOOL compressed;
 }
-- (id)initWithSHA1:(NSString *)theSHA1 stretchEncryptionKey:(BOOL)isStretchedKey;
+- (id)initWithSHA1:(NSString *)theSHA1 archiveId:(NSString *)theArchiveId archiveSize:(uint64_t)theArchiveSize archiveUploadedDate:(NSDate *)theArchiveUploadedDate compressed:(BOOL)isCompressed;
+- (id)initWithSHA1:(NSString *)theSHA1 storageType:(StorageType)theStorageType stretchEncryptionKey:(BOOL)isStretchedKey compressed:(BOOL)isCompressed;
+- (id)initWithStorageType:(StorageType)theStorageType archiveId:(NSString *)theArchiveId archiveSize:(uint64_t)theArchiveSize archiveUploadedDate:(NSDate *)theArchiveUploadedDate sha1:(NSString *)theSHA1 stretchEncryptionKey:(BOOL)isStretchedKey compressed:(BOOL)isCompressed;
+
+- (StorageType)storageType;
+- (NSString *)archiveId;
+- (uint64_t)archiveSize;
+- (NSDate *)archiveUploadedDate;
 - (NSString *)sha1;
 - (BOOL)stretchEncryptionKey;
+- (BOOL)compressed;
+- (BOOL)isEqualToBlobKey:(BlobKey *)other;
 @end

BlobKey.m

@@ -1,53 +1,118 @@
 //
 //  BlobKey.m
-//  Arq
 //
 //  Created by Stefan Reitshamer on 6/27/11.
-//  Copyright 2011 __MyCompanyName__. All rights reserved.
+//  Copyright 2011 Haystack Software. All rights reserved.
 //
 
 #import "BlobKey.h"
+#import "BufferedInputStream.h"
+#import "StringIO.h"
+#import "BooleanIO.h"
+#import "IntegerIO.h"
+#import "NSObject_extra.h"
 
 
 @implementation BlobKey
-- (id)initWithSHA1:(NSString *)theSHA1 stretchEncryptionKey:(BOOL)isStretchedKey {
+- (id)initWithSHA1:(NSString *)theSHA1 archiveId:(NSString *)theArchiveId archiveSize:(uint64_t)theArchiveSize archiveUploadedDate:(NSDate *)theArchiveUploadedDate compressed:(BOOL)isCompressed {
     if (self = [super init]) {
+        storageType = StorageTypeGlacier;
+        sha1 = [theSHA1 retain];
+        archiveId = [theArchiveId retain];
+        archiveSize = theArchiveSize;
+        archiveUploadedDate = [theArchiveUploadedDate retain];
+        compressed = isCompressed;
+    }
+    return self;
+}
+- (id)initWithSHA1:(NSString *)theSHA1 storageType:(StorageType)theStorageType stretchEncryptionKey:(BOOL)isStretchedKey compressed:(BOOL)isCompressed {
+    if (self = [super init]) {
+        storageType = theStorageType;
         sha1 = [theSHA1 retain];
         stretchEncryptionKey = isStretchedKey;
+        compressed = isCompressed;
+    }
+    return self;
+}
+- (id)initWithStorageType:(StorageType)theStorageType archiveId:(NSString *)theArchiveId archiveSize:(uint64_t)theArchiveSize archiveUploadedDate:(NSDate *)theArchiveUploadedDate sha1:(NSString *)theSHA1 stretchEncryptionKey:(BOOL)isStretchedKey compressed:(BOOL)isCompressed {
+    if (self = [super init]) {
+        storageType = theStorageType;
+        archiveId = [theArchiveId retain];
+        archiveSize = theArchiveSize;
+        archiveUploadedDate = [theArchiveUploadedDate retain];
+        sha1 = [theSHA1 retain];
+        stretchEncryptionKey = isStretchedKey;
+        compressed = isCompressed;
     }
     return self;
 }
 - (void)dealloc {
+    [archiveId release];
+    [archiveUploadedDate release];
     [sha1 release];
     [super dealloc];
 }
 
+- (StorageType)storageType {
+    return storageType;
+}
+- (NSString *)archiveId {
+    return archiveId;
+}
+- (uint64_t)archiveSize {
+    return archiveSize;
+}
+- (NSDate *)archiveUploadedDate {
+    return archiveUploadedDate;
+}
 - (NSString *)sha1 {
     return sha1;
 }
 - (BOOL)stretchEncryptionKey {
     return stretchEncryptionKey;
 }
+- (BOOL)compressed {
+    return compressed;
+}
+- (BOOL)isEqualToBlobKey:(BlobKey *)other {
+    if (![[other sha1] isEqualToString:sha1]) {
+        return NO;
+    }
+    if (stretchEncryptionKey != [other stretchEncryptionKey]) {
+        return NO;
+    }
+    return YES;
+}
 
 
 #pragma mark NSCopying
 - (id)copyWithZone:(NSZone *)zone {
-    return [[BlobKey alloc] initWithSHA1:sha1 stretchEncryptionKey:stretchEncryptionKey];
+    return [[BlobKey alloc] initWithStorageType:storageType archiveId:archiveId archiveSize:archiveSize archiveUploadedDate:archiveUploadedDate sha1:sha1 stretchEncryptionKey:stretchEncryptionKey compressed:compressed];
 }
 
 
 #pragma mark NSObject
 - (NSString *)description {
-    return [NSString stringWithFormat:@"<BlobKey %@,stretchedkey=%@>", sha1, (stretchEncryptionKey ? @"YES" : @"NO")];
+    if (storageType == StorageTypeS3) {
+        return [NSString stringWithFormat:@"<BlobKey sha1=%@,stretchedkey=%@,compressed=%@>", sha1, (stretchEncryptionKey ? @"YES" : @"NO"), (compressed ? @"YES" : @"NO")];
+    }
+    return [NSString stringWithFormat:@"<BlobKey archiveId=%@,archiveUploadedDate=%@,stretchedkey=%@,compressed=%@>", archiveId, archiveUploadedDate, (stretchEncryptionKey ? @"YES" : @"NO"), (compressed ? @"YES" : @"NO")];
 }
 - (BOOL)isEqual:(id)anObject {
     if (![anObject isKindOfClass:[BlobKey class]]) {
         return NO;
     }
     BlobKey *other = (BlobKey *)anObject;
-    return [[other sha1] isEqualToString:sha1] && [other stretchEncryptionKey] == stretchEncryptionKey;
+    
+    return [NSObject equalObjects:sha1 and:[other sha1]]
+    && stretchEncryptionKey == [other stretchEncryptionKey]
+    && storageType == [other storageType]
+    && [NSObject equalObjects:archiveId and:[other archiveId]]
+    && archiveSize == [other archiveSize]
+    && [NSObject equalObjects:archiveUploadedDate and:[other archiveUploadedDate]]
+    && compressed == [other compressed];
 }
 - (NSUInteger)hash {
-    return [sha1 hash] + (stretchEncryptionKey ? 1 : 0);
+    return [sha1 hash] + (stretchEncryptionKey ? 1 : 0) + (compressed ? 1 : 0);
 }
 @end

BlobKeyIO.h

@@ -0,0 +1,17 @@
+//
+//  BlobKeyIO.h
+//
+//  Created by Stefan Reitshamer on 9/14/12.
+//
+//
+
+@class BufferedInputStream;
+@class BlobKey;
+
+
+@interface BlobKeyIO : NSObject {
+    
+}
++ (void)write:(BlobKey *)theBlobKey to:(NSMutableData *)data;
++ (BOOL)read:(BlobKey **)theBlobKey from:(BufferedInputStream *)is treeVersion:(int)theTreeVersion compressed:(BOOL)isCompressed error:(NSError **)error;
+@end

BlobKeyIO.m

@@ -0,0 +1,54 @@
+//
+//  BlobKeyIO.m
+//
+//  Created by Stefan Reitshamer on 9/14/12.
+//
+//
+
+#import "BlobKeyIO.h"
+#import "BooleanIO.h"
+#import "StringIO.h"
+#import "IntegerIO.h"
+#import "StorageType.h"
+#import "BlobKey.h"
+#import "DateIO.h"
+
+
+@implementation BlobKeyIO
++ (void)write:(BlobKey *)theBlobKey to:(NSMutableData *)data {
+    [StringIO write:[theBlobKey sha1] to:data];
+    [BooleanIO write:[theBlobKey stretchEncryptionKey] to:data];
+    [IntegerIO writeUInt32:(uint32_t)[theBlobKey storageType] to:data];
+    [StringIO write:[theBlobKey archiveId] to:data];
+    [IntegerIO writeUInt64:[theBlobKey archiveSize] to:data];
+    [DateIO write:[theBlobKey archiveUploadedDate] to:data];
+}
++ (BOOL)read:(BlobKey **)theBlobKey from:(BufferedInputStream *)is treeVersion:(int)theTreeVersion compressed:(BOOL)isCompressed error:(NSError **)error {
+    NSString *dataSHA1;
+    BOOL stretchEncryptionKey = NO;
+    StorageType storageType = StorageTypeS3;
+    NSString *archiveId = nil;
+    uint64_t archiveSize = 0;
+    NSDate *archiveUploadedDate = nil;
+    
+    if (![StringIO read:&dataSHA1 from:is error:error]) {
+        [self release];
+        return NO;
+    }
+    if (theTreeVersion >= 14 && ![BooleanIO read:&stretchEncryptionKey from:is error:error]) {
+        [self release];
+        return NO;
+    }
+    if (theTreeVersion >= 17) {
+        if (![IntegerIO readUInt32:&storageType from:is error:error]
+            || ![StringIO read:&archiveId from:is error:error]
+            || ![IntegerIO readUInt64:&archiveSize from:is error:error]
+            || ![DateIO read:&archiveUploadedDate from:is error:error]) {
+            [self release];
+            return NO;
+        }
+    }
+    *theBlobKey = [[[BlobKey alloc] initWithStorageType:storageType archiveId:archiveId archiveSize:archiveSize archiveUploadedDate:archiveUploadedDate sha1:dataSHA1 stretchEncryptionKey:stretchEncryptionKey compressed:isCompressed] autorelease];
+    return YES;
+}
+@end

Commit.h

@@ -11,19 +11,20 @@
 #import "BufferedInputStream.h"
 @class BlobKey;
 
-#define CURRENT_COMMIT_VERSION 7
+#define CURRENT_COMMIT_VERSION 9
 
 @interface Commit : NSObject {
     int commitVersion;
 	NSString *_author;
 	NSString *_comment;
-	NSMutableSet *_parentCommitBlobKeys;
+    BlobKey *_parentCommitBlobKey;
 	BlobKey *_treeBlobKey;
 	NSString *_location;
     NSString *_computer;
-	BlobKey *_mergeCommonAncestorCommitBlobKey;
 	NSDate *_creationDate;
     NSArray *_commitFailedFiles;
+    BOOL _hasMissingNodes;
+    BOOL _isComplete;
     NSData *_bucketXMLData;
 }
 + (NSString *)errorDomain;
@@ -31,27 +32,29 @@
 
 - (id)             initWithAuthor:(NSString *)theAuthor
                           comment:(NSString *)theComment
-             parentCommitBlobKeys:(NSSet *)theParentCommitBlobKeys 
+              parentCommitBlobKey:(BlobKey *)theParentCommitBlobKey
                       treeBlobKey:(BlobKey *)theTreeBlobKey
                          location:(NSString *)theLocation
- mergeCommonAncestorCommitBlobKey:(BlobKey *)theMergeCommonAncestorCommitBlobKey
+                     creationDate:(NSDate *)theCreationDate
                 commitFailedFiles:(NSArray *)theCommitFailedFiles
+                  hasMissingNodes:(BOOL)theHasMissingNodes
+                       isComplete:(BOOL)theIsComplete
                     bucketXMLData:(NSData *)theBucketXMLData;
 
 - (id)initWithBufferedInputStream:(BufferedInputStream *)is error:(NSError **)error;
 
+@property(readonly) int commitVersion;
 @property(readonly,copy) NSString *author;
 @property(readonly,copy) NSString *comment;
 @property(readonly,copy) BlobKey *treeBlobKey;
-@property(readonly,retain) NSSet *parentCommitBlobKeys;
+@property(readonly,retain) BlobKey *parentCommitBlobKey;
 @property(readonly,copy) NSString *location;
 @property(readonly,copy) NSString *computer;
-@property(readonly,copy) BlobKey *mergeCommonAncestorCommitBlobKey;
 @property(readonly,retain) NSDate *creationDate;
 @property(readonly,retain) NSArray *commitFailedFiles;
+@property(readonly) BOOL hasMissingNodes;
+@property(readonly) BOOL isComplete;
 @property(readonly, retain) NSData *bucketXMLData;
 
-- (NSNumber *)isMergeCommit;
+- (NSData *)toData;
-- (Blob *)toBlob;
-
 @end

Commit.m

@@ -13,8 +13,7 @@
 #import "Blob.h"
 #import "DataInputStream.h"
 #import "RegexKitLite.h"
-#import "SetNSError.h"
+#import "StorageType.h"
-#import "NSErrorCodes.h"
 #import "CommitFailedFile.h"
 #import "BufferedInputStream.h"
 #import "BooleanIO.h"
@@ -33,15 +32,17 @@
 }
 
 
-@synthesize author = _author, 
+@synthesize commitVersion,
+author = _author,
 comment = _comment,
 treeBlobKey = _treeBlobKey,
-parentCommitBlobKeys = _parentCommitBlobKeys, 
+parentCommitBlobKey = _parentCommitBlobKey,
 location = _location,
 computer = _computer,
-mergeCommonAncestorCommitBlobKey = _mergeCommonAncestorCommitBlobKey, 
 creationDate = _creationDate,
 commitFailedFiles = _commitFailedFiles,
+hasMissingNodes = _hasMissingNodes,
+isComplete = _isComplete,
 bucketXMLData = _bucketXMLData;
 
 
@@ -49,17 +50,14 @@ bucketXMLData = _bucketXMLData;
     if (self = [super init]) {
         _author = [[theCommit author] copy];
         _comment = [[theCommit comment] copy];
-        if (theParentBlobKey != nil) {
+        _parentCommitBlobKey = [theParentBlobKey retain];
-            _parentCommitBlobKeys = [[NSMutableSet alloc] initWithObjects:theParentBlobKey, nil];
-        } else {
-            _parentCommitBlobKeys = [[NSMutableSet alloc] init];
-        }
         _treeBlobKey = [[theCommit treeBlobKey] copy];
         _location = [[theCommit location] copy];
         _computer = [[theCommit computer] copy];
-        _mergeCommonAncestorCommitBlobKey = nil;
         _creationDate = [[theCommit creationDate] copy];
         _commitFailedFiles = [[theCommit commitFailedFiles] copy];
+        _hasMissingNodes = [theCommit hasMissingNodes];
+        _isComplete = [theCommit isComplete];
         _bucketXMLData = [[theCommit bucketXMLData] copy];
     }
     return self;
@@ -67,16 +65,19 @@ bucketXMLData = _bucketXMLData;
 
 - (id)             initWithAuthor:(NSString *)theAuthor
                           comment:(NSString *)theComment
-             parentCommitBlobKeys:(NSSet *)theParentCommitBlobKeys 
+              parentCommitBlobKey:(BlobKey *)theParentCommitBlobKey
                       treeBlobKey:(BlobKey *)theTreeBlobKey
                          location:(NSString *)theLocation
- mergeCommonAncestorCommitBlobKey:(BlobKey *)theMergeCommonAncestorCommitBlobKey 
+                     creationDate:(NSDate *)theCreationDate
                 commitFailedFiles:(NSArray *)theCommitFailedFiles
+                  hasMissingNodes:(BOOL)theHasMissingNodes
+                       isComplete:(BOOL)theIsComplete
                     bucketXMLData:(NSData *)theBucketXMLData {
     if (self = [super init]) {
+        commitVersion = CURRENT_COMMIT_VERSION;
         _author = [theAuthor copy];
         _comment = [theComment copy];
-        _parentCommitBlobKeys = [[NSMutableSet alloc] initWithSet:theParentCommitBlobKeys];
+        _parentCommitBlobKey = [theParentCommitBlobKey retain];
         _treeBlobKey = [theTreeBlobKey retain];
         _location = [theLocation copy];
         NSRange computerRange = [_location rangeOfRegex:@"^file://([^/]+)/" capture:1];
@@ -86,16 +87,16 @@ bucketXMLData = _bucketXMLData;
             _computer = @"";
         }
         [_computer retain];
-        _mergeCommonAncestorCommitBlobKey = [theMergeCommonAncestorCommitBlobKey retain];
+        _creationDate = [theCreationDate retain];
-        _creationDate = [[NSDate alloc] init];
         _commitFailedFiles = [theCommitFailedFiles copy];
+        _hasMissingNodes = theHasMissingNodes;
+        _isComplete = theIsComplete;
         _bucketXMLData = [theBucketXMLData copy];
     }
     return self;
 }
 - (id)initWithBufferedInputStream:(BufferedInputStream *)is error:(NSError **)error {
 	if (self = [super init]) {
-        _parentCommitBlobKeys = [[NSMutableSet alloc] init];
         if (![self readHeader:is error:error]) {
             goto init_error;
         }
@@ -124,9 +125,11 @@ bucketXMLData = _bucketXMLData;
                     goto init_error;
                 }
             }
-            BlobKey *parentBlobKey = [[BlobKey alloc] initWithSHA1:key stretchEncryptionKey:cryptoKeyStretched];
+            if (_parentCommitBlobKey != nil) {
-            [_parentCommitBlobKeys addObject:parentBlobKey];
+                HSLogError(@"IGNORING EXTRA PARENT COMMIT BLOB KEY!");
-            [parentBlobKey release];
+            } else {
+                _parentCommitBlobKey = [[BlobKey alloc] initWithSHA1:key storageType:StorageTypeS3 stretchEncryptionKey:cryptoKeyStretched compressed:NO];
+            }
         }
         
         NSString *treeSHA1 = nil;
@@ -139,7 +142,13 @@ bucketXMLData = _bucketXMLData;
                 goto init_error;
             }
         }
-        _treeBlobKey = [[BlobKey alloc] initWithSHA1:treeSHA1 stretchEncryptionKey:treeStretchedKey];
+        BOOL treeIsCompressed = NO;
+        if (commitVersion >= 8) {
+            if (![BooleanIO read:&treeIsCompressed from:is error:error]) {
+                goto init_error;
+            }
+        }
+        _treeBlobKey = [[BlobKey alloc] initWithSHA1:treeSHA1 storageType:StorageTypeS3 stretchEncryptionKey:treeStretchedKey compressed:treeIsCompressed];
         
         if (![StringIO read:&_location from:is error:error]) {
             goto init_error;
@@ -154,6 +163,8 @@ bucketXMLData = _bucketXMLData;
         }
         [_computer retain];
         
+        // Removed mergeCommonAncestorCommitBlobKey in Commit version 8. It was never used.
+        if (commitVersion < 8) {
             NSString *mergeCommonAncestorCommitSHA1 = nil;
             BOOL mergeCommonAncestorCommitStretchedKey = NO;
             if (![StringIO read:&mergeCommonAncestorCommitSHA1 from:is error:error]) {
@@ -164,8 +175,9 @@ bucketXMLData = _bucketXMLData;
                     goto init_error;
                 }
             }
-        if (mergeCommonAncestorCommitSHA1 != nil) {
+            //            if (mergeCommonAncestorCommitSHA1 != nil) {
-            _mergeCommonAncestorCommitBlobKey = [[BlobKey alloc] initWithSHA1:mergeCommonAncestorCommitSHA1 stretchEncryptionKey:mergeCommonAncestorCommitStretchedKey];
+            //                _mergeCommonAncestorCommitBlobKey = [[BlobKey alloc] initWithSHA1:mergeCommonAncestorCommitSHA1 stretchEncryptionKey:mergeCommonAncestorCommitStretchedKey];
+            //            }
         }
         
         if (![DateIO read:&_creationDate from:is error:error]) {
@@ -189,6 +201,18 @@ bucketXMLData = _bucketXMLData;
             _commitFailedFiles = [commitFailedFiles retain];
         }
         
+        if (commitVersion >= 8) {
+            if (![BooleanIO read:&_hasMissingNodes from:is error:error]) {
+                goto init_error;
+            }
+        }
+        if (commitVersion >= 9) {
+            if (![BooleanIO read:&_isComplete from:is error:error]) {
+                goto init_error;
+            }
+        } else {
+            _isComplete = YES;
+        }
         if (commitVersion >= 5) {
             if (![DataIO read:&_bucketXMLData from:is error:error]) {
                 goto init_error;
@@ -208,53 +232,48 @@ init_done:
 - (void)dealloc {
     [_author release];
     [_comment release];
-    [_parentCommitBlobKeys release];
+    [_parentCommitBlobKey release];
     [_treeBlobKey release];
     [_location release];
     [_computer release];
-    [_mergeCommonAncestorCommitBlobKey release];
     [_creationDate release];
     [_commitFailedFiles release];
     [_bucketXMLData release];
     [super dealloc];
 }
-- (NSNumber *)isMergeCommit {
+- (NSData *)toData {
-    return [NSNumber numberWithBool:([_parentCommitBlobKeys count] > 1)];
+    NSMutableData *data = [[[NSMutableData alloc] init] autorelease];
-}
-- (Blob *)toBlob {
-    Blob *ret = nil;
-    NSMutableData *data = [[NSMutableData alloc] init];
     char header[HEADER_LENGTH + 1];
     sprintf(header, "CommitV%03d", CURRENT_COMMIT_VERSION);
     [data appendBytes:header length:HEADER_LENGTH];
     [StringIO write:_author to:data];
     [StringIO write:_comment to:data];
-    uint64_t parentCommitBlobKeysCount = (uint64_t)[_parentCommitBlobKeys count];
+    if (_parentCommitBlobKey == nil) {
-    [IntegerIO writeUInt64:parentCommitBlobKeysCount to:data];
+        [IntegerIO writeUInt64:0 to:data];
-    for (BlobKey *parentCommitBlobKey in _parentCommitBlobKeys) {
+    } else {
-        [StringIO write:[parentCommitBlobKey sha1] to:data];
+        [IntegerIO writeUInt64:1 to:data];
-        [BooleanIO write:[parentCommitBlobKey stretchEncryptionKey] to:data];
+        [StringIO write:[_parentCommitBlobKey sha1] to:data];
+        [BooleanIO write:[_parentCommitBlobKey stretchEncryptionKey] to:data];
     }
     [StringIO write:[_treeBlobKey sha1] to:data];
     [BooleanIO write:[_treeBlobKey stretchEncryptionKey] to:data];
+    [BooleanIO write:[_treeBlobKey compressed] to:data];
     [StringIO write:_location to:data];
-    [StringIO write:[_mergeCommonAncestorCommitBlobKey sha1] to:data];
-    [BooleanIO write:[_mergeCommonAncestorCommitBlobKey stretchEncryptionKey] to:data];
     [DateIO write:_creationDate to:data];
     uint64_t commitFailedFilesCount = (uint64_t)[_commitFailedFiles count];
     [IntegerIO writeUInt64:commitFailedFilesCount to:data];
     for (CommitFailedFile *cff in _commitFailedFiles) {
         [cff writeTo:data];
     }
+    [BooleanIO write:_hasMissingNodes to:data];
+    [BooleanIO write:_isComplete to:data];
     [DataIO write:_bucketXMLData to:data];
-    ret = [[[Blob alloc] initWithData:data mimeType:@"binary/octet-stream" downloadName:@"commit" dataDescription:@"commit"] autorelease];
+    return data;
-    [data release];
-    return ret;
 }
 
 #pragma mark NSObject
 - (NSString *)description {
-    return [NSString stringWithFormat:@"<Commit: created=%@ tree=%@ parents=%@>", _creationDate, _treeBlobKey, _parentCommitBlobKeys];
+    return [NSString stringWithFormat:@"<Commit: created=%@ tree=%@ parent=%@ complete=%@ missingnodes=%@>", _creationDate, _treeBlobKey, _parentCommitBlobKey, (_isComplete ? @"YES" : @"NO"), (_hasMissingNodes ? @"YES" : @"NO")];
 }
 @end
 
@@ -266,16 +285,14 @@ init_done:
         goto readHeader_error;
     }
     NSString *header = [[[NSString alloc] initWithBytes:buf length:HEADER_LENGTH encoding:NSASCIIStringEncoding] autorelease];
-    NSRange versionRange = [header rangeOfRegex:@"^CommitV(\\d{3})$" capture:1];
+    if (![header hasPrefix:@"CommitV"] || [header length] < 8) {
-    commitVersion = 0;
+        HSLogDebug(@"current Commit version: %d", CURRENT_COMMIT_VERSION);
-    if (versionRange.location != NSNotFound) {
+        SETNSERROR([Commit errorDomain], ERROR_INVALID_COMMIT_HEADER, @"invalid header %@", header);
-        NSNumberFormatter *nf = [[NSNumberFormatter alloc] init];
+        goto readHeader_error;
-        NSNumber *number = [nf numberFromString:[header substringWithRange:versionRange]];
-        commitVersion = [number intValue];
-        [nf release];
     }
+    commitVersion = [[header substringFromIndex:7] intValue];
     if (commitVersion > CURRENT_COMMIT_VERSION || commitVersion < 2) {
-        SETNSERROR([Commit errorDomain], ERROR_INVALID_OBJECT_VERSION, @"invalid Commit header");
+        SETNSERROR([Commit errorDomain], ERROR_INVALID_OBJECT_VERSION, @"invalid Commit version %d", commitVersion);
         goto readHeader_error;
     }
     ret = YES;

Node.h

@@ -10,19 +10,17 @@
 
 @protocol InputStream;
 @class BlobKey;
+@class BufferedInputStream;
+
 
 @interface Node : NSObject {
     int treeVersion;
 	BOOL isTree;
+    BOOL treeContainsMissingItems;
     unsigned long long uncompressedDataSize;
-    BOOL dataAreCompressed;
     NSMutableArray *dataBlobKeys;
-    BlobKey *thumbnailBlobKey;
-    BlobKey *previewBlobKey;
-    BOOL xattrsAreCompressed;
     BlobKey *xattrsBlobKey;
     unsigned long long xattrsSize;
-    BOOL aclIsCompressed;
     BlobKey *aclBlobKey;
     int uid;
     int gid;
@@ -46,22 +44,20 @@
     int64_t st_blocks;
     uint32_t st_blksize;
 }
-- (id)initWithInputStream:(id <InputStream>)is treeVersion:(int)theTreeVersion error:(NSError **)error;
+- (id)initWithInputStream:(BufferedInputStream *)is treeVersion:(int)theTreeVersion error:(NSError **)error;
 - (void)writeToData:(NSMutableData *)data;
-- (BOOL)dataMatchesStatData:(struct stat *)st;
+- (BOOL)dataMatchesStat:(struct stat *)st;
+- (BOOL)ctimeMatchesStat:(struct stat *)st;
 
 @property(readonly) BOOL isTree;
+@property(readonly) BOOL treeContainsMissingItems;
 @property(readonly,copy) BlobKey *treeBlobKey;
-@property(readonly) BOOL dataAreCompressed;
 @property(readonly,copy) NSArray *dataBlobKeys;
 
 @property(readonly) unsigned long long uncompressedDataSize;
-@property(readonly,copy) BlobKey *thumbnailBlobKey;
+@property(readonly) unsigned long long aggregateGlacierArchiveSize;
-@property(readonly,copy) BlobKey *previewBlobKey;
-@property(readonly) BOOL xattrsAreCompressed;
 @property(readonly,copy) BlobKey *xattrsBlobKey;
 @property(readonly) unsigned long long xattrsSize;
-@property(readonly) BOOL aclIsCompressed;
 @property(readonly,copy) BlobKey *aclBlobKey;
 @property(readonly) int uid;
 @property(readonly) int gid;
@@ -85,5 +81,4 @@
 @property(readonly) int st_ino;
 @property(readonly) int64_t st_blocks;
 @property(readonly) uint32_t st_blksize;
-- (uint64_t)sizeOnDisk;
 @end

Node.m

@@ -14,12 +14,16 @@
 #import "BufferedInputStream.h"
 #import "BlobKey.h"
 #import "NSObject_extra.h"
+#import "Tree.h"
+#import "BlobKeyIO.h"
+
 
 @implementation Node
-@synthesize isTree, uncompressedDataSize, thumbnailBlobKey, previewBlobKey, xattrsBlobKey, xattrsSize, aclBlobKey, uid, gid, mode, mtime_sec, mtime_nsec, flags, finderFlags, extendedFinderFlags, finderFileType, finderFileCreator, isFileExtensionHidden, st_dev, treeVersion, st_rdev;
+@synthesize isTree, treeContainsMissingItems, uncompressedDataSize, xattrsBlobKey, xattrsSize, aclBlobKey, uid, gid, mode, mtime_sec, mtime_nsec, flags, finderFlags, extendedFinderFlags, finderFileType, finderFileCreator, isFileExtensionHidden, st_dev, treeVersion, st_rdev;
 @synthesize ctime_sec, ctime_nsec, createTime_sec, createTime_nsec, st_nlink, st_ino, st_blocks, st_blksize;
 @dynamic treeBlobKey, dataBlobKeys;
-@synthesize dataAreCompressed, xattrsAreCompressed, aclIsCompressed;
+@dynamic aggregateGlacierArchiveSize;
+
 
 - (id)initWithInputStream:(BufferedInputStream *)is treeVersion:(int)theTreeVersion error:(NSError **)error {
     if (self = [super init]) {
@@ -31,6 +35,16 @@
             return nil;
         }
         
+        if (theTreeVersion >= 18) {
+            if (![BooleanIO read:&treeContainsMissingItems from:is error:error]) {
+                [self release];
+                return nil;
+            }
+        }
+        
+        BOOL dataAreCompressed = NO;
+        BOOL xattrsAreCompressed = NO;
+        BOOL aclIsCompressed = NO;
         if (treeVersion >= 12) {
             if (![BooleanIO read:&dataAreCompressed from:is error:error]
                 || ![BooleanIO read:&xattrsAreCompressed from:is error:error]
@@ -46,38 +60,32 @@
             return nil;
         }
         for (int i = 0; i < dataBlobKeysCount; i++) {
-            NSString *dataSHA1;
+            BlobKey *dataBlobKey = nil;
-            BOOL stretchEncryptionKey = NO;
+            if (![BlobKeyIO read:&dataBlobKey from:is treeVersion:treeVersion compressed:dataAreCompressed error:error]) {
-            if (![StringIO read:&dataSHA1 from:is error:error]) {
                 [self release];
                 return nil;
             }
-            if (treeVersion >= 14 && ![BooleanIO read:&stretchEncryptionKey from:is error:error]) {
+            [dataBlobKeys addObject:dataBlobKey];
+        }
+        if (![IntegerIO readUInt64:&uncompressedDataSize from:is error:error]) {
             [self release];
             return nil;
         }
-            BlobKey *bk = [[BlobKey alloc] initWithSHA1:dataSHA1 stretchEncryptionKey:stretchEncryptionKey];
+        
-            [dataBlobKeys addObject:bk];
+        // As of Tree version 18 thumbnailBlobKey and previewBlobKey have been removed. They were never used.
-            [bk release];
+        if (theTreeVersion < 18) {
+            BlobKey *theThumbnailBlobKey = nil;
+            BlobKey *thePreviewBlobKey = nil;
+            if (![BlobKeyIO read:&theThumbnailBlobKey from:is treeVersion:treeVersion compressed:NO error:error]
+                || ![BlobKeyIO read:&thePreviewBlobKey from:is treeVersion:treeVersion compressed:NO error:error]) {
+                [self release];
+                return nil;
             }
-        NSString *thumbnailSHA1 = nil;
+        }
-        BOOL thumbnailStretchedKey = NO;
+        
-        NSString *previewSHA1 = nil;
+        BOOL ret = [BlobKeyIO read:&xattrsBlobKey from:is treeVersion:treeVersion compressed:xattrsAreCompressed error:error]
-        BOOL previewStretchedKey = NO;
-        NSString *xattrsSHA1 = nil;
-        BOOL xattrsStretchedKey = NO;
-        NSString *aclSHA1 = nil;
-        BOOL aclStretchedKey = NO;
-        BOOL ret = [IntegerIO readUInt64:&uncompressedDataSize from:is error:error]
-        && [StringIO read:&thumbnailSHA1 from:is error:error]
-        && (treeVersion < 14 || [BooleanIO read:&thumbnailStretchedKey from:is error:error])
-        && [StringIO read:&previewSHA1 from:is error:error]
-        && (treeVersion < 14 || [BooleanIO read:&previewStretchedKey from:is error:error])
-        && [StringIO read:&xattrsSHA1 from:is error:error]
-        && (treeVersion < 14 || [BooleanIO read:&xattrsStretchedKey from:is error:error])
         && [IntegerIO readUInt64:&xattrsSize from:is error:error]
-        && [StringIO read:&aclSHA1 from:is error:error]
+        && [BlobKeyIO read:&aclBlobKey from:is treeVersion:treeVersion compressed:aclIsCompressed error:error]
-        && (treeVersion < 14 || [BooleanIO read:&aclStretchedKey from:is error:error])
         && [IntegerIO readInt32:&uid from:is error:error]
         && [IntegerIO readInt32:&gid from:is error:error]
         && [IntegerIO readInt32:&mode from:is error:error]
@@ -99,31 +107,30 @@
         && [IntegerIO readInt64:&createTime_nsec from:is error:error]
         && [IntegerIO readInt64:&st_blocks from:is error:error]
         && [IntegerIO readUInt32:&st_blksize from:is error:error];
+        [xattrsBlobKey retain];
+        [aclBlobKey retain];
         [finderFileType retain];
         [finderFileCreator retain];
         if (!ret) {
             [self release];
             return nil;
         }
-        if (thumbnailSHA1 != nil) {
+        
-            thumbnailBlobKey = [[BlobKey alloc] initWithSHA1:thumbnailSHA1 stretchEncryptionKey:thumbnailStretchedKey];
+        // If any BlobKey has a nil sha1, drop it.
+        
+        if ([xattrsBlobKey sha1] == nil) {
+            [xattrsBlobKey release];
+            xattrsBlobKey = nil;
         }
-        if (previewSHA1 != nil) {
+        if ([aclBlobKey sha1] == nil) {
-            previewBlobKey = [[BlobKey alloc] initWithSHA1:previewSHA1 stretchEncryptionKey:previewStretchedKey];
+            [aclBlobKey release];
-        }
+            aclBlobKey = nil;
-        if (xattrsSHA1 != nil) {
-            xattrsBlobKey = [[BlobKey alloc] initWithSHA1:xattrsSHA1 stretchEncryptionKey:xattrsStretchedKey];
-        }
-        if (aclSHA1 != nil) {
-            aclBlobKey = [[BlobKey alloc] initWithSHA1:aclSHA1 stretchEncryptionKey:aclStretchedKey];
         }
     }
     return self;
 }
 - (void)dealloc {
 	[dataBlobKeys release];
-    [thumbnailBlobKey release];
-    [previewBlobKey release];
 	[xattrsBlobKey release];
 	[aclBlobKey release];
 	[finderFileType release];
@@ -137,29 +144,38 @@
 - (NSArray *)dataBlobKeys {
     return dataBlobKeys;
 }
-- (BOOL)dataMatchesStatData:(struct stat *)st {
+- (uint64_t)aggregateGlacierArchiveSize {
-    return (st->st_mtimespec.tv_sec == mtime_sec && st->st_mtimespec.tv_nsec == mtime_nsec && st->st_size == uncompressedDataSize);
+    uint64_t ret = 0;
+    ret += [aclBlobKey archiveSize];
+    ret += [xattrsBlobKey archiveSize];
+    for (BlobKey *dataBlobKey in dataBlobKeys) {
+        ret += [dataBlobKey archiveSize];
+    }
+    return ret;
+}
+
+- (BOOL)dataMatchesStat:(struct stat *)st {
+    return st->st_mtimespec.tv_sec == mtime_sec && st->st_mtimespec.tv_nsec == mtime_nsec && st->st_size == uncompressedDataSize;
+}
+- (BOOL)ctimeMatchesStat:(struct stat *)st {
+    return st->st_ctimespec.tv_sec == ctime_sec && st->st_ctimespec.tv_nsec == ctime_nsec;
 }
 - (void)writeToData:(NSMutableData *)data {
     [BooleanIO write:isTree to:data];
+    [BooleanIO write:treeContainsMissingItems to:data];
+    BOOL dataAreCompressed = [dataBlobKeys count] == 0 ? NO : [[dataBlobKeys objectAtIndex:0] compressed];
     [BooleanIO write:dataAreCompressed to:data];
-    [BooleanIO write:xattrsAreCompressed to:data];
+    [BooleanIO write:[xattrsBlobKey compressed] to:data];
-    [BooleanIO write:aclIsCompressed to:data];
+    [BooleanIO write:[aclBlobKey compressed] to:data];
     [IntegerIO writeInt32:(int32_t)[dataBlobKeys count] to:data];
     for (BlobKey *dataBlobKey in dataBlobKeys) {
-        [StringIO write:[dataBlobKey sha1] to:data];
+        NSAssert([dataBlobKey compressed] == dataAreCompressed, @"all dataBlobKeys must have same compressed flag value");
-        [BooleanIO write:[dataBlobKey stretchEncryptionKey] to:data];
+        [BlobKeyIO write:dataBlobKey to:data];
     }
     [IntegerIO writeUInt64:uncompressedDataSize to:data];
-    [StringIO write:[thumbnailBlobKey sha1] to:data];
+    [BlobKeyIO write:xattrsBlobKey to:data];
-    [BooleanIO write:[thumbnailBlobKey stretchEncryptionKey] to:data];
-    [StringIO write:[previewBlobKey sha1] to:data];
-    [BooleanIO write:[previewBlobKey stretchEncryptionKey] to:data];
-    [StringIO write:[xattrsBlobKey sha1] to:data];
-    [BooleanIO write:[xattrsBlobKey stretchEncryptionKey] to:data];
     [IntegerIO writeUInt64:xattrsSize to:data];
-    [StringIO write:[aclBlobKey sha1] to:data];
+    [BlobKeyIO write:aclBlobKey to:data];
-    [BooleanIO write:[aclBlobKey stretchEncryptionKey] to:data];
     [IntegerIO writeInt32:uid to:data];
     [IntegerIO writeInt32:gid to:data];
     [IntegerIO writeInt32:mode to:data];
@@ -182,9 +198,7 @@
     [IntegerIO writeInt64:st_blocks to:data];
     [IntegerIO writeUInt32:st_blksize to:data];
 }
-- (uint64_t)sizeOnDisk {
+
-    return (uint64_t)st_blocks * (uint64_t)512;
-}
 
 #pragma mark NSObject
 - (BOOL)isEqual:(id)object {
@@ -195,14 +209,9 @@
     return treeVersion == [other treeVersion]
     && isTree == [other isTree]
     && uncompressedDataSize == [other uncompressedDataSize]
-    && dataAreCompressed == [other dataAreCompressed]
     && [dataBlobKeys isEqualToArray:[other dataBlobKeys]]
-    && [NSObject equalObjects:thumbnailBlobKey and:[other thumbnailBlobKey]]
-    && [NSObject equalObjects:previewBlobKey and:[other previewBlobKey]]
-    && xattrsAreCompressed == [other xattrsAreCompressed]
     && [NSObject equalObjects:xattrsBlobKey and:[other xattrsBlobKey]]
     && xattrsSize == [other xattrsSize]
-    && aclIsCompressed == [other aclIsCompressed]
     && [NSObject equalObjects:aclBlobKey and:[other aclBlobKey]]
     && uid == [other uid]
     && gid == [other gid]
@@ -226,6 +235,6 @@
     && st_blksize == [other st_blksize];
 }
 - (NSUInteger)hash {
-    return (NSUInteger)treeVersion + (dataAreCompressed ? 1 : 0) + [dataBlobKeys hash];
+    return (NSUInteger)treeVersion + [dataBlobKeys hash];
 }
 @end

ReflogEntry.m

@@ -29,10 +29,14 @@
             return nil;
         }
         oldHeadBlobKey = [[BlobKey alloc] initWithSHA1:[[dictNode stringNodeForKey:@"oldHeadSHA1"] stringValue]
-                                  stretchEncryptionKey:[[dictNode booleanNodeForKey:@"oldHeadStretchKey"] booleanValue]];
+                                           storageType:StorageTypeS3
+                                  stretchEncryptionKey:[[dictNode booleanNodeForKey:@"oldHeadStretchKey"] booleanValue]
+                                            compressed:NO];
         
         newHeadBlobKey = [[BlobKey alloc] initWithSHA1:[[dictNode stringNodeForKey:@"newHeadSHA1"] stringValue]
-                                  stretchEncryptionKey:[[dictNode booleanNodeForKey:@"newHeadStretchKey"] booleanValue]];
+                                           storageType:StorageTypeS3
+                                  stretchEncryptionKey:[[dictNode booleanNodeForKey:@"newHeadStretchKey"] booleanValue]
+                                            compressed:NO];
     }
     return self;
 }

Restorer.m

@@ -69,9 +69,9 @@
 - (BOOL)applyTree:(Tree *)tree toPath:(NSString *)restorePath error:(NSError **)error;
 - (BOOL)applyNode:(Node *)node toPath:(NSString *)restorePath error:(NSError **)error;
 - (BOOL)createFile:(Node *)node atPath:(NSString *)path error:(NSError **)error;
-- (BOOL)createFileAtPath:(NSString *)path fromBlobKeys:(NSArray *)dataBlobKeys uncompress:(BOOL)uncompress error:(NSError **)error;
+- (BOOL)createFileAtPath:(NSString *)path fromBlobKeys:(NSArray *)dataBlobKeys error:(NSError **)error;
-- (BOOL)appendBlobForBlobKey:(BlobKey *)theBlobKey uncompress:(BOOL)uncompress to:(FileOutputStream *)fos error:(NSError **)error;
+- (BOOL)appendBlobForBlobKey:(BlobKey *)theBlobKey to:(FileOutputStream *)fos error:(NSError **)error;
-- (BOOL)doAppendBlobForBlobKey:(BlobKey *)theBlobKey uncompress:(BOOL)uncompress to:(BufferedOutputStream *)bos error:(NSError **)error;
+- (BOOL)doAppendBlobForBlobKey:(BlobKey *)theBlobKey to:(BufferedOutputStream *)bos error:(NSError **)error;
 - (BOOL)createSymLink:(Node *)node path:(NSString *)symLinkFile target:(NSString *)target error:(NSError **)error;
 - (BOOL)applyACLBlobKey:(BlobKey *)aclBlobKey uncompress:(BOOL)uncompress toPath:(NSString *)path error:(NSError **)error;
 - (BOOL)applyXAttrsBlobKey:(BlobKey *)xattrsBlobKey uncompress:(BOOL)uncompress toFile:(NSString *)path error:(NSError **)error;
@@ -117,13 +117,13 @@
     Commit *commit = nil;
     NSError *myError = nil;
     if (commitSHA1 != nil) {
-        commitBlobKey = [[[BlobKey alloc] initWithSHA1:commitSHA1 stretchEncryptionKey:YES] autorelease];
+        commitBlobKey = [[[BlobKey alloc] initWithSHA1:commitSHA1 storageType:StorageTypeS3 stretchEncryptionKey:YES compressed:NO] autorelease];
         commit = [repo commitForBlobKey:commitBlobKey error:&myError];
         if (commit == nil) {
             HSLogError(@"error attempting to read commit for %@", commitBlobKey);
             
             // Try without stretched encryption key.
-            commitBlobKey = [[[BlobKey alloc] initWithSHA1:commitSHA1 stretchEncryptionKey:NO] autorelease];
+            commitBlobKey = [[[BlobKey alloc] initWithSHA1:commitSHA1 storageType:StorageTypeS3 stretchEncryptionKey:NO compressed:NO] autorelease];
             commit = [repo commitForBlobKey:commitBlobKey error:&myError];
             if (commit == nil) {
                 HSLogError(@"error attempting to read commit for %@", commitBlobKey);
@@ -452,10 +452,10 @@
     if (!fa) {
         return NO;
     }
-    if (![self applyXAttrsBlobKey:[node xattrsBlobKey] uncompress:[node xattrsAreCompressed] toFile:path error:error]) {
+    if (![self applyXAttrsBlobKey:[node xattrsBlobKey] uncompress:[[node xattrsBlobKey] compressed] toFile:path error:error]) {
         return NO;
     }
-    if (![self applyACLBlobKey:[node aclBlobKey] uncompress:[node aclIsCompressed] toPath:path error:error]) {
+    if (![self applyACLBlobKey:[node aclBlobKey] uncompress:[[node aclBlobKey] compressed] toPath:path error:error]) {
         return NO;
     }
     if (!S_ISFIFO([node mode])) {
@@ -503,7 +503,7 @@
                 HSLogError(@"error getting data for %@", dataBlobKey);
                 return NO;
             }
-            if ([node dataAreCompressed]) {
+            if ([dataBlobKey compressed]) {
                 blobData = [blobData gzipInflate];
             }
             [data appendData:blobData];
@@ -514,7 +514,7 @@
             return NO;
         }
     } else if ([node uncompressedDataSize] > 0) {
-        if (![self createFileAtPath:path fromBlobKeys:[node dataBlobKeys] uncompress:[node dataAreCompressed] error:error]) {
+        if (![self createFileAtPath:path fromBlobKeys:[node dataBlobKeys] error:error]) {
             NSError *myError = nil;
             if ([[NSFileManager defaultManager] fileExistsAtPath:path] && ![[NSFileManager defaultManager] removeItemAtPath:path error:&myError]) {
                 HSLogError(@"error deleting incorrectly-restored file %@: %@", path, myError);
@@ -535,12 +535,12 @@
     HSLogDetail(@"restored %@", path);
     return YES;
 }
-- (BOOL)createFileAtPath:(NSString *)path fromBlobKeys:(NSArray *)dataBlobKeys uncompress:(BOOL)uncompress error:(NSError **)error {
+- (BOOL)createFileAtPath:(NSString *)path fromBlobKeys:(NSArray *)dataBlobKeys error:(NSError **)error {
     FileOutputStream *fos = [[FileOutputStream alloc] initWithPath:path append:NO];
     BOOL ret = YES;
     writtenToCurrentFile = 0;
     for (BlobKey *dataBlobKey in dataBlobKeys) {
-        if (![self appendBlobForBlobKey:dataBlobKey uncompress:uncompress to:fos error:error]) {
+        if (![self appendBlobForBlobKey:dataBlobKey to:fos error:error]) {
             ret = NO;
             break;
         }
@@ -548,7 +548,7 @@
     [fos release];
     return ret;
 }
-- (BOOL)appendBlobForBlobKey:(BlobKey *)theBlobKey uncompress:(BOOL)uncompress to:(FileOutputStream *)fos error:(NSError **)error {
+- (BOOL)appendBlobForBlobKey:(BlobKey *)theBlobKey to:(FileOutputStream *)fos error:(NSError **)error {
     BOOL ret = NO;
     NSError *myError = nil;
     NSAutoreleasePool *pool = nil;
@@ -558,7 +558,7 @@
         [pool drain];
         pool = [[NSAutoreleasePool alloc] init];
         BufferedOutputStream *bos = [[[BufferedOutputStream alloc] initWithUnderlyingOutputStream:fos] autorelease];
-        if ([self doAppendBlobForBlobKey:theBlobKey uncompress:uncompress to:bos error:&myError] && [bos flush:&myError]) {
+        if ([self doAppendBlobForBlobKey:theBlobKey to:bos error:&myError] && [bos flush:&myError]) {
             ret = YES;
             break;
         }
@@ -589,13 +589,13 @@
     }
     return ret;
 }
-- (BOOL)doAppendBlobForBlobKey:(BlobKey *)theBlobKey uncompress:(BOOL)uncompress to:(BufferedOutputStream *)bos error:(NSError **)error {
+- (BOOL)doAppendBlobForBlobKey:(BlobKey *)theBlobKey to:(BufferedOutputStream *)bos error:(NSError **)error {
     ServerBlob *sb = [[repo newServerBlobForBlobKey:theBlobKey error:error] autorelease];
     if (sb == nil) {
         return NO;
     }
     id <InputStream> is = [[sb newInputStream] autorelease];
-    if (uncompress) {
+    if ([theBlobKey compressed]) {
         is = [[[GunzipInputStream alloc] initWithUnderlyingStream:is] autorelease];
     }
     HSLogDebug(@"writing %@ to %@", is, bos);

Tree.h

@@ -6,13 +6,14 @@
 //  Copyright 2009 PhotoMinds LLC. All rights reserved.
 //
 
+#include <sys/stat.h>
 
 #import "Blob.h"
 @class BufferedInputStream;
 @class Node;
 @class BlobKey;
 
-#define CURRENT_TREE_VERSION 15
+#define CURRENT_TREE_VERSION 18
 #define TREE_HEADER_LENGTH (8)
 
 @interface Tree : NSObject {
@@ -40,7 +41,7 @@
     int64_t createTime_nsec;
     int64_t st_blocks;
     uint32_t st_blksize;
-    uint64_t aggregateSizeOnDisk;
+    NSMutableDictionary *missingNodes;
 	NSMutableDictionary *nodes;
 }
 + (NSString *)errorDomain;
@@ -49,7 +50,12 @@
 - (Node *)childNodeWithName:(NSString *)name;
 - (BOOL)containsNodeNamed:(NSString *)name;
 - (NSDictionary *)nodes;
-- (Blob *)toBlob;
+- (BOOL)containsMissingItems;
+- (NSArray *)missingChildNodeNames;
+- (Node *)missingChildNodeWithName:(NSString *)name;
+- (NSDictionary *)missingNodes;
+- (NSData *)toData;
+- (BOOL)ctimeMatchesStat:(struct stat *)st;
 
 @property(readonly) BOOL xattrsAreCompressed;
 @property(readonly,copy) BlobKey *xattrsBlobKey;
@@ -75,5 +81,5 @@
 @property(readonly) int st_ino;
 @property(readonly) int64_t st_blocks;
 @property(readonly) uint32_t st_blksize;
-@property(readonly) uint64_t aggregateSizeOnDisk;
+@property(readonly) uint64_t aggregateUncompressedDataSize;
 @end

Tree.m

@@ -6,6 +6,7 @@
 //  Copyright 2009 PhotoMinds LLC. All rights reserved.
 //
 
+#include <sys/stat.h>
 #import "StringIO.h"
 #import "IntegerIO.h"
 #import "BooleanIO.h"
@@ -13,14 +14,14 @@
 #import "Tree.h"
 #import "Blob.h"
 #import "DataInputStream.h"
-#import "SetNSError.h"
 #import "RegexKitLite.h"
-#import "NSErrorCodes.h"
 #import "BufferedInputStream.h"
 #import "NSData-Gzip.h"
 #import "GunzipInputStream.h"
 #import "BlobKey.h"
 #import "NSObject_extra.h"
+#import "BlobKeyIO.h"
+
 
 @interface Tree (internal)
 - (BOOL)readHeader:(BufferedInputStream *)is error:(NSError **)error;
@@ -29,7 +30,8 @@
 @implementation Tree
 @synthesize xattrsAreCompressed, xattrsBlobKey, xattrsSize, aclIsCompressed, aclBlobKey, uid, gid, mode, mtime_sec, mtime_nsec, flags, finderFlags, extendedFinderFlags, st_dev, treeVersion, st_rdev;
 @synthesize ctime_sec, ctime_nsec, createTime_sec, createTime_nsec, st_nlink, st_ino, st_blocks, st_blksize;
-@synthesize aggregateSizeOnDisk;
+@dynamic aggregateUncompressedDataSize;
+
 
 + (NSString *)errorDomain {
     return @"TreeErrorDomain";
@@ -37,11 +39,15 @@
 - (id)init {
     if (self = [super init]) {
         nodes = [[NSMutableDictionary alloc] init];
+        missingNodes = [[NSMutableDictionary alloc] init];
     }
     return self;
 }
 - (id)initWithBufferedInputStream:(BufferedInputStream *)is error:(NSError **)error {
 	if (self = [super init]) {
+        nodes = [[NSMutableDictionary alloc] init];
+        missingNodes = [[NSMutableDictionary alloc] init];
+        
         if (![self readHeader:is error:error]) {
             [self release];
             return nil;
@@ -54,15 +60,9 @@
             }
         }
         
-        NSString *xattrsSHA1 = nil;
+        BOOL ret = [BlobKeyIO read:&xattrsBlobKey from:is treeVersion:treeVersion compressed:xattrsAreCompressed error:error]
-        BOOL xattrsStretchedKey = NO;
-        NSString *aclSHA1 = nil;
-        BOOL aclStretchedKey = NO;
-        BOOL ret = [StringIO read:&xattrsSHA1 from:is error:error]
-        && (treeVersion < 14 || [BooleanIO read:&xattrsStretchedKey from:is error:error])
         && [IntegerIO readUInt64:&xattrsSize from:is error:error]
-        &&[StringIO read:&aclSHA1 from:is error:error]
+        && [BlobKeyIO read:&aclBlobKey from:is treeVersion:treeVersion compressed:aclIsCompressed error:error]
-        && (treeVersion < 14 || [BooleanIO read:&aclStretchedKey from:is error:error])
         && [IntegerIO readInt32:&uid from:is error:error]
         && [IntegerIO readInt32:&gid from:is error:error]
         && [IntegerIO readInt32:&mode from:is error:error]
@@ -79,18 +79,23 @@
         && [IntegerIO readInt64:&ctime_nsec from:is error:error]
         && [IntegerIO readInt64:&st_blocks from:is error:error]
         && [IntegerIO readUInt32:&st_blksize from:is error:error];
+        [xattrsBlobKey retain];
+        [aclBlobKey retain];
         if (!ret) {
             goto initError;
         }
-        if (xattrsSHA1 != nil) {
+        if ([xattrsBlobKey sha1] == nil) {
-            xattrsBlobKey = [[BlobKey alloc] initWithSHA1:xattrsSHA1 stretchEncryptionKey:xattrsStretchedKey];
+            [xattrsBlobKey release];
+            xattrsBlobKey = nil;
         }
-        if (aclSHA1 != nil) {
+        if ([aclBlobKey sha1] == nil) {
-            aclBlobKey = [[BlobKey alloc] initWithSHA1:aclSHA1 stretchEncryptionKey:aclStretchedKey];
+            [aclBlobKey release];
+            aclBlobKey = nil;
         }
         
-        if (treeVersion >= 11) {
+        if (treeVersion >= 11 && treeVersion <= 16) {
-            if (![IntegerIO readUInt64:&aggregateSizeOnDisk from:is error:error]) {
+            uint64_t unusedAggregateSizeOnDisk;
+            if (![IntegerIO readUInt64:&unusedAggregateSizeOnDisk from:is error:error]) {
                 goto initError;
             }
         }
@@ -101,12 +106,29 @@
             }
         }
         
-        unsigned int nodeCount;
+        if (treeVersion >= 18) {
+            uint32_t missingNodeCount;
+            if (![IntegerIO readUInt32:&missingNodeCount from:is error:error]) {
+                goto initError;
+            }
+            for (uint32_t i = 0; i < missingNodeCount; i++) {
+                NSString *missingNodeName = nil;
+                if (![StringIO read:&missingNodeName from:is error:error]) {
+                    goto initError;
+                }
+                Node *node = [[[Node alloc] initWithInputStream:is treeVersion:treeVersion error:error] autorelease];
+                if (node == nil) {
+                    goto initError;
+                }
+                [missingNodes setObject:node forKey:missingNodeName];
+            }
+        }
+        
+        uint32_t nodeCount;
         if (![IntegerIO readUInt32:&nodeCount from:is error:error]) {
             goto initError;
         }
-        nodes = [[NSMutableDictionary alloc] init];
+        for (uint32_t i = 0; i < nodeCount; i++) {
-        for (unsigned int i = 0; i < nodeCount; i++) {
             NSString *nodeName;
             if (![StringIO read:&nodeName from:is error:error]) {
                 goto initError;
@@ -130,6 +152,7 @@ initDone:
     [xattrsBlobKey release];
     [aclBlobKey release];
 	[nodes release];
+    [missingNodes release];
 	[super dealloc];
 }
 - (NSArray *)childNodeNames {
@@ -141,18 +164,36 @@ initDone:
 - (BOOL)containsNodeNamed:(NSString *)name {
 	return [nodes objectForKey:name] != nil;
 }
+- (BOOL)containsMissingItems {
+    if ([missingNodes count] > 0) {
+        return YES;
+    }
+    for (Node *node in [nodes allValues]) {
+        if ([node isTree] && [node treeContainsMissingItems]) {
+            return YES;
+        }
+    }
+    return NO;
+}
+- (NSArray *)missingChildNodeNames {
+    return [missingNodes allKeys];
+}
+- (Node *)missingChildNodeWithName:(NSString *)name {
+    return [missingNodes objectForKey:name];
+}
 - (NSDictionary *)nodes {
     return nodes;
 }
-- (Blob *)toBlob {
+- (NSDictionary *)missingNodes {
-    NSMutableData *data = [[NSMutableData alloc] init];
+    return missingNodes;
+}
+- (NSData *)toData {
+    NSMutableData *data = [[[NSMutableData alloc] init] autorelease];
     [BooleanIO write:xattrsAreCompressed to:data];
     [BooleanIO write:aclIsCompressed to:data];
-    [StringIO write:[xattrsBlobKey sha1] to:data];
+    [BlobKeyIO write:xattrsBlobKey to:data];
-    [BooleanIO write:[xattrsBlobKey stretchEncryptionKey] to:data];
     [IntegerIO writeUInt64:xattrsSize to:data];
-    [StringIO write:[aclBlobKey sha1] to:data];
+    [BlobKeyIO write:aclBlobKey to:data];
-    [BooleanIO write:[aclBlobKey stretchEncryptionKey] to:data];
     [IntegerIO writeInt32:uid to:data];
     [IntegerIO writeInt32:gid to:data];
     [IntegerIO writeInt32:mode to:data];
@@ -169,7 +210,14 @@ initDone:
     [IntegerIO writeInt64:ctime_nsec to:data];
     [IntegerIO writeInt64:st_blocks to:data];
     [IntegerIO writeUInt32:st_blksize to:data];
-    [IntegerIO writeUInt64:aggregateSizeOnDisk to:data];
+    [IntegerIO writeInt64:createTime_sec to:data];
+    [IntegerIO writeInt64:createTime_nsec to:data];
+    
+    [IntegerIO writeUInt32:(uint32_t)[missingNodes count] to:data];
+    for (NSString *missingNodeName in [missingNodes allKeys]) {
+        [StringIO write:missingNodeName to:data];
+        [[missingNodes objectForKey:missingNodeName] writeToData:data];
+    }
     
     [IntegerIO writeUInt32:(uint32_t)[nodes count] to:data];
     NSMutableArray *nodeNames = [NSMutableArray arrayWithArray:[nodes allKeys]];
@@ -182,14 +230,21 @@ initDone:
     
     char header[TREE_HEADER_LENGTH + 1];
     sprintf(header, "TreeV%03d", CURRENT_TREE_VERSION);
-    NSMutableData *completeData = [[NSMutableData alloc] init];
+    NSMutableData *completeData = [[[NSMutableData alloc] init] autorelease];
     [completeData appendBytes:header length:TREE_HEADER_LENGTH];
     
     [completeData appendBytes:[data bytes] length:[data length]];
-    
+    return completeData;
-    Blob *ret =[[[Blob alloc] initWithData:completeData mimeType:@"binary/octet-stream" downloadName:@"Tree" dataDescription:@"tree"] autorelease];
+}
-    [completeData release];
+- (BOOL)ctimeMatchesStat:(struct stat *)st {
-    [data release];
+    return st->st_ctimespec.tv_sec == ctime_sec && st->st_ctimespec.tv_nsec == ctime_nsec;
+}
+- (uint64_t)aggregateUncompressedDataSize {
+    //FIXME: This doesn't include the size of the ACL.
+    uint64_t ret = xattrsSize;
+    for (Node *node in [nodes allValues]) {
+        ret += [node uncompressedDataSize];
+    }
     return ret;
 }
 
@@ -199,7 +254,7 @@ initDone:
         return NO;
     }
     Tree *other = (Tree *)object;
-    return treeVersion == [other treeVersion] 
+    BOOL ret = (treeVersion == [other treeVersion]
                 && xattrsAreCompressed == [other xattrsAreCompressed]
                 && [NSObject equalObjects:xattrsBlobKey and:[other xattrsBlobKey]]
                 && xattrsSize == [other xattrsSize]
@@ -221,10 +276,11 @@ initDone:
                 && ctime_nsec == [other ctime_nsec]
                 && createTime_sec == [other createTime_sec]
                 && createTime_nsec == [other createTime_nsec]
+                && [missingNodes isEqual:[other missingNodes]]
                 && st_blocks == [other st_blocks]
                 && st_blksize == [other st_blksize]
-    && aggregateSizeOnDisk == [other aggregateSizeOnDisk]
+                && [nodes isEqual:[other nodes]]);
-    && [nodes isEqual:[other nodes]];
+    return ret;
 }
 - (NSUInteger)hash {
     return (NSUInteger)treeVersion + [nodes hash];
@@ -239,14 +295,11 @@ initDone:
         goto readHeader_error;
     }
     NSString *header = [[[NSString alloc] initWithBytes:buf length:TREE_HEADER_LENGTH encoding:NSASCIIStringEncoding] autorelease];
-    NSRange versionRange = [header rangeOfRegex:@"^TreeV(\\d{3})$" capture:1];
+    if (![header hasPrefix:@"TreeV"] || [header length] < 6) {
-    treeVersion = 0;
+        SETNSERROR([Tree errorDomain], ERROR_INVALID_OBJECT_VERSION, @"invalid Tree header: %@", header);
-    if (versionRange.location != NSNotFound) {
+        goto readHeader_error;
-        NSNumberFormatter *nf = [[NSNumberFormatter alloc] init];
-        NSNumber *number = [nf numberFromString:[header substringWithRange:versionRange]];
-        treeVersion = [number intValue];
-        [nf release];
     }
+    treeVersion = [[header substringFromIndex:5] intValue];
     if (treeVersion < 10) {
         SETNSERROR([Tree errorDomain], ERROR_INVALID_OBJECT_VERSION, @"invalid Tree header: %@", header);
         goto readHeader_error;

arq_restore.xcodeproj/project.pbxproj

@@ -51,6 +51,7 @@
 		F805B8C21160EC41007EC01E /* Security.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = F805B8C11160EC41007EC01E /* Security.framework */; };
 		F805B8CE1160ECD7007EC01E /* CoreServices.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = F805B8CD1160ECD7007EC01E /* CoreServices.framework */; };
 		F81426D714541A6C00D7E50A /* BackupSet.m in Sources */ = {isa = PBXBuildFile; fileRef = F81426D614541A6C00D7E50A /* BackupSet.m */; };
+		F8146BEB16EB70EE006AD471 /* BlobKeyIO.m in Sources */ = {isa = PBXBuildFile; fileRef = F8146BEA16EB70EE006AD471 /* BlobKeyIO.m */; };
 		F8373E0F14794D01005AFBE6 /* ReflogEntry.m in Sources */ = {isa = PBXBuildFile; fileRef = F8373E0E14794D01005AFBE6 /* ReflogEntry.m */; };
 		F8373E5A147A8DEC005AFBE6 /* ReflogPrinter.m in Sources */ = {isa = PBXBuildFile; fileRef = F8373E59147A8DEC005AFBE6 /* ReflogPrinter.m */; };
 		F83C1A7411CA7C170001958F /* ArqRestoreCommand.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B54C1160D3E6007EC01E /* ArqRestoreCommand.m */; };
@@ -363,6 +364,9 @@
 		F805B8CD1160ECD7007EC01E /* CoreServices.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreServices.framework; path = System/Library/Frameworks/CoreServices.framework; sourceTree = SDKROOT; };
 		F81426D514541A6C00D7E50A /* BackupSet.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BackupSet.h; sourceTree = "<group>"; };
 		F81426D614541A6C00D7E50A /* BackupSet.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BackupSet.m; sourceTree = "<group>"; };
+		F8146BE816EB7054006AD471 /* StorageType.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = StorageType.h; path = storage/StorageType.h; sourceTree = "<group>"; };
+		F8146BE916EB70EE006AD471 /* BlobKeyIO.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BlobKeyIO.h; sourceTree = "<group>"; };
+		F8146BEA16EB70EE006AD471 /* BlobKeyIO.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BlobKeyIO.m; sourceTree = "<group>"; };
 		F8373E0D14794D01005AFBE6 /* ReflogEntry.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ReflogEntry.h; sourceTree = "<group>"; };
 		F8373E0E14794D01005AFBE6 /* ReflogEntry.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ReflogEntry.m; sourceTree = "<group>"; };
 		F8373E58147A8DEC005AFBE6 /* ReflogPrinter.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ReflogPrinter.h; sourceTree = "<group>"; };
@@ -577,6 +581,7 @@
 				F89A1EB613FAC3750071D321 /* repo */,
 				F805B7651160DD60007EC01E /* s3 */,
 				F805B7A61160DEF2007EC01E /* shared */,
+				F8146BE716EB704A006AD471 /* storage */,
 				F8F4D1E0121D7BC3002D09C1 /* AppKeychain.h */,
 				F8F4D1E1121D7BC3002D09C1 /* AppKeychain.m */,
 				F805B54B1160D3E6007EC01E /* ArqRestoreCommand.h */,
@@ -588,6 +593,8 @@
 				32A70AAB03705E1F00C91783 /* arq_restore_Prefix.pch */,
 				F89A1F4313FAC6C40071D321 /* BlobKey.h */,
 				F89A1F4413FAC6C40071D321 /* BlobKey.m */,
+				F8146BE916EB70EE006AD471 /* BlobKeyIO.h */,
+				F8146BEA16EB70EE006AD471 /* BlobKeyIO.m */,
 				F83C1D0811CA929D0001958F /* BucketVerifier.h */,
 				F83C1D0911CA929D0001958F /* BucketVerifier.m */,
 				F8D67F6E1161443600CC270E /* RestoreNode.h */,
@@ -856,6 +863,14 @@
 			path = crypto;
 			sourceTree = "<group>";
 		};
+		F8146BE716EB704A006AD471 /* storage */ = {
+			isa = PBXGroup;
+			children = (
+				F8146BE816EB7054006AD471 /* StorageType.h */,
+			);
+			name = storage;
+			sourceTree = "<group>";
+		};
 		F89A1EB613FAC3750071D321 /* repo */ = {
 			isa = PBXGroup;
 			children = (
@@ -1074,6 +1089,7 @@
 				F841670515E279BB00B6ECED /* Sysctl.m in Sources */,
 				F841670B15E279DE00B6ECED /* DNS_SDErrors.m in Sources */,
 				F841672615E27A5200B6ECED /* RemoteS3Signer.m in Sources */,
+				F8146BEB16EB70EE006AD471 /* BlobKeyIO.m in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};

arq_restore_Prefix.pch

@@ -1,4 +1,6 @@
 #ifdef __OBJC__
     #import <Foundation/Foundation.h>
     #import "HSLog.h"
+#import "SetNSError.h"
+#import "NSErrorCodes.h"
 #endif

io/CryptoKey.m

@@ -28,6 +28,11 @@
             [self release];
             return nil;
         }
+        if ([thePassword length] == 0) {
+            SETNSERROR([Encryption errorDomain], -1, @"missing encryption password");
+            [self release];
+            return nil;
+        }
         if (theSalt != nil && [theSalt length] != 8) {
             SETNSERROR([Encryption errorDomain], -1, @"salt must be 8 bytes or nil");
             [self release];

shared/NSErrorCodes.h

@@ -48,3 +48,4 @@
 #define ERROR_RRS_NOT_FOUND (-17)
 #define ERROR_INVALID_FILE (-18)
 #define ERROR_DELAYS_IN_S3_EVENTUAL_CONSISTENCY (-19)
+#define ERROR_INVALID_COMMIT_HEADER (-20)

storage/StorageType.h

@@ -0,0 +1,5 @@
+enum {
+    StorageTypeS3 = 1,
+    StorageTypeGlacier = 2
+};
+typedef uint32_t StorageType;